extras-buildsys/server BuildJob.py, NONE, 1.1 CONFIG.py, 1.4, 1.5 buildmaster.py, 1.2, 1.3 buildserver.py, 1.2, 1.3 client_manager.py, 1.4, 1.5 buildjob.py, 1.6, NONE
Daniel Williams (dcbw)
fedora-extras-commits at redhat.com
Thu Jun 9 19:31:31 UTC 2005
Author: dcbw
Update of /cvs/fedora/extras-buildsys/server
In directory cvs-int.fedora.redhat.com:/tmp/cvs-serv8175
Modified Files:
CONFIG.py buildmaster.py buildserver.py client_manager.py
Added Files:
BuildJob.py
Removed Files:
buildjob.py
Log Message:
2005-06-09 Dan Williams <dcbw at redhat.com>
* common/FileDownloader.py
- Don't traceback on refused or dropped connections
* server/buildjob.py -> server/BuildJob.py
- BuildJob objects are now threads
- Clean up console output a lot
- Delete arch-specific jobs that don't actually start
* server/buildmaster.py
- Clean up console output
* server/buildserver.py
- Clean up 'enqueue' command return messages
* server/client_manager.py
- Try to tighten up exception handling around socket operations
- Don't print so much stuff to the console
- Prettier printing of downloaded file lists from client
--- NEW FILE BuildJob.py ---
#!/usr/bin/python
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# copyright 2005 Duke University
# written by Seth Vidal
import os
import os.path
import sys
import commands
import threading
import time
import popen2
import rpmUtils
import exceptions
import shutil
import tempfile
import smtplib
from email.MIMEText import MIMEText
import string
import SimpleXMLRPCServer
import xmlrpclib
import CONFIG
import socket
from client_manager import BuildClientManager
from client_manager import BuildClientJob
os.environ['CVSROOT'] = CONFIG.get('pkg_cvs_root')
if len(CONFIG.get('pkg_cvs_rsh')) > 0:
os.environ['CVS_RSH'] = CONFIG.get('pkg_cvs_rsh')
DEBUG = False
def debugprint(stuff=''):
if DEBUG:
print stuff
def log(stuff=''):
print stuff
class PrepError(exceptions.Exception):
def __init__(self, errno=0, args=None):
exceptions.Exception.__init__(self)
self.args = args
self.errno = errno
def __str__(self):
return self.args
class BuildError(exceptions.Exception):
def __init__(self, errno=0, args=None):
exceptions.Exception.__init__(self)
self.args = args
self.errno = errno
def __str__(self):
return self.args
http_dir = os.path.join(CONFIG.get('server_work_dir'), "srpm_http_dir")
class BuildJob(threading.Thread):
""" Controller object for building 1 SRPM on multiple arches """
def __init__(self, uid, username, package, cvs_tag, target, client_manager):
self.curstage = 'initialize'
self.bcm = client_manager
self.uid = uid
self.username = username
self.starttime = time.time()
self.endtime = None
self.package = package
self.target = target
self.buildarches = []
self.sub_jobs = {}
self.failed = False
self.no_cvs = CONFIG.get('use_srpm_not_cvs')
self.cvs_tag = cvs_tag
self.stage_dir = None
self.srpm_path = None
self.srpm_http_path = None
# Deal with straight SRPM builds
if self.no_cvs and self.curstage is 'initialize':
self.curstage = 'make_srpm'
threading.Thread.__init__(self)
def get_cur_stage(self):
return self.curstage
def get_uid(self):
return self.uid
def arch_handling(self, hdr):
archs = []
targets = CONFIG.get('targets')
buildable_arches = targets[self.target]
ba = hdr['buildarchs']
exclusive = hdr['exclusivearch']
exclude = hdr['excludearch']
if ba == ['noarch']:
return ba
# default to building all arches
tmparchs = []
for arch in buildable_arches:
tmparchs.append(arch)
if ba:
tmparchs = ba
else:
if exclusive:
tmparchs = exclusive
if exclude:
for arch in exclude:
if arch in tmparchs:
tmparchs.remove(arch)
# we probably need to check the archs, and make sure they are what
# we can build for
for thisarch in tmparchs:
if thisarch in buildable_arches:
archs.append(thisarch)
return archs
def _make_stage_dir(self, rootdir):
# The dir will look like this:
# <rootdir>/devel/95-foo-1.1.0-23
pkgsubdir = '%d-%s-%s-%s' % (self.uid, self.name, self.ver, self.release)
stage_dir = os.path.join(rootdir, self.target, pkgsubdir)
if os.path.exists(stage_dir):
shutil.rmtree(stage_dir, ignore_errors=True)
os.makedirs(stage_dir)
return stage_dir
def _checkout(self):
self.curstage = 'checkout'
dir_prefix = self.cvs_tag + "-"
self.checkout_tmpdir = tempfile.mkdtemp(prefix=dir_prefix, dir=CONFIG.get('tmpdir'))
os.chdir(self.checkout_tmpdir)
# Checkout the module
cmd = '%s co -r %s %s' % (CONFIG.get('cvs_cmd'), self.cvs_tag, self.package)
debugprint("%d: Running %s" % (self.uid, cmd))
s, o = commands.getstatusoutput(cmd)
if s != 0:
subj = 'Prep Error: %s on %s' % (self.cvs_tag, self.target)
msg = "could not check out %s from %s - output was:\n %s" % (self.cvs_tag, self.target, o)
self.email_result(resultstring=msg, subject=subj)
self.curstage = 'finished'
self.failed = True
shutil.rmtree(self.checkout_tmpdir, True)
return
# Just in case the 'common' directory didn't come along for the ride,
# get it from CVS
pkg_path = os.path.join(self.checkout_tmpdir, self.package)
if not os.path.exists(os.path.join(pkg_path, "common")):
os.chdir(pkg_path)
cmd = '%s co common' % CONFIG.get('cvs_cmd')
debugprint("%d: Running %s" % (self.uid, cmd))
s, o = commands.getstatusoutput(cmd)
os.chdir(self.checkout_tmpdir)
if s != 0:
subj = 'Prep Error: %s on %s' % (self.cvs_tag, self.target)
msg = "could not check out common directory - output was:\n %s" % (self.cvs_tag, self.target, o)
self.email_result(resultstring=msg, subject=subj)
self.curstage = 'finished'
self.failed = True
shutil.rmtree(self.checkout_tmpdir, True)
return
def _make_srpm(self):
self.curstage = 'make_srpm'
self.srpm_path = None
srpm_dir = os.path.join(self.checkout_tmpdir, self.package, self.target)
if not os.path.exists(srpm_dir):
subj = 'Prep Error: %s on %s' % (self.cvs_tag, self.target)
msg = "could not find path %s for %s." % (srpm_dir, self.cvs_tag)
self.email_result(resultstring=msg, subject=subj)
self.curstage = 'finished'
self.failed = True
shutil.rmtree(self.checkout_tmpdir, True)
return
os.chdir(srpm_dir)
cmd = '%s srpm' % CONFIG.get('make_cmd')
debugprint("%d: Running %s in %s" % (self.uid, cmd, srpm_dir))
s, o = commands.getstatusoutput(cmd)
if s != 0:
subj = 'Prep Error: %s on %s' % (self.cvs_tag, self.target)
msg = "could not make srpm for %s - output was:\n %s" % (self.cvs_tag, o)
self.email_result(resultstring=msg, subject=subj)
self.curstage = 'finished'
self.failed = True
#shutil.rmtree(self.checkout_tmpdir, True)
return
srpmpath = None
for line in o.split("\n"):
if line.startswith("Wrote:"):
line.replace("\n", "")
(garbage, path) = line.split(':')
srpmpath = path.strip()
break
if not srpmpath:
subj = 'Prep Error: %s on %s' % (self.cvs_tag, self.target)
msg = "could not find srpm for %s - output was:\n %s" % (self.cvs_tag, o)
self.email_result(resultstring=msg, subject=subj)
self.curstage = 'finished'
self.failed = True
shutil.rmtree(self.checkout_tmpdir, True)
return
self.srpm_path = srpmpath
def _prep(self):
self.curstage = 'prep'
# In SRPM-only mode, cvs_tag is path to the SRPM to build
if self.no_cvs:
self.srpm_path = self.cvs_tag
ts = rpmUtils.transaction.initReadOnlyTransaction()
hdr = rpmUtils.miscutils.hdrFromPackage(ts, self.srpm_path)
self.name = hdr['name']
self.ver = hdr['version']
self.release = hdr['release']
self.buildarches = self.arch_handling(hdr)
del hdr
del ts
if len(self.buildarches) == 0:
self.failed = True
self.curstage = 'finished'
return
# Make sure build clients see latest packages
self._createrepo()
self.stage_dir = self._make_stage_dir(CONFIG.get('server_work_dir'))
for arch in self.buildarches:
thisdir = os.path.join(self.stage_dir, arch)
if not os.path.exists(thisdir):
os.makedirs(thisdir)
# Copy the SRPM to the final package product dir
srpm = os.path.basename(self.srpm_path)
srpm_in_dir = os.path.join(self.stage_dir, srpm)
if os.path.exists(srpm_in_dir):
os.unlink(srpm_in_dir)
shutil.copy(self.srpm_path, self.stage_dir)
# Must also copy SRPM to where the build client can get it over HTTP
http_pkg_path = self._make_stage_dir(http_dir)
self.srpm_http_path = os.path.join(http_pkg_path, srpm)
shutil.copy(self.srpm_path, self.srpm_http_path)
self.srpm_path = srpm_in_dir
# Remove CVS checkout and make_srpm dirs
if not self.no_cvs:
shutil.rmtree(self.checkout_tmpdir, ignore_errors=True)
def run(self):
while self.curstage != 'needsign' and self.curstage != 'failed':
# Advance to next stage based on current stage
oldstage = self.curstage
if oldstage == 'initialize':
self._checkout()
elif oldstage == 'checkout':
self._make_srpm()
elif oldstage == 'make_srpm':
self._prep()
elif oldstage == 'prep' or oldstage == 'building':
self._monitor()
elif oldstage == 'finished':
self._cleanup()
elif oldstage == 'cleanup':
if self.failed:
self._failed()
else:
self._succeeded()
def job_server_gone(self, job):
""" Remove a job from our building queue if its server went away """
print "%s (%s/%s): Build client disappeared. Requeuing arch..." % (self.uid, self.package, job.arch)
del self.sub_jobs[job.arch]
def _start_unspawned_builds(self):
for arch in self.buildarches:
if not self.sub_jobs.has_key(arch):
# Construct SPRM URL
srpm_http_base = self.srpm_http_path[len(http_dir):]
srpm_url = "https://" + CONFIG.get('hostname') + ":8886" + srpm_http_base
job = self.bcm.new_job_on_arch(self, self.target, arch, srpm_url)
if job:
if job.start() == True:
self.bcm.track_job(job)
self.sub_jobs[arch] = job
log("%s (%s/%s): Builder UID is %s" % (self.uid, self.package, arch, job.jobid))
else:
del job
def _monitor(self):
self.curstage = 'building'
self._start_unspawned_builds()
have_jobs = False
jobs_running = False
for job in self.sub_jobs.values():
have_jobs = True
job_status = job.get_status()
if job_status is 'downloading' or job_status is 'running':
jobs_running = True
continue
elif job_status is 'done':
builder_result = job.get_builder_result()
if builder_result == 'failed' or builder_result == 'killed':
self.failed = True
if self.failed or (have_jobs == True and jobs_running == False):
self.curstage = 'finished'
time.sleep(5)
def _cleanup(self):
self.curstage = 'cleanup'
if self.failed:
# Kill remaining jobs on other arches
for job in self.sub_jobs.values():
job_status = job.get_status()
if job_status is 'initialize' or job_status is 'running':
job.die()
def get_stage_dir(self):
return self.stage_dir
def _failed(self):
old_stage = self.curstage
self.curstage = 'failed'
resultstring = """
%s (%s): %s on %s failed to complete on one or more archs.
""" % (self.uid, self.package, self.cvs_tag, self.target)
self.email_result(resultstring)
def _succeeded(self):
self.curstage = 'needsign'
# Copy completed RPMs to repo dir here
for job in self.sub_jobs.values():
file_list = job.get_files()
for f in file_list:
if not f.endswith(".rpm"):
continue
src_file = os.path.join(self.stage_dir, job.arch, f)
verrel = "%s-%s" % (self.ver, self.release)
dst_path = os.path.join(CONFIG.get('repo_dir'), self.target, self.name, verrel, job.arch)
if not os.path.exists(dst_path):
os.makedirs(dst_path)
shutil.copy(src_file, dst_path)
resultstring = "%s (%s): Build on target %s succeeded." % (self.uid, self.name, self.target)
self.email_result(resultstring)
# Udpate the repo with new packages
self._createrepo()
def email_result(self, resultstring, subject=None):
"""send 'resultstring' to self.email from self.email_from"""
print "%s (%s): Result: '%s'" % (self.uid, self.package, resultstring)
return
msg = MIMEText(resultstring)
if not subject:
subject = 'Build Result: %s on %s' % (self.name, self.target)
msg['Subject'] = subject
msg['From'] = CONFIG.get('email_from')
email_to = '%s@%s' % (self.username, CONFIG.get('email_to_domain'))
msg['To'] = email_to
s = smtplib.SMTP()
s.connect()
s.sendmail(CONFIG.get('email_from'), [email_to], msg.as_string())
s.close()
def _createrepo(self):
# createrepo on the needsign tree for new changes
repodir = os.path.join(CONFIG.get('repo_dir'), self.target)
debugprint("%d: updating repodir %s" % (self.uid, repodir))
if not os.path.exists(repodir):
os.makedirs(repodir)
s, o = commands.getstatusoutput('/usr/bin/createrepo -q %s' % repodir)
if s != 0:
print "Createrepo failed!!!"
#self.curstage = 'failed'
#raise PrepError(5, 'Error generating repodata for %s: %s' % (repodir, o))
Index: CONFIG.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/server/CONFIG.py,v
retrieving revision 1.4
retrieving revision 1.5
diff -u -r1.4 -r1.5
--- CONFIG.py 9 Jun 2005 01:57:41 -0000 1.4
+++ CONFIG.py 9 Jun 2005 19:31:28 -0000 1.5
@@ -35,8 +35,7 @@
'devel' : ['i386']
}
-#config_opts['builders'] = [ 'http://172.16.83.112:8888', 'http://172.16.83.22:8888' ]
-config_opts['builders'] = [ 'http://127.0.0.1:8888' ]
+config_opts['builders'] = [ 'https://127.0.0.1:8888' ]
def get(key):
if config_opts.has_key(key):
Index: buildmaster.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/server/buildmaster.py,v
retrieving revision 1.2
retrieving revision 1.3
diff -u -r1.2 -r1.3
--- buildmaster.py 7 Jun 2005 12:10:23 -0000 1.2
+++ buildmaster.py 9 Jun 2005 19:31:28 -0000 1.3
@@ -18,8 +18,7 @@
import time
import CONFIG
-from buildjob import BuildJob
-from buildjob import PrepError
+import BuildJob
import sqlite
import threading
@@ -71,10 +70,9 @@
# Allow each job some processing time
for job in self.building_jobs:
- job.process()
self.set_job_status(job)
if job.get_cur_stage() == 'failed' or job.get_cur_stage() == 'needsign':
- print "%d: Job finished." % job.get_uid()
+ print "%s (%s): Job finished." % (job.get_uid(), job.package)
self.building_jobs.remove(job)
# Grab one waiting job from database and start it
@@ -83,11 +81,12 @@
self.dbcx.commit()
item = self.curs.fetchone()
if item:
- print "%d: Adding (%s/'%s'/%s) to build queue" % (item['uid'], \
+ print "%s (%s): Starting tag '%s' on target '%s'" % (item['uid'], \
item['package'], item['cvs_tag'], item['target'])
- job = BuildJob(item['uid'], item['username'], item['package'],
+ job = BuildJob.BuildJob(item['uid'], item['username'], item['package'],
item['cvs_tag'], item['target'], self.bcm)
self.building_jobs.append(job)
+ job.start()
time.sleep(5)
if self.should_stop == True:
Index: buildserver.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/server/buildserver.py,v
retrieving revision 1.2
retrieving revision 1.3
diff -u -r1.2 -r1.3
--- buildserver.py 9 Jun 2005 01:57:41 -0000 1.2
+++ buildserver.py 9 Jun 2005 19:31:28 -0000 1.3
@@ -22,7 +22,6 @@
import os
import SimpleXMLRPCServer
import xmlrpclib
-from buildjob import BuildJob
import sqlite
import smtplib
from email.MIMEText import MIMEText
@@ -62,12 +61,10 @@
def enqueue(self, username, package, cvs_tag, target, buildreq=None):
""" Accept a job to build and stuff it into the job database """
- # FIXME we should be passing back a tuple of (error/success, reason)
-
if CONFIG.get('use_srpm_not_cvs') == True:
email_result(username, cvs_tag, "Error setting up build for %s on "\
"%s: this server builds SRPMs, not CVS checkouts." % (cvs_tag, target))
- return -1
+ return (-1, "This build server is set up for building SRPMS only. Use the 'enqueue_srpm' command instead.")
print "Request to enqueue '%s' tag '%s' for target '%s' (user '%s')" \
% (package, cvs_tag, target, username)
@@ -77,7 +74,7 @@
% (cvs_tag, target)
email_result(username, cvs_tag, "Error setting up build for %s on "\
"%s: target does not exist." % (cvs_tag, target))
- return -1
+ return (-1, "This build server does not support the target %s." % target)
else:
# Insert request into the database
self.curs.execute('INSERT INTO jobs (uid, username, package,' \
@@ -86,23 +83,21 @@
% (username, package, cvs_tag, target, buildreq, \
time.time(), 'waiting'))
self.dbcx.commit()
- return 0
+ return (0, "Success: package has been queued.")
def enqueue_srpm(self, username, package, srpm_file, target, buildreq=None):
""" Accept a job to build from SRPM file and stuff it into the job database """
- # FIXME we should be passing back a tuple of (error/success, reason)
-
if CONFIG.get('use_srpm_not_cvs') == False:
email_result(username, srpm_file, "Error setting up build for %s on "\
"%s: this server builds CVS checkouts, not SRPMS." % (srpm_file, target))
- return -1
+ return (-1, "This build server is set up for building from CVS. Use the 'enqueue' command instead.")
# We limit the database field to 255 chars
if len(srpm_file) > 255:
email_result(username, srpm_file, "Error setting up build for %s on "\
"%s: try using a shorter path to the SRPM (< 255 chars)." % (srpm_file, target))
- return -1
+ return (-1, "Pathname to SRPM is limited to 255 characters.")
print "Request to enqueue '%s' file '%s' for target '%s' (user '%s')" \
% (package, srpm_file, target, username)
@@ -112,7 +107,7 @@
% (srpm_file, target)
email_result(username, srpm_file, "Error setting up build for %s on "\
"%s: target does not exist." % (srpm_file, target))
- return -1
+ return (-1, "This build server does not support the target %s." % target)
else:
# Insert request into the database
self.curs.execute('INSERT INTO jobs (uid, username, package,' \
@@ -121,7 +116,7 @@
% (username, package, srpm_file, target, buildreq, \
time.time(), 'waiting'))
self.dbcx.commit()
- return 0
+ return (0, "Success: package has been queued.")
def list_waiting_jobs(self):
self.curs.execute('SELECT uid, username, package, cvs_tag, target' \
Index: client_manager.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/server/client_manager.py,v
retrieving revision 1.4
retrieving revision 1.5
diff -u -r1.4 -r1.5
--- client_manager.py 9 Jun 2005 03:18:03 -0000 1.4
+++ client_manager.py 9 Jun 2005 19:31:28 -0000 1.5
@@ -60,11 +60,12 @@
failed = False
try:
self.jobid = self._server.start(self.target, self.arch, self.srpm_url)
- except Exception, e:
+ except socket.error, e:
+ if e[0] != 104 and e[0] != 111:
+ print "Error starting job on host %s\n\t---error: %s" % (self.bci.address(), e)
failed = True
if failed or self.jobid == 0:
- print "Error starting job on host %s\n\t---error: %s" % (self.bci.address(), e)
self.status = 'failed'
return False
else:
@@ -80,20 +81,18 @@
result = ''
try:
result = self._server.status(self.jobid)
- except Exception, e:
- print "BuildClientJob(%s): got error '%s' from build client while attempting " \
- "to get its status." % (self.bci.address(), e)
+ except socket.error, e:
+ if e[0] != 104 and e[0] != 111:
+ print "%s (%s/%s): [ %s ] Unknown error when getting status: '%s'" % (self.parent_job.uid,
+ self.parent_job.package, self.arch, self.bci.address(), e)
self.client_result = result
# if the builder is done, grab list of files to download
if result == 'done' or result == 'killed' or result == 'failed':
self.status = 'downloading'
- print "%s (%s): Files to download from build client:" % (self.jobid, self.arch)
for f in self._server.files(self.jobid):
uf = urllib.unquote(f)
- print " %s" % uf
self.downloads[uf] = 0
- print "%s (%s): Done" % (self.jobid, self.arch)
elif self.status == 'downloading':
# Start grabbing the next undownloaded file, but only
# if we aren't already pulling one down
@@ -119,6 +118,7 @@
break
elif dl_status == 1:
# in progress
+ undownloaded = True
break
elif dl_status == 2:
# error
@@ -126,17 +126,32 @@
elif dl_status == 3:
# this one is done
continue
+
+ # All done downloading?
if not undownloaded:
+ # Print out downloaded file list
+ file_string = ""
+ ndownloads = len(self.downloads.keys())
+ for url in self.downloads.keys():
+ filename = os.path.basename(url)
+ dl_status = self.downloads[url]
+ string = "'" + filename + "'"
+ if dl_status == 2:
+ string = string + " (failed)"
+ file_string = file_string + string
+ if url != self.downloads.keys()[ndownloads - 1]:
+ file_string = file_string + ", "
+
+ print "%s (%s/%s): Build result files - [ %s ]" % (self.parent_job.uid,
+ self.parent_job.package, self.arch, file_string)
self.status = 'done'
def dl_callback(self, status, cb_data):
url = cb_data
if status == 'done':
self.downloads[url] = 3
- print("%s/%s: Finished downloading %s" % (self.jobid, self.arch, url))
elif status == 'failed':
self.downloads[url] = 2
- print("%s/%s: Failed to download %s" % (self.jobid, self.arch, url))
def get_status(self):
return self.status
--- buildjob.py DELETED ---
More information about the fedora-extras-commits
mailing list