extras-buildsys/server fileserver.py, NONE, 1.1 CONFIG.py, 1.1.1.1, 1.2 aw_manager.py, 1.1.1.1, 1.2 bm_server.py, 1.1.1.1, 1.2 buildjob.py, 1.1.1.1, 1.2
Daniel Williams (dcbw)
fedora-extras-commits at redhat.com
Sat Jun 4 23:14:53 UTC 2005
- Previous message (by thread): extras-buildsys/client srpm_download.py, NONE, 1.1 archwelder.py, 1.1.1.1, 1.2 archwelder_config.py, 1.1.1.1, 1.2 fileserver.py, 1.1.1.1, 1.2
- Next message (by thread): web/html/docs/selinux-faq index.php,1.1.1.1,1.2
- Messages sorted by:
[ date ]
[ thread ]
[ subject ]
[ author ]
Author: dcbw
Update of /cvs/fedora/extras-buildsys/server
In directory cvs-int.fedora.redhat.com:/tmp/cvs-serv10030/server
Modified Files:
CONFIG.py aw_manager.py bm_server.py buildjob.py
Added Files:
fileserver.py
Log Message:
First (incomplete) cut of HTTP transfer stuff.
--- NEW FILE fileserver.py ---
#!/usr/bin/python -t
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
# Copyright 2005 Dan Williams and Red Hat, Inc.
#
import SimpleHTTPServer
import SocketServer
import threading
import os
import urllib
import posixpath
import CONFIG
BaseRequestHandler = SimpleHTTPServer.SimpleHTTPRequestHandler
BaseHttpServer = SocketServer.TCPServer
class BMHttpRequestHandler(BaseRequestHandler):
def __init__(self, request, client_address, server):
self._server = server
BaseRequestHandler.__init__(self, request, client_address, server)
def list_directory(self, path):
self.send_error(404, "No permission to list directory")
def translate_path(self, path):
"""Translate a /-separated PATH to the local filename syntax.
Components that mean special things to the local file system
(e.g. drive or directory names) are ignored. (XXX They should
probably be diagnosed.)
This code is lifted from SimpleHTTPRequestHandler so that we can
make sure the request is always based in our srpm_http_dir.
"""
path = posixpath.normpath(urllib.unquote(path))
words = path.split('/')
words = filter(None, words)
path = CONFIG.get('srpm_http_dir')
for word in words:
drive, word = os.path.splitdrive(word)
head, word = os.path.split(word)
if word in (os.curdir, os.pardir): continue
path = os.path.join(path, word)
return path
def do_GET(self):
BaseRequestHandler.do_GET(self)
# try:
# BaseRequestHandler.do_GET(self)
# except Exception, e:
# # We get an exception if the client drops the transfer
# pass
class BMThreadingHttpServer(SocketServer.ThreadingTCPServer):
def __init__(self, server_address, RequestHandlerClass):
self.protocol_version = "HTTP/1.0" # Don't want keepalive
self.allow_reuse_address = 1
os.chdir(CONFIG.get('srpm_http_dir'))
BaseHttpServer.__init__(self, server_address, RequestHandlerClass)
class BMHttpServerThread(threading.Thread):
def __init__(self, address_tuple):
self._server = BMThreadingHttpServer(address_tuple, BMHttpRequestHandler)
threading.Thread.__init__(self)
def run(self):
self._server.serve_forever()
Index: CONFIG.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/server/CONFIG.py,v
retrieving revision 1.1.1.1
retrieving revision 1.2
diff -u -r1.1.1.1 -r1.2
--- CONFIG.py 1 Jun 2005 04:15:01 -0000 1.1.1.1
+++ CONFIG.py 4 Jun 2005 23:14:51 -0000 1.2
@@ -13,6 +13,8 @@
config_opts['redhat_internal_cvs'] = 1
config_opts['log_url'] = "http://foo.foo.org/logs/"
+config_opts['srpm_http_dir'] = "/tmp/srpm_stage_dir"
+
# This option disables pulling from CVS. Allowing jobs to be submitted
# as unknown SRPMs from random people may be a security risk, so don't
# do this unless you really really want to.
Index: aw_manager.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/server/aw_manager.py,v
retrieving revision 1.1.1.1
retrieving revision 1.2
diff -u -r1.1.1.1 -r1.2
--- aw_manager.py 1 Jun 2005 04:15:01 -0000 1.1.1.1
+++ aw_manager.py 4 Jun 2005 23:14:51 -0000 1.2
@@ -32,21 +32,20 @@
class ArchWelderJob:
""" Tracks a single build instance for a single arch on an ArchWelder """
- def __init__(self, parent_job, awi, server, srpm, target, mydir, arch):
+ def __init__(self, awi, parent_job, server, target, arch, srpm_url):
self.parent_job = parent_job
self.awi = awi
self.jobid = None
self.status = None
- self.srpm = srpm
self.target = target
- self.mydir = mydir
self.arch = arch
+ self.srpm_url = srpm_url
self._server = server
self.starttime = None
def start(self):
try:
- self.jobid = self._server.start(self.srpm, self.target, self.mydir, self.arch)
+ self.jobid = self._server.start(self.target, self.arch, self.srpm_url)
except Exception, e:
print "Error starting job on host %s\n\t---error: %s" % (self.awi.address(), e)
return False
@@ -103,8 +102,8 @@
def address(self):
return self._address
- def new_job(self, parent_job, srpm, target, mydir):
- return ArchWelderJob(parent_job, self, self._server, srpm, target, mydir, self._arch)
+ def new_job(self, parent_job, target, srpm_url):
+ return ArchWelderJob(self, parent_job, self._server, target, self._arch, srpm_url)
def track_job(self, job):
self._jobs.append(job)
@@ -184,7 +183,7 @@
server = xmlrpclib.Server(address)
try:
arches = server.supported_arches()
- except Exception, e:
+ except socket.error, e:
pass
if arches:
arches.append('noarch')
@@ -194,8 +193,6 @@
self.running_aw[a] = []
awi = ArchWelderInstance(self, address, a)
self.running_aw[a].append(awi)
- else:
- self.possible_aw.remove(address)
del server
def process(self):
@@ -211,12 +208,12 @@
if job:
job.awi.track_job(job)
- def new_job_on_arch(self, parent_job, arch, srpm, target, mydir):
+ def new_job_on_arch(self, parent_job, target, arch, srpm_url):
""" Create a job on a free builder for this arch """
if self.running_aw.has_key(arch):
for aw in self.running_aw[arch]:
if aw.available():
- return aw.new_job(parent_job, srpm, target, mydir)
+ return aw.new_job(parent_job, target, srpm_url)
return None
Index: bm_server.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/server/bm_server.py,v
retrieving revision 1.1.1.1
retrieving revision 1.2
diff -u -r1.1.1.1 -r1.2
--- bm_server.py 1 Jun 2005 04:15:01 -0000 1.1.1.1
+++ bm_server.py 4 Jun 2005 23:14:51 -0000 1.2
@@ -19,6 +19,7 @@
import time
import CONFIG
import socket
+import os
import SimpleXMLRPCServer
import xmlrpclib
from buildjob import BuildJob
@@ -30,6 +31,7 @@
from buildmaster import BuildMaster
from buildmaster import ensure_build_db_tables
from aw_manager import ArchWelderManager
+from fileserver import BMHttpServerThread
def email_result(username, cvs_tag, resultstring, subject=None):
"""send 'resultstring' to username"""
@@ -176,6 +178,11 @@
xmlrpc_bm = XMLRPCBuildMaster(awm)
bm_server = MyXMLRPCServer((CONFIG.get('hostname'), 8887))
bm_server.register_instance(xmlrpc_bm)
+
+ # SRPM fileserver
+ fileserver = BMHttpServerThread((CONFIG.get('hostname'), 8886))
+ fileserver.start()
+
print "BuildMaster accepting requests on %s:8887.\n" % CONFIG.get('hostname')
try:
bm_server.serve_forever()
@@ -184,3 +191,7 @@
print "Shutting down..."
bm.stop()
+ print "Done."
+ os._exit(0)
+
+
Index: buildjob.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/server/buildjob.py,v
retrieving revision 1.1.1.1
retrieving revision 1.2
diff -u -r1.1.1.1 -r1.2
--- buildjob.py 1 Jun 2005 04:15:01 -0000 1.1.1.1
+++ buildjob.py 4 Jun 2005 23:14:51 -0000 1.2
@@ -69,6 +69,7 @@
""" Controller object for building 1 SRPM on multiple arches """
def __init__(self, uid, username, package, cvs_tag, target, arch_welder_manager):
+ self.curstage = 'initialize'
self.awm = arch_welder_manager
self.uid = uid
self.username = username
@@ -80,17 +81,14 @@
self.buildarches = []
self.sub_jobs = {}
self.failed = False
-
- # If we are building and SRPM, cvs_tag will be the SRPM file
- if CONFIG.get('use_srpm_not_cvs') == True:
- self.srpmpath = cvs_tag
- # SRPM builds short-circut to the prep stage (which is right after make_srpm)
+ self.no_cvs = CONFIG.get('use_srpm_not_cvs')
+ self.cvs_tag = cvs_tag
+ self.stage_dir = None
+ self.srpm_path = None
+ self.srpm_http_path = None
+ # Deal with straight SRPM builds
+ if self.no_cvs and self.curstage is 'initialize':
self.curstage = 'make_srpm'
- self.cvs_tag = None
- else:
- self.srpmpath = None
- self.curstage = 'initialize'
- self.cvs_tag = cvs_tag
def get_cur_stage(self):
return self.curstage
@@ -135,19 +133,21 @@
return archs
- def _make_stage_dir(self):
- self.pkgsubdir = '%s-%d/%s-%s' % (self.name, self.starttime, self.ver, self.release)
- self.target_and_pkg_subdir = '%s/%s' % (self.target, self.pkgsubdir)
- self.stage_dir = os.path.join(self.stages_root, self.curstage, self.target_and_pkg_subdir)
- if not os.path.exists(self.stage_dir):
- os.makedirs(self.stage_dir)
- return self.stage_dir
+ def _make_stage_dir(self, rootdir):
+ # The dir will look like this:
+ # /builder/devel/finished/foo-3463467347734/1.1.0-23
+ pkgsubdir = '%s-%d/%s-%s' % (self.name, self.starttime, self.ver, self.release)
+ stage_dir = os.path.join(rootdir, self.target, self.curstage, pkgsubdir)
+ if not os.path.exists(stage_dir):
+ os.makedirs(stage_dir)
+ return stage_dir
def _checkout(self):
self.curstage = 'checkout'
- self.tmpdir = tempfile.mkdtemp(prefix=self.cvs_tag, dir=CONFIG.get('tmpdir'))
- os.chdir(self.tmpdir)
+ dir_prefix = self.cvs_tag + "-"
+ self.checkout_tmpdir = tempfile.mkdtemp(prefix=dir_prefix, dir=CONFIG.get('tmpdir'))
+ os.chdir(self.checkout_tmpdir)
cmd = '%s co -r %s %s' % (CONFIG.get('cvs_cmd'), self.cvs_tag, self.package)
debugprint("%d: Running %s" % (self.uid, cmd))
s, o = commands.getstatusoutput(cmd)
@@ -157,10 +157,11 @@
self.email_result(resultstring=msg, subject=subj)
self.curstage = 'finished'
self.failed = True
+ shutil.rmtree(self.checkout_tmpdir, True)
return
if CONFIG.get('redhat_internal_cvs') == 1:
- os.chdir(os.path.join(self.tmpdir, self.package))
+ os.chdir(os.path.join(self.checkout_tmpdir, self.package))
cmd = '%s co common' % CONFIG.get('cvs_cmd')
debugprint("%d: Running %s" % (self.uid, cmd))
s, o = commands.getstatusoutput(cmd)
@@ -170,18 +171,20 @@
self.email_result(resultstring=msg, subject=subj)
self.curstage = 'finished'
self.failed = True
+ shutil.rmtree(self.checkout_tmpdir, True)
return
def _make_srpm(self):
self.curstage = 'make_srpm'
- self.srpmpath = None
- packagedir = os.path.join(self.tmpdir, self.package)
+ self.srpm_path = None
+ packagedir = os.path.join(self.checkout_tmpdir, self.package)
if not os.path.exists(packagedir):
subj = 'Prep Error: %s on %s' % (self.cvs_tag, self.target)
msg = "could not find path %s for %s." % (packagedir, self.cvs_tag)
self.email_result(resultstring=msg, subject=subj)
self.curstage = 'finished'
self.failed = True
+ shutil.rmtree(self.checkout_tmpdir, True)
return
if CONFIG.get('redhat_internal_cvs') == 1:
@@ -199,6 +202,7 @@
self.email_result(resultstring=msg, subject=subj)
self.curstage = 'finished'
self.failed = True
+ shutil.rmtree(self.checkout_tmpdir, True)
return
srpmpath = None
@@ -214,53 +218,62 @@
self.email_result(resultstring=msg, subject=subj)
self.curstage = 'finished'
self.failed = True
+ shutil.rmtree(self.checkout_tmpdir, True)
return
- self.srpmpath = srpmpath
+ self.srpm_path = srpmpath
def _prep(self):
self.curstage = 'prep'
ts = rpmUtils.transaction.initReadOnlyTransaction()
- hdr = rpmUtils.miscutils.hdrFromPackage(ts, self.srpmpath)
+ hdr = rpmUtils.miscutils.hdrFromPackage(ts, self.srpm_path)
self.name = hdr['name']
self.ver = hdr['version']
self.release = hdr['release']
self.buildarches = self.arch_handling(hdr)
+ del hdr
+ del ts
+
if len(self.buildarches) == 0:
self.failed = True
self.curstage = 'finished'
- del hdr
- del ts
- return;
- self._make_stage_dir()
+ return
+ self.stage_dir = self._make_stage_dir(self.stages_root)
self._createrepo(stage='needsign')
-
+
for arch in self.buildarches:
thisdir = os.path.join(self.stage_dir, arch)
if not os.path.exists(thisdir):
os.makedirs(thisdir)
- srpm = os.path.basename(self.srpmpath)
+ srpm = os.path.basename(self.srpm_path)
srpm_in_dir = os.path.join(self.stage_dir, srpm)
if os.path.exists(srpm_in_dir):
os.unlink(srpm_in_dir)
- shutil.move(self.srpmpath, self.stage_dir)
- self.srpmpath = srpm_in_dir
- del hdr
- del ts
-
+ shutil.copy(self.srpm_path, self.stage_dir)
+
+ # Must also copy it to where the build client can get it over HTTP
+ http_pkg_path = self._make_stage_dir(CONFIG.get('srpm_http_dir'))
+ self.srpm_http_path = os.path.join(http_pkg_path, srpm)
+ shutil.copy(self.srpm_path, self.srpm_http_path)
+ self.srpm_path = srpm_in_dir
+
+ # Remove CVS checkout and make_srpm dirs
+ shutil.rmtree(self.checkout_tmpdir, True)
+
def process(self):
# Advance to next stage based on current stage
- if self.curstage == 'initialize':
+ oldstage = self.curstage
+ if oldstage == 'initialize':
self._checkout()
- elif self.curstage == 'checkout':
+ elif oldstage == 'checkout':
self._make_srpm()
- elif self.curstage == 'make_srpm':
+ elif oldstage == 'make_srpm':
self._prep()
- elif self.curstage == 'prep' or self.curstage == 'building':
+ elif oldstage == 'prep' or oldstage == 'building':
self._monitor()
- elif self.curstage == 'finished':
+ elif oldstage == 'finished':
self._cleanup()
- elif self.curstage == 'cleanup':
+ elif oldstage == 'cleanup':
if self.failed:
self._failed()
else:
@@ -276,7 +289,11 @@
def _start_unspawned_builds(self):
for arch in self.buildarches:
if not self.sub_jobs.has_key(arch):
- job = self.awm.new_job_on_arch(self, arch, self.srpmpath, self.target, self.stage_dir)
+ # Construct SPRM URL
+ srpm_http_base = self.srpm_http_path[len(CONFIG.get('srpm_http_dir')):]
+ srpm_url = "http://" + CONFIG.get('hostname') + ":8886" + srpm_http_base
+ print "SRPM URL: " + srpm_url
+ job = self.awm.new_job_on_arch(self, self.target, arch, srpm_url)
if job:
if job.start() == True:
self.awm.track_job(job)
@@ -314,51 +331,57 @@
job.die()
def _failed(self):
+ old_stage = self.curstage
self.curstage = 'failed'
- old_stage = self.stage_dir
- dest = self._make_stage_dir()
- if os.path.exists(dest):
- shutil.rmtree(dest, ignore_errors=True)
- shutil.move(old_stage, dest)
- for job in self.sub_jobs.values():
- buildroot = 'fedora-%s-%s-core' % (self.target, job.arch)
- stage_arch = os.path.join(self.stage_dir, job.arch)
- build_log = '%s/mach/%s/%s-%s-%s/rpm.log' % (CONFIG.get('tmpdir'), buildroot,
- self.name, self.ver, self.release)
- if os.path.exists(build_log):
- bl = open(build_log, 'r')
- else:
- bl = None
- if not os.path.exists(stage_arch):
- os.makedirs(stage_arch)
- fn = '%s/%s-%s-%s.failure.log' % (stage_arch, self.name, self.ver, self.release)
- print "%d: Logfile (%s) is: %s" % (self.uid, job.arch, fn)
- logfile = open(fn, 'w')
+ if old_stage is not 'initialize' and old_stage is not 'checkout' and old_stage is not 'make_srpm':
+ old_stage_dir = self.stage_dir
+ dest = self._make_stage_dir(self.stages_root)
+ if os.path.exists(dest):
+ shutil.rmtree(dest, ignore_errors=True)
+ shutil.move(old_stage_dir, dest)
+ for job in self.sub_jobs.values():
+ buildroot = 'fedora-%s-%s-core' % (self.target, job.arch)
+ stage_arch = os.path.join(self.stage_dir, job.arch)
+ build_log = '%s/mach/%s/%s-%s-%s/rpm.log' % (CONFIG.get('tmpdir'), buildroot,
+ self.name, self.ver, self.release)
+ if os.path.exists(build_log):
+ bl = open(build_log, 'r')
+ else:
+ bl = None
+ if not os.path.exists(stage_arch):
+ os.makedirs(stage_arch)
+ fn = '%s/%s-%s-%s.failure.log' % (stage_arch, self.name, self.ver, self.release)
+ print "%d: Logfile (%s) is: %s" % (self.uid, job.arch, fn)
+ logfile = open(fn, 'w')
- if job.status == 'killed':
- lines = ['Build process terminated due to failure on another arch\n']
- else:
- lines = job.logs()
+ if job.status == 'killed':
+ lines = ['Build process terminated due to failure on another arch\n']
+ else:
+ lines = job.logs()
- for line in lines:
- logfile.write(line)
- if bl:
- for line in bl.readlines():
+ for line in lines:
logfile.write(line)
- bl.close()
- logfile.close()
-
- # markup status file
- resultstring = """
-%s: Build of %s on %s failed to complete on one or more archs. Please see logs at:
-%s/%s/%s""" % (self.uid, self.name, self.target, CONFIG.get('log_url'), self.target, self.name)
+ if bl:
+ for line in bl.readlines():
+ logfile.write(line)
+ bl.close()
+ logfile.close()
+
+ # markup status file
+ resultstring = """
+ %s: Build of %s on %s failed to complete on one or more archs. Please see logs at:
+ %s/%s/%s""" % (self.uid, self.name, self.target, CONFIG.get('log_url'), self.target, self.name)
+ else:
+ resultstring = """
+ %s: Build of %s (%s) on %s failed to complete on one or more archs.
+""" % (self.uid, self.package, self.cvs_tag, self.target)
self.email_result(resultstring)
return False
def _succeeded(self):
self.curstage = 'needsign'
old_stage = self.stage_dir
- dest = self._make_stage_dir()
+ dest = self._make_stage_dir(self.stages_root)
if os.path.exists(dest):
shutil.rmtree(dest, ignore_errors=True)
shutil.move(old_stage, dest)
- Previous message (by thread): extras-buildsys/client srpm_download.py, NONE, 1.1 archwelder.py, 1.1.1.1, 1.2 archwelder_config.py, 1.1.1.1, 1.2 fileserver.py, 1.1.1.1, 1.2
- Next message (by thread): web/html/docs/selinux-faq index.php,1.1.1.1,1.2
- Messages sorted by:
[ date ]
[ thread ]
[ subject ]
[ author ]
More information about the fedora-extras-commits
mailing list