extras-buildsys/builder CONFIG.py,1.3,1.4 builder.py,1.4,1.5
Daniel Williams (dcbw)
fedora-extras-commits at redhat.com
Tue Jul 5 21:07:54 UTC 2005
Author: dcbw
Update of /cvs/fedora/extras-buildsys/builder
In directory cvs-int.fedora.redhat.com:/tmp/cvs-serv3768/builder
Modified Files:
CONFIG.py builder.py
Log Message:
2005-07-05 Dan Williams <dcbw at redhat.com>
* Rework a bunch of stuff so the build server doesn't use quite as much CPU,
also split out stuff from server/client_manager.py
* Generalize the common/ classes to provide both SSL and non-SSL facilities,
renaming a lot of those files in the process
* Fix non-SSL builder/server and client/server communication
Note: At this time, SSL may be broken.
Index: CONFIG.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/builder/CONFIG.py,v
retrieving revision 1.3
retrieving revision 1.4
diff -u -r1.3 -r1.4
--- CONFIG.py 1 Jul 2005 11:51:27 -0000 1.3
+++ CONFIG.py 5 Jul 2005 21:07:52 -0000 1.4
@@ -18,6 +18,7 @@
config_opts['distro_name'] = "fedora"
config_opts['repo_name'] = "core"
+config_opts['ssl_buildserver'] = False
config_opts['fileserver_port'] = 8889
config_opts['xmlrpc_port'] = 8888
@@ -34,3 +35,6 @@
# WARNING: this directory is world-readable via HTTP!
config_opts['builder_work_dir'] = "/tmp/builder_work"
+# User to drop privilege level to since we can't run mock as root,
+# but we have to run the HTTP server as root so it can chroot
+config_opts['builder_user'] = "builder"
Index: builder.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/builder/builder.py,v
retrieving revision 1.4
retrieving revision 1.5
diff -u -r1.4 -r1.5
--- builder.py 1 Jul 2005 11:51:27 -0000 1.4
+++ builder.py 5 Jul 2005 21:07:52 -0000 1.5
@@ -31,9 +31,9 @@
import urllib
import errno
from plague import FileDownloader
-from plague import SimpleSSLXMLRPCServer
+from plague import AuthedXMLRPCServer
from plague import lighttpdManager
-from plague import SimpleHTTPSServer
+from plague import HTTPServer
from M2Crypto import threading as m2thread
@@ -57,11 +57,15 @@
return None
file_part = file_path[len(config_opts["builder_work_dir"]) + 1:]
port = "%s" % config_opts['fileserver_port']
- full_url = "https://" + g_our_hostname + ":" + port + "/" + file_part
+ if config_opts['ssl_buildserver']:
+ method = "https://"
+ else:
+ method = "http://"
+ full_url = method + g_our_hostname + ":" + port + "/" + file_part
return urllib.quote(full_url)
-class BuildClientMock:
+class BuilderMock:
"""puts things together for an arch - baseclass for handling builds for
other arches"""
@@ -372,47 +376,47 @@
return False
-class i386Arch(BuildClientMock):
+class i386Arch(BuilderMock):
def __init__(self, uniqid, target, buildarch, srpm_url):
self.buildroot = '%s-%s-i386-%s' % (config_opts['distro_name'], target, config_opts['repo_name'])
self.buildarch = buildarch
self.arch_command = '/usr/bin/setarch i686'
- BuildClientMock.__init__(self, uniqid, target, srpm_url)
+ BuilderMock.__init__(self, uniqid, target, srpm_url)
-class x86_64Arch(BuildClientMock):
+class x86_64Arch(BuilderMock):
def __init__(self, uniqid, target, buildarch, srpm_url):
self.buildroot = '%s-%s-x86_64-%s' % (config_opts['distro_name'], target, config_opts['repo_name'])
self.buildarch = buildarch
self.arch_command = ''
- BuildClientMock.__init__(self, uniqid, target, srpm_url)
+ BuilderMock.__init__(self, uniqid, target, srpm_url)
-class PPCArch(BuildClientMock):
+class PPCArch(BuilderMock):
def __init__(self, uniqid, target, buildarch, srpm_url):
self.buildroot = '%s-%s-ppc-%s' % (config_opts['distro_name'], target, config_opts['repo_name'])
self.buildarch = buildarch
self.arch_command = ''
- BuildClientMock.__init__(self, uniqid, target, srpm_url)
+ BuilderMock.__init__(self, uniqid, target, srpm_url)
-class PPC64Arch(BuildClientMock):
+class PPC64Arch(BuilderMock):
def __init__(self, uniqid, target, buildarch, srpm_url):
self.buildroot = '%s-%s-ppc64-%s' % (config_opts['distro_name'], target, config_opts['repo_name'])
self.buildarch = buildarch
self.arch_command = ''
- BuildClientMock.__init__(self, uniqid, target, srpm_url)
+ BuilderMock.__init__(self, uniqid, target, srpm_url)
-class SparcArch(BuildClientMock):
+class SparcArch(BuilderMock):
def __init__(self, uniqid, target, buildarch, srpm_url):
self.buildroot = '%s-%s-sparc-%s' % (config_opts['distro_name'], target, config_opts['repo_name'])
self.buildarch = buildarch
self.arch_command = '/usr/bin/sparc32'
- BuildClientMock.__init__(self, uniqid, target, srpm_url)
+ BuilderMock.__init__(self, uniqid, target, srpm_url)
-class Sparc64Arch(BuildClientMock):
+class Sparc64Arch(BuilderMock):
def __init__(self, uniqid, target, buildarch, srpm_url):
self.buildroot = '%s-%s-sparc64-%s' % (config_opts['distro_name'], target, config_opts['repo_name'])
self.buildarch = buildarch
self.arch_command = '/usr/bin/sparc64'
- BuildClientMock.__init__(self, uniqid, target, srpm_url)
+ BuilderMock.__init__(self, uniqid, target, srpm_url)
# Keep this global scope, used in __main__
builder_dict = {'i386': i386Arch,
@@ -432,7 +436,7 @@
'sparc64': Sparc64Arch
}
-def getBuildClient(uniqid, target, buildarch, srpm_url, localarches):
+def getArchBuilder(uniqid, target, buildarch, srpm_url, localarches):
"""hand it an arch it hands you back the builder instance you need"""
if not builder_dict.has_key(buildarch):
@@ -454,7 +458,7 @@
if config_opts['debug']:
print string
-class XMLRPCBuildClientServer:
+class XMLRPCBuilderServer:
def __init__(self, localarches):
self.ids = {} # unique id => awclass instance
self.localarches = localarches
@@ -462,75 +466,122 @@
def process(self):
# Give jobs some time to update their status and do their thing
- job = 0
- for (uniqid, bcp) in self.ids.iteritems():
- if not bcp.is_done_status():
- bcp.process()
- job = uniqid
- self.cur_job = job # Update current job
+ jobid = 0
+ for (uniqid, job) in self.ids.iteritems():
+ if not job.is_done_status():
+ job.process()
+ jobid = uniqid
+ self.cur_job = jobid # Update current job
- def start(self, target, buildarch, srpm_url):
+ def start(self, target, arch, srpm_url):
if self.cur_job != 0:
log("Tried to build '%s' when already buiding something" % srpm_url)
return 0
cur_time = time.time()
- check = '%d %s %s %s' % (cur_time, target, buildarch, srpm_url)
+ check = '%d %s %s %s' % (cur_time, target, arch, srpm_url)
sum = sha.new()
sum.update(check)
uniqid = sum.hexdigest()
if target == 'devel':
target = 'development'
- bcp = getBuildClient(uniqid, target, buildarch, srpm_url, self.localarches)
- if bcp != None:
- self.ids[uniqid] = bcp
- bcp.start()
+ job = getArchBuilder(uniqid, target, arch, srpm_url, self.localarches)
+ if job != None:
+ self.ids[uniqid] = job
+ job.start()
filename = os.path.basename(srpm_url)
log("%s: started %s on %s arch %s at time %d" % (uniqid, filename,
- target, buildarch, cur_time))
+ target, arch, cur_time))
else:
log("%s: Failed request for %s on %s UNSUPPORTED arch %s at time %d" %
- (uniqid, srpm_url, target, buildarch, cur_time))
+ (uniqid, srpm_url, target, arch, cur_time))
uniqid = 0
self.cur_job = uniqid
return uniqid
- def status(self, uniqid=None):
- if not uniqid:
- uniqid = self.cur_job
- if not uniqid:
- return 'idle'
- try:
- bcp = self.ids[uniqid]
- except KeyError, e:
- bcp = None
- if not bcp:
- return 'idle'
- return bcp.status()
-
def die(self, uniqid):
- bcp = self.ids[uniqid]
- return bcp.die()
+ job = self.ids[uniqid]
+ return job.die()
def files(self, uniqid):
- bcp = self.ids[uniqid]
- return bcp.files()
+ job = self.ids[uniqid]
+ return job.files()
def repo_unlocked(self, uniqid):
- bcp = self.ids[uniqid]
- return bcp.repo_unlocked()
+ job = self.ids[uniqid]
+ return job.repo_unlocked()
def listjobs(self):
return self.ids.keys()
def get_cur_job(self):
""" Are we currently building something? """
- return self.cur_job
+ status = 'idle'
+ try:
+ job = self.ids[self.cur_job]
+ status = job.status()
+ except KeyError:
+ status = 'idle'
+ return (self.cur_job, status)
def supported_arches(self):
return self.localarches
+def drop_privs():
+ """
+ We can't and shouldn't run mock as root, so we drop privs.
+ We have to run the HTTP server as root though so it can chroot
+ to the fileserver directory.
+ """
+
+ if os.getuid() != 0:
+ return
+
+ import pwd
+ import grp
+
+ eu = config_opts['builder_user']
+ try:
+ uid = int(eu)
+ except ValueError:
+ try:
+ pwrec = pwd.getpwnam(eu)
+ except KeyError:
+ print "Username '%s' does not exist." % eu
+ return -1
+ uid = pwrec[2]
+ else:
+ try:
+ pwrec = pwd.getpwuid(uid)
+ except KeyError:
+ print "User ID %d doesn't exist." % uid
+ return -1
+ gid = pwrec[3]
+
+ if uid == 0:
+ print "You cannot use the superuser as the 'builder_user' option."
+ return -1
+
+ # Make ourself members of the mock group build_user's group
+ try:
+ mock_req = grp.getgrnam('mock')
+ except KeyError:
+ print "Mock group doesn't exist."
+ return -1
+ groups = [mock_req[2], gid]
+ os.setgroups(groups)
+
+ try:
+ os.setgid(gid)
+ except OSError:
+ print "Could drop group privileges. Error: '%s'" % sys.exc_info()
+ return -1
+
+ os.setuid(uid)
+ return 0
+
+
if __name__ == '__main__':
if len(sys.argv) < 3:
print "Usage:\n"
@@ -552,26 +603,46 @@
g_our_hostname = sys.argv[1]
localarches = sys.argv[2:]
- print "Binding to address '%s' with arches: [%s]" % (g_our_hostname, string.join(localarches))
+ work_dir = config_opts['builder_work_dir']
+ if not os.path.exists(work_dir) or not os.access(work_dir, os.R_OK):
+ print "%s does not exist or is not readable." % work_dir
+ os._exit(1)
m2thread.init()
- xmlrpc_port = config_opts['xmlrpc_port']
- xmlserver = SimpleSSLXMLRPCServer.SimpleSSLXMLRPCServer(certs, (g_our_hostname, xmlrpc_port))
- bcs = XMLRPCBuildClientServer(localarches)
- xmlserver.register_instance(bcs)
-
# Start up the HTTP server thread which the build server
# pulls completed RPMs from
- work_dir = config_opts['builder_work_dir']
port = config_opts['fileserver_port']
if use_lighttpd:
http_cnf_file = "/var/tmp/plague-builder-%s.conf" % os.getpid()
- http_server = lighttpdManager.lighttpdManager(http_cnf_file, g_our_hostname, port, work_dir, config_opts['builder_key_and_cert'])
+ if config_opts['ssl_buildserver']:
+ key_and_cert = config_opts['builder_key_and_cert']
+ else:
+ key_and_cert = None
+ http_server = lighttpdManager.lighttpdManager(http_cnf_file, g_our_hostname, port, work_dir, True, key_and_cert)
else:
- http_server = SimpleHTTPSServer.SimpleHTTPSServer(certs, (g_our_hostname, port), work_dir)
+ if config_opts['ssl_buildserver']:
+ http_server = HTTPServer.HTTPServer(certs, (g_our_hostname, port), work_dir, certs)
+ else:
+ http_server = HTTPServer.HTTPServer((g_our_hostname, port), work_dir, None)
http_server.start()
+ # Stop running as root
+ if drop_privs() == -1:
+ http_server.stop()
+ time.sleep(1)
+ m2thread.cleanup()
+ os._exit(1)
+
+ print "Binding to address '%s' with arches: [%s]" % (g_our_hostname, string.join(localarches))
+ xmlrpc_port = config_opts['xmlrpc_port']
+ if config_opts['ssl_buildserver']:
+ xmlserver = AuthedXMLRPCServer.AuthedSSLXMLRPCServer((g_our_hostname, xmlrpc_port), certs)
+ else:
+ xmlserver = AuthedXMLRPCServer.AuthedXMLRPCServer((g_our_hostname, xmlrpc_port))
+ bcs = XMLRPCBuilderServer(localarches)
+ xmlserver.register_instance(bcs)
+
last_time = time.time()
while True:
try:
More information about the fedora-extras-commits
mailing list