extras-buildsys/server CONFIG.py, 1.3, 1.4 buildjob.py, 1.4, 1.5 buildserver.py, 1.1, 1.2 client_manager.py, 1.2, 1.3
Daniel Williams (dcbw)
fedora-extras-commits at redhat.com
Thu Jun 9 01:57:43 UTC 2005
- Previous message (by thread): extras-buildsys/common HTTPSURLopener.py, NONE, 1.1 SSLXMLRPCServerProxy.py, NONE, 1.1 SimpleHTTPSServer.py, NONE, 1.1 SimpleSSLXMLRPCServer.py, NONE, 1.1 FileDownloader.py, 1.1, 1.2 FileServer.py, 1.1, NONE
- Next message (by thread): extras-buildsys ChangeLog,1.4,1.5
- Messages sorted by:
[ date ]
[ thread ]
[ subject ]
[ author ]
Author: dcbw
Update of /cvs/fedora/extras-buildsys/server
In directory cvs-int.fedora.redhat.com:/tmp/cvs-serv15057/server
Modified Files:
CONFIG.py buildjob.py buildserver.py client_manager.py
Log Message:
2005-06-08 Dan Williams <dcbw at redhat.com>
* Convert all client/server communication to SSL. You will now need certificates
(see the README file for how to set them all up) to get them to talk to each
other. Now requires m2crypto module as well.
Index: CONFIG.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/server/CONFIG.py,v
retrieving revision 1.3
retrieving revision 1.4
diff -u -r1.3 -r1.4
--- CONFIG.py 7 Jun 2005 12:10:23 -0000 1.3
+++ CONFIG.py 9 Jun 2005 01:57:41 -0000 1.4
@@ -1,7 +1,7 @@
# Configuration file for buildmaster.py
config_opts = {}
-config_opts['hostname'] = "localhost"
+config_opts['hostname'] = "127.0.0.1"
config_opts['email_to_domain'] = "redhat.com"
config_opts['email_from'] = "buildsys at fedoraproject.org"
config_opts['pkg_cvs_root'] = ":pserver:anonymous at cvs.fedora.redhat.com:/cvs/dist"
@@ -11,6 +11,12 @@
config_opts['tmpdir'] = "/tmp"
config_opts['log_url'] = "http://foo.foo.org/logs/"
+# SSL Cert and key bits
+# MUST be full path to cert
+config_opts['server_cert'] = "/work/fedora-cvs/extras-buildsys/server/certs/server_cert.pem"
+config_opts['server_key'] = "/work/fedora-cvs/extras-buildsys/server/certs/server_key.pem"
+config_opts['ca_cert'] = "/work/fedora-cvs/extras-buildsys/server/certs/ca_cert.pem"
+
# server_work_dir
# - Where logs and finished RPMs are stored
config_opts['server_work_dir'] = "/rpmbuild"
@@ -29,6 +35,7 @@
'devel' : ['i386']
}
+#config_opts['builders'] = [ 'http://172.16.83.112:8888', 'http://172.16.83.22:8888' ]
config_opts['builders'] = [ 'http://127.0.0.1:8888' ]
def get(key):
Index: buildjob.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/server/buildjob.py,v
retrieving revision 1.4
retrieving revision 1.5
diff -u -r1.4 -r1.5
--- buildjob.py 8 Jun 2005 15:55:57 -0000 1.4
+++ buildjob.py 9 Jun 2005 01:57:41 -0000 1.5
@@ -299,7 +299,7 @@
if not self.sub_jobs.has_key(arch):
# Construct SPRM URL
srpm_http_base = self.srpm_http_path[len(http_dir):]
- srpm_url = "http://" + CONFIG.get('hostname') + ":8886" + srpm_http_base
+ srpm_url = "https://" + CONFIG.get('hostname') + ":8886" + srpm_http_base
# print "SRPM URL: " + srpm_url
job = self.bcm.new_job_on_arch(self, self.target, arch, srpm_url)
if job:
Index: buildserver.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/server/buildserver.py,v
retrieving revision 1.1
retrieving revision 1.2
diff -u -r1.1 -r1.2
--- buildserver.py 7 Jun 2005 12:10:23 -0000 1.1
+++ buildserver.py 9 Jun 2005 01:57:41 -0000 1.2
@@ -30,7 +30,7 @@
from buildmaster import BuildMaster
from buildmaster import ensure_build_db_tables
from client_manager import BuildClientManager
-import FileServer
+import SimpleHTTPSServer
def email_result(username, cvs_tag, resultstring, subject=None):
"""send 'resultstring' to username"""
@@ -177,9 +177,15 @@
bm_server = MyXMLRPCServer((CONFIG.get('hostname'), 8887))
bm_server.register_instance(xmlrpc_bm)
+ # SSL certificate and key filenames
+ server_cert = CONFIG.get('server_cert')
+ server_key = CONFIG.get('server_key')
+ ca_cert = CONFIG.get('ca_cert')
+
# SRPM fileserver
http_dir = os.path.join(CONFIG.get('server_work_dir'), "srpm_http_dir")
- srpm_server = FileServer.FileServer((CONFIG.get('hostname'), 8886), http_dir)
+ srpm_server = SimpleHTTPSServer.SimpleHTTPSServer(server_cert, server_key, ca_cert,
+ (CONFIG.get('hostname'), 8886), http_dir)
srpm_server.start()
print "BuildMaster accepting requests on %s:8887.\n" % CONFIG.get('hostname')
Index: client_manager.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/server/client_manager.py,v
retrieving revision 1.2
retrieving revision 1.3
diff -u -r1.2 -r1.3
--- client_manager.py 8 Jun 2005 15:55:57 -0000 1.2
+++ client_manager.py 9 Jun 2005 01:57:41 -0000 1.3
@@ -24,6 +24,14 @@
import FileDownloader
import os
import urllib
+import SSLXMLRPCServerProxy
+
+
+# SSL certificate and key filenames
+server_cert = CONFIG.get('server_cert')
+server_key = CONFIG.get('server_key')
+ca_cert = CONFIG.get('ca_cert')
+
def result_is_finished(result):
if result == 'done' or result == 'killed' or result == 'failed':
@@ -48,9 +56,13 @@
self.downloads = {}
def start(self):
+ failed = False
try:
self.jobid = self._server.start(self.target, self.arch, self.srpm_url)
except Exception, e:
+ failed = True
+
+ if failed or self.jobid == 0:
print "Error starting job on host %s\n\t---error: %s" % (self.bci.address(), e)
self.status = 'failed'
return False
@@ -98,7 +110,8 @@
target_dir = os.path.join(self.parent_job.get_stage_dir(), self.arch)
if not os.path.exists(target_dir):
os.makedirs(target_dir)
- dl_thread = FileDownloader.FileDownloader(self.dl_callback, url, url, target_dir, ['.rpm', '.log'])
+ dl_thread = FileDownloader.FileDownloader(self.dl_callback, url, url,
+ target_dir, ['.rpm', '.log'], server_cert, server_key, ca_cert)
dl_thread.start()
undownloaded = True
self.downloads[url] = 1
@@ -155,12 +168,12 @@
class BuildClientInstance:
""" Tracks an single arch on an BuildClient """
- def __init__(self, awm, address, arch):
- self._awm = awm
+ def __init__(self, bcm, address, arch):
+ self._bcm = bcm
self._jobs = []
self._arch = arch
self._address = address
- self._server = xmlrpclib.Server(self._address)
+ self._server = SSLXMLRPCServerProxy.SSLXMLRPCServerProxy(server_cert, server_key, ca_cert, self._address)
self._unavail_count = 0
self._cur_job = self._get_cur_job()
@@ -187,6 +200,9 @@
else:
print "bci(%s): got error '%s' from AW while trying to get " \
"current job number" % (self._address, e)
+ except SSLError, e:
+ if e == "unexpected eof":
+ self._unavail_count = self._unavail_count + 1
else:
self._unavail_count = 0
if cur_job == 0:
@@ -220,13 +236,13 @@
class BuildClientManager:
def __init__(self):
# List of addresses of possible builders
- self.possible_aw = CONFIG.get('builders')
+ self.possible_clients = CONFIG.get('builders')
# Dict: arches => available builders
# Like so: [ 'i386':['10.0.0.1', '10.0.0.2'],
# 'x86_64':['10.0.0.3']
# ]
- self.running_aw = {}
+ self.running_clients = {}
print "-----------------------------------------------------"
print " Looking for BuildClients..."
@@ -237,10 +253,10 @@
# Figure out which clients are alive, and what they support
# We create a separate client instance for each arch on each builder,
# even though both instances talk to the same XMLRPC server on the builder
- for address in self.possible_aw:
- # If the address is already in our running_aw list, skip it
+ for address in self.possible_clients:
+ # If the address is already in our running_clients list, skip it
skip = False
- for bci_list in self.running_aw.values():
+ for bci_list in self.running_clients.values():
for bci in bci_list:
if address == bci.address():
skip = True
@@ -248,24 +264,27 @@
continue
arches = None
- server = xmlrpclib.Server(address)
+ server_cert = CONFIG.get('server_cert')
+ server_key = CONFIG.get('server_key')
+ ca_cert = CONFIG.get('ca_cert')
+ server = SSLXMLRPCServerProxy.SSLXMLRPCServerProxy(server_cert, server_key, ca_cert, address)
try:
arches = server.supported_arches()
except socket.error, e:
pass
if arches:
arches.append('noarch')
- print " New AW: '%s' [%s]" % (address, string.join(arches))
+ print " New Client: '%s' [%s]" % (address, string.join(arches))
for a in arches:
- if not self.running_aw.has_key(a):
- self.running_aw[a] = []
+ if not self.running_clients.has_key(a):
+ self.running_clients[a] = []
bci = BuildClientInstance(self, address, a)
- self.running_aw[a].append(bci)
+ self.running_clients[a].append(bci)
del server
def process(self):
""" Allow each BuildClientInstance to update its status and do some processing """
- for bci_list in self.running_aw.values():
+ for bci_list in self.running_clients.values():
for bci in bci_list:
if bci.process() == 1:
# Remove the BuildClientInstance from our lists
@@ -279,9 +298,9 @@
def new_job_on_arch(self, parent_job, target, arch, srpm_url):
""" Create a job on a free builder for this arch """
- if self.running_aw.has_key(arch):
- for aw in self.running_aw[arch]:
- if aw.available():
- return aw.new_job(parent_job, target, srpm_url)
+ if self.running_clients.has_key(arch):
+ for client in self.running_clients[arch]:
+ if client.available():
+ return client.new_job(parent_job, target, srpm_url)
return None
- Previous message (by thread): extras-buildsys/common HTTPSURLopener.py, NONE, 1.1 SSLXMLRPCServerProxy.py, NONE, 1.1 SimpleHTTPSServer.py, NONE, 1.1 SimpleSSLXMLRPCServer.py, NONE, 1.1 FileDownloader.py, 1.1, 1.2 FileServer.py, 1.1, NONE
- Next message (by thread): extras-buildsys ChangeLog,1.4,1.5
- Messages sorted by:
[ date ]
[ thread ]
[ subject ]
[ author ]
More information about the fedora-extras-commits
mailing list