extras-buildsys/server BuildMaster.py, NONE, 1.1 BuildJob.py, 1.2, 1.3 CONFIG.py, 1.8, 1.9 UserInterface.py, 1.1, 1.2 buildserver.py, 1.6, 1.7 client_manager.py, 1.8, 1.9 buildmaster.py, 1.6, NONE
Daniel Williams (dcbw)
fedora-extras-commits at redhat.com
Wed Jun 15 04:10:10 UTC 2005
Author: dcbw
Update of /cvs/fedora/extras-buildsys/server
In directory cvs-int.fedora.redhat.com:/tmp/cvs-serv23432/server
Modified Files:
BuildJob.py CONFIG.py UserInterface.py buildserver.py
client_manager.py
Added Files:
BuildMaster.py
Removed Files:
buildmaster.py
Log Message:
2005-06-14 Dan Williams <dcbw at redhat.com>
* client/buildclient.py
- Write mock output to a log on failure
* server/buildmaster.py -> server/BuildMaster.py
* server/BuildJob.py
server/UserInterface.py
server/buildserver.py
- Fix for buildmaster.py -> BuildMaster.py
* server/BuildJob.py
server/BuildMaster.py
- BuildMaster object now has a 'createrepo' method, which BuildJobs
call when they need to update the repo. BuildMaster.createrepo() is
locked so that we never run two createrepos at the same time
* server/CONFIG.py
- Clarify some options
* server/UserInterface.py
- Remove list_waiting_jobs and list_building_jobs, we now have a more
flexible "list_jobs" interface that will be extended to allow more
search parameters
- In update_clients(), return new clients that we've found
* server/client_manager.py
- Print out clients we find when we start up
- In update_clients(), return new clients that we've found
* utils/package-builder.py
- Implement most of remaining functionality:
- hook up enqueue and enqueue_srpm commands
- Add a list_own_jobs command
- grab user's email address from certificate or ~/.package-builder
- Allow non-SSL connections to build server
--- NEW FILE BuildMaster.py ---
#!/usr/bin/python
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# copyright 2005 Duke University
# written by Seth Vidal
import time
import CONFIG
import BuildJob
import sqlite
import threading
import os
import commands
def ensure_build_db_tables(dbcx):
""" Central routine to create the database table structure """
curs = dbcx.cursor()
try:
curs.execute('SELECT * FROM jobs')
dbcx.commit()
except Exception, e:
# If DB wasn't created, try to create it
curs.execute('CREATE TABLE jobs (uid INTEGER PRIMARY KEY, ' \
'username VARCHAR(20), package VARCHAR(50), ' \
'cvs_tag VARCHAR(255), target VARCHAR(20), ' \
'buildreq VARCHAR(75), time_submitted BIGINT, ' \
'status VARCHAR(15))')
dbcx.commit()
class BuildMaster(threading.Thread):
def __init__(self, hostname, client_manager):
self.bcm = client_manager
self.hostname = hostname
self.building_jobs = []
self.should_stop = False
self.dbcx = sqlite.connect("jobdb", encoding="utf-8", timeout=2)
self.curs = self.dbcx.cursor()
self.createrepo_lock = threading.Lock()
ensure_build_db_tables(self.dbcx)
threading.Thread.__init__(self)
def __del__(self):
self.dbcx.close()
def stop(self):
self.should_stop = True
def set_job_status(self, job):
status = job.get_cur_stage()
job_uid = job.get_uid()
self.curs.execute('UPDATE jobs SET status="%s" WHERE uid=%d' \
% (status, job_uid))
def getClientManager(self):
return self.bcm
def createrepo(self, repodir):
""" We need to lock calls to createrepo so they don't get run at the same time """
if not os.path.exists(repodir):
os.makedirs(repodir)
self.createrepo_lock.acquire()
s, o = commands.getstatusoutput('/usr/bin/createrepo -q %s' % repodir)
self.createrepo_lock.release()
if s != 0:
print "createrepo failed with exit status %d" % s
def run(self):
while self.should_stop == False:
# Update all build clients and known jobs
self.bcm.process()
# Allow each job some processing time
for job in self.building_jobs:
self.set_job_status(job)
if job.get_cur_stage() == 'failed' or job.get_cur_stage() == 'needsign':
print "%s (%s): Job finished." % (job.get_uid(), job.package)
self.building_jobs.remove(job)
# Grab one waiting job from database and start it
self.curs.execute('SELECT uid, username, package, cvs_tag, target' \
' FROM jobs WHERE status="waiting"')
self.dbcx.commit()
item = self.curs.fetchone()
if item:
print "%s (%s): Starting tag '%s' on target '%s'" % (item['uid'], \
item['package'], item['cvs_tag'], item['target'])
job = BuildJob.BuildJob(item['uid'], item['username'], item['package'],
item['cvs_tag'], item['target'], self, self.hostname)
self.building_jobs.append(job)
job.start()
time.sleep(5)
Index: BuildJob.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/server/BuildJob.py,v
retrieving revision 1.2
retrieving revision 1.3
diff -u -r1.2 -r1.3
--- BuildJob.py 12 Jun 2005 08:23:43 -0000 1.2
+++ BuildJob.py 15 Jun 2005 04:10:07 -0000 1.3
@@ -72,9 +72,9 @@
class BuildJob(threading.Thread):
""" Controller object for building 1 SRPM on multiple arches """
- def __init__(self, uid, username, package, cvs_tag, target, client_manager, hostname):
+ def __init__(self, uid, username, package, cvs_tag, target, buildmaster, hostname):
self.curstage = 'initialize'
- self.bcm = client_manager
+ self.bm = buildmaster
self.hostname = hostname
self.uid = uid
self.username = username
@@ -363,7 +363,7 @@
# Construct SPRM URL
srpm_http_base = self.srpm_http_path[len(http_dir):]
srpm_url = "https://" + self.hostname + ":8886" + srpm_http_base
- job = self.bcm.start_arch_job(self, self.target, arch, srpm_url)
+ job = self.bm.bcm.start_arch_job(self, self.target, arch, srpm_url)
if job:
self.sub_jobs[arch] = job
log("%s (%s/%s): Builder UID is %s" % (self.uid, self.package, arch, job.jobid))
@@ -461,13 +461,7 @@
def _createrepo(self):
# createrepo on the needsign tree for new changes
repodir = os.path.join(CONFIG.get('repo_dir'), self.target)
- debugprint("%d: updating repodir %s" % (self.uid, repodir))
- if not os.path.exists(repodir):
- os.makedirs(repodir)
- s, o = commands.getstatusoutput('/usr/bin/createrepo -q %s' % repodir)
- if s != 0:
- print "Createrepo failed!!!"
- #self.curstage = 'failed'
- #raise PrepError(5, 'Error generating repodata for %s: %s' % (repodir, o))
+ self.bm.createrepo(repodir)
+ debugprint("%d: updated repodir %s" % (self.uid, repodir))
Index: CONFIG.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/server/CONFIG.py,v
retrieving revision 1.8
retrieving revision 1.9
diff -u -r1.8 -r1.9
--- CONFIG.py 14 Jun 2005 13:37:42 -0000 1.8
+++ CONFIG.py 15 Jun 2005 04:10:07 -0000 1.9
@@ -1,4 +1,4 @@
-# Configuration file for buildmaster.py
+# Configuration file for build system server
config_opts = {}
config_opts['email_to_domain'] = "redhat.com"
@@ -26,9 +26,14 @@
# MUST be full path to cert
config_opts['server_cert'] = SERVER_BASE_DIR + "/certs/server_cert.pem"
config_opts['server_key'] = SERVER_BASE_DIR + "/certs/server_key.pem"
+
+# CA cert that signs build client certificates
config_opts['ca_cert'] = SERVER_BASE_DIR + "/certs/ca_cert.pem"
+
+# CA cert that signed build job submitters' certificates
config_opts['ui_ca_cert'] = SERVER_BASE_DIR + "/certs/fedora-upload-ca.pem"
+
# server_work_dir
# - Where logs and finished RPMs are stored
config_opts['server_work_dir'] = "/rpmbuild"
Index: UserInterface.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/server/UserInterface.py,v
retrieving revision 1.1
retrieving revision 1.2
diff -u -r1.1 -r1.2
--- UserInterface.py 14 Jun 2005 13:37:42 -0000 1.1
+++ UserInterface.py 15 Jun 2005 04:10:08 -0000 1.2
@@ -22,7 +22,7 @@
import time
import sys
import os
-from buildmaster import ensure_build_db_tables
+import BuildMaster
def email_result(email, cvs_tag, resultstring, subject=None):
@@ -51,7 +51,7 @@
self.bcm = client_manager
self.dbcx = sqlite.connect("jobdb", encoding="utf-8", timeout=2)
self.curs = self.dbcx.cursor()
- ensure_build_db_tables(self.dbcx)
+ BuildMaster.ensure_build_db_tables(self.dbcx)
def __del__(self):
@@ -128,21 +128,22 @@
return (0, "Success: package has been queued.", jobid)
- def list_waiting_jobs(self):
- self.curs.execute('SELECT uid, username, package, cvs_tag, target' \
- ' FROM jobs WHERE status="waiting"')
- self.dbcx.commit()
- data = self.curs.fetchall()
- job_list = []
- for row in data:
- tempX = [ item for item in row]
- job_list.append(tempX)
- return job_list
-
-
- def list_building_jobs(self):
- self.curs.execute('SELECT uid, username, package, cvs_tag, target' \
- ' FROM jobs WHERE status="building"')
+ def list_jobs(self, args_dict):
+ sql = 'SELECT uid, username, package, cvs_tag, target, status FROM jobs'
+ search = ''
+ first = True
+
+ if args_dict.has_key('email') and args_dict['email']:
+ if first:
+ search = search + " WHERE "
+ else:
+ search = search + "AND "
+ search = search + 'username LIKE "%%%s%%" ' % args_dict['email']
+ first = False
+
+ if len(search):
+ sql = sql + search
+ self.curs.execute(sql)
self.dbcx.commit()
data = self.curs.fetchall()
job_list = []
@@ -154,11 +155,8 @@
def update_clients(self):
reload(CONFIG)
- print "-----------------------------------------------------"
- print " Looking for Build Clients..."
- self.bcm.update_clients()
- print "-----------------------------------------------------\n"
- return (0, "Success.")
+ client_list = self.bcm.update_clients()
+ return (0, "Success.", client_list)
@@ -179,12 +177,8 @@
return UserInterface.enqueue_srpm(self, user.email, package, cvs_tag, target, buildreq)
- def list_waiting_jobs(self, user):
- return UserInterface.list_waiting_jobs(self)
-
-
- def list_building_jobs(self, user):
- return UserInterface.list_building_jobs(self)
+ def list_jobs(self, user, args_dict):
+ return UserInterface.list_jobs(self, args_dict)
def update_clients(self, user):
@@ -198,22 +192,3 @@
Allow all operations, NULL authentication
"""
- def enqueue(self, email, package, cvs_tag, target, buildreq=None):
- return UserInterface.enqueue(self, email, package, cvs_tag, target, buildreq)
-
-
- def enqueue_srpm(self, email, package, srpm_file, target, buildreq=None):
- return UserInterface.enqueue_srpm(self, email, package, cvs_tag, target, buildreq)
-
-
- def list_waiting_jobs(self):
- return UserInterface.list_waiting_jobs(self)
-
-
- def list_building_jobs(self):
- return UserInterface.list_building_jobs(self)
-
-
- def update_clients(self):
- return UserInterface.update_clients(self)
-
Index: buildserver.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/server/buildserver.py,v
retrieving revision 1.6
retrieving revision 1.7
diff -u -r1.6 -r1.7
--- buildserver.py 14 Jun 2005 13:37:42 -0000 1.6
+++ buildserver.py 15 Jun 2005 04:10:08 -0000 1.7
@@ -19,7 +19,7 @@
import CONFIG
import sys
import os
-from buildmaster import BuildMaster
+import BuildMaster
from client_manager import BuildClientManager
import SimpleHTTPSServer
import SimpleSSLXMLRPCServer
@@ -75,7 +75,7 @@
bcm = BuildClientManager()
# Create the BuildMaster thread
- bm = BuildMaster(hostname, bcm)
+ bm = BuildMaster.BuildMaster(hostname, bcm)
bm.start()
# SSL certificate and key filenames
@@ -106,7 +106,7 @@
srpm_server = SimpleHTTPSServer.SimpleHTTPSServer(srpm_server_certs, (hostname, 8886), http_dir)
srpm_server.start()
- print "BuildMaster accepting requests on %s:8887.\n" % hostname
+ print "Build Server accepting requests on %s:8887.\n" % hostname
try:
bm_server.serve_forever()
except KeyboardInterrupt:
Index: client_manager.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/server/client_manager.py,v
retrieving revision 1.8
retrieving revision 1.9
diff -u -r1.8 -r1.9
--- client_manager.py 15 Jun 2005 01:41:12 -0000 1.8
+++ client_manager.py 15 Jun 2005 04:10:08 -0000 1.9
@@ -285,19 +285,37 @@
else:
return True
+ def to_dict(self):
+ client_dict = {}
+ client_dict['address'] = self._address
+ client_dict['arches'] = self._arches
+ if self._cur_job:
+ client_dict['status'] = 'building'
+ else:
+ client_dict['status'] = 'idle'
+ return client_dict
class BuildClientManager:
def __init__(self):
# List of addresses of possible builders
self.possible_clients = CONFIG.get('builders')
self.running_clients = []
+ client_list = self.update_clients()
+
+ # Print out client list when starting up
+ print "\nBuild Clients:"
+ print "-" * 60
+ for client in client_list:
+ string = " " + client['address']
+ string = string + " " * (40 - len(client['address']))
+ for arch in client['arches']:
+ string = string + arch + " "
+ print string
+ print ""
- print "-----------------------------------------------------"
- print " Looking for BuildClients..."
- self.update_clients()
- print "-----------------------------------------------------\n"
def update_clients(self):
+ client_list = []
for address in self.possible_clients:
# If the address is already in our running_clients list, skip it
skip = False
@@ -311,11 +329,13 @@
# list if we can
client = BuildClient(self, address)
if client.valid():
- print " New Client: '%s' [%s]" % (address, string.join(client.arches()))
+ client_list.append(client.to_dict())
self.running_clients.append(client)
else:
del client
+ return client_list
+
def process(self):
""" Allow each BuildClient to update its status and do some processing """
for client in self.running_clients:
--- buildmaster.py DELETED ---
More information about the fedora-extras-commits
mailing list