extras-buildsys/server ArchJob.py, 1.29, 1.30 Builder.py, 1.37, 1.38 BuilderManager.py, 1.22, 1.23
Daniel Williams (dcbw)
fedora-extras-commits at redhat.com
Tue May 9 06:05:01 UTC 2006
Author: dcbw
Update of /cvs/fedora/extras-buildsys/server
In directory cvs-int.fedora.redhat.com:/tmp/cvs-serv10735/server
Modified Files:
ArchJob.py Builder.py BuilderManager.py
Log Message:
2006-05-09 Dan Williams <dcbw at redhat.com>
* builder/Builder.py
- (upload_files): fix extraction of server address, and pass
the jobid to the server so it knows what the result files
are for
* builder/BuilderMock.py
- Fix name of ul_callback()
* common/FileTransfer.py
- Trap operational errors
* server/ArchJob.py
- (get_upload_dir): new function; return the directory into
which result files should be written
* server/Builder.py
- (get_archjob): new function; return archjob object for a
particular jobid
- (_handle_building_jobs): correct scoping of bits that get
status for certain jobs
- Increase the required active builder contact interval slightly
- (request): reset the unavailable count every time the builder
contacts us
* server/BuilderManager.py
- (__init__): Enable the HTTP POST handler when there are
active builders
- (upload_callback): handle incoming files by writing them
to the correct location based on their jobid
Index: ArchJob.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/server/ArchJob.py,v
retrieving revision 1.29
retrieving revision 1.30
diff -u -r1.29 -r1.30
--- ArchJob.py 9 May 2006 02:52:58 -0000 1.29
+++ ArchJob.py 9 May 2006 06:04:54 -0000 1.30
@@ -225,6 +225,12 @@
print "%s (%s/%s): Build result files - [ %s ]" % (self.par_job.uid,
self.par_job.package, self._target_dict['arch'], file_string)
+ def get_upload_dir(self):
+ upload_dir = os.path.join(self.par_job.get_stage_dir(), self._target_dict['arch'])
+ if not os.path.exists(upload_dir):
+ os.makedirs(upload_dir)
+ return upload_dir
+
def _status_downloading(self):
# Start grabbing the next undownloaded file, but only
# if we aren't already pulling one down
Index: Builder.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/server/Builder.py,v
retrieving revision 1.37
retrieving revision 1.38
diff -u -r1.37 -r1.38
--- Builder.py 9 May 2006 02:57:12 -0000 1.37
+++ Builder.py 9 May 2006 06:04:54 -0000 1.38
@@ -117,6 +117,13 @@
def stop(self):
self._stop = True
+ def get_archjob(self, archjob_id):
+ try:
+ return self._jobs[archjob_id]
+ except KeyError:
+ pass
+ return None
+
def _handle_builder_suspend(self, reason, msg):
for jobid in self._jobs.keys():
job = self._jobs[jobid]
@@ -196,22 +203,23 @@
except KeyError:
pass
- # We have to check jobs that weren't reported
- # as 'building' by the builder, since the job
- # may have finished on the builder and was
- # removed from the building job list before we
- # were able to know that it was done. HACK
- self._prepping_jobs = False
- for jobid in self._jobs.keys():
- # If the builder didn't report this job as building,
- # and its not done, explicitly get its status
- job = self._jobs[jobid]
- if jobid not in reported_uniqids and job.get_status() != 'done':
- new_cmds.append(Commands.PlgCommandJobStatus(jobid, self._seq_gen.next()))
-
- # Check for prepping jobs
- if job.prepping():
- self._prepping_jobs = True
+ # We have to check jobs that weren't reported
+ # as 'building' by the builder, since the job
+ # may have finished on the builder and was
+ # removed from the building job list before we
+ # were able to know that it was done. HACK
+ self._prepping_jobs = False
+ for jobid in self._jobs.keys():
+ # If the builder didn't report this job as building,
+ # and its not done, explicitly get its status
+ job = self._jobs[jobid]
+ if jobid not in reported_uniqids and job.get_status() != 'done':
+ print "Requesting job status for %s" % jobid
+ new_cmds.append(Commands.PlgCommandJobStatus(jobid, self._seq_gen.next()))
+
+ # Check for prepping jobs
+ if job.prepping():
+ self._prepping_jobs = True
del reported_uniqids
return new_cmds
@@ -510,7 +518,7 @@
punching holes through it.
"""
- _REQUIRED_CONTACT_INTERVAL = 20
+ _REQUIRED_CONTACT_INTERVAL = 25
def __init__(self, manager, cfg, address, weight, btype):
Builder.__init__(self, manager, cfg, address, weight, btype)
@@ -568,6 +576,8 @@
from the BuildMaster's XML-RPC server."""
self._last_contact = time.time()
+ self._unavail_count = 0
+ print "%s: builder contact" % self._last_contact
if not self._available:
self._handle_builder_reactivate(cmd_list)
Index: BuilderManager.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/server/BuilderManager.py,v
retrieving revision 1.22
retrieving revision 1.23
diff -u -r1.22 -r1.23
--- BuilderManager.py 28 Apr 2006 03:17:41 -0000 1.22
+++ BuilderManager.py 9 May 2006 06:04:54 -0000 1.23
@@ -185,8 +185,51 @@
certs['key_and_cert'] = cfg.get_str("SSL", "server_key_and_cert")
certs['ca_cert'] = cfg.get_str("SSL", "ca_cert")
certs['peer_ca_cert'] = cfg.get_str("SSL", "ca_cert")
- self._srpm_server = HTTPServer.PlgHTTPServerManager((hostname, port), http_dir, certs)
- self._srpm_server.start()
+ self._fileserver = HTTPServer.PlgHTTPServerManager((hostname, port), http_dir, certs)
+ if any_active:
+ self._fileserver.set_POST_handler('/upload', self.upload_callback)
+ self._fileserver.start()
+
+ def upload_callback(self, request_handler, fs):
+ # Ensure we know this builder
+ addr = request_handler.client_address[0]
+ builder = self.get_builder(addr, addr)
+ if not builder:
+ request_handler.send_error(403, "Unauthorized")
+ return
+
+ # Search for filename
+ fslist = [fs]
+ if not fs.name and not fs.filename and fs.value:
+ fslist = fs.value
+ jobid = filename = tmpfile = None
+ for item in fslist:
+ if item.name == 'archjob_id':
+ try:
+ jobid = str(item.value)
+ except ValueError:
+ pass
+ elif item.name == 'filedata':
+ filename = item.filename
+ tmpfile = item.file
+
+ if jobid and filename and tmpfile:
+ archjob = builder.get_archjob(jobid)
+ if archjob:
+ upload_dir = archjob.get_upload_dir()
+ import shutil, urllib
+ destpath = os.path.join(upload_dir, urllib.unquote(filename))
+ dest = file(destpath, "w+b")
+ shutil.copyfileobj(tmpfile, dest)
+ dest.close()
+ request_handler.send_response(200, "Success")
+ request_handler.send_header("Content-type", "text/html")
+ request_handler.end_headers()
+ request_handler.wfile.write("<html><body>Success!</body></html>")
+ else:
+ request_handler.send_error(400, "Invalid request for archjob_id %s" % jobid)
+ else:
+ request_handler.send_error(400, "Invalid request for %s" % request_handler.path)
def _print_builders(self):
# Print out builder list when starting up
@@ -213,7 +256,7 @@
builder.stop()
if self._xmlrpc_server:
self._xmlrpc_server.stop()
- self._srpm_server.stop()
+ self._fileserver.stop()
def _load_builders(self):
self._builders_lock.acquire()
More information about the fedora-extras-commits
mailing list