extras-buildsys/common lighttpdManager.py, NONE, 1.1 FileDownloader.py, 1.8, 1.9 Makefile, 1.2, 1.3 SSLCommon.py, 1.6, 1.7 SimpleHTTPSServer.py, 1.5, 1.6
Daniel Williams (dcbw)
fedora-extras-commits at redhat.com
Fri Jul 1 11:51:35 UTC 2005
- Previous message (by thread): extras-buildsys/server BuildJob.py, 1.14, 1.15 CONFIG.py, 1.13, 1.14 buildserver.py, 1.11, 1.12 client_manager.py, 1.26, 1.27
- Next message (by thread): common tobuild,1.1658,1.1659
- Messages sorted by:
[ date ]
[ thread ]
[ subject ]
[ author ]
Author: dcbw
Update of /cvs/fedora/extras-buildsys/common
In directory cvs-int.fedora.redhat.com:/tmp/cvs-serv11305/common
Modified Files:
FileDownloader.py Makefile SSLCommon.py SimpleHTTPSServer.py
Added Files:
lighttpdManager.py
Log Message:
2005-07-01 Dan Williams <dcbw at redhat.com>
* We now use lighttpd as the fileserver in both the server and the builder. It's
a lot more robust than the python implementation, but the python implementation
code still exists and can be used.
* The server key and certificate must now be in the same file. You can 'cat' them
together with 'cat key.pem cert.pem > key_and_cert.pem'
* Initialize m2crypto threading in buildservery.py and builder.py
* Change some config options for the builder that said 'client' to 'builder'
* Add some testing code to SimpleHTTPSServer.py and FileDownloader.py
--- NEW FILE lighttpdManager.py ---
#!/usr/bin/python -t
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
# Copyright 2005 Dan Williams and Red Hat, Inc.
#
import os
import shutil
import fcntl
import threading
import time
import popen2
class lighttpdManager(threading.Thread):
def __init__(self, conffile, hostname, port, basedir, key_and_cert):
self._stop = False
self._pobj = None
self._conffile = conffile
self._port = port
self._hostname = hostname
self._basedir = basedir
self._key_and_cert = key_and_cert
threading.Thread.__init__(self)
if not self._setup():
os._exit(1)
self._start_server()
def _setup(self):
if not os.path.exists(self._basedir):
os.makedirs(self._basedir)
if not os.access(self._key_and_cert, os.R_OK):
print "%s does not exist or is not readable." % self._key_and_cert
return False
if not os.path.exists(os.path.dirname(self._conffile)):
os.makedirs(os.path.dirname(self._conffile))
cnf = '''
server.chroot = "%s"
server.document-root = "/"
mimetype.assign = (
".rpm" => "application/x-rpm",
".log" => "text/plain"
)
server.port = %d
server.bind = "%s"
server.username = "lighttpd"
server.groupname = "lighttpd"
server.dir-listing = "disable"
ssl.engine = "enable"
ssl.pemfile = "%s"
''' % (self._basedir, self._port, self._hostname, self._key_and_cert)
f = open(self._conffile, "w")
f.write(cnf)
f.close()
return True
def _start_server(self):
cmd = "/usr/sbin/lighttpd -f %s -D" % self._conffile
self._pobj = popen2.Popen4(cmd=cmd, bufsize=1024)
fcntl.fcntl(self._pobj.fromchild.fileno(), fcntl.F_SETFL, os.O_NONBLOCK)
def run(self):
while not self._stop:
if self._pobj:
st = self._pobj.poll()
# Restart it if it fails
if st >= 0:
print "HTTP server failed, trying to restart..."
print "----------------------------------------"
for line in self._pobj.fromchild.readlines():
print line
print "----------------------------------------\n"
del self._pobj
if not self._setup():
os._exit(1)
self._start_server()
time.sleep(5)
if self._pobj:
os.kill(self._pobj.pid, 15)
os.remove(self._conffile)
def stop(self):
self._stop = True
Index: FileDownloader.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/common/FileDownloader.py,v
retrieving revision 1.8
retrieving revision 1.9
diff -u -r1.8 -r1.9
--- FileDownloader.py 21 Jun 2005 17:54:58 -0000 1.8
+++ FileDownloader.py 1 Jul 2005 11:51:33 -0000 1.9
@@ -102,9 +102,83 @@
except socket.error, e:
if CommonErrors.canIgnoreSocketError(e):
pass
+ except IOError, e:
+ if CommonErrors.canIgnoreSocketError(e):
+ pass
if result:
self._callback('done', self._cb_data)
else:
self._callback('failed', self._cb_data)
+
+
+###########################################################
+# Testing stuff
+###########################################################
+
+import sys, time
+from M2Crypto import threading as m2thread
+
+class dlwr:
+ def __init__(self, x, t):
+ self.num = x
+ self.tracker = t
+ self.dl = None
+
+ def set_dl(self, dl):
+ self.dl = dl
+
+
+class dl_tracker:
+ def __init__(self):
+ self.lst = []
+
+ def add(self, dlwr):
+ self.lst.append(dlwr)
+
+ def remove(self, dlwr):
+ self.lst.remove(dlwr)
+
+ def num(self):
+ return len(self.lst)
+
+
+def dl_callback(status, dlwr):
+ print "Finished with %d (%s)" % (dlwr.num, status)
+ dlwr.tracker.remove(dlwr)
+
+
+if __name__ == '__main__':
+ if len(sys.argv) < 4:
+ print "Usage: python FileDownloader.py key_and_cert ca_cert peer_ca_cert"
+ sys.exit(1)
+
+ certs = {}
+ certs['key_and_cert'] = sys.argv[1]
+ certs['ca_cert'] = sys.argv[2]
+ certs['peer_ca_cert'] = sys.argv[3]
+
+ print "Starting..."
+ dlt = dl_tracker()
+
+ m2thread.init()
+ x = 0
+ while x < 100:
+ wr = dlwr(x, dlt)
+ dstdir = os.path.join("/tmp", "client_dir", "%s" % x)
+ if not os.path.exists(dstdir):
+ os.makedirs(dstdir)
+ time.sleep(0.25)
+ dl = FileDownloader(dl_callback, wr, "https://localhost:8886/testfile.dat", dstdir, ['.dat'], certs)
+ dlt.add(wr)
+ dl.start()
+ x = x + 1
+
+ while dlt.num() > 0:
+ try:
+ time.sleep(1)
+ except KeyboardInterrupt:
+ print "Quitting..."
+ m2thread.cleanup()
+ os._exit(0)
Index: Makefile
===================================================================
RCS file: /cvs/fedora/extras-buildsys/common/Makefile,v
retrieving revision 1.2
retrieving revision 1.3
diff -u -r1.2 -r1.3
--- Makefile 28 Jun 2005 01:41:28 -0000 1.2
+++ Makefile 1 Jul 2005 11:51:33 -0000 1.3
@@ -15,6 +15,7 @@
SimpleSSLXMLRPCServer.py \
SSLCommon.py \
SSLXMLRPCServerProxy.py \
+ lighttpdManager.py \
__init__.py
clean:
Index: SSLCommon.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/common/SSLCommon.py,v
retrieving revision 1.6
retrieving revision 1.7
diff -u -r1.6 -r1.7
--- SSLCommon.py 27 Jun 2005 02:40:54 -0000 1.6
+++ SSLCommon.py 1 Jul 2005 11:51:33 -0000 1.7
@@ -29,17 +29,16 @@
"""
Helper method for m2crypto's SSL libraries.
"""
- cert = certs['cert']
- key = certs['key']
+ key_and_cert = certs['key_and_cert']
ca_cert = certs['ca_cert']
peer_ca_cert = certs['peer_ca_cert']
- for f in cert, key, ca_cert, peer_ca_cert:
+ for f in key_and_cert, ca_cert, peer_ca_cert:
if f and not os.access(f, os.R_OK):
print "%s does not exist or is not readable." % f
os._exit(1)
ctx = SSL.Context('sslv3') # SSLv3 only
- ctx.load_cert(cert, key)
+ ctx.load_cert(key_and_cert)
ctx.load_client_ca(ca_cert)
ctx.load_verify_info(peer_ca_cert)
ctx.set_allow_unknown_ca(False)
@@ -58,6 +57,7 @@
# because it hung or tracebacked
timeout = SSL.timeout(10)
self.socket.set_socket_read_timeout(timeout)
+ self.socket.set_socket_write_timeout(timeout)
def handle_request(self):
request = None
@@ -66,6 +66,7 @@
request, client_address = self.get_request()
timeout = SSL.timeout(10)
request.set_socket_read_timeout(timeout)
+ request.set_socket_write_timeout(timeout)
if self.verify_request(request, client_address):
self.process_request(request, client_address)
except SSL.SSLError, e:
Index: SimpleHTTPSServer.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/common/SimpleHTTPSServer.py,v
retrieving revision 1.5
retrieving revision 1.6
diff -u -r1.5 -r1.6
--- SimpleHTTPSServer.py 27 Jun 2005 02:40:54 -0000 1.5
+++ SimpleHTTPSServer.py 1 Jul 2005 11:51:33 -0000 1.6
@@ -19,7 +19,6 @@
import SimpleHTTPServer
import SocketServer
import threading
-import os
import urllib
import posixpath
import os, sys
@@ -29,6 +28,7 @@
import SSLCommon
import socket
import time
+from M2Crypto import threading as m2thread
class HttpRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
@@ -90,6 +90,7 @@
# because it hung or tracebacked
timeout = SSL.timeout(10)
self.socket.set_socket_read_timeout(timeout)
+ self.socket.set_socket_write_timeout(timeout)
def finish(self):
if self.request:
@@ -116,8 +117,55 @@
def __init__(self, certs, addr, http_dir):
self._server = ThreadingHTTPSServer(certs, addr, http_dir)
+ self._stop = False
threading.Thread.__init__(self)
def run(self):
- self._server.serve_forever()
+ while not self._stop:
+ self._server.handle_request()
+ def stop(self):
+ self._stop = True
+
+
+###########################################################
+# Testing stuff
+###########################################################
+
+if __name__ == '__main__':
+ if len(sys.argv) < 4:
+ print "Usage: python SimpleHTTPSServer.py key_and_cert ca_cert peer_ca_cert"
+ sys.exit(1)
+
+ certs = {}
+ certs['key_and_cert'] = sys.argv[1]
+ certs['ca_cert'] = sys.argv[2]
+ certs['peer_ca_cert'] = sys.argv[3]
+
+ print "Starting..."
+
+ srcdir = os.path.join("/tmp", "server_dir")
+ if not os.path.exists(srcdir):
+ os.makedirs(srcdir)
+
+ print "Creating random test data..."
+ f = open(os.path.join(srcdir, "testfile.dat"), "w")
+ x = 1
+ while x < 10000:
+ f.write(os.urandom(50))
+ x = x + 1
+ f.close()
+
+ m2thread.init()
+
+ print "Starting the server."
+ server = SimpleHTTPSServer(certs, ('localhost', 8886), srcdir)
+ server.start()
+
+ while True:
+ try:
+ time.sleep(1)
+ except KeyboardInterrupt:
+ print "Quitting..."
+ m2thread.cleanup()
+ os._exit(0)
- Previous message (by thread): extras-buildsys/server BuildJob.py, 1.14, 1.15 CONFIG.py, 1.13, 1.14 buildserver.py, 1.11, 1.12 client_manager.py, 1.26, 1.27
- Next message (by thread): common tobuild,1.1658,1.1659
- Messages sorted by:
[ date ]
[ thread ]
[ subject ]
[ author ]
More information about the fedora-extras-commits
mailing list