extras-buildsys/common FileTransfer.py, NONE, 1.1 FileUploader.py, 1.1, 1.2 FileDownloader.py, 1.18, 1.19
Daniel Williams (dcbw)
fedora-extras-commits at redhat.com
Mon May 8 14:24:37 UTC 2006
Author: dcbw
Update of /cvs/fedora/extras-buildsys/common
In directory cvs-int.fedora.redhat.com:/tmp/cvs-serv28620/common
Modified Files:
FileUploader.py FileDownloader.py
Added Files:
FileTransfer.py
Log Message:
2006-05-08 Dan Williams <dcbw at redhat.com>
* common/FileTransfer.py
- Refactor common code from FileDownload.py and
FileUpload.py into FileTransfer.py
* common/FileDownload.py
- pylint cleanups
- Refactor for FileTransfer.py
- Allow multiple transfers for one FileDownloader object
* common/FileUploader.py
- pylint cleanups
- Refactor for FileTransfer.py
- Allow multiple uploads for one FileUploader object
--- NEW FILE FileTransfer.py ---
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
# Copyright 2006 Dan Williams <dcbw at redhat.com> and Red Hat, Inc.
import threading
import URLopener
import time
FT_RESULT_SUCCESS = 'success'
FT_RESULT_FAILED = 'failed'
FT_RESULT_CANCELED = 'canceled'
class FileTransfer(threading.Thread):
def __init__(self, url, certs=None):
self._callback = None
self._cb_data = None
self._cancel = False
self._tries = 5
self._opener = None
if not url:
raise Exception("Require a URL to download.")
self._url = url
if certs:
if type(certs) is not type({}):
raise ValueError("Certs must be a dict of certificate paths.")
self._certs = certs
self._opener = URLopener.PlgURLopener(self._certs, 20)
threading.Thread.__init__(self)
def set_tries(self, tries):
if tries <= 0:
raise ValueError("Tries must be larger than 0")
self._tries = tries
def set_callback(self, callback, cb_data):
self._callback = callback
self._cb_data = cb_data
def cancel(self):
self._cancel = True
def _action(self, data=None):
# Should be implemented by subclasses
return (False, "_action should be implemented by subclass")
def _process_one_transfer(self, data=None):
"""Retries a transfer for a specified number of times before failing."""
status = FT_RESULT_FAILED
result = msg = None
for attempt in range(1, self._tries + 1):
(result, msg) = self._action(data)
if result:
status = FT_RESULT_SUCCESS
break
if self._cancel:
status = FT_RESULT_CANCELED
break
time.sleep(5)
msg = "Transfer of timed out."
return (status, msg)
def run(self):
(status, msg) = self._process_one_transfer()
if self._callback:
self._callback(status, self._cb_data, msg)
Index: FileUploader.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/common/FileUploader.py,v
retrieving revision 1.1
retrieving revision 1.2
diff -u -r1.1 -r1.2
--- FileUploader.py 27 Mar 2006 04:12:11 -0000 1.1
+++ FileUploader.py 8 May 2006 14:24:30 -0000 1.2
@@ -23,40 +23,62 @@
import OpenSSL
import CommonErrors
import exceptions
+import FileTransfer
-class FileUploader(threading.Thread):
- def __init__(self, callback, cb_data, url, data, certs=None):
- self._callback = callback
- self._cb_data = cb_data
- self._url = url
- self._data = data
- self._opener = URLopener.PlgURLopener(certs, 20)
- threading.Thread.__init__(self)
+class FileUploader(FileTransfer.FileTransfer):
+ def __init__(self, url, files, filevar, cgi_vars=None, certs=None):
+ FileTransfer.FileTransfer.__init__(self, url, certs)
+ self._files = {}
+
+ if files and type(files) == type(""):
+ files = [files]
+ if not files or type(files) is not type([]):
+ raise ValueError("Require a list of files to upload.")
+ for fpath in files:
+ self._files[os.path.abspath(fpath)] = (None, None)
+ self._filevar = filevar
+
+ if cgi_vars:
+ if type(cgi_vars) is not type([]):
+ raise ValueError("cgi_vars must be a list of tuples")
+ for var in cgi_vars:
+ if type(var) is not type(()):
+ raise ValueError("cgi_vars must be a list of tuples")
+ self._cgi_vars = cgi_vars
- def run(self):
+ def _action(self, cgivars=None):
result = None
err_msg = None
- if self._url and self._data:
- try:
- result = self._opener.open(self._url, self._data)
-# except SSL.SSLError, e:
-# # Don't traceback on dropped connections or timeouts
-# if CommonErrors.canIgnoreSSLError(e):
-# pass
- except socket.error, e:
- if not CommonErrors.canIgnoreSocketError(e):
- err_msg = "Socket Error: %s" % e
- except IOError, e:
- if not CommonErrors.canIgnoreSocketError(e):
- err_msg = "IOError Error: %s" % e
-
- if result:
- self._callback('done', self._cb_data, err_msg)
- else:
- self._callback('failed', self._cb_data, err_msg)
-
+ try:
+ result = self._opener.open(self._url, cgivars)
+ except socket.error, exc:
+ if not CommonErrors.canIgnoreSocketError(exc):
+ err_msg = "Socket Error: %s" % exc
+ except IOError, exc:
+ if not CommonErrors.canIgnoreSocketError(exc):
+ err_msg = "IOError Error: %s" % exc
+ return (result, err_msg)
+ def run(self):
+ final_result = FileTransfer.FT_RESULT_SUCCESS
+ msg = None
+ for fpath in self._files.keys():
+ fd = open(fpath, "r")
+ cgivars = [(self._filevar, fd)]
+ cgivars = cgivars + self._cgi_vars
+ (result, msg) = self._process_one_transfer(cgivars)
+ fd.close()
+ self._files[fpath] = (result, msg)
+ if result == FileTransfer.FT_RESULT_FAILED:
+ final_result = FileTransfer.FT_RESULT_FAILED
+ msg = "Upload of %s failed because: %s" % (fpath, msg)
+ break
+ if self._cancel:
+ final_result = FileTransfer.FT_RESULT_CANCELED
+ break
+ if self._callback:
+ self._callback(final_result, self._cb_data, msg)
###########################################################
# Testing stuff
@@ -64,7 +86,7 @@
import sys, time
-class dlwr:
+class UlCallbackData:
def __init__(self, x, t):
self.num = x
self.tracker = t
@@ -74,62 +96,57 @@
self.dl = dl
-class dl_tracker:
+class UlTracker:
def __init__(self):
self.lst = []
- def add(self, dlwr):
- self.lst.append(dlwr)
+ def add(self, ulcbdata):
+ self.lst.append(ulcbdata)
- def remove(self, dlwr):
- self.lst.remove(dlwr)
+ def remove(self, ulcbdata):
+ self.lst.remove(ulcbdata)
def num(self):
return len(self.lst)
-def ul_callback(status, dlwr, msg=""):
- print "Finished with %d (%s: %s)" % (dlwr.num, status, msg)
- dlwr.tracker.remove(dlwr)
+def UploadCallback(status, ulcbdata, msg=""):
+ print "Finished with %d (%s: %s)" % (ulcbdata.num, status, msg)
+ ulcbdata.tracker.remove(ulcbdata)
if __name__ == '__main__':
if len(sys.argv) < 5:
- print "Usage: python FileUploader.py filename key_and_cert ca_cert peer_ca_cert"
+ print "Usage: python FileUploader.py key_and_cert ca_cert peer_ca_cert fname1 fname2 ..."
sys.exit(1)
certs = {}
- certs['key_and_cert'] = sys.argv[2]
- certs['ca_cert'] = sys.argv[3]
- certs['peer_ca_cert'] = sys.argv[4]
+ certs['key_and_cert'] = sys.argv[1]
+ certs['ca_cert'] = sys.argv[2]
+ certs['peer_ca_cert'] = sys.argv[3]
print "Starting..."
- dlt = dl_tracker()
+ dlt = UlTracker()
x = 0
- filename = sys.argv[1]
+ files = []
+ for fname in sys.argv[4:]:
+ files.append(os.path.abspath(fname))
+
while x < 100:
- wr = dlwr(x, dlt)
- time.sleep(0.25)
+ wr = UlCallbackData(x, dlt)
+ try:
+ time.sleep(0.25)
+ except KeyboardInterrupt:
+ break
data = []
data.append(('jobid', x))
data.append(('arch', 'i386'))
- import shutil, string
- name = os.path.basename(filename)
- parts = name.split(".")
- ext = parts[-1]
- firstpart = string.join(parts[:len(parts)-1], ".")
- try:
- os.makedirs("/tmp/foo")
- except:
- pass
- newname = os.path.join("/tmp/foo", firstpart + "-%s.%s" % (x, ext))
- shutil.copyfile(filename, newname)
- fd = open(newname, "r")
- data.append(('filedata', fd))
- dl = FileUploader(ul_callback, wr, "https://localhost:8886/upload", data, certs)
+ ful = FileUploader("https://localhost:8886/upload", files,
+ 'filedata', data, certs)
+ ful.set_callback(UploadCallback, wr)
dlt.add(wr)
- dl.start()
+ ful.start()
x = x + 1
while dlt.num() > 0:
Index: FileDownloader.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/common/FileDownloader.py,v
retrieving revision 1.18
retrieving revision 1.19
diff -u -r1.18 -r1.19
--- FileDownloader.py 27 Mar 2006 04:12:11 -0000 1.18
+++ FileDownloader.py 8 May 2006 14:24:30 -0000 1.19
@@ -14,18 +14,17 @@
#
# Copyright 2005 Dan Williams <dcbw at redhat.com> and Red Hat, Inc.
-import threading
import urllib
-import string
import os
import socket
-import URLopener
-import OpenSSL
import CommonErrors
import exceptions
+import time
+import FileTransfer
-class FileNameException(exceptions.Exception): pass
+class FileNameException(exceptions.Exception):
+ pass
def get_base_filename_from_url(url, legal_exts):
""" Safely unquotes a URL and gets the base file name from it.
@@ -65,91 +64,102 @@
# FIXME: what other validation can we do here?
safe_list = ['_', '-', '.', '+']
- for c in filename:
+ for char in filename:
# For now, legal characters are '_-.' plus alphanumeric
- if c in safe_list or c.isalnum():
+ if char in safe_list or char.isalnum():
pass
else:
- raise FileNameException("Illegal character '%s' encountered." % c)
+ raise FileNameException("Illegal character '%s' encountered." % char)
return filename
-class FileDownloader(threading.Thread):
- def __init__(self, callback, cb_data, url, target_dir, legal_exts, certs):
- self._callback = callback
- self._cb_data = cb_data
- self._url = url
+class FileDownloader(FileTransfer.FileTransfer):
+ def __init__(self, urls, target_dir, legal_exts, certs=None):
+ FileTransfer.FileTransfer.__init__(self, url, certs)
+
+ if not target_dir:
+ raise Exception("Require a target directory to download to.")
self._target_dir = target_dir
- # May fail with FileNameException, trapped elsewhere
- self._filename = get_base_filename_from_url(self._url, legal_exts)
- self._opener = URLopener.PlgURLopener(certs, 20)
- threading.Thread.__init__(self)
+ if type(urls) == type(""):
+ urls = [urls]
+ if type(urls) is not type([]):
+ raise ValueError("urls argument must be a list of URLs.")
+ for url in urls:
+ fname = get_base_filename_from_url(url, legal_exts)
+ if not fname:
+ raise FileNameException("Bad file name from url %s" % url)
+ self._files[url] = fname
- def run(self):
+ def _action(self, (url, fname)):
result = None
err_msg = None
- if self._url and self._target_dir and self._filename:
- if not os.path.exists(self._target_dir):
- os.makedirs(self._target_dir)
- target_file = os.path.join(self._target_dir, self._filename)
- try:
- result = self._opener.retrieve(self._url, target_file)
-# except SSL.SSLError, e:
-# # Don't traceback on dropped connections or timeouts
-# if CommonErrors.canIgnoreSSLError(e):
-# pass
- except socket.error, e:
- if not CommonErrors.canIgnoreSocketError(e):
- err_msg = "Socket Error: %s" % e
- except IOError, e:
- if not CommonErrors.canIgnoreSocketError(e):
- err_msg = "IOError Error: %s" % e
-
- if result:
- self._callback('done', self._cb_data, err_msg)
- else:
- self._callback('failed', self._cb_data, err_msg)
+ if not os.path.exists(self._target_dir):
+ os.makedirs(self._target_dir)
+ target_file = os.path.join(self._target_dir, fname)
+ try:
+ result = self._opener.retrieve(url, target_file)
+ except socket.error, exc:
+ if not CommonErrors.canIgnoreSocketError(exc):
+ err_msg = "Socket Error: %s" % exc
+ except IOError, exc:
+ if not CommonErrors.canIgnoreSocketError(exc):
+ err_msg = "IOError Error: %s" % exc
+ return (result, err_msg)
+ def run(self):
+ final_result = FileTransfer.FT_RESULT_SUCCESS
+ msg = None
+ for url in self._files.keys():
+ (result, msg) = self._process_one_transfer((url, self._files[url]))
+ if result == FileTransfer.FT_RESULT_FAILED:
+ final_result = FileTransfer.FT_RESULT_FAILED
+ msg = "Download of %s failed because: %s" % (url, msg)
+ break
+ if self._cancel:
+ final_result = FileTransfer.FT_RESULT_CANCELED
+ break
+ if self._callback:
+ self._callback(final_result, self._cb_data, msg)
###########################################################
# Testing stuff
###########################################################
-import sys, time
+import sys
-class dlwr:
- def __init__(self, x, t):
- self.num = x
- self.tracker = t
- self.dl = None
+class DlCallbackData:
+ def __init__(self, num, tracker):
+ self.num = num
+ self.tracker = tracker
+ self.fdl = None
- def set_dl(self, dl):
- self.dl = dl
+ def set_dl(self, fdl):
+ self.fdl = fdl
-class dl_tracker:
+class DlTracker:
def __init__(self):
self.lst = []
- def add(self, dlwr):
- self.lst.append(dlwr)
+ def add(self, dlcb):
+ self.lst.append(dlcbdata)
- def remove(self, dlwr):
- self.lst.remove(dlwr)
+ def remove(self, dlcbdata):
+ self.lst.remove(dlcbdata)
def num(self):
return len(self.lst)
-def dl_callback(status, dlwr, msg=""):
- print "Finished with %d (%s: %s)" % (dlwr.num, status, msg)
- dlwr.tracker.remove(dlwr)
+def dl_callback(status, dlcbdata, msg=""):
+ print "Finished with %d (%s: %s)" % (dlcbdata.num, status, msg)
+ dlcbdata.tracker.remove(dlcbdata)
-if __name__ == '__main__':
+def main():
if len(sys.argv) < 4:
print "Usage: python FileDownloader.py key_and_cert ca_cert peer_ca_cert"
sys.exit(1)
@@ -160,19 +170,21 @@
certs['peer_ca_cert'] = sys.argv[3]
print "Starting..."
- dlt = dl_tracker()
+ dlt = DlTracker()
- x = 0
- while x < 100:
- wr = dlwr(x, dlt)
- dstdir = os.path.join("/tmp", "client_dir", "%s" % x)
+ count = 0
+ while count < 100:
+ dlcb = DlCallbackData(count, dlt)
+ dstdir = os.path.join("/tmp", "client_dir", "%s" % count)
if not os.path.exists(dstdir):
os.makedirs(dstdir)
time.sleep(0.25)
- dl = FileDownloader(dl_callback, wr, "https://localhost:8886/testfile.dat", dstdir, ['.dat'], certs)
- dlt.add(wr)
- dl.start()
- x = x + 1
+ fdl = FileDownloader("https://localhost:8886/testfile.dat",
+ dstdir, ['.dat'], certs)
+ fdl.set_callback(dl_callback, dlcb)
+ dlt.add(dlcb)
+ fdl.start()
+ count = count + 1
while dlt.num() > 0:
try:
@@ -180,3 +192,6 @@
except KeyboardInterrupt:
print "Quitting..."
os._exit(0)
+
+if __name__ == '__main__':
+ main()
More information about the fedora-extras-commits
mailing list