check-mirrors check-mirrors.py,1.20,1.21
Michael Patrick McGrath (mmcgrath)
fedora-extras-commits at redhat.com
Wed Aug 2 14:50:58 UTC 2006
Author: mmcgrath
Update of /cvs/fedora/check-mirrors
In directory cvs-int.fedora.redhat.com:/tmp/cvs-serv18140
Modified Files:
check-mirrors.py
Log Message:
Created RepoDB for various db functions. This works but is not widly tested.
Index: check-mirrors.py
===================================================================
RCS file: /cvs/fedora/check-mirrors/check-mirrors.py,v
retrieving revision 1.20
retrieving revision 1.21
diff -u -r1.20 -r1.21
--- check-mirrors.py 1 Aug 2006 19:31:00 -0000 1.20
+++ check-mirrors.py 2 Aug 2006 14:50:55 -0000 1.21
@@ -66,38 +66,82 @@
def errorprint(error):
print >> sys.stderr, error
-def check_and_make_db(db):
- """
- verify that we can create the sqlite DB file
- """
- try:
- con = sqlite.connect(db)
- cursor = con.cursor()
- except sqlite.Error, errmsg:
- errorprint('Failed to connect to database: %s' % db)
- errorprint('Err: %s ' % errmsg)
- return None, None
+def debugprint(debugmsg):
+ if debug:
+ print debugmsg
- try:
- query = "insert into mirrors (repo, arch, country, url, failures, lastgood) VALUES ('testrepo', 'testarch', 'testcountry', 'http://nowhere/', 0, DATETIME('now'));"
- if debug:
- print "Executing %s" % query
- cursor.execute(query)
- if debug:
- print "deleting test %i" % cursor.lastrowid
- cursor.execute("delete from mirrors where m_id =" + str(cursor.lastrowid) + ";")
- con.commit()
- except sqlite.Error, errmsg:
- if debug:
- print 'db IO test failed: %s' % errmsg
-
+
+class RepoDB:
+ def __init__(self, db):
+ (self.dbconn, self.dbcursor) = self.connect(db)
+ self.initdb(0)
+ return None
+
+ def connect(self, db):
try:
- cursor.execute('CREATE TABLE mirrors (m_id INTEGER PRIMARY KEY, repo varchar(30), arch varchar(8), country varchar(2), url text, failures integer, lastgood date);')
- con.commit()
+ conn = sqlite.connect(db)
+ cursor = conn.cursor()
except sqlite.Error, errmsg:
- errorprint('Err: %s' % errmsg)
+ errorprint('Failed to connect to database: %s' % db)
+ errorprint('Err: %s ' % errmsg)
return None, None
- return con, cursor
+ else:
+ return(conn, cursor)
+
+ def delete(self, where):
+ """ Accepts a where clause """
+ sql = 'delete from mirrors where %s' % where
+ if self.run(sql): return True
+ else: return False
+
+ def run(self, sql):
+ """ Accepts and runs full sql query """
+ try:
+ debugprint("SQL: %s" % sql)
+ self.dbcursor.execute(sql)
+ self.dbconn.commit()
+ except sqlite.Error, errmsg:
+ errorprint("Query: %s" % sql)
+ errorprint("Query Error: %s" % errmsg)
+ return False
+ else:
+ return self.dbcursor.rowcount
+
+ def goodmirror(self, repo, arch, country, url):
+ """ Update or insert a good mirror """
+ if self.run("update mirrors set failures='0', lastgood=DATETIME('now') where url='%s' and repo='%s' and arch='%s';" % (url, repo, arch)):
+ return True
+ elif self.run("insert into mirrors (repo, arch, country, url, failures, lastgood) VALUES ('%s', '%s', '%s', '%s', 0, DATETIME('now'));" % (repo, arch, country, url)):
+ return True
+ else:
+ return False
+
+ def badmirror(self, repo, arch, country, url):
+ """ Update or insert a bad mirror """
+ if self.run("update mirrors set failures=(select failures from mirrors where url='%s')+1 where url='%s' and repo='%s' and arch='%s';" % (url, url, repo, arch)):
+ return True
+ elif self.run("insert into mirrors (repo, arch, country, url, failures, lastgood) VALUES ('%s', '%s', '%s', '%s', 1, '0');" % (repo, arch, country, url)):
+ return True
+ else:
+ return True
+
+ def initdb(self, runno):
+ """ Initializes the database, creating it if necessary """
+ if self.goodmirror('testrepo', 'testarch', 'TS', 'http://testurl/path/to/repo/'):
+ if self.delete("repo='testrepo' and arch='testarch' and country='TS' and url='http://testurl/path/to/repo/'"):
+ return True
+ else:
+ errorprint('db IO error: could not delete - Exiting')
+ sys.exit(2)
+ return False
+ else:
+ self.run('CREATE TABLE mirrors (m_id INTEGER PRIMARY KEY, repo varchar(30), arch varchar(8), country varchar(2), url text, failures integer, lastgood date);')
+ if runno == 0:
+ self.initdb(1)
+ else:
+ errorprint('Could not create database - Exiting')
+ sys.exit(2) # could not create database
+ return True
def check_and_make_dir(dir):
@@ -124,41 +168,40 @@
result = True
return result
-def update_db(repo, arch, country, url, failure, dbconn, dbcursor):
- updated = 0
- if not dbcursor:
- errorprint('sqlite database check failed')
-
- if failure:
- query = "update mirrors set failures=(select failures from mirrors where url='%s')+1 where url='%s' and repo='%s' and arch='%s';" % (url, url, repo, arch)
- else:
- query = "update mirrors set failures='0', lastgood=DATETIME('now') where url='%s' and repo='%s' and arch='%s';" % (url, repo, arch)
- try:
- if debug:
- print "Executing: %s" % query
- dbcursor.execute(query)
- updated = dbcursor.rowcount
- dbconn.commit()
- except sqlite.Error, errmsg:
- errorprint('DBerr: %s ' errmsg)
- errorprint(query)
- if not updated:
- try:
- if failure:
- lastgoodsql='0'
- else:
- lastgoodsql="DATETIME('now')"
- query = "insert into mirrors (repo, arch, country, url, failures, lastgood) VALUES ('%s', '%s', '%s', '%s', '%s', %s);" % (repo, arch, country, url, failure, lastgoodsql)
- if debug:
- print "Executing: %s" % query
- dbcursor.execute(query)
- updated = dbcursor.rowcount
- dbconn.commit()
- except sqlite.Error, errmsg:
- errorprint('DBErr: %s' errmsg)
- errorprint(query)
- return None
- return updated
+#def update_db(repo, arch, country, url, failure, dbconn, dbcursor):
+# updated = 0
+# if not dbcursor:
+# errorprint('sqlite database check failed')
+#
+# if failure:
+# query = "update mirrors set failures=(select failures from mirrors where url='%s')+1 where url='%s' and repo='%s' and arch='%s';" % (url, url, repo, arch)
+# else:
+# try:
+# if debug:
+# print "Executing: %s" % query
+# dbcursor.execute(query)
+# updated = dbcursor.rowcount
+# dbconn.commit()
+# except sqlite.Error, errmsg:
+# errorprint('DBerr: %s ' % errmsg)
+# errorprint(query)
+# if not updated:
+# try:
+# if failure:
+# lastgoodsql='0'
+# else:
+# lastgoodsql="DATETIME('now')"
+# query = "insert into mirrors (repo, arch, country, url, failures, lastgood) VALUES ('%s', '%s', '%s', '%s', '%s', %s);" % (repo, arch, country, url, failure, lastgoodsql)
+# if debug:
+# print "Executing: %s" % query
+# dbcursor.execute(query)
+# updated = dbcursor.rowcount
+# dbconn.commit()
+# except sqlite.Error, errmsg:
+# errorprint('DBErr: %s' % errmsg)
+# errorprint(query)
+# return None
+# return updated
class RepoData:
@@ -395,7 +438,6 @@
# grab the canonical mirrors info
for s in sections:
mirrors = []
- badmirrors = []
ug = URLGrabber(timeout=s.timeout)
s.populate_mirrorlist(ug)
@@ -403,8 +445,9 @@
errorprint("no mirrors to look at for %s, something is broken, skipping" % s.mirrorid)
continue
- dbconn, dbcursor = check_and_make_db(s.db)
-
+# dbconn, dbcursor = check_and_make_db(s.db)
+ DB = RepoDB(s.db)
+
if not check_and_make_dir(s.outputpath):
errorprint('Error creating output path %s for %s' % (s.outputpath, s.mirrorid))
continue
@@ -460,9 +503,8 @@
if debug: print 'adding to %s: %s' % (m.country, m.url)
country_specific[m.country].append(m.url)
if not goodmirror:
- print "Bad: %s, %s, %s, %s" % (s.mirrorid, arch, m.country, m.url)
- if not update_db(s.mirrorid, arch, m.country, m.url, '1', dbconn, dbcursor):
- errorprint("Error updating: %s" % url)
+ debugprint("Bad: %s, %s, %s, %s" % (s.mirrorid, arch, m.country, m.url))
+ DB.badmirror(s.mirrorid, arch, m.country, m.url)
global_file = '%s/%s-global-%s.txt' % (s.outputpath, s.mirrorid, arch)
glob_fo = open(global_file, 'w')
for url in glob_urls:
@@ -475,8 +517,7 @@
country_fo = open(country_file, 'w')
for url in country_specific[code]:
country_fo.write('%s\n' % url)
- if not update_db(s.mirrorid, arch, code, url, 0, dbconn, dbcursor):
- errorprint("Error updating: %s" % url)
+ DB.goodmirror(s.mirrorid, arch, code, url)
new_file_list.append(os.path.normpath(country_file))
# clean up
@@ -485,9 +526,6 @@
if fn not in new_file_list:
if debug: print "removing old file %s" % fn
os.unlink(fn)
-# dbconn.close()
-# dbcursor.close()
-
if __name__ == '__main__':
if len(sys.argv) < 2:
More information about the fedora-extras-commits
mailing list