extras-repoclosure rc-report.py,1.49,1.50
Michael Schwendt (mschwendt)
fedora-extras-commits at redhat.com
Thu Apr 3 13:37:45 UTC 2008
- Previous message (by thread): rpms/namazu/F-7 filter-requires-namazu.sh, 1.3, 1.4 namazu.spec, 1.9, 1.10
- Next message (by thread): rpms/kernel/devel patch-2.6.25-rc8-git2.bz2.sign, NONE, 1.1 .cvsignore, 1.798, 1.799 kernel.spec, 1.579, 1.580 sources, 1.759, 1.760 upstream, 1.678, 1.679 patch-2.6.25-rc8-git1.bz2.sign, 1.1, NONE
- Messages sorted by:
[ date ]
[ thread ]
[ subject ]
[ author ]
Author: mschwendt
Update of /cvs/fedora/extras-repoclosure
In directory cvs-int.fedora.redhat.com:/tmp/cvs-serv24742
Modified Files:
rc-report.py
Log Message:
s/owners/owner/ here too
Index: rc-report.py
===================================================================
RCS file: /cvs/fedora/extras-repoclosure/rc-report.py,v
retrieving revision 1.49
retrieving revision 1.50
diff -u -r1.49 -r1.50
--- rc-report.py 25 Jul 2007 08:58:47 -0000 1.49
+++ rc-report.py 3 Apr 2008 13:37:45 -0000 1.50
@@ -1,38 +1,85 @@
#!/usr/bin/python
# -*- mode: Python; indent-tabs-mode: nil; -*-
-import errno
-import commands
-import datetime, time
-import os, sys, stat, shutil
-import pickle
+import errno, os, sys, stat
import re
import smtplib
-import getopt
+import datetime, time
+from optparse import OptionParser
+import ConfigParser
from PackageOwners import PackageOwners
-from History import History
+#from FakeOwners import FakeOwners as PackageOwners
-sys.path.append('/srv/extras-push/work/buildsys-utils/pushscript')
-import Utils
-cfg = Utils.load_config_module('Extras')
+FAS = {
+ 'project' : "Fedora EPEL",
+ 'user' : "",
+ 'passwd' : "",
+ }
+
+Mail = {
+ 'server' : "localhost",
+ 'user' : "",
+ 'passwd' : "",
+ 'maxsize' : 39*1024,
+ 'from' : "root at localhost",
+ 'replyto' : "root at localhost",
+ 'subject' : "Broken dependencies in EPEL",
+}
-combinedreport = False
+class BrokenDep:
+ def __init__(self):
+ self.pkgid = None # 'name - EVR.arch'
+ self.repoid = None # e.g. 'fedora-core-6-i386'
+ self.srp_mname = None
+ self.age = '' # e.g. '(14 days)'
+ self.owner = ''
+ self.coowners = []
+ # disabled/stripped feature
+ self.mail = True # whether to notify owner by mail
+ # disabled/stripped feature
+ self.new = False
+ self.report = []
-maxmailsize = 39*1024
+ def GetRequires(self):
+ pkgid2 = self.pkgid.replace(' ','')
+ r = []
+ for line in self.report:
+ if len(line) and not line.isspace() and not line.startswith('package: ') and line.find('unresolved deps:') < 0:
+ r.append( ' '+pkgid2+' requires '+line.lstrip() )
+ return '\n'.join(r)
-fromaddr = 'Fedora Extras repoclosure <buildsys at fedoraproject.org>'
-replytoaddr = 'fedora-devel-list at redhat.com'
-subject = 'Broken dependencies in Fedora'
-mailserver = 'localhost'
-mailuser = ''
-mailpasswd = ''
+def whiteListed(b): # Just a hook, not a generic white-list feature.
+ # These two in Fedora 7 Everything most likely won't be fixed.
+ if b.pkgid.startswith('kmod-em8300') and b.repoid.startswith('fedora-7'):
+ return True
+ elif b.pkgid.startswith('kmod-sysprof') and b.repoid.startswith('fedora-7'):
+ return True
+ elif b.pkgid.startswith('kmod'): # gah ;) temporarily catch them all
+ return True
+ else:
+ return False
-today = datetime.date.today()
-# Where to check out owners/owners.list
-ownersworkdir = '/srv/extras-push/work/extras-repoclosure'
+def makeOwners(brokendeps):
+ owners = PackageOwners()
+ try:
+ #if not owners.FromURL():
+ if not owners.FromURL(repoid=FAS['project'],username=FAS['user'],password=FAS['passwd']):
+ raise IOError('ERROR: Could not retrieve package owner data.')
+ except IOError, e:
+ print e
+ sys.exit(1)
+ for b in brokendeps:
+ toaddr = owners.GetOwner(b.srpm_name)
+ if toaddr == '':
+ toaddr = 'UNKNOWN OWNER'
+ e = 'ERROR: "%s" not in owners.list!\n\n' % b.srpm_name
+ if e not in errcache:
+ errcache.append(e)
+ b.owner = toaddr
+ b.coowners = owners.GetCoOwnerList(b.srpm_name)
def mail(smtp, fromaddr, toaddrs, replytoaddr, subject, body):
@@ -69,7 +116,6 @@
print 'ERROR: SMTPException'
-
def mailsplit(smtp, fromaddr, toaddrs, replytoaddr, subject, body):
# Split mail body at line positions to keep it below maxmailsize.
parts = 0
@@ -77,8 +123,8 @@
end = len(body)
slices = []
while ( start < end ):
- if ( (end-start) > maxmailsize ):
- nextstart = body.rfind( '\n', start, start+maxmailsize )
+ if ( (end-start) > Mail['maxsize'] ):
+ nextstart = body.rfind( '\n', start, start+Mail['maxsize'] )
if ( nextstart<0 or nextstart==start ):
print 'ERROR: cannot split mail body cleanly'
nextstart = end
@@ -100,176 +146,91 @@
curpart += 1
-class BrokenDep:
- def __init__(self):
- self.pkgid = None # 'name - EVR.arch'
- self.repoid = None # e.g. 'fedora-core-6-i386'
- self.age = '' # e.g. '(14 days')
- self.owner = None
- self.coowners = None
- self.mail = False # whether to notify owner by mail
- self.new = False
- self.report = []
-
- def GetRequires(self):
- pkgid2 = self.pkgid.replace(' ','')
- r = ''
- for line in self.report:
- if not line.isspace() and not line.startswith('package: ') and line.find('unresolved deps:') < 0:
- r += ' '+pkgid2+' requires '+line.lstrip()
- return r
-
+def loadConfigFile(filename):
+ if not filename:
+ return
-def WhiteListed(b): # Just a hook, not a generic white-list feature.
- # These two in Fedora 7 Everything most likely won't be fixed.
- if b.pkgid.startswith('kmod-em8300') and b.repoid.startswith('fedora-7'):
- return True
- elif b.pkgid.startswith('kmod-sysprof') and b.repoid.startswith('fedora-7'):
- return True
- else:
- return False
-
-# ====================================================================
+ config = ConfigParser.ConfigParser()
+ try:
+ config.readfp(open(filename))
+ except IOError, (e, errstr):
+ print filename, ':', errstr
+ sys.exit(e)
+
+ try:
+ if config.has_section('FAS'):
+ for v in ['project','user','passwd']:
+ if config.has_option('FAS',v):
+ FAS[v] = config.get('FAS',v)
+ if config.has_section('Mail'):
+ for v in ['server','user','passwd','from','replyto','subject']:
+ if config.has_option('Mail',v):
+ Mail[v] = config.get('Mail',v)
+ if config.has_option('Mail','maxsize'):
+ Mail['maxsize'] = config.getint('Mail','maxsize')
+
+ except (ConfigParser.NoSectionError, ConfigParser.NoOptionError), e:
+ print 'Configuration file error:', e
+
+
+### main
+
+usage = "Usage: %s <options> <Extras repoclosure report file(s)>" % sys.argv[0]
+parser = OptionParser(usage=usage)
+parser.add_option("-c", "--config", default=None,
+ help="config file to use")
+parser.add_option("-k", "--keyword", default=[], action='append',
+ help="a keyword to look for in repoids")
+parser.add_option("-m", "--mail", default=[], action='append',
+ help="what mail to send (owner, summary)")
+parser.add_option("-w", "--warn", default=[], action='append',
+ help="repository warnings to include (needsign, testing)")
+parser.add_option("", "--noowners", default=False, action="store_true",
+ help="don't fetch package owner data from FAS")
+(opts, args) = parser.parse_args()
-try:
- opts, args = getopt.getopt(sys.argv[1:], "", ["mail=","needsign","testing"])
- if len(args) < 2:
- raise getopt.GetoptError('not enough arguments')
-except getopt.GetoptError:
- print 'SYNTAX: %s [--mail=<owners|summary>] <release> <logfile> [release] [logfile]...' % sys.argv[0]
- sys.exit(errno.EINVAL)
+loadConfigFile(opts.config)
-mailowners = False
-mailsummary = False
-withneedsign = False
-withtesting = False
-for o, a in opts:
- if o == "--mail":
- if a == "owners":
- mailowners = True
- if a == "summary":
- mailsummary = True
- elif o == "--needsign":
- withneedsign = True
- elif o == "--testing":
- withtesting = True
-
-domail = (mailowners or mailsummary)
-
-owners = PackageOwners()
-#owners.FromCVS(workdir=ownersworkdir)
-if not owners.FromURL():
- sys.exit(1)
+domail = len(opts.mail)>0
brokendeps = [] # list of BrokenDeps
-summail = ''
-errcache = []
-
-if len(args) > 2:
- combinedreport = True
-while args:
- (release,logfilename) = (args[0],args[1])
- del args[0:2]
-
- # This is ugly to get the report date from the report file name.
- datere = re.compile('-(?P<timestamp>[0-9]+-[0-9]+-[0-9]+).txt$')
- res = datere.search(logfilename)
- if res:
- datestring = res.group('timestamp')
- else:
- datestring = today
+errcache = [] # error messages to be included in the summary mail
- if not combinedreport:
- if withtesting:
- mailsubject = subject + ' ' + release + ' + Test Updates - ' + datestring
- else:
- mailsubject = subject + ' ' + release + ' - ' + datestring
- else:
- mailsubject = subject + ' - ' + datestring
-
- sumsubject = 'Summary - ' + mailsubject
- history = History(release)
- history.today = today
+if not len(args):
+ print usage
+ sys.exit(errno.EINVAL)
+# Parse extras-repoclosure output files and fill brokendeps array.
+while len(args):
+ logfilename = args[0]
+ del args[0]
+ f = file( logfilename )
pkgre = re.compile('(?P<name>.*)-[^-]+-[^-]+$')
inbody = False
srcrpm = ''
- createreport = False
- try:
- f = file( logfilename )
- except:
- print 'ERROR: could not open input file'
- sys.exit(5)
for line in f:
if line.startswith('source rpm: '):
w = line.rstrip().split(' ')
- if ( len(w) >= 3 ):
- srcrpm = w[2]
- inbody = True
- createreport = not history.Get(srcrpm)
- else:
+ srcrpm = w[2]
+ res = pkgre.search( srcrpm ) # try to get src.rpm "name"
+ if not res: # only true for invalid input
inbody = False
+ else:
+ srpm_name = res.group('name')
+ inbody = True
continue
+
elif inbody and line.startswith('package: '):
w = line.rstrip().split(' ')
repoid = w[5]
- # Only report broken packages in Fedora Extras.
- needle = 'extras'
- if ( release=='7' ):
- needle = 'fedora'
- elif ( release=='development'):
- needle = 'rawhide'
- if ( repoid.find(needle) < 0 ):
- inbody = False
- continue
-
- res = pkgre.search(srcrpm) # try to get src.rpm "name"
- if not res: # only true for invalid input
- inbody = False
- continue
- srpm_name = res.group('name')
-
b = BrokenDep()
- b.pkgid = w[1]+' - '+w[3] # name - EVR.arch
- b.age = history.GetAge(srcrpm)
+ b.pkgid = w[1]+' - '+w[3] # name - EVR.arch
b.repoid = repoid
- b.mail = createreport
- # createrepo==True => resend broken deps report to owner
- # however, only include new reports in summary
- b.new = (b.age=='')
- if WhiteListed(b):
- continue
+ b.srpm_name = srpm_name
brokendeps.append(b)
- # Don't let -kmod src.rpms in development repo trigger a new summary report.
- if w[1].startswith('kmod-') and repoid.find('-development-')>0:
- b.new = False
- # Fill in package owner(s).
- toaddr = owners.GetOwner(srpm_name)
- if toaddr == '':
- toaddr = 'UNKNOWN OWNER'
- e = 'ERROR: "%s" not in owners.list!\n\n' % srpm_name
- if e not in errcache:
- errcache.append(e)
- summail += e
- b.owner = toaddr
- b.coowners = owners.GetCoOwnerList(srpm_name)
if inbody:
- # Construct report per broken package.
- b.report.append(line)
-
- history.Save()
-
-
-# Mail init.
-if domail:
- srv = smtplib.SMTP( mailserver )
- #srv.set_debuglevel(1)
- if ( len(mailuser) and len(mailpasswd) ):
- try:
- srv.login( mailuser, mailpasswd )
- except smtplib.SMTPException:
- print 'ERROR: mailserver login failed'
- sys.exit(-1)
+ # Copy report per broken package.
+ b.report.append( line.rstrip() )
def bdSortByOwnerAndName(a,b):
@@ -278,63 +239,74 @@
def bdSortByRepoAndName(a,b):
return cmp(a.repoid+a.pkgid,b.repoid+b.pkgid)
-reportssummary = '' # any NEW stuff for the summary
-sep = '='*70+'\n'
-reports = {} # map of lists [new,body] - a flag and the full report for a package owner
+
+# Filter out unwanted repoids.
+for b in list(brokendeps):
+ for needle in opts.keyword:
+ if b.repoid.find( needle ) >= 0: # wanted?
+ break
+ else:
+ brokendeps.remove(b)
+
+# Filter out entries from whitelist.
+for b in list(brokendeps):
+ if whiteListed(b):
+ brokendeps.remove(b)
+
+# Fill in package owners.
+if not opts.noowners:
+ makeOwners(brokendeps)
# Build full mail report per owner. Use a flag for new breakage.
-brokendeps.sort(bdSortByOwnerAndName)
-for b in brokendeps:
- if b.new:
- print 'NEW breakage: %s in %s' % (b.pkgid, b.repoid)
- if b.mail:
- print ' mailing owner(s)'
- r = ''.join(b.report)
- reports.setdefault(b.owner,[b.new,''])
- reports[b.owner][1] += r
- # Also build mails for co-owners.
- for toaddr in b.coowners:
- reports.setdefault(toaddr,[None,''])
- reports[toaddr][1] += r
-
-if withneedsign:
- reportssummary += sep
- reportssummary += "The results in this summary consider unreleased updates in the\nbuild-system's needsign-queue!\n"
- reportssummary += sep
-
-if withtesting:
- reportssummary += sep
- reportssummary += "The results in this summary consider Test Updates!\n"
- reportssummary += sep
+reports = {} # map of lists [new,body] - a flag and the full report for a package owner
+if not opts.noowners:
+ brokendeps.sort(bdSortByOwnerAndName)
+ for b in brokendeps:
+ if b.new:
+ print 'NEW breakage: %s in %s' % (b.pkgid, b.repoid)
+ if b.mail:
+ r = '\n'.join(b.report)+'\n'
+ reports.setdefault(b.owner,[b.new,''])
+ reports[b.owner][1] += r
+ # Also build mails for co-owners.
+ for toaddr in b.coowners:
+ reports.setdefault(toaddr,[None,''])
+ reports[toaddr][1] += r
-# Mail reports and construct summary of new reports.
-for toaddr,(new,body) in reports.iteritems():
- if new: # Include new reports in summary.
- reportssummary += 'New report for: %s\n\n%s\n' % ( toaddr.replace('@',' AT '), body)
- reportssummary += sep
- # Send mail to every package owner with broken package dependencies.
- mailtext = 'This is an automated mail created by an experimental script.\nYour following packages in the repository contain broken dependencies:\n\n'
- if withneedsign:
- mailtext += sep
- mailtext += "The results in this report consider unreleased updates in the\nbuild-system's needsign-queue!\n"
- mailtext += sep+'\n'
- if withtesting:
- mailtext += sep
- mailtext += "The results in this report consider Test Updates!\n"
- mailtext += sep+'\n'
- mailtext += body
- if mailowners and toaddr!='UNKNOWN OWNER':
- mailsplit( srv, fromaddr, toaddr, replytoaddr, mailsubject, mailtext )
-summail += reportssummary
+sep = '='*70+'\n'
+summail = '' # main summary mail text
+reportssummary = '' # any NEW stuff for the summary
+
+def giveNeedsignMsg():
+ if 'needsign' in opts.warn:
+ return sep+"The results in this summary consider unreleased updates in the\nbuild-system's needsign-queue!\n"+sep+'\n'
+ else:
+ return ''
-summail += ('Summary of broken packages (by owner):\n')
-o = None
-for b in brokendeps:
- if o != b.owner:
- o = b.owner
- summail += '\n '+b.owner.replace('@',' AT ')+'\n'
- summail += ' '+b.pkgid+' '+b.age+'\n'
+def giveTestingMsg():
+ if 'testing' in opts.warn:
+ return sep+"The results in this summary consider Test Updates!\n"+sep+'\n'
+ else:
+ return ''
+
+# Create summary mail text.
+reportssummary += giveNeedsignMsg()
+reportssummary += giveTestingMsg()
+summail += reportssummary
+
+if not opts.noowners and len(brokendeps):
+ summail += ('Summary of broken packages (by owner):\n')
+ brokendeps.sort(bdSortByOwnerAndName)
+ o = None
+ for b in brokendeps:
+ if o != b.owner:
+ o = b.owner
+ seenbefore = []
+ summail += '\n '+b.owner.replace('@',' AT ')+'\n'
+ if b.pkgid not in seenbefore:
+ summail += ' '+b.pkgid+' '+b.age+'\n'
+ seenbefore.append(b.pkgid)
# Broken deps sorted by repository id.
brokendeps.sort(bdSortByRepoAndName)
@@ -343,30 +315,37 @@
if r != b.repoid:
r = b.repoid
summail += '\n\n'+sep+('Broken packages in %s:\n\n' % b.repoid)
- summail += b.GetRequires()
+ summail += b.GetRequires()+'\n'
+
+# Mail init.
+if domail:
+ srv = smtplib.SMTP( Mail['server'] )
+ if ( len(Mail['user']) and len(Mail['passwd']) ):
+ try:
+ srv.login( Mail['user'], Mail['passwd'] )
+ except smtplib.SMTPException:
+ print 'ERROR: mailserver login failed'
+ sys.exit(-1)
-# Decrease the "spam" a bit. Mail a summary (even if there are no new
-# reports) every 7 days.
-timestampfile = os.path.join(ownersworkdir,'lastsummary.timestamp')
-try:
- timestamp = os.stat(timestampfile).st_mtime
-except OSError:
- timestamp = 0
- f = open(timestampfile,'w')
- f.close()
-if mailsummary and (len(reportssummary) or (datetime.date.fromtimestamp(timestamp)+datetime.timedelta(days=7) < today)):
- f = open(timestampfile,'w')
- f.close()
-else:
- mailsummary = False
-
-# Send summary to mailing-list
-# if there are new reports.
-if mailsummary and (len(summail) or mailsubject.find('development') >= 0): # avoid sending empty reports for stable/old branches
- if not len(summail):
- summail = 'no broken dependencies'
- toaddr = replytoaddr # mail summary to list
- mailsplit( srv, fromaddr, toaddr, '', sumsubject, summail )
+# Mail reports to owners.
+for toaddr,(new,body) in reports.iteritems():
+ # Send mail to every package owner with broken package dependencies.
+ mailtext = 'Your following packages in the repository suffer from broken dependencies:\n\n'
+ mailtext += giveNeedsignMsg()
+ mailtext += giveTestingMsg()
+ mailtext += body
+ if domail and ('owner' in opts.mail) and toaddr!='UNKNOWN OWNER':
+ subject = Mail['subject'] + ' - %s' % datetime.date.today()
+ mail( srv, Mail['from'], toaddr, Mail['replyto'], subject, mailtext )
+
+# Mail summary to mailing-list.
+if domail and ('summary' in opts.mail):
+ subject = Mail['subject'] + ' - %s' % datetime.date.today()
+ toaddr = Mail['replyto']
+ mailsplit( srv, Mail['from'], toaddr, '', subject, summail )
if domail:
srv.quit()
+
+if len(summail):
+ print summail
- Previous message (by thread): rpms/namazu/F-7 filter-requires-namazu.sh, 1.3, 1.4 namazu.spec, 1.9, 1.10
- Next message (by thread): rpms/kernel/devel patch-2.6.25-rc8-git2.bz2.sign, NONE, 1.1 .cvsignore, 1.798, 1.799 kernel.spec, 1.579, 1.580 sources, 1.759, 1.760 upstream, 1.678, 1.679 patch-2.6.25-rc8-git1.bz2.sign, 1.1, NONE
- Messages sorted by:
[ date ]
[ thread ]
[ subject ]
[ author ]
More information about the fedora-extras-commits
mailing list