extras-buildsys/server EmailUtils.py, NONE, 1.1 BuildMaster.py, 1.21, 1.22 BuilderManager.py, 1.7, 1.8 CONFIG.py, 1.20, 1.21 Makefile, 1.5, 1.6 PackageJob.py, 1.13, 1.14 UserInterface.py, 1.27, 1.28
Daniel Williams (dcbw)
fedora-extras-commits at redhat.com
Mon Jul 25 19:47:17 UTC 2005
Author: dcbw
Update of /cvs/fedora/extras-buildsys/server
In directory cvs-int.fedora.redhat.com:/tmp/cvs-serv26150/server
Modified Files:
BuildMaster.py BuilderManager.py CONFIG.py Makefile
PackageJob.py UserInterface.py
Added Files:
EmailUtils.py
Log Message:
2005-07-25 Dan Williams <dcbw at redhat.com>
* server/CONFIG.py
- Move stuff around, two new options for admin_emails and success_emails
* server/BuildMaster.py
server/PackageJob.py
- Set endtime when all builds complete or fail, to not include all
the repo time in the job's build time
* server/PackageJob.py
server/UserInterface.py
server/BuilderManager.py
server/EmailUtils.py
- Notify admins when a builder times out
- Consolidate email sending into one place
- Notify success_emails when a build job succeeds
--- NEW FILE EmailUtils.py ---
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
# Copyright 2005 Dan Williams <dcbw at redhat.com> and Red Hat, Inc.
import smtplib
from email.MIMEText import MIMEText
# Load in the config
CONFIG_LOCATION = "/etc/plague/server/"
execfile(CONFIG_LOCATION + "CONFIG.py")
def email_result(to, resultstring, subject=None):
msg = MIMEText(resultstring)
msg['Subject'] = subject
msg['From'] = config_opts['email_from']
msg['To'] = to
s = smtplib.SMTP()
s.connect()
s.sendmail(config_opts['email_from'], [to], msg.as_string())
s.close()
Index: BuildMaster.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/server/BuildMaster.py,v
retrieving revision 1.21
retrieving revision 1.22
diff -u -r1.21 -r1.22
--- BuildMaster.py 25 Jul 2005 19:37:31 -0000 1.21
+++ BuildMaster.py 25 Jul 2005 19:47:15 -0000 1.22
@@ -223,7 +223,7 @@
# Update job end time
try:
- self.curs.execute('UPDATE jobs SET endtime=%d WHERE uid=%d' % (time.time(), uid))
+ self.curs.execute('UPDATE jobs SET endtime=%d WHERE uid=%d' % (job.endtime, uid))
except sqlite.OperationalError, e:
print "DB Error: could not access jobs database. Reason: '%s'" % e
self.dbcx.commit()
Index: BuilderManager.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/server/BuilderManager.py,v
retrieving revision 1.7
retrieving revision 1.8
diff -u -r1.7 -r1.8
--- BuilderManager.py 20 Jul 2005 20:50:07 -0000 1.7
+++ BuilderManager.py 25 Jul 2005 19:47:15 -0000 1.8
@@ -22,6 +22,7 @@
import os
import threading
import Builder
+import EmailUtils
# Load in the config
execfile("/etc/plague/server/CONFIG.py")
@@ -147,6 +148,14 @@
for builder in self.running_builders:
if not builder.alive():
print "Removing builder '%s' because it timed out." % builder.address()
+
+ # Notify admins
+ subject = "Builder Timeout: %s" % builder.address()
+ msg = "The builder '%s' timed out and was removed from the active builder list." % builder.address()
+ for addr in config_opts['admin_emails']:
+ EmailUtils.email_result(addr, msg, subject)
+
+ # Forget about the builder
builder.stop()
self.running_builders.remove(builder)
Index: CONFIG.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/server/CONFIG.py,v
retrieving revision 1.20
retrieving revision 1.21
diff -u -r1.20 -r1.21
--- CONFIG.py 22 Jul 2005 02:54:07 -0000 1.20
+++ CONFIG.py 25 Jul 2005 19:47:15 -0000 1.21
@@ -1,24 +1,52 @@
# Configuration file for build system server
config_opts = {}
-config_opts['email_from'] = "buildsys at fedoraproject.org"
config_opts['pkg_cvs_root'] = ":pserver:anonymous at cvs.fedora.redhat.com:/cvs/dist"
config_opts['pkg_cvs_rsh'] = ""
config_opts['cvs_cmd'] = "/usr/bin/cvs"
config_opts['make_cmd'] = "/usr/bin/make"
-config_opts['tmpdir'] = "/tmp"
config_opts['log_url'] = "http://foo.foo.org/logs/"
config_opts['guest_allowed'] = True
+
+#
+# Directories
+#
+
+# server_work_dir: where logs and finished RPMs are stored
+config_opts['server_work_dir'] = "/rpmbuild"
+
+# repo_dir: repository dir of built RPMs
+config_opts['repo_dir'] = "/repodir"
+
+# tmpdir: where to store temporary stuff, like CVS checkout directories
+config_opts['tmpdir'] = "/tmp"
+
+
+#
+# Email options
+#
+
+# email_from: the address emails from the build server should appear to come from
+config_opts['email_from'] = "buildsys at fedoraproject.org"
+
+# admin_emails: who to email when things go wrong
+config_opts['admin_emails'] = []
+
+# success_email: addresses to send notification of build successes to
+config_opts['success_emails'] = []
+
+
+#
# SSL options
#
+
# ssl_frontend: True = package submitters need SSL to connect to the server
config_opts['ssl_frontend'] = True
# ssl_buildclients: True = all communication between server & build client
# be over an SSL connecction
config_opts['ssl_builders'] = True
-
SERVER_BASE_DIR = "/etc/plague/server"
# SSL Cert and key bits
@@ -32,14 +60,6 @@
config_opts['ui_ca_cert'] = SERVER_BASE_DIR + "/certs/fedora-upload-ca.pem"
-# server_work_dir
-# - Where logs and finished RPMs are stored
-config_opts['server_work_dir'] = "/rpmbuild"
-
-# repo_dir
-# - Repository dir of up-to-date RPMs
-config_opts['repo_dir'] = "/repodir"
-
# This option disables pulling from CVS. Allowing jobs to be submitted
# as unknown SRPMs from random people may be a security risk, so don't
@@ -55,6 +75,10 @@
# Further architectures on a per-package basis are configured in each target's
# package file. See the config option 'addl_package_arches_dir'.
#
+# Mapping - our_target: [list of base arches]
+#
+# ex: { 'development' : ['i386', 'x86_64'], '5' : ['i386'] }
+#
config_opts['targets'] = { 'development' : ['i386'] }
@@ -86,9 +110,13 @@
# Target Optional Arches
#
-# These are arches that submitters _may_ build packages on, but packages won't
+# These are arches that packages may build on, but won't
# be built on these arches by default.
#
+# Mapping - our_target: [list of optional arches]
+#
+# ex: { 'development' : ['i486', 'i586', 'i686', 'ppc32'] }
+#
config_opts['target_optional_arches'] = { }
@@ -113,7 +141,7 @@
config_opts['addl_package_arches_dir'] = SERVER_BASE_DIR + "/addl_pkg_arches"
-# Builder Clients
+# Builders
config_opts['builders'] = [ 'https://127.0.0.1:8888' ]
Index: Makefile
===================================================================
RCS file: /cvs/fedora/extras-buildsys/server/Makefile,v
retrieving revision 1.5
retrieving revision 1.6
diff -u -r1.5 -r1.6
--- Makefile 5 Jul 2005 21:08:03 -0000 1.5
+++ Makefile 25 Jul 2005 19:47:15 -0000 1.6
@@ -10,11 +10,12 @@
rm -f *.pyc *.pyo *~ *.bak
FILES = \
- ArchJob.py \
- BuilderManager.py \
- Builder.py \
+ ArchJob.py \
+ BuilderManager.py \
+ Builder.py \
BuildMaster.py \
- PackageJob.py \
+ EmailUtils.py \
+ PackageJob.py \
Repo.py \
UserInterface.py \
User.py
Index: PackageJob.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/server/PackageJob.py,v
retrieving revision 1.13
retrieving revision 1.14
diff -u -r1.13 -r1.14
--- PackageJob.py 22 Jul 2005 21:35:27 -0000 1.13
+++ PackageJob.py 25 Jul 2005 19:47:15 -0000 1.14
@@ -21,15 +21,12 @@
import commands
import threading
import time
-import popen2
import rpmUtils
import exceptions
import shutil
-import tempfile
-import smtplib
import copy
-from email.MIMEText import MIMEText
import string
+import EmailUtils
import SimpleXMLRPCServer
import xmlrpclib
import socket
@@ -124,6 +121,7 @@
self.hostname = hostname
self.username = username
self.starttime = time.time()
+ self.endtime = 0
self.target = repo.target()
self.repo = repo
self.no_cvs = config_opts['use_srpm_not_cvs']
@@ -196,15 +194,14 @@
# Remove arches we don't support from addl_arches
for arch in addl_arches:
- # arch_utils is only used to determine which arches to build on by default,
+ # ArchUtils.sub_arches is only used to determine which arches to build on by default,
# so that if we have an Additional Package Arches file that specifies
# 'sparcv9' for a package that we don't try to build sparcv9 for that
# package unless 'sparc' is also listed in our 'targets' config option.
if ArchUtils.sub_arches.has_key(arch):
master_addl_arch = ArchUtils.sub_arches[arch]
if master_addl_arch not in buildable_arches:
- if master_addl_arch not in opt_arches:
- addl_arches.remove(arch)
+ addl_arches.remove(arch)
ba = hdr['buildarchs']
exclusive = hdr['exclusivearch']
@@ -433,7 +430,7 @@
resultstring = "%s (%s): Build on target %s was killed by %s." % (self.uid, self.name, self.target, username)
self._set_cur_stage('killed', resultstring)
- self.email_result(resultstring)
+ self.email_result(self.username, resultstring)
self._archjobs_lock.acquire()
for job in self.archjobs.values():
@@ -442,6 +439,7 @@
self.archjobs = {}
self._archjobs_lock.release()
+ self.endtime = time.time()
self.bm.notify_job_done(self)
def wake(self):
@@ -471,7 +469,7 @@
if not self.no_cvs:
shutil.rmtree(self.checkout_tmpdir, ignore_errors=True)
subj = 'Prep Error (Job %s): %s on %s' % (self.uid, self.cvs_tag, self.target)
- self.email_result(resultstring=e.args, subject=subj)
+ self.email_result(self.username, resultstring=e.args, subject=subj)
self._failed(e.args)
except BuildError, e:
subj = 'Build Error (Job %s): %s on %s' % (self.uid, self.cvs_tag, self.target)
@@ -479,7 +477,7 @@
msg = "%s\n\n Build logs may be found at %s\n\n" % (e.msg, log_url)
logtail = self._get_log_tail(e.arch)
msg = "%s\n-------------------------------------------------\n\n%s\n" % (msg, logtail)
- self.email_result(resultstring=msg, subject=subj)
+ self.email_result(self.username, resultstring=msg, subject=subj)
# Kill remaining jobs on other arches
self._archjobs_lock.acquire()
for job in self.archjobs.values():
@@ -523,6 +521,7 @@
def _failed(self, msg=None):
self._set_cur_stage('failed', msg)
+ self.endtime = time.time()
self.bm.notify_job_done(self)
def _add_to_repo(self):
@@ -552,6 +551,8 @@
if len(self.repofiles):
self.repo.request_copy(self)
+ self.endtime = time.time()
+
def repo_add_callback(self):
self._set_cur_stage('repodone')
self.wake()
@@ -562,7 +563,11 @@
log_url = make_job_log_url(self.target, self.uid, self.name, self.ver, self.release)
resultstring = resultstring + "\n Build logs may be found at %s\n" % (log_url)
- self.email_result(resultstring)
+ self.email_result(self.username, resultstring)
+
+ # Notify everyone else who might want to know that the build succeeded
+ for addr in config_opts['success_emails']:
+ self.email_result(addr, resultstring)
self.bm.notify_job_done(self)
@@ -610,20 +615,13 @@
f.close()
return "".join(lines)
- def email_result(self, resultstring, subject=None):
- """send 'resultstring' to self.email from self.email_from"""
-
- msg = MIMEText(resultstring)
+ def email_result(self, to, resultstring, subject=None):
+ """send 'resultstring' to self.username"""
+
if not subject:
name = self.name
if not name:
name = self.package
subject = 'Build Result: %d - %s on %s' % (self.uid, name, self.target)
- msg['Subject'] = subject
- msg['From'] = config_opts['email_from']
- msg['To'] = self.username
- s = smtplib.SMTP()
- s.connect()
- s.sendmail(config_opts['email_from'], [self.username], msg.as_string())
- s.close()
+ EmailUtils.email_result(to, resultstring, subject)
Index: UserInterface.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/server/UserInterface.py,v
retrieving revision 1.27
retrieving revision 1.28
diff -u -r1.27 -r1.28
--- UserInterface.py 23 Jul 2005 00:55:26 -0000 1.27
+++ UserInterface.py 25 Jul 2005 19:47:15 -0000 1.28
@@ -16,8 +16,7 @@
import sqlite
-import smtplib
-from email.MIMEText import MIMEText
+import EmailUtils
import time
import sys
import os
@@ -32,21 +31,6 @@
execfile(CONFIG_LOCATION + "CONFIG.py")
-def email_result(email, cvs_tag, resultstring, subject=None):
- """send 'resultstring' to email"""
-
- msg = MIMEText(resultstring)
- if not subject:
- subject = 'Build Result: %s' % cvs_tag
- msg['Subject'] = subject
- msg['From'] = config_opts['email_from']
- msg['To'] = email
- s = smtplib.SMTP()
- s.connect()
- s.sendmail(config_opts['email_from'], [email], msg.as_string())
- s.close()
-
-
def get_dbcx():
dbcx = None
dbcx = sqlite.connect(CONFIG_LOCATION + "jobdb", encoding="utf-8", timeout=3)
@@ -102,6 +86,12 @@
raise InvalidTargetError()
+def email_result(email, source, resultstring):
+ """send 'resultstring' to email"""
+
+ subject = 'Enqueue Result: %s' % source
+ EmailUtils.email_result(email, resultstring, subject)
+
class UserInterface:
"""
More information about the fedora-extras-commits
mailing list