extras-buildsys/server Builder.py, 1.32, 1.33 BuilderManager.py, 1.19, 1.20 Config.py, 1.13, 1.14
Daniel Williams (dcbw)
fedora-extras-commits at redhat.com
Mon Mar 20 12:43:28 UTC 2006
Author: dcbw
Update of /cvs/fedora/extras-buildsys/server
In directory cvs-int.fedora.redhat.com:/tmp/cvs-serv25775/server
Modified Files:
Builder.py BuilderManager.py Config.py
Log Message:
2006-03-20 Dan Williams <dcbw at redhat.com>
* server/Config.py
- Change config file format for builders. Each builder is now listed on
a different line like this: builder1 = 20 https://127.0.0.1:8888
Each builder's tag (what's before the = sign) must start with
'builder'. The number right after the = is the builder's
weight, or preference. Higher numbers mean the builder is more
likely to be used.
* server/BuilderManager.py
- Use builder weights when picking which builder to use for a job
* server/Builder.py
- Add accessort for builder weight
Index: Builder.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/server/Builder.py,v
retrieving revision 1.32
retrieving revision 1.33
diff -u -r1.32 -r1.33
--- Builder.py 20 Mar 2006 04:21:09 -0000 1.32
+++ Builder.py 20 Mar 2006 12:43:21 -0000 1.33
@@ -67,7 +67,7 @@
_BUILDER_PING_INTERVAL = 60 * 5 # In seconds
- def __init__(self, manager, cfg, address):
+ def __init__(self, manager, cfg, address, weight):
self._cur_jobid = None
self._manager = manager
self._jobs = {}
@@ -85,6 +85,7 @@
self._ping_now = False
self._when_died = 0
self._server_cfg = cfg
+ self._weight = weight
certs = None
if self._server_cfg.get_bool("Builders", "use_ssl"):
@@ -201,6 +202,9 @@
def free_slots(self):
return self._free_slots
+
+ def weight(self):
+ return self._weight
def start_job(self, par_job, target_dict, srpm_url):
if not self.available():
Index: BuilderManager.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/server/BuilderManager.py,v
retrieving revision 1.19
retrieving revision 1.20
diff -u -r1.19 -r1.20
--- BuilderManager.py 26 Nov 2005 06:10:22 -0000 1.19
+++ BuilderManager.py 20 Mar 2006 12:43:21 -0000 1.20
@@ -55,6 +55,7 @@
del builder_dict
print string
print ""
+ os._exit(1)
self._queue_lock = threading.Lock()
self._queue = []
@@ -72,21 +73,23 @@
def add_new_builders(self):
self._builders_lock.acquire()
- builder_list = []
- tmp_list = self._cfg.get_list("Builders", "builders")
+ tmp_list = self._cfg.builders()
prefix = "http://"
if self._cfg.get_bool("Builders", "use_ssl") == True:
prefix = "https://"
- for addr in tmp_list:
+
+ builder_list = {}
+ for addr in tmp_list.keys():
+ new_addr = addr
# Rewrite addresses to match current builder connection method
if addr.startswith("http://"):
- addr = addr[7:]
+ new_addr = addr[7:]
elif addr.startswith("https://"):
- addr = addr[8:]
-
- builder_list.append(prefix + addr)
+ new_addr = addr[8:]
+ if new_addr:
+ builder_list[prefix + new_addr] = tmp_list[addr]
- for address in builder_list:
+ for address in builder_list.keys():
# If the address is already in our _builders list, skip it
skip = False
for builder in self._builders:
@@ -96,7 +99,8 @@
continue
# Add the builder to our build list
- builder = Builder.Builder(self, self._cfg, address)
+ weight = builder_list[address]
+ builder = Builder.Builder(self, self._cfg, address, weight)
builder.start()
self._builders.append(builder)
@@ -124,15 +128,26 @@
return True
return self._have_work
- def _builder_sort_func(self, builder):
- return builder.free_slots()
-
def _builder_cmp_func(self, builder1, builder2):
- if builder1.free_slots() > builder2.free_slots():
+ # If both builders have at least one free slot, sort on
+ # weight, not free slots
+ b1_free = builder1.free_slots()
+ b2_free = builder2.free_slots()
+ if b1_free > 0 and b2_free > 0:
+ b1_weight = builder1.weight()
+ b2_weight = builder2.weight()
+ # Equal weight case falls through to
+ # sort on free slots
+ if b1_weight > b2_weight:
+ return 1
+ elif b1_weight < b2_weight:
+ return -1
+
+ if b1_free > b2_free:
return 1
- elif builder1.free_slots() == builder2.free_slots():
+ elif b1_free == b2_free:
return 0
- elif builder1.free_slots() < builder2.free_slots():
+ elif b1_free < b2_free:
return -1
return 1
@@ -154,22 +169,21 @@
continue
# Find all free builders that could satisfy the request
- builder_list = []
+ possible_builders = []
for builder in self._builders:
if builder.available() and builder.can_build_for_target(req['target_dict']):
- builder_list.append(builder)
+ possible_builders.append(builder)
- # Use the builder with the most free build slots
- if sys.version_info[:3] >= (2, 4, 0):
- builder_list.sort(key=self._builder_sort_func, reverse=True)
- else:
- builder_list.sort(self._builder_cmp_func)
- builder_list.reverse()
+ # Sort builder list by free slots and weights
+ possible_builders.sort(self._builder_cmp_func)
+ possible_builders.reverse()
- for builder in builder_list:
+ for builder in possible_builders:
try:
job = builder.start_job(parent, req['target_dict'], req['srpm_url'])
- except RuntimeError:
+ except RuntimeError, e:
+ print "Builder (%s) couldn't start job %s because: '%s'" % (builder.address(),
+ req['target_dict'], e)
continue
if not new_jobs.has_key(parent):
Index: Config.py
===================================================================
RCS file: /cvs/fedora/extras-buildsys/server/Config.py,v
retrieving revision 1.13
retrieving revision 1.14
diff -u -r1.13 -r1.14
--- Config.py 13 Mar 2006 03:45:48 -0000 1.13
+++ Config.py 20 Mar 2006 12:43:21 -0000 1.14
@@ -39,10 +39,14 @@
self._targets = []
self._target_strings = {}
self._target_aliases = {}
+ self._builders = self._load_builders()
def targets(self):
return self._targets
+ def builders(self):
+ return self._builders
+
def resolve_target_user_alias(self, user_alias):
try:
cfg = self._target_aliases[user_alias.lower()]
@@ -91,7 +95,30 @@
try:
self._add_target(target_cfg)
except InvalidTargetException, e:
- print "Error: could not add target %s because: %s" % (f, e)
+ print "Error: could not add target %s because: %s" % (f, e)
+
+ def _load_builders(self):
+ if not self._config.has_section("Builders"):
+ return {}
+ items = self._config.items("Builders")
+ print items
+ builder_list = {}
+ for (tag, builder) in items:
+ if not tag.startswith("builder"):
+ continue
+ weight = 0
+ weight_str = "0"
+ addr = None
+ try:
+ (weight_str, addr) = builder.split()
+ except ValueError:
+ pass
+ try:
+ weight = int(weight_str)
+ except ValueError:
+ weight = 0
+ builder_list[addr] = weight
+ return builder_list
def save_default_config(self, filename=None):
self.add_section("General")
@@ -113,7 +140,8 @@
self.add_section("Builders")
self.set_option("Builders", "use_ssl", "yes")
- self.set_option("Builders", "builders", "127.0.0.1:8888")
+ self.set_option("Builders", "0 builder1", "127.0.0.1:8888")
+ self.set_option("Builders", "10 builder2", "127.0.0.2:8888")
self.add_section("SSL")
self.set_option("SSL", "server_key_and_cert", "/etc/plague/server/certs/server_key_and_cert.pem")
More information about the fedora-extras-commits
mailing list