[edk2-devel] [Patch] BaseTools: Improve the method of checking queue empty
Liming Gao
liming.gao at intel.com
Tue Aug 4 09:00:01 UTC 2020
Reviewed-by: Liming Gao <liming.gao at intel.com>
-----Original Message-----
From: devel at edk2.groups.io <devel at edk2.groups.io> On Behalf Of Bob Feng
Sent: 2020年8月3日 10:04
To: devel at edk2.groups.io
Cc: Feng, Bob C <bob.c.feng at intel.com>; Gao, Liming <liming.gao at intel.com>; Chen, Christine <yuwei.chen at intel.com>; Lucy Yan <lucyyan at google.com>
Subject: [edk2-devel] [Patch] BaseTools: Improve the method of checking queue empty
From: "Feng, Bob C" <bob.c.feng at intel.com>
Ref: https://bugzilla.tianocore.org/show_bug.cgi?id=2807
The Queue.empty() method is not reliable in the multiple process runtime environment. This patch uses a new method to check if all modules are processed and workers need to be stopped. That is to add a None item at the bottom of the queue. Worker check if it gets that None item to know if all the module is processed.
Signed-off-by: Bob Feng <bob.c.feng at intel.com>
Cc: Liming Gao <liming.gao at intel.com>
Cc: Yuwei Chen <yuwei.chen at intel.com>
Cc: Lucy Yan <lucyyan at google.com>
---
.../Source/Python/AutoGen/AutoGenWorker.py | 26 ++++++++++++++-----
BaseTools/Source/Python/build/build.py | 3 ++-
2 files changed, 22 insertions(+), 7 deletions(-)
diff --git a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
index 563d91b421..017f676399 100755
--- a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
+++ b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
@@ -22,10 +22,11 @@ except:
from Queue import Empty
import traceback
import sys
from AutoGen.DataPipe import MemoryDataPipe import logging
+import time
def clearQ(q):
try:
while True:
q.get_nowait()
@@ -109,11 +110,15 @@ class AutoGenManager(threading.Thread):
badnews = self.feedback_q.get()
if badnews is None:
break
if badnews == "Done":
fin_num += 1
+ elif badnews == "QueueEmpty":
+ EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" % (os.getpid(), badnews))
+ self.TerminateWorkers()
else:
+ EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s"
+ % (os.getpid(), badnews))
self.Status = False
self.TerminateWorkers()
if fin_num == len(self.autogen_workers):
self.clearQueue()
for w in self.autogen_workers:
@@ -225,16 +230,25 @@ class AutoGenWorkerInProcess(mp.Process):
FfsCmd = {}
GlobalData.FfsCmd = FfsCmd
PlatformMetaFile = self.GetPlatformMetaFile(self.data_pipe.Get("P_Info").get("ActivePlatform"),
self.data_pipe.Get("P_Info").get("WorkspaceDir"))
while True:
- if self.module_queue.empty():
- break
if self.error_event.is_set():
break
module_count += 1
- module_file,module_root,module_path,module_basename,module_originalpath,module_arch,IsLib = self.module_queue.get_nowait()
+ try:
+ module_file,module_root,module_path,module_basename,module_originalpath,module_arch,IsLib = self.module_queue.get_nowait()
+ except Empty:
+ EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" % (os.getpid(), "Fake Empty."))
+ time.sleep(0.01)
+ continue
+ if module_file is None:
+ EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" % (os.getpid(), "Worker get the last item in the queue."))
+ self.feedback_q.put("QueueEmpty")
+ time.sleep(0.01)
+ continue
+
modulefullpath = os.path.join(module_root,module_file)
taskname = " : ".join((modulefullpath,module_arch))
module_metafile = PathClass(module_file,module_root)
if module_path:
module_metafile.Path = module_path @@ -278,15 +292,15 @@ class AutoGenWorkerInProcess(mp.Process):
self.cache_q.put((Ma.MetaFile.Path, Ma.Arch, "MakeCache", True))
continue
else:
self.cache_q.put((Ma.MetaFile.Path, Ma.Arch, "MakeCache", False))
- except Empty:
- pass
- except:
+ except Exception as e:
+ EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" %
+ (os.getpid(), str(e)))
self.feedback_q.put(taskname)
finally:
+ EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" %
+ (os.getpid(), "Done"))
self.feedback_q.put("Done")
self.cache_q.put("CacheDone")
def printStatus(self):
print("Processs ID: %d Run %d modules in AutoGen " % (os.getpid(),len(AutoGen.Cache())))
diff --git a/BaseTools/Source/Python/build/build.py b/BaseTools/Source/Python/build/build.py
index 1ab1e60a64..59ceacfed0 100755
--- a/BaseTools/Source/Python/build/build.py
+++ b/BaseTools/Source/Python/build/build.py
@@ -1215,11 +1215,11 @@ class Build():
if Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']:
# for target which must generate AutoGen code and makefile
mqueue = mp.Queue()
for m in AutoGenObject.GetAllModuleInfo:
mqueue.put(m)
-
+ mqueue.put((None,None,None,None,None,None,None))
AutoGenObject.DataPipe.DataContainer = {"CommandTarget": self.Target}
AutoGenObject.DataPipe.DataContainer = {"Workspace_timestamp": AutoGenObject.Workspace._SrcTimeStamp}
AutoGenObject.CreateLibModuelDirs()
AutoGenObject.DataPipe.DataContainer = {"LibraryBuildDirectoryList":AutoGenObject.LibraryBuildDirectoryList}
AutoGenObject.DataPipe.DataContainer = {"ModuleBuildDirectoryList":AutoGenObject.ModuleBuildDirectoryList}
@@ -2172,10 +2172,11 @@ class Build():
ToolChain, Arch, self.PlatformFile,Pa.DataPipe)
self.AllModules.add(Ma)
data_pipe_file = os.path.join(Pa.BuildDir, "GlobalVar_%s_%s.bin" % (str(Pa.Guid),Pa.Arch))
Pa.DataPipe.dump(data_pipe_file)
+ mqueue.put((None,None,None,None,None,None,None))
autogen_rt, errorcode = self.StartAutoGen(mqueue, Pa.DataPipe, self.SkipAutoGen, PcdMaList, cqueue)
if not autogen_rt:
self.AutoGenMgr.TerminateWorkers()
self.AutoGenMgr.join(1)
--
2.20.1.windows.1
-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.
View/Reply Online (#63693): https://edk2.groups.io/g/devel/message/63693
Mute This Topic: https://groups.io/mt/75958155/1813853
Group Owner: devel+owner at edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub [edk2-devel-archive at redhat.com]
-=-=-=-=-=-=-=-=-=-=-=-
More information about the edk2-devel-archive
mailing list