[edk2-devel] [Patch 10/10 V5] BaseTools: Enable block queue log agent.
Bob Feng
bob.c.feng at intel.com
Wed Jul 31 05:52:44 UTC 2019
BZ: https://bugzilla.tianocore.org/show_bug.cgi?id=1875
To support Ctrl+S and Ctrl+Q, we enable block queue
for log.
Cc: Liming Gao <liming.gao at intel.com>
Signed-off-by: Bob Feng <bob.c.feng at intel.com>
---
.../Source/Python/AutoGen/AutoGenWorker.py | 6 ++-
BaseTools/Source/Python/Common/EdkLogger.py | 10 +++--
BaseTools/Source/Python/build/build.py | 39 ++++++++++---------
3 files changed, 31 insertions(+), 24 deletions(-)
diff --git a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
index 1f388ee178be..635ddfe5203f 100644
--- a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
+++ b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
@@ -21,17 +21,18 @@ try:
except:
from Queue import Empty
import traceback
import sys
from AutoGen.DataPipe import MemoryDataPipe
+import logging
+
def clearQ(q):
try:
while True:
q.get_nowait()
except Empty:
pass
-import logging
class LogAgent(threading.Thread):
def __init__(self,log_q,log_level,log_file=None):
super(LogAgent,self).__init__()
self.log_q = log_q
@@ -121,13 +122,14 @@ class AutoGenManager(threading.Thread):
except Exception:
return
def clearQueue(self):
taskq = self.autogen_workers[0].module_queue
+ logq = self.autogen_workers[0].log_q
clearQ(taskq)
clearQ(self.feedback_q)
-
+ clearQ(logq)
def TerminateWorkers(self):
self.error_event.set()
def kill(self):
self.feedback_q.put(None)
class AutoGenWorkerInProcess(mp.Process):
diff --git a/BaseTools/Source/Python/Common/EdkLogger.py b/BaseTools/Source/Python/Common/EdkLogger.py
index 15fd1458a95a..06da4a9d0a1d 100644
--- a/BaseTools/Source/Python/Common/EdkLogger.py
+++ b/BaseTools/Source/Python/Common/EdkLogger.py
@@ -93,11 +93,13 @@ except:
"""
try:
self.enqueue(self.prepare(record))
except Exception:
self.handleError(record)
-
+class BlockQueueHandler(QueueHandler):
+ def enqueue(self, record):
+ self.queue.put(record,True)
## Log level constants
DEBUG_0 = 1
DEBUG_1 = 2
DEBUG_2 = 3
DEBUG_3 = 4
@@ -290,23 +292,23 @@ def LogClientInitialize(log_q):
# Since we use different format to log different levels of message into different
# place (stdout or stderr), we have to use different "Logger" objects to do this.
#
# For DEBUG level (All DEBUG_0~9 are applicable)
_DebugLogger.setLevel(INFO)
- _DebugChannel = QueueHandler(log_q)
+ _DebugChannel = BlockQueueHandler(log_q)
_DebugChannel.setFormatter(_DebugFormatter)
_DebugLogger.addHandler(_DebugChannel)
# For VERBOSE, INFO, WARN level
_InfoLogger.setLevel(INFO)
- _InfoChannel = QueueHandler(log_q)
+ _InfoChannel = BlockQueueHandler(log_q)
_InfoChannel.setFormatter(_InfoFormatter)
_InfoLogger.addHandler(_InfoChannel)
# For ERROR level
_ErrorLogger.setLevel(INFO)
- _ErrorCh = QueueHandler(log_q)
+ _ErrorCh = BlockQueueHandler(log_q)
_ErrorCh.setFormatter(_ErrorFormatter)
_ErrorLogger.addHandler(_ErrorCh)
## Set log level
#
diff --git a/BaseTools/Source/Python/build/build.py b/BaseTools/Source/Python/build/build.py
index 1c03f063668b..f06fd47a8916 100644
--- a/BaseTools/Source/Python/build/build.py
+++ b/BaseTools/Source/Python/build/build.py
@@ -707,11 +707,11 @@ class Build():
self.Fdf = BuildOptions.FdfFile
self.FdList = BuildOptions.RomImage
self.FvList = BuildOptions.FvImage
self.CapList = BuildOptions.CapName
self.SilentMode = BuildOptions.SilentMode
- self.ThreadNumber = BuildOptions.ThreadNumber
+ self.ThreadNumber = 1
self.SkipAutoGen = BuildOptions.SkipAutoGen
self.Reparse = BuildOptions.Reparse
self.SkuId = BuildOptions.SkuId
if self.SkuId:
GlobalData.gSKUID_CMD = self.SkuId
@@ -881,23 +881,10 @@ class Build():
ToolChainFamily.append(TAB_COMPILER_MSFT)
else:
ToolChainFamily.append(ToolDefinition[TAB_TOD_DEFINES_FAMILY][Tool])
self.ToolChainFamily = ToolChainFamily
- if self.ThreadNumber is None:
- self.ThreadNumber = self.TargetTxt.TargetTxtDictionary[TAB_TAT_DEFINES_MAX_CONCURRENT_THREAD_NUMBER]
- if self.ThreadNumber == '':
- self.ThreadNumber = 0
- else:
- self.ThreadNumber = int(self.ThreadNumber, 0)
-
- if self.ThreadNumber == 0:
- try:
- self.ThreadNumber = multiprocessing.cpu_count()
- except (ImportError, NotImplementedError):
- self.ThreadNumber = 1
-
if not self.PlatformFile:
PlatformFile = self.TargetTxt.TargetTxtDictionary[TAB_TAT_DEFINES_ACTIVE_PLATFORM]
if not PlatformFile:
# Try to find one in current directory
WorkingDirectory = os.getcwd()
@@ -911,11 +898,11 @@ class Build():
else:
EdkLogger.error("build", RESOURCE_NOT_AVAILABLE,
ExtraData="No active platform specified in target.txt or command line! Nothing can be built.\n")
self.PlatformFile = PathClass(NormFile(PlatformFile, self.WorkspaceDir), self.WorkspaceDir)
-
+ self.ThreadNumber = ThreadNum()
## Initialize build configuration
#
# This method will parse DSC file and merge the configurations from
# command line and target.txt, then get the final build configurations.
#
@@ -2052,16 +2039,17 @@ class Build():
for m in Pa.GetAllModuleInfo:
mqueue.put(m)
data_pipe_file = os.path.join(Pa.BuildDir, "GlobalVar_%s_%s.bin" % (str(Pa.Guid),Pa.Arch))
Pa.DataPipe.dump(data_pipe_file)
autogen_rt, errorcode = self.StartAutoGen(mqueue, Pa.DataPipe, self.SkipAutoGen, PcdMaList,self.share_data)
- self.Progress.Stop("done!")
- self.AutoGenTime += int(round((time.time() - AutoGenStart)))
+
if not autogen_rt:
self.AutoGenMgr.TerminateWorkers()
self.AutoGenMgr.join(0.1)
raise FatalError(errorcode)
+ self.AutoGenTime += int(round((time.time() - AutoGenStart)))
+ self.Progress.Stop("done!")
for Arch in Wa.ArchList:
MakeStart = time.time()
for Ma in BuildModules:
# Generate build task for the module
if not Ma.IsBinaryModule:
@@ -2292,27 +2280,42 @@ def LogBuildTime(Time):
else:
TimeDurStr = time.strftime("%H:%M:%S", TimeDur)
return TimeDurStr
else:
return None
+def ThreadNum():
+ ThreadNumber = BuildOption.ThreadNumber
+ if ThreadNumber is None:
+ ThreadNumber = TargetTxt.TargetTxtDictionary[TAB_TAT_DEFINES_MAX_CONCURRENT_THREAD_NUMBER]
+ if ThreadNumber == '':
+ ThreadNumber = 0
+ else:
+ ThreadNumber = int(ThreadNumber, 0)
+ if ThreadNumber == 0:
+ try:
+ ThreadNumber = multiprocessing.cpu_count()
+ except (ImportError, NotImplementedError):
+ ThreadNumber = 1
+ return ThreadNumber
## Tool entrance method
#
# This method mainly dispatch specific methods per the command line options.
# If no error found, return zero value so the caller of this tool can know
# if it's executed successfully or not.
#
# @retval 0 Tool was successful
# @retval 1 Tool failed
#
+LogQMaxSize = ThreadNum() * 10
def Main():
StartTime = time.time()
#
# Create a log Queue
#
- LogQ = mp.Queue()
+ LogQ = mp.Queue(LogQMaxSize)
# Initialize log system
EdkLogger.LogClientInitialize(LogQ)
GlobalData.gCommand = sys.argv[1:]
#
# Parse the options and args
--
2.20.1.windows.1
-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.
View/Reply Online (#44663): https://edk2.groups.io/g/devel/message/44663
Mute This Topic: https://groups.io/mt/32662301/1813853
Group Owner: devel+owner at edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub [edk2-devel-archive at redhat.com]
-=-=-=-=-=-=-=-=-=-=-=-
More information about the edk2-devel-archive
mailing list