[edk2-devel] [Patch 4/4 V4] BaseTools: Enhance Basetool for incremental build
Bob Feng via Groups.Io
bob.c.feng=intel.com at groups.io
Tue Dec 3 08:52:47 UTC 2019
BZ: https://bugzilla.tianocore.org/show_bug.cgi?id=2311
Include dependency file in Makefile to enhance
incremental build
Cc: Liming Gao <liming.gao at intel.com>
Cc: Steven Shi <steven.shi at intel.com>
Signed-off-by: Bob Feng <bob.c.feng at intel.com>
---
BaseTools/Source/Python/AutoGen/GenMake.py | 83 ++----
.../Source/Python/AutoGen/IncludesAutoGen.py | 258 ++++++++++++++++++
.../Source/Python/AutoGen/ModuleAutoGen.py | 23 ++
BaseTools/Source/Python/build/build.py | 63 ++++-
4 files changed, 354 insertions(+), 73 deletions(-)
create mode 100644 BaseTools/Source/Python/AutoGen/IncludesAutoGen.py
diff --git a/BaseTools/Source/Python/AutoGen/GenMake.py b/BaseTools/Source/Python/AutoGen/GenMake.py
index 59a01a7f24..fe94f9a4c2 100755
--- a/BaseTools/Source/Python/AutoGen/GenMake.py
+++ b/BaseTools/Source/Python/AutoGen/GenMake.py
@@ -183,10 +183,16 @@ class BuildFile(object):
EdkLogger.error("build", PARAMETER_INVALID, "Invalid build type [%s]" % FileType,
ExtraData="[%s]" % str(self._AutoGenObject))
self._FileType = FileType
FileContent = self._TEMPLATE_.Replace(self._TemplateDict)
FileName = self._FILE_NAME_[FileType]
+ if not os.path.exists(os.path.join(self._AutoGenObject.MakeFileDir, "deps.txt")):
+ with open(os.path.join(self._AutoGenObject.MakeFileDir, "deps.txt"),"w+") as fd:
+ fd.write("")
+ if not os.path.exists(os.path.join(self._AutoGenObject.MakeFileDir, "dependency")):
+ with open(os.path.join(self._AutoGenObject.MakeFileDir, "dependency"),"w+") as fd:
+ fd.write("")
return SaveFileOnChange(os.path.join(self._AutoGenObject.MakeFileDir, FileName), FileContent, False)
## Return a list of directory creation command string
#
# @param DirList The list of directory to be created
@@ -302,13 +308,10 @@ MAKE_FILE = ${makefile_path}
# Build Macro
#
${BEGIN}${file_macro}
${END}
-COMMON_DEPS = ${BEGIN}${common_dependency_file} \\
- ${END}
-
#
# Overridable Target Macro Definitions
#
FORCE_REBUILD = force_build
INIT_TARGET = init
@@ -380,10 +383,12 @@ gen_libs:
#
gen_fds:
\t@"$(MAKE)" $(MAKE_FLAGS) -f $(BUILD_DIR)${separator}${makefile_name} fds
\t at cd $(MODULE_BUILD_DIR)
+${INCLUDETAG}
+
#
# Individual Object Build Targets
#
${BEGIN}${file_build_target}
${END}
@@ -513,13 +518,10 @@ cleanlib:
if Tool == "MAKE":
continue
# Remove duplicated include path, if any
if Attr == "FLAGS":
Value = RemoveDupOption(Value, IncPrefix, MyAgo.IncludePathList)
- if self._AutoGenObject.BuildRuleFamily == TAB_COMPILER_MSFT and Tool == 'CC' and '/GM' in Value:
- Value = Value.replace(' /MP', '')
- MyAgo.BuildOption[Tool][Attr] = Value
if Tool == "OPTROM" and PCI_COMPRESS_Flag:
ValueList = Value.split()
if ValueList:
for i, v in enumerate(ValueList):
if '-e' == v:
@@ -538,11 +540,11 @@ cleanlib:
RespFile = os.path.join(MyAgo.OutputDir, str(Resp).lower() + '.txt')
StrList = RespDict[Resp].split(' ')
UnexpandMacro = []
NewStr = []
for Str in StrList:
- if '$' in Str:
+ if '$' in Str or '-MMD' in Str or '-MF' in Str:
UnexpandMacro.append(Str)
else:
NewStr.append(Str)
UnexpandMacroStr = ' '.join(UnexpandMacro)
NewRespStr = ' '.join(NewStr)
@@ -588,14 +590,13 @@ cleanlib:
"source_file" : IncludePathList
}
)
FileMacroList.append(FileMacro)
# Add support when compiling .nasm source files
- for File in self.FileCache.keys():
- if not str(File).endswith('.nasm'):
- continue
- IncludePathList = []
+ IncludePathList = []
+ asmsource = [item for item in MyAgo.SourceFileList if item.File.upper().endswith((".NASM",".ASM",".NASMB","S"))]
+ if asmsource:
for P in MyAgo.IncludePathList:
IncludePath = self._INC_FLAG_['NASM'] + self.PlaceMacro(P, self.Macros)
if IncludePath.endswith(os.sep):
IncludePath = IncludePath.rstrip(os.sep)
# When compiling .nasm files, need to add a literal backslash at each path
@@ -604,11 +605,10 @@ cleanlib:
IncludePath = ''.join([IncludePath, '^', os.sep])
else:
IncludePath = os.path.join(IncludePath, '')
IncludePathList.append(IncludePath)
FileMacroList.append(self._FILE_MACRO_TEMPLATE.Replace({"macro_name": "NASM_INC", "source_file": IncludePathList}))
- break
# Generate macros used to represent files containing list of input files
for ListFileMacro in self.ListFileMacros:
ListFileName = os.path.join(MyAgo.OutputDir, "%s.lst" % ListFileMacro.lower()[:len(ListFileMacro) - 5])
FileMacroList.append("%s = %s" % (ListFileMacro, ListFileName))
@@ -694,10 +694,11 @@ cleanlib:
"dependent_library_build_directory" : self.LibraryBuildDirectoryList,
"library_build_command" : LibraryMakeCommandList,
"file_macro" : FileMacroList,
"file_build_target" : self.BuildTargetList,
"backward_compatible_target": BcTargetList,
+ "INCLUDETAG" : self._INCLUDE_CMD_[self._FileType] + " " + os.path.join("$(MODULE_BUILD_DIR)","dependency")
}
return MakefileTemplateDict
def ParserGenerateFfsCmd(self):
@@ -901,20 +902,14 @@ cleanlib:
if OutPutFileList:
for Item in OutPutFileList:
if Item in SourceFileList:
SourceFileList.remove(Item)
- FileDependencyDict = self.GetFileDependency(
- SourceFileList,
- ForceIncludedFile,
- self._AutoGenObject.IncludePathList + self._AutoGenObject.BuildOptionIncPathList
- )
-
+ FileDependencyDict = {item:ForceIncludedFile for item in SourceFileList}
- if FileDependencyDict:
- for Dependency in FileDependencyDict.values():
- self.DependencyHeaderFileSet.update(set(Dependency))
+ for Dependency in FileDependencyDict.values():
+ self.DependencyHeaderFileSet.update(set(Dependency))
# Get a set of unique package includes from MetaFile
parentMetaFileIncludes = set()
for aInclude in self._AutoGenObject.PackageIncludePathList:
aIncludeName = str(aInclude)
@@ -970,46 +965,20 @@ cleanlib:
GlobalData.gModuleBuildTracking[self._AutoGenObject] = 'FAIL_METAFILE'
EdkLogger.warn("build","Module MetaFile [Sources] is missing local header!",
ExtraData = "Local Header: " + aFile + " not found in " + self._AutoGenObject.MetaFile.Path
)
- DepSet = None
for File,Dependency in FileDependencyDict.items():
if not Dependency:
- FileDependencyDict[File] = ['$(FORCE_REBUILD)']
continue
self._AutoGenObject.AutoGenDepSet |= set(Dependency)
- # skip non-C files
- if File.Ext not in [".c", ".C"] or File.Name == "AutoGen.c":
- continue
- elif DepSet is None:
- DepSet = set(Dependency)
- else:
- DepSet &= set(Dependency)
- # in case nothing in SourceFileList
- if DepSet is None:
- DepSet = set()
- #
- # Extract common files list in the dependency files
- #
- for File in DepSet:
- self.CommonFileDependency.append(self.PlaceMacro(File.Path, self.Macros))
-
CmdSumDict = {}
CmdTargetDict = {}
CmdCppDict = {}
DependencyDict = FileDependencyDict.copy()
- for File in FileDependencyDict:
- # skip non-C files
- if File.Ext not in [".c", ".C"] or File.Name == "AutoGen.c":
- continue
- NewDepSet = set(FileDependencyDict[File])
- NewDepSet -= DepSet
- FileDependencyDict[File] = ["$(COMMON_DEPS)"] + list(NewDepSet)
- DependencyDict[File] = list(NewDepSet)
# Convert target description object to target string in makefile
if self._AutoGenObject.BuildRuleFamily == TAB_COMPILER_MSFT and TAB_C_CODE_FILE in self._AutoGenObject.Targets:
for T in self._AutoGenObject.Targets[TAB_C_CODE_FILE]:
NewFile = self.PlaceMacro(str(T), self.Macros)
@@ -1078,21 +1047,17 @@ cleanlib:
if CmdCppDict.get(item.Target.SubDir):
CmdCppDict[item.Target.SubDir].append(Path)
else:
CmdCppDict[item.Target.SubDir] = ['$(MAKE_FILE)', Path]
if CppPath.Path in DependencyDict:
- if '$(FORCE_REBUILD)' in DependencyDict[CppPath.Path]:
- if '$(FORCE_REBUILD)' not in (self.CommonFileDependency + CmdCppDict[item.Target.SubDir]):
- CmdCppDict[item.Target.SubDir].append('$(FORCE_REBUILD)')
- else:
- for Temp in DependencyDict[CppPath.Path]:
- try:
- Path = self.PlaceMacro(Temp.Path, self.Macros)
- except:
- continue
- if Path not in (self.CommonFileDependency + CmdCppDict[item.Target.SubDir]):
- CmdCppDict[item.Target.SubDir].append(Path)
+ for Temp in DependencyDict[CppPath.Path]:
+ try:
+ Path = self.PlaceMacro(Temp.Path, self.Macros)
+ except:
+ continue
+ if Path not in (self.CommonFileDependency + CmdCppDict[item.Target.SubDir]):
+ CmdCppDict[item.Target.SubDir].append(Path)
if T.Commands:
CommandList = T.Commands[:]
for Item in CommandList[:]:
SingleCommandList = Item.split()
if len(SingleCommandList) > 0 and self.CheckCCCmd(SingleCommandList):
@@ -1107,11 +1072,11 @@ cleanlib:
CmdTargetDict[CmdSign] = "%s %s" % (CmdTargetDict[CmdSign], SingleCommandList[-1])
Index = CommandList.index(Item)
CommandList.pop(Index)
if SingleCommandList[-1].endswith("%s%s.c" % (TAB_SLASH, CmdSumDict[CmdSign[3:].rsplit(TAB_SLASH, 1)[0]])):
Cpplist = CmdCppDict[T.Target.SubDir]
- Cpplist.insert(0, '$(OBJLIST_%d): $(COMMON_DEPS)' % list(self.ObjTargetDict.keys()).index(T.Target.SubDir))
+ Cpplist.insert(0, '$(OBJLIST_%d): ' % list(self.ObjTargetDict.keys()).index(T.Target.SubDir))
T.Commands[Index] = '%s\n\t%s' % (' \\\n\t'.join(Cpplist), CmdTargetDict[CmdSign])
else:
T.Commands.pop(Index)
return T, CmdSumDict, CmdTargetDict, CmdCppDict
diff --git a/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py b/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py
new file mode 100644
index 0000000000..fd85650423
--- /dev/null
+++ b/BaseTools/Source/Python/AutoGen/IncludesAutoGen.py
@@ -0,0 +1,258 @@
+## @file
+# Build cache intermediate result and state
+#
+# Copyright (c) 2019, Intel Corporation. All rights reserved.<BR>
+# SPDX-License-Identifier: BSD-2-Clause-Patent
+#
+from Common.caching import cached_property
+import Common.EdkLogger as EdkLogger
+import Common.LongFilePathOs as os
+from Common.BuildToolError import *
+from Common.Misc import SaveFileOnChange, PathClass
+from Common.Misc import TemplateString
+import sys
+gIsFileMap = {}
+if sys.platform == "win32":
+ _INCLUDE_DEPS_TEMPLATE = TemplateString('''
+${BEGIN}
+!IF EXIST(${deps_file})
+!INCLUDE ${deps_file}
+!ENDIF
+${END}
+ ''')
+else:
+ _INCLUDE_DEPS_TEMPLATE = TemplateString('''
+${BEGIN}
+-include ${deps_file}
+${END}
+ ''')
+
+DEP_FILE_TAIL = "# Updated \n"
+
+class IncludesAutoGen():
+ """ This class is to manage the dependent files witch are used in Makefile to support incremental build.
+ 1. C files:
+ 1. MSVS.
+ cl.exe has a build option /showIncludes to display include files on stdout. Build tool captures
+ that messages and generate dependency files, .deps files.
+ 2. CLANG and GCC
+ -MMD -MF build option are used to generate dependency files by compiler. Build tool updates the
+ .deps files.
+ 2. ASL files:
+ 1. Trim find out all the included files with asl specific include format and generate .trim.deps file.
+ 2. ASL PP use c preprocessor to find out all included files with #include format and generate a .deps file
+ 3. build tool updates the .deps file
+ 3. ASM files (.asm, .s or .nasm):
+ 1. Trim find out all the included files with asl specific include format and generate .trim.deps file.
+ 2. ASM PP use c preprocessor to find out all included files with #include format and generate a deps file
+ 3. build tool updates the .deps file
+ """
+ def __init__(self, makefile_folder, ModuleAuto):
+ self.d_folder = makefile_folder
+ self.makefile_folder = makefile_folder
+ self.module_autogen = ModuleAuto
+ self.ToolChainFamily = ModuleAuto.ToolChainFamily
+ self.workspace = ModuleAuto.WorkspaceDir
+
+ def CreateModuleDeps(self):
+ SaveFileOnChange(os.path.join(self.makefile_folder,"deps.txt"),"\n".join(self.DepsCollection),False)
+
+ def CreateDepsInclude(self):
+ deps_file = {'deps_file':self.deps_files}
+ try:
+ deps_include_str = _INCLUDE_DEPS_TEMPLATE.Replace(deps_file)
+ except Exception as e:
+ print(e)
+ SaveFileOnChange(os.path.join(self.makefile_folder,"dependency"),deps_include_str,False)
+
+ @cached_property
+ def deps_files(self):
+ """ Get all .deps file under module build folder. """
+ deps_files = []
+ for root, _, files in os.walk(self.d_folder, topdown=False):
+ for name in files:
+ if not name.endswith(".deps"):
+ continue
+ abspath = os.path.join(root, name)
+ deps_files.append(abspath)
+ return deps_files
+
+ @cached_property
+ def DepsCollection(self):
+ """ Collect all the dependency files list from all .deps files under a module's build folder """
+ includes = set()
+ targetname = [item[0].Name for item in self.TargetFileList.values()]
+ for abspath in self.deps_files:
+ try:
+ with open(abspath,"r") as fd:
+ lines = fd.readlines()
+
+ firstlineitems = lines[0].split(": ")
+ dependency_file = firstlineitems[1].strip(" \\\n")
+ dependency_file = dependency_file.strip('''"''')
+ if dependency_file:
+ if os.path.normpath(dependency_file +".deps") == abspath:
+ continue
+ filename = os.path.basename(dependency_file).strip()
+ if filename not in self.SourceFileList and filename not in targetname:
+ includes.add(dependency_file.strip())
+
+ for item in lines[1:]:
+ if item == DEP_FILE_TAIL:
+ continue
+ dependency_file = item.strip(" \\\n")
+ dependency_file = dependency_file.strip('''"''')
+ if os.path.normpath(dependency_file +".deps") == abspath:
+ continue
+ filename = os.path.basename(dependency_file).strip()
+ if filename in self.SourceFileList:
+ continue
+ if filename in targetname:
+ continue
+ includes.add(dependency_file.strip())
+ except Exception as e:
+ EdkLogger.error("build",FILE_NOT_FOUND, "%s doesn't exist" % abspath, ExtraData=str(e), RaiseError=False)
+ continue
+ rt = sorted(list(set([item.strip(' " \\\n') for item in includes])))
+ return rt
+
+ @cached_property
+ def SourceFileList(self):
+ """ Get a map of module's source files name to module's source files path """
+ source = {os.path.basename(item.File):item.Path for item in self.module_autogen.SourceFileList}
+ middle_file = {}
+ for afile in source:
+ if afile.upper().endswith(".VFR"):
+ middle_file.update({afile.split(".")[0]+".c":os.path.join(self.module_autogen.DebugDir,afile.split(".")[0]+".c")})
+ if afile.upper().endswith((".S","ASM")):
+ middle_file.update({afile.split(".")[0]+".i":os.path.join(self.module_autogen.OutputDir,afile.split(".")[0]+".i")})
+ if afile.upper().endswith(".ASL"):
+ middle_file.update({afile.split(".")[0]+".i":os.path.join(self.module_autogen.OutputDir,afile.split(".")[0]+".i")})
+ source.update({"AutoGen.c":os.path.join(self.module_autogen.OutputDir,"AutoGen.c")})
+ source.update(middle_file)
+ return source
+
+ @cached_property
+ def TargetFileList(self):
+ """ Get a map of module's target name to a tuple of module's targets path and whose input file path """
+ targets = {}
+ targets["AutoGen.obj"] = (PathClass(os.path.join(self.module_autogen.OutputDir,"AutoGen.obj")),PathClass(os.path.join(self.module_autogen.DebugDir,"AutoGen.c")))
+ for item in self.module_autogen.Targets.values():
+ for block in item:
+ targets[block.Target.Path] = (block.Target,block.Inputs[0])
+ return targets
+
+ def GetRealTarget(self,source_file_abs):
+ """ Get the final target file based on source file abspath """
+ source_target_map = {item[1].Path:item[0].Path for item in self.TargetFileList.values()}
+ source_name_map = {item[1].File:item[0].Path for item in self.TargetFileList.values()}
+ target_abs = source_target_map.get(source_file_abs)
+ if target_abs is None:
+ if source_file_abs.strip().endswith(".i"):
+ sourcefilename = os.path.basename(source_file_abs.strip())
+ for sourcefile in source_name_map:
+ if sourcefilename.split(".")[0] == sourcefile.split(".")[0]:
+ target_abs = source_name_map[sourcefile]
+ break
+ else:
+ target_abs = source_file_abs
+ else:
+ target_abs = source_file_abs
+ return target_abs
+
+ def CreateDepsFileForMsvc(self, DepList):
+ """ Generate dependency files, .deps file from /showIncludes output message """
+ if not DepList:
+ return
+ ModuleDepDict = {}
+ current_source = ""
+ for line in DepList:
+ if line.strip() in self.SourceFileList:
+ current_source = line.strip()
+ ModuleDepDict[current_source] = []
+ elif "Note: including file:" == line.lstrip()[:21]:
+ if not current_source:
+ EdkLogger.error("build",BUILD_ERROR, "Parse /showIncludes output failed. line: %s. \n" % line, RaiseError=False)
+ else:
+ ModuleDepDict[current_source].append(line.lstrip()[22:].strip())
+
+ for source in ModuleDepDict:
+ if ModuleDepDict[source]:
+ source_abs = self.SourceFileList.get(source,source)
+ target_abs = self.GetRealTarget(source_abs)
+ dep_file_name = source + ".deps"
+ SaveFileOnChange(os.path.join(self.makefile_folder,dep_file_name)," \\\n".join([target_abs+":"] + ['''"''' + item +'''"''' for item in ModuleDepDict[source]]),False)
+
+ def UpdateDepsFileforNonMsvc(self):
+ """ Update .deps files.
+ 1. Update target path to absolute path.
+ 2. Update middle target to final target.
+ """
+
+ for abspath in self.deps_files:
+ if abspath.endswith(".trim.deps"):
+ continue
+ try:
+ newcontent = []
+ with open(abspath,"r") as fd:
+ lines = fd.readlines()
+ if lines[-1] == DEP_FILE_TAIL:
+ continue
+ firstlineitems = lines[0].strip().split(" ")
+
+ if len(firstlineitems) > 2:
+ sourceitem = firstlineitems[1]
+ else:
+ sourceitem = lines[1].strip().split(" ")[0]
+
+ source_abs = self.SourceFileList.get(sourceitem,sourceitem)
+ firstlineitems[0] = self.GetRealTarget(source_abs)
+ p_target = firstlineitems
+ if not p_target[0].strip().endswith(":"):
+ p_target[0] += ": "
+
+ if len(p_target) == 2:
+ p_target[0] += lines[1]
+ newcontent.append(p_target[0])
+ newcontent.extend(lines[2:])
+ else:
+ line1 = " ".join(p_target).strip()
+ line1 += "\n"
+ newcontent.append(line1)
+ newcontent.extend(lines[1:])
+
+ newcontent.append("\n")
+ newcontent.append(DEP_FILE_TAIL)
+ with open(abspath,"w") as fw:
+ fw.write("".join(newcontent))
+ except Exception as e:
+ EdkLogger.error("build",FILE_NOT_FOUND, "%s doesn't exist" % abspath, ExtraData=str(e), RaiseError=False)
+ continue
+
+ def UpdateDepsFileforTrim(self):
+ """ Update .deps file which generated by trim. """
+
+ for abspath in self.deps_files:
+ if not abspath.endswith(".trim.deps"):
+ continue
+ try:
+ newcontent = []
+ with open(abspath,"r") as fd:
+ lines = fd.readlines()
+ if lines[-1] == DEP_FILE_TAIL:
+ continue
+
+ source_abs = lines[0].strip().split(" ")[0]
+ targetitem = self.GetRealTarget(source_abs.strip(" :"))
+
+ targetitem += ": "
+ targetitem += lines[1]
+ newcontent.append(targetitem)
+ newcontent.extend(lines[2:])
+ newcontent.append("\n")
+ newcontent.append(DEP_FILE_TAIL)
+ with open(abspath,"w") as fw:
+ fw.write("".join(newcontent))
+ except Exception as e:
+ EdkLogger.error("build",FILE_NOT_FOUND, "%s doesn't exist" % abspath, ExtraData=str(e), RaiseError=False)
+ continue
diff --git a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py
index e6d6c43810..1111d5de25 100755
--- a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py
+++ b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py
@@ -1127,12 +1127,35 @@ class ModuleAutoGen(AutoGen):
if not self.MetaFile.OriginalPath.Path.startswith(PackageDir):
IncludesList = list(set(Package.Includes).difference(set(Package._PrivateIncludes)))
for Inc in IncludesList:
if Inc not in RetVal:
RetVal.append(str(Inc))
+ RetVal.extend(self.IncPathFromBuildOptions)
return RetVal
+ @cached_property
+ def IncPathFromBuildOptions(self):
+ IncPathList = []
+ for tool in self.BuildOption:
+ if 'FLAGS' in self.BuildOption[tool]:
+ flags = self.BuildOption[tool]['FLAGS']
+ whitespace = False
+ for flag in flags.split(" "):
+ flag = flag.strip()
+ if flag.startswith(("/I","-I")):
+ if len(flag)>2:
+ if os.path.exists(flag[2:]):
+ IncPathList.append(flag[2:])
+ else:
+ whitespace = True
+ continue
+ if whitespace and flag:
+ if os.path.exists(flag):
+ IncPathList.append(flag)
+ whitespace = False
+ return IncPathList
+
@cached_property
def IncludePathLength(self):
return sum(len(inc)+1 for inc in self.IncludePathList)
## Get the list of include paths from the packages
diff --git a/BaseTools/Source/Python/build/build.py b/BaseTools/Source/Python/build/build.py
index 4b31356a42..6fcfc7ec7a 100755
--- a/BaseTools/Source/Python/build/build.py
+++ b/BaseTools/Source/Python/build/build.py
@@ -55,11 +55,11 @@ from GenFds.GenFds import GenFds, GenFdsApi
import multiprocessing as mp
from multiprocessing import Manager
from AutoGen.DataPipe import MemoryDataPipe
from AutoGen.ModuleAutoGenHelper import WorkSpaceInfo, PlatformInfo
from GenFds.FdfParser import FdfParser
-
+from AutoGen.IncludesAutoGen import IncludesAutoGen
## standard targets of build command
gSupportedTarget = ['all', 'genc', 'genmake', 'modules', 'libraries', 'fds', 'clean', 'cleanall', 'cleanlib', 'run']
## build configuration file
@@ -173,33 +173,46 @@ def NormFile(FilePath, Workspace):
#
# @param From The stream message read from
# @param To The stream message put on
# @param ExitFlag The flag used to indicate stopping reading
#
-def ReadMessage(From, To, ExitFlag):
+def ReadMessage(From, To, ExitFlag,MemTo=None):
while True:
# read one line a time
Line = From.readline()
# empty string means "end"
if Line is not None and Line != b"":
- To(Line.rstrip().decode(encoding='utf-8', errors='ignore'))
+ LineStr = Line.rstrip().decode(encoding='utf-8', errors='ignore')
+ if MemTo is not None:
+ if "Note: including file:" == LineStr.lstrip()[:21]:
+ MemTo.append(LineStr)
+ else:
+ To(LineStr)
+ MemTo.append(LineStr)
+ else:
+ To(LineStr)
else:
break
if ExitFlag.isSet():
break
+class MakeSubProc(Popen):
+ def __init__(self,*args, **argv):
+ super(MakeSubProc,self).__init__(*args, **argv)
+ self.ProcOut = []
+
## Launch an external program
#
# This method will call subprocess.Popen to execute an external program with
# given options in specified directory. Because of the dead-lock issue during
# redirecting output of the external program, threads are used to to do the
# redirection work.
#
# @param Command A list or string containing the call of the program
# @param WorkingDir The directory in which the program will be running
#
-def LaunchCommand(Command, WorkingDir):
+def LaunchCommand(Command, WorkingDir,ModuleAuto = None):
BeginTime = time.time()
# if working directory doesn't exist, Popen() will raise an exception
if not os.path.isdir(WorkingDir):
EdkLogger.error("build", FILE_NOT_FOUND, ExtraData=WorkingDir)
@@ -214,23 +227,23 @@ def LaunchCommand(Command, WorkingDir):
Proc = None
EndOfProcedure = None
try:
# launch the command
- Proc = Popen(Command, stdout=PIPE, stderr=PIPE, env=os.environ, cwd=WorkingDir, bufsize=-1, shell=True)
+ Proc = MakeSubProc(Command, stdout=PIPE, stderr=PIPE, env=os.environ, cwd=WorkingDir, bufsize=-1, shell=True)
# launch two threads to read the STDOUT and STDERR
EndOfProcedure = Event()
EndOfProcedure.clear()
if Proc.stdout:
- StdOutThread = Thread(target=ReadMessage, args=(Proc.stdout, EdkLogger.info, EndOfProcedure))
+ StdOutThread = Thread(target=ReadMessage, args=(Proc.stdout, EdkLogger.info, EndOfProcedure,Proc.ProcOut))
StdOutThread.setName("STDOUT-Redirector")
StdOutThread.setDaemon(False)
StdOutThread.start()
if Proc.stderr:
- StdErrThread = Thread(target=ReadMessage, args=(Proc.stderr, EdkLogger.quiet, EndOfProcedure))
+ StdErrThread = Thread(target=ReadMessage, args=(Proc.stderr, EdkLogger.quiet, EndOfProcedure,Proc.ProcOut))
StdErrThread.setName("STDERR-Redirector")
StdErrThread.setDaemon(False)
StdErrThread.start()
# waiting for program exit
@@ -261,10 +274,19 @@ def LaunchCommand(Command, WorkingDir):
RespContent = f.read()
f.close()
EdkLogger.info(RespContent)
EdkLogger.error("build", COMMAND_FAILURE, ExtraData="%s [%s]" % (Command, WorkingDir))
+ if ModuleAuto:
+ iau = IncludesAutoGen(WorkingDir,ModuleAuto)
+ if ModuleAuto.ToolChainFamily == TAB_COMPILER_MSFT:
+ iau.CreateDepsFileForMsvc(Proc.ProcOut)
+ else:
+ iau.UpdateDepsFileforNonMsvc()
+ iau.UpdateDepsFileforTrim()
+ iau.CreateModuleDeps()
+ iau.CreateDepsInclude()
return "%dms" % (int(round((time.time() - BeginTime) * 1000)))
## The smallest unit that can be built in multi-thread build mode
#
# This is the base class of build unit. The "Obj" parameter must provide
@@ -606,11 +628,11 @@ class BuildTask:
# @param Command A list or string contains the call of the command
# @param WorkingDir The directory in which the program will be running
#
def _CommandThread(self, Command, WorkingDir):
try:
- self.BuildItem.BuildObject.BuildTime = LaunchCommand(Command, WorkingDir)
+ self.BuildItem.BuildObject.BuildTime = LaunchCommand(Command, WorkingDir,self.BuildItem.BuildObject)
self.CompleteFlag = True
# Run hash operation post dependency, to account for libs
if GlobalData.gUseHashCache and self.BuildItem.BuildObject.IsLibrary:
HashFile = path.join(self.BuildItem.BuildObject.BuildDir, self.BuildItem.BuildObject.Name + ".hash")
@@ -1274,23 +1296,36 @@ class Build():
self.BuildModules = []
return True
# build library
if Target == 'libraries':
- for Lib in AutoGenObject.LibraryBuildDirectoryList:
+ DirList = []
+ for Lib in AutoGenObject.LibraryAutoGenList:
+ if not Lib.IsBinaryModule:
+ DirList.append((os.path.join(AutoGenObject.BuildDir, Lib.BuildDir),Lib))
+ for Lib, LibAutoGen in DirList:
NewBuildCommand = BuildCommand + ['-f', os.path.normpath(os.path.join(Lib, makefile)), 'pbuild']
- LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir)
+ LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir,LibAutoGen)
return True
# build module
if Target == 'modules':
- for Lib in AutoGenObject.LibraryBuildDirectoryList:
+ DirList = []
+ for Lib in AutoGenObject.LibraryAutoGenList:
+ if not Lib.IsBinaryModule:
+ DirList.append((os.path.join(AutoGenObject.BuildDir, Lib.BuildDir),Lib))
+ for Lib, LibAutoGen in DirList:
NewBuildCommand = BuildCommand + ['-f', os.path.normpath(os.path.join(Lib, makefile)), 'pbuild']
- LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir)
- for Mod in AutoGenObject.ModuleBuildDirectoryList:
+ LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir,LibAutoGen)
+
+ DirList = []
+ for ModuleAutoGen in AutoGenObject.ModuleAutoGenList:
+ if not ModuleAutoGen.IsBinaryModule:
+ DirList.append((os.path.join(AutoGenObject.BuildDir, ModuleAutoGen.BuildDir),ModuleAutoGen))
+ for Mod,ModAutoGen in DirList:
NewBuildCommand = BuildCommand + ['-f', os.path.normpath(os.path.join(Mod, makefile)), 'pbuild']
- LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir)
+ LaunchCommand(NewBuildCommand, AutoGenObject.MakeFileDir,ModAutoGen)
self.CreateAsBuiltInf()
if GlobalData.gBinCacheDest:
self.UpdateBuildCache()
self.BuildModules = []
return True
--
2.20.1.windows.1
-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.
View/Reply Online (#51592): https://edk2.groups.io/g/devel/message/51592
Mute This Topic: https://groups.io/mt/65666186/1813853
Group Owner: devel+owner at edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub [edk2-devel-archive at redhat.com]
-=-=-=-=-=-=-=-=-=-=-=-
More information about the edk2-devel-archive
mailing list