[edk2-devel] [Patch 1/1] BaseTools: Fixed issue for IgnoreAutoGen
Bob Feng
bob.c.feng at intel.com
Thu Sep 5 14:04:01 UTC 2019
Liming,
Yes, this change only impact -u option.
I separate _MultiThreadBuildPlatform function into two functions and create 2 new functions specially for IgnoreAutoGen, so the there looks big change for build.py in this patch. I verified this patch on Ovmf, E
-Bob
-----Original Message-----
From: Gao, Liming
Sent: Thursday, September 5, 2019 9:12 PM
To: devel at edk2.groups.io; Feng, Bob C <bob.c.feng at intel.com>
Cc: Shi, Steven <steven.shi at intel.com>
Subject: RE: [edk2-devel] [Patch 1/1] BaseTools: Fixed issue for IgnoreAutoGen
Bob:
Does this change only impact -u option behavior?
Thanks
Liming
> -----Original Message-----
> From: devel at edk2.groups.io [mailto:devel at edk2.groups.io] On Behalf Of
> Bob Feng
> Sent: Thursday, September 5, 2019 5:05 PM
> To: devel at edk2.groups.io
> Cc: Gao, Liming <liming.gao at intel.com>; Shi, Steven
> <steven.shi at intel.com>; Feng, Bob C <bob.c.feng at intel.com>
> Subject: [edk2-devel] [Patch 1/1] BaseTools: Fixed issue for
> IgnoreAutoGen
>
> https://bugzilla.tianocore.org/show_bug.cgi?id=2080
>
> This patch is to improve build -u option to re-use
> GlobalVar_<platformguid>_<arch>.bin file which is introduced by
> multiple-process-autogen feature.
>
> Cc: Liming Gao <liming.gao at intel.com>
> Cc: Steven Shi <steven.shi at intel.com>
> Signed-off-by: Bob Feng <bob.c.feng at intel.com>
> ---
> .../Source/Python/AutoGen/AutoGenWorker.py | 2 +-
> .../Source/Python/AutoGen/ModuleAutoGen.py | 4 +-
> .../Python/AutoGen/ModuleAutoGenHelper.py | 29 ++
> .../Source/Python/AutoGen/PlatformAutoGen.py | 2 +
> .../Source/Python/AutoGen/WorkspaceAutoGen.py | 1 -
> BaseTools/Source/Python/build/build.py | 355 +++++++++++-------
> 6 files changed, 255 insertions(+), 138 deletions(-)
>
> diff --git a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
> b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
> index 2e68538b1cb4..f488ae9d5f80 100755
> --- a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
> +++ b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py
> @@ -236,11 +236,11 @@ class AutoGenWorkerInProcess(mp.Process):
> Ma.ReferenceModules = Refes[(Ma.MetaFile.File,Ma.MetaFile.Root,Ma.Arch,Ma.MetaFile.Path)]
> if GlobalData.gBinCacheSource and CommandTarget in [None, "", "all"]:
> Ma.GenModuleFilesHash(GlobalData.gCacheIR)
> Ma.GenPreMakefileHash(GlobalData.gCacheIR)
> if Ma.CanSkipbyPreMakefileCache(GlobalData.gCacheIR):
> - continue
> + continue
>
> Ma.CreateCodeFile(False)
>
> Ma.CreateMakeFile(False,GenFfsList=FfsCmd.get((Ma.MetaFile.File,
> Ma.Arch),[]))
>
> if GlobalData.gBinCacheSource and CommandTarget in [None, "", "all"]:
> diff --git a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py
> b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py
> index 3bb7e91154ac..5f28681e3146 100755
> --- a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py
> +++ b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py
> @@ -2058,12 +2058,12 @@ class ModuleAutoGen(AutoGen):
> not gDict[(self.MetaFile.Path, self.Arch)].ModuleFilesHashDigest:
> self.GenModuleFilesHash(gDict)
>
> if not (self.MetaFile.Path, self.Arch) in gDict or \
> not gDict[(self.MetaFile.Path, self.Arch)].ModuleFilesHashDigest:
> - EdkLogger.quiet("[cache warning]: Cannot generate ModuleFilesHashDigest for module %s[%s]" %(self.MetaFile.Path,
> self.Arch))
> - return
> + EdkLogger.quiet("[cache warning]: Cannot generate
> + ModuleFilesHashDigest for module %s[%s]" %(self.MetaFile.Path,
> self.Arch))
> + return
>
> # Initialze hash object
> m = hashlib.md5()
>
> # Add Platform level hash
> diff --git a/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py
> b/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py
> index c7591253debd..9dd93b9beb12 100644
> --- a/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py
> +++ b/BaseTools/Source/Python/AutoGen/ModuleAutoGenHelper.py
> @@ -234,22 +234,51 @@ class AutoGenInfo(object):
> #
> #This class is the pruned WorkSpaceAutoGen for ModuleAutoGen in
> multiple thread # class WorkSpaceInfo(AutoGenInfo):
> def __init__(self,Workspace, MetaFile, Target, ToolChain, Arch):
> + if not hasattr(self, "_Init"):
> + self.do_init(Workspace, MetaFile, Target, ToolChain, Arch)
> + self._Init = True
> + def do_init(self,Workspace, MetaFile, Target, ToolChain, Arch):
> self._SrcTimeStamp = 0
> self.Db = BuildDB
> self.BuildDatabase = self.Db.BuildObject
> self.Target = Target
> self.ToolChain = ToolChain
> self.WorkspaceDir = Workspace
> self.ActivePlatform = MetaFile
> self.ArchList = Arch
> + self.AutoGenObjectList = []
> + @property
> + def BuildDir(self):
> + return self.AutoGenObjectList[0].BuildDir
>
> + @property
> + def Name(self):
> + return self.AutoGenObjectList[0].Platform.PlatformName
> +
> + @property
> + def FlashDefinition(self):
> + return self.AutoGenObjectList[0].Platform.FlashDefinition
> + @property
> + def GenFdsCommandDict(self):
> + FdsCommandDict = self.AutoGenObjectList[0].DataPipe.Get("FdsCommandDict")
> + if FdsCommandDict:
> + return FdsCommandDict
> + return {}
> +
> + @cached_property
> + def FvDir(self):
> + return os.path.join(self.BuildDir, TAB_FV_DIRECTORY)
>
> class PlatformInfo(AutoGenInfo):
> def __init__(self, Workspace, MetaFile, Target, ToolChain, Arch,DataPipe):
> + if not hasattr(self, "_Init"):
> + self.do_init(Workspace, MetaFile, Target, ToolChain, Arch,DataPipe)
> + self._Init = True
> + def do_init(self,Workspace, MetaFile, Target, ToolChain, Arch,DataPipe):
> self.Wa = Workspace
> self.WorkspaceDir = self.Wa.WorkspaceDir
> self.MetaFile = MetaFile
> self.Arch = Arch
> self.Target = Target
> diff --git a/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py
> b/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py
> index 565424a95ead..1e17b6687129 100644
> --- a/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py
> +++ b/BaseTools/Source/Python/AutoGen/PlatformAutoGen.py
> @@ -186,11 +186,13 @@ class PlatformAutoGen(AutoGen):
> key = (Ma.MetaFile.File, self.Arch)
> if key in FfsCommand:
> Ma.CreateMakeFile(CreateModuleMakeFile, FfsCommand[key])
> else:
> Ma.CreateMakeFile(CreateModuleMakeFile)
> + self.CreateLibModuelDirs()
>
> + def CreateLibModuelDirs(self):
> # no need to create makefile for the platform more than once
> if self.IsMakeFileCreated:
> return
>
> # create library/module build dirs for platform diff --git
> a/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py
> b/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py
> index 24942674721f..f9d2c216169c 100644
> --- a/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py
> +++ b/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py
> @@ -126,11 +126,10 @@ class WorkspaceAutoGen(AutoGen):
> self._CheckPcdDefineAndType()
>
> self.CreateBuildOptionsFile()
> self.CreatePcdTokenNumberFile()
> self.CreateModuleHashInfo()
> - GlobalData.gAutoGenPhase = False
>
> #
> # Merge Arch
> #
> def MergeArch(self):
> diff --git a/BaseTools/Source/Python/build/build.py
> b/BaseTools/Source/Python/build/build.py
> index 0406ac314b65..db57c3e526da 100755
> --- a/BaseTools/Source/Python/build/build.py
> +++ b/BaseTools/Source/Python/build/build.py
> @@ -52,10 +52,13 @@ from PatchPcdValue.PatchPcdValue import
> PatchBinaryFile
>
> import Common.GlobalData as GlobalData from GenFds.GenFds import
> GenFds, GenFdsApi import multiprocessing as mp from multiprocessing
> import Manager
> +from AutoGen.DataPipe import MemoryDataPipe from
> +AutoGen.ModuleAutoGenHelper import WorkSpaceInfo, PlatformInfo from
> +GenFds.FdfParser import FdfParser
>
>
> ## standard targets of build command
> gSupportedTarget = ['all', 'genc', 'genmake', 'modules', 'libraries',
> 'fds', 'clean', 'cleanall', 'cleanlib', 'run']
>
> @@ -837,11 +840,11 @@ class Build():
> for PcdMa in PcdMaList:
> if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]:
> PcdMa.GenModuleFilesHash(share_data)
> PcdMa.GenPreMakefileHash(share_data)
> if PcdMa.CanSkipbyPreMakefileCache(share_data):
> - continue
> + continue
>
> PcdMa.CreateCodeFile(False)
> PcdMa.CreateMakeFile(False,GenFfsList =
> DataPipe.Get("FfsCommand").get((PcdMa.MetaFile.File,
> PcdMa.Arch),[]))
>
> if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]:
> @@ -1218,18 +1221,28 @@ class Build():
> mqueue = mp.Queue()
> for m in AutoGenObject.GetAllModuleInfo:
> mqueue.put(m)
>
> AutoGenObject.DataPipe.DataContainer = {"CommandTarget":
> self.Target}
> + AutoGenObject.DataPipe.DataContainer = {"Workspace_timestamp": AutoGenObject.Workspace._SrcTimeStamp}
> + AutoGenObject.CreateLibModuelDirs()
> + AutoGenObject.DataPipe.DataContainer = {"LibraryBuildDirectoryList":AutoGenObject.LibraryBuildDirectoryList}
> + AutoGenObject.DataPipe.DataContainer = {"ModuleBuildDirectoryList":AutoGenObject.ModuleBuildDirectoryList}
> + AutoGenObject.DataPipe.DataContainer = {"FdsCommandDict":
> + AutoGenObject.Workspace.GenFdsCommandDict}
> self.Progress.Start("Generating makefile and code")
> data_pipe_file = os.path.join(AutoGenObject.BuildDir,
> "GlobalVar_%s_%s.bin" %
> (str(AutoGenObject.Guid),AutoGenObject.Arch))
> AutoGenObject.DataPipe.dump(data_pipe_file)
> autogen_rt,errorcode = self.StartAutoGen(mqueue,
> AutoGenObject.DataPipe, self.SkipAutoGen, PcdMaList,
> GlobalData.gCacheIR)
> + AutoGenIdFile = os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt")
> + with open(AutoGenIdFile,"w") as fw:
> + fw.write("Arch=%s\n" % "|".join((AutoGenObject.Workspace.ArchList)))
> + fw.write("BuildDir=%s\n" % AutoGenObject.Workspace.BuildDir)
> + fw.write("PlatformGuid=%s\n" %
> + str(AutoGenObject.Guid))
> self.Progress.Stop("done!")
> if not autogen_rt:
> self.AutoGenMgr.TerminateWorkers()
> - self.AutoGenMgr.join(0.1)
> + self.AutoGenMgr.join(1)
> raise FatalError(errorcode)
> AutoGenObject.CreateCodeFile(False)
> AutoGenObject.CreateMakeFile(False)
> else:
> # always recreate top/platform makefile when clean, just
> in case of inconsistency @@ -1778,10 +1791,11 @@ class Build():
> self._CollectFvMapBuffer(MapBuffer, Wa, ModuleList)
> #
> # Save MAP buffer into MAP file.
> #
> self._SaveMapFile (MapBuffer, Wa)
> + self.CreateGuidedSectionToolsFile(Wa)
>
> ## Build active module for different build targets, different tool chains and different archs
> #
> def _BuildModule(self):
> for BuildTarget in self.BuildTargetList:
> @@ -1852,13 +1866,13 @@ class Build():
>
> if GlobalData.gBinCacheSource and self.Target in [None, "", "all"]:
> Ma.GenModuleFilesHash(GlobalData.gCacheIR)
> Ma.GenPreMakefileHash(GlobalData.gCacheIR)
> if Ma.CanSkipbyPreMakefileCache(GlobalData.gCacheIR):
> - self.HashSkipModules.append(Ma)
> - EdkLogger.quiet("cache hit: %s[%s]" % (Ma.MetaFile.Path, Ma.Arch))
> - continue
> + self.HashSkipModules.append(Ma)
> + EdkLogger.quiet("cache hit: %s[%s]" % (Ma.MetaFile.Path, Ma.Arch))
> + continue
>
> # Not to auto-gen for targets 'clean', 'cleanlib', 'cleanall', 'run', 'fds'
> if self.Target not in ['clean', 'cleanlib', 'cleanall', 'run', 'fds']:
> # for target which must generate AutoGen code and makefile
> if not self.SkipAutoGen or self.Target == 'genc':
> @@ -1987,125 +2001,221 @@ class Build():
> GenFfsDict = GenFds.GenFfsMakefile('', GlobalData.gFdfParser, self, ArchList, GlobalData)
> for Cmd in GenFfsDict:
> tmpInf, tmpArch = GenFfsDict[Cmd]
> CmdSetDict[tmpInf, tmpArch].add(Cmd)
> return CmdSetDict
> + def VerifyAutoGenFiles(self):
> + AutoGenIdFile = os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt")
> + try:
> + with open(AutoGenIdFile) as fd:
> + lines = fd.readlines()
> + except:
> + return None
> + for line in lines:
> + if "Arch" in line:
> + ArchList = line.strip().split("=")[1].split("|")
> + if "BuildDir" in line:
> + BuildDir = line.split("=")[1].strip()
> + if "PlatformGuid" in line:
> + PlatformGuid = line.split("=")[1].strip()
> + GlobalVarList = []
> + for arch in ArchList:
> + global_var = os.path.join(BuildDir, "GlobalVar_%s_%s.bin" % (str(PlatformGuid),arch))
> + if not os.path.exists(global_var):
> + return None
> + GlobalVarList.append(global_var)
> + for global_var in GlobalVarList:
> + data_pipe = MemoryDataPipe()
> + data_pipe.load(global_var)
> + target = data_pipe.Get("P_Info").get("Target")
> + toolchain = data_pipe.Get("P_Info").get("ToolChain")
> + archlist = data_pipe.Get("P_Info").get("ArchList")
> + Arch = data_pipe.Get("P_Info").get("Arch")
> + active_p = data_pipe.Get("P_Info").get("ActivePlatform")
> + workspacedir = data_pipe.Get("P_Info").get("WorkspaceDir")
> + PackagesPath = os.getenv("PACKAGES_PATH")
> + mws.setWs(workspacedir, PackagesPath)
> + LibraryBuildDirectoryList = data_pipe.Get("LibraryBuildDirectoryList")
> + ModuleBuildDirectoryList =
> + data_pipe.Get("ModuleBuildDirectoryList")
> +
> + for m_build_dir in LibraryBuildDirectoryList:
> + if not os.path.exists(os.path.join(m_build_dir,GenMake.BuildFile._FILE_NAME_[GenMake.gMakeType])):
> + return None
> + for m_build_dir in ModuleBuildDirectoryList:
> + if not os.path.exists(os.path.join(m_build_dir,GenMake.BuildFile._FILE_NAME_[GenMake.gMakeType])):
> + return None
> + Wa = WorkSpaceInfo(
> + workspacedir,active_p,target,toolchain,archlist
> + )
> + Pa = PlatformInfo(Wa, active_p, target, toolchain, Arch,data_pipe)
> + Wa.AutoGenObjectList.append(Pa)
> + return Wa
> + def SetupMakeSetting(self,Wa):
> + BuildModules = []
> + for Pa in Wa.AutoGenObjectList:
> + for m in Pa._MbList:
> + ma = ModuleAutoGen(Wa,m.MetaFile, Pa.BuildTarget, Wa.ToolChain, Pa.Arch, Pa.MetaFile,Pa.DataPipe)
> + BuildModules.append(ma)
> + fdf_file = Wa.FlashDefinition
> + if fdf_file:
> + Fdf = FdfParser(fdf_file.Path)
> + Fdf.ParseFile()
> + GlobalData.gFdfParser = Fdf
> + if Fdf.CurrentFdName and Fdf.CurrentFdName in Fdf.Profile.FdDict:
> + FdDict = Fdf.Profile.FdDict[Fdf.CurrentFdName]
> + for FdRegion in FdDict.RegionList:
> + if str(FdRegion.RegionType) is 'FILE' and self.Platform.VpdToolGuid in str(FdRegion.RegionDataList):
> + if int(FdRegion.Offset) % 8 != 0:
> + EdkLogger.error("build", FORMAT_INVALID,
> + 'The VPD Base Address %s must be 8-byte aligned.' %
> (FdRegion.Offset))
> + Wa.FdfProfile = Fdf.Profile
> + self.Fdf = Fdf
> + else:
> + self.Fdf = None
> + return BuildModules
>
> ## Build a platform in multi-thread mode
> #
> - def _MultiThreadBuildPlatform(self):
> - SaveFileOnChange(self.PlatformBuildPath, '# DO NOT EDIT \n# FILE auto-generated\n', False)
> - for BuildTarget in self.BuildTargetList:
> - GlobalData.gGlobalDefines['TARGET'] = BuildTarget
> - index = 0
> - for ToolChain in self.ToolChainList:
> - WorkspaceAutoGenTime = time.time()
> - GlobalData.gGlobalDefines['TOOLCHAIN'] = ToolChain
> - GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = ToolChain
> - GlobalData.gGlobalDefines['FAMILY'] = self.ToolChainFamily[index]
> - index += 1
> - Wa = WorkspaceAutoGen(
> - self.WorkspaceDir,
> - self.PlatformFile,
> - BuildTarget,
> - ToolChain,
> - self.ArchList,
> - self.BuildDatabase,
> - self.TargetTxt,
> - self.ToolDef,
> - self.Fdf,
> - self.FdList,
> - self.FvList,
> - self.CapList,
> - self.SkuId,
> - self.UniFlag,
> - self.Progress
> - )
> - self.Fdf = Wa.FdfFile
> - self.LoadFixAddress = Wa.Platform.LoadFixAddress
> - self.BuildReport.AddPlatformReport(Wa)
> - Wa.CreateMakeFile(False)
> + def PerformAutoGen(self,BuildTarget,ToolChain):
> + WorkspaceAutoGenTime = time.time()
> + Wa = WorkspaceAutoGen(
> + self.WorkspaceDir,
> + self.PlatformFile,
> + BuildTarget,
> + ToolChain,
> + self.ArchList,
> + self.BuildDatabase,
> + self.TargetTxt,
> + self.ToolDef,
> + self.Fdf,
> + self.FdList,
> + self.FvList,
> + self.CapList,
> + self.SkuId,
> + self.UniFlag,
> + self.Progress
> + )
> + self.Fdf = Wa.FdfFile
> + self.LoadFixAddress = Wa.Platform.LoadFixAddress
> + self.BuildReport.AddPlatformReport(Wa)
> + Wa.CreateMakeFile(False)
>
> # Add ffs build to makefile
> - CmdListDict = {}
> - if GlobalData.gEnableGenfdsMultiThread and self.Fdf:
> - CmdListDict = self._GenFfsCmd(Wa.ArchList)
> + CmdListDict = {}
> + if GlobalData.gEnableGenfdsMultiThread and self.Fdf:
> + CmdListDict = self._GenFfsCmd(Wa.ArchList)
>
> - # Add Platform and Package level hash in share_data for module hash calculation later
> - if GlobalData.gBinCacheSource or GlobalData.gBinCacheDest:
> - GlobalData.gCacheIR[('PlatformHash')] = GlobalData.gPlatformHash
> - for PkgName in GlobalData.gPackageHash.keys():
> - GlobalData.gCacheIR[(PkgName, 'PackageHash')] = GlobalData.gPackageHash[PkgName]
> + # Add Platform and Package level hash in share_data for module hash calculation later
> + if GlobalData.gBinCacheSource or GlobalData.gBinCacheDest:
> + GlobalData.gCacheIR[('PlatformHash')] = GlobalData.gPlatformHash
> + for PkgName in GlobalData.gPackageHash.keys():
> + GlobalData.gCacheIR[(PkgName, 'PackageHash')] =
> + GlobalData.gPackageHash[PkgName]
>
> - # multi-thread exit flag
> - ExitFlag = threading.Event()
> - ExitFlag.clear()
> - self.AutoGenTime += int(round((time.time() - WorkspaceAutoGenTime)))
> - self.BuildModules = []
> - TotalModules = []
> - for Arch in Wa.ArchList:
> - PcdMaList = []
> - AutoGenStart = time.time()
> - GlobalData.gGlobalDefines['ARCH'] = Arch
> - Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, ToolChain, Arch)
> - if Pa is None:
> - continue
> - ModuleList = []
> - for Inf in Pa.Platform.Modules:
> - ModuleList.append(Inf)
> + self.AutoGenTime += int(round((time.time() - WorkspaceAutoGenTime)))
> + BuildModules = []
> + TotalModules = []
> + for Arch in Wa.ArchList:
> + PcdMaList = []
> + AutoGenStart = time.time()
> + GlobalData.gGlobalDefines['ARCH'] = Arch
> + Pa = PlatformAutoGen(Wa, self.PlatformFile, BuildTarget, ToolChain, Arch)
> + if Pa is None:
> + continue
> + ModuleList = []
> + for Inf in Pa.Platform.Modules:
> + ModuleList.append(Inf)
> # Add the INF only list in FDF
> - if GlobalData.gFdfParser is not None:
> - for InfName in GlobalData.gFdfParser.Profile.InfList:
> - Inf = PathClass(NormPath(InfName), self.WorkspaceDir, Arch)
> - if Inf in Pa.Platform.Modules:
> - continue
> - ModuleList.append(Inf)
> - Pa.DataPipe.DataContainer = {"FfsCommand":CmdListDict}
> - Pa.DataPipe.DataContainer = {"Workspace_timestamp": Wa._SrcTimeStamp}
> - Pa.DataPipe.DataContainer = {"CommandTarget": self.Target}
> - for Module in ModuleList:
> + if GlobalData.gFdfParser is not None:
> + for InfName in GlobalData.gFdfParser.Profile.InfList:
> + Inf = PathClass(NormPath(InfName), self.WorkspaceDir, Arch)
> + if Inf in Pa.Platform.Modules:
> + continue
> + ModuleList.append(Inf)
> + Pa.DataPipe.DataContainer = {"FfsCommand":CmdListDict}
> + Pa.DataPipe.DataContainer = {"Workspace_timestamp": Wa._SrcTimeStamp}
> + Pa.DataPipe.DataContainer = {"CommandTarget": self.Target}
> + Pa.CreateLibModuelDirs()
> + Pa.DataPipe.DataContainer = {"LibraryBuildDirectoryList":Pa.LibraryBuildDirectoryList}
> + Pa.DataPipe.DataContainer = {"ModuleBuildDirectoryList":Pa.ModuleBuildDirectoryList}
> + Pa.DataPipe.DataContainer = {"FdsCommandDict": Wa.GenFdsCommandDict}
> + ModuleCodaFile = {}
> + for ma in Pa.ModuleAutoGenList:
> +
> + ModuleCodaFile[(ma.MetaFile.File,ma.MetaFile.Root,ma.Arch,ma.MetaFil
> + e.Path)] = [item.Target for item in
> ma.CodaTargetList]
> + Pa.DataPipe.DataContainer = {"ModuleCodaFile":ModuleCodaFile}
> + for Module in ModuleList:
> # Get ModuleAutoGen object to generate C code file and makefile
> - Ma = ModuleAutoGen(Wa, Module, BuildTarget, ToolChain, Arch, self.PlatformFile,Pa.DataPipe)
> + Ma = ModuleAutoGen(Wa, Module, BuildTarget,
> + ToolChain, Arch, self.PlatformFile,Pa.DataPipe)
>
> - if Ma is None:
> + if Ma is None:
> + continue
> + if Ma.PcdIsDriver:
> + Ma.PlatformInfo = Pa
> + Ma.Workspace = Wa
> + PcdMaList.append(Ma)
> + TotalModules.append(Ma)
> + # Initialize all modules in tracking to 'FAIL'
> + GlobalData.gModuleBuildTracking[Ma] = 'FAIL'
> +
> +
> + mqueue = mp.Queue()
> + for m in Pa.GetAllModuleInfo:
> + mqueue.put(m)
> + data_pipe_file = os.path.join(Pa.BuildDir, "GlobalVar_%s_%s.bin" % (str(Pa.Guid),Pa.Arch))
> + Pa.DataPipe.dump(data_pipe_file)
> +
> + autogen_rt, errorcode = self.StartAutoGen(mqueue,
> + Pa.DataPipe, self.SkipAutoGen, PcdMaList,GlobalData.gCacheIR)
> +
> + # Skip cache hit modules
> + if GlobalData.gBinCacheSource:
> + for Ma in TotalModules:
> + if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \
> + GlobalData.gCacheIR[(Ma.MetaFile.Path, Ma.Arch)].PreMakeCacheHit:
> + self.HashSkipModules.append(Ma)
> + continue
> + if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \
> + GlobalData.gCacheIR[(Ma.MetaFile.Path, Ma.Arch)].MakeCacheHit:
> + self.HashSkipModules.append(Ma)
> continue
> - if Ma.PcdIsDriver:
> - Ma.PlatformInfo = Pa
> - Ma.Workspace = Wa
> - PcdMaList.append(Ma)
> - TotalModules.append(Ma)
> - # Initialize all modules in tracking to 'FAIL'
> - GlobalData.gModuleBuildTracking[Ma] = 'FAIL'
> + BuildModules.append(Ma)
> + else:
> + BuildModules.extend(TotalModules)
>
> - mqueue = mp.Queue()
> - for m in Pa.GetAllModuleInfo:
> - mqueue.put(m)
> - data_pipe_file = os.path.join(Pa.BuildDir, "GlobalVar_%s_%s.bin" % (str(Pa.Guid),Pa.Arch))
> - Pa.DataPipe.dump(data_pipe_file)
> - autogen_rt, errorcode = self.StartAutoGen(mqueue, Pa.DataPipe, self.SkipAutoGen, PcdMaList,
> GlobalData.gCacheIR)
> + if not autogen_rt:
> + self.AutoGenMgr.TerminateWorkers()
> + self.AutoGenMgr.join(1)
> + raise FatalError(errorcode)
> + self.AutoGenTime += int(round((time.time() - AutoGenStart)))
> + AutoGenIdFile = os.path.join(GlobalData.gConfDirectory,".AutoGenIdFile.txt")
> + with open(AutoGenIdFile,"w") as fw:
> + fw.write("Arch=%s\n" % "|".join((Wa.ArchList)))
> + fw.write("BuildDir=%s\n" % Wa.BuildDir)
> + fw.write("PlatformGuid=%s\n" % str(Wa.AutoGenObjectList[0].Guid))
> + self.Progress.Stop("done!")
> + return Wa, BuildModules
>
> - # Skip cache hit modules
> - if GlobalData.gBinCacheSource:
> - for Ma in TotalModules:
> - if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \
> - GlobalData.gCacheIR[(Ma.MetaFile.Path, Ma.Arch)].PreMakeCacheHit:
> - self.HashSkipModules.append(Ma)
> - continue
> - if (Ma.MetaFile.Path, Ma.Arch) in GlobalData.gCacheIR and \
> - GlobalData.gCacheIR[(Ma.MetaFile.Path, Ma.Arch)].MakeCacheHit:
> - self.HashSkipModules.append(Ma)
> - continue
> - self.BuildModules.append(Ma)
> + def _MultiThreadBuildPlatform(self):
> + SaveFileOnChange(self.PlatformBuildPath, '# DO NOT EDIT \n# FILE auto-generated\n', False)
> + for BuildTarget in self.BuildTargetList:
> + GlobalData.gGlobalDefines['TARGET'] = BuildTarget
> + index = 0
> + for ToolChain in self.ToolChainList:
> + GlobalData.gGlobalDefines['TOOLCHAIN'] = ToolChain
> + GlobalData.gGlobalDefines['TOOL_CHAIN_TAG'] = ToolChain
> + GlobalData.gGlobalDefines['FAMILY'] = self.ToolChainFamily[index]
> + index += 1
> + ExitFlag = threading.Event()
> + ExitFlag.clear()
> + if self.SkipAutoGen:
> + Wa = self.VerifyAutoGenFiles()
> + if Wa is None:
> + Wa, self.BuildModules =
> + self.PerformAutoGen(BuildTarget,ToolChain)
> else:
> - self.BuildModules.extend(TotalModules)
> -
> - if not autogen_rt:
> - self.AutoGenMgr.TerminateWorkers()
> - self.AutoGenMgr.join(0.1)
> - raise FatalError(errorcode)
> - self.AutoGenTime += int(round((time.time() - AutoGenStart)))
> - self.Progress.Stop("done!")
> + GlobalData.gAutoGenPhase = True
> + self.BuildModules = self.SetupMakeSetting(Wa)
> + else:
> + Wa, self.BuildModules = self.PerformAutoGen(BuildTarget,ToolChain)
> + Pa = Wa.AutoGenObjectList[0]
> + GlobalData.gAutoGenPhase = False
>
> if GlobalData.gBinCacheSource:
> EdkLogger.quiet("Total cache hit driver num: %s,
> cache miss driver num: %s" % (len(set(self.HashSkipModules)),
> len(set(self.BuildModules))))
> CacheHitMa = set()
> CacheNotHitMa = set() @@ -2136,18 +2246,17 @@
> class Build():
> # Start task scheduler
> if not BuildTask.IsOnGoing():
>
> BuildTask.StartScheduler(self.ThreadNumber, ExitFlag)
>
> # in case there's an interruption. we need a full version of makefile for platform
> - Pa.CreateMakeFile(False)
> +
> if BuildTask.HasError():
> self.invalidateHash()
> EdkLogger.error("build", BUILD_ERROR, "Failed to build module", ExtraData=GlobalData.gBuildingModule)
> self.MakeTime += int(round((time.time() -
> MakeStart)))
>
> MakeContiue = time.time()
> -
> #
> #
> # All modules have been put in build tasks queue. Tell task scheduler
> # to exit if all tasks are completed
> #
> @@ -2175,17 +2284,12 @@ class Build():
> if (Arch == 'IA32' or Arch == 'ARM') and
> self.LoadFixAddress != 0xFFFFFFFFFFFFFFFF and self.LoadFixAddress >= 0x100000000:
> EdkLogger.error("build",
> PARAMETER_INVALID, "FIX_LOAD_TOP_MEMORY_ADDRESS can't be set to larger than or equal to 4G for the platorm with IA32 or ARM arch modules")
> #
> # Get Module List
> #
> - ModuleList = {}
> - for Pa in Wa.AutoGenObjectList:
> - for Ma in Pa.ModuleAutoGenList:
> - if Ma is None:
> - continue
> - if not Ma.IsLibrary:
> - ModuleList[Ma.Guid.upper()] = Ma
> + ModuleList = {ma.Guid.upper():ma for ma in
> + self.BuildModules}
> +
> #
> # Rebase module to the preferred memory address before GenFds
> #
> MapBuffer = []
> if self.LoadFixAddress != 0:
> @@ -2206,33 +2310,17 @@ class Build():
> self.GenFdsTime += int(round((time.time() - GenFdsStart)))
> #
> # Save MAP buffer into MAP file.
> #
> self._SaveMapFile(MapBuffer, Wa)
> + self.CreateGuidedSectionToolsFile(Wa)
> self.invalidateHash()
> -
> ## Generate GuidedSectionTools.txt in the FV directories.
> #
> - def CreateGuidedSectionToolsFile(self):
> + def CreateGuidedSectionToolsFile(self,Wa):
> for BuildTarget in self.BuildTargetList:
> for ToolChain in self.ToolChainList:
> - Wa = WorkspaceAutoGen(
> - self.WorkspaceDir,
> - self.PlatformFile,
> - BuildTarget,
> - ToolChain,
> - self.ArchList,
> - self.BuildDatabase,
> - self.TargetTxt,
> - self.ToolDef,
> - self.Fdf,
> - self.FdList,
> - self.FvList,
> - self.CapList,
> - self.SkuId,
> - self.UniFlag
> - )
> FvDir = Wa.FvDir
> if not os.path.exists(FvDir):
> continue
>
> for Arch in self.ArchList:
> @@ -2285,11 +2373,10 @@ class Build():
> if not self.SpawnMode or self.Target not in ["", "all"]:
> self.SpawnMode = False
> self._BuildPlatform()
> else:
> self._MultiThreadBuildPlatform()
> - self.CreateGuidedSectionToolsFile()
> else:
> self.SpawnMode = False
> self._BuildModule()
>
> if self.Target == 'cleanall':
> --
> 2.20.1.windows.1
>
>
>
-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.
View/Reply Online (#46900): https://edk2.groups.io/g/devel/message/46900
Mute This Topic: https://groups.io/mt/33151466/1813853
Group Owner: devel+owner at edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub [edk2-devel-archive at redhat.com]
-=-=-=-=-=-=-=-=-=-=-=-
More information about the edk2-devel-archive
mailing list