[edk2-devel] [edk2-staging/EdkRepo] [PATCH v2 2/2] EdkRepo: Update commands to use new submodule code
Ashley E Desimone
ashley.e.desimone at intel.com
Thu May 28 02:17:09 UTC 2020
Pushed: ddb92ec59bfb872a8f69442f99faf64619e2f751
-----Original Message-----
From: Bjorge, Erik C <erik.c.bjorge at intel.com>
Sent: Tuesday, May 19, 2020 2:51 PM
To: devel at edk2.groups.io
Cc: Desimone, Ashley E <ashley.e.desimone at intel.com>; Desimone, Nathaniel L <nathaniel.l.desimone at intel.com>; Pandya, Puja <puja.pandya at intel.com>; Bret Barkelew <Bret.Barkelew at microsoft.com>; Agyeman, Prince <prince.agyeman at intel.com>
Subject: [edk2-staging/EdkRepo] [PATCH v2 2/2] EdkRepo: Update commands to use new submodule code
Replaced use of maintain_submodules to use new common code. This allows for selective submodule initialization via the manifest file.
Cc: Ashley E Desimone <ashley.e.desimone at intel.com>
Cc: Nate DeSimone <nathaniel.l.desimone at intel.com>
Cc: Puja Pandya <puja.pandya at intel.com>
Cc: Bret Barkelew <Bret.Barkelew at microsoft.com>
Cc: Prince Agyeman <prince.agyeman at intel.com>
Cc: Erik Bjorge <erik.c.bjorge at intel.com>
Signed-off-by: Erik Bjorge <erik.c.bjorge at intel.com>
---
edkrepo/commands/checkout_pin_command.py | 6 ++++
edkrepo/commands/clone_command.py | 20 +++++++----
edkrepo/commands/sync_command.py | 25 +++++++++-----
edkrepo/common/common_repo_functions.py | 43 +++++-------------------
4 files changed, 45 insertions(+), 49 deletions(-)
diff --git a/edkrepo/commands/checkout_pin_command.py b/edkrepo/commands/checkout_pin_command.py
index 4aaf2b5..9d7346a 100644
--- a/edkrepo/commands/checkout_pin_command.py
+++ b/edkrepo/commands/checkout_pin_command.py
@@ -22,6 +22,8 @@ from edkrepo.common.workspace_maintenance.manifest_repos_maintenance import list from edkrepo.common.workspace_maintenance.manifest_repos_maintenance import find_source_manifest_repo from edkrepo.config.config_factory import get_workspace_path, get_workspace_manifest from edkrepo_manifest_parser.edk_manifest import ManifestXml
+from project_utils.submodule import deinit_submodules,
+maintain_submodules
+
class CheckoutPinCommand(EdkrepoCommand):
def __init__(self):
@@ -68,11 +70,15 @@ class CheckoutPinCommand(EdkrepoCommand):
if sparse_enabled:
print(SPARSE_RESET)
reset_sparse_checkout(workspace_path, manifest_sources)
+ submodule_combo = pin.general_config.current_combo
+ deinit_submodules(workspace_path, manifest, manifest.general_config.current_combo,
+ pin, submodule_combo, args.verbose)
pin_repo_sources = pin.get_repo_sources(pin.general_config.current_combo)
try:
checkout_repos(args.verbose, args.override, pin_repo_sources, workspace_path, manifest)
manifest.write_current_combo(humble.PIN_COMBO.format(args.pinfile))
finally:
+ maintain_submodules(workspace_path, pin, submodule_combo,
+ args.verbose)
if sparse_enabled:
print(SPARSE_CHECKOUT)
sparse_checkout(workspace_path, pin_repo_sources, manifest) diff --git a/edkrepo/commands/clone_command.py b/edkrepo/commands/clone_command.py
index 3f2e6e1..f638090 100644
--- a/edkrepo/commands/clone_command.py
+++ b/edkrepo/commands/clone_command.py
@@ -25,6 +25,7 @@ from edkrepo.common.workspace_maintenance.manifest_repos_maintenance import pull from edkrepo.common.workspace_maintenance.manifest_repos_maintenance import list_available_manifest_repos from edkrepo.common.workspace_maintenance.humble.manifest_repos_maintenance_humble import PROJ_NOT_IN_REPO, SOURCE_MANIFEST_REPO_NOT_FOUND from edkrepo_manifest_parser.edk_manifest import CiIndexXml, ManifestXml
+from project_utils.submodule import maintain_submodules
class CloneCommand(EdkrepoCommand):
@@ -108,7 +109,6 @@ class CloneCommand(EdkrepoCommand):
manifest = ManifestXml(local_manifest_path)
# Process the combination name and make sure it can be found in the manifest
- combo_name = None
if args.Combination is not None:
try:
combo_name = case_insensitive_single_match(args.Combination, combinations_in_manifest(manifest))
@@ -122,13 +122,15 @@ class CloneCommand(EdkrepoCommand):
# Since pin files are subset of manifest files they do not have a "default combo" it is set to None. In this
# case use the current_combo instead.
combo_name = manifest.general_config.current_combo
+ else:
+ # If a combo was not specified or a pin file used the default combo should be cloned. Also ensure that the
+ # current combo is updated to match.
+ combo_name = manifest.general_config.default_combo
+ manifest.write_current_combo(combo_name)
# Get the list of repos to clone and clone them
- manifest_config = manifest.general_config
- if combo_name:
- repo_sources_to_clone = manifest.get_repo_sources(combo_name)
- else:
- repo_sources_to_clone = manifest.get_repo_sources(manifest_config.default_combo)
+ repo_sources_to_clone = manifest.get_repo_sources(combo_name)
+
#check that the repo sources do not contain duplicated local roots
local_roots = [r.root for r in repo_sources_to_clone]
for root in local_roots:
@@ -141,7 +143,11 @@ class CloneCommand(EdkrepoCommand):
# Set up submodule alt url config settings prior to cloning any repos
submodule_included_configs = write_included_config(manifest.remotes, manifest.submodule_alternate_remotes, local_manifest_dir)
write_conditional_include(workspace_dir, repo_sources_to_clone, submodule_included_configs)
- clone_repos(args, workspace_dir, repo_sources_to_clone, project_client_side_hooks, config, args.skip_submodule, manifest)
+ clone_repos(args, workspace_dir, repo_sources_to_clone,
+ project_client_side_hooks, config, manifest)
+
+ # Init submodules
+ if not args.skip_submodule:
+ maintain_submodules(workspace_dir, manifest, combo_name,
+ args.verbose)
# Perform a sparse checkout if requested.
use_sparse = args.sparse
diff --git a/edkrepo/commands/sync_command.py b/edkrepo/commands/sync_command.py
index b662c62..13b46e8 100644
--- a/edkrepo/commands/sync_command.py
+++ b/edkrepo/commands/sync_command.py
@@ -38,7 +38,7 @@ from edkrepo.common.common_repo_functions import checkout_repos, check_dirty_rep from edkrepo.common.common_repo_functions import update_editor_config from edkrepo.common.common_repo_functions import update_repo_commit_template, get_latest_sha from edkrepo.common.common_repo_functions import has_primary_repo_remote, fetch_from_primary_repo, in_sync_with_primary -from edkrepo.common.common_repo_functions import update_hooks, maintain_submodules, combinations_in_manifest
+from edkrepo.common.common_repo_functions import update_hooks,
+combinations_in_manifest
from edkrepo.common.common_repo_functions import write_included_config, remove_included_config from edkrepo.common.workspace_maintenance.workspace_maintenance import generate_name_for_obsolete_backup from edkrepo.common.workspace_maintenance.manifest_repos_maintenance import pull_workspace_manifest_repo @@ -48,6 +48,7 @@ from edkrepo.common.ui_functions import init_color_console from edkrepo.config.config_factory import get_workspace_path, get_workspace_manifest, get_edkrepo_global_data_directory from edkrepo.config.config_factory import get_workspace_manifest_file from edkrepo_manifest_parser.edk_manifest import CiIndexXml, ManifestXml
+from project_utils.submodule import deinit_submodules,
+maintain_submodules
class SyncCommand(EdkrepoCommand):
@@ -84,6 +85,7 @@ class SyncCommand(EdkrepoCommand):
current_combo = initial_manifest.general_config.current_combo
initial_sources = initial_manifest.get_repo_sources(current_combo)
initial_hooks = initial_manifest.repo_hooks
+ initial_combo = current_combo
source_global_manifest_repo = find_source_manifest_repo(initial_manifest, config['cfg_file'], config['user_cfg_file'], args.source_manifest_repo)
pull_workspace_manifest_repo(initial_manifest, config['cfg_file'], config['user_cfg_file'], args.source_manifest_repo, False) @@ -100,6 +102,7 @@ class SyncCommand(EdkrepoCommand):
if not args.update_local_manifest:
self.__check_for_new_manifest(args, config, initial_manifest, workspace_path, global_manifest_directory)
check_dirty_repos(initial_manifest, workspace_path)
+
# Determine if sparse checkout needs to be disabled for this operation
sparse_settings = initial_manifest.sparse_settings
sparse_enabled = sparse_checkout_enabled(workspace_path, initial_sources) @@ -113,7 +116,7 @@ class SyncCommand(EdkrepoCommand):
reset_sparse_checkout(workspace_path, initial_sources)
# Get the latest manifest if requested
- if args.update_local_manifest: #NOTE: hyphens in arg name replaced with underscores due to argparse
+ if args.update_local_manifest: # NOTE: hyphens in arg name
+ replaced with underscores due to argparse
self.__update_local_manifest(args, config, initial_manifest, workspace_path, global_manifest_directory)
manifest = get_workspace_manifest()
if args.update_local_manifest:
@@ -127,6 +130,12 @@ class SyncCommand(EdkrepoCommand):
repo_sources_to_sync = manifest.get_repo_sources(current_combo)
manifest.write_current_combo(current_combo)
+ # At this point both new and old manifest files are ready so we can deinit any
+ # submodules that are removed due to a manifest update.
+ if not args.skip_submodule:
+ deinit_submodules(workspace_path, initial_manifest, initial_combo,
+ manifest, current_combo, args.verbose)
+
sync_error = False
# Calculate the hooks which need to be updated, added or removed for the sync
if args.update_local_manifest:
@@ -195,16 +204,16 @@ class SyncCommand(EdkrepoCommand):
elif args.verbose:
print(NO_SYNC_DETACHED_HEAD.format(repo_to_sync.root))
- if not args.skip_submodule:
- if repo_to_sync.enable_submodule:
- # Perform submodule updates and url redirection
- maintain_submodules(repo_to_sync, repo)
# Update commit message templates
- update_repo_commit_template(workspace_path, repo, repo_to_sync, config, global_manifest_directory)
+ update_repo_commit_template(workspace_path, repo,
+ repo_to_sync, config, global_manifest_directory)
if sync_error:
print(SYNC_ERROR)
+ # Initialize submodules
+ if not args.skip_submodule:
+ maintain_submodules(workspace_path, manifest,
+ current_combo, args.verbose)
+
# Restore sparse checkout state
if sparse_enabled:
print(SPARSE_CHECKOUT)
@@ -326,7 +335,7 @@ class SyncCommand(EdkrepoCommand):
print(path_to_source)
if len(sources_to_remove) > 0:
print(SYNC_REMOVE_LIST_END_FORMATTING)
- clone_repos(args, workspace_path, sources_to_clone, new_manifest_to_check.repo_hooks, config, args.skip_submodule, new_manifest_to_check)
+ clone_repos(args, workspace_path, sources_to_clone,
+ new_manifest_to_check.repo_hooks, config, new_manifest_to_check)
# Make a list of and only checkout repos that were newly cloned. Sync keeps repos on their initial active branches
# cloning the entire combo can prevent existing repos from correctly being returned to their proper branch
repos_to_checkout = []
diff --git a/edkrepo/common/common_repo_functions.py b/edkrepo/common/common_repo_functions.py
index 61133f1..8c79f3d 100644
--- a/edkrepo/common/common_repo_functions.py
+++ b/edkrepo/common/common_repo_functions.py
@@ -68,12 +68,13 @@ from edkrepo_manifest_parser.edk_manifest_validation import validate_manifestrep from edkrepo_manifest_parser.edk_manifest_validation import get_manifest_validation_status from edkrepo_manifest_parser.edk_manifest_validation import print_manifest_errors from edkrepo_manifest_parser.edk_manifest_validation import validate_manifestfiles
+from project_utils.submodule import deinit_submodules,
+maintain_submodules
CLEAR_LINE = '\x1b[K'
DEFAULT_REMOTE_NAME = 'origin'
PRIMARY_REMOTE_NAME = 'primary'
-def clone_repos(args, workspace_dir, repos_to_clone, project_client_side_hooks, config, skip_submodule, manifest):
+def clone_repos(args, workspace_dir, repos_to_clone, project_client_side_hooks, config, manifest):
for repo_to_clone in repos_to_clone:
local_repo_path = os.path.join(workspace_dir, repo_to_clone.root)
local_repo_url = repo_to_clone.remote_url @@ -108,10 +109,6 @@ def clone_repos(args, workspace_dir, repos_to_clone, project_client_side_hooks,
else:
raise EdkrepoManifestInvalidException(MISSING_BRANCH_COMMIT)
- if not skip_submodule:
- if repo_to_clone.enable_submodule:
- maintain_submodules(repo_to_clone, repo, args.verbose)
-
try:
if 'source_manifest_repo' in vars(args).keys():
src_manifest_repo = find_source_manifest_repo(manifest, config['cfg_file'], config['user_cfg_file'], args.source_manifest_repo) @@ -184,32 +181,6 @@ def write_conditional_include(workspace_path, repo_sources, included_configs):
gitglobalconfig.add_section(section)
gitglobalconfig.set(section, 'path', path)
-def maintain_submodules(repo_sources, repo, verbose = False):
- try:
- output_data = repo.git.execute(['git', 'submodule', 'init'], with_extended_output=True, with_stdout=True)
- if verbose and output_data[0]:
- print(output_data[0])
- if output_data[1]:
- print(output_data[1])
- if verbose and output_data[2]:
- print(output_data[2])
- output_data = repo.git.execute(['git', 'submodule', 'sync', '--recursive'], with_extended_output=True, with_stdout=True)
- if verbose and output_data[0]:
- print(output_data[0])
- if output_data[1]:
- print(output_data[1])
- if verbose and output_data[2]:
- print(output_data[2])
- output_data = repo.git.execute(['git', 'submodule', 'update', '--recursive'], with_extended_output=True, with_stdout=True)
- if verbose and output_data[0]:
- print(output_data[0])
- if output_data[1]:
- print(output_data[1])
- if verbose and output_data[2]:
- print(output_data[2])
- except:
- raise EdkrepoGitException(SUBMODULE_FAILURE.format(repo_sources.remote_name))
-
def install_hooks(hooks, local_repo_path, repo_for_install, config, global_manifest_directory):
# Determine the which hooks are for the repo in question and which are from a URL based source or are in a global
# manifest repo relative path
@@ -399,9 +370,6 @@ def checkout_repos(verbose, override, repos_to_checkout, workspace_path, manifes
else:
raise EdkrepoManifestInvalidException(MISSING_BRANCH_COMMIT)
- if repo_to_checkout.enable_submodule:
- maintain_submodules(repo_to_checkout, repo, verbose)
-
def validate_manifest_repo(manifest_repo, verbose=False, archived=False):
print(VERIFY_GLOBAL)
if archived:
@@ -514,9 +482,11 @@ def checkout(combination_or_sha, verbose=False, override=False, log=None):
# Create combo_or_sha so we have original input and do not introduce any
# unintended behavior by messing with parameters.
combo_or_sha = combination_or_sha
+ submodule_combo = manifest.general_config.current_combo
try:
# Try to handle normalize combo name to match the manifest file.
combo_or_sha = case_insensitive_single_match(combo_or_sha, combinations_in_manifest(manifest))
+ submodule_combo = combo_or_sha
except:
# No match so leave it alone. It must be a SHA1 or a bad combo name.
pass
@@ -528,6 +498,10 @@ def checkout(combination_or_sha, verbose=False, override=False, log=None):
log=log)
initial_repo_sources = manifest.get_repo_sources(manifest.general_config.current_combo)
+ # Deinit any submodules that have been removed.
+ deinit_submodules(workspace_path, manifest, manifest.general_config.current_combo,
+ manifest, submodule_combo, verbose)
+
# Disable sparse checkout
current_repos = initial_repo_sources
sparse_enabled = sparse_checkout_enabled(workspace_path, initial_repo_sources) @@ -566,6 +540,7 @@ def checkout(combination_or_sha, verbose=False, override=False, log=None):
# Return to the initial combo, since there was an issue with cheking out the selected combo
checkout_repos(verbose, override, initial_repo_sources, workspace_path, manifest)
finally:
+ maintain_submodules(workspace_path, manifest, submodule_combo,
+ verbose)
if sparse_enabled or sparse_diff:
print(SPARSE_CHECKOUT)
sparse_checkout(workspace_path, current_repos, manifest)
--
2.21.0.windows.1
-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.
View/Reply Online (#60377): https://edk2.groups.io/g/devel/message/60377
Mute This Topic: https://groups.io/mt/74336547/1813853
Group Owner: devel+owner at edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub [edk2-devel-archive at redhat.com]
-=-=-=-=-=-=-=-=-=-=-=-
More information about the edk2-devel-archive
mailing list