[edk2-devel] [edk2-staging/EdkRepo] [PATCH v2 2/2] EdkRepo: Enable use of repo cache support.
Bjorge, Erik C
erik.c.bjorge at intel.com
Tue Nov 17 04:49:45 UTC 2020
This changes enables the local repo cache to be used when cloning
and syncing changes. The repo cache applies to submodules as well.
Cc: Ashley E Desimone <ashley.e.desimone at intel.com>
Cc: Nate DeSimone <nathaniel.l.desimone at intel.com>
Cc: Puja Pandya <puja.pandya at intel.com>
Cc: Bret Barkelew <Bret.Barkelew at microsoft.com>
Cc: Prince Agyeman <prince.agyeman at intel.com>
Cc: Erik Bjorge <erik.c.bjorge at intel.com>
Signed-off-by: Erik Bjorge <erik.c.bjorge at intel.com>
---
edkrepo/commands/checkout_command.py | 3 ++-
edkrepo/commands/checkout_pin_command.py | 8 +++++++-
edkrepo/commands/clone_command.py | 15 ++++++++++++--
edkrepo/commands/sync_command.py | 12 +++++++++---
edkrepo/common/common_repo_functions.py | 25 ++++++++++++++++++++----
project_utils/submodule.py | 13 ++++++++----
6 files changed, 61 insertions(+), 15 deletions(-)
diff --git a/edkrepo/commands/checkout_command.py b/edkrepo/commands/checkout_command.py
index 0169f30..2ce26c0 100644
--- a/edkrepo/commands/checkout_command.py
+++ b/edkrepo/commands/checkout_command.py
@@ -16,6 +16,7 @@ import os
from edkrepo.commands.edkrepo_command import EdkrepoCommand, OverrideArgument
import edkrepo.commands.arguments.checkout_args as arguments
import edkrepo.commands.humble.checkout_humble as humble
+from edkrepo.common.common_cache_functions import get_repo_cache_obj
from edkrepo.common.common_repo_functions import checkout, combination_is_in_manifest
from edkrepo.common.edkrepo_exception import EdkrepoInvalidParametersException
from edkrepo.config.config_factory import get_workspace_manifest
@@ -42,6 +43,6 @@ class CheckoutCommand(EdkrepoCommand):
def run_command(self, args, config):
if combination_is_in_manifest(args.Combination, get_workspace_manifest()):
- checkout(args.Combination, args.verbose, args.override)
+ checkout(args.Combination, args.verbose, args.override, get_repo_cache_obj(config))
else:
raise EdkrepoInvalidParametersException(humble.NO_COMBO.format(args.Combination))
diff --git a/edkrepo/commands/checkout_pin_command.py b/edkrepo/commands/checkout_pin_command.py
index 1c58113..0ad1b48 100644
--- a/edkrepo/commands/checkout_pin_command.py
+++ b/edkrepo/commands/checkout_pin_command.py
@@ -14,6 +14,7 @@ from git import Repo
from edkrepo.commands.edkrepo_command import EdkrepoCommand, OverrideArgument, SourceManifestRepoArgument
import edkrepo.commands.arguments.checkout_pin_args as arguments
import edkrepo.commands.humble.checkout_pin_humble as humble
+from edkrepo.common.common_cache_functions import get_repo_cache_obj
from edkrepo.common.common_repo_functions import sparse_checkout_enabled, reset_sparse_checkout, sparse_checkout
from edkrepo.common.common_repo_functions import check_dirty_repos, checkout_repos, combinations_in_manifest
from edkrepo.common.humble import SPARSE_CHECKOUT, SPARSE_RESET, SUBMODULE_DEINIT_FAILED
@@ -21,6 +22,7 @@ from edkrepo.common.edkrepo_exception import EdkrepoInvalidParametersException,
from edkrepo.common.workspace_maintenance.manifest_repos_maintenance import list_available_manifest_repos
from edkrepo.common.workspace_maintenance.manifest_repos_maintenance import find_source_manifest_repo
from edkrepo.config.config_factory import get_workspace_path, get_workspace_manifest
+from edkrepo.config.tool_config import SUBMODULE_CACHE_REPO_NAME
from edkrepo_manifest_parser.edk_manifest import ManifestXml
from project_utils.submodule import deinit_full, maintain_submodules
@@ -82,7 +84,11 @@ class CheckoutPinCommand(EdkrepoCommand):
checkout_repos(args.verbose, args.override, pin_repo_sources, workspace_path, manifest)
manifest.write_current_combo(humble.PIN_COMBO.format(args.pinfile))
finally:
- maintain_submodules(workspace_path, pin, submodule_combo, args.verbose)
+ cache_path = None
+ cache_obj = get_repo_cache_obj(config)
+ if cache_obj is not None:
+ cache_path = cache_obj.get_cache_path(SUBMODULE_CACHE_REPO_NAME)
+ maintain_submodules(workspace_path, pin, submodule_combo, args.verbose, cache_path)
if sparse_enabled:
print(SPARSE_CHECKOUT)
sparse_checkout(workspace_path, pin_repo_sources, manifest)
diff --git a/edkrepo/commands/clone_command.py b/edkrepo/commands/clone_command.py
index 8769102..56c15c9 100644
--- a/edkrepo/commands/clone_command.py
+++ b/edkrepo/commands/clone_command.py
@@ -14,6 +14,8 @@ import sys
from edkrepo.commands.edkrepo_command import EdkrepoCommand
from edkrepo.commands.edkrepo_command import SubmoduleSkipArgument, SourceManifestRepoArgument
import edkrepo.commands.arguments.clone_args as arguments
+from edkrepo.common.common_cache_functions import get_repo_cache_obj
+from edkrepo.common.common_cache_functions import add_missing_cache_repos
from edkrepo.common.common_repo_functions import clone_repos, sparse_checkout, verify_single_manifest
from edkrepo.common.common_repo_functions import update_editor_config, combinations_in_manifest
from edkrepo.common.common_repo_functions import write_included_config, write_conditional_include
@@ -28,6 +30,7 @@ from edkrepo.common.workspace_maintenance.manifest_repos_maintenance import list
from edkrepo.common.workspace_maintenance.humble.manifest_repos_maintenance_humble import PROJ_NOT_IN_REPO, SOURCE_MANIFEST_REPO_NOT_FOUND
from edkrepo_manifest_parser.edk_manifest import CiIndexXml, ManifestXml
from project_utils.submodule import maintain_submodules
+from edkrepo.config.tool_config import SUBMODULE_CACHE_REPO_NAME
class CloneCommand(EdkrepoCommand):
@@ -151,11 +154,19 @@ class CloneCommand(EdkrepoCommand):
# Set up submodule alt url config settings prior to cloning any repos
submodule_included_configs = write_included_config(manifest.remotes, manifest.submodule_alternate_remotes, local_manifest_dir)
write_conditional_include(workspace_dir, repo_sources_to_clone, submodule_included_configs)
- clone_repos(args, workspace_dir, repo_sources_to_clone, project_client_side_hooks, config, manifest)
+
+ # Determine if caching is going to be used and then clone
+ cache_obj = get_repo_cache_obj(config)
+ if cache_obj is not None:
+ add_missing_cache_repos(cache_obj, manifest, args.verbose)
+ clone_repos(args, workspace_dir, repo_sources_to_clone, project_client_side_hooks, config, manifest, cache_obj)
# Init submodules
if not args.skip_submodule:
- maintain_submodules(workspace_dir, manifest, combo_name, args.verbose)
+ cache_path = None
+ if cache_obj is not None:
+ cache_path = cache_obj.get_cache_path(SUBMODULE_CACHE_REPO_NAME)
+ maintain_submodules(workspace_dir, manifest, combo_name, args.verbose, cache_path)
# Perform a sparse checkout if requested.
use_sparse = args.sparse
diff --git a/edkrepo/commands/sync_command.py b/edkrepo/commands/sync_command.py
index c4ee330..ff48f50 100644
--- a/edkrepo/commands/sync_command.py
+++ b/edkrepo/commands/sync_command.py
@@ -32,6 +32,7 @@ from edkrepo.common.humble import MIRROR_BEHIND_PRIMARY_REPO, SYNC_NEEDS_REBASE,
from edkrepo.common.humble import SYNC_BRANCH_CHANGE_ON_LOCAL, SYNC_INCOMPATIBLE_COMBO
from edkrepo.common.humble import SYNC_REBASE_CALC_FAIL
from edkrepo.common.pathfix import get_actual_path, expanduser
+from edkrepo.common.common_cache_functions import get_repo_cache_obj
from edkrepo.common.common_repo_functions import clone_repos, sparse_checkout_enabled
from edkrepo.common.common_repo_functions import reset_sparse_checkout, sparse_checkout, verify_single_manifest
from edkrepo.common.common_repo_functions import checkout_repos, check_dirty_repos
@@ -47,6 +48,7 @@ from edkrepo.common.workspace_maintenance.manifest_repos_maintenance import list
from edkrepo.common.ui_functions import init_color_console
from edkrepo.config.config_factory import get_workspace_path, get_workspace_manifest, get_edkrepo_global_data_directory
from edkrepo.config.config_factory import get_workspace_manifest_file
+from edkrepo.config.tool_config import SUBMODULE_CACHE_REPO_NAME
from edkrepo_manifest_parser.edk_manifest import CiIndexXml, ManifestXml
from project_utils.submodule import deinit_submodules, maintain_submodules
@@ -102,7 +104,7 @@ class SyncCommand(EdkrepoCommand):
if not args.update_local_manifest:
self.__check_for_new_manifest(args, config, initial_manifest, workspace_path, global_manifest_directory)
check_dirty_repos(initial_manifest, workspace_path)
-
+
# Determine if sparse checkout needs to be disabled for this operation
sparse_settings = initial_manifest.sparse_settings
sparse_enabled = sparse_checkout_enabled(workspace_path, initial_sources)
@@ -116,7 +118,7 @@ class SyncCommand(EdkrepoCommand):
reset_sparse_checkout(workspace_path, initial_sources)
# Get the latest manifest if requested
- if args.update_local_manifest: # NOTE: hyphens in arg name replaced with underscores due to argparse
+ if args.update_local_manifest: # NOTE: hyphens in arg name replaced with underscores due to argparse
self.__update_local_manifest(args, config, initial_manifest, workspace_path, global_manifest_directory)
manifest = get_workspace_manifest()
if args.update_local_manifest:
@@ -212,7 +214,11 @@ class SyncCommand(EdkrepoCommand):
# Initialize submodules
if not args.skip_submodule:
- maintain_submodules(workspace_path, manifest, current_combo, args.verbose)
+ cache_path = None
+ cache_obj = get_repo_cache_obj(config)
+ if cache_obj is not None:
+ cache_path = cache_obj.get_cache_path(SUBMODULE_CACHE_REPO_NAME)
+ maintain_submodules(workspace_path, manifest, current_combo, args.verbose, cache_path)
# Restore sparse checkout state
if sparse_enabled:
diff --git a/edkrepo/common/common_repo_functions.py b/edkrepo/common/common_repo_functions.py
index 2277c1e..336661a 100644
--- a/edkrepo/common/common_repo_functions.py
+++ b/edkrepo/common/common_repo_functions.py
@@ -56,6 +56,7 @@ from project_utils.sparse import BuildInfo, process_sparse_checkout
from edkrepo.config.config_factory import get_workspace_path
from edkrepo.config.config_factory import get_workspace_manifest
from edkrepo.config.tool_config import CI_INDEX_FILE_NAME
+from edkrepo.config.tool_config import SUBMODULE_CACHE_REPO_NAME
from edkrepo.common.edkrepo_exception import EdkrepoInvalidParametersException
from edkrepo_manifest_parser.edk_manifest import CiIndexXml, ManifestXml
from edkrepo.common.edkrepo_exception import EdkrepoNotFoundException, EdkrepoGitException, EdkrepoWarningException
@@ -75,12 +76,25 @@ CLEAR_LINE = '\x1b[K'
DEFAULT_REMOTE_NAME = 'origin'
PRIMARY_REMOTE_NAME = 'primary'
-def clone_repos(args, workspace_dir, repos_to_clone, project_client_side_hooks, config, manifest):
+
+def clone_repos(args, workspace_dir, repos_to_clone, project_client_side_hooks, config, manifest, cache_obj=None):
for repo_to_clone in repos_to_clone:
local_repo_path = os.path.join(workspace_dir, repo_to_clone.root)
local_repo_url = repo_to_clone.remote_url
+ cache_path = None
+ if cache_obj is not None:
+ cache_path = cache_obj.get_cache_path(local_repo_url)
print("Cloning from: " + str(local_repo_url))
- repo = Repo.clone_from(local_repo_url, local_repo_path, progress=GitProgressHandler(), no_checkout=True)
+ if cache_path is not None:
+ print('+ Using cache at {}'.format(cache_path))
+ repo = Repo.clone_from(local_repo_url, local_repo_path,
+ progress=GitProgressHandler(),
+ reference_if_able=cache_path,
+ no_checkout=True)
+ else:
+ repo = Repo.clone_from(local_repo_url, local_repo_path,
+ progress=GitProgressHandler(),
+ no_checkout=True)
# Fetch notes
repo.remotes.origin.fetch("refs/notes/*:refs/notes/*")
@@ -427,7 +441,7 @@ def combination_is_in_manifest(combination, manifest):
return combination in combination_names
-def checkout(combination, verbose=False, override=False, log=None):
+def checkout(combination, verbose=False, override=False, log=None, cache_obj=None):
workspace_path = get_workspace_path()
manifest = get_workspace_manifest()
@@ -493,7 +507,10 @@ def checkout(combination, verbose=False, override=False, log=None):
# Return to the initial combo, since there was an issue with cheking out the selected combo
checkout_repos(verbose, override, initial_repo_sources, workspace_path, manifest)
finally:
- maintain_submodules(workspace_path, manifest, submodule_combo, verbose)
+ cache_path = None
+ if cache_obj is not None:
+ cache_path = cache_obj.get_cache_path(SUBMODULE_CACHE_REPO_NAME)
+ maintain_submodules(workspace_path, manifest, submodule_combo, verbose, cache_path)
if sparse_enabled or sparse_diff:
print(SPARSE_CHECKOUT)
sparse_checkout(workspace_path, current_repos, manifest)
diff --git a/project_utils/submodule.py b/project_utils/submodule.py
index 3d1b620..f735125 100644
--- a/project_utils/submodule.py
+++ b/project_utils/submodule.py
@@ -61,7 +61,7 @@ def _deinit(repo, submodules=None, verbose=False):
return
-def _update(repo, submodules=None, verbose=False, recursive=False):
+def _update(repo, submodules=None, verbose=False, recursive=False, cache_path=None):
"""
Performs the update of submodules. This includes the sync and update operations.
@@ -82,6 +82,8 @@ def _update(repo, submodules=None, verbose=False, recursive=False):
cmd = ['git', 'submodule', 'update', '--init']
if recursive:
cmd.append('--recursive')
+ if cache_path is not None:
+ cmd.extend(['--reference', cache_path])
output_data = repo.git.execute(cmd, with_extended_output=True, with_stdout=True)
display_git_output(output_data, verbose)
else:
@@ -99,6 +101,8 @@ def _update(repo, submodules=None, verbose=False, recursive=False):
cmd = ['git', 'submodule', 'update', '--init']
if sub.recursive:
cmd.append('--recursive')
+ if cache_path is not None:
+ cmd.extend(['--reference', cache_path])
cmd.extend(['--', sub.path])
output_data = repo.git.execute(cmd, with_extended_output=True, with_stdout=True)
display_git_output(output_data, verbose)
@@ -269,7 +273,7 @@ def deinit_submodules(workspace, start_manifest, start_combo,
_deinit(repo, deinit_list, verbose)
-def maintain_submodules(workspace, manifest, combo_name, verbose=False):
+def maintain_submodules(workspace, manifest, combo_name, verbose=False, cache_path=None):
"""
Updates the submodules for a specific repo.
@@ -277,6 +281,7 @@ def maintain_submodules(workspace, manifest, combo_name, verbose=False):
manifest - The manifest parser object for the project.
combo_name - The combination name to use for submodule maintenance.
verbose - Enable verbose messages.
+ cache_path - Path to the submodule cache repo. A value of None indicates that no cache repo exists.
"""
# Process each repo that may have submodules enabled
print(strings.SUBMOD_INIT_UPDATE)
@@ -303,9 +308,9 @@ def maintain_submodules(workspace, manifest, combo_name, verbose=False):
# Perform sync/update
if len(repo_subs) == 0:
- _update(repo, None, verbose)
+ _update(repo, None, verbose, cache_path=cache_path)
else:
- _update(repo, repo_subs, verbose)
+ _update(repo, repo_subs, verbose, cache_path=cache_path)
if __name__ == '__main__':
--
2.21.0.windows.1
-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.
View/Reply Online (#67645): https://edk2.groups.io/g/devel/message/67645
Mute This Topic: https://groups.io/mt/78310536/1813853
Group Owner: devel+owner at edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub [edk2-devel-archive at redhat.com]
-=-=-=-=-=-=-=-=-=-=-=-
More information about the edk2-devel-archive
mailing list