[edk2-devel] [edk2-staging/EdkRepo] [PATCH v1 1/2] EdkRepo: Add cache command
Bjorge, Erik C
erik.c.bjorge at intel.com
Wed Nov 11 23:07:28 UTC 2020
Adds a module to add a repo cache and mange it. Also adds a command
to manage the repo cache from EdkRepo. No other commands use the
functionality at this point.
Cc: Ashley E Desimone <ashley.e.desimone at intel.com>
Cc: Nate DeSimone <nathaniel.l.desimone at intel.com>
Cc: Puja Pandya <puja.pandya at intel.com>
Cc: Bret Barkelew <Bret.Barkelew at microsoft.com>
Cc: Prince Agyeman <prince.agyeman at intel.com>
Cc: Erik Bjorge <erik.c.bjorge at intel.com>
Signed-off-by: Erik Bjorge <erik.c.bjorge at intel.com>
---
edkrepo/commands/arguments/cache_args.py | 19 ++
edkrepo/commands/cache_command.py | 118 ++++++++++++
edkrepo/commands/humble/cache_humble.py | 17 ++
edkrepo/common/common_cache_functions.py | 41 +++++
edkrepo/common/edkrepo_exception.py | 3 +
edkrepo/config/config_factory.py | 14 +-
edkrepo/config/tool_config.py | 5 +-
project_utils/cache.py | 224 +++++++++++++++++++++++
project_utils/project_utils_strings.py | 11 ++
9 files changed, 448 insertions(+), 4 deletions(-)
create mode 100644 edkrepo/commands/arguments/cache_args.py
create mode 100644 edkrepo/commands/cache_command.py
create mode 100644 edkrepo/commands/humble/cache_humble.py
create mode 100644 edkrepo/common/common_cache_functions.py
create mode 100644 project_utils/cache.py
diff --git a/edkrepo/commands/arguments/cache_args.py b/edkrepo/commands/arguments/cache_args.py
new file mode 100644
index 0000000..0080536
--- /dev/null
+++ b/edkrepo/commands/arguments/cache_args.py
@@ -0,0 +1,19 @@
+#!/usr/bin/env python3
+#
+## @file
+# cache_args.py
+#
+# Copyright (c) 2020, Intel Corporation. All rights reserved.<BR>
+# SPDX-License-Identifier: BSD-2-Clause-Patent
+#
+
+''' Contains the help and description strings for arguments in the
+cache command meta data.
+'''
+COMMAND_DESCRIPTION = ('Manages local caching support for project repos. The goal of this feature '
+ 'is to improve clone performance')
+COMMAND_ENABLE_HELP = 'Enables caching support on the system.'
+COMMAND_DISABLE_HELP = 'Disables caching support on the system.'
+COMMAND_UPDATE_HELP = 'Update the repo cache for all cached projects.'
+COMMAND_INFO_HELP = 'Display the current cache information.'
+COMMAND_PROJECT_HELP = 'Project to add to the cache.'
diff --git a/edkrepo/commands/cache_command.py b/edkrepo/commands/cache_command.py
new file mode 100644
index 0000000..9f0d6e9
--- /dev/null
+++ b/edkrepo/commands/cache_command.py
@@ -0,0 +1,118 @@
+#!/usr/bin/env python3
+#
+## @file
+# cache_command.py
+#
+# Copyright (c) 2020, Intel Corporation. All rights reserved.<BR>
+# SPDX-License-Identifier: BSD-2-Clause-Patent
+#
+
+import edkrepo.commands.arguments.cache_args as arguments
+from edkrepo.commands.edkrepo_command import EdkrepoCommand
+from edkrepo.commands.edkrepo_command import SourceManifestRepoArgument
+from edkrepo.commands.humble.cache_humble import CACHE_ENABLED, CACHE_FETCH, CACHE_INFO
+from edkrepo.commands.humble.cache_humble import CACHE_INFO_LINE, PROJECT_NOT_FOUND, NO_INSTANCE
+from edkrepo.commands.humble.cache_humble import UNABLE_TO_LOAD_MANIFEST, UNABLE_TO_PARSE_MANIFEST
+from edkrepo.common.common_cache_functions import add_missing_cache_repos
+from edkrepo.common.common_cache_functions import get_repo_cache_obj
+from edkrepo.common.edkrepo_exception import EdkrepoCacheException
+from edkrepo.common.workspace_maintenance.manifest_repos_maintenance import find_project_in_all_indices
+from edkrepo.config.config_factory import get_workspace_manifest
+from edkrepo_manifest_parser.edk_manifest import ManifestXml
+
+
+class CacheCommand(EdkrepoCommand):
+ def __init__(self):
+ super().__init__()
+
+ def get_metadata(self):
+ metadata = {}
+ metadata['name'] = 'cache'
+ metadata['help-text'] = arguments.COMMAND_DESCRIPTION
+ args = []
+ metadata['arguments'] = args
+ args.append({'name': 'enable',
+ 'positional': False,
+ 'required': False,
+ 'help-text': arguments.COMMAND_ENABLE_HELP})
+ args.append({'name': 'disable',
+ 'positional': False,
+ 'required': False,
+ 'help-text': arguments.COMMAND_DISABLE_HELP})
+ args.append({'name': 'update',
+ 'positional': False,
+ 'required': False,
+ 'help-text': arguments.COMMAND_UPDATE_HELP})
+ args.append({'name': 'info',
+ 'positional': False,
+ 'required': False,
+ 'help-text': arguments.COMMAND_INFO_HELP})
+ args.append({'name': 'project',
+ 'positional': True,
+ 'required': False,
+ 'help-text': arguments.COMMAND_PROJECT_HELP})
+ args.append(SourceManifestRepoArgument)
+ return metadata
+
+ def run_command(self, args, config):
+ # Process enable disable requests
+ if args.disable:
+ config['user_cfg_file'].set_caching_state(False)
+ elif args.enable:
+ config['user_cfg_file'].set_caching_state(True)
+
+ # Get the current state now that we have processed enable/disable
+ cache_state = config['user_cfg_file'].caching_state
+ print(CACHE_ENABLED.format(cache_state))
+ if not cache_state:
+ return
+
+ # State is enabled so make sure cache directory exists
+ cache_obj = get_repo_cache_obj(config)
+
+ # Check to see if a manifest was provided and add any missing remotes
+ manifest = None
+ if args.project is not None:
+ manifest = _get_manifest(args.project, config, args.source_manifest_repo)
+ else:
+ try:
+ manifest = get_workspace_manifest()
+ except Exception:
+ pass
+
+ # If manifest is provided attempt to add any remotes that do not exist
+ if manifest is not None:
+ add_missing_cache_repos(cache_obj, manifest, True)
+
+ # Display all the cache information
+ if args.info:
+ print(CACHE_INFO)
+ info = cache_obj.get_cache_info(args.verbose)
+ for item in info:
+ print(CACHE_INFO_LINE.format(item.path, item.remote, item.url))
+
+ # Do an update if requested
+ if args.update:
+ print(CACHE_FETCH)
+ cache_obj.update_cache(verbose=True)
+
+ # Close the cache repos
+ cache_obj.close(args.verbose)
+
+
+def _get_manifest(project, config, source_manifest_repo=None):
+ try:
+ manifest_repo, source_cfg, manifest_path = find_project_in_all_indices(
+ project,
+ config['cfg_file'],
+ config['user_cfg_file'],
+ PROJECT_NOT_FOUND.format(project),
+ NO_INSTANCE.format(project),
+ source_manifest_repo)
+ except Exception:
+ raise EdkrepoCacheException(UNABLE_TO_LOAD_MANIFEST)
+ try:
+ manifest = ManifestXml(manifest_path)
+ except Exception:
+ raise EdkrepoCacheException(UNABLE_TO_PARSE_MANIFEST)
+ return manifest
diff --git a/edkrepo/commands/humble/cache_humble.py b/edkrepo/commands/humble/cache_humble.py
new file mode 100644
index 0000000..4f318ac
--- /dev/null
+++ b/edkrepo/commands/humble/cache_humble.py
@@ -0,0 +1,17 @@
+#!/usr/bin/env python3
+#
+## @file
+# cache_humble.py
+#
+# Copyright (c) 2020, Intel Corporation. All rights reserved.<BR>
+# SPDX-License-Identifier: BSD-2-Clause-Patent
+#
+
+CACHE_ENABLED = 'Caching Enabled: {}'
+CACHE_INFO = 'Cache Information:'
+CACHE_INFO_LINE = '+ {}\n {} ({})'
+CACHE_FETCH = 'Fetching all remotes... (this could take a while)'
+PROJECT_NOT_FOUND = 'Project {} does not exist'
+NO_INSTANCE = 'Unable to determine instance to use for {}'
+UNABLE_TO_LOAD_MANIFEST = 'Unable to load manifest file.'
+UNABLE_TO_PARSE_MANIFEST = 'Failed to parse manifest file.'
diff --git a/edkrepo/common/common_cache_functions.py b/edkrepo/common/common_cache_functions.py
new file mode 100644
index 0000000..84bd3ed
--- /dev/null
+++ b/edkrepo/common/common_cache_functions.py
@@ -0,0 +1,41 @@
+#!/usr/bin/env python3
+#
+## @file
+# common_cache_functions.py
+#
+# Copyright (c) 2020, Intel Corporation. All rights reserved.<BR>
+# SPDX-License-Identifier: BSD-2-Clause-Patent
+#
+
+import os
+
+from edkrepo.config.config_factory import get_edkrepo_global_data_directory
+from edkrepo.config.tool_config import SUBMODULE_CACHE_REPO_NAME
+from project_utils.cache import RepoCache
+
+
+def get_global_cache_directory(config):
+ if config['user_cfg_file'].caching_state:
+ return os.path.join(get_edkrepo_global_data_directory(), '.cache')
+ return None
+
+
+def get_repo_cache_obj(config):
+ cache_obj = None
+ cache_directory = get_global_cache_directory(config)
+ if cache_directory is not None:
+ cache_obj = RepoCache(cache_directory)
+ cache_obj.open()
+ return cache_obj
+
+
+def add_missing_cache_repos(cache_obj, manifest, verbose=False):
+ print('Adding and fetching new remotes... (this could take a while)')
+ for remote in manifest.remotes:
+ cache_obj.add_repo(url=remote.url, verbose=verbose)
+ alt_submodules = manifest.submodule_alternate_remotes
+ if alt_submodules:
+ print('Adding and fetching new submodule remotes... (this could also take a while)')
+ cache_obj.add_repo(name=SUBMODULE_CACHE_REPO_NAME, verbose=verbose)
+ for alt in alt_submodules:
+ cache_obj.add_remote(alt.alternate_url, SUBMODULE_CACHE_REPO_NAME, verbose)
diff --git a/edkrepo/common/edkrepo_exception.py b/edkrepo/common/edkrepo_exception.py
index a56e709..b3f2300 100644
--- a/edkrepo/common/edkrepo_exception.py
+++ b/edkrepo/common/edkrepo_exception.py
@@ -98,3 +98,6 @@ class EdkrepoGitConfigSetupException(EdkrepoException):
def __init__(self, message):
super().__init__(message, 131)
+class EdkrepoCacheException(EdkrepoException):
+ def __init__(self, message):
+ super().__init__(message, 132)
diff --git a/edkrepo/config/config_factory.py b/edkrepo/config/config_factory.py
index fe69460..3680c0b 100644
--- a/edkrepo/config/config_factory.py
+++ b/edkrepo/config/config_factory.py
@@ -225,10 +225,20 @@ class GlobalUserConfig(BaseConfig):
self.filename = os.path.join(get_edkrepo_global_data_directory(), "edkrepo_user.cfg")
self.prop_list = [
CfgProp('scm', 'mirror_geo', 'geo', 'none', False),
- CfgProp('send-review', 'max-patch-set', 'max_patch_set', '10', False)
- ]
+ CfgProp('send-review', 'max-patch-set', 'max_patch_set', '10', False),
+ CfgProp('caching', 'enable-caching', 'enable_caching_text', 'false', False)]
super().__init__(self.filename, get_edkrepo_global_data_directory(), False)
+ @property
+ def caching_state(self):
+ return self.enable_caching_text.lower() == 'true'
+
+ def set_caching_state(self, enable):
+ if enable:
+ self.enable_caching_text = 'true'
+ else:
+ self.enable_caching_text = 'false'
+
@property
def max_patch_set_int(self):
try:
diff --git a/edkrepo/config/tool_config.py b/edkrepo/config/tool_config.py
index eee1326..81f4ddf 100644
--- a/edkrepo/config/tool_config.py
+++ b/edkrepo/config/tool_config.py
@@ -1,10 +1,11 @@
#!/usr/bin/env python3
#
## @file
-# tool)config.py
+# tool_config.py
#
# Copyright (c) 2020, Intel Corporation. All rights reserved.<BR>
# SPDX-License-Identifier: BSD-2-Clause-Patent
#
-CI_INDEX_FILE_NAME = 'CiIndex.xml'
\ No newline at end of file
+CI_INDEX_FILE_NAME = 'CiIndex.xml'
+SUBMODULE_CACHE_REPO_NAME = 'submodule-cache'
diff --git a/project_utils/cache.py b/project_utils/cache.py
new file mode 100644
index 0000000..8efd411
--- /dev/null
+++ b/project_utils/cache.py
@@ -0,0 +1,224 @@
+#!/usr/bin/env python3
+#
+## @file
+# cache.py
+#
+# Copyright (c) 2020, Intel Corporation. All rights reserved.<BR>
+# SPDX-License-Identifier: BSD-2-Clause-Patent
+#
+from collections import namedtuple
+import os
+import shutil
+
+from git import Repo
+
+from edkrepo.common.progress_handler import GitProgressHandler
+from project_utils.project_utils_strings import CACHE_ADD_REMOTE, CACHE_ADDING_REPO, CACHE_CHECK_ROOT_DIR
+from project_utils.project_utils_strings import CACHE_FAILED_TO_CLOSE, CACHE_FAILED_TO_OPEN, CACHE_FETCH_REMOTE
+from project_utils.project_utils_strings import CACHE_REMOTE_EXISTS, CACHE_REMOVE_REPO, CACHE_REPO_EXISTS
+
+CacheInfo = namedtuple('CacheInfo', ['path', 'remote', 'url'])
+
+
+class RepoCache(object):
+ """
+ Provides basic management of a cache repo.
+ """
+ def __init__(self, path):
+ self._cache_root_path = path
+ self._repos = {}
+
+ def _create_name(self, url_or_name):
+ """
+ Used to create consistent repo and remote names
+ """
+ dir_name = url_or_name.split('/')[-1]
+ if not dir_name.endswith('.git'):
+ dir_name += '.git'
+ return dir_name
+
+ def _get_repo_path(self, dir_name):
+ return os.path.join(self._cache_root_path, dir_name)
+
+ def _get_repo(self, dir_name):
+ """
+ Returns the git repo object for the cache repo.
+
+ Raises FileNotFoundError if the cache directory does not exist.
+ Raises IOError if the repo cannot be opened
+ """
+ repo_path = self._get_repo_path(dir_name)
+ if not os.path.isdir(repo_path):
+ raise FileNotFoundError
+ try:
+ repo = Repo(repo_path)
+ except Exception:
+ raise IOError
+ return repo
+
+ def _get_cache_dirs(self):
+ if not os.path.isdir(self._cache_root_path):
+ raise FileNotFoundError
+ return [x for x in os.listdir(self._cache_root_path) if os.path.isdir(self._get_repo_path(x))]
+
+ def _add_and_fetch_remote(self, repo, remote_name, url, verbose=False):
+ if verbose:
+ print(CACHE_ADD_REMOTE.format(remote_name, url))
+ repo.create_remote(remote_name, url)
+ if verbose:
+ print(CACHE_FETCH_REMOTE.format(remote_name, url))
+ repo.remotes[remote_name].fetch(progress=GitProgressHandler())
+
+ def open(self, verbose=False):
+ """
+ Opens all cache repos.
+
+ Raises FileNotFoundError if the cache directory does not exist.
+ """
+ if not self._repos:
+ if not os.path.isdir(self._cache_root_path):
+ if verbose:
+ print(CACHE_CHECK_ROOT_DIR.format(self._cache_root_path))
+ os.makedirs(self._cache_root_path)
+
+ for dir_name in self._get_cache_dirs():
+ try:
+ self._repos[dir_name] = self._get_repo(dir_name)
+ except Exception:
+ if verbose:
+ print(CACHE_FAILED_TO_OPEN.format(dir_name))
+
+ def close(self, verbose=False):
+ """
+ Closes all cache repos.
+ """
+ for dir_name in self._repos:
+ try:
+ self._repos[dir_name].close()
+ except Exception:
+ if verbose:
+ print(CACHE_FAILED_TO_CLOSE.format(dir_name))
+ self._repos = {}
+
+ def get_cache_path(self, url_or_name):
+ dir_name = self._create_name(url_or_name)
+ if dir_name not in self._repos:
+ return None
+ return self._get_repo_path(dir_name)
+
+ def get_cache_info(self, verbose=False):
+ """
+ Returns a list of remotes currently configured in the cache.
+
+ Raises FileNotFoundError if the cache repo is not open.
+ """
+ ret_val = []
+ for dir_name in self._repos:
+ for remote in self._repos[dir_name].remotes:
+ ret_val.append(CacheInfo(self._get_repo_path(dir_name), remote.name, remote.url))
+ return ret_val
+
+ def delete_cache_root(self, verbose=False):
+ """
+ Deletes the cache root directory and all caches.
+ """
+ if os.path.isdir(self._cache_root_path):
+ if self._repos:
+ self.close()
+ shutil.rmtree(self._cache_root_path, ignore_errors=True)
+
+ def add_repo(self, url=None, name=None, verbose=False):
+ """
+ Adds a repo to the cache if it does not already exist.
+
+ """
+ remote_name = None
+ if url is None and name is None:
+ raise ValueError
+ elif name is not None:
+ dir_name = self._create_name(name)
+ else:
+ dir_name = self._create_name(url)
+ if url is not None:
+ remote_name = self._create_name(url)
+ repo_path = self._get_repo_path(dir_name)
+
+ if dir_name in self._repos:
+ if verbose:
+ print(CACHE_REPO_EXISTS.format(dir_name))
+ else:
+ if verbose:
+ print(CACHE_ADDING_REPO.format(dir_name))
+ os.makedirs(repo_path)
+ self._repos[dir_name] = Repo.init(repo_path, bare=True)
+
+ if remote_name is not None and remote_name not in self._repos[dir_name].remotes:
+ self._add_and_fetch_remote(self._get_repo(dir_name), remote_name, url)
+ return dir_name
+
+ def remove_repo(self, url=None, name=None, verbose=False):
+ """
+ Removes a remote from the cache repo if it exists
+
+ Raises FileNotFoundError if the cache repo is not open.
+ """
+ if url is None and name is None:
+ raise ValueError
+ elif name is not None:
+ dir_name = self._create_name(name)
+ else:
+ dir_name = self._create_name(url)
+ if dir_name not in self._repos:
+ return
+ if verbose:
+ print(CACHE_REMOVE_REPO.format(dir_name))
+ self._repos.pop(dir_name).close()
+ shutil.rmtree(os.path.join(self._cache_root_path, dir_name), ignore_errors=True)
+
+ def add_remote(self, url, name, verbose=False):
+ remote_name = self._create_name(url)
+ dir_name = self._create_name(name)
+ if dir_name not in self._repos:
+ raise ValueError
+ repo = self._get_repo(dir_name)
+ if remote_name in repo.remotes:
+ if verbose:
+ print(CACHE_REMOTE_EXISTS.format(remote_name))
+ return
+ self._add_and_fetch_remote(repo, remote_name, url, verbose)
+
+ def remove_remote(self, url, name, verbose=False):
+ remote_name = self._create_name(url)
+ dir_name = self._create_name(name)
+ if dir_name not in self._repos:
+ raise ValueError
+ repo = self._get_repo(dir_name)
+ if remote_name not in repo.remotes:
+ raise IndexError
+ repo.remove_remote(repo.remotes[remote_name])
+
+ def update_cache(self, url_or_name=None, verbose=False):
+ if not self._repos:
+ raise FileNotFoundError
+ repo_dirs = self._repos.keys()
+
+ if url_or_name is not None:
+ dir_name = self._create_name(url_or_name)
+ if dir_name in self._repos:
+ repo_dirs = [dir_name]
+ else:
+ return
+
+ for dir_name in repo_dirs:
+ try:
+ repo = self._get_repo(dir_name)
+ except Exception:
+ print(CACHE_FAILED_TO_OPEN.format(dir_name))
+ continue
+ for remote in repo.remotes:
+ if verbose:
+ print(CACHE_FETCH_REMOTE.format(dir_name, remote.url))
+ remote.fetch(progress=GitProgressHandler())
+
+ def clean_cache(self, verbose=False):
+ raise NotImplementedError
diff --git a/project_utils/project_utils_strings.py b/project_utils/project_utils_strings.py
index 33c22d2..1547978 100644
--- a/project_utils/project_utils_strings.py
+++ b/project_utils/project_utils_strings.py
@@ -22,3 +22,14 @@ SUBMOD_DEINIT_PATH = 'Submodule deinit: {}'
SUBMOD_SYNC_PATH = 'Submodule sync: {}'
SUBMOD_UPDATE_PATH = 'Submodule update: {}'
SUBMOD_EXCEPTION = '- Exception: {}'
+
+# Caching support strings
+CACHE_ADD_REMOTE = '+ Adding remote {} ({})'
+CACHE_FETCH_REMOTE = '+ Fetching data for {} ({})'
+CACHE_CHECK_ROOT_DIR = '+ Creating cache root directory: {}'
+CACHE_FAILED_TO_OPEN = '- Failed to open cache: {}'
+CACHE_FAILED_TO_CLOSE = '- Failed to close cache: {}'
+CACHE_REPO_EXISTS = '- Repo {} already exists.'
+CACHE_ADDING_REPO = '+ Adding cache repo {}'
+CACHE_REMOVE_REPO = '- Removing cache repo: {}'
+CACHE_REMOTE_EXISTS = '- Remote {} already exists.'
--
2.21.0.windows.1
-=-=-=-=-=-=-=-=-=-=-=-
Groups.io Links: You receive all messages sent to this group.
View/Reply Online (#67329): https://edk2.groups.io/g/devel/message/67329
Mute This Topic: https://groups.io/mt/78195378/1813853
Group Owner: devel+owner at edk2.groups.io
Unsubscribe: https://edk2.groups.io/g/devel/unsub [edk2-devel-archive at redhat.com]
-=-=-=-=-=-=-=-=-=-=-=-
More information about the edk2-devel-archive
mailing list