[virt-tools-list] [virt-bootstrap] [PATCH v4 22/26] Add flag --faster

[Cedric Bosdonnat]

On Thu, 2017-08-03 at 14:13 +0100, Radostin Stoyanov wrote:
> When using this flag with format='dir' the overhead when extracting the
> root file system of container image will be reduce by running `tar` on
> the host instead of in sandbox.
> ---
>  src/virtBootstrap/sources/docker_source.py |  9 ++++++++-
>  src/virtBootstrap/sources/file_source.py   |  4 +++-
>  src/virtBootstrap/utils.py                 | 29 ++++++++++++++++++-----------
>  src/virtBootstrap/virt_bootstrap.py        |  6 ++++++
>  4 files changed, 35 insertions(+), 13 deletions(-)
> 
> diff --git a/src/virtBootstrap/sources/docker_source.py b/src/virtBootstrap/sources/docker_source.py
> index 207d166..675466a 100644
> --- a/src/virtBootstrap/sources/docker_source.py
> +++ b/src/virtBootstrap/sources/docker_source.py
> @@ -53,6 +53,7 @@ class DockerSource(object):
>          @param gid_map: Mappings for GID of files in rootfs
>          @param fmt: Format used to store image [dir, qcow2]
>          @param not_secure: Do not require HTTPS and certificate verification
> +        @param faster: Reduce the overhead when untar images to rootfs
>          @param no_cache: Whether to store downloaded images or not
>          @param progress: Instance of the progress module
>  
> @@ -68,6 +69,7 @@ class DockerSource(object):
>          self.root_password = kwargs.get('root_password', None)
>          self.output_format = kwargs.get('fmt', utils.DEFAULT_OUTPUT_FORMAT)
>          self.insecure = kwargs.get('not_secure', False)
> +        self.faster = kwargs.get('faster', False)
>          self.no_cache = kwargs.get('no_cache', False)
>          self.progress = kwargs['progress'].update_progress
>          self.images_dir = utils.get_image_dir(self.no_cache)
> @@ -271,7 +273,12 @@ class DockerSource(object):
>              if self.output_format == 'dir':
>                  self.progress("Extracting container layers", value=50,
>                                logger=logger)
> -                utils.untar_layers(self.layers, dest, self.progress)
> +                utils.untar_layers(
> +                    self.layers,
> +                    dest,
> +                    self.progress,
> +                    self.faster
> +                )
>              elif self.output_format == 'qcow2':
>                  self.progress("Extracting container layers into qcow2 images",
>                                value=50, logger=logger)
> diff --git a/src/virtBootstrap/sources/file_source.py b/src/virtBootstrap/sources/file_source.py
> index e10cee3..2769927 100644
> --- a/src/virtBootstrap/sources/file_source.py
> +++ b/src/virtBootstrap/sources/file_source.py
> @@ -43,6 +43,7 @@ class FileSource(object):
>          @param fmt: Format used to store image [dir, qcow2]
>          @param uid_map: Mappings for UID of files in rootfs
>          @param gid_map: Mappings for GID of files in rootfs
> +        @param faster: Reduce the overhead when untar images to rootfs
>          @param progress: Instance of the progress module
>          """
>          self.path = kwargs['uri'].path
> @@ -50,6 +51,7 @@ class FileSource(object):
>          self.uid_map = kwargs.get('uid_map', None)
>          self.gid_map = kwargs.get('gid_map', None)
>          self.root_password = kwargs.get('root_password', None)
> +        self.faster = kwargs.get('faster', False)
>          self.progress = kwargs['progress'].update_progress
>  
>      def unpack(self, dest):
> @@ -65,7 +67,7 @@ class FileSource(object):
>          if self.output_format == 'dir':
>              self.progress("Extracting files into destination directory",
>                            value=0, logger=logger)
> -            utils.safe_untar(self.path, dest)
> +            utils.untar(self.path, dest, self.faster)
>  
>          elif self.output_format == 'qcow2':
>              self.progress("Extracting files into qcow2 image", value=0,
> diff --git a/src/virtBootstrap/utils.py b/src/virtBootstrap/utils.py
> index 1731b2a..460e066 100644
> --- a/src/virtBootstrap/utils.py
> +++ b/src/virtBootstrap/utils.py
> @@ -293,18 +293,25 @@ def execute(cmd):
>          raise subprocess.CalledProcessError(proc.returncode, cmd_str)
>  
>  
> -def safe_untar(src, dest):
> +def untar(src, dest, faster=False):
>      """
> -    Extract tarball within LXC container for safety.
> +    Extract tarball to destination path.
> +
> +    @param faster: If True reduce the overhead by running tar on the host
>      """
> -    virt_sandbox = ['virt-sandbox',
> -                    '-c', LIBVIRT_CONN,
> -                    '-m', 'host-bind:/mnt=' + dest]  # Bind destination folder
> +    cmd = ['/bin/tar',
> +           'xf', src,
> +           '-C', dest if faster else '/mnt',
> +           '--exclude', 'dev/*']
>  
> -    # Compression type is auto detected from tar
> -    # Exclude files under /dev to avoid "Cannot mknod: Operation not permitted"
> -    params = ['--', '/bin/tar', 'xf', src, '-C', '/mnt', '--exclude', 'dev/*']
> -    execute(virt_sandbox + params)
> +    if not faster:
> +        cmd = ['virt-sandbox',
> +               # Set connection
> +               '-c', LIBVIRT_CONN,
> +               # Bind destination folder
> +               '-m', 'host-bind:/mnt=' + dest,
> +               '--'] + cmd
> +    execute(cmd)
>  
>  
>  def bytes_to_size(number):
> @@ -351,7 +358,7 @@ def log_layer_extract(layer, current, total, progress):
>      logger.debug('Untar layer: (%s:%s) %s', sum_type, sum_value, layer_file)
>  
>  
> -def untar_layers(layers_list, dest_dir, progress):
> +def untar_layers(layers_list, dest_dir, progress, faster=False):
>      """
>      Untar each of layers from container image.
>      """
> @@ -361,7 +368,7 @@ def untar_layers(layers_list, dest_dir, progress):
>          layer_file = layer[2]
>  
>          # Extract layer tarball into destination directory
> -        safe_untar(layer_file, dest_dir)
> +        untar(layer_file, dest_dir, faster)
>  
>          # Update progress value
>          progress(value=(float(index + 1) / nlayers * 50) + 50)
> diff --git a/src/virtBootstrap/virt_bootstrap.py b/src/virtBootstrap/virt_bootstrap.py
> index cbd9f0c..e465fb2 100755
> --- a/src/virtBootstrap/virt_bootstrap.py
> +++ b/src/virtBootstrap/virt_bootstrap.py
> @@ -100,6 +100,7 @@ def bootstrap(uri, dest,
>                uid_map=None,
>                gid_map=None,
>                not_secure=False,
> +              faster=None,
>                no_cache=False,
>                progress_cb=None):
>      """
> @@ -127,6 +128,7 @@ def bootstrap(uri, dest,
>             uid_map=uid_map,
>             gid_map=gid_map,
>             not_secure=not_secure,
> +           faster=faster,
>             no_cache=no_cache,
>             root_password=root_password,
>             progress=prog).unpack(dest)
> @@ -210,6 +212,9 @@ def main():
>      parser.add_argument("--status-only", action="store_const",
>                          const=utils.write_progress,
>                          help=_("Show only progress information"))
> +    parser.add_argument("--faster", action="store_true",
> +                        help=_("Reduce the overhead when extracting layers "
> +                               "(use this option only for trusted images)"))
>  
>      try:
>          args = parser.parse_args()
> @@ -235,6 +240,7 @@ def main():
>                    uid_map=uid_map,
>                    gid_map=gid_map,
>                    not_secure=args.not_secure,
> +                  faster=args.faster,
>                    no_cache=args.no_cache,
>                    progress_cb=args.status_only)
>  

ACK

--
Cedric