[virt-tools-list] [virt-bootstrap] [PATCH v6 14/26] Create qcow2 images with guestfs-python

Cedric Bosdonnat cbosdonnat at suse.com
Sat Aug 19 12:54:27 UTC 2017


On Thu, 2017-08-17 at 10:39 +0100, Radostin Stoyanov wrote:
> Use the python bindings of libguestfs to create qcow2 image with
> backing chains to mimic the layers of container image.
> 
> This commit also changes the behaviour of FileSource when 'qcow2'
> output format is used. Now the string layer-0.qcow2 will be used
> as name of the output file.
> 
> This change is appied in the test suite as an update to the function

typo: 'applied'

> get_image_path().
> 
> Note: The following commits which introduce UID/GID mapping
> and setting root password for 'qcow2' with FileSource will apply
> these changes to layer-1.qcow2 image with backing file layer-0.qcow2.

What is the interest of this note for this commit? The info could be
interesting in the corresponding commit though.

> ---
>  src/virtBootstrap/sources/docker_source.py |  10 +-
>  src/virtBootstrap/sources/file_source.py   |  14 +--
>  src/virtBootstrap/utils.py                 | 146 +++++++++++++++++------------
>  tests/file_source.py                       |   8 +-
>  4 files changed, 106 insertions(+), 72 deletions(-)
> 
> diff --git a/src/virtBootstrap/sources/docker_source.py b/src/virtBootstrap/sources/docker_source.py
> index 58e30ce..99247ab 100644
> --- a/src/virtBootstrap/sources/docker_source.py
> +++ b/src/virtBootstrap/sources/docker_source.py
> @@ -275,7 +275,15 @@ class DockerSource(object):
>              elif self.output_format == 'qcow2':
>                  self.progress("Extracting container layers into qcow2 images",
>                                value=50, logger=logger)
> -                utils.extract_layers_in_qcow2(self.layers, dest, self.progress)
> +
> +                img = utils.BuildImage(
> +                    layers=self.layers,
> +                    dest=dest,
> +                    progress=self.progress
> +                )
> +                img.create_base_layer()
> +                img.create_backing_chains()
> +
>              else:
>                  raise Exception("Unknown format:" + self.output_format)
>  
> diff --git a/src/virtBootstrap/sources/file_source.py b/src/virtBootstrap/sources/file_source.py
> index 412db8a..69f024c 100644
> --- a/src/virtBootstrap/sources/file_source.py
> +++ b/src/virtBootstrap/sources/file_source.py
> @@ -64,14 +64,16 @@ class FileSource(object):
>              utils.untar_layers(layer, dest, self.progress)
>  
>          elif self.output_format == 'qcow2':
> -            # Remove the old path
> -            file_name = os.path.basename(self.path)
> -            qcow2_file = os.path.realpath('{}/{}.qcow2'.format(dest,
> -                                                               file_name))
> -
>              self.progress("Extracting files into qcow2 image", value=0,
>                            logger=logger)
> -            utils.create_qcow2(self.path, qcow2_file)
> +
> +            img = utils.BuildImage(
> +                layers=layer,
> +                dest=dest,
> +                progress=self.progress
> +            )
> +            img.create_base_layer()
> +
>          else:
>              raise Exception("Unknown format:" + self.output_format)
>  
> diff --git a/src/virtBootstrap/utils.py b/src/virtBootstrap/utils.py
> index 77177a3..482e4aa 100644
> --- a/src/virtBootstrap/utils.py
> +++ b/src/virtBootstrap/utils.py
> @@ -33,6 +33,7 @@ import tempfile
>  import logging
>  import re
>  
> +import guestfs
>  import passlib.hosts
>  
>  # pylint: disable=invalid-name
> @@ -42,6 +43,8 @@ logger = logging.getLogger(__name__)
>  DEFAULT_OUTPUT_FORMAT = 'dir'
>  # Default virtual size of qcow2 image
>  DEF_QCOW2_SIZE = '5G'
> +DEF_BASE_IMAGE_SIZE = 5 * 1024 * 1024 * 1024
> +
>  if os.geteuid() == 0:
>      LIBVIRT_CONN = "lxc:///"
>      DEFAULT_IMG_DIR = "/var/lib/virt-bootstrap/docker_images"
> @@ -51,6 +54,88 @@ else:
>      DEFAULT_IMG_DIR += "/.local/share/virt-bootstrap/docker_images"
>  
>  
> +class BuildImage(object):
> +    """
> +    Use guestfs-python to create qcow2 disk images.
> +    """
> +
> +    def __init__(self, layers, dest, progress):
> +        """
> +        @param tar_files: Tarballs to be converted to qcow2 images
> +        @param dest: Directory where the qcow2 images will be created
> +        @param progress: Instance of the progress module
> +        """
> +        self.g = guestfs.GuestFS(python_return_dict=True)
> +        self.layers = layers
> +        self.nlayers = len(layers)
> +        self.dest = dest
> +        self.progress = progress
> +        self.qcow2_files = []
> +
> +    def create_base_layer(self, fmt='qcow2', size=DEF_BASE_IMAGE_SIZE):
> +        """
> +        Create and format qcow2 disk image which represnts the base layer.
> +        """
> +        self.qcow2_files = [os.path.join(self.dest, 'layer-0.qcow2')]
> +        self.progress("Creating base layer", logger=logger)
> +        self.g.disk_create(self.qcow2_files[0], fmt, size)
> +        self.g.add_drive(self.qcow2_files[0], format=fmt)
> +        self.g.launch()
> +        self.progress("Formating disk image", logger=logger)
> +        self.g.mkfs("ext3", '/dev/sda')
> +        self.extract_layer(0, '/dev/sda')
> +        # Shutdown qemu instance to avoid hot-plugging of devices.
> +        self.g.shutdown()
> +
> +    def create_backing_chains(self):
> +        """
> +        Convert other layers to qcow2 images linked as backing chains.
> +        """
> +        for i in range(1, self.nlayers):
> +            self.qcow2_files.append(
> +                os.path.join(self.dest, 'layer-%d.qcow2' % i)
> +            )
> +            self.progress(
> +                "Creating image (%d/%d)" % (i + 1, self.nlayers),
> +                logger=logger
> +            )
> +            self.g.disk_create(
> +                filename=self.qcow2_files[i],
> +                format='qcow2',
> +                size=-1,
> +                backingfile=self.qcow2_files[i - 1],
> +                backingformat='qcow2'
> +            )
> +            self.g.add_drive(self.qcow2_files[i], format='qcow2')
> +        self.g.launch()
> +        devices = self.g.list_devices()
> +        # Tar-in layers (skip the base layer)
> +        for index in range(1, self.nlayers):
> +            self.extract_layer(index, devices[index - 1])
> +        self.g.shutdown()
> +
> +    def extract_layer(self, index, dev):
> +        """
> +        Extract tarball of layer to device
> +        """
> +        tar_file, tar_size = self.layers[index]
> +        log_layer_extract(
> +            tar_file, tar_size, index + 1, self.nlayers, self.progress
> +        )
> +        self.tar_in(dev, tar_file)
> +
> +    def tar_in(self, dev, tar_file):
> +        """
> +        Common pattern used to tar-in archive into image
> +        """
> +        self.g.mount(dev, '/')
> +        # Restore extended attributes, SELinux contexts and POSIX ACLs
> +        # from tar file.
> +        self.g.tar_in(tar_file, '/', get_compression_type(tar_file),
> +                      xattrs=True, selinux=True, acls=True)
> +        self.g.umount('/')
> +
> +
>  def get_compression_type(tar_file):
>      """
>      Get compression type of tar file.
> @@ -209,67 +294,6 @@ def get_mime_type(path):
>          return output.read().decode('utf-8').split()[1]
>  
>  
> -def create_qcow2(tar_file, layer_file, backing_file=None, size=DEF_QCOW2_SIZE):
> -    """
> -    Create qcow2 image from tarball.
> -    """
> -    qemu_img_cmd = ["qemu-img", "create", "-f", "qcow2", layer_file, size]
> -
> -    if not backing_file:
> -        logger.info("Creating base qcow2 image")
> -        execute(qemu_img_cmd)
> -
> -        logger.info("Formatting qcow2 image")
> -        execute(['virt-format',
> -                 '--format=qcow2',
> -                 '--partition=none',
> -                 '--filesystem=ext3',
> -                 '-a', layer_file])
> -    else:
> -        # Add backing chain
> -        qemu_img_cmd.insert(2, "-b")
> -        qemu_img_cmd.insert(3, backing_file)
> -
> -        logger.info("Creating qcow2 image with backing chain")
> -        execute(qemu_img_cmd)
> -
> -    # Extract tarball using "tar-in" command from libguestfs
> -    tar_in_cmd = ["guestfish",
> -                  "-a", layer_file,
> -                  '-m', '/dev/sda',
> -                  'tar-in', tar_file, "/"]
> -
> -    # Check if tarball is compressed
> -    compression = get_compression_type(tar_file)
> -    if compression is not None:
> -        tar_in_cmd.append('compress:' + compression)
> -
> -    # Execute virt-tar-in command
> -    execute(tar_in_cmd)
> -
> -
> -def extract_layers_in_qcow2(layers_list, dest_dir, progress):
> -    """
> -    Extract docker layers in qcow2 images with backing chains.
> -    """
> -    qcow2_backing_file = None
> -
> -    nlayers = len(layers_list)
> -    for index, layer in enumerate(layers_list):
> -        tar_file, tar_size = layer
> -        log_layer_extract(tar_file, tar_size, index + 1, nlayers, progress)
> -
> -        # Name format for the qcow2 image
> -        qcow2_layer_file = "{}/layer-{}.qcow2".format(dest_dir, index)
> -        # Create the image layer
> -        create_qcow2(tar_file, qcow2_layer_file, qcow2_backing_file)
> -        # Keep the file path for the next layer
> -        qcow2_backing_file = qcow2_layer_file
> -
> -        # Update progress value
> -        progress(value=(float(index + 1) / nlayers * 50) + 50)
> -
> -
>  def get_image_dir(no_cache=False):
>      """
>      Get the directory where image layers are stored.
> diff --git a/tests/file_source.py b/tests/file_source.py
> index dd7fd00..2a0f964 100644
> --- a/tests/file_source.py
> +++ b/tests/file_source.py
> @@ -50,13 +50,13 @@ class Qcow2BuildImage(BuildTarFiles):
>          g.umount('/')
>          g.shutdown()
>  
> -    def get_image_path(self):
> +    def get_image_path(self, n=0):
>          """
>          Returns the path where the qcow2 image will be stored.
>          """
>          return os.path.join(
>              self.dest_dir,
> -            "%s.qcow2" % os.path.basename(self.tar_file)
> +            "layer-%d.qcow2" % n
>          )
>  
>      def runTest(self):
> @@ -92,7 +92,7 @@ class Qcow2OwnershipMapping(Qcow2BuildImage):
>              gid_map=self.gid_map
>          )
>          self.apply_mapping()
> -        self.check_qcow2_images(self.get_image_path())
> +        self.check_qcow2_images(self.get_image_path(1))
>  
>  
>  @unittest.skip("Not implemented")
> @@ -114,7 +114,7 @@ class Qcow2SettingRootPassword(Qcow2BuildImage):
>              root_password=self.root_password
>          )
>          self.check_image = self.validate_shadow_file_in_image
> -        self.check_qcow2_images(self.get_image_path())
> +        self.check_qcow2_images(self.get_image_path(1))
>  
>  
>  @unittest.skipIf(os.geteuid() != 0, "Root privileges required")

ACK

--
Cedric




More information about the virt-tools-list mailing list