[virt-tools-list] [virt-bootstrap] [PATCH v7 17/26] Add virt-builder source
Radostin Stoyanov
rstoyanov1 at gmail.com
Sat Aug 26 20:42:06 UTC 2017
Add implementation for virt-builder source which aims to create
container root file system from VM image build with virt-builder.
Usage examples:
$ virt-bootstrap virt-builder://fedora-25 /tmp/foo
$ virt-bootstrap virt-builder://ubuntu-16.04 /tmp/bar --root-password secret
$ virt-bootstrap virt-builder://fedora-25 /tmp/foo -f qcow2 --idmap 0:1000:10
$ sudo virt-bootstrap virt-builder://fedora-25 /tmp/foo --idmap 0:1000:10
Tests are also introduced along with the implementation. They cover
creation of root file system and UID/GID mapping for 'dir' and 'qcow2'
output format by mocking the build_image() method to avoid the time
consuming call to virt-builder which might also require network
connection with function which creates dummy disk image.
Setting root password is handled by virt-builder and hence the
introduced test only ensures that the password string is passed
correctly.
---
src/virtBootstrap/sources/__init__.py | 1 +
src/virtBootstrap/sources/virt_builder_source.py | 154 ++++++++++++++
src/virtBootstrap/virt_bootstrap.py | 2 +-
tests/virt_builder_source.py | 243 +++++++++++++++++++++++
4 files changed, 399 insertions(+), 1 deletion(-)
create mode 100644 src/virtBootstrap/sources/virt_builder_source.py
create mode 100644 tests/virt_builder_source.py
diff --git a/src/virtBootstrap/sources/__init__.py b/src/virtBootstrap/sources/__init__.py
index e891e9b..be6b25c 100644
--- a/src/virtBootstrap/sources/__init__.py
+++ b/src/virtBootstrap/sources/__init__.py
@@ -24,3 +24,4 @@ sources - Class definitions which process container image or
from virtBootstrap.sources.file_source import FileSource
from virtBootstrap.sources.docker_source import DockerSource
+from virtBootstrap.sources.virt_builder_source import VirtBuilderSource
diff --git a/src/virtBootstrap/sources/virt_builder_source.py b/src/virtBootstrap/sources/virt_builder_source.py
new file mode 100644
index 0000000..0343091
--- /dev/null
+++ b/src/virtBootstrap/sources/virt_builder_source.py
@@ -0,0 +1,154 @@
+# -*- coding: utf-8 -*-
+# Authors: Radostin Stoyanov <rstoyanov1 at gmail.com>
+#
+# Copyright (C) 2017 Radostin Stoyanov
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+"""
+VirtBuilderSource aim is to extract the root file system from VM image
+build with virt-builder from template.
+"""
+
+import os
+import logging
+import subprocess
+import tempfile
+
+import guestfs
+from virtBootstrap import utils
+
+
+# pylint: disable=invalid-name
+# Create logger
+logger = logging.getLogger(__name__)
+
+
+class VirtBuilderSource(object):
+ """
+ Extract root file system from image build with virt-builder.
+ """
+ def __init__(self, **kwargs):
+ """
+ Create container rootfs by building VM from virt-builder template
+ and extract the rootfs.
+
+ @param uri: Template name
+ @param fmt: Format used to store the output [dir, qcow2]
+ @param uid_map: Mappings for UID of files in rootfs
+ @param gid_map: Mappings for GID of files in rootfs
+ @param root_password: Root password to set in rootfs
+ @param progress: Instance of the progress module
+ """
+ # Parsed URIs:
+ # - "virt-builder:///<template>"
+ # - "virt-builder://<template>"
+ # - "virt-builder:/<template>"
+ self.template = kwargs['uri'].netloc or kwargs['uri'].path[1:]
+ self.output_format = kwargs.get('fmt', utils.DEFAULT_OUTPUT_FORMAT)
+ self.uid_map = kwargs.get('uid_map', [])
+ self.gid_map = kwargs.get('gid_map', [])
+ self.root_password = kwargs.get('root_password', None)
+ self.progress = kwargs['progress'].update_progress
+
+ def build_image(self, output_file):
+ """
+ Build VM from virt-builder template
+ """
+ cmd = ['virt-builder', self.template,
+ '-o', output_file,
+ '--no-network',
+ '--delete', '/dev/*',
+ '--delete', '/boot/*',
+ # Comment out all lines in fstab
+ '--edit', '/etc/fstab:s/^/#/']
+ if self.root_password is not None:
+ cmd += ['--root-password', "password:%s" % self.root_password]
+ self.run_buider(cmd)
+
+ def run_buider(self, cmd):
+ """
+ Execute virt-builder command
+ """
+ subprocess.check_call(cmd)
+
+ def unpack(self, dest):
+ """
+ Build image and extract root file system
+
+ @param dest: Directory path where output files will be stored.
+ """
+
+ with tempfile.NamedTemporaryFile(prefix='bootstrap_') as tmp_file:
+ if self.output_format == 'dir':
+ self.progress("Building image", value=0, logger=logger)
+ self.build_image(tmp_file.name)
+ self.progress("Extracting rootfs", value=50, logger=logger)
+ g = guestfs.GuestFS(python_return_dict=True)
+ g.add_drive_opts(tmp_file.name, readonly=False, format='raw')
+ g.launch()
+
+ # Get the device with file system
+ root_dev = g.inspect_os()
+ if not root_dev:
+ raise Exception("No file system was found")
+ g.mount(root_dev[0], '/')
+
+ # Extract file system to destination directory
+ g.copy_out('/', dest)
+
+ g.umount('/')
+ g.shutdown()
+
+ self.progress("Extraction completed successfully!",
+ value=100, logger=logger)
+ logger.info("Files are stored in: %s", dest)
+
+ elif self.output_format == 'qcow2':
+ output_file = os.path.join(dest, 'layer-0.qcow2')
+
+ self.progress("Building image", value=0, logger=logger)
+ self.build_image(tmp_file.name)
+
+ self.progress("Extracting rootfs", value=50, logger=logger)
+ g = guestfs.GuestFS(python_return_dict=True)
+ g.add_drive_opts(tmp_file.name, readonly=True, format='raw')
+ # Create qcow2 disk image
+ g.disk_create(
+ filename=output_file,
+ format='qcow2',
+ size=os.path.getsize(tmp_file.name)
+ )
+ g.add_drive_opts(output_file, readonly=False, format='qcow2')
+ g.launch()
+ # Get the device with file system
+ root_dev = g.inspect_os()
+ if not root_dev:
+ raise Exception("No file system was found")
+ output_dev = g.list_devices()[1]
+ # Copy the file system to the new qcow2 disk
+ g.copy_device_to_device(root_dev[0], output_dev, sparse=True)
+ g.shutdown()
+
+ # UID/GID mapping
+ if self.uid_map or self.gid_map:
+ logger.info("Mapping UID/GID")
+ utils.map_id_in_image(1, dest, self.uid_map, self.gid_map)
+
+ self.progress("Extraction completed successfully!", value=100,
+ logger=logger)
+ logger.info("Image is stored in: %s", output_file)
+
+ else:
+ raise Exception("Unknown format:" + self.output_format)
diff --git a/src/virtBootstrap/virt_bootstrap.py b/src/virtBootstrap/virt_bootstrap.py
index e387842..fe95f5e 100755
--- a/src/virtBootstrap/virt_bootstrap.py
+++ b/src/virtBootstrap/virt_bootstrap.py
@@ -62,7 +62,7 @@ def get_source(source_type):
Get object which match the source type
"""
try:
- class_name = "%sSource" % source_type.capitalize()
+ class_name = "%sSource" % source_type.title().replace('-', '')
clazz = getattr(sources, class_name)
return clazz
except Exception:
diff --git a/tests/virt_builder_source.py b/tests/virt_builder_source.py
new file mode 100644
index 0000000..3cd812b
--- /dev/null
+++ b/tests/virt_builder_source.py
@@ -0,0 +1,243 @@
+# -*- coding: utf-8 -*-
+# Authors: Radostin Stoyanov <rstoyanov1 at gmail.com>
+#
+# Copyright (C) 2017 Radostin Stoyanov
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+
+"""
+Tests which aim to exercise the extraction of root file system from disk image
+created with virt-builder.
+
+Brief description of these tests:
+1. Create dummy root file system on raw disk image.
+2. Create index file of local repository for virt-builder.
+3. Call bootstrap() with modified virt-builder commnad to use local repository
+ as source.
+4. Check the result.
+"""
+
+import copy
+import platform
+import os
+import shutil
+import tempfile
+import unittest
+import subprocess
+
+import guestfs
+
+from . import virt_bootstrap
+from . import mock
+from . import DEFAULT_FILE_MODE
+from . import ROOTFS_TREE
+from . import Qcow2ImageAccessor
+from . import NOT_ROOT
+
+
+# pylint: disable=invalid-name, too-many-instance-attributes
+class TestVirtBuilderSource(Qcow2ImageAccessor):
+ """
+ Test cases for virt-builder source.
+ """
+
+ def create_local_repository(self):
+ """
+ Create raw disk image with dummy root file system and index file which
+ contains the metadata used by virt-builder.
+ """
+ g = guestfs.GuestFS(python_return_dict=True)
+ g.disk_create(
+ self.image['path'],
+ format=self.image['format'],
+ size=self.image['size']
+ )
+ g.add_drive(
+ self.image['path'],
+ readonly=False,
+ format=self.image['format']
+ )
+ g.launch()
+ g.mkfs('ext2', '/dev/sda')
+ g.mount('/dev/sda', '/')
+ for user in self.rootfs_tree:
+ usr_uid = self.rootfs_tree[user]['uid']
+ usr_gid = self.rootfs_tree[user]['gid']
+
+ for member in self.rootfs_tree[user]['dirs']:
+ dir_name = '/' + member
+ g.mkdir_p(dir_name)
+ g.chown(usr_uid, usr_gid, dir_name)
+
+ for member in self.rootfs_tree[user]['files']:
+ if isinstance(member, tuple):
+ m_name, m_permissions, m_data = member
+ file_name = '/' + m_name
+ g.write(file_name, m_data)
+ g.chmod(m_permissions & 0o777, file_name)
+ else:
+ file_name = '/' + member
+ g.touch(file_name)
+ g.chmod(DEFAULT_FILE_MODE & 0o777, file_name)
+
+ g.chown(usr_uid, usr_gid, file_name)
+
+ # Create index file
+ with open(self.repo_index, 'w') as index_file:
+ index_file.write(
+ '[{template}]\n'
+ 'name=Test\n'
+ 'arch={arch}\n'
+ 'file={filename}\n' # Relative (not real) path must be used.
+ 'format={format}\n'
+ 'expand=/dev/sda\n'
+ 'size={size}\n'.format(**self.image)
+ # The new line at the end of the index file is required.
+ # Otherwise, virt-builder will return "syntax error".
+ )
+
+ def setUp(self):
+ self.rootfs_tree = copy.deepcopy(ROOTFS_TREE)
+
+ self.fmt = None
+ self.uid_map = None
+ self.gid_map = None
+ self.root_password = None
+ self.checked_members = set()
+
+ self.dest_dir = tempfile.mkdtemp('_bootstrap_dest')
+ self.repo_dir = tempfile.mkdtemp('_local_builder_repo')
+ # Set permissions for tmp directories to avoid
+ # "Permission denied" errors from Libvirt.
+ os.chmod(self.repo_dir, 0o755)
+ os.chmod(self.dest_dir, 0o755)
+ self.repo_index = os.path.join(self.repo_dir, 'index')
+
+ self.image = {
+ 'template': 'test',
+ 'filename': 'test.img',
+ 'path': os.path.join(self.repo_dir, 'test.img'),
+ 'format': 'raw',
+ 'size': (1 * 1024 * 1024),
+ 'arch': platform.processor(),
+ }
+ self.create_local_repository()
+
+ def mocked_run_builder(self, cmd):
+ """
+ Modify the virt-builder command to use the dummy disk image
+ and capture the 'stdout'.
+ """
+ subprocess.check_call(
+ cmd + [
+ '--source',
+ 'file://%s' % self.repo_index,
+ '--no-check-signature',
+ '--no-cache'
+ ],
+ stdout=subprocess.PIPE
+ )
+
+ def tearDown(self):
+ """
+ Clean up
+ """
+ shutil.rmtree(self.repo_dir)
+ shutil.rmtree(self.dest_dir)
+
+ def call_bootstrap(self):
+ """
+ Mock out run_builder() with mocked_run_builder() and
+ call bootstrap() method from virtBootstrap.
+ """
+ # By default virt-builder sets random root password which leads to
+ # modification in /etc/shadow file. If we don't test this we simplify
+ # the test by not adding shadow file in our dummy root file system.
+ if not self.root_password:
+ self.rootfs_tree['root']['files'] = ['etc/hosts', 'etc/fstab']
+
+ target = ('virtBootstrap.sources.VirtBuilderSource.run_buider')
+ with mock.patch(target) as m_run_builder:
+ m_run_builder.side_effect = self.mocked_run_builder
+
+ virt_bootstrap.bootstrap(
+ progress_cb=mock.Mock(),
+ uri='virt-builder://%s' % self.image['template'],
+ dest=self.dest_dir,
+ fmt=self.fmt,
+ gid_map=self.gid_map,
+ uid_map=self.uid_map,
+ root_password=self.root_password
+ )
+
+ def test_dir_extract_rootfs(self):
+ """
+ Ensures that the root file system is extracted correctly.
+ """
+ self.fmt = 'dir'
+ self.call_bootstrap()
+ self.check_rootfs(skip_ownership=NOT_ROOT)
+
+ @unittest.skipIf(NOT_ROOT, "Root privileges required")
+ def test_dir_ownership_mapping(self):
+ """
+ Ensures that UID/GID mapping is applied to extracted root file system.
+ """
+ self.fmt = 'dir'
+ self.gid_map = [[1000, 2000, 10], [0, 1000, 10], [500, 500, 10]]
+ self.uid_map = [[1000, 2000, 10], [0, 1000, 10], [500, 500, 10]]
+ self.call_bootstrap()
+ self.apply_mapping()
+ self.check_rootfs()
+
+ def test_dir_setting_root_password(self):
+ """
+ Ensures that password for root is set correctly.
+ """
+ self.root_password = 'my secret root password'
+ self.fmt = 'dir'
+ self.call_bootstrap()
+ self.validate_shadow_file()
+
+ def test_qcow2_build_image(self):
+ """
+ Ensures that the root file system is copied correctly within single
+ partition qcow2 image.
+ """
+ self.fmt = 'qcow2'
+ self.call_bootstrap()
+ self.check_qcow2_images(self.get_image_path())
+
+ def test_qcow2_ownership_mapping(self):
+ """
+ Ensures that UID/GID mapping is applied in qcow2 image "layer-1.qcow2".
+ """
+ self.fmt = 'qcow2'
+ self.gid_map = [[1000, 2000, 10], [0, 1000, 10], [500, 500, 10]]
+ self.uid_map = [[1000, 2000, 10], [0, 1000, 10], [500, 500, 10]]
+ self.call_bootstrap()
+ self.apply_mapping()
+ self.check_qcow2_images(self.get_image_path(1))
+
+ def test_qcow2_setting_root_password(self):
+ """
+ Ensures that the root password is set in the shadow file of
+ "layer-1.qcow2"
+ """
+ self.fmt = 'qcow2'
+ self.root_password = "My secret password"
+ self.call_bootstrap()
+ self.check_image = self.validate_shadow_file_in_image
+ self.check_qcow2_images(self.get_image_path())
--
2.13.5
More information about the virt-tools-list
mailing list