[Fedora-livecd-list] [PATCH] re-factor imgcreate/fs.py module
David Huff
dhuff at redhat.com
Mon Jul 21 21:26:55 UTC 2008
From: Daniel P. Berrange <berrange redhat com>
Most of the change here involves re-factoring the imgcreate/fs.py module.
The SparseExtLoopbackMount, SparseLoopbackMount, LoopbackMount classes all
have the built-in limitation that the image being produced corresponds to
a single filesystem / loop mount. With the disk creator, the image being
produced can be partitioned into multiple chunks, with many filesystems.
Furthermore, not all of them require loopback mounts, as the partitions
themselves are already visible via /dev/mapper/
So this patch separates the roles. There are now classes which deal with
accessing / creating disks:
Disk - generic base for disks
RawDisk - a disk backed by a block device
LoopbackDisk - a disk backed by a file
SparseLoopbackDisk - a disk backed by a sparse file
The 'create' method must make the disk visible as a block device - eg
by calling losetup. For RawDisk, this is obviously a no-op. The 'cleanup'
method must undo the 'create' operation.
There are then classes which deal with mounting things:
Mount - generic base for mounts
DiskMount - able to mount a Disk object
ExtDiskMount - able to format/resize ext3 filesystems when mounting
Patch modified by David Huff <dhuff at redhat.com>
Also fixed comments to use the standard pydoc """ format.
---
imgcreate/fs.py | 346 +++++++++++++++++++++++++++++++++++++++++++++++++++----
1 files changed, 322 insertions(+), 24 deletions(-)
diff --git a/imgcreate/fs.py b/imgcreate/fs.py
index 98c0db4..16cbbba 100644
--- a/imgcreate/fs.py
+++ b/imgcreate/fs.py
@@ -24,6 +24,7 @@ import stat
import subprocess
import random
import string
+import logging
from imgcreate.errors import *
@@ -288,6 +289,303 @@ class SparseExtLoopbackMount(SparseLoopbackMount):
self.resize(size)
return minsize
+class Disk:
+ """
+ With the new disk API the image being produced can be partitioned into
+ multiple chunks, with many filesystems. Furthermore, not all of them
+ require loopback mounts, as the partitions themselves are already
+ visible via /dev/mapper/
+
+ There are now classes which deal with accessing / creating disks:
+
+ Disk - generic base for disks
+ RawDisk - a disk backed by a block device
+ LoopbackDisk - a disk backed by a file
+ SparseLoopbackDisk - a disk backed by a sparse file
+
+ The 'create' method must make the disk visible as a block device - eg
+ by calling losetup. For RawDisk, this is obviously a no-op. The 'cleanup'
+ method must undo the 'create' operation.
+
+ There are then classes which deal with mounting things:
+
+ Mount - generic base for mounts
+ DiskMount - able to mount a Disk object
+ ExtDiskMount - able to format/resize ext3 filesystems when mounting
+ """
+ def __init__(self, size, device = None):
+ self._device = device
+ self._size = size
+
+ def create(self):
+ pass
+
+ def cleanup(self):
+ pass
+
+ def get_device(self):
+ return self._device
+ def set_device(self, path):
+ self._device = path
+ device = property(get_device, set_device)
+
+ def get_size(self):
+ return self._size
+ size = property(get_size)
+
+
+class RawDisk(Disk):
+ def __init__(self, size, device):
+ Disk.__init__(self, size, device)
+
+ def fixed(self):
+ return True
+
+ def exists(self):
+ return True
+
+class LoopbackDisk(Disk):
+ def __init__(self, lofile, size):
+ Disk.__init__(self, size)
+ self.lofile = lofile
+
+ def fixed(self):
+ return False
+
+ def exists(self):
+ return os.path.exists(self.lofile)
+
+ def create(self):
+ if self.device is not None:
+ return
+
+ losetupProc = subprocess.Popen(["/sbin/losetup", "-f"],
+ stdout=subprocess.PIPE)
+ losetupOutput = losetupProc.communicate()[0]
+
+ if losetupProc.returncode:
+ raise MountError("Failed to allocate loop device for '%s'" %
+ self.lofile)
+
+ device = losetupOutput.split()[0]
+
+ logging.debug("Losetup add %s mapping to %s" % (device, self.lofile))
+ rc = subprocess.call(["/sbin/losetup", device, self.lofile])
+ if rc != 0:
+ raise MountError("Failed to allocate loop device for '%s'" %
+ self.lofile)
+ self.device = device
+
+ def cleanup(self):
+ if self.device is None:
+ return
+ logging.debug("Losetup remove %s" % self.device)
+ rc = subprocess.call(["/sbin/losetup", "-d", self.device])
+ self.device = None
+
+
+
+class SparseLoopbackDisk(LoopbackDisk):
+ def __init__(self, lofile, size):
+ LoopbackDisk.__init__(self, lofile, size)
+
+ def expand(self, create = False, size = None):
+ flags = os.O_WRONLY
+ if create:
+ flags |= os.O_CREAT
+ makedirs(os.path.dirname(self.lofile))
+
+ if size is None:
+ size = self.size
+
+ logging.debug("Extending sparse file %s to %d" % (self.lofile, size))
+ fd = os.open(self.lofile, flags)
+
+ os.lseek(fd, size, 0)
+ os.write(fd, '\x00')
+ os.close(fd)
+
+ def truncate(self, size = None):
+ if size is None:
+ size = self.size
+
+ logging.debug("Truncating sparse file %s to %d" % (self.lofile, size))
+ fd = os.open(self.lofile, os.O_WRONLY)
+ os.ftruncate(fd, size)
+ os.close(fd)
+
+ def create(self):
+ self.expand(create = True)
+ LoopbackDisk.create(self)
+
+class Mount:
+ def __init__(self, mountdir):
+ self.mountdir = mountdir
+
+ def cleanup(self):
+ self.unmount()
+
+ def mount(self):
+ pass
+
+ def unmount(self):
+ pass
+
+class DiskMount(Mount):
+ def __init__(self, disk, mountdir, fstype = None, rmmountdir = True):
+ Mount.__init__(self, mountdir)
+
+ self.disk = disk
+ self.fstype = fstype
+ self.rmmountdir = rmmountdir
+
+ self.mounted = False
+ self.rmdir = False
+
+ def cleanup(self):
+ Mount.cleanup(self)
+ self.disk.cleanup()
+
+ def unmount(self):
+ if self.mounted:
+ logging.debug("Unmounting directory %s" % self.mountdir)
+ rc = subprocess.call(["/bin/umount", self.mountdir])
+ if rc == 0:
+ self.mounted = False
+
+ if self.rmdir and not self.mounted:
+ try:
+ os.rmdir(self.mountdir)
+ except OSError, e:
+ pass
+ self.rmdir = False
+
+
+ def __create(self):
+ self.disk.create()
+
+
+ def mount(self):
+ if self.mounted:
+ return
+
+ if not os.path.isdir(self.mountdir):
+ logging.debug("Creating mount point %s" % self.mountdir)
+ os.makedirs(self.mountdir)
+ self.rmdir = self.rmmountdir
+
+ self.__create()
+
+ logging.debug("Mounting %s at %s" % (self.disk.device, self.mountdir))
+ args = [ "/bin/mount", self.disk.device, self.mountdir ]
+ if self.fstype:
+ args.extend(["-t", self.fstype])
+
+ rc = subprocess.call(args)
+ if rc != 0:
+ raise MountError("Failed to mount '%s' to '%s'" %
+ (self.disk.device, self.mountdir))
+
+ self.mounted = True
+
+class ExtDiskMount(DiskMount):
+ def __init__(self, disk, mountdir, fstype, blocksize, fslabel, rmmountdir=True):
+ DiskMount.__init__(self, disk, mountdir, fstype, rmmountdir)
+ self.blocksize = blocksize
+ self.fslabel = fslabel
+
+ def __format_filesystem(self):
+ logging.debug("Formating %s filesystem on %s" % (self.fstype, self.disk.device))
+ rc = subprocess.call(["/sbin/mkfs." + self.fstype,
+ "-F", "-L", self.fslabel,
+ "-m", "1", "-b", str(self.blocksize),
+ self.disk.device])
+ # str(self.disk.size / self.blocksize)])
+ if rc != 0:
+ raise MountError("Error creating %s filesystem" % (self.fstype,))
+ logging.debug("Tuning filesystem on %s" % self.disk.device)
+ subprocess.call(["/sbin/tune2fs", "-c0", "-i0", "-Odir_index",
+ "-ouser_xattr,acl", self.disk.device])
+
+ def __resize_filesystem(self, size = None):
+ current_size = os.stat(self.disk.lofile)[stat.ST_SIZE]
+
+ if size is None:
+ size = self.size
+
+ if size == current_size:
+ return
+
+ if size > current_size:
+ self.expand(size)
+
+ self.__fsck()
+
+ resize2fs(self.disk.lofile, size)
+ return size
+
+ def __create(self):
+ resize = False
+ if not self.disk.fixed() and self.disk.exists():
+ resize = True
+
+ self.disk.create()
+
+ if resize:
+ self.__resize_filesystem()
+ else:
+ self.__format_filesystem()
+
+ def mount(self):
+ self.__create()
+ DiskMount.mount(self)
+
+ def __fsck(self):
+ logging.debug("Checking filesystem %s" % self.disk.lofile)
+ subprocess.call(["/sbin/e2fsck", "-f", "-y", self.disk.lofile])
+
+ def __get_size_from_filesystem(self):
+ def parse_field(output, field):
+ for line in output.split("\n"):
+ if line.startswith(field + ":"):
+ return line[len(field) + 1:].strip()
+
+ raise KeyError("Failed to find field '%s' in output" % field)
+
+ dev_null = os.open("/dev/null", os.O_WRONLY)
+ try:
+ out = subprocess.Popen(['/sbin/dumpe2fs', '-h', self.disk.lofile],
+ stdout = subprocess.PIPE,
+ stderr = dev_null).communicate()[0]
+ finally:
+ os.close(dev_null)
+
+ return int(parse_field(out, "Block count")) * self.blocksize
+
+ def __resize_to_minimal(self):
+ self.__fsck()
+
+ #
+ # Use a binary search to find the minimal size
+ # we can resize the image to
+ #
+ bot = 0
+ top = self.__get_size_from_filesystem()
+ while top != (bot + 1):
+ t = bot + ((top - bot) / 2)
+
+ if not resize2fs(self.disk.lofile, t):
+ top = t
+ else:
+ bot = t
+ return top
+
+ def resparse(self, size = None):
+ self.cleanup()
+ minsize = self.__resize_to_minimal()
+ self.disk.truncate(minsize)
+ return minsize
+
class DeviceMapperSnapshot(object):
def __init__(self, imgloop, cowloop):
self.imgloop = imgloop
@@ -306,8 +604,8 @@ class DeviceMapperSnapshot(object):
if self.__created:
return
- self.imgloop.loopsetup()
- self.cowloop.loopsetup()
+ self.imgloop.create()
+ self.cowloop.create()
self.__name = "imgcreate-%d-%d" % (os.getpid(),
random.randint(0, 2**16))
@@ -315,8 +613,8 @@ class DeviceMapperSnapshot(object):
size = os.stat(self.imgloop.lofile)[stat.ST_SIZE]
table = "0 %d snapshot %s %s p 8" % (size / 512,
- self.imgloop.loopdev,
- self.cowloop.loopdev)
+ self.imgloop.device,
+ self.cowloop.device)
args = ["/sbin/dmsetup", "create", self.__name, "--table", table]
if subprocess.call(args) != 0:
@@ -365,32 +663,31 @@ class DeviceMapperSnapshot(object):
except ValueError:
raise SnapshotError("Failed to parse dmsetup status: " + out)
-#
-# Builds a copy-on-write image which can be used to
-# create a device-mapper snapshot of an image where
-# the image's filesystem is as small as possible
-#
-# The steps taken are:
-# 1) Create a sparse COW
-# 2) Loopback mount the image and the COW
-# 3) Create a device-mapper snapshot of the image
-# using the COW
-# 4) Resize the filesystem to the minimal size
-# 5) Determine the amount of space used in the COW
-# 6) Restroy the device-mapper snapshot
-# 7) Truncate the COW, removing unused space
-# 8) Create a squashfs of the COW
-#
def create_image_minimizer(path, image, minimal_size):
- imgloop = LoopbackMount(image, "None")
+ """
+ Builds a copy-on-write image which can be used to
+ create a device-mapper snapshot of an image where
+ the image's filesystem is as small as possible
+
+ The steps taken are:
+ 1) Create a sparse COW
+ 2) Loopback mount the image and the COW
+ 3) Create a device-mapper snapshot of the image
+ using the COW
+ 4) Resize the filesystem to the minimal size
+ 5) Determine the amount of space used in the COW
+ 6) Restroy the device-mapper snapshot
+ 7) Truncate the COW, removing unused space
+ 8) Create a squashfs of the COW
+ """
+ imgloop = LoopbackDisk(image, None) # Passing bogus size - doesn't matter
- cowloop = SparseLoopbackMount(os.path.join(os.path.dirname(path), "osmin"),
- None, 64L * 1024L * 1024L)
+ cowloop = SparseLoopbackDisk(os.path.join(os.path.dirname(path), "osmin"),
+ 64L * 1024L * 1024L)
snapshot = DeviceMapperSnapshot(imgloop, cowloop)
try:
- cowloop.create()
snapshot.create()
resize2fs(snapshot.path, minimal_size)
@@ -404,3 +701,4 @@ def create_image_minimizer(path, image, minimal_size):
mksquashfs(cowloop.lofile, path)
os.unlink(cowloop.lofile)
+
--
1.5.4.1
More information about the Fedora-livecd-list
mailing list