[libvirt] [sandbox PATCH 01/11] Add virt-sandbox-image

Eren Yagdiran erenyagdiran at gmail.com
Thu Jul 23 15:57:27 UTC 2015


From: Daniel P Berrange <berrange at redhat.com>

virt-sandbox-image.py is a python script that lets you download Docker
images easily. It is a proof of concept code and consumes Docker Rest API.
---
 po/POTFILES.in                           |   1 +
 virt-sandbox-image/virt-sandbox-image.py | 397 +++++++++++++++++++++++++++++++
 2 files changed, 398 insertions(+)
 create mode 100644 virt-sandbox-image/virt-sandbox-image.py

diff --git a/po/POTFILES.in b/po/POTFILES.in
index afcb050..7204112 100644
--- a/po/POTFILES.in
+++ b/po/POTFILES.in
@@ -11,3 +11,4 @@ libvirt-sandbox/libvirt-sandbox-context-interactive.c
 libvirt-sandbox/libvirt-sandbox-init-common.c
 libvirt-sandbox/libvirt-sandbox-rpcpacket.c
 libvirt-sandbox/libvirt-sandbox-util.c
+virt-sandbox-image/virt-sandbox-image.py
diff --git a/virt-sandbox-image/virt-sandbox-image.py b/virt-sandbox-image/virt-sandbox-image.py
new file mode 100644
index 0000000..4da3dde
--- /dev/null
+++ b/virt-sandbox-image/virt-sandbox-image.py
@@ -0,0 +1,397 @@
+#!/usr/bin/python -Es
+#
+# Authors: Daniel P. Berrange <berrange at redhat.com>
+#
+# Copyright (C) 2013 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+#
+
+import argparse
+import gettext
+import hashlib
+import json
+import os
+import os.path
+import shutil
+import sys
+import urllib2
+import subprocess
+
+default_index_server = "index.docker.io"
+default_template_dir = "/var/lib/libvirt/templates"
+
+debug = True
+verbose = True
+
+gettext.bindtextdomain("libvirt-sandbox", "/usr/share/locale")
+gettext.textdomain("libvirt-sandbox")
+try:
+    gettext.install("libvirt-sandbox",
+                    localedir="/usr/share/locale",
+                    unicode=False,
+                    codeset = 'utf-8')
+except IOError:
+    import __builtin__
+    __builtin__.__dict__['_'] = unicode
+
+
+def debug(msg):
+    sys.stderr.write(msg)
+
+def info(msg):
+    sys.stdout.write(msg)
+
+def get_url(server, path, headers):
+    url = "https://" + server + path
+    debug("  Fetching %s..." % url)
+    
+    req = urllib2.Request(url=url)
+
+    if json:
+        req.add_header("Accept", "application/json")
+
+    for h in headers.keys():
+        req.add_header(h, headers[h])
+
+    return urllib2.urlopen(req)
+
+def get_json(server, path, headers):
+    try:
+        res = get_url(server, path, headers)
+        data = json.loads(res.read())
+        debug("OK\n")
+        return (data, res)
+    except Exception, e:
+        debug("FAIL %s\n" % str(e))
+        raise
+
+def save_data(server, path, headers, dest, checksum=None, datalen=None):
+    try:
+        res = get_url(server, path, headers)
+
+        csum = None
+        if checksum is not None:
+            csum = hashlib.sha256()
+
+        pattern = [".", "o", "O", "o"]
+        patternIndex = 0
+        donelen = 0
+
+        with open(dest, "w") as f:
+            while 1:
+                buf = res.read(1024*64)
+                if not buf:
+                    break
+                if csum is not None:
+                    csum.update(buf)
+                f.write(buf)
+
+                if datalen is not None:
+                    donelen = donelen + len(buf)
+                    debug("\x1b[s%s (%5d Kb of %5d Kb)\x1b8" % (
+                        pattern[patternIndex], (donelen/1024), (datalen/1024)
+                    ))
+                    patternIndex = (patternIndex + 1) % 4
+
+        debug("\x1b[K")
+        if csum is not None:
+            csumstr = "sha256:" + csum.hexdigest()
+            if csumstr != checksum:
+                debug("FAIL checksum '%s' does not match '%s'" % (csumstr, checksum))
+                os.remove(dest)
+                raise IOError("Checksum '%s' for data does not match '%s'" % (csumstr, checksum))
+    
+        debug("OK\n")
+        return res
+    except Exception, e:
+        debug("FAIL %s\n" % str(e))
+        raise
+
+
+def download_template(name, server, destdir):
+    tag = "latest"
+
+    offset = name.find(':')
+    if offset != -1:
+        tag = name[offset + 1:]
+        name = name[0:offset]
+
+    # First we must ask the index server about the image name. THe
+    # index server will return an auth token we can use when talking
+    # to the registry server. We need this token even when anonymous
+    try:
+        (data, res) = get_json(server, "/v1/repositories/" + name + "/images",
+                               {"X-Docker-Token": "true"})
+    except urllib2.HTTPError, e:
+        raise ValueError(["Image '%s' does not exist" % name])
+
+    registryserver = res.info().getheader('X-Docker-Endpoints')
+    token = res.info().getheader('X-Docker-Token')
+    checksums = {}
+    for layer in data:
+        pass
+        # XXX Checksums here don't appear to match the data in
+        # image download later. Find out what these are sums of
+        #checksums[layer["id"]] = layer["checksum"]
+
+    # Now we ask the registry server for the list of tags associated
+    # with the image. Tags usually reflect some kind of version of
+    # the image, but they aren't officially "versions". There is
+    # always a "latest" tag which is the most recent upload
+    #
+    # We must pass in the auth token from the index server. This
+    # token can only be used once, and we're given a cookie back
+    # in return to use for later RPC calls.
+    (data, res) = get_json(registryserver, "/v1/repositories/" + name + "/tags",
+                           { "Authorization": "Token " + token })
+
+    cookie = res.info().getheader('Set-Cookie')
+
+    if not tag in data:
+        raise ValueError(["Tag '%s' does not exist for image '%s'" % (tag, name)])
+    imagetagid = data[tag]
+
+    # Only base images are self-contained, most images reference one
+    # or more parents, in a linear stack. Here we are getting the list
+    # of layers for the image with the tag we used.
+    (data, res) = get_json(registryserver, "/v1/images/" + imagetagid + "/ancestry",
+                           { "Cookie": cookie })
+
+    if data[0] != imagetagid:
+        raise ValueError(["Expected first layer id '%s' to match image id '%s'",
+                          data[0], imagetagid])
+
+    try:
+        createdFiles = []
+        createdDirs = []
+
+        for layerid in data:
+            templatedir = destdir + "/" + layerid
+            if not os.path.exists(templatedir):
+                os.mkdir(templatedir)
+                createdDirs.append(templatedir)
+
+            jsonfile = templatedir + "/template.json"
+            datafile = templatedir + "/template.tar.gz"
+
+            if not os.path.exists(jsonfile) or not os.path.exists(datafile):
+                # The '/json' URL gives us some metadata about the layer
+                res = save_data(registryserver, "/v1/images/" + layerid + "/json",
+                                { "Cookie": cookie }, jsonfile)
+                createdFiles.append(jsonfile)
+                layersize = int(res.info().getheader("x-docker-size"))
+
+                datacsum = None
+                if layerid in checksums:
+                    datacsum = checksums[layerid]
+
+                # and the '/layer' URL is the actual payload, provided
+                # as a tar.gz archive
+                save_data(registryserver, "/v1/images/" + layerid + "/layer",
+                          { "Cookie": cookie }, datafile, datacsum, layersize)
+                createdFiles.append(datafile)
+
+        # Strangely the 'json' data for a layer doesn't include
+        # its actual name, so we save that in a json file of our own
+        index = {
+            "name": name,
+        }
+
+        indexfile = destdir + "/" + imagetagid + "/index.json"
+        with open(indexfile, "w") as f:
+            f.write(json.dumps(index))
+    except Exception, e:
+        for f in createdFiles:
+            try:
+                os.remove(f)
+            except:
+                pass
+        for d in createdDirs:
+            try:
+                os.rmdir(d)
+            except:
+                pass
+
+
+def delete_template(name, destdir):
+    imageusage = {}
+    imageparent = {}
+    imagenames = {}
+    imagedirs = os.listdir(destdir)
+    for imagetagid in imagedirs:
+        indexfile = destdir + "/" + imagetagid + "/index.json"
+        if os.path.exists(indexfile):
+            with open(indexfile, "r") as f:
+                index = json.load(f)
+            imagenames[index["name"]] = imagetagid
+        jsonfile = destdir + "/" + imagetagid + "/template.json"
+        if os.path.exists(jsonfile):
+            with open(jsonfile, "r") as f:
+                template = json.load(f)
+
+            parent = template.get("parent", None)
+            if parent:
+                if parent not in imageusage:
+                    imageusage[parent] = []
+                imageusage[parent].append(imagetagid)
+                imageparent[imagetagid] = parent
+
+    if not name in imagenames:
+        raise ValueError(["Image %s does not exist locally" % name])
+
+    imagetagid = imagenames[name]
+    while imagetagid != None:
+        debug("Remove %s\n" %  imagetagid)
+        parent = imageparent.get(imagetagid, None)
+
+        indexfile = destdir + "/" + imagetagid + "/index.json"
+        if os.path.exists(indexfile):
+            os.remove(indexfile)
+        jsonfile = destdir + "/" + imagetagid + "/template.json"
+        if os.path.exists(jsonfile):
+            os.remove(jsonfile)
+        datafile = destdir + "/" + imagetagid + "/template.tar.gz"
+        if os.path.exists(datafile):
+            os.remove(datafile)
+        imagedir = destdir + "/" + imagetagid
+        os.rmdir(imagedir)
+
+        if parent:
+            if len(imageusage[parent]) != 1:
+                debug("Parent %s is shared\n" % parent)
+                parent = None
+        imagetagid = parent
+
+
+def get_image_list(name, destdir):
+    imageparent = {}
+    imagenames = {}
+    imagedirs = os.listdir(destdir)
+    for imagetagid in imagedirs:
+        indexfile = destdir + "/" + imagetagid + "/index.json"
+        if os.path.exists(indexfile):
+            with open(indexfile, "r") as f:
+                index = json.load(f)
+            imagenames[index["name"]] = imagetagid
+        jsonfile = destdir + "/" + imagetagid + "/template.json"
+        if os.path.exists(jsonfile):
+            with open(jsonfile, "r") as f:
+                template = json.load(f)
+
+            parent = template.get("parent", None)
+            if parent:
+                imageparent[imagetagid] = parent
+
+    if not name in imagenames:
+        raise ValueError(["Image %s does not exist locally" % name])
+
+    imagetagid = imagenames[name]
+    imagelist = []
+    while imagetagid != None:
+        imagelist.append(imagetagid)
+        parent = imageparent.get(imagetagid, None)
+        imagetagid = parent
+
+    return imagelist
+
+def create_template(name, imagepath, format, destdir):
+    if not format in ["qcow2"]:
+        raise ValueError(["Unsupported image format %s" % format])
+
+    imagelist = get_image_list(name, destdir)
+    imagelist.reverse()
+
+    parentImage = None
+    for imagetagid in imagelist:
+        templateImage = destdir + "/" + imagetagid + "/template." + format
+        cmd = ["qemu-img", "create", "-f", "qcow2"]
+        if parentImage is not None:
+            cmd.append("-o")
+            cmd.append("backing_fmt=qcow2,backing_file=%s" % parentImage)
+        cmd.append(templateImage)
+        if parentImage is None:
+            cmd.append("10G")
+        debug("Run %s\n" % " ".join(cmd))
+        subprocess.call(cmd)
+        parentImage = templateImage
+        
+
+def download(args):
+    info("Downloading %s from %s to %s\n" % (args.name, default_index_server, default_template_dir))
+    download_template(args.name, default_index_server, default_template_dir)
+
+def delete(args):
+    info("Deleting %s from %s\n" % (args.name, default_template_dir))
+    delete_template(args.name, default_template_dir)
+
+def create(args):
+    info("Creating %s from %s in format %s\n" % (args.imagepath, args.name, args.format))
+    create_template(args.name, args.imagepath, args.format, default_template_dir)
+
+def requires_name(parser):
+    parser.add_argument("name",
+                        help=_("name of the template"))
+
+def gen_download_args(subparser):
+    parser = subparser.add_parser("download",
+                                   help=_("Download template data"))
+    requires_name(parser)
+    parser.set_defaults(func=download)
+
+def gen_delete_args(subparser):
+    parser = subparser.add_parser("delete",
+                                   help=_("Delete template data"))
+    requires_name(parser)
+    parser.set_defaults(func=delete)
+
+def gen_create_args(subparser):
+    parser = subparser.add_parser("create",
+                                   help=_("Create image from template data"))
+    requires_name(parser)
+    parser.add_argument("imagepath",
+                        help=_("path for image"))
+    parser.add_argument("format",
+                        help=_("format"))
+    parser.set_defaults(func=create)
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Sandbox Container Image Tool')
+
+    subparser = parser.add_subparsers(help=_("commands"))
+    gen_download_args(subparser)
+    gen_delete_args(subparser)
+    gen_create_args(subparser)
+
+    try:
+        args = parser.parse_args()
+        args.func(args)
+        sys.exit(0)
+    except KeyboardInterrupt, e:
+        sys.exit(0)
+    except ValueError, e:
+        for line in e:
+            for l in line:
+                sys.stderr.write("%s: %s\n" % (sys.argv[0], l))
+        sys.stderr.flush()
+        sys.exit(1)
+    except IOError, e:
+        sys.stderr.write("%s: %s: %s\n" % (sys.argv[0], e.filename, e.reason))
+        sys.stderr.flush()
+        sys.exit(1)
+    except OSError, e:
+        sys.stderr.write("%s: %s\n" % (sys.argv[0], e))
+        sys.stderr.flush()
+        sys.exit(1)
-- 
2.1.0




More information about the libvir-list mailing list