[libvirt] [sandbox PATCH 04/11] Image: Add download function

Eren Yagdiran erenyagdiran at gmail.com
Thu Jul 23 15:57:30 UTC 2015


Refactor download function from virt-sandbox-image to use
the newly introduced Source abstract class. The docker-specific
download code is moved to a new DockerSource class.
---
 virt-sandbox-image/sources/DockerSource.py | 193 +++++++++++++++++++++++++++
 virt-sandbox-image/sources/Source.py       |   5 +
 virt-sandbox-image/virt-sandbox-image.py   | 202 ++++-------------------------
 3 files changed, 225 insertions(+), 175 deletions(-)
 create mode 100644 virt-sandbox-image/sources/DockerSource.py

diff --git a/virt-sandbox-image/sources/DockerSource.py b/virt-sandbox-image/sources/DockerSource.py
new file mode 100644
index 0000000..5bcd613
--- /dev/null
+++ b/virt-sandbox-image/sources/DockerSource.py
@@ -0,0 +1,193 @@
+#!/usr/bin/python
+
+from Source import Source
+import urllib2
+import sys
+import json
+import traceback
+import os
+import subprocess
+import shutil
+
+class DockerSource(Source):
+    default_index_server = "index.docker.io"
+    default_template_dir = "/var/lib/libvirt/templates"
+    default_image_path = "/var/lib/libvirt/templates"
+    default_disk_format = "qcow2"
+
+    www_auth_username = None
+    www_auth_password = None
+
+    def __init__(self,server="index.docker.io",destdir="/var/lib/libvirt/templates"):
+        self.default_index_server = server
+        self.default_template_dir = destdir
+
+    def download_template(self,**args):
+        name = args['name']
+        registry = args['registry'] if args['registry'] is not None else self.default_index_server
+        username = args['username']
+        password = args['password']
+        templatedir = args['templatedir'] if args['templatedir'] is not None else self.default_template_dir
+        self.__download_template(name,registry,username,password,templatedir)
+
+    def __download_template(self,name, server,username,password,destdir):
+
+        if username is not None:
+            self.www_auth_username = username
+            self.www_auth_password = password
+
+        tag = "latest"
+        offset = name.find(':')
+        if offset != -1:
+            tag = name[offset + 1:]
+            name = name[0:offset]
+        try:
+            (data, res) = self.__get_json(server, "/v1/repositories/" + name + "/images",
+                               {"X-Docker-Token": "true"})
+        except urllib2.HTTPError, e:
+            raise ValueError(["Image '%s' does not exist" % name])
+
+        registryserver = res.info().getheader('X-Docker-Endpoints')
+        token = res.info().getheader('X-Docker-Token')
+        checksums = {}
+        for layer in data:
+            pass
+        (data, res) = self.__get_json(registryserver, "/v1/repositories/" + name + "/tags",
+                           { "Authorization": "Token " + token })
+
+        cookie = res.info().getheader('Set-Cookie')
+
+        if not tag in data:
+            raise ValueError(["Tag '%s' does not exist for image '%s'" % (tag, name)])
+        imagetagid = data[tag]
+
+        (data, res) = self.__get_json(registryserver, "/v1/images/" + imagetagid + "/ancestry",
+                               { "Authorization": "Token "+token })
+
+        if data[0] != imagetagid:
+            raise ValueError(["Expected first layer id '%s' to match image id '%s'",
+                          data[0], imagetagid])
+
+        try:
+            createdFiles = []
+            createdDirs = []
+
+            for layerid in data:
+                templatedir = destdir + "/" + layerid
+                if not os.path.exists(templatedir):
+                    os.mkdir(templatedir)
+                    createdDirs.append(templatedir)
+
+                jsonfile = templatedir + "/template.json"
+                datafile = templatedir + "/template.tar.gz"
+
+                if not os.path.exists(jsonfile) or not os.path.exists(datafile):
+                    res = self.__save_data(registryserver, "/v1/images/" + layerid + "/json",
+                                { "Authorization": "Token " + token }, jsonfile)
+                    createdFiles.append(jsonfile)
+
+                    layersize = int(res.info().getheader("Content-Length"))
+
+                    datacsum = None
+                    if layerid in checksums:
+                        datacsum = checksums[layerid]
+
+                    self.__save_data(registryserver, "/v1/images/" + layerid + "/layer",
+                          { "Authorization": "Token "+token }, datafile, datacsum, layersize)
+                    createdFiles.append(datafile)
+
+            index = {
+                "name": name,
+            }
+
+            indexfile = destdir + "/" + imagetagid + "/index.json"
+            print("Index file " + indexfile)
+            with open(indexfile, "w") as f:
+                 f.write(json.dumps(index))
+        except Exception as e:
+            traceback.print_exc()
+            for f in createdFiles:
+                try:
+                    os.remove(f)
+                except:
+                    pass
+            for d in createdDirs:
+                try:
+                    shutil.rmtree(d)
+                except:
+                    pass
+    def __save_data(self,server, path, headers, dest, checksum=None, datalen=None):
+        try:
+            res = self.__get_url(server, path, headers)
+
+            csum = None
+            if checksum is not None:
+                csum = hashlib.sha256()
+
+            pattern = [".", "o", "O", "o"]
+            patternIndex = 0
+            donelen = 0
+
+            with open(dest, "w") as f:
+                while 1:
+                    buf = res.read(1024*64)
+                    if not buf:
+                        break
+                    if csum is not None:
+                        csum.update(buf)
+                    f.write(buf)
+
+                    if datalen is not None:
+                        donelen = donelen + len(buf)
+                        debug("\x1b[s%s (%5d Kb of %5d Kb)\x1b8" % (
+                            pattern[patternIndex], (donelen/1024), (datalen/1024)
+                        ))
+                        patternIndex = (patternIndex + 1) % 4
+
+            debug("\x1b[K")
+            if csum is not None:
+                csumstr = "sha256:" + csum.hexdigest()
+                if csumstr != checksum:
+                    debug("FAIL checksum '%s' does not match '%s'" % (csumstr, checksum))
+                    os.remove(dest)
+                    raise IOError("Checksum '%s' for data does not match '%s'" % (csumstr, checksum))
+            debug("OK\n")
+            return res
+        except Exception, e:
+            debug("FAIL %s\n" % str(e))
+            raise
+
+    def __get_url(self,server, path, headers):
+        url = "https://" + server + path
+        debug("Fetching %s..." % url)
+
+        req = urllib2.Request(url=url)
+        if json:
+            req.add_header("Accept", "application/json")
+        for h in headers.keys():
+            req.add_header(h, headers[h])
+
+        #www Auth header starts
+        if self.www_auth_username is not None:
+            base64string = base64.encodestring('%s:%s' % (self.www_auth_username, self.www_auth_password)).replace('\n', '')
+            req.add_header("Authorization", "Basic %s" % base64string)
+        #www Auth header finish
+
+        return urllib2.urlopen(req)
+
+    def __get_json(self,server, path, headers):
+        try:
+            res = self.__get_url(server, path, headers)
+            data = json.loads(res.read())
+            debug("OK\n")
+            return (data, res)
+        except Exception, e:
+            debug("FAIL %s\n" % str(e))
+            raise
+
+def debug(msg):
+    sys.stderr.write(msg)
+
+if __name__ == "sources.DockerSource":
+    from __builtin__ import add_hook
+    add_hook('docker',DockerSource)
diff --git a/virt-sandbox-image/sources/Source.py b/virt-sandbox-image/sources/Source.py
index cfc75d3..99f44bb 100644
--- a/virt-sandbox-image/sources/Source.py
+++ b/virt-sandbox-image/sources/Source.py
@@ -6,3 +6,8 @@ class Source():
     __metaclass__ = ABCMeta
     def __init__(self):
         pass
+
+    @abstractmethod
+    def download_template(self,**args):
+        pass
+
diff --git a/virt-sandbox-image/virt-sandbox-image.py b/virt-sandbox-image/virt-sandbox-image.py
index 99ed46e..1392e8f 100644
--- a/virt-sandbox-image/virt-sandbox-image.py
+++ b/virt-sandbox-image/virt-sandbox-image.py
@@ -40,7 +40,6 @@ verbose = True
 
 sys.dont_write_bytecode = True
 
-
 ##Hook mechanism starts##
 import __builtin__
 from sources.Source import Source
@@ -84,178 +83,6 @@ def debug(msg):
 def info(msg):
     sys.stdout.write(msg)
 
-def get_url(server, path, headers):
-    url = "https://" + server + path
-    debug("  Fetching %s..." % url)
-    
-    req = urllib2.Request(url=url)
-
-    if json:
-        req.add_header("Accept", "application/json")
-
-    for h in headers.keys():
-        req.add_header(h, headers[h])
-
-    return urllib2.urlopen(req)
-
-def get_json(server, path, headers):
-    try:
-        res = get_url(server, path, headers)
-        data = json.loads(res.read())
-        debug("OK\n")
-        return (data, res)
-    except Exception, e:
-        debug("FAIL %s\n" % str(e))
-        raise
-
-def save_data(server, path, headers, dest, checksum=None, datalen=None):
-    try:
-        res = get_url(server, path, headers)
-
-        csum = None
-        if checksum is not None:
-            csum = hashlib.sha256()
-
-        pattern = [".", "o", "O", "o"]
-        patternIndex = 0
-        donelen = 0
-
-        with open(dest, "w") as f:
-            while 1:
-                buf = res.read(1024*64)
-                if not buf:
-                    break
-                if csum is not None:
-                    csum.update(buf)
-                f.write(buf)
-
-                if datalen is not None:
-                    donelen = donelen + len(buf)
-                    debug("\x1b[s%s (%5d Kb of %5d Kb)\x1b8" % (
-                        pattern[patternIndex], (donelen/1024), (datalen/1024)
-                    ))
-                    patternIndex = (patternIndex + 1) % 4
-
-        debug("\x1b[K")
-        if csum is not None:
-            csumstr = "sha256:" + csum.hexdigest()
-            if csumstr != checksum:
-                debug("FAIL checksum '%s' does not match '%s'" % (csumstr, checksum))
-                os.remove(dest)
-                raise IOError("Checksum '%s' for data does not match '%s'" % (csumstr, checksum))
-    
-        debug("OK\n")
-        return res
-    except Exception, e:
-        debug("FAIL %s\n" % str(e))
-        raise
-
-
-def download_template(name, server, destdir):
-    tag = "latest"
-
-    offset = name.find(':')
-    if offset != -1:
-        tag = name[offset + 1:]
-        name = name[0:offset]
-
-    # First we must ask the index server about the image name. THe
-    # index server will return an auth token we can use when talking
-    # to the registry server. We need this token even when anonymous
-    try:
-        (data, res) = get_json(server, "/v1/repositories/" + name + "/images",
-                               {"X-Docker-Token": "true"})
-    except urllib2.HTTPError, e:
-        raise ValueError(["Image '%s' does not exist" % name])
-
-    registryserver = res.info().getheader('X-Docker-Endpoints')
-    token = res.info().getheader('X-Docker-Token')
-    checksums = {}
-    for layer in data:
-        pass
-        # XXX Checksums here don't appear to match the data in
-        # image download later. Find out what these are sums of
-        #checksums[layer["id"]] = layer["checksum"]
-
-    # Now we ask the registry server for the list of tags associated
-    # with the image. Tags usually reflect some kind of version of
-    # the image, but they aren't officially "versions". There is
-    # always a "latest" tag which is the most recent upload
-    #
-    # We must pass in the auth token from the index server. This
-    # token can only be used once, and we're given a cookie back
-    # in return to use for later RPC calls.
-    (data, res) = get_json(registryserver, "/v1/repositories/" + name + "/tags",
-                           { "Authorization": "Token " + token })
-
-    cookie = res.info().getheader('Set-Cookie')
-
-    if not tag in data:
-        raise ValueError(["Tag '%s' does not exist for image '%s'" % (tag, name)])
-    imagetagid = data[tag]
-
-    # Only base images are self-contained, most images reference one
-    # or more parents, in a linear stack. Here we are getting the list
-    # of layers for the image with the tag we used.
-    (data, res) = get_json(registryserver, "/v1/images/" + imagetagid + "/ancestry",
-                           { "Authorization": "Token " + token })
-
-    if data[0] != imagetagid:
-        raise ValueError(["Expected first layer id '%s' to match image id '%s'",
-                          data[0], imagetagid])
-
-    try:
-        createdFiles = []
-        createdDirs = []
-
-        for layerid in data:
-            templatedir = destdir + "/" + layerid
-            if not os.path.exists(templatedir):
-                os.mkdir(templatedir)
-                createdDirs.append(templatedir)
-
-            jsonfile = templatedir + "/template.json"
-            datafile = templatedir + "/template.tar.gz"
-
-            if not os.path.exists(jsonfile) or not os.path.exists(datafile):
-                # The '/json' URL gives us some metadata about the layer
-                res = save_data(registryserver, "/v1/images/" + layerid + "/json",
-                                { "Authorization": "Token " + token }, jsonfile)
-                createdFiles.append(jsonfile)
-                layersize = int(res.info().getheader("Content-Length"))
-
-                datacsum = None
-                if layerid in checksums:
-                    datacsum = checksums[layerid]
-
-                # and the '/layer' URL is the actual payload, provided
-                # as a tar.gz archive
-                save_data(registryserver, "/v1/images/" + layerid + "/layer",
-                          { "Authorization": "Token " + token }, datafile, datacsum, layersize)
-                createdFiles.append(datafile)
-
-        # Strangely the 'json' data for a layer doesn't include
-        # its actual name, so we save that in a json file of our own
-        index = {
-            "name": name,
-        }
-
-        indexfile = destdir + "/" + imagetagid + "/index.json"
-        with open(indexfile, "w") as f:
-            f.write(json.dumps(index))
-    except Exception, e:
-        for f in createdFiles:
-            try:
-                os.remove(f)
-            except:
-                pass
-        for d in createdDirs:
-            try:
-                os.rmdir(d)
-            except:
-                pass
-
-
 def delete_template(name, destdir):
     imageusage = {}
     imageparent = {}
@@ -360,8 +187,16 @@ def create_template(name, imagepath, format, destdir):
         
 
 def download(args):
-    info("Downloading %s from %s to %s\n" % (args.name, default_index_server, default_template_dir))
-    download_template(args.name, default_index_server, default_template_dir)
+    try:
+        dynamic_source_loader(args.source).download_template(name=args.name,
+                                                             registry=args.registry,
+                                                             username=args.username,
+                                                             password=args.password,
+                                                             templatedir=args.template_dir)
+    except IOError,e:
+        print "Source %s cannot be found in given path" %args.source
+    except Exception,e:
+        print "Download Error %s" % str(e)
 
 def delete(args):
     info("Deleting %s from %s\n" % (args.name, default_template_dir))
@@ -375,10 +210,27 @@ def requires_name(parser):
     parser.add_argument("name",
                         help=_("name of the template"))
 
+def requires_source(parser):
+    parser.add_argument("-s","--source",
+                        default="docker",
+                        help=_("name of the template"))
+
+def requires_auth_conn(parser):
+    parser.add_argument("-r","--registry",
+                        help=_("Url of the custom registry"))
+    parser.add_argument("-u","--username",
+                        help=_("Username for the custom registry"))
+    parser.add_argument("-p","--password",
+                        help=_("Password for the custom registry"))
+    parser.add_argument("-t","--template-dir",
+                        help=_("Template directory for saving templates"))
+
 def gen_download_args(subparser):
     parser = subparser.add_parser("download",
                                    help=_("Download template data"))
+    requires_source(parser)
     requires_name(parser)
+    requires_auth_conn(parser)
     parser.set_defaults(func=download)
 
 def gen_delete_args(subparser):
-- 
2.1.0




More information about the libvir-list mailing list