[Libguestfs] [PATCH v2 3/3] v2v: Add -o rhv-upload output mode.
Tomáš Golembiovský
tgolembi at redhat.com
Wed Feb 28 14:00:15 UTC 2018
On Tue, 27 Feb 2018 21:33:24 +0000
"Richard W.M. Jones" <rjones at redhat.com> wrote:
> PROBLEMS:
> - Spools to a temporary disk
> - Target cluster
Seeing how you parse and reconstruct the URI we may encode the cluster
into the fragment or query part.
> - Delete disks on failure, or rename disks on success?
> - Handling of sparseness in raw format disks
> - Manual page
>
> This adds a new output mode to virt-v2v. virt-v2v -o rhv-upload
> streams images directly to an oVirt or RHV >= 4 Data Domain using the
> oVirt SDK v4. It is more efficient than -o rhv because it does not
> need to go via the Export Storage Domain, and is possible for humans
> to use unlike -o vdsm.
>
> The implementation uses the Python SDK by running snippets of Python
> code to interact with the ‘ovirtsdk4’ module. It requires both Python 3
> and the Python SDK v4 to be installed at run time (these are not,
> however, new dependencies of virt-v2v since most people wouldn't have
> them).
> ---
> v2v/Makefile.am | 2 +
> v2v/cmdline.ml | 38 ++++
> v2v/output_rhv_upload.ml | 451 ++++++++++++++++++++++++++++++++++++++++++++++
> v2v/output_rhv_upload.mli | 27 +++
> 4 files changed, 518 insertions(+)
>
> diff --git a/v2v/Makefile.am b/v2v/Makefile.am
> index c2eb31097..cd44dfc2a 100644
> --- a/v2v/Makefile.am
> +++ b/v2v/Makefile.am
> @@ -64,6 +64,7 @@ SOURCES_MLI = \
> output_null.mli \
> output_qemu.mli \
> output_rhv.mli \
> + output_rhv_upload.mli \
> output_vdsm.mli \
> parse_ovf_from_ova.mli \
> parse_libvirt_xml.mli \
> @@ -116,6 +117,7 @@ SOURCES_ML = \
> output_local.ml \
> output_qemu.ml \
> output_rhv.ml \
> + output_rhv_upload.ml \
> output_vdsm.ml \
> inspect_source.ml \
> target_bus_assignment.ml \
> diff --git a/v2v/cmdline.ml b/v2v/cmdline.ml
> index d725ae022..c53d1703b 100644
> --- a/v2v/cmdline.ml
> +++ b/v2v/cmdline.ml
> @@ -65,6 +65,8 @@ let parse_cmdline () =
> let output_password = ref None in
> let output_storage = ref None in
> let password_file = ref None in
> + let rhv_cafile = ref None in
> + let rhv_direct = ref false in
> let vddk_config = ref None in
> let vddk_cookie = ref None in
> let vddk_libdir = ref None in
> @@ -143,6 +145,8 @@ let parse_cmdline () =
> | "disk" | "local" -> output_mode := `Local
> | "null" -> output_mode := `Null
> | "ovirt" | "rhv" | "rhev" -> output_mode := `RHV
> + | "ovirt-upload" | "ovirt_upload" | "rhv-upload" | "rhv_upload" ->
> + output_mode := `RHV_Upload
> | "qemu" -> output_mode := `QEmu
> | "vdsm" -> output_mode := `VDSM
> | s ->
> @@ -229,6 +233,9 @@ let parse_cmdline () =
> [ L"print-source" ], Getopt.Set print_source,
> s_"Print source and stop";
> [ L"qemu-boot" ], Getopt.Set qemu_boot, s_"Boot in qemu (-o qemu only)";
> + [ L"rhv-cafile" ], Getopt.String ("ca.pem", set_string_option_once "--rhv-cafile" rhv_cafile),
> + s_"For -o rhv-upload, set ‘ca.pem’ file";
> + [ L"rhv-direct" ], Getopt.Set rhv_direct, s_"Use direct transfer mode";
> [ L"root" ], Getopt.String ("ask|... ", set_root_choice),
> s_"How to choose root filesystem";
> [ L"vddk-config" ], Getopt.String ("filename", set_string_option_once "--vddk-config" vddk_config),
> @@ -322,6 +329,8 @@ read the man page virt-v2v(1).
> let password_file = !password_file in
> let print_source = !print_source in
> let qemu_boot = !qemu_boot in
> + let rhv_cafile = !rhv_cafile in
> + let rhv_direct = !rhv_direct in
> let root_choice = !root_choice in
> let vddk_options =
> { vddk_config = !vddk_config;
> @@ -546,6 +555,35 @@ read the man page virt-v2v(1).
> Output_rhv.output_rhv os output_alloc,
> output_format, output_alloc
>
> + | `RHV_Upload ->
> + let output_conn =
> + match output_conn with
> + | None ->
> + error (f_"-o rhv-upload: use ‘-oc’ to point to the oVirt or RHV server REST API URL, which is usually https://servername/ovirt-engine/api")
> + | Some oc -> oc in
> + (* In theory we could make the password optional in future. *)
> + let output_password =
> + match output_password with
> + | None ->
> + error (f_"-o rhv-upload: output password file was not specified, use ‘-op’ to point to a file which contains the password used to connect to the oVirt or RHV server")
> + | Some op -> op in
> + let os =
> + match output_storage with
> + | None ->
> + error (f_"-o rhv-upload: output storage was not specified, use ‘-os’");
> + | Some os -> os in
> + if qemu_boot then
> + error_option_cannot_be_used_in_output_mode "rhv-upload" "--qemu-boot";
> + let rhv_cafile =
> + match rhv_cafile with
> + | None ->
> + error (f_"-o rhv-upload: must use ‘--rhv-cafile’ to supply the path to the oVirt or RHV server’s ‘ca.pem’ file")
> + | Some rhv_cafile -> rhv_cafile in
> + Output_rhv_upload.output_rhv_upload output_alloc output_conn
> + output_password os
> + rhv_cafile rhv_direct,
> + output_format, output_alloc
> +
> | `VDSM ->
> if output_password <> None then
> error_option_cannot_be_used_in_output_mode "vdsm" "-op";
> diff --git a/v2v/output_rhv_upload.ml b/v2v/output_rhv_upload.ml
> new file mode 100644
> index 000000000..c5bcd45e3
> --- /dev/null
> +++ b/v2v/output_rhv_upload.ml
> @@ -0,0 +1,451 @@
> +(* virt-v2v
> + * Copyright (C) 2009-2018 Red Hat Inc.
> + *
> + * This program is free software; you can redistribute it and/or modify
> + * it under the terms of the GNU General Public License as published by
> + * the Free Software Foundation; either version 2 of the License, or
> + * (at your option) any later version.
> + *
> + * This program is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
> + * GNU General Public License for more details.
> + *
> + * You should have received a copy of the GNU General Public License along
> + * with this program; if not, write to the Free Software Foundation, Inc.,
> + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
> + *)
> +
> +open Printf
> +open Unix
> +
> +open Std_utils
> +open Tools_utils
> +open Unix_utils
> +open Common_gettext.Gettext
> +
> +open Types
> +open Utils
> +
> +(* Timeout to wait for oVirt disks to change status, or the transfer
> + * object to finish initializing [seconds].
> + *)
> +let ovirt_timeout = 5*60
> +
> +(* These correspond mostly to the fields in the Python
> + * sdk.Connection object, except for the password which
> + * is handled separately.
> + *)
> +type connection = {
> + conn_url : string;
> + conn_username : string;
> + conn_debug : bool;
> +}
> +
> +let string_of_connection conn =
> + sprintf "url=%s username=%s debug=%b"
> + conn.conn_url conn.conn_username conn.conn_debug
> +
> +(* Python code fragments go first. Note these must not be
> + * indented because of Python's stupid whitespace thing.
> + *)
If you want some indentation it is possible. But you will have to modify
your python invocation to something ugly like:
let cmd = sprintf "%s -c '%s' %s" (quote python)
"import sys\nimport textwrap\nwith open(sys.argv[1]) as f: exec(textwrap.dedent(f.read()))"
(quote filename) in
You need to be careful that the code snippets you glue together have
same leading indentation though. So the usefulness of this is
questionable at best.
Tomas
> +
> +(* Print the Python version. *)
> +let python_get_version = "
> +import sys
> +print (sys.version[0]) # syntax works on py2 or py3
> +"
> +
> +(* Import all the Python modules needed. *)
> +let python_imports = "
> +import logging
> +import ovirtsdk4 as sdk
> +import ovirtsdk4.types as types
> +import ssl
> +import sys
> +import time
> +
> +from http.client import HTTPSConnection
> +
> +try:
> + from urllib.parse import urlparse
> +except ImportError:
> + from urlparse import urlparse
> +"
> +
> +(* Create the Python prologue which connects to the system service.
> + * This returns a string of Python code.
> + *)
> +let python_connect tmpdir conn output_password =
> + sprintf "
> +password_file = %s
> +with open(password_file, 'r') as file:
> + password = file.read()
> +password = password.rstrip()
> +
> +# Open the connection.
> +connection = sdk.Connection(
> + url = %s,
> + username = %s,
> + password = password,
> + debug = %s,
> + log = logging.getLogger(),
> + insecure = True,
> +)
> +system_service = connection.system_service()
> +" (py_quote output_password)
> + (py_quote conn.conn_url)
> + (py_quote conn.conn_username)
> + (py_bool conn.conn_debug)
> +
> +let python_get_storage_domain_id output_storage =
> + let search_term = sprintf "name=%s" output_storage in
> + sprintf "
> +sds_service = system_service.storage_domains_service()
> +sd = sds_service.list(search=%s)[0]
> +print(sd.id)
> +" (py_quote search_term)
> +
> +let python_create_one_disk disk_name disk_format
> + output_alloc sd_id disk_size =
> + sprintf "
> +disks_service = system_service.disks_service()
> +disk = disks_service.add(
> + disk = types.Disk(
> + name = %s,
> + format = %s,
> + sparse = %s,
> + provisioned_size = %Ld,
> + storage_domains = [types.StorageDomain(id = %s)],
> + )
> +)
> +disk_id = disk.id
> +
> +# Wait til the disk is up. The transfer cannot start if the
> +# disk is locked.
> +disk_service = disks_service.disk_service(disk_id)
> +timeout = time.time() + %d
> +while True:
> + time.sleep(5)
> + disk = disk_service.get()
> + if disk.status == types.DiskStatus.OK:
> + break
> + if time.time() > timeout:
> + raise RuntimeError(\"Timed out waiting for disk to become unlocked\")
> +
> +# Return the disk ID.
> +print(disk_id)
> +" (py_quote disk_name)
> + disk_format (* it's a raw Python expression, don't quote *)
> + (py_bool (output_alloc = Sparse))
> + disk_size
> + (py_quote sd_id)
> + ovirt_timeout
> +
> +(* XXX Temporary function. *)
> +let python_upload_one_disk disk_id disk_size filename rhv_cafile rhv_direct =
> + sprintf "
> +transfers_service = system_service.image_transfers_service()
> +
> +transfer = transfers_service.add(
> + types.ImageTransfer(
> + image = types.Image (id=%s)
> + )
> +)
> +
> +transfer_service = transfers_service.image_transfer_service(transfer.id)
> +
> +# After adding a new transfer for the disk, the transfer's status will
> +# be INITIALIZING. Wait until the init phase is over.
> +timeout = time.time() + %d
> +while True:
> + time.sleep(5)
> + transfer = transfer_service.get()
> + if transfer.phase != types.ImageTransferPhase.INITIALIZING:
> + break
> + if time.time() > timeout:
> + raise RuntimeError(\"Timed out waiting for transfer status != INITIALIZING\")
> +
> +if %s:
> + if transfer.transfer_url is None:
> + raise RuntimeError(\"Direct upload to host not supported, requires ovirt-engine >= 4.2 and only works when virt-v2v is run within the oVirt/RHV environment, eg. on an ovirt node.\")
> + destination_url = urlparse(transfer.transfer_url)
> +else:
> + destination_url = urlparse(transfer.proxy_url)
> +
> +context = ssl.create_default_context()
> +context.load_verify_locations(cafile = %s)
> +
> +proxy_connection = HTTPSConnection(
> + destination_url.hostname,
> + destination_url.port,
> + context = context
> +)
> +
> +image_path = %s
> +image_size = %Ld
> +
> +proxy_connection.putrequest(\"PUT\", destination_url.path)
> +proxy_connection.putheader('Content-Length', image_size)
> +proxy_connection.endheaders()
> +
> +# This seems to give the best throughput when uploading from Yaniv's
> +# machine to a server that drops the data. You may need to tune this
> +# on your setup.
> +BUF_SIZE = 128 * 1024
> +
> +with open(image_path, \"rb\") as disk:
> + pos = 0
> + while pos < image_size:
> + # Send the next chunk to the proxy.
> + to_read = min(image_size - pos, BUF_SIZE)
> + chunk = disk.read(to_read)
> + if not chunk:
> + transfer_service.pause()
> + raise RuntimeError(\"Unexpected end of file at pos=%%d\" %% pos)
> +
> + proxy_connection.send(chunk)
> + pos += len(chunk)
> +
> +# Get the response
> +response = proxy_connection.getresponse()
> +if response.status != 200:
> + transfer_service.pause()
> + raise RuntimeError(\"Upload failed: %%s %%s\" %%
> + (response.status, response.reason))
> +
> +# Successful cleanup
> +transfer_service.finalize()
> +connection.close()
> +proxy_connection.close()
> +" (py_quote disk_id)
> + ovirt_timeout
> + (py_bool rhv_direct)
> + (py_quote rhv_cafile)
> + (py_quote filename)
> + disk_size
> +
> +let python_create_virtual_machine ovf =
> + sprintf "
> +vms_service = system_service.vms_service()
> +vm = vms_service.add(
> + types.Vm(
> + cluster=types.Cluster(name = %s),
> + initialization=types.Initialization(
> + configuration = types.Configuration(
> + type = types.ConfigurationType.OVA,
> + data = %s
> + )
> + )
> + )
> +)
> +" (py_quote "Default" (* XXX target cluster *))
> + (py_quote (DOM.doc_to_string ovf))
> +
> +(* Find the Python 3 binary. *)
> +let find_python3 () =
> + let rec loop = function
> + | [] ->
> + error "could not locate Python 3 binary on the $PATH. You may have to install Python 3. If Python 3 is already installed then you may need to create a directory containing a binary called ‘python3’ which runs Python 3."
> + | python :: rest ->
> + (* Use shell_command first to check the binary exists. *)
> + let cmd = sprintf "%s --help >/dev/null 2>&1" (quote python) in
> + if shell_command cmd = 0 &&
> + run_python ~python python_get_version = ["3"] then (
> + debug "rhv-upload: python binary: %s" python;
> + python
> + )
> + else
> + loop rest
> + in
> + loop ["python3"; "python"]
> +
> +(* Parse the -oc URI. *)
> +let parse_output_conn oc =
> + let uri = Xml.parse_uri oc in
> + if uri.Xml.uri_scheme <> Some "https" then
> + error (f_"rhv-upload: -oc: URI must start with https://...");
> + if uri.Xml.uri_server = None then
> + error (f_"rhv-upload: -oc: no remote server name in the URI");
> + if uri.Xml.uri_path = None || uri.Xml.uri_path = Some "/" then
> + error (f_"rhv-upload: -oc: URI path component looks incorrect");
> + let username =
> + match uri.Xml.uri_user with
> + | None ->
> + warning (f_"rhv-upload: -oc: username was missing from URI, assuming ‘admin at internal’");
> + "admin at internal"
> + | Some user -> user in
> + (* Reconstruct the URI without the username. *)
> + let url = sprintf "%s://%s%s"
> + (Option.default "https" uri.Xml.uri_scheme)
> + (Option.default "localhost" uri.Xml.uri_server)
> + (Option.default "" uri.Xml.uri_path) in
> +
> + let conn = { conn_url = url; conn_username = username;
> + conn_debug = verbose () } in
> + debug "rhv-upload: connection=%s" (string_of_connection conn);
> + conn
> +
> +(* Get the storage domain ID. *)
> +let get_storage_domain_id run_python tmpdir conn output_password
> + output_storage =
> + let code =
> + python_imports ^
> + python_connect tmpdir conn output_password ^
> + python_get_storage_domain_id output_storage in
> + match run_python code with
> + | [id] -> id
> + | _ ->
> + error (f_"rhv-upload: get_storage_domain_id: could not fetch storage domain ID of ‘%s’ (does it exist on the server?). See previous output for more details.") output_storage
> +
> +(* Create a single, empty disk on the target. *)
> +let create_one_disk run_python tmpdir conn
> + output_password output_format
> + output_alloc sd_id
> + source target =
> + (* Give the disk a predictable name based on the source
> + * name and disk index.
> + *)
> + let disk_name =
> + let id = target.target_overlay.ov_source.s_disk_id in
> + sprintf "%s-%03d" source.s_name id in
> +
> + let disk_format =
> + match output_format with
> + | `Raw -> "types.DiskFormat.RAW"
> + | `COW -> "types.DiskFormat.COW" in
> +
> + (* This is the virtual size in bytes. *)
> + let disk_size = target.target_overlay.ov_virtual_size in
> +
> + let code =
> + python_imports ^
> + python_connect tmpdir conn output_password ^
> + python_create_one_disk disk_name disk_format
> + output_alloc sd_id disk_size in
> + match run_python code with
> + | [id] -> id
> + | _ ->
> + error (f_"rhv-upload: create_one_disk: error creating disks, see previous output")
> +
> +(* XXX Temporary function to upload spooled disk. *)
> +let upload_one_disk run_python tmpdir conn output_password
> + rhv_cafile rhv_direct
> + t filename disk_id =
> + let disk_size = t.target_overlay.ov_virtual_size in
> +
> + let code =
> + python_imports ^
> + python_connect tmpdir conn output_password ^
> + python_upload_one_disk disk_id disk_size filename rhv_cafile rhv_direct in
> + ignore (run_python code)
> +
> +(* Upload the virtual machine metadata (ie OVF) and create a VM. *)
> +let create_virtual_machine run_python tmpdir conn output_password ovf =
> + let code =
> + python_imports ^
> + python_connect tmpdir conn output_password ^
> + python_create_virtual_machine ovf in
> + ignore (run_python code)
> +
> +class output_rhv_upload output_alloc output_conn
> + output_password output_storage
> + rhv_cafile rhv_direct =
> + let run_python =
> + let python = find_python3 () in
> + run_python ~python in
> + let conn = parse_output_conn output_conn in
> +
> + (* The temporary directory is used for a few things such as passing
> + * passwords securely and (temporarily) for spooling disks (XXX).
> + *)
> + let tmpdir =
> + let base_dir = (open_guestfs ())#get_cachedir () in
> + let t = Mkdtemp.temp_dir ~base_dir "rhvupload." in
> + rmdir_on_exit t;
> + t in
> +
> + (* Storage domain ID. *)
> + let sd_id =
> + get_storage_domain_id run_python tmpdir conn output_password
> + output_storage in
> +object
> + inherit output
> +
> + method precheck () =
> + (* Check all the dependencies including the Python 3 oVirt SDK v4
> + * module are installed. This will fail with a Python error message.
> + *)
> + ignore (run_python python_imports)
> +
> + method as_options =
> + "-o rhv-upload" ^
> + (match output_alloc with
> + | Sparse -> "" (* default, don't need to print it *)
> + | Preallocated -> " -oa preallocated") ^
> + sprintf " -oc %s -op %s -os %s"
> + output_conn output_password output_storage
> +
> + method supported_firmware = [ TargetBIOS ]
> +
> + (* List of disks we have created. There will be one per target. *)
> + val mutable target_disk_ids = []
> +
> + method prepare_targets source targets =
> + let targets =
> + List.map (
> + fun t ->
> + (* Only allow output format "raw" or "qcow2". *)
> + let output_format =
> + match t.target_format with
> + | "raw" -> `Raw
> + | "qcow2" -> `COW
> + | _ ->
> + error (f_"rhv-upload: -of %s: Only output format ‘raw’ or ‘qcow2’ is supported. If the input is in a different format then force one of these output formats by adding either ‘-of raw’ or ‘-of qcow2’ on the command line.")
> + t.target_format in
> +
> + let disk_id = create_one_disk run_python tmpdir conn output_password
> + output_format output_alloc
> + sd_id source t in
> +
> + (* XXX Temporarily spool disks to tmpdir. *)
> + let target_file = TargetFile (tmpdir // t.target_overlay.ov_sd) in
> + { t with target_file }, disk_id
> + ) targets in
> + target_disk_ids <- List.map snd targets;
> + List.map fst targets
> +
> + method create_metadata source targets _ guestcaps inspect target_firmware =
> + (* Upload the spooled disks. *)
> + List.iter (
> + fun (t, disk_id) ->
> + let filename =
> + match t.target_file with
> + | TargetFile filename -> filename
> + | TargetURI _ -> assert false in
> + upload_one_disk run_python tmpdir conn output_password
> + rhv_cafile rhv_direct
> + t filename disk_id
> + ) (List.combine targets target_disk_ids);
> +
> + let image_uuids = target_disk_ids
> + and vol_uuids = List.map (fun _ -> uuidgen ()) target_disk_ids
> + and vm_uuid = uuidgen () in
> +
> + (* Create the metadata. *)
> + let ovf =
> + Create_ovf.create_ovf source targets guestcaps inspect
> + Sparse
> + sd_id (* storage UUID *)
> + image_uuids
> + vol_uuids
> + vm_uuid
> + OVirt in
> +
> + (* Add the virtual machine. *)
> + create_virtual_machine run_python tmpdir conn output_password ovf
> +
> +end
> +
> +let output_rhv_upload = new output_rhv_upload
> +let () = Modules_list.register_output_module "rhv-upload"
> diff --git a/v2v/output_rhv_upload.mli b/v2v/output_rhv_upload.mli
> new file mode 100644
> index 000000000..3e7086f85
> --- /dev/null
> +++ b/v2v/output_rhv_upload.mli
> @@ -0,0 +1,27 @@
> +(* virt-v2v
> + * Copyright (C) 2009-2018 Red Hat Inc.
> + *
> + * This program is free software; you can redistribute it and/or modify
> + * it under the terms of the GNU General Public License as published by
> + * the Free Software Foundation; either version 2 of the License, or
> + * (at your option) any later version.
> + *
> + * This program is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
> + * GNU General Public License for more details.
> + *
> + * You should have received a copy of the GNU General Public License along
> + * with this program; if not, write to the Free Software Foundation, Inc.,
> + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
> + *)
> +
> +(** [-o rhv-upload] target. *)
> +
> +val output_rhv_upload : Types.output_allocation -> string -> string ->
> + string -> string -> bool ->
> + Types.output
> +(** [output_rhv_upload output_alloc output_conn output_password output_storage
> + rhv_cafile rhv_direct]
> + creates and returns a new {!Types.output} object specialized for writing
> + output to oVirt or RHV directly via RHV APIs. *)
> --
> 2.13.2
>
--
Tomáš Golembiovský <tgolembi at redhat.com>
More information about the Libguestfs
mailing list