[Libguestfs] [PATCH v12 3/3] New tool: virt-builder-repository
Cédric Bosdonnat
cbosdonnat at suse.com
Mon Nov 13 13:56:12 UTC 2017
virt-builder-repository allows users to easily create or update
a virt-builder source repository out of disk images. The tool can
be run in either interactive or automated mode.
---
.gitignore | 4 +
builder/Makefile.am | 87 ++++-
builder/repository_main.ml | 621 ++++++++++++++++++++++++++++++++
builder/test-docs.sh | 2 +
builder/test-virt-builder-repository.sh | 98 +++++
builder/utils.ml | 4 +
builder/utils.mli | 3 +
builder/virt-builder-repository.pod | 213 +++++++++++
builder/virt-builder.pod | 4 +
fish/guestfish.pod | 1 +
installcheck.sh.in | 1 +
lib/guestfs.pod | 1 +
12 files changed, 1037 insertions(+), 2 deletions(-)
create mode 100644 builder/repository_main.ml
create mode 100755 builder/test-virt-builder-repository.sh
create mode 100644 builder/virt-builder-repository.pod
diff --git a/.gitignore b/.gitignore
index f3569aa73..e54ea2d45 100644
--- a/.gitignore
+++ b/.gitignore
@@ -98,14 +98,18 @@ Makefile.in
/builder/oUnit-*
/builder/*.out
/builder/*.qcow2
+/builder/repository-testdata
/builder/stamp-virt-builder.pod
+/builder/stamp-virt-builder-repository.pod
/builder/stamp-virt-index-validate.pod
/builder/test-config/virt-builder/repos.d/test-index.conf
/builder/test-console-*.sh
/builder/test-simplestreams/virt-builder/repos.d/cirros.conf
/builder/test-website/virt-builder/repos.d/libguestfs.conf
/builder/virt-builder
+/builder/virt-builder-repository
/builder/virt-builder.1
+/builder/virt-builder-repository.1
/builder/virt-index-validate
/builder/virt-index-validate.1
/builder/*.xz
diff --git a/builder/Makefile.am b/builder/Makefile.am
index 757af75c7..e132335f5 100644
--- a/builder/Makefile.am
+++ b/builder/Makefile.am
@@ -21,6 +21,8 @@ AM_YFLAGS = -d
EXTRA_DIST = \
$(SOURCES_MLI) $(SOURCES_ML) $(SOURCES_C) \
+ $(REPOSITORY_SOURCES_ML) \
+ $(REPOSITORY_SOURCES_MLI) \
index_parser_tests.ml \
libguestfs.gpg \
opensuse.gpg \
@@ -33,6 +35,7 @@ EXTRA_DIST = \
test-virt-builder-list.sh \
test-virt-builder-list-simplestreams.sh \
test-virt-builder-planner.sh \
+ test-virt-builder-repository.sh \
test-virt-index-validate.sh \
test-virt-index-validate-bad-1 \
test-virt-index-validate-good-1 \
@@ -40,6 +43,7 @@ EXTRA_DIST = \
test-virt-index-validate-good-3 \
test-virt-index-validate-good-4 \
virt-builder.pod \
+ virt-builder-repository.pod \
virt-index-validate.pod \
yajl_tests.ml
@@ -90,13 +94,45 @@ SOURCES_C = \
setlocale-c.c \
yajl-c.c
+REPOSITORY_SOURCES_ML = \
+ yajl.ml \
+ utils.ml \
+ index.ml \
+ cache.ml \
+ downloader.ml \
+ sigchecker.ml \
+ ini_reader.ml \
+ index_parser.ml \
+ paths.ml \
+ sources.ml \
+ osinfo_config.ml \
+ osinfo.ml \
+ repository_main.ml
+
+REPOSITORY_SOURCES_MLI = \
+ cache.mli \
+ downloader.mli \
+ index.mli \
+ index_parser.mli \
+ ini_reader.mli \
+ sigchecker.mli \
+ sources.mli \
+ yajl.mli
+
+REPOSITORY_SOURCES_C = \
+ index-scan.c \
+ index-struct.c \
+ index-parse.c \
+ index-parser-c.c \
+ yajl-c.c
+
man_MANS =
noinst_DATA =
bin_PROGRAMS =
if HAVE_OCAML
-bin_PROGRAMS += virt-builder
+bin_PROGRAMS += virt-builder virt-builder-repository
virt_builder_SOURCES = $(SOURCES_C)
virt_builder_CPPFLAGS = \
@@ -119,12 +155,31 @@ virt_builder_CFLAGS = \
BOBJECTS = $(SOURCES_ML:.ml=.cmo)
XOBJECTS = $(BOBJECTS:.cmo=.cmx)
+virt_builder_repository_SOURCES = $(REPOSITORY_SOURCES_C)
+virt_builder_repository_CPPFLAGS = \
+ -I. \
+ -I$(top_builddir) \
+ -I$(top_srcdir)/gnulib/lib -I$(top_builddir)/gnulib/lib \
+ -I$(shell $(OCAMLC) -where) \
+ -I$(top_srcdir)/gnulib/lib \
+ -I$(top_srcdir)/lib
+virt_builder_repository_CFLAGS = \
+ -pthread \
+ $(WARN_CFLAGS) $(WERROR_CFLAGS) \
+ -Wno-unused-macros \
+ $(LIBTINFO_CFLAGS) \
+ $(LIBXML2_CFLAGS) \
+ $(YAJL_CFLAGS)
+REPOSITORY_BOBJECTS = $(REPOSITORY_SOURCES_ML:.ml=.cmo)
+REPOSITORY_XOBJECTS = $(REPOSITORY_BOBJECTS:.cmo=.cmx)
+
# -I $(top_builddir)/lib/.libs is a hack which forces corresponding -L
# option to be passed to gcc, so we don't try linking against an
# installed copy of libguestfs.
OCAMLPACKAGES = \
-package str,unix \
-I $(top_builddir)/common/utils/.libs \
+ -I $(top_builddir)/common/mlxml \
-I $(top_builddir)/lib/.libs \
-I $(top_builddir)/gnulib/lib/.libs \
-I $(top_builddir)/ocaml \
@@ -157,13 +212,16 @@ OCAMLFLAGS = $(OCAML_FLAGS) $(OCAML_WARN_ERROR)
if !HAVE_OCAMLOPT
OBJECTS = $(BOBJECTS)
+REPOSITORY_OBJECTS = $(REPOSITORY_BOBJECTS)
else
OBJECTS = $(XOBJECTS)
+REPOSITORY_OBJECTS = $(REPOSITORY_XOBJECTS)
endif
OCAMLLINKFLAGS = \
mlgettext.$(MLARCHIVE) \
mlpcre.$(MLARCHIVE) \
+ mlxml.$(MLARCHIVE) \
mlstdutils.$(MLARCHIVE) \
mlguestfs.$(MLARCHIVE) \
mlcutils.$(MLARCHIVE) \
@@ -185,6 +243,16 @@ virt_builder_LINK = \
$(OCAMLFIND) $(BEST) $(OCAMLFLAGS) $(OCAMLPACKAGES) $(OCAMLLINKFLAGS) \
$(OBJECTS) -o $@
+virt_builder_repository_DEPENDENCIES = \
+ $(REPOSITORY_OBJECTS) \
+ ../common/mltools/mltools.$(MLARCHIVE) \
+ ../common/mlxml/mlxml.$(MLARCHIVE) \
+ $(top_srcdir)/ocaml-link.sh
+virt_builder_repository_LINK = \
+ $(top_srcdir)/ocaml-link.sh -cclib '$(OCAMLCLIBS)' -- \
+ $(OCAMLFIND) $(BEST) $(OCAMLFLAGS) $(OCAMLPACKAGES) $(OCAMLLINKFLAGS) \
+ $(REPOSITORY_OBJECTS) -o $@
+
# Manual pages and HTML files for the website.
man_MANS += virt-builder.1
@@ -203,6 +271,20 @@ stamp-virt-builder.pod: virt-builder.pod $(top_srcdir)/customize/customize-synop
$<
touch $@
+man_MANS += virt-builder-repository.1
+noinst_DATA += $(top_builddir)/website/virt-builder-repository.1.html
+
+virt-builder-repository.1 $(top_builddir)/website/virt-builder-repository.1.html: stamp-virt-builder-repository.pod
+
+stamp-virt-builder-repository.pod: virt-builder-repository.pod
+ $(PODWRAPPER) \
+ --man virt-builder-repository.1 \
+ --html $(top_builddir)/website/virt-builder-repository.1.html \
+ --license GPLv2+ \
+ --warning safe \
+ $<
+ touch $@
+
# Tests.
TESTS_ENVIRONMENT = $(top_builddir)/run --test
@@ -321,7 +403,8 @@ check-valgrind:
SLOW_TESTS = \
$(console_test_scripts) \
- test-virt-builder-planner.sh
+ test-virt-builder-planner.sh \
+ test-virt-builder-repository.sh
check-slow:
$(MAKE) check TESTS="$(SLOW_TESTS)" SLOW=1
diff --git a/builder/repository_main.ml b/builder/repository_main.ml
new file mode 100644
index 000000000..75ece806d
--- /dev/null
+++ b/builder/repository_main.ml
@@ -0,0 +1,621 @@
+(* virt-builder
+ * Copyright (C) 2016-2017 SUSE Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ *)
+
+open Std_utils
+open Common_gettext.Gettext
+open Tools_utils
+open Unix_utils
+open Getopt.OptionName
+open Utils
+open Yajl
+open Xpath_helpers
+
+open Printf
+
+type cmdline = {
+ gpg : string;
+ gpgkey : string option;
+ interactive : bool;
+ keep_unsigned : bool;
+ compression : bool;
+ repo : string;
+}
+
+type disk_image_info = {
+ format : string;
+ size : int64;
+}
+
+let parse_cmdline () =
+ let gpg = ref "gpg" in
+ let gpgkey = ref None in
+ let set_gpgkey arg = gpgkey := Some arg in
+
+ let interactive = ref false in
+ let keep_unsigned = ref false in
+ let compression = ref true in
+ let machine_readable = ref false in
+
+ let argspec = [
+ [ L"gpg" ], Getopt.Set_string ("gpg", gpg), s_"Set GPG binary/command";
+ [ S 'K'; L"gpg-key" ], Getopt.String ("gpgkey", set_gpgkey),
+ s_"ID of the GPG key to sign the repo with";
+ [ S 'i'; L"interactive" ], Getopt.Set interactive, s_"Ask the user about missing data";
+ [ L"keep-index" ], Getopt.Set keep_unsigned, s_"Keep unsigned index";
+ [ L"no-compression" ], Getopt.Clear compression, s_"Don’t compress the new images in the index";
+ [ L"machine-readable" ], Getopt.Set machine_readable, s_"Make output machine readable";
+ ] in
+
+ let args = ref [] in
+ let anon_fun s = push_front s args in
+ let usage_msg =
+ sprintf (f_"\
+%s: create a repository for virt-builder
+
+ virt-builder-repository REPOSITORY_PATH
+
+A short summary of the options is given below. For detailed help please
+read the man page virt-builder-repository(1).
+")
+ prog in
+ let opthandle = create_standard_options argspec ~anon_fun usage_msg in
+ Getopt.parse opthandle;
+
+ (* Machine-readable mode? Print out some facts about what
+ * this binary supports.
+ *)
+ if !machine_readable then (
+ printf "virt-builder-repository\n";
+ exit 0
+ );
+
+ (* Dereference options. *)
+ let args = List.rev !args in
+ let gpg = !gpg in
+ let gpgkey = !gpgkey in
+ let interactive = !interactive in
+ let keep_unsigned = !keep_unsigned in
+ let compression = !compression in
+
+ (* Check options *)
+ let repo =
+ match args with
+ | [repo] -> repo
+ | [] ->
+ error (f_"virt-builder-repository /path/to/repo
+
+Use ‘/path/to/repo’ to point to the repository folder.")
+ | _ ->
+ error (f_"too many parameters, only one path to repository is allowed") in
+
+ {
+ gpg = gpg;
+ gpgkey = gpgkey;
+ interactive = interactive;
+ keep_unsigned = keep_unsigned;
+ compression = compression;
+ repo = repo;
+ }
+
+let do_mv src dest =
+ let cmd = [ "mv"; src; dest ] in
+ let r = run_command cmd in
+ if r <> 0 then
+ error (f_"moving file ‘%s’ to ‘%s’ failed") src dest
+
+let checksums_get_sha512 = function
+ | None -> None
+ | Some csums ->
+ try
+ Some (List.find (
+ function
+ | Checksums.SHA512 _ -> true
+ | _ -> false
+ ) csums)
+ with Not_found -> None
+
+let osinfo_ids = ref None
+
+let rec osinfo_get_short_ids () =
+ match !osinfo_ids with
+ | Some ids -> ids
+ | None ->
+ osinfo_ids :=
+ Some (
+ Osinfo.fold (
+ fun set filepath ->
+ let doc = Xml.parse_file filepath in
+ let xpathctx = Xml.xpath_new_context doc in
+ let nodes = xpath_get_nodes xpathctx "/libosinfo/os/short-id" in
+ List.fold_left (
+ fun set node ->
+ let id = Xml.node_as_string node in
+ StringSet.add id set
+ ) set nodes
+ ) StringSet.empty
+ );
+ osinfo_get_short_ids ()
+
+let compress_to file outdir =
+ let outimg = outdir // (Filename.basename file) ^ ".xz" in
+
+ info "Compressing ...%!";
+ let cmd = [ "xz"; "-f"; "--best"; "--block-size=16777216"; "-c"; file ] in
+ let file_flags = [ Unix.O_WRONLY; Unix.O_CREAT; Unix.O_TRUNC; ] in
+ let outfd = Unix.openfile outimg file_flags 0o666 in
+ let res = run_command cmd ~stdout_chan:outfd in
+ if res <> 0 then
+ error (f_"‘xz’ command failed");
+ outimg
+
+let get_mime_type filepath =
+ let file_cmd = "file --mime-type --brief " ^ (quote filepath) in
+ match external_command file_cmd with
+ | [] -> None
+ | line :: _ -> Some line
+
+let get_disk_image_info filepath =
+ let infos = get_image_infos filepath in
+ {
+ format = object_get_string "format" infos;
+ size = object_get_number "virtual-size" infos
+ }
+
+let compute_short_id distro major minor =
+ match distro with
+ | "centos" when major >= 7 ->
+ sprintf "%s%d.0" distro major
+ | "debian" when major >= 4 ->
+ sprintf "%s%d" distro major
+ | ("fedora"|"mageia") ->
+ sprintf "%s%d" distro major
+ | "sles" when major = 0 ->
+ sprintf "%s%d" distro major
+ | "sles" ->
+ sprintf "%s%dsp%d" distro major minor
+ | "ubuntu" ->
+ sprintf "%s%d.%02d" distro major minor
+ | _ (* Any other combination. *) ->
+ sprintf "%s%d.%d" distro major minor
+
+let cmp a b =
+ let string_of_arch = function Index.Arch s -> s | Index.GuessedArch s -> s in
+ (string_of_arch a) = (string_of_arch b)
+
+let has_entry id arch index =
+ List.exists (
+ fun (item_id, { Index.arch = item_arch }) ->
+ item_id = id && (cmp item_arch arch)
+ ) index
+
+let process_image acc_entries filename repo tmprepo index interactive
+ compression sigchecker =
+ message (f_"Preparing %s") filename;
+
+ let filepath = repo // filename in
+ let { format = format; size = size } = get_disk_image_info filepath in
+ let out_path =
+ if not compression then filepath
+ else compress_to filepath tmprepo in
+ let out_filename = Filename.basename out_path in
+ let checksum = Checksums.compute_checksum "sha512" out_path in
+ let compressed_size = (Unix.LargeFile.stat out_path).Unix.LargeFile.st_size in
+
+ let ask ?default ?values message =
+ let default_str = match default with
+ | None -> ""
+ | Some x -> sprintf " [%s] " x in
+
+ let list_str = match values with
+ | None -> ""
+ | Some x ->
+ sprintf (f_"Choose one from the list below:\n %s\n")
+ (String.concat "\n " x) in
+
+ printf "%s%s%s%!" message default_str list_str;
+
+ let value = read_line () in
+ match value with
+ | "" -> default
+ | "-" -> None
+ | s -> Some s
+ in
+
+ let rec ask_id default =
+ match ask (s_"Identifier: ") ~default with
+ | None -> default
+ | Some id ->
+ if not (Str.string_match (Str.regexp "[a-zA-Z0-9-_.]+") id 0) then (
+ warning (f_"Allowed characters are letters, digits, - _ and .");
+ ask_id default
+ ) else
+ id in
+
+ let rec ask_arch guess =
+ let arches = [ "x86_64"; "aarch64"; "armv7l"; "i686"; "ppc64"; "ppc64le"; "s390x" ] in
+ match (ask (s_"Architecture: ") ~default:guess ~values:arches) with
+ | None -> ask_arch guess
+ | Some x ->
+ if x = "" then
+ ask_arch guess
+ else
+ Index.Arch x
+ in
+
+ let ask_osinfo default =
+ match ask (s_ "osinfo short ID: ")
+ ~default with
+ | None -> None
+ | Some osinfo ->
+ let osinfo_ids = osinfo_get_short_ids () in
+ if not (StringSet.mem osinfo osinfo_ids) then
+ warning (f_"‘%s’ is not a recognized osinfo OS id; using it anyway") osinfo;
+ Some osinfo in
+
+ let extract_entry_data ?entry () =
+ message (f_"Extracting data from the image...");
+ let g = new Guestfs.guestfs () in
+ g#add_drive_ro filepath;
+ g#launch ();
+
+ let roots = g#inspect_os () in
+ let nroots = Array.length roots in
+ if nroots <> 1 then
+ error (f_"virt-builder template images must have one and only one root file system, found %d")
+ nroots;
+
+ let root = Array.get roots 0 in
+ let inspected_arch = g#inspect_get_arch root in
+ let product = g#inspect_get_product_name root in
+ let distro = g#inspect_get_distro root in
+ let version_major = g#inspect_get_major_version root in
+ let version_minor = g#inspect_get_minor_version root in
+ let lvs = g#lvs () in
+ let filesystems = g#inspect_get_filesystems root in
+
+ let shortid = compute_short_id distro version_major version_minor in
+
+ g#close ();
+
+ let id =
+ match entry with
+ | Some (id, _) -> id
+ | None -> (
+ if interactive then ask_id shortid
+ else error (f_"missing image identifier")
+ ) in
+
+ let arch =
+ match entry with
+ | Some (_, { Index.arch = arch }) -> (
+ match arch with
+ | Index.Arch arch -> Index.Arch arch
+ | Index.GuessedArch arch ->
+ if interactive then ask_arch arch
+ else Index.Arch arch )
+ | None ->
+ if interactive then ask_arch inspected_arch
+ else Index.Arch inspected_arch in
+
+ if has_entry id arch acc_entries then (
+ let arch =
+ match arch with
+ | Index.Arch arch
+ | Index.GuessedArch arch -> arch in
+ error (f_"Already existing image with id %s and architecture %s") id arch
+ );
+
+ let printable_name =
+ match entry with
+ | Some (_, { Index.printable_name = printable_name }) ->
+ if printable_name = None then
+ if interactive then ask (s_"Display name: ") ~default:product
+ else Some product
+ else
+ printable_name
+ | None -> Some product in
+
+ let osinfo =
+ match entry with
+ | Some (_, { Index.osinfo = osinfo }) ->
+ if osinfo = None then
+ if interactive then ask_osinfo shortid else Some shortid
+ else
+ osinfo
+ | None ->
+ if interactive then ask_osinfo shortid else Some shortid in
+
+ let expand =
+ match entry with
+ | Some (_, { Index.expand = expand }) ->
+ if expand = None then
+ if interactive then ask (s_"Expandable partition: ") ~default:root
+ ~values:(Array.to_list filesystems)
+ else Some root
+ else
+ expand
+ | None ->
+ if interactive then ask (s_"Expandable partition: ") ~default:root
+ ~values:(Array.to_list filesystems)
+ else Some root in
+
+ let lvexpand =
+ match entry with
+ | Some (_, { Index.lvexpand = lvexpand }) ->
+ if lvexpand = None && lvs <> [||] then
+ if interactive then
+ ask (s_"Expandable volume: ") ~values:(Array.to_list lvs)
+ else Some (Array.get lvs 0)
+ else
+ lvexpand
+ | None ->
+ if interactive then
+ ask (s_"Expandable volume: ") ~values:(Array.to_list lvs)
+ else Some (Array.get lvs 0) in
+
+ let revision =
+ match entry with
+ | Some (_, { Index.revision = revision }) ->
+ Utils.increment_revision revision
+ | None -> Rev_int 1 in
+
+ let notes =
+ match entry with
+ | Some (_, { Index.notes = notes }) -> notes
+ | None -> [] in
+
+ let hidden =
+ match entry with
+ | Some (_, { Index.hidden = hidden }) -> hidden
+ | None -> false in
+
+ let aliases =
+ match entry with
+ | Some (_, { Index.aliases = aliases }) -> aliases
+ | None -> None in
+
+ (id, { Index.printable_name = printable_name;
+ osinfo = osinfo;
+ file_uri = Filename.basename out_path;
+ arch = arch;
+ signature_uri = None;
+ checksums = Some [checksum];
+ revision = revision;
+ format = Some format;
+ size = size;
+ compressed_size = Some compressed_size;
+ expand = expand;
+ lvexpand = lvexpand;
+ notes = notes;
+ hidden = hidden;
+ aliases = aliases;
+ sigchecker = sigchecker;
+ proxy = Curl.SystemProxy })
+ in
+
+ (* Do we have an entry for that file already? *)
+ let file_entry =
+ try
+ List.hd (
+ List.filter (
+ fun (_, { Index.file_uri = file_uri }) ->
+ let basename = Filename.basename file_uri in
+ basename = out_filename || basename = filename
+ ) index
+ )
+ with
+ | Failure _ -> extract_entry_data () in
+
+ let _, { Index.checksums = checksums } = file_entry in
+ let old_checksum = checksums_get_sha512 checksums in
+
+ match old_checksum with
+ | Some old_sum ->
+ if old_sum = checksum then
+ let id, entry = file_entry in
+ (id, { entry with Index.file_uri = out_filename })
+ else
+ extract_entry_data ~entry:file_entry ()
+ | None ->
+ extract_entry_data ~entry:file_entry ()
+
+let main () =
+ let cmdline = parse_cmdline () in
+
+ (* If debugging, echo the command line arguments. *)
+ debug "command line: %s" (String.concat " " (Array.to_list Sys.argv));
+
+ (* Check that the paths are existing *)
+ if not (Sys.file_exists cmdline.repo) then
+ error (f_"repository folder ‘%s’ doesn’t exist") cmdline.repo;
+
+ (* Create a temporary folder to work in *)
+ let tmpdir = Mkdtemp.temp_dir ~base_dir:cmdline.repo
+ "virt-builder-repository." in
+ rmdir_on_exit tmpdir;
+
+ let tmprepo = tmpdir // "repo" in
+ mkdir_p tmprepo 0o700;
+
+ let sigchecker = Sigchecker.create ~gpg:cmdline.gpg
+ ~check_signature:false
+ ~gpgkey:No_Key
+ ~tmpdir in
+
+ let index =
+ try
+ let index_filename =
+ List.find (
+ fun filename -> Sys.file_exists (cmdline.repo // filename)
+ ) [ "index.asc"; "index" ] in
+
+ let downloader = Downloader.create ~curl:"do-not-use-curl"
+ ~cache:None ~tmpdir in
+
+ let source = { Sources.name = index_filename;
+ uri = cmdline.repo // index_filename;
+ gpgkey = No_Key;
+ proxy = Curl.SystemProxy;
+ format = Sources.FormatNative } in
+
+ Index_parser.get_index ~downloader ~sigchecker ~template:true source
+ with Not_found -> [] in
+
+ (* Check for index/interactive consistency *)
+ if not cmdline.interactive && index = [] then
+ error (f_"the repository must contain an index file when running in automated mode");
+
+ debug "Searching for images ...";
+
+ let images =
+ let is_supported_format file =
+ let extension = last_part_of file '.' in
+ match extension with
+ | Some ext -> List.mem ext [ "qcow2"; "raw"; "img" ]
+ | None ->
+ match (get_mime_type file) with
+ | None -> false
+ | Some mime -> mime = "application/octet-stream" in
+ let is_new file =
+ try
+ let _, { Index.checksums = checksums } =
+ List.find (
+ fun (_, { Index.file_uri = file_uri }) ->
+ Filename.basename file_uri = file
+ ) index in
+ let checksum = checksums_get_sha512 checksums in
+ let path = cmdline.repo // file in
+ let file_checksum = Checksums.compute_checksum "sha512" path in
+ match checksum with
+ | None -> true
+ | Some sum -> sum <> file_checksum
+ with Not_found -> true in
+ let files = Array.to_list (Sys.readdir cmdline.repo) in
+ let files = List.filter (
+ fun file -> is_regular_file (cmdline.repo // file)
+ ) files in
+ List.filter (
+ fun file ->
+ if is_supported_format (cmdline.repo // file) then
+ is_new file
+ else
+ false
+ ) files in
+
+ if images = [] then (
+ info (f_ "No new image found");
+ exit 0
+ );
+
+ info (f_ "Found new images: %s") (String.concat " " images);
+
+ let outindex_path = tmprepo // "index" in
+ let index_channel = open_out outindex_path in
+
+ (* Generate entries for uncompressed images *)
+ let images_entries = List.fold_right (
+ fun filename acc ->
+ let image_entry = process_image acc
+ filename
+ cmdline.repo
+ tmprepo
+ index
+ cmdline.interactive
+ cmdline.compression
+ sigchecker in
+ image_entry :: acc
+ ) images [] in
+
+ (* Filter out entries for newly found images and entries
+ without a corresponding image file or with empty arch *)
+ let index = List.filter (
+ fun (id, { Index.arch = arch;
+ Index.file_uri = file_uri }) ->
+ not (has_entry id arch images_entries) && Sys.file_exists file_uri
+ ) index in
+
+ (* Convert all URIs back to relative ones *)
+ let index = List.map (
+ fun (id, entry) ->
+ let { Index.file_uri = file_uri } = entry in
+ let rel_path =
+ try
+ subdirectory cmdline.repo file_uri
+ with
+ | Invalid_argument _ ->
+ file_uri in
+ let rel_entry = { entry with Index.file_uri = rel_path } in
+ (id, rel_entry)
+ ) index in
+
+ (* Write all the entries *)
+ List.iter (
+ fun entry ->
+ Index_parser.write_entry index_channel entry;
+ ) (index @ images_entries);
+
+ close_out index_channel;
+
+ (* GPG sign the generated index *)
+ (match cmdline.gpgkey with
+ | None ->
+ debug "Skip index signing"
+ | Some gpgkey ->
+ message (f_"Signing index with the GPG key %s") gpgkey;
+ let cmd = sprintf "%s --armor --output %s --export %s"
+ (quote (cmdline.gpg // "index.gpg"))
+ (quote tmprepo) (quote gpgkey) in
+ if shell_command cmd <> 0 then
+ error (f_"failed to export the GPG key %s") gpgkey;
+
+ let cmd = sprintf "%s --armor --default-key %s --clearsign %s"
+ (quote cmdline.gpg) (quote gpgkey)
+ (quote (tmprepo // "index" )) in
+ if shell_command cmd <> 0 then
+ error (f_"failed to sign index");
+
+ (* Remove the index file since we have the signed version of it *)
+ if not cmdline.keep_unsigned then
+ Sys.remove (tmprepo // "index")
+ );
+
+ message (f_"Creating index backup copy");
+
+ List.iter (
+ fun filename ->
+ let filepath = cmdline.repo // filename in
+ if Sys.file_exists filepath then
+ do_mv filepath (filepath ^ ".bak")
+ ) ["index"; "index.asc"];
+
+ message (f_"Moving files to final destination");
+
+ Array.iter (
+ fun filename ->
+ do_mv (tmprepo // filename) cmdline.repo
+ ) (Sys.readdir tmprepo);
+
+ debug "Cleanup";
+
+ (* Remove the processed image files *)
+ if cmdline.compression then
+ List.iter (
+ fun filename -> Sys.remove (cmdline.repo // filename)
+ ) images
+
+let () = run_main_and_handle_errors main
diff --git a/builder/test-docs.sh b/builder/test-docs.sh
index 884135de6..6f39b906d 100755
--- a/builder/test-docs.sh
+++ b/builder/test-docs.sh
@@ -25,3 +25,5 @@ $top_srcdir/podcheck.pl virt-builder.pod virt-builder \
--insert $top_srcdir/customize/customize-synopsis.pod:__CUSTOMIZE_SYNOPSIS__ \
--insert $top_srcdir/customize/customize-options.pod:__CUSTOMIZE_OPTIONS__ \
--ignore=--check-signatures,--no-check-signatures
+
+$srcdir/../podcheck.pl virt-builder-repository.pod virt-builder-repository
diff --git a/builder/test-virt-builder-repository.sh b/builder/test-virt-builder-repository.sh
new file mode 100755
index 000000000..5ff270bcc
--- /dev/null
+++ b/builder/test-virt-builder-repository.sh
@@ -0,0 +1,98 @@
+#!/bin/bash -
+# libguestfs
+# Copyright (C) 2017 SUSE Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+set -e
+
+$TEST_FUNCTIONS
+slow_test
+skip_if_skipped "$script"
+
+test_data=repository-testdata
+rm -rf $test_data
+mkdir $test_data
+
+# Make a copy of the Fedora image
+cp ../test-data/phony-guests/fedora.img $test_data
+
+# Create minimal index file
+cat > $test_data/index << EOF
+[fedora]
+file=fedora.img
+EOF
+
+# Run virt-builder-repository (no compression, interactive)
+echo 'x86_64
+Fedora Test Image
+fedora14
+/dev/sda1
+/dev/VG/Root
+' | virt-builder-repository -v -x --no-compression -i $test_data
+
+assert_config () {
+ item=$1
+ regex=$2
+
+ sed -n -e "/\[$item]/,/^$/p" $test_data/index | grep "$regex"
+}
+
+# Check the generated index file
+assert_config 'fedora' 'revision=1'
+assert_config 'fedora' 'arch=x86_64'
+assert_config 'fedora' 'name=Fedora Test Image'
+assert_config 'fedora' 'osinfo=fedora14'
+assert_config 'fedora' 'checksum'
+assert_config 'fedora' 'format=raw'
+assert_config 'fedora' '^size='
+assert_config 'fedora' 'compressed_size='
+assert_config 'fedora' 'expand=/dev/'
+
+
+# Copy the debian image and add the minimal piece to index
+cp ../test-data/phony-guests/debian.img $test_data
+
+cat >> $test_data/index << EOF
+
+[debian]
+file=debian.img
+EOF
+
+# Run virt-builder-repository again
+echo 'x86_64
+Debian Test Image
+debian9
+
+' | virt-builder-repository --no-compression -i $test_data
+
+# Check that the new image is complete and the first one hasn't changed
+assert_config 'fedora' 'revision=1'
+
+assert_config 'debian' 'revision=1'
+assert_config 'debian' 'checksum'
+
+# Modify the fedora image
+export EDITOR='echo newline >>'
+virt-edit -a $test_data/fedora.img /etc/test3
+
+# Rerun the tool (with compression)
+virt-builder-repository -i $test_data
+
+# Check that the revision, file and size have been updated
+assert_config 'fedora' 'revision=2'
+assert_config 'fedora' 'file=fedora.img.xz'
+test -e $test_data/fedora.img.xz
+! test -e $test_data/fedora.img
diff --git a/builder/utils.ml b/builder/utils.ml
index d0e51a049..4cca4c21d 100644
--- a/builder/utils.ml
+++ b/builder/utils.ml
@@ -35,6 +35,10 @@ let string_of_revision = function
| Rev_int n -> string_of_int n
| Rev_string s -> s
+let increment_revision = function
+ | Rev_int n -> Rev_int (n + 1)
+ | Rev_string s -> Rev_int ((int_of_string s) + 1)
+
let get_image_infos filepath =
let qemuimg_cmd = "qemu-img info --output json " ^ quote filepath in
let lines = external_command qemuimg_cmd in
diff --git a/builder/utils.mli b/builder/utils.mli
index 5c7bfa1e4..7440e4b3d 100644
--- a/builder/utils.mli
+++ b/builder/utils.mli
@@ -32,3 +32,6 @@ val string_of_revision : revision -> string
val get_image_infos : string -> Yajl.yajl_val
(** [get_image_infos path] Run qemu-img info on the image pointed at
path as YAJL tree. *)
+
+val increment_revision : revision -> revision
+(** Add one to the version number *)
diff --git a/builder/virt-builder-repository.pod b/builder/virt-builder-repository.pod
new file mode 100644
index 000000000..11fec8f07
--- /dev/null
+++ b/builder/virt-builder-repository.pod
@@ -0,0 +1,213 @@
+=begin html
+
+<img src="virt-builder.svg" width="250"
+ style="float: right; clear: right;" />
+
+=end html
+
+=head1 NAME
+
+virt-builder-repository - Build virt-builder source repository easily
+
+=head1 SYNOPSIS
+
+ virt-builder-repository /path/to/repository
+ [-i|--interactive] [--gpg-key KEYID]
+
+=head1 DESCRIPTION
+
+Virt-builder is a tool for quickly building new virtual machines. It can
+be configured to use template repositories. However creating and
+maintaining a repository involves many tasks which can be automated.
+virt-builder-repository is a tool helping to manage these repositories.
+
+Virt-builder-repository loops over the files in the directory specified
+as argument, compresses the files with a name ending by C<qcow2>, C<raw>,
+C<img> or without extension, extracts data from them and creates or
+updates the C<index> file.
+
+Some of the image-related data needed for the index file can’t be
+computed from the image file. virt-builder-repository first tries to
+find them in the existing index file. If data are still missing after
+this, they are prompted in interactive mode, otherwise an error will
+be triggered.
+
+If a C<KEYID> is provided, the generated index file will be signed
+with this GPG key.
+
+=head1 EXAMPLES
+
+=head2 Create the initial repository
+
+Create a folder and copy the disk image template files in it. Then
+run a command like the following one:
+
+ virt-builder-repository --gpg-key "joe at hacker.org" -i /path/to/folder
+
+Note that this example command runs in interactive mode. To run in
+automated mode, a minimal index file needs to be created before running
+the command containing sections like this one:
+
+ [template_id]
+ file=template_filename.qcow.xz
+
+The file value needs to match the image name extended with the C<.xz>
+suffix if the I<--no-compression> parameter is not provided or the
+image name if no compression is involved. Other optional data can be
+prefilled. Default values are computed by inspecting the disk image.
+For more informations, see
+L<virt-builder(1)/Creating and signing the index file>.
+
+=head2 Update images in an existing repository
+
+In this use case, an new image or a new revision of an existing image
+needs to be added to the repository. Place the corresponding image
+template files in the repository folder.
+
+To update the revision of an image, the file needs to have the same
+name than the existing one (without the C<xz> extension).
+
+As in the repository creation use case, a minimal fragment can be
+added to the index file for the automated mode. This can be done
+on the signed index even if it may sound a strange idea: the index
+will be signed again by the tool.
+
+To remove an image from the repository, just remove the corresponding
+image file before running virt-builder-repository.
+
+Then running the following command will complete and update the index
+file:
+
+ virt-builder-repository --gpg-key "joe at hacker.org" -i /path/to/folder
+
+virt-builder-repository works in a temporary folder inside the repository
+one. If anything wrong happens when running the tool, the repository is
+left untouched.
+
+=head1 OPTIONS
+
+=over 4
+
+=item B<--help>
+
+Display help.
+
+=item B<--gpg> GPG
+
+Specify an alternate L<gpg(1)> (GNU Privacy Guard) binary. You can
+also use this to add gpg parameters, for example to specify an
+alternate home directory:
+
+ virt-builder-repository --gpg "gpg --homedir /tmp" [...]
+
+This can also be used to avoid gpg asking for the key passphrase:
+
+ virt-builder-repository --gpg "gpg --passphrase-file /tmp/pass --batch" [...]
+
+=item B<-K> KEYID
+
+=item B<--gpg-key> KEYID
+
+Specify the GPG key to be used to sign the repository index file.
+If not provided, the index will left unsigned. C<KEYID> is used to
+identify the GPG key to use. This value is passed to gpg’s
+I<--default-key> option and can thus be an email address or a
+fingerprint.
+
+B<NOTE>: by default, virt-builder-repository searches for the key
+in the user’s GPG keyring.
+
+=item B<-i>
+
+=item B<--interactive>
+
+Prompt for missing data. Default values are computed from the disk
+image.
+
+When prompted for data, inputting C<-> corresponds to leaving the
+value empty. This can be used to avoid setting the default computed value.
+
+=item B<--keep-index>
+
+When using a GPG key, don’t remove the unsigned index.
+
+=item B<--no-compression>
+
+Don’t compress the template images.
+
+=item B<--machine-readable>
+
+This option is used to make the output more machine friendly
+when being parsed by other programs. See
+L</MACHINE READABLE OUTPUT> below.
+
+
+=item B<--colors>
+
+=item B<--colours>
+
+Use ANSI colour sequences to colourize messages. This is the default
+when the output is a tty. If the output of the program is redirected
+to a file, ANSI colour sequences are disabled unless you use this
+option.
+
+=item B<-q>
+
+=item B<--quiet>
+
+Don’t print ordinary progress messages.
+
+=item B<-v>
+
+=item B<--verbose>
+
+Enable debug messages and/or produce verbose output.
+
+When reporting bugs, use this option and attach the complete output to
+your bug report.
+
+=item B<-V>
+
+=item B<--version>
+
+Display version number and exit.
+
+=item B<-x>
+
+Enable tracing of libguestfs API calls.
+
+
+=back
+
+=head1 MACHINE READABLE OUTPUT
+
+The I<--machine-readable> option can be used to make the output more
+machine friendly, which is useful when calling virt-builder-repository from
+other programs, GUIs etc.
+
+Use the option on its own to query the capabilities of the
+virt-builder-repository binary. Typical output looks like this:
+
+ $ virt-builder-repository --machine-readable
+ virt-builder-repository
+
+A list of features is printed, one per line, and the program exits
+with status 0.
+
+=head1 EXIT STATUS
+
+This program returns 0 if successful, or non-zero if there was an
+error.
+
+=head1 SEE ALSO
+
+L<virt-builder(1)>
+L<http://libguestfs.org/>.
+
+=head1 AUTHOR
+
+Cédric Bosdonnat L<mailto:cbosdonnat at suse.com>
+
+=head1 COPYRIGHT
+
+Copyright (C) 2016-2017 SUSE Inc.
diff --git a/builder/virt-builder.pod b/builder/virt-builder.pod
index 74bf7bb11..1ed18a7c7 100644
--- a/builder/virt-builder.pod
+++ b/builder/virt-builder.pod
@@ -1319,6 +1319,9 @@ digital signature):
The part in square brackets is the C<os-version>, which is the same
string that is used on the virt-builder command line to build that OS.
+The index file creation and signature can be eased with the
+L<virt-builder-repository(1)> tool.
+
After preparing the C<index> file in the correct format, clearsign it
using the following command:
@@ -1875,6 +1878,7 @@ error.
L<guestfs(3)>,
L<guestfish(1)>,
L<guestmount(1)>,
+L<virt-builder-repository(1)>,
L<virt-copy-out(1)>,
L<virt-customize(1)>,
L<virt-get-kernel(1)>,
diff --git a/fish/guestfish.pod b/fish/guestfish.pod
index 11f2bbeb5..c37189bd8 100644
--- a/fish/guestfish.pod
+++ b/fish/guestfish.pod
@@ -1617,6 +1617,7 @@ L<guestfs(3)>,
L<http://libguestfs.org/>,
L<virt-alignment-scan(1)>,
L<virt-builder(1)>,
+L<virt-builder-repository(1)>,
L<virt-cat(1)>,
L<virt-copy-in(1)>,
L<virt-copy-out(1)>,
diff --git a/installcheck.sh.in b/installcheck.sh.in
index 6b05ab812..a4829cda6 100644
--- a/installcheck.sh.in
+++ b/installcheck.sh.in
@@ -46,6 +46,7 @@ cp @bindir@/guestfish fish/
cp @bindir@/guestmount fuse/
cp @bindir@/virt-alignment-scan align/
cp @bindir@/virt-builder builder/
+cp @bindir@/virt-builder-repository builder/
cp @bindir@/virt-cat cat/
cp @bindir@/virt-copy-in fish/
cp @bindir@/virt-copy-out fish/
diff --git a/lib/guestfs.pod b/lib/guestfs.pod
index 8d31f3200..55467e92e 100644
--- a/lib/guestfs.pod
+++ b/lib/guestfs.pod
@@ -3417,6 +3417,7 @@ L<guestfish(1)>,
L<guestmount(1)>,
L<virt-alignment-scan(1)>,
L<virt-builder(1)>,
+L<virt-builder-repository(1)>,
L<virt-cat(1)>,
L<virt-copy-in(1)>,
L<virt-copy-out(1)>,
--
2.14.3
More information about the Libguestfs
mailing list