[Virtio-fs] Regression: Docker in vms broken.

Wed May 19 16:17:22 UTC 2021

On Wed, May 19, 2021 at 08:35:59AM -0500, Harry G. Coin wrote:
> Something changed in virtiofs, likely to do with overlay support? 

I am not aware of any change in virtiofs w.r.t overlay. Are there any
messages on VM serail console from guest kernel? 

Vivek

> 
> Docker in a vm with virtiofs as root running with kernel 
> 5.8.0-53-generic works normally.  However, later 5.8 versions and also
> 5.11 versions fail with:
> 
> 
> May 18 22:00:07 nocsupport4 systemd[1]: docker.service: Scheduled
> restart job, restart counter is at 2.
> May 18 22:00:10 nocsupport4 dockerd[1104]:
> time="2021-05-18T22:00:10.061661124-05:00" level=info msg="Starting up"
> May 18 22:00:10 nocsupport4 dockerd[1104]:
> time="2021-05-18T22:00:10.109655841-05:00" level=info msg="detected
> 127.0.0.53 nameserver, assuming systemd-resolved, so using resolv.conf:
> /run/systemd/resolve/resolv.conf"
> May 18 22:00:10 nocsupport4 dockerd[1104]:
> time="2021-05-18T22:00:10.253580688-05:00" level=info msg="parsed
> scheme: \"unix\"" module=grpc
> May 18 22:00:10 nocsupport4 dockerd[1104]:
> time="2021-05-18T22:00:10.253663917-05:00" level=info msg="scheme
> \"unix\" not registered, fallback to default scheme" module=grpc
> May 18 22:00:10 nocsupport4 dockerd[1104]:
> time="2021-05-18T22:00:10.253737208-05:00" level=info
> msg="ccResolverWrapper: sending update to cc:
> {[{unix:///run/containerd/containerd.sock  <nil> 0 <nil>}] <nil> <nil>}"
> module=grpc
> May 18 22:00:10 nocsupport4 dockerd[1104]:
> time="2021-05-18T22:00:10.253777767-05:00" level=info msg="ClientConn
> switching balancer to \"pick_first\"" module=grpc
> May 18 22:00:10 nocsupport4 dockerd[1104]:
> time="2021-05-18T22:00:10.272602278-05:00" level=info msg="parsed
> scheme: \"unix\"" module=grpc
> May 18 22:00:10 nocsupport4 dockerd[1104]:
> time="2021-05-18T22:00:10.272713227-05:00" level=info msg="scheme
> \"unix\" not registered, fallback to default scheme" module=grpc
> May 18 22:00:10 nocsupport4 dockerd[1104]:
> time="2021-05-18T22:00:10.272778900-05:00" level=info
> msg="ccResolverWrapper: sending update to cc:
> {[{unix:///run/containerd/containerd.sock  <nil> 0 <nil>}] <nil> <nil>}"
> module=grpc
> May 18 22:00:10 nocsupport4 dockerd[1104]:
> time="2021-05-18T22:00:10.272816740-05:00" level=info msg="ClientConn
> switching balancer to \"pick_first\"" module=grpc
> May 18 22:00:10 nocsupport4 dockerd[1104]:
> time="2021-05-18T22:00:10.367601799-05:00" level=error msg="failed to
> mount overlay: invalid argument" storage-driver=overlay2
> May 18 22:00:10 nocsupport4 dockerd[1104]:
> time="2021-05-18T22:00:10.367792469-05:00" level=error
> msg="[graphdriver] prior storage driver overlay2 failed: driver not
> supported"
> May 18 22:00:10 nocsupport4 dockerd[1104]:
> time="2021-05-18T22:00:10.370914131-05:00" level=info msg="stopping
> event stream following graceful shutdown" error="context canceled"
> module=libcontainerd namespace=plugins.moby
> May 18 22:00:10 nocsupport4 dockerd[1104]: failed to start daemon: error
> initializing graphdriver: driver not supported
> May 18 22:00:10 nocsupport4 systemd[1]: docker.service: Main process
> exited, code=exited, status=1/FAILURE
> May 18 22:00:10 nocsupport4 systemd[1]: docker.service: Failed with
> result 'exit-code'.
> 
> root at nocsupport4:~# cat /etc/fstab
> myfs / virtiofs defaults 0 0
> 
> On the host:
> 
> root at noc4:/vmsystems# uname -a
> Linux noc4.xxxx 5.11.0-17-generic
> 
> root      225434  0.0  0.0  80068  3544 ?        Sl   21:32   0:00
> /usr/lib/qemu/virtiofsd --fd=32 -o
> source=/vmsystems/nocsupport4,xattr,flock,no_posix_lock -o writeback
> libvirt+  225458 85.5  0.7 4366196 236724 ?      Sl   21:32   7:34
> /usr/bin/qemu-system-x86_64 -name guest=nocsupport4,debug-threads=on -S
> -object
> secret,id=masterKey0,format=raw,file=/var/lib/libvirt/qemu/domain-6-nocsupport4/master-key.aes
> -machine pc-q35-5.0,accel=kvm,usb=off,vmport=off,dump-guest-core=off
> -cpu
> Conroe,vme=on,vmx=on,cx16=on,pdcm=on,x2apic=on,tsc-deadline=on,hypervisor=on,arat=on,tsc-adjust=on,arch-capabilities=on,pdpe1gb=on,skip-l1dfl-vmentry=on,pschange-mc-no=on
> -m 3584 -overcommit mem-lock=off -smp 2,sockets=2,cores=1,threads=1
> -object
> memory-backend-file,id=ram-node0,mem-path=/dev/hugepages/libvirt/qemu/6-nocsupport4,share=yes,prealloc=yes,size=3758096384
> -numa node,nodeid=0,cpus=0-1,memdev=ram-node0 -uuid
> c7b055aa-1874-4ba9-9302-26fad866e749 -no-user-config -nodefaults
> -chardev socket,id=charmonitor,fd=32,server,nowait -mon
> chardev=charmonitor,id=monitor,mode=control -rtc base=utc,driftfix=slew
> -global kvm-pit.lost_tick_policy=delay -no-hpet -no-shutdown -global
> ICH9-LPC.disable_s3=1 -global ICH9-LPC.disable_s4=1 -boot strict=on
> -kernel /vmsystems/nocsupport4/boot/vmlinuz -initrd
> /vmsystems/nocsupport4/boot/initrd.img -append rootfstype=virtiofs
> root=myfs  rw -device
> pcie-root-port,port=0x10,chassis=1,id=pci.1,bus=pcie.0,multifunction=on,addr=0x2
> -device
> pcie-root-port,port=0x11,chassis=2,id=pci.2,bus=pcie.0,addr=0x2.0x1
> -device
> pcie-root-port,port=0x12,chassis=3,id=pci.3,bus=pcie.0,addr=0x2.0x2
> -device
> pcie-root-port,port=0x13,chassis=4,id=pci.4,bus=pcie.0,addr=0x2.0x3
> -device
> pcie-root-port,port=0x14,chassis=5,id=pci.5,bus=pcie.0,addr=0x2.0x4
> -device
> pcie-root-port,port=0x15,chassis=6,id=pci.6,bus=pcie.0,addr=0x2.0x5
> -device
> pcie-root-port,port=0x16,chassis=7,id=pci.7,bus=pcie.0,addr=0x2.0x6
> -device
> pcie-root-port,port=0x17,chassis=8,id=pci.8,bus=pcie.0,addr=0x2.0x7
> -device pcie-pci-bridge,id=pci.9,bus=pci.4,addr=0x0 -device
> ich9-usb-ehci1,id=usb,bus=pcie.0,addr=0x1d.0x7 -device
> ich9-usb-uhci1,masterbus=usb.0,firstport=0,bus=pcie.0,multifunction=on,addr=0x1d
> -device
> ich9-usb-uhci2,masterbus=usb.0,firstport=2,bus=pcie.0,addr=0x1d.0x1
> -device
> ich9-usb-uhci3,masterbus=usb.0,firstport=4,bus=pcie.0,addr=0x1d.0x2
> -device virtio-serial-pci,id=virtio-serial0,bus=pci.3,addr=0x0 -chardev
> socket,id=chr-vu-fs0,path=/var/lib/libvirt/qemu/domain-6-nocsupport4/fs0-fs.sock
> -device
> vhost-user-fs-pci,chardev=chr-vu-fs0,queue-size=1024,tag=myfs,bus=pci.1,addr=0x0
> -netdev tap,fd=46,id=hostnet0,vhost=on,vhostfd=47 -device
> virtio-net-pci,netdev=hostnet0,id=net0,mac=52:54:00:30:4d:ca,bus=pci.2,addr=0x0
> -chardev pty,id=charserial0 -device
> isa-serial,chardev=charserial0,id=serial0 -chardev
> socket,id=charchannel0,fd=48,server,nowait -device
> virtserialport,bus=virtio-serial0.0,nr=1,chardev=charchannel0,id=channel0,name=org.qemu.guest_agent.0
> -chardev spicevmc,id=charchannel1,name=vdagent -device
> virtserialport,bus=virtio-serial0.0,nr=2,chardev=charchannel1,id=channel1,name=com.redhat.spice.0
> -device usb-tablet,id=input0,bus=usb.0,port=1 -spice
> port=5900,addr=127.0.0.1,disable-ticketing,seamless-migration=on -device
> qxl-vga,id=video0,ram_size=67108864,vram_size=67108864,vram64_size_mb=0,vgamem_mb=16,max_outputs=1,bus=pcie.0,addr=0x1
> -device ich9-intel-hda,id=sound0,bus=pcie.0,addr=0x1b -device
> hda-duplex,id=sound0-codec0,bus=sound0.0,cad=0 -device
> i6300esb,id=watchdog0,bus=pci.9,addr=0x1 -watchdog-action reset -chardev
> spicevmc,id=charredir0,name=usbredir -device
> usb-redir,chardev=charredir0,id=redir0,bus=usb.0,port=2 -chardev
> spicevmc,id=charredir1,name=usbredir -device
> usb-redir,chardev=charredir1,id=redir1,bus=usb.0,port=3 -device
> virtio-balloon-pci,id=balloon0,bus=pci.5,addr=0x0 -object
> rng-random,id=objrng0,filename=/dev/urandom -device
> virtio-rng-pci,rng=objrng0,id=rng0,bus=pci.6,addr=0x0 -sandbox
> on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny
> -msg timestamp=on
> root      225472 11.0  0.0 6889960 10848 ?       Sl   21:32   0:58
> /usr/lib/qemu/virtiofsd --fd=32 -o
> source=/vmsystems/nocsupport4,xattr,flock,no_posix_lock -o writeback
> 
> 
> 
> 
> 
> 
> _______________________________________________
> Virtio-fs mailing list
> Virtio-fs at redhat.com
> https://listman.redhat.com/mailman/listinfo/virtio-fs