[dm-devel] [Question] multipathd add/remove paths takes a long time

[Wu Guanghao]

在 2022/7/21 22:34, Benjamin Marzinski 写道:
> On Tue, Jul 19, 2022 at 08:13:39PM +0800, Wu Guanghao wrote:
>> The system has 1K multipath devices, each device has 16 paths.
>> Execute multipathd add/multipathd remove or uev_add_path/
>> uev_remove_path to add/remove paths, which takes over 20s.
>> What's more, the second checkloop may be execed immediately
>> after finishing first checkloop. It's too long.
>>
>> We found that time was mostly spent waiting for locks.
>>
>> checkerloop(){
>> 	...
>> 	lock(&vecs->lock);
>> 	vector_foreach_slot (vecs->pathvec, pp, i) {
>> 		rc = check_path(...); // Too many paths, it takes a long time
>> 		...
>> 	}
>> 	lock_cleanup_pop(vecs->lock);
>> 	...
>> }
>>
>> Can the range of vecs->lock locks be adjusted to reduce the time consuming
>> when adding/removing paths?
> 
> As long as we make sure not to skip any paths or double-check any paths,
> we don't need to hold the vecs->lock between checking paths. There is
> certainly some optimization that could get done here.
> 
> could you post the output of:
> 
> # multipath -l <sample_multipath_device>
> # multipathd show config local
> 

This is the output and time consuming of 'multipath -l'

# time multipath -l 364cf55b10097699e01a9c4d4000003c4
174181.329435 | loading /lib64/multipath/libchecktur.so checker
364cf55b10097699e01a9c4d4000003c4 dm-604 HUAWEI,XSG1
size=1.0G features='1 queue_if_no_path' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=0 status=active
  |- 27:0:0:591  sdlgf 70:8368   active undef running
  |- 28:0:0:591  sdlhs 129:8224  active undef running
  |- 39:0:0:591  sdlst 131:8464  active undef running
  |- 24:0:0:591  sdmdv 133:8720  active undef running
  |- 40:0:0:591  sdmej 133:8944  active undef running
  |- 29:0:0:591  sdmes 134:8832  active undef running
  |- 37:0:0:591  sdmhl 66:9200   active undef running
  |- 38:0:0:591  sdmju 70:9152   active undef running
  |- 26:0:0:591  sdmjx 70:9200   active undef running
  |- 41:0:0:591  sdnre 133:9728  active undef running
  |- 43:0:0:591  sdnrz 134:9808  active undef running
  `- 42:0:0:591  sdnsp 135:9808  active undef running

real    0m37.570s
user    0m28.676s
sys     0m8.112s

This is the output of 'multipathd show config local'

timeout
defaults {
	verbosity 2
	polling_interval 5
	max_polling_interval 20
	reassign_maps "no"
	multipath_dir "/lib64/multipath"
	path_selector "service-time 0"
	path_grouping_policy "group_by_prio"
	uid_attribute "ID_SERIAL"
	prio "const"
	prio_args ""
	features "0"
	path_checker "tur"
	alias_prefix "mpath"
	failback "immediate"
	rr_min_io 1000
	rr_min_io_rq 1
	max_fds "max"
	rr_weight "uniform"
	no_path_retry 18
	queue_without_daemon "no"
	flush_on_last_del "no"
	user_friendly_names "no"
	fast_io_fail_tmo 5
	bindings_file "/etc/multipath/bindings"
	wwids_file "/etc/multipath/wwids"
	prkeys_file "/etc/multipath/prkeys"
	log_checker_err once
	reservation_key file
	all_tg_pt "no"
	retain_attached_hw_handler "yes"
	detect_prio "yes"
	detect_checker "yes"
	force_sync "no"
	strict_timing "no"
	deferred_remove "yes"
	config_dir "/etc/multipath/conf.d"
	delay_watch_checks "no"
	delay_wait_checks "no"
	san_path_err_threshold "no"
	san_path_err_forget_rate "no"
	san_path_err_recovery_time "no"
	marginal_path_err_sample_time "no"
	marginal_path_err_rate_threshold "no"
	marginal_path_err_recheck_gap_time "no"
	marginal_path_double_failed_time "no"
	find_multipaths "off"
	uxsock_timeout 4000
	retrigger_tries 3
	retrigger_delay 10
	missing_uev_wait_timeout 30
	skip_kpartx "no"
	disable_changed_wwids ignored
	remove_retries 0
	ghost_delay "no"
	find_multipaths_timeout -10
	enable_foreign ""
	marginal_pathgroups "no"
}
blacklist {
	devnode "^(ram|zram|raw|loop|fd|md|dm-|sr|scd|st|dcssblk)[0-9]"
	devnode "^(td|hd|vd)[a-z]"
	device {
		vendor "SGI"
		product "Universal Xport"
	}
	device {
		vendor "^DGC"
		product "LUNZ"
	}
	device {
		vendor "EMC"
		product "LUNZ"
	}
	device {
		vendor "DELL"
		product "Universal Xport"
	}
	device {
		vendor "IBM"
		product "Universal Xport"
	}
	device {
		vendor "LENOVO"
		product "Universal Xport"
	}
	device {
		vendor "(NETAPP|LSI|ENGENIO)"
		product "Universal Xport"
	}
	device {
		vendor "STK"
		product "Universal Xport"
	}
	device {
		vendor "SUN"
		product "Universal Xport"
	}
	device {
		vendor "(Intel|INTEL)"
		product "VTrak V-LUN"
	}
	device {
		vendor "Promise"
		product "VTrak V-LUN"
	}
	device {
		vendor "Promise"
		product "Vess V-LUN"
	}
	device {
		vendor "IBM"
		product "S/390.*"
	}
}
blacklist_exceptions {
}
devices {
	device {
		vendor "HUAWEI"
		product "XSG1"
		path_grouping_policy "group_by_prio"
		prio "alua"
	}
}
overrides {
}
multipaths {
	multipath {
		wwid "364cf55b10097699e0197320300000174"
	}
	multipath {
		wwid "364cf55b10097699e018d9d9500000171"
	}
	...
}
> -Ben
> 
> .
>