[Cluster-devel] [RFC RHEL7 GFS2 PATCH 3/3] gfs2: ordered write list addendum patch

[Steven Whitehouse]

On 12/12/17 17:22, Abhi Das wrote:
> Trim the list in gfs2_ordered_write() as we run through it
> to write out inodes.
> Also attempt to remove an inode from the list after it is
> fsync'ed.
> Finally, call gfs2_ordered_write() in case we were not able
> to shrink the list in gfs2_ordered_shrink() in the hopes that
> it will eventually cause the list to shrink.
> ---
>   fs/gfs2/file.c   |  3 +++
>   fs/gfs2/incore.h |  1 +
>   fs/gfs2/log.c    | 13 +++++++++++--
>   fs/gfs2/log.h    |  4 +++-
>   fs/gfs2/quota.c  |  4 ++--
>   5 files changed, 20 insertions(+), 5 deletions(-)
A further thought... eventually we'll probably need to build the bios 
ourselves in order that we can hook the I/O completion functions. That 
would be the best way to get notification that the I/O has completed. So 
something like a copy of the existing filemap/writepage functions but 
allowing us custom completions.

It would also be worth looking at the existing pagecache/mm code too, so 
see how the inode dirty flags get cleared after an fsync for example. 
That might be a useful reference for how to check that the inode is 
clean in a race free manner,

Steve.

>
> diff --git a/fs/gfs2/file.c b/fs/gfs2/file.c
> index 757ec66..75f9ac0 100644
> --- a/fs/gfs2/file.c
> +++ b/fs/gfs2/file.c
> @@ -697,6 +697,9 @@ static int gfs2_fsync(struct file *file, loff_t start, loff_t end,
>   	if (mapping->nrpages)
>   		ret = filemap_fdatawait_range(mapping, start, end);
>   
> +	if (!ret && !ret1)
> +		gfs2_ordered_del_inode(ip, ORD_WHENCE_FSYNC);
> +
>   	return ret ? ret : ret1;
>   }
>   
> diff --git a/fs/gfs2/incore.h b/fs/gfs2/incore.h
> index 6fcad2a..93da360 100644
> --- a/fs/gfs2/incore.h
> +++ b/fs/gfs2/incore.h
> @@ -661,6 +661,7 @@ struct ord_stats {
>   	unsigned long os_rm_trunc;
>   	unsigned long os_rm_evict;
>   	unsigned long os_rm_wait;
> +	unsigned long os_rm_fsync;
>   	unsigned long os_rm_syncfs;
>   	unsigned long os_rm_write;
>   	unsigned long os_rm_setflags;
> diff --git a/fs/gfs2/log.c b/fs/gfs2/log.c
> index 6d618a1..4cfef47 100644
> --- a/fs/gfs2/log.c
> +++ b/fs/gfs2/log.c
> @@ -507,9 +507,13 @@ static void gfs2_ordered_write(struct gfs2_sbd *sdp)
>   	list_sort(NULL, &sdp->sd_log_le_ordered, &ip_cmp);
>   	while (!list_empty(&sdp->sd_log_le_ordered)) {
>   		ip = list_entry(sdp->sd_log_le_ordered.next, struct gfs2_inode, i_ordered);
> -		list_move(&ip->i_ordered, &written);
> -		if (ip->i_inode.i_mapping->nrpages == 0)
> +		if (ip->i_inode.i_mapping->nrpages == 0) {
> +			test_and_clear_bit(GIF_ORDERED, &ip->i_flags);
> +			list_del(&ip->i_ordered);
> +			ord_stats_adjust(sdp, -1, ORD_WHENCE_ORD_WRITE);
>   			continue;
> +		}
> +		list_move(&ip->i_ordered, &written);
>   		spin_unlock(&sdp->sd_ordered_lock);
>   		filemap_fdatawrite(ip->i_inode.i_mapping);
>   		spin_lock(&sdp->sd_ordered_lock);
> @@ -540,17 +544,22 @@ static void gfs2_ordered_wait(struct gfs2_sbd *sdp)
>   void gfs2_ordered_shrink(struct gfs2_sbd *sdp, int whence)
>   {
>   	struct gfs2_inode *ip, *tmp;
> +	bool removed;
>   
>   	spin_lock(&sdp->sd_ordered_lock);
>   	list_for_each_entry_safe(ip, tmp, &sdp->sd_log_le_ordered, i_ordered) {
>   		if (ip->i_inode.i_mapping->nrpages != 0)
>   			continue;
>   		if (test_and_clear_bit(GIF_ORDERED, &ip->i_flags)) {
> +			removed = true;
>   			list_del(&ip->i_ordered);
>   			ord_stats_adjust(sdp, -1, whence);
>   		}
>   	}
>   	spin_unlock(&sdp->sd_ordered_lock);
> +
> +	if (!removed)
> +		gfs2_ordered_write(sdp);
>   }
>   
>   void gfs2_ordered_del_inode(struct gfs2_inode *ip, int whence)
> diff --git a/fs/gfs2/log.h b/fs/gfs2/log.h
> index 80c8861..0bc3620 100644
> --- a/fs/gfs2/log.h
> +++ b/fs/gfs2/log.h
> @@ -55,7 +55,8 @@ enum {
>   	ORD_WHENCE_SYNCFS       = 3,
>   	ORD_WHENCE_ORD_WRITE    = 4,
>   	ORD_WHENCE_SETFLAGS     = 5,
> -	ORD_WHENCE_ADD          = 6,
> +	ORD_WHENCE_FSYNC        = 6,
> +	ORD_WHENCE_ADD          = 7,
>   };
>   
>   static inline void ord_stats_adjust(struct gfs2_sbd *sdp, int count, int whence)
> @@ -70,6 +71,7 @@ static inline void ord_stats_adjust(struct gfs2_sbd *sdp, int count, int whence)
>   	case ORD_WHENCE_SYNCFS:     os->os_rm_syncfs += -(count); break;
>   	case ORD_WHENCE_ORD_WRITE:  os->os_rm_write += -(count); break;
>   	case ORD_WHENCE_SETFLAGS:   os->os_rm_setflags += -(count); break;
> +	case ORD_WHENCE_FSYNC:      os->os_rm_fsync += -(count); break;
>   
>   	case ORD_WHENCE_ADD:        os->os_add += count; break;
>   	default: break;
> diff --git a/fs/gfs2/quota.c b/fs/gfs2/quota.c
> index 66c5126..63e3afa 100644
> --- a/fs/gfs2/quota.c
> +++ b/fs/gfs2/quota.c
> @@ -1575,11 +1575,11 @@ int gfs2_quotad(void *data)
>   
>   		printk(KERN_WARNING "Ord list Size:%lu +[add_inode:%lu] "
>   		       "-[trunc:%lu evict:%lu wait:%lu syncfs:%lu ord_write:%lu"
> -		       " setflags:%lu]\n", sdp->sd_ord_stats.os_ct,
> +		       " setflags:%lu fsync:%lu]\n", sdp->sd_ord_stats.os_ct,
>   		       sdp->sd_ord_stats.os_add, sdp->sd_ord_stats.os_rm_trunc,
>   		       sdp->sd_ord_stats.os_rm_evict, sdp->sd_ord_stats.os_rm_wait,
>   		       sdp->sd_ord_stats.os_rm_syncfs, sdp->sd_ord_stats.os_rm_write,
> -		       sdp->sd_ord_stats.os_rm_setflags);
> +		       sdp->sd_ord_stats.os_rm_setflags, sdp->sd_ord_stats.os_rm_fsync);
>   	}
>   
>   	return 0;