[Linux-cluster] Slowness above 500 RRDs

[David Teigland]

On Tue, Apr 24, 2007 at 02:36:00PM -0500, David Teigland wrote:
> On Mon, Apr 23, 2007 at 04:17:18PM -0500, David Teigland wrote:
> > > Also, what's that new infrastructure?  Do you mean GFS2?  I read it
> > > was not production quality yet, so I didn't mean to try it.  But again
> > > you may have got something else in your head...
> > 
> > GFS1 and GFS2 both run on the new openais-based cluster infrastructure.
> > (in the cluster-2.00.00 release, and the RHEL5 and HEAD cvs branches).
> 
> I've attached a little flock/plock performance test that emulates what
> you're doing; could you run it on your cluster and send the results?

Attached now

-------------- next part --------------
#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
#include <string.h>
#include <errno.h>
#include <sys/stat.h>
#include <sys/time.h>
#include <sys/types.h>
#include <sys/file.h>
#include <fcntl.h>
#include <unistd.h>

#define die(fmt, args...) \
{ \
	fprintf(stderr, "%s: ", prog_name); \
	fprintf(stderr, fmt, ##args); \
	exit(EXIT_FAILURE); \
}

char *prog_name;
unsigned int wait = 1;
int delay;
int pid;
int files[5000];

int do_plock(int fd, unsigned long long offset, int len, int type)
{
	struct flock lock;
	int action;

	lock.l_type = type;
	lock.l_start = offset;
	lock.l_whence = SEEK_SET;
	lock.l_len = len;

	if (wait)
		action = F_SETLKW;
	else
		action = F_SETLK;

	return fcntl(fd, action, &lock);
}

void read_plock(int fd, unsigned long long offset, int len)
{
	int error;

	error = do_plock(fd, offset, len, F_RDLCK);
	if (error < 0) {
		printf("%d read lock failed %d errno %d\n", pid, error, errno);
		exit(EXIT_FAILURE);
	}
}

void write_plock(int fd, unsigned long long offset, int len)
{
	int error;

	error = do_plock(fd, offset, len, F_WRLCK);
	if (error < 0) {
		printf("%d write lock failed %d errno %d\n", pid, error, errno);
		exit(EXIT_FAILURE);
	}
}

void unlock_plock(int fd, unsigned long long offset, int len)
{
	int error;

	error = do_plock(fd, offset, len, F_UNLCK);
	if (error < 0) {
		printf("%d unlock failed %d errno %d\n", pid, error, errno);
		exit(EXIT_FAILURE);
	}
}

void read_flock(int fd)
{
	int error;
	int mode = LOCK_SH;

	if (!wait)
		mode |= LOCK_NB;

	error = flock(fd, mode);
	if (error < 0) {
		printf("%d read lock failed %d errno %d\n", pid, error, errno);
		exit(EXIT_FAILURE);
	}
}

void write_flock(int fd)
{
	int error;
	int mode = LOCK_EX;

	if (!wait)
		mode |= LOCK_NB;

	error = flock(fd, mode);
	if (error < 0) {
		printf("%d write lock failed %d errno %d\n", pid, error, errno);
		exit(EXIT_FAILURE);
	}
}

void unlock_flock(int fd)
{
	int error;

	error = flock(fd, LOCK_UN);
	if (error < 0) {
		printf("%d unlock failed %d errno %d\n", pid, error, errno);
		exit(EXIT_FAILURE);
	}
}

uint64_t dt_usec(struct timeval *start, struct timeval *stop)
{
	uint64_t dt;

	dt = stop->tv_sec - start->tv_sec;
	dt *= 1000000;
	dt += stop->tv_usec - start->tv_usec;
	return dt;
}

void file_loop(int filecount, int flock)
{
	char path[64];
	struct timeval enter, exit, begin, end;
	uint64_t et_usec;
	int x, i, fd;

	gettimeofday(&enter, NULL);
	for (x = 0; x < 5; x++) {
		gettimeofday(&begin, NULL);
		for (i = 0; i < filecount; i++) {

			memset(path, 0, sizeof(path));
			sprintf(path, "file%.10u", i);
			fd = open(path, O_RDWR, 0644);

			if (flock) {
				read_flock(fd);
				unlock_flock(fd);
			} else {
				read_plock(fd, 0, 0);
				unlock_plock(fd, 0, 0);
			}

			close(fd);
		}
		gettimeofday(&end, NULL);

		et_usec = dt_usec(&begin, &end);
		printf("%s: filecount=%d iteration=%d elapsed time=%.3f s\n",
			flock ? "flock" : "plock", filecount, x,
			(1.e-6 * et_usec));
	}
	gettimeofday(&exit, NULL);

	et_usec = dt_usec(&enter, &exit);
	printf("total elapsed time=%.3f s\n", (1.e-6 * et_usec));

	if (delay) {
		printf("%d sec delay...\n", delay);
		sleep(delay);
	}
}

void create_all(int filecount)
{
	int i, fd;
	char path[64];

	for (i = 0; i < filecount; i++) {
		memset(path, 0, sizeof(path));
		sprintf(path, "file%.10u", i);
		fd = open(path, O_RDWR | O_CREAT, 0644);
		if (fd < 0)
			die("can't create file %s\n", strerror(errno));
		close(fd);
	}
}

void open_all(int filecount)
{
	int i, fd;
	char path[64];

	for (i = 0; i < filecount; i++) {
		memset(path, 0, sizeof(path));
		sprintf(path, "file%.10u", i);
		files[i] = fd = open(path, O_RDWR | O_CREAT, 0644);
		if (fd < 0)
			die("can't create file %s\n", strerror(errno));
	}
}

void close_all(int filecount)
{
	int i;

	for (i = 0; i < filecount; i++)
		close(files[i]);
}

int main(int argc, char *argv[])
{
	pid = getpid();
	prog_name = argv[0];

	if (argc > 1)
		delay = atoi(argv[1]);

	create_all(5000);

	file_loop(100, 1);
	file_loop(500, 1);
	file_loop(1000, 1);
	file_loop(2000, 1);
	file_loop(5000, 1);

	file_loop(100, 0);
	file_loop(500, 0);
	file_loop(1000, 0);
	file_loop(2000, 0);
	file_loop(5000, 0);

	exit(EXIT_SUCCESS);
}