Merge pull request #7 from hjelmn/master

Add mdtest to IOR repo and update backends to support mdtest

NEWS

@@ -0,0 +1,9 @@
+IOR NEWS
+========
+
+Last updated 2017-06
+
+3.0.2
+
+- IOR and mdtest now share a common codebase. This will make it easier
+run performance benchmarks on new hardware.

bootstrap

@@ -1,11 +1,3 @@
 #!/bin/sh
 
-set -x
-aclocal -I config || exit 1
-#libtoolize --automake --copy --force || exit 1
-autoheader || exit 1
-automake --add-missing --copy --force-missing --foreign || exit 1
-#autoconf --force --warnings=all || exit 1
-autoconf --force || exit 1
-rm -fr autom4te*.cache
-exit 0
+autoreconf -ivf

config/x_ac_meta.m4

@@ -11,6 +11,10 @@
 #    Read metadata from the META file.
 ##*****************************************************************************
 
+m4_define([META_FILE], m4_include([META]))
+m4_define([META_PACKAGE_NAME], regexp(META_FILE, [^Package:\s*\(.*\)], [\1]))
+m4_define([META_PACKAGE_VERSION], regexp(META_FILE, [^Version:\s*\(.*\)$], [\1]))
+
 AC_DEFUN([X_AC_META], [
   AC_MSG_CHECKING([metadata])
 

configure.ac

@@ -2,31 +2,38 @@
 # Process this file with autoconf to produce a configure script.
 
 AC_PREREQ([2.62])
-AC_INIT
+AC_CONFIG_MACRO_DIR([config])
+
+AC_INIT([META_PACKAGE_NAME],[META_PACKAGE_VERSION],[],[META_PACKAGE_NAME])
+
 X_AC_META
+
 AC_CONFIG_AUX_DIR([config])
 AC_CONFIG_SRCDIR([src/ior.c])
 AC_CONFIG_HEADER([src/config.h])
 
 AC_CANONICAL_HOST
 
 # Automake support
-AM_INIT_AUTOMAKE([$META_NAME], [$META_VERSION], [check-news dist-bzip2 gnu no-define])
+AM_INIT_AUTOMAKE([check-news dist-bzip2 gnu no-define])
 AM_MAINTAINER_MODE
 
 # Checks for programs.
 AX_PROG_CC_MPI
 
+# No reason not to require modern C at this point
+AC_PROG_CC_C99
+
 # Checks for libraries.
 
 # Checks for header files.
-AC_CHECK_HEADERS([fcntl.h libintl.h stdlib.h string.h strings.h sys/ioctl.h sys/param.h sys/statfs.h sys/statvfs.h sys/time.h unistd.h wchar.h plfs.h hdfs.h beegfs/beegfs.h])
+AC_CHECK_HEADERS([fcntl.h libintl.h stdlib.h string.h strings.h sys/ioctl.h sys/param.h sys/statfs.h sys/statvfs.h sys/time.h sys/param.h sys/mount.h unistd.h wchar.h hdfs.h beegfs/beegfs.h])
 
 # Checks for typedefs, structures, and compiler characteristics.
 AC_TYPE_SIZE_T
 
 # Checks for library functions.
-AC_CHECK_FUNCS([getpagesize gettimeofday memset mkdir pow putenv realpath regcomp sqrt strcasecmp strchr strerror strncasecmp strstr uname])
+AC_CHECK_FUNCS([getpagesize gettimeofday memset mkdir pow putenv realpath regcomp sqrt strcasecmp strchr strerror strncasecmp strstr uname statfs statvfs])
 AC_SEARCH_LIBS([sqrt], [m], [],
         [AC_MSG_ERROR([Math library not found])])
 
@@ -109,17 +116,6 @@ AM_COND_IF([USE_NCMPI_AIORI],[
         AC_DEFINE([USE_NCMPI_AIORI], [], [Build NCMPI backend AIORI])
 ])
 
-# PLFS IO support
-AC_ARG_WITH([plfs],
-        [AS_HELP_STRING([--with-plfs],
-           [support IO with PLFS backend @<:@default=no@:>@])],
-        [],
-        [with_plfs=no])
-AM_CONDITIONAL([USE_PLFS_AIORI], [test x$with_plfs = xyes])
-AM_COND_IF([USE_PLFS_AIORI],[
-        AC_DEFINE([USE_PLFS_AIORI], [], [Build PLFS backend AIORI])
-])
-
 # POSIX IO support
 AC_ARG_WITH([posix],
         [AS_HELP_STRING([--with-posix],

doc/Makefile.am

@@ -1 +1,2 @@
 dist_data_DATA = USER_GUIDE
+dist_man_MANS = mdtest.1

doc/mdtest.1

@@ -0,0 +1,188 @@
+.TH mdtest 1 "2010-05-05" "mdtest-1.8.3" "mdtest"
+.SH NAME
+mdtest \- test file system metadata performance
+.SH SYNOPSIS
+.B mdtest
+.I "[-options]"
+.SH DESCRIPTION
+.B mdtest
+is a file system metadata performance test designed to run
+in a cluster MPI environment against parallel file systems.
+.PP
+In each iteration of the test, each MPI task creates, stats, and removes
+the specified number of directories and/or files and measures the performance
+in ops/second.  After all the iterations complete, the maximum, minimum,
+mean ops/sec and the std. deviation are reported for each operation.
+.SH OPTIONS
+.TP
+.I "-b" branching_factor
+The branching factor of the hierarchical directory structure [default: 1].
+.TP
+.I "-B"
+No barriers will be taken between the phases (create/stat/remove) of the tests.
+.TP
+.I "-c"
+Use ``collective creates'', meaning task 0 does all the creates.
+.TP
+.I "-C"
+Only perform the create phase of the tests.
+.TP
+.I "-d" testdir[@testdir2]
+The directory in which the tests will run.  For multiple pathes, must use fully-qualified pathnames.
+[default: working directory of mdtest].
+.TP
+.I "-D"
+Perform test on directories only (no files).
+.TP
+.I "-e" bytes
+Set the number of Bytes to read from each file [default: 0].
+.TP
+.I "-E"
+Only perform the read phase of the tests.
+.TP
+.I "-f" first
+The first number of tasks on which the test will run
+[default: 0].
+.TP
+.I "-F"
+Perform test on files only (no directories).
+.TP
+.I "-h"
+Display help message.
+.TP
+.I "-i" iterations
+The number of iterations the test will run
+[default: 1].
+.TP
+.I "-I" items_per_directory
+The number of items per directory in the tree [default: 0].
+.TP
+.I "-l" last
+The last number of tasks on which the test will run
+[default: 0].
+.TP
+.I "-L"
+Files/directories only created at the leaf level of the tree.
+.TP
+.I "-n" number_of_items
+Every process will creat/stat/remove # directories and files
+[default: 0].
+.TP
+.I "-N" stride
+Stride # between neighbor tasks for file/dir stat, 0 = local
+[default: 0].
+.TP
+.I "-p" seconds
+Pre-iteration delay (in seconds).
+.TP
+.I "-r"
+Only perform the remove phase of the tests.
+.TP
+.I "-R[seed]"
+Randomly stat files. There is an optional argument that provides a seed
+to the random number generator. (Note: There is no space between the 
+.I "-R"
+ and
+the seed if one is provided.)
+.TP
+.I "-s" stride
+Stride between the number of tasks for each test
+[default: 1].
+.TP
+.I "-S"
+Shared file access (file only, no directories).
+.TP
+.I "-t"
+Include unique working directory management overhead in the results
+(presumes
+.I "-u"
+option).
+.TP
+.I "-T"
+Only perform the stat phase of the tests.
+.TP
+.I "-u"
+Create a unique working directory for each task
+(presumes
+.I "-d"
+option).
+.TP
+.I "-v"
+Increase verbosity (each instance of option increments by one).
+.TP
+.I "-V" value
+Set verbosity value
+[default: 0].
+.TP
+.I "-w" bytes
+Set the number of Bytes to write to each file after it is created
+[default: 0].
+.TP
+.I "-z" tree_depth
+The depth of the hierarchical directory tree [default: 0].
+.SH EXAMPLES
+.SS "Example 1"
+.nf
+$ mpirun -n 2 ./mdtest -d /tmp/z -n 100 -i 2
+
+-- started at 11/23/2009 09:05:29 --
+
+mdtest-1.8.1 was launched with 2 total task(s) on 1 nodes
+Command line used: ./mdtest -d /tmp/z -n 100 -i 2
+Path: /tmp
+FS: 28.8 GiB   Used FS: 8.6%   8.6%Inodes: 1.8 Mi   Used Inodes: 5.1%
+
+time to create tree: 0.000078 sec
+tree creation rate: 12826.617737 ops/sec
+
+2 tasks, 200 files/directories
+
+SUMMARY: (of 2 iterations)
+   Operation                  Max        Min       Mean    Std Dev
+   ---------                  ---        ---       ----    -------
+   Directory creation:  21489.415  17447.551  19468.483   2020.932
+   Directory stat    : 154657.227  28731.061  91694.144  62963.083
+   Directory removal : 146756.613  21489.415  84123.014  62633.599
+   File creation     :  42024.989  28731.061  35378.025   6646.964
+   File stat         : 146756.613  17447.551  82102.082  64654.531
+   File removal      : 156884.384  42024.989  99454.686  57429.698
+
+time to remove tree: 0.001031 sec
+tree removal rate: 970.005550 ops/sec
+
+-- finished at 11/23/2009 09:05:29 --
+.fi
+.SS "Example 2"
+.nf
+$ mpirun -np 2 -H pc6 ./mdtest -d /tmp/z -b 2 -z 3 -I 10
+
+-- started at 11/23/2009 09:09:23 --
+
+mdtest-1.8.1 was launched with 2 total task(s) on 1 nodes
+Command line used: ./mdtest -d /tmp/z -b 2 -z 3 -I 10
+Path: /tmp
+FS: 28.8 GiB   Used FS: 8.6%   8.6%Inodes: 1.8 Mi   Used Inodes: 5.1%
+
+time to create tree: 0.000765 sec
+tree creation rate: 19605.659084 ops/sec
+
+2 tasks, 300 files/directories
+
+SUMMARY: (of 1 iterations)
+   Operation                  Max        Min       Mean    Std Dev
+   ---------                  ---        ---       ----    -------
+   Directory creation:  29365.707  29365.707  29365.707      0.000
+   Directory stat    : 123701.455 123701.455 123701.455      0.000
+   Directory removal :  25623.459  25623.459  25623.459      0.000
+   File creation     :  38704.743  38704.743  38704.743      0.000
+   File stat         : 125477.782 125477.782 125477.782      0.000
+   File removal      :  51911.845  51911.845  51911.845      0.000
+
+time to remove tree: 0.000940 sec
+tree removal rate: 15960.060883 ops/sec
+
+-- finished at 11/23/2009 09:09:23 --
+.fi
+
+.SH "SEE ALSO"
+\fBhttp://sourceforge.net/projects/mdtest\fR