Add OceanFS support for romio

src/mpi/romio/adio/Makefile.mk

@@ -37,4 +37,5 @@ include $(top_srcdir)/adio/ad_ufs/Makefile.mk
 include $(top_srcdir)/adio/ad_xfs/Makefile.mk
 include $(top_srcdir)/adio/ad_ime/Makefile.mk
 include $(top_srcdir)/adio/ad_quobytefs/Makefile.mk
+include $(top_srcdir)/adio/ad_oceanfs/Makefile.mk
 include $(top_srcdir)/adio/common/Makefile.mk

src/mpi/romio/adio/ad_oceanfs/Makefile.mk

@@ -0,0 +1,14 @@
+##
+## Copyright (C) by Argonne National Laboratory
+## See COPYRIGHT in top-level directory.
+##
+
+if BUILD_AD_OCEANFS
+
+noinst_HEADERS += adio/ad_oceanfs/ad_oceanfs.h
+
+romio_other_sources += \
+ adio/ad_oceanfs/ad_oceanfs.c \
+ adio/ad_oceanfs/ad_oceanfs_open.c
+
+endif BUILD_AD_OCEANFS

src/mpi/romio/adio/ad_oceanfs/ad_oceanfs.c

@@ -0,0 +1,42 @@
+/*
+* Copyright (C) by Argonne National Laboratory
+* See COPYRIGHT in top-level directory.
+*/
+
+#include "ad_oceanfs.h"
+#include "adioi.h"
+
+struct ADIOI_Fns_struct ADIO_OCEANFS_operations = {
+ ADIOI_OCEANFS_Open, /* Open */
+ ADIOI_GEN_OpenColl, /* OpenColl */
+ ADIOI_GEN_ReadContig, /* ReadContig */
+ ADIOI_GEN_WriteContig, /* WriteContig */
+ ADIOI_GEN_ReadStridedColl, /* ReadStridedColl */
+ ADIOI_GEN_WriteStridedColl, /* WriteStridedColl */
+ ADIOI_GEN_SeekIndividual, /* SeekIndividual */
+ ADIOI_GEN_Fcntl, /* Fcntl */
+ ADIOI_GEN_SetInfo, /* SetInfo */
+ ADIOI_GEN_ReadStrided, /* ReadStrided */
+ ADIOI_GEN_WriteStrided, /* WriteStrided */
+ ADIOI_GEN_Close, /* Close */
+ ADIOI_FAKE_IreadContig, /* IreadContig */
+ ADIOI_FAKE_IwriteContig, /* IwriteContig */
+ ADIOI_FAKE_IODone, /* ReadDone */
+ ADIOI_FAKE_IODone, /* WriteDone */
+ ADIOI_FAKE_IOComplete, /* ReadComplete */
+ ADIOI_FAKE_IOComplete, /* WriteComplete */
+ ADIOI_FAKE_IreadStrided, /* IreadStrided */
+ ADIOI_FAKE_IwriteStrided, /* IwriteStrided */
+ ADIOI_GEN_Flush, /* Flush */
+ ADIOI_GEN_Resize, /* Resize */
+ ADIOI_GEN_Delete, /* Delete */
+ ADIOI_GEN_Feature, /* Feature */
+ "OCEANFS: ROMIO driver for OCEANFS",
+ ADIOI_GEN_IreadStridedColl, /* IreadStridedColl */
+ ADIOI_GEN_IwriteStridedColl, /* IwriteStridedColl */
+#if defined(F_SETLKW64)
+ ADIOI_GEN_SetLock /* SetLock */
+#else
+ ADIOI_GEN_SetLock64 /* SetLock */
+#endif
+};

src/mpi/romio/adio/ad_oceanfs/ad_oceanfs.h

@@ -0,0 +1,13 @@
+/*
+* Copyright (C) by Argonne National Laboratory
+* See COPYRIGHT in top-level directory.
+*/
+
+#ifndef AD_OCEANFS_H_INCLUDED
+#define AD_OCEANFS_H_INCLUDED
+
+#include "adio.h"
+
+void ADIOI_OCEANFS_Open(ADIO_File fd, int *error_code);
+
+#endif /* AD_OCEANFS_H_INCLUDED */

src/mpi/romio/adio/ad_oceanfs/ad_oceanfs_open.c

@@ -0,0 +1,150 @@
+/*
+* Copyright (C) by Argonne National Laboratory
+* See COPYRIGHT in top-level directory.
+*/
+
+#include "ad_oceanfs.h"
+#include <sys/ioctl.h>
+
+typedef struct oceanfs_group_lock {
+ int fd;
+ u_int64_t lock_id;
+} oceanfs_group_lock_t;
+
+#define OCEANFS_IOCTL_GET_GROUPLOCK _IOWR('S', 103, oceanfs_group_lock_t)
+#define OCEANFS_IOCTL_SET_GROUPLOCK _IOWR('S', 111, oceanfs_group_lock_t)
+
+int ADIOI_OCEANFS_GetGroupLock(int fd, u_int64_t * lock_id)
+{
+ oceanfs_group_lock_t group_lock;
+ group_lock.fd = fd;
+ int ret = ioctl(fd, OCEANFS_IOCTL_GET_GROUPLOCK, &group_lock);
+ if (ret) {
+ *lock_id = 0;
+ return ret;
+ }
+
+ *lock_id = group_lock.lock_id;
+ return ret;
+}
+
+int ADIOI_OCEANFS_SetGroupLock(int fd, u_int64_t lock_id)
+{
+ oceanfs_group_lock_t group_lock;
+ group_lock.fd = fd;
+ group_lock.lock_id = lock_id;
+ return ioctl(fd, OCEANFS_IOCTL_SET_GROUPLOCK, &group_lock);
+}
+
+int ADIOI_OCEANFS_GetMode(ADIO_File fd)
+{
+ int amode = 0;
+ /* setup the file access mode */
+ if (fd->access_mode & ADIO_CREATE) {
+ amode = amode | O_CREAT;
+ }
+ if (fd->access_mode & ADIO_RDONLY) {
+ amode = amode | O_RDONLY;
+ }
+ if (fd->access_mode & ADIO_WRONLY) {
+ amode = amode | O_WRONLY;
+ }
+ if (fd->access_mode & ADIO_RDWR) {
+ amode = amode | O_RDWR;
+ }
+ if (fd->access_mode & ADIO_EXCL) {
+ amode = amode | O_EXCL;
+ }
+ if (fd->access_mode & ADIO_APPEND) {
+ amode = amode | O_APPEND;
+ }
+
+ return amode;
+}
+
+// using group_lock in default.
+static int ADIOI_OCEANFS_GroupLockEnable()
+{
+ int group_lock = 1;
+ char *env = NULL;
+ env = getenv("OCEANFS_MPIO_GROUP_LOCK_ENABLE");
+ if (env) {
+ group_lock = atoi(env);
+ }
+
+ return group_lock;
+}
+
+static void ADIOI_OCEANFS_SyncGroupLock(ADIO_File fd, int rank)
+{
+ int ret;
+ u_int64_t lock_id = 0;
+
+ if (!ADIOI_OCEANFS_GroupLockEnable()) {
+ return;
+ }
+ if (rank == 0) {
+ ret = ADIOI_OCEANFS_GetGroupLock(fd->fd_sys, &lock_id);
+ ADIOI_Assert(ret == 0);
+ MPI_Bcast(&lock_id, 1, MPI_UNSIGNED_LONG_LONG, 0, fd->comm);
+ } else {
+ MPI_Bcast(&lock_id, 1, MPI_UNSIGNED_LONG_LONG, 0, fd->comm);
+ ret = ADIOI_OCEANFS_SetGroupLock(fd->fd_sys, lock_id);
+ ADIOI_Assert(ret == 0);
+ }
+}
+
+static int ADIOI_OCEANFS_SetupFilePerm(ADIO_File fd)
+{
+ static const int umask_param = 022;
+ static const int mask_param = 0666;
+ mode_t old_mask;
+ int perm;
+ if (fd->perm == ADIO_PERM_NULL) {
+ old_mask = umask(umask_param);
+ umask(old_mask);
+ perm = old_mask ^ mask_param;
+ } else {
+ perm = fd->perm;
+ }
+ perm &= ~S_IFMT;
+ perm |= S_IFREG;
+
+ return perm;
+}
+
+void ADIOI_OCEANFS_Open(ADIO_File fd, int *error_code)
+{
+ static char myname[] = "ADIOI_OCEANFS_OPEN";
+ int perm, amode, ret, rank;
+ *error_code = MPI_SUCCESS;
+
+ /* setup file permissions */
+ perm = ADIOI_OCEANFS_SetupFilePerm(fd);
+ amode = ADIOI_OCEANFS_GetMode(fd);
+ /* init OCEANFS */
+ fd->fs_ptr = NULL;
+ MPI_Comm_rank(fd->comm, &rank);
+
+ /* all processes open the file */
+ ret = open(fd->filename, amode, perm);
+ if (ret < 0) {
+ *error_code = ADIOI_Err_create_code(myname, fd->filename, errno);
+ return;
+ }
+
+ fd->fd_sys = ret;
+ fd->fd_direct = -1;
+
+ ADIOI_OCEANFS_SyncGroupLock(fd, rank);
+
+ if ((fd->fd_sys != -1) && ((u_int32_t) fd->access_mode & ADIO_APPEND)) {
+ ret = lseek(fd->fd_sys, 0, SEEK_END);
+ if (ret < 0) {
+ *error_code = ADIOI_Err_create_code(myname, fd->filename, errno);
+ return;
+ }
+ fd->fp_ind = ret;
+ fd->fp_sys_posn = ret;
+ }
+}

src/mpi/romio/adio/common/ad_fstype.c

@@ -94,6 +94,10 @@
 #define GPFS_SUPER_MAGIC 0x47504653
 #endif
 
+#if defined(ROMIO_OCEANFS) && !defined(OCEANFS_MAGIC)
+#define OCEANFS_MAGIC 0xFFEA36969
+#endif
+
 #ifndef LL_SUPER_MAGIC
 #define LL_SUPER_MAGIC 0x0BD00BD0
 #endif
@@ -102,6 +106,10 @@
 #define DAOS_SUPER_MAGIC (0xDA05AD10)
 #endif
 
+#if defined(ROMIO_OCEANFS) && !defined(OCEANFS_SUPER_MAGIC)
+#define OCEANFS_SUPER_MAGIC (0x0CEAAEC0)
+#endif
+
 #define UNKNOWN_SUPER_MAGIC (0xDEADBEEF)
 
 #ifdef HAVE_STRUCT_STATVFS_WITH_F_BASETYPE
@@ -197,6 +205,9 @@ static struct ADIO_FSTypes fstypes[] = {
 #ifdef ROMIO_QUOBYTEFS
  /* userspace driver only selected via prefix */
  {&ADIO_QUOBYTEFS_operations, ADIO_QUOBYTEFS, "quobyte:", 0},
+#endif
+#ifdef ROMIO_OCEANFS
+ {&ADIO_OCEANFS_operations, ADIO_OCEANFS, "oceanfs:", OCEANFS_SUPER_MAGIC},
 #endif
  {0, 0, 0, 0} /* guard entry */
 };
@@ -217,6 +228,7 @@ static const char *fstype_prefix[] = {
  "testfs",
  "ime",
  "quobyte",
+ "oceanfs",
  NULL /* guard entry */
 };
 
@@ -475,6 +487,11 @@ static void ADIO_FileSysType_fncall(const char *filename, int *fstype, int *erro
  case PVFS2_SUPER_MAGIC:
  *fstype = ADIO_PVFS2;
  return;
+#endif
+#ifdef ROMIO_OCEANFS
+ case OCEANFS_SUPER_MAGIC:
+ *fstype = ADIO_OCEANFS;
+ return;
 #endif
  default:
  /* UFS support if we don't know what else to use */

src/mpi/romio/adio/include/adio.h

@@ -293,6 +293,7 @@ typedef struct {
 #define ADIO_IME 169 /* IME burst buffer */
 #define ADIO_DAOS 170
 #define ADIO_QUOBYTEFS 171 /* Quobyte FS */
+#define ADIO_OCEANFS 172 /* OceanFS */
 
 #define ADIO_SEEK_SET SEEK_SET
 #define ADIO_SEEK_CUR SEEK_CUR

src/mpi/romio/adio/include/adioi_fs_proto.h

@@ -63,4 +63,9 @@ extern struct ADIOI_Fns_struct ADIO_QUOBYTEFS_operations;
 extern void ADIOI_QUOBYTEFS_CreateAdapter(const char *, int *);
 #endif
 
+#ifdef ROMIO_OCEANFS
+/* prototypes are in adio/ad_oceanfs/ad_oceanfs.h */
+extern struct ADIOI_Fns_struct ADIO_OCEANFS_operations;
+#endif
+
 #endif /* ADIOI_FS_PROTO_H_INCLUDED */

src/mpi/romio/adio/include/mpio_error.h

@@ -63,6 +63,7 @@
 #define MPIR_ERR_NO_LUSTRE 37
 #define MPIR_ERR_NO_GPFS 38
 #define MPIR_ERR_NO_IM 39
+#define MPIR_ERR_NO_OCEANFS 40
 
 /* MPI_ERR_COMM */
 #ifndef MPIR_ERR_COMM_NULL

src/mpi/romio/configure.ac

@@ -196,7 +196,7 @@ dnl An m4 macro for use with m4_foreach_w and friends. You should modify this
 dnl list if you want to add a known file system. The list is just whitespace
 dnl separated, so you can use newlines and tabs as well.
 m4_define([known_filesystems_m4_w],
- [daos nfs ufs pvfs2 testfs xfs panfs lustre gpfs ime quobytefs])dnl
+ [daos nfs ufs pvfs2 testfs xfs panfs lustre gpfs ime quobytefs oceanfs])dnl
 dnl
 dnl An m4 macro for use with m4_foreach and friends. Expands to a quoted list of
 dnl quoted elements. A bit easier to use without unintended expansion than the
@@ -993,6 +993,10 @@ if test -n "$file_system_quobytefs" ; then
  PAC_APPEND_FLAG([-lquobyte],[LIBS])
 fi
 
+if test -n "$file_system_oceanfs" ; then
+ AC_DEFINE(ROMIO_OCEANFS,1,[Define for ROMIO with OCEANFS])
+fi
+
 #
 # Verify presence of pvfs2.h
 #