Main Page   Namespace List   Class Hierarchy   Alphabetical List   Compound List   File List   Namespace Members   Compound Members   File Members   Related Pages  

fastio.h

Go to the documentation of this file.
00001 /***************************************************************************
00002  *cr
00003  *cr            (C) Copyright 1995-2016 The Board of Trustees of the
00004  *cr                        University of Illinois
00005  *cr                         All Rights Reserved
00006  *cr
00007  ***************************************************************************/
00008 /***************************************************************************
00009  * RCS INFORMATION:
00010  *
00011  *      $RCSfile: fastio.h,v $
00012  *      $Author: johns $       $Locker:  $             $State: Exp $
00013  *      $Revision: 1.34 $       $Date: 2016/11/28 05:01:53 $
00014  *
00015  ***************************************************************************
00016  * DESCRIPTION:
00017  *   This is a simple abstraction layer for system-dependent I/O calls
00018  * that allow plugins to do binary I/O using the fastest possible method.
00019  *
00020  * This code is intended for use by binary trajectory reader plugins that
00021  * work with multi-gigabyte data sets, reading only binary data.
00022  *
00023  ***************************************************************************/
00024 
00025 #define FIO_READ    0x01
00026 #define FIO_WRITE   0x02
00027 #define FIO_DIRECT  0x04 /* emulate Unix O_DIRECT flag */
00028  
00029 /* Compiling on windows */
00030 #if defined(_MSC_VER) || defined(__MINGW32__)
00031 
00032 #if 1 
00033 /* use native Windows I/O calls */
00034 #define FASTIO_NATIVEWIN32 1
00035 
00036 #include <stdio.h>
00037 #include <string.h>
00038 #include <windows.h>
00039 
00040 typedef HANDLE fio_fd;
00041 typedef LONGLONG fio_size_t;
00042 typedef void * fio_caddr_t;
00043 
00044 typedef struct {
00045   fio_caddr_t iov_base;
00046   int iov_len;
00047 } fio_iovec;
00048 
00049 
00050 #define FIO_SEEK_CUR  FILE_CURRENT
00051 #define FIO_SEEK_SET  FILE_BEGIN
00052 #define FIO_SEEK_END  FILE_END
00053 
00054 static int fio_win32convertfilename(const char *filename, char *newfilename, int maxlen) {
00055   int i;
00056   int len=strlen(filename);
00057  
00058   if ((len + 1) >= maxlen)
00059     return -1;
00060    
00061   for (i=0; i<len; i++) {
00062     if (filename[i] == '/')
00063       newfilename[i] = '\\';
00064     else
00065       newfilename[i] = filename[i];
00066   }
00067   newfilename[len] = '\0'; /* NUL terminate the string */
00068 
00069   return 0;
00070 }
00071 
00072 static int fio_open(const char *filename, int mode, fio_fd *fd) {
00073   HANDLE fp;
00074   char winfilename[8192];
00075   DWORD access;
00076   DWORD sharing;
00077   LPSECURITY_ATTRIBUTES security;
00078   DWORD createmode;
00079   DWORD flags;
00080 
00081   if (fio_win32convertfilename(filename, winfilename, sizeof(winfilename)))
00082     return -1;  
00083 
00084   access = 0;
00085   if (mode & FIO_READ)
00086     access |= GENERIC_READ;
00087   if (mode & FIO_WRITE)
00088     access |= GENERIC_WRITE;
00089 #if 0
00090   access = FILE_ALL_ACCESS; /* XXX hack if above modes fail */
00091 #endif
00092 #if 1
00093   if (mode & FIO_DIRECT)
00094     flags = FILE_FLAG_NO_BUFFERING;
00095   else
00096     flags = FILE_ATTRIBUTE_NORMAL;
00097 #else
00098   if (mode & FIO_DIRECT)
00099     return -1; /* not supported yet */
00100 #endif
00101 
00102   sharing = 0;       /* disallow sharing with other processes  */
00103   security = NULL;   /* child processes don't inherit anything */
00104 
00105   /* since we never append, blow away anything that's already there */
00106   if (mode & FIO_WRITE)
00107     createmode = CREATE_ALWAYS;
00108   else 
00109     createmode = OPEN_EXISTING;
00110 
00111   fp = CreateFile(winfilename, access, sharing, security, 
00112                   createmode, flags, NULL);
00113 
00114   if (fp == NULL) {
00115     return -1;
00116   } else {
00117     *fd = fp;
00118     return 0;
00119   }
00120 }
00121 
00122 
00123 static int fio_fclose(fio_fd fd) {
00124   BOOL rc;
00125   rc = CloseHandle(fd);
00126   if (rc) 
00127     return 0;
00128   else 
00129     return -1;
00130 }
00131 
00132 static fio_size_t fio_fread(void *ptr, fio_size_t size, 
00133                             fio_size_t nitems, fio_fd fd) {
00134   BOOL rc;
00135   DWORD len;
00136   DWORD readlen;
00137 
00138   len = size * nitems;
00139 
00140   rc = ReadFile(fd, ptr, len, &readlen, NULL);
00141   if (rc) {
00142     if (readlen == len)
00143       return nitems;
00144     else 
00145       return 0;
00146   } else {
00147     return 0;
00148   }
00149 }
00150 
00151 static fio_size_t fio_readv(fio_fd fd, const fio_iovec * iov, int iovcnt) {
00152   int i;
00153   fio_size_t len = 0; 
00154 
00155   for (i=0; i<iovcnt; i++) {
00156     fio_size_t rc = fio_fread(iov[i].iov_base, iov[i].iov_len, 1, fd);
00157     if (rc != 1)
00158       break;
00159     len += iov[i].iov_len;
00160   }
00161 
00162   return len;
00163 }
00164 
00165 static fio_size_t fio_fwrite(void *ptr, fio_size_t size, 
00166                              fio_size_t nitems, fio_fd fd) {
00167   BOOL rc;
00168   DWORD len;
00169   DWORD writelen;
00170 
00171   len = size * nitems; 
00172  
00173   rc = WriteFile(fd, ptr, len, &writelen, NULL);
00174   if (rc) {
00175     if (writelen == len)
00176       return nitems;
00177     else
00178       return 0;
00179   } else {
00180     return 0;
00181   }
00182 }
00183 
00184 static fio_size_t fio_fseek(fio_fd fd, fio_size_t offset, int whence) {
00185 #if 1
00186   /* code that works with older MSVC6 compilers */
00187   LONGLONG finaloffset;
00188   LARGE_INTEGER bigint;
00189   LARGE_INTEGER finalint;
00190 
00191   bigint.QuadPart = offset;
00192   finalint = bigint;      /* set the high part, which will be overwritten */
00193   finalint.LowPart = SetFilePointer(fd, bigint.LowPart, &finalint.HighPart, whence);
00194   if (finalint.LowPart == -1) {
00195     /* if (finalint.LowPart == INVALID_SET_FILE_POINTER) { */
00196     /* INVALID_SET_FILE_POINTER is a possible "ok" low order result when */
00197     /* working with 64-bit offsets, so we have to also check the system  */
00198     /* error value for this thread to be sure */
00199     if (GetLastError() != ERROR_SUCCESS) {
00200       return -1;
00201     }
00202   } 
00203 
00204   finaloffset = finalint.QuadPart;
00205   return 0;
00206 #else
00207   BOOL rc;
00208   LONGLONG finaloffset;
00209 
00210   /* SetFilePointerEx() only exists with new .NET compilers */
00211   rc = SetFilePointerEx(fd, offset, &finaloffset, whence);
00212 
00213   if (rc) 
00214     return 0;
00215   else
00216     return -1;
00217 #endif
00218 }
00219 
00220 static fio_size_t fio_ftell(fio_fd fd) {
00221   /* code that works with older MSVC6 compilers */
00222   LONGLONG finaloffset;
00223   LARGE_INTEGER bigint;
00224   LARGE_INTEGER finalint;
00225 
00226   bigint.QuadPart = 0;
00227   finalint = bigint;      /* set the high part, which will be overwritten */
00228 
00229   finalint.LowPart = SetFilePointer(fd, bigint.LowPart, &finalint.HighPart, FILE_CURRENT);
00230   if (finalint.LowPart == -1) {
00231     /* if (finalint.LowPart == INVALID_SET_FILE_POINTER) { */
00232     /* INVALID_SET_FILE_POINTER is a possible "ok" low order result when */
00233     /* working with 64-bit offsets, so we have to also check the system  */
00234     /* error value for this thread to be sure */
00235     if (GetLastError() != ERROR_SUCCESS) {
00236       return -1;
00237     }
00238   }
00239 
00240   finaloffset = finalint.QuadPart;
00241 
00242   return finaloffset;
00243 }
00244 
00245 
00246 #else
00247 
00248 /* Version for machines with plain old ANSI C  */
00249 
00250 #include <stdio.h>
00251 #include <string.h>
00252 
00253 typedef FILE * fio_fd;
00254 typedef size_t fio_size_t;  /* MSVC doesn't uinversally support ssize_t */
00255 typedef void * fio_caddr_t; /* MSVC doesn't universally support caddr_t */
00256 
00257 typedef struct {
00258   fio_caddr_t iov_base;
00259   int iov_len;
00260 } fio_iovec;
00261 
00262 #define FIO_SEEK_CUR SEEK_CUR
00263 #define FIO_SEEK_SET SEEK_SET
00264 #define FIO_SEEK_END SEEK_END
00265 
00266 static int fio_open(const char *filename, int mode, fio_fd *fd) {
00267   char * modestr;
00268   FILE *fp;
00269 
00270   if (mode & FIO_READ) 
00271     modestr = "rb";
00272 
00273   if (mode & FIO_WRITE) 
00274     modestr = "wb";
00275 
00276   if (mode & FIO_DIRECT)
00277     return -1; /* not supported yet */
00278  
00279   fp = fopen(filename, modestr);
00280   if (fp == NULL) {
00281     return -1;
00282   } else {
00283     *fd = fp;
00284     return 0;
00285   }
00286 }
00287 
00288 static int fio_fclose(fio_fd fd) {
00289   return fclose(fd);
00290 }
00291 
00292 static fio_size_t fio_fread(void *ptr, fio_size_t size, 
00293                             fio_size_t nitems, fio_fd fd) {
00294   return fread(ptr, size, nitems, fd);
00295 }
00296 
00297 static fio_size_t fio_readv(fio_fd fd, const fio_iovec * iov, int iovcnt) {
00298   int i;
00299   fio_size_t len = 0; 
00300 
00301   for (i=0; i<iovcnt; i++) {
00302     fio_size_t rc = fread(iov[i].iov_base, iov[i].iov_len, 1, fd);
00303     if (rc != 1)
00304       break;
00305     len += iov[i].iov_len;
00306   }
00307 
00308   return len;
00309 }
00310 
00311 static fio_size_t fio_fwrite(void *ptr, fio_size_t size, 
00312                              fio_size_t nitems, fio_fd fd) {
00313   return fwrite(ptr, size, nitems, fd);
00314 }
00315 
00316 static fio_size_t fio_fseek(fio_fd fd, fio_size_t offset, int whence) {
00317   return fseek(fd, offset, whence);
00318 }
00319 
00320 static fio_size_t fio_ftell(fio_fd fd) {
00321   return ftell(fd);
00322 }
00323 #endif /* plain ANSI C */
00324 
00325 #else 
00326 
00327 /* Version for UNIX machines */
00328 #if defined(__linux)
00329 #ifndef _GNU_SOURCE
00330 #define _GNU_SOURCE            /* required for O_DIRECT */
00331 #endif
00332 #endif
00333 #include <unistd.h>
00334 #include <stdio.h>
00335 #include <sys/types.h>
00336 #include <sys/stat.h>
00337 #include <fcntl.h>
00338 #include <string.h>
00339 
00340 typedef int fio_fd;
00341 typedef off_t fio_size_t;      /* off_t is 64-bits with LFS builds */
00342 
00343 /*
00344  * Enable use of kernel readv() if available and reliable
00345  *
00346  * Note: Some Linux implementations incorporate readv() code in libc 
00347  * that does userspace copying of I/O vectors to internal temporary
00348  * buffers in order to meet the atomicity requirements of the POSIX standard.
00349  * Such copies make the use of vectorized I/O APIs much less useful for
00350  * large trajectory files because the internal buffer allocations can fail
00351  * badly when performing large aggregate I/O operations.  It may be that
00352  * other implementations of vector I/O have similar problems, and in these
00353  * cases it is probably best not to use it at all, and to fall back to 
00354  * non-vectorized I/O APIs to avoid such extra copies.
00355  */
00356 #if defined(__sun) || defined(__APPLE_CC__) || defined(__linux)
00357 #define USE_KERNEL_READV 1
00358 #endif
00359 
00360 typedef void * fio_caddr_t;
00361 
00362 #if defined(USE_KERNEL_READV)
00363 #include <errno.h>
00364 #include <sys/uio.h>
00365 typedef struct iovec fio_iovec;
00366 #else
00367 
00368 typedef struct {
00369   fio_caddr_t iov_base;
00370   int iov_len;
00371 } fio_iovec;
00372 #endif
00373 
00374 #define FIO_SEEK_CUR SEEK_CUR
00375 #define FIO_SEEK_SET SEEK_SET
00376 #define FIO_SEEK_END SEEK_END
00377 
00378 static int fio_open(const char *filename, int mode, fio_fd *fd) {
00379   int nfd;
00380   int oflag = 0;
00381   
00382   if (mode & FIO_READ) 
00383     oflag = O_RDONLY;
00384 
00385   if (mode & FIO_WRITE) 
00386     oflag = O_WRONLY | O_CREAT | O_TRUNC;
00387 
00388 #if defined(__linux)
00389   /* enable direct I/O, requires block-aligned buffers and I/O sizes */
00390   if (mode & FIO_DIRECT)
00391     oflag |= O_DIRECT;
00392 #else
00393   if (mode & FIO_DIRECT)
00394     return -1; /* not supported yet */
00395 #endif
00396 
00397   nfd = open(filename, oflag, 0666);
00398   if (nfd < 0) {
00399     return -1;
00400   } else {
00401     *fd = nfd;
00402     return 0;
00403   }
00404 }
00405 
00406 static int fio_fclose(fio_fd fd) {
00407   return close(fd);
00408 }
00409 
00410 static fio_size_t fio_fread(void *ptr, fio_size_t size, 
00411                             fio_size_t nitems, fio_fd fd) {
00412   fio_size_t i;
00413   fio_size_t len = 0; 
00414   fio_size_t cnt = 0;
00415 
00416 #if 1
00417   /*
00418    * On Linux individual calls to read() can end up doing short reads when
00419    * reading more than 2GB in a single read call, even on 64-bit machines.  
00420    * For large structures, e.g. 240M-atoms or larger, we have to use a loop
00421    * to continue reading into the memory buffer until completion.
00422    */ 
00423   for (i=0; i<nitems; i++) {
00424     fio_size_t szleft = size;
00425     fio_size_t rc = 0;
00426     for (szleft=size; szleft > 0; szleft -= rc) {
00427       rc = read(fd, ((char*) ptr) + (cnt*size) + (size-szleft), szleft);
00428        if (rc == 0) {
00429           return cnt;  /* end of file scenario */
00430        }
00431 //      if (rc != szleft) {
00432 //        printf("fio_fread(): rc %ld  sz: %ld\n", rc, szleft);
00433 //      }
00434       if (rc < 0) {
00435         printf("fio_fread(): rc %ld  sz: %ld\n", rc, size);
00436         perror("  perror fio_fread(): ");
00437         break;
00438       }
00439     }
00440     len += rc;
00441     cnt++;
00442   }
00443 #else
00444   for (i=0; i<nitems; i++) {
00445     fio_size_t rc = read(fd, (void*) (((char *) ptr) + (cnt * size)), size);
00446     if (rc != size) {
00447 //      printf("fio_fread(): rc %ld  sz: %ld\n", rc, size);
00448 //      perror("  perror fio_fread(): ");
00449       break;
00450     }
00451     len += rc;
00452     cnt++;
00453   }
00454 #endif
00455 
00456   return cnt;
00457 }
00458 
00459 static fio_size_t fio_readv(fio_fd fd, const fio_iovec * iov, int iovcnt) {
00460   fio_size_t len;
00461   int i;
00462 
00463 #if 0
00464   fio_size_t tlen;
00465   for (tlen=0,i=0; i<iovcnt; i++) {
00466     tlen += iov[i].iov_len;
00467   }
00468 
00469 #if defined(USE_KERNEL_READV)
00470   len = readv(fd, iov, iovcnt);
00471   if (len != tlen) {
00472     printf("fio_readv(): readv() rc: %ld  sz: %ld\n", len, tlen);
00473     printf("fio_readv(): readv() errno %d\n", errno);
00474   }
00475 
00476   if ((len < 0 && errno == ENOSYS) ||
00477       (len != tlen && errno == EINVAL)) 
00478 #endif
00479   {
00480     /* XXX this loop doesn't meet the atomicity requirements of
00481      *     real POSIX readv(), since we don't need that feature 
00482      */
00483     len = 0; 
00484     for (i=0; i<iovcnt; i++) {
00485       void *ptr = iov[i].iov_base;
00486       fio_size_t sz = iov[i].iov_len;
00487       fio_size_t szleft = sz;
00488       fio_size_t rc=0;
00489 
00490       for (szleft=sz; szleft > 0; szleft -= rc) {
00491         rc = read(fd, ((char*) ptr)+(sz-szleft), szleft);
00492         if (rc == 0) {
00493           return len;  /* end of file scenario */
00494         }
00495         if (rc != szleft) {
00496           printf("fio_readv(): read() rc %ld  sz: %ld\n", rc, szleft);
00497         }
00498         if (rc < 0) {
00499           printf("fio_readv(): read() rc %ld  sz: %ld\n", rc, szleft);
00500           perror("  perror fio_readv(): ");
00501           break;
00502         }
00503       }
00504       len += iov[i].iov_len;
00505     }
00506   }
00507 #else
00508 #if defined(USE_KERNEL_READV)
00509   len = readv(fd, iov, iovcnt);
00510   if (len < 0 && errno == ENOSYS)
00511 #endif
00512   {
00513     /* XXX this loop doesn't meet the atomicity requirements of
00514      *     real POSIX readv(), since we don't need that feature 
00515      */
00516     len = 0; 
00517     for (i=0; i<iovcnt; i++) {
00518       fio_size_t rc = read(fd, iov[i].iov_base, iov[i].iov_len);
00519       if (rc != iov[i].iov_len)
00520         break;
00521       len += iov[i].iov_len;
00522     }
00523   }
00524 #endif
00525 
00526   return len;
00527 }
00528 
00529 static fio_size_t fio_fwrite(void *ptr, fio_size_t size, 
00530                              fio_size_t nitems, fio_fd fd) {
00531   fio_size_t i;
00532   fio_size_t len = 0; 
00533   fio_size_t cnt = 0;
00534 
00535 #if 1
00536   /*
00537    * On Linux individual calls to write() can end up doing short writes when
00538    * writing more than 2GB in a single write call, even on 64-bit machines.  
00539    * For large structures, e.g. 240M-atoms or larger, we have to use a loop
00540    * to continue writing the memory buffer until completion.
00541    */ 
00542   int writecalls=0;
00543   for (i=0; i<nitems; i++) {
00544     fio_size_t szleft = size;
00545     fio_size_t rc = 0;
00546     for (szleft=size; szleft > 0; szleft -= rc) {
00547       fio_size_t writesz = szleft;
00548 
00549 #if 0
00550       /* On some kernel versions write calls beyond 2GB may not do */
00551       /* a partial write and may just return an error immediately. */
00552       /* Clamp maximum write size to 1GB per write call.           */
00553       if (writesz > (1024L * 1024L * 1024L))
00554         writesz = (1024L * 1024L * 1024L);
00555 #endif
00556 
00557       writecalls++;
00558       rc = write(fd, ((char*) ptr)+(size-szleft), writesz);
00559       if (rc < 0) {
00560         printf("fio_fwrite(): rc %ld  sz: %ld  szleft: %ld  calls: %d\n", 
00561                rc, size, szleft, writecalls);
00562         perror("  perror fio_fwrite(): ");
00563         return cnt;
00564       }
00565     }
00566     len += rc;
00567     cnt++;
00568   }
00569 #else
00570   for (i=0; i<nitems; i++) {
00571     fio_size_t rc = write(fd, ptr, size);
00572     if (rc != size) {
00573       printf("fio_fwrite(): rc %ld  sz: %ld\n", rc, size);
00574       perror("  perror fio_fwrite(): ");
00575       break;
00576     }
00577     len += rc;
00578     cnt++;
00579   }
00580 #endif
00581 
00582   return cnt;
00583 }
00584 
00585 static fio_size_t fio_fseek(fio_fd fd, fio_size_t offset, int whence) {
00586  if (lseek(fd, offset, whence) >= 0)
00587    return 0;  /* success (emulate behavior of fseek) */
00588  else 
00589    return -1; /* failure (emulate behavior of fseek) */
00590 }
00591 
00592 static fio_size_t fio_ftell(fio_fd fd) {
00593   return lseek(fd, 0, SEEK_CUR);
00594 }
00595 
00596 #endif
00597 
00598 
00599 /* higher level routines that are OS independent */
00600 
00601 static int fio_write_int32(fio_fd fd, int i) {
00602   return (fio_fwrite(&i, 4, 1, fd) != 1);
00603 }
00604 
00605 static int fio_read_int32(fio_fd fd, int *i) {
00606   return (fio_fread(i, 4, 1, fd) != 1);
00607 }
00608 
00609 static int fio_write_str(fio_fd fd, const char *str) {
00610   int len = strlen(str);
00611   return (fio_fwrite((void *) str, len, 1, fd) != 1);
00612 }
00613 

Generated on Thu Oct 29 03:07:58 2020 for VMD Plugins (current) by doxygen1.2.14 written by Dimitri van Heesch, © 1997-2002