diff --git a/DiskFile.c b/DiskFile.c index 59e12668..8cfaedfd 100644 --- a/DiskFile.c +++ b/DiskFile.c @@ -54,6 +54,14 @@ static int torch_DiskFile_bigEndianEncoding(lua_State *L) return 1; } +static int torch_DiskFile_longSize(lua_State *L) +{ + THFile *self = luaT_checkudata(L, 1, "torch.DiskFile"); + THDiskFile_longSize(self, lua_tointeger(L, 2)); + lua_settop(L, 1); + return 1; +} + static int torch_DiskFile___tostring__(lua_State *L) { THFile *self = luaT_checkudata(L, 1, "torch.DiskFile"); @@ -71,6 +79,7 @@ static const struct luaL_Reg torch_DiskFile__ [] = { {"nativeEndianEncoding", torch_DiskFile_nativeEndianEncoding}, {"littleEndianEncoding", torch_DiskFile_littleEndianEncoding}, {"bigEndianEncoding", torch_DiskFile_bigEndianEncoding}, + {"longSize", torch_DiskFile_longSize}, {"__tostring__", torch_DiskFile___tostring__}, {NULL, NULL} }; diff --git a/MemoryFile.c b/MemoryFile.c index 114dbc49..a22dc17e 100644 --- a/MemoryFile.c +++ b/MemoryFile.c @@ -29,6 +29,14 @@ static int torch_MemoryFile_storage(lua_State *L) return 1; } +static int torch_longSize(lua_State *L) +{ + THFile *self = luaT_checkudata(L, 1, "torch.MemoryFile"); + THMemoryFile_longSize(self, lua_tointeger(L, 2)); + lua_settop(L, 1); + return 1; +} + static int torch_MemoryFile_free(lua_State *L) { THFile *self = luaT_checkudata(L, 1, "torch.MemoryFile"); @@ -48,6 +56,7 @@ static int torch_MemoryFile___tostring__(lua_State *L) static const struct luaL_Reg torch_MemoryFile__ [] = { {"storage", torch_MemoryFile_storage}, + {"longSize", torch_longSize}, {"__tostring__", torch_MemoryFile___tostring__}, {NULL, NULL} }; diff --git a/doc/diskfile.md b/doc/diskfile.md index 22506ef0..96fdde1c 100644 --- a/doc/diskfile.md +++ b/doc/diskfile.md @@ -62,4 +62,8 @@ addresses) In [binary](file.md#torch.File.binary) mode, force encoding in _native endian_. + +### longSize([size]) ### +Longs will be written and read from the file as `size` bytes long, which +can be 0, 4 or 8. 0 means system default. diff --git a/doc/memoryfile.md b/doc/memoryfile.md index c39b151c..acd4426f 100644 --- a/doc/memoryfile.md +++ b/doc/memoryfile.md @@ -35,3 +35,8 @@ Returns the [storage](storage.md) which contains all the data of the size of the storage is the size of the data in the `File`, plus one, the last character being `NULL`. + +### longSize([size]) ### + +Longs will be written and read from the file as `size` bytes long, which +can be 0, 4 or 8. 0 means system default. diff --git a/lib/TH/THDiskFile.c b/lib/TH/THDiskFile.c index 7163e664..b07a8a7f 100644 --- a/lib/TH/THDiskFile.c +++ b/lib/TH/THDiskFile.c @@ -9,6 +9,7 @@ typedef struct THDiskFile__ FILE *handle; char *name; int isNativeEncoding; + int longSize; } THDiskFile; @@ -282,6 +283,14 @@ void THDiskFile_bigEndianEncoding(THFile *self) /* End of Little and Big Endian Stuff */ +void THDiskFile_longSize(THFile *self, int size) +{ + THDiskFile *dfself = (THDiskFile*)(self); + THArgCheck(dfself->handle != NULL, 1, "attempt to use a closed file"); + THArgCheck(size == 0 || size == 4 || size == 8, 1, "Invalid long size specified"); + dfself->longSize = size; +} + static void THDiskFile_free(THFile *self) { THDiskFile *dfself = (THDiskFile*)(self); @@ -313,9 +322,9 @@ READ_WRITE_METHODS(int, Int, int ret = fscanf(dfself->handle, "%d", &data[i]); if(ret <= 0) break; else nread++, int ret = fprintf(dfself->handle, "%d", data[i]); if(ret <= 0) break; else nwrite++) -READ_WRITE_METHODS(long, Long, +/*READ_WRITE_METHODS(long, Long, int ret = fscanf(dfself->handle, "%ld", &data[i]); if(ret <= 0) break; else nread++, - int ret = fprintf(dfself->handle, "%ld", data[i]); if(ret <= 0) break; else nwrite++) + int ret = fprintf(dfself->handle, "%ld", data[i]); if(ret <= 0) break; else nwrite++)*/ READ_WRITE_METHODS(float, Float, int ret = fscanf(dfself->handle, "%g", &data[i]); if(ret <= 0) break; else nread++, @@ -325,7 +334,142 @@ READ_WRITE_METHODS(double, Double, int ret = fscanf(dfself->handle, "%lg", &data[i]); if(ret <= 0) break; else nread++, int ret = fprintf(dfself->handle, "%.17g", data[i]); if(ret <= 0) break; else nwrite++) -static size_t THDiskFile_readString(THFile *self, const char *format, char **str_) + +/* For Long we need to rewrite everything, because of the special management of longSize */ +static size_t THDiskFile_readLong(THFile *self, long *data, size_t n) +{ + THDiskFile *dfself = (THDiskFile*)(self); + size_t nread = 0L; + + THArgCheck(dfself->handle != NULL, 1, "attempt to use a closed file"); + THArgCheck(dfself->file.isReadable, 1, "attempt to read in a write-only file"); + + if(dfself->file.isBinary) + { + if(dfself->longSize == 0 || dfself->longSize == sizeof(long)) + { + nread = fread__(data, sizeof(long), n, dfself->handle); + if(!dfself->isNativeEncoding && (sizeof(long) > 1) && (nread > 0)) + THDiskFile_reverseMemory(data, data, sizeof(long), nread); + } else if(dfself->longSize == 4) + { + int i; + nread = fread__(data, 4, n, dfself->handle); + if(!dfself->isNativeEncoding && (nread > 0)) + THDiskFile_reverseMemory(data, data, 4, nread); + for(i = nread-1; i >= 0; i--) + data[i] = ((int *)data)[i]; + } + else /* if(dfself->longSize == 8) */ + { + int i, big_endian = !THDiskFile_isLittleEndianCPU(); + long *buffer = THAlloc(8*n); + nread = fread__(buffer, 8, n, dfself->handle); + for(i = nread-1; i >= 0; i--) + data[i] = buffer[2*i + big_endian]; + THFree(buffer); + if(!dfself->isNativeEncoding && (nread > 0)) + THDiskFile_reverseMemory(data, data, 4, nread); + } + } + else + { + size_t i; + for(i = 0; i < n; i++) + { + int ret = fscanf(dfself->handle, "%ld", &data[i]); if(ret <= 0) break; else nread++; + } + if(dfself->file.isAutoSpacing && (n > 0)) + { + int c = fgetc(dfself->handle); + if( (c != '\n') && (c != EOF) ) + ungetc(c, dfself->handle); + } + } + + if(nread != n) + { + dfself->file.hasError = 1; /* shouldn't we put hasError to 0 all the time ? */ + if(!dfself->file.isQuiet) + THError("read error: read %d blocks instead of %d", nread, n); + } + + return nread; +} + +static size_t THDiskFile_writeLong(THFile *self, long *data, size_t n) +{ + THDiskFile *dfself = (THDiskFile*)(self); + size_t nwrite = 0L; + + THArgCheck(dfself->handle != NULL, 1, "attempt to use a closed file"); + THArgCheck(dfself->file.isWritable, 1, "attempt to write in a read-only file"); + + if(dfself->file.isBinary) + { + if(dfself->longSize == 0 || dfself->longSize == sizeof(long)) + { + if(dfself->isNativeEncoding) + { + nwrite = fwrite(data, sizeof(long), n, dfself->handle); + } + else + { + char *buffer = THAlloc(sizeof(long)*n); + THDiskFile_reverseMemory(buffer, data, sizeof(long), n); + nwrite = fwrite(buffer, sizeof(long), n, dfself->handle); + THFree(buffer); + } + } else if(dfself->longSize == 4) + { + int i; + int *buffer = THAlloc(4*n); + for(i = 0; i < n; i++) + buffer[i] = data[i]; + if(!dfself->isNativeEncoding) + THDiskFile_reverseMemory(buffer, buffer, 4, n); + nwrite = fwrite(buffer, 4, n, dfself->handle); + THFree(buffer); + } + else /* if(dfself->longSize == 8) */ + { + int i, big_endian = !THDiskFile_isLittleEndianCPU(); + long *buffer = THAlloc(8*n); + for(i = 0; i < n; i++) + { + buffer[2*i + !big_endian] = 0; + buffer[2*i + big_endian] = data[i]; + } + if(!dfself->isNativeEncoding) + THDiskFile_reverseMemory(buffer, buffer, 8, n); + nwrite = fwrite(buffer, 8, n, dfself->handle); + THFree(buffer); + } + } + else + { + size_t i; + for(i = 0; i < n; i++) + { + int ret = fprintf(dfself->handle, "%ld", data[i]); if(ret <= 0) break; else nwrite++; + if( dfself->file.isAutoSpacing && (i < n-1) ) + fprintf(dfself->handle, " "); + } + if(dfself->file.isAutoSpacing && (n > 0)) + fprintf(dfself->handle, "\n"); + } + + if(nwrite != n) + { + dfself->file.hasError = 1; + if(!dfself->file.isQuiet) + THError("write error: wrote %d blocks instead of %d", nwrite, n); + } + + return nwrite; +} + +static long THDiskFile_readString(THFile *self, const char *format, char **str_) { THDiskFile *dfself = (THDiskFile*)(self); THArgCheck(dfself->handle != NULL, 1, "attempt to use a closed file"); @@ -502,6 +646,7 @@ THFile *THDiskFile_new(const char *name, const char *mode, int isQuiet) self->name = THAlloc(strlen(name)+1); strcpy(self->name, name); self->isNativeEncoding = 1; + self->longSize = 0; self->file.vtable = &vtable; self->file.isQuiet = isQuiet; diff --git a/lib/TH/THDiskFile.h b/lib/TH/THDiskFile.h index f7c93c22..f216ec80 100644 --- a/lib/TH/THDiskFile.h +++ b/lib/TH/THDiskFile.h @@ -13,5 +13,6 @@ TH_API int THDiskFile_isBigEndianCPU(void); TH_API void THDiskFile_nativeEndianEncoding(THFile *self); TH_API void THDiskFile_littleEndianEncoding(THFile *self); TH_API void THDiskFile_bigEndianEncoding(THFile *self); +TH_API void THDiskFile_longSize(THFile *self, int size); #endif diff --git a/lib/TH/THMemoryFile.c b/lib/TH/THMemoryFile.c index 8e5cba38..f2b09d1f 100644 --- a/lib/TH/THMemoryFile.c +++ b/lib/TH/THMemoryFile.c @@ -7,6 +7,7 @@ typedef struct THMemoryFile__ THCharStorage *storage; size_t size; size_t position; + int longSize; } THMemoryFile; @@ -214,6 +215,13 @@ static int THMemoryFile_mode(const char *mode, int *isReadable, int *isWritable) } +void THMemoryFile_longSize(THFile *self, int size) +{ + THMemoryFile *dfself = (THMemoryFile*)(self); + THArgCheck(size == 0 || size == 4 || size == 8, 1, "Invalid long size specified"); + dfself->longSize = size; +} + THCharStorage *THMemoryFile_storage(THFile *self) { THMemoryFile *mfself = (THMemoryFile*)self; @@ -323,10 +331,10 @@ READ_WRITE_METHODS(int, Int, nByteWritten = snprintf(mfself->storage->data+mfself->position, mfself->storage->size-mfself->position, "%d", data[i]), 1) -READ_WRITE_METHODS(long, Long, +/*READ_WRITE_METHODS(long, Long, int nByteRead_; int ret = sscanf(mfself->storage->data+mfself->position, "%ld%n", &data[i], &nByteRead_); nByteRead = nByteRead_; if(ret <= 0) break; else nread++, nByteWritten = snprintf(mfself->storage->data+mfself->position, mfself->storage->size-mfself->position, "%ld", data[i]), - 1) + 1)*/ READ_WRITE_METHODS(float, Float, int nByteRead_; int ret = sscanf(mfself->storage->data+mfself->position, "%g%n", &data[i], &nByteRead_); nByteRead = nByteRead_; if(ret <= 0) break; else nread++, @@ -338,7 +346,177 @@ READ_WRITE_METHODS(double, Double, nByteWritten = snprintf(mfself->storage->data+mfself->position, mfself->storage->size-mfself->position, "%.17g", data[i]), 1) -static char* THMemoryFile_cloneString(const char *str, size_t size) +int THDiskFile_isLittleEndianCPU(void); + +static size_t THMemoryFile_readLong(THFile *self, long *data, size_t n) +{ + THMemoryFile *mfself = (THMemoryFile*)self; + size_t nread = 0L; + + THArgCheck(mfself->storage != NULL, 1, "attempt to use a closed file"); + THArgCheck(mfself->file.isReadable, 1, "attempt to read in a write-only file"); + + if (n == 0) + return 0; + + if(mfself->file.isBinary) + { + if(mfself->longSize == 0 || mfself->longSize == sizeof(long)) + { + size_t nByte = sizeof(long)*n; + size_t nByteRemaining = (mfself->position + nByte <= mfself->size ? nByte : mfself->size-mfself->position); + nread = nByteRemaining/sizeof(long); + memmove(data, mfself->storage->data+mfself->position, nread*sizeof(long)); + mfself->position += nread*sizeof(long); + } else if(mfself->longSize == 4) + { + size_t i; + size_t nByte = 4*n; + size_t nByteRemaining = (mfself->position + nByte <= mfself->size ? nByte : mfself->size-mfself->position); + int *storage = (int *)(mfself->storage->data + mfself->position); + nread = nByteRemaining/4; + for(i = 0; i < nread; i++) + data[i] = storage[i]; + mfself->position += nread*4; + } + else /* if(mfself->longSize == 8) */ + { + int i, big_endian = !THDiskFile_isLittleEndianCPU(); + size_t nByte = 8*n; + long *storage = (long *)(mfself->storage->data + mfself->position); + size_t nByteRemaining = (mfself->position + nByte <= mfself->size ? nByte : mfself->size-mfself->position); + nread = nByteRemaining/8; + for(i = 0; i < nread; i++) + data[i] = storage[2*i + big_endian]; + mfself->position += nread*4; + } + } + else + { + size_t i; + for(i = 0; i < n; i++) + { + size_t nByteRead = 0; + char spaceChar = 0; + char *spacePtr = THMemoryFile_strnextspace(mfself->storage->data+mfself->position, &spaceChar); + int nByteRead_; int ret = sscanf(mfself->storage->data+mfself->position, "%ld%n", &data[i], &nByteRead_); nByteRead = nByteRead_; if(ret <= 0) break; else nread++; + if(ret == EOF) + { + while(mfself->storage->data[mfself->position]) + mfself->position++; + } + else + mfself->position += nByteRead; + if(spacePtr) + *spacePtr = spaceChar; + } + if(mfself->file.isAutoSpacing && (n > 0)) + { + if( (mfself->position < mfself->size) && (mfself->storage->data[mfself->position] == '\n') ) + mfself->position++; + } + } + + if(nread != n) + { + mfself->file.hasError = 1; /* shouldn't we put hasError to 0 all the time ? */ + if(!mfself->file.isQuiet) + THError("read error: read %d blocks instead of %d", nread, n); + } + + return nread; +} + +static size_t THMemoryFile_writeLong(THFile *self, long *data, size_t n) +{ + THMemoryFile *mfself = (THMemoryFile*)self; + + THArgCheck(mfself->storage != NULL, 1, "attempt to use a closed file"); + THArgCheck(mfself->file.isWritable, 1, "attempt to write in a read-only file"); + + if (n == 0) + return 0; + + if(mfself->file.isBinary) + { + if(mfself->longSize == 0 || mfself->longSize == sizeof(long)) + { + size_t nByte = sizeof(long)*n; + THMemoryFile_grow(mfself, mfself->position+nByte); + memmove(mfself->storage->data+mfself->position, data, nByte); + mfself->position += nByte; + } else if(mfself->longSize == 4) + { + int i; + size_t nByte = 4*n; + int *storage = (int *)(mfself->storage->data + mfself->position); + THMemoryFile_grow(mfself, mfself->position+nByte); + for(i = 0; i < n; i++) + storage[i] = data[i]; + mfself->position += nByte; + } + else /* if(mfself->longSize == 8) */ + { + int i, big_endian = !THDiskFile_isLittleEndianCPU(); + size_t nByte = 8*n; + long *storage = (long *)(mfself->storage->data + mfself->position); + THMemoryFile_grow(mfself, mfself->position+nByte); + for(i = 0; i < n; i++) + { + storage[2*i + !big_endian] = 0; + storage[2*i + big_endian] = data[i]; + } + mfself->position += nByte; + } + if(mfself->position > mfself->size) + { + mfself->size = mfself->position; + mfself->storage->data[mfself->size] = '\0'; + } + } + else + { + size_t i; + for(i = 0; i < n; i++) + { + size_t nByteWritten; + while (1) + { + nByteWritten = snprintf(mfself->storage->data+mfself->position, mfself->storage->size-mfself->position, "%ld", data[i]); + if( (nByteWritten > -1) && (nByteWritten < mfself->storage->size-mfself->position) ) + { + mfself->position += nByteWritten; + break; + } + THMemoryFile_grow(mfself, mfself->storage->size + (mfself->storage->size/2) + 2); + } + if(mfself->file.isAutoSpacing) + { + if(i < n-1) + { + THMemoryFile_grow(mfself, mfself->position+1); + sprintf(mfself->storage->data+mfself->position, " "); + mfself->position++; + } + if(i == n-1) + { + THMemoryFile_grow(mfself, mfself->position+1); + sprintf(mfself->storage->data+mfself->position, "\n"); + mfself->position++; + } + } + } + if(mfself->position > mfself->size) + { + mfself->size = mfself->position; + mfself->storage->data[mfself->size] = '\0'; + } + } + + return n; +} + +static char* THMemoryFile_cloneString(const char *str, long size) { char *cstr = THAlloc(size); memcpy(cstr, str, size); @@ -481,6 +659,7 @@ THFile *THMemoryFile_newWithStorage(THCharStorage *storage, const char *mode) mfself->storage = storage; mfself->size = (storage ? storage->size-1 : 0); mfself->position = 0; + mfself->longSize = 0; mfself->file.vtable = &vtable; mfself->file.isQuiet = 0; diff --git a/lib/TH/THMemoryFile.h b/lib/TH/THMemoryFile.h index a6c19a57..b54cdcc2 100644 --- a/lib/TH/THMemoryFile.h +++ b/lib/TH/THMemoryFile.h @@ -8,5 +8,6 @@ TH_API THFile *THMemoryFile_newWithStorage(THCharStorage *storage, const char *m TH_API THFile *THMemoryFile_new(const char *mode); TH_API THCharStorage *THMemoryFile_storage(THFile *self); +TH_API void THMemoryFile_longSize(THFile *self, int size); #endif diff --git a/test/longSize.lua b/test/longSize.lua new file mode 100644 index 00000000..82eef043 --- /dev/null +++ b/test/longSize.lua @@ -0,0 +1,42 @@ +tensor = torch.rand(2,3) +f = torch.DiskFile('tensor8.bin','w') +f:binary() +f:longSize(8) +f:writeObject(tensor) +f:close() +f = torch.DiskFile('tensor8.bin','r') +f:binary() +f:longSize(8) +tensor2 = f:readObject() +f:close() +print('Tensors are same: ',tensor:norm()==tensor2:norm()) + +f = torch.DiskFile('tensor4.bin','w') +f:binary() +f:longSize(4) +f:writeObject(tensor) +f:close() +f = torch.DiskFile('tensor4.bin','r') +f:binary() +f:longSize(4) +tensor2 = f:readObject() +f:close() +print('Tensors are same: ',tensor:norm()==tensor2:norm()) + +f = torch.MemoryFile() +f:binary() +f:longSize(8) +f:writeObject(tensor) +f:seek(1) +tensor2 = f:readObject() +f:close() +print('Tensors are same: ',tensor:norm()==tensor2:norm()) + +f = torch.MemoryFile() +f:binary() +f:longSize(4) +f:writeObject(tensor) +f:seek(1) +tensor2 = f:readObject() +f:close() +print('Tensors are same: ',tensor:norm()==tensor2:norm())