diff --git a/DiskFile.c b/DiskFile.c
index 59e12668..8cfaedfd 100644
--- a/DiskFile.c
+++ b/DiskFile.c
@@ -54,6 +54,14 @@ static int torch_DiskFile_bigEndianEncoding(lua_State *L)
return 1;
}
+static int torch_DiskFile_longSize(lua_State *L)
+{
+ THFile *self = luaT_checkudata(L, 1, "torch.DiskFile");
+ THDiskFile_longSize(self, lua_tointeger(L, 2));
+ lua_settop(L, 1);
+ return 1;
+}
+
static int torch_DiskFile___tostring__(lua_State *L)
{
THFile *self = luaT_checkudata(L, 1, "torch.DiskFile");
@@ -71,6 +79,7 @@ static const struct luaL_Reg torch_DiskFile__ [] = {
{"nativeEndianEncoding", torch_DiskFile_nativeEndianEncoding},
{"littleEndianEncoding", torch_DiskFile_littleEndianEncoding},
{"bigEndianEncoding", torch_DiskFile_bigEndianEncoding},
+ {"longSize", torch_DiskFile_longSize},
{"__tostring__", torch_DiskFile___tostring__},
{NULL, NULL}
};
diff --git a/MemoryFile.c b/MemoryFile.c
index 114dbc49..a22dc17e 100644
--- a/MemoryFile.c
+++ b/MemoryFile.c
@@ -29,6 +29,14 @@ static int torch_MemoryFile_storage(lua_State *L)
return 1;
}
+static int torch_longSize(lua_State *L)
+{
+ THFile *self = luaT_checkudata(L, 1, "torch.MemoryFile");
+ THMemoryFile_longSize(self, lua_tointeger(L, 2));
+ lua_settop(L, 1);
+ return 1;
+}
+
static int torch_MemoryFile_free(lua_State *L)
{
THFile *self = luaT_checkudata(L, 1, "torch.MemoryFile");
@@ -48,6 +56,7 @@ static int torch_MemoryFile___tostring__(lua_State *L)
static const struct luaL_Reg torch_MemoryFile__ [] = {
{"storage", torch_MemoryFile_storage},
+ {"longSize", torch_longSize},
{"__tostring__", torch_MemoryFile___tostring__},
{NULL, NULL}
};
diff --git a/doc/diskfile.md b/doc/diskfile.md
index 22506ef0..96fdde1c 100644
--- a/doc/diskfile.md
+++ b/doc/diskfile.md
@@ -62,4 +62,8 @@ addresses)
In [binary](file.md#torch.File.binary) mode, force encoding in _native endian_.
+
+### longSize([size]) ###
+Longs will be written and read from the file as `size` bytes long, which
+can be 0, 4 or 8. 0 means system default.
diff --git a/doc/memoryfile.md b/doc/memoryfile.md
index c39b151c..acd4426f 100644
--- a/doc/memoryfile.md
+++ b/doc/memoryfile.md
@@ -35,3 +35,8 @@ Returns the [storage](storage.md) which contains all the data of the
size of the storage is the size of the data in the `File`, plus one, the
last character being `NULL`.
+
+### longSize([size]) ###
+
+Longs will be written and read from the file as `size` bytes long, which
+can be 0, 4 or 8. 0 means system default.
diff --git a/lib/TH/THDiskFile.c b/lib/TH/THDiskFile.c
index 7163e664..b07a8a7f 100644
--- a/lib/TH/THDiskFile.c
+++ b/lib/TH/THDiskFile.c
@@ -9,6 +9,7 @@ typedef struct THDiskFile__
FILE *handle;
char *name;
int isNativeEncoding;
+ int longSize;
} THDiskFile;
@@ -282,6 +283,14 @@ void THDiskFile_bigEndianEncoding(THFile *self)
/* End of Little and Big Endian Stuff */
+void THDiskFile_longSize(THFile *self, int size)
+{
+ THDiskFile *dfself = (THDiskFile*)(self);
+ THArgCheck(dfself->handle != NULL, 1, "attempt to use a closed file");
+ THArgCheck(size == 0 || size == 4 || size == 8, 1, "Invalid long size specified");
+ dfself->longSize = size;
+}
+
static void THDiskFile_free(THFile *self)
{
THDiskFile *dfself = (THDiskFile*)(self);
@@ -313,9 +322,9 @@ READ_WRITE_METHODS(int, Int,
int ret = fscanf(dfself->handle, "%d", &data[i]); if(ret <= 0) break; else nread++,
int ret = fprintf(dfself->handle, "%d", data[i]); if(ret <= 0) break; else nwrite++)
-READ_WRITE_METHODS(long, Long,
+/*READ_WRITE_METHODS(long, Long,
int ret = fscanf(dfself->handle, "%ld", &data[i]); if(ret <= 0) break; else nread++,
- int ret = fprintf(dfself->handle, "%ld", data[i]); if(ret <= 0) break; else nwrite++)
+ int ret = fprintf(dfself->handle, "%ld", data[i]); if(ret <= 0) break; else nwrite++)*/
READ_WRITE_METHODS(float, Float,
int ret = fscanf(dfself->handle, "%g", &data[i]); if(ret <= 0) break; else nread++,
@@ -325,7 +334,142 @@ READ_WRITE_METHODS(double, Double,
int ret = fscanf(dfself->handle, "%lg", &data[i]); if(ret <= 0) break; else nread++,
int ret = fprintf(dfself->handle, "%.17g", data[i]); if(ret <= 0) break; else nwrite++)
-static size_t THDiskFile_readString(THFile *self, const char *format, char **str_)
+
+/* For Long we need to rewrite everything, because of the special management of longSize */
+static size_t THDiskFile_readLong(THFile *self, long *data, size_t n)
+{
+ THDiskFile *dfself = (THDiskFile*)(self);
+ size_t nread = 0L;
+
+ THArgCheck(dfself->handle != NULL, 1, "attempt to use a closed file");
+ THArgCheck(dfself->file.isReadable, 1, "attempt to read in a write-only file");
+
+ if(dfself->file.isBinary)
+ {
+ if(dfself->longSize == 0 || dfself->longSize == sizeof(long))
+ {
+ nread = fread__(data, sizeof(long), n, dfself->handle);
+ if(!dfself->isNativeEncoding && (sizeof(long) > 1) && (nread > 0))
+ THDiskFile_reverseMemory(data, data, sizeof(long), nread);
+ } else if(dfself->longSize == 4)
+ {
+ int i;
+ nread = fread__(data, 4, n, dfself->handle);
+ if(!dfself->isNativeEncoding && (nread > 0))
+ THDiskFile_reverseMemory(data, data, 4, nread);
+ for(i = nread-1; i >= 0; i--)
+ data[i] = ((int *)data)[i];
+ }
+ else /* if(dfself->longSize == 8) */
+ {
+ int i, big_endian = !THDiskFile_isLittleEndianCPU();
+ long *buffer = THAlloc(8*n);
+ nread = fread__(buffer, 8, n, dfself->handle);
+ for(i = nread-1; i >= 0; i--)
+ data[i] = buffer[2*i + big_endian];
+ THFree(buffer);
+ if(!dfself->isNativeEncoding && (nread > 0))
+ THDiskFile_reverseMemory(data, data, 4, nread);
+ }
+ }
+ else
+ {
+ size_t i;
+ for(i = 0; i < n; i++)
+ {
+ int ret = fscanf(dfself->handle, "%ld", &data[i]); if(ret <= 0) break; else nread++;
+ }
+ if(dfself->file.isAutoSpacing && (n > 0))
+ {
+ int c = fgetc(dfself->handle);
+ if( (c != '\n') && (c != EOF) )
+ ungetc(c, dfself->handle);
+ }
+ }
+
+ if(nread != n)
+ {
+ dfself->file.hasError = 1; /* shouldn't we put hasError to 0 all the time ? */
+ if(!dfself->file.isQuiet)
+ THError("read error: read %d blocks instead of %d", nread, n);
+ }
+
+ return nread;
+}
+
+static size_t THDiskFile_writeLong(THFile *self, long *data, size_t n)
+{
+ THDiskFile *dfself = (THDiskFile*)(self);
+ size_t nwrite = 0L;
+
+ THArgCheck(dfself->handle != NULL, 1, "attempt to use a closed file");
+ THArgCheck(dfself->file.isWritable, 1, "attempt to write in a read-only file");
+
+ if(dfself->file.isBinary)
+ {
+ if(dfself->longSize == 0 || dfself->longSize == sizeof(long))
+ {
+ if(dfself->isNativeEncoding)
+ {
+ nwrite = fwrite(data, sizeof(long), n, dfself->handle);
+ }
+ else
+ {
+ char *buffer = THAlloc(sizeof(long)*n);
+ THDiskFile_reverseMemory(buffer, data, sizeof(long), n);
+ nwrite = fwrite(buffer, sizeof(long), n, dfself->handle);
+ THFree(buffer);
+ }
+ } else if(dfself->longSize == 4)
+ {
+ int i;
+ int *buffer = THAlloc(4*n);
+ for(i = 0; i < n; i++)
+ buffer[i] = data[i];
+ if(!dfself->isNativeEncoding)
+ THDiskFile_reverseMemory(buffer, buffer, 4, n);
+ nwrite = fwrite(buffer, 4, n, dfself->handle);
+ THFree(buffer);
+ }
+ else /* if(dfself->longSize == 8) */
+ {
+ int i, big_endian = !THDiskFile_isLittleEndianCPU();
+ long *buffer = THAlloc(8*n);
+ for(i = 0; i < n; i++)
+ {
+ buffer[2*i + !big_endian] = 0;
+ buffer[2*i + big_endian] = data[i];
+ }
+ if(!dfself->isNativeEncoding)
+ THDiskFile_reverseMemory(buffer, buffer, 8, n);
+ nwrite = fwrite(buffer, 8, n, dfself->handle);
+ THFree(buffer);
+ }
+ }
+ else
+ {
+ size_t i;
+ for(i = 0; i < n; i++)
+ {
+ int ret = fprintf(dfself->handle, "%ld", data[i]); if(ret <= 0) break; else nwrite++;
+ if( dfself->file.isAutoSpacing && (i < n-1) )
+ fprintf(dfself->handle, " ");
+ }
+ if(dfself->file.isAutoSpacing && (n > 0))
+ fprintf(dfself->handle, "\n");
+ }
+
+ if(nwrite != n)
+ {
+ dfself->file.hasError = 1;
+ if(!dfself->file.isQuiet)
+ THError("write error: wrote %d blocks instead of %d", nwrite, n);
+ }
+
+ return nwrite;
+}
+
+static long THDiskFile_readString(THFile *self, const char *format, char **str_)
{
THDiskFile *dfself = (THDiskFile*)(self);
THArgCheck(dfself->handle != NULL, 1, "attempt to use a closed file");
@@ -502,6 +646,7 @@ THFile *THDiskFile_new(const char *name, const char *mode, int isQuiet)
self->name = THAlloc(strlen(name)+1);
strcpy(self->name, name);
self->isNativeEncoding = 1;
+ self->longSize = 0;
self->file.vtable = &vtable;
self->file.isQuiet = isQuiet;
diff --git a/lib/TH/THDiskFile.h b/lib/TH/THDiskFile.h
index f7c93c22..f216ec80 100644
--- a/lib/TH/THDiskFile.h
+++ b/lib/TH/THDiskFile.h
@@ -13,5 +13,6 @@ TH_API int THDiskFile_isBigEndianCPU(void);
TH_API void THDiskFile_nativeEndianEncoding(THFile *self);
TH_API void THDiskFile_littleEndianEncoding(THFile *self);
TH_API void THDiskFile_bigEndianEncoding(THFile *self);
+TH_API void THDiskFile_longSize(THFile *self, int size);
#endif
diff --git a/lib/TH/THMemoryFile.c b/lib/TH/THMemoryFile.c
index 8e5cba38..f2b09d1f 100644
--- a/lib/TH/THMemoryFile.c
+++ b/lib/TH/THMemoryFile.c
@@ -7,6 +7,7 @@ typedef struct THMemoryFile__
THCharStorage *storage;
size_t size;
size_t position;
+ int longSize;
} THMemoryFile;
@@ -214,6 +215,13 @@ static int THMemoryFile_mode(const char *mode, int *isReadable, int *isWritable)
}
+void THMemoryFile_longSize(THFile *self, int size)
+{
+ THMemoryFile *dfself = (THMemoryFile*)(self);
+ THArgCheck(size == 0 || size == 4 || size == 8, 1, "Invalid long size specified");
+ dfself->longSize = size;
+}
+
THCharStorage *THMemoryFile_storage(THFile *self)
{
THMemoryFile *mfself = (THMemoryFile*)self;
@@ -323,10 +331,10 @@ READ_WRITE_METHODS(int, Int,
nByteWritten = snprintf(mfself->storage->data+mfself->position, mfself->storage->size-mfself->position, "%d", data[i]),
1)
-READ_WRITE_METHODS(long, Long,
+/*READ_WRITE_METHODS(long, Long,
int nByteRead_; int ret = sscanf(mfself->storage->data+mfself->position, "%ld%n", &data[i], &nByteRead_); nByteRead = nByteRead_; if(ret <= 0) break; else nread++,
nByteWritten = snprintf(mfself->storage->data+mfself->position, mfself->storage->size-mfself->position, "%ld", data[i]),
- 1)
+ 1)*/
READ_WRITE_METHODS(float, Float,
int nByteRead_; int ret = sscanf(mfself->storage->data+mfself->position, "%g%n", &data[i], &nByteRead_); nByteRead = nByteRead_; if(ret <= 0) break; else nread++,
@@ -338,7 +346,177 @@ READ_WRITE_METHODS(double, Double,
nByteWritten = snprintf(mfself->storage->data+mfself->position, mfself->storage->size-mfself->position, "%.17g", data[i]),
1)
-static char* THMemoryFile_cloneString(const char *str, size_t size)
+int THDiskFile_isLittleEndianCPU(void);
+
+static size_t THMemoryFile_readLong(THFile *self, long *data, size_t n)
+{
+ THMemoryFile *mfself = (THMemoryFile*)self;
+ size_t nread = 0L;
+
+ THArgCheck(mfself->storage != NULL, 1, "attempt to use a closed file");
+ THArgCheck(mfself->file.isReadable, 1, "attempt to read in a write-only file");
+
+ if (n == 0)
+ return 0;
+
+ if(mfself->file.isBinary)
+ {
+ if(mfself->longSize == 0 || mfself->longSize == sizeof(long))
+ {
+ size_t nByte = sizeof(long)*n;
+ size_t nByteRemaining = (mfself->position + nByte <= mfself->size ? nByte : mfself->size-mfself->position);
+ nread = nByteRemaining/sizeof(long);
+ memmove(data, mfself->storage->data+mfself->position, nread*sizeof(long));
+ mfself->position += nread*sizeof(long);
+ } else if(mfself->longSize == 4)
+ {
+ size_t i;
+ size_t nByte = 4*n;
+ size_t nByteRemaining = (mfself->position + nByte <= mfself->size ? nByte : mfself->size-mfself->position);
+ int *storage = (int *)(mfself->storage->data + mfself->position);
+ nread = nByteRemaining/4;
+ for(i = 0; i < nread; i++)
+ data[i] = storage[i];
+ mfself->position += nread*4;
+ }
+ else /* if(mfself->longSize == 8) */
+ {
+ int i, big_endian = !THDiskFile_isLittleEndianCPU();
+ size_t nByte = 8*n;
+ long *storage = (long *)(mfself->storage->data + mfself->position);
+ size_t nByteRemaining = (mfself->position + nByte <= mfself->size ? nByte : mfself->size-mfself->position);
+ nread = nByteRemaining/8;
+ for(i = 0; i < nread; i++)
+ data[i] = storage[2*i + big_endian];
+ mfself->position += nread*4;
+ }
+ }
+ else
+ {
+ size_t i;
+ for(i = 0; i < n; i++)
+ {
+ size_t nByteRead = 0;
+ char spaceChar = 0;
+ char *spacePtr = THMemoryFile_strnextspace(mfself->storage->data+mfself->position, &spaceChar);
+ int nByteRead_; int ret = sscanf(mfself->storage->data+mfself->position, "%ld%n", &data[i], &nByteRead_); nByteRead = nByteRead_; if(ret <= 0) break; else nread++;
+ if(ret == EOF)
+ {
+ while(mfself->storage->data[mfself->position])
+ mfself->position++;
+ }
+ else
+ mfself->position += nByteRead;
+ if(spacePtr)
+ *spacePtr = spaceChar;
+ }
+ if(mfself->file.isAutoSpacing && (n > 0))
+ {
+ if( (mfself->position < mfself->size) && (mfself->storage->data[mfself->position] == '\n') )
+ mfself->position++;
+ }
+ }
+
+ if(nread != n)
+ {
+ mfself->file.hasError = 1; /* shouldn't we put hasError to 0 all the time ? */
+ if(!mfself->file.isQuiet)
+ THError("read error: read %d blocks instead of %d", nread, n);
+ }
+
+ return nread;
+}
+
+static size_t THMemoryFile_writeLong(THFile *self, long *data, size_t n)
+{
+ THMemoryFile *mfself = (THMemoryFile*)self;
+
+ THArgCheck(mfself->storage != NULL, 1, "attempt to use a closed file");
+ THArgCheck(mfself->file.isWritable, 1, "attempt to write in a read-only file");
+
+ if (n == 0)
+ return 0;
+
+ if(mfself->file.isBinary)
+ {
+ if(mfself->longSize == 0 || mfself->longSize == sizeof(long))
+ {
+ size_t nByte = sizeof(long)*n;
+ THMemoryFile_grow(mfself, mfself->position+nByte);
+ memmove(mfself->storage->data+mfself->position, data, nByte);
+ mfself->position += nByte;
+ } else if(mfself->longSize == 4)
+ {
+ int i;
+ size_t nByte = 4*n;
+ int *storage = (int *)(mfself->storage->data + mfself->position);
+ THMemoryFile_grow(mfself, mfself->position+nByte);
+ for(i = 0; i < n; i++)
+ storage[i] = data[i];
+ mfself->position += nByte;
+ }
+ else /* if(mfself->longSize == 8) */
+ {
+ int i, big_endian = !THDiskFile_isLittleEndianCPU();
+ size_t nByte = 8*n;
+ long *storage = (long *)(mfself->storage->data + mfself->position);
+ THMemoryFile_grow(mfself, mfself->position+nByte);
+ for(i = 0; i < n; i++)
+ {
+ storage[2*i + !big_endian] = 0;
+ storage[2*i + big_endian] = data[i];
+ }
+ mfself->position += nByte;
+ }
+ if(mfself->position > mfself->size)
+ {
+ mfself->size = mfself->position;
+ mfself->storage->data[mfself->size] = '\0';
+ }
+ }
+ else
+ {
+ size_t i;
+ for(i = 0; i < n; i++)
+ {
+ size_t nByteWritten;
+ while (1)
+ {
+ nByteWritten = snprintf(mfself->storage->data+mfself->position, mfself->storage->size-mfself->position, "%ld", data[i]);
+ if( (nByteWritten > -1) && (nByteWritten < mfself->storage->size-mfself->position) )
+ {
+ mfself->position += nByteWritten;
+ break;
+ }
+ THMemoryFile_grow(mfself, mfself->storage->size + (mfself->storage->size/2) + 2);
+ }
+ if(mfself->file.isAutoSpacing)
+ {
+ if(i < n-1)
+ {
+ THMemoryFile_grow(mfself, mfself->position+1);
+ sprintf(mfself->storage->data+mfself->position, " ");
+ mfself->position++;
+ }
+ if(i == n-1)
+ {
+ THMemoryFile_grow(mfself, mfself->position+1);
+ sprintf(mfself->storage->data+mfself->position, "\n");
+ mfself->position++;
+ }
+ }
+ }
+ if(mfself->position > mfself->size)
+ {
+ mfself->size = mfself->position;
+ mfself->storage->data[mfself->size] = '\0';
+ }
+ }
+
+ return n;
+}
+
+static char* THMemoryFile_cloneString(const char *str, long size)
{
char *cstr = THAlloc(size);
memcpy(cstr, str, size);
@@ -481,6 +659,7 @@ THFile *THMemoryFile_newWithStorage(THCharStorage *storage, const char *mode)
mfself->storage = storage;
mfself->size = (storage ? storage->size-1 : 0);
mfself->position = 0;
+ mfself->longSize = 0;
mfself->file.vtable = &vtable;
mfself->file.isQuiet = 0;
diff --git a/lib/TH/THMemoryFile.h b/lib/TH/THMemoryFile.h
index a6c19a57..b54cdcc2 100644
--- a/lib/TH/THMemoryFile.h
+++ b/lib/TH/THMemoryFile.h
@@ -8,5 +8,6 @@ TH_API THFile *THMemoryFile_newWithStorage(THCharStorage *storage, const char *m
TH_API THFile *THMemoryFile_new(const char *mode);
TH_API THCharStorage *THMemoryFile_storage(THFile *self);
+TH_API void THMemoryFile_longSize(THFile *self, int size);
#endif
diff --git a/test/longSize.lua b/test/longSize.lua
new file mode 100644
index 00000000..82eef043
--- /dev/null
+++ b/test/longSize.lua
@@ -0,0 +1,42 @@
+tensor = torch.rand(2,3)
+f = torch.DiskFile('tensor8.bin','w')
+f:binary()
+f:longSize(8)
+f:writeObject(tensor)
+f:close()
+f = torch.DiskFile('tensor8.bin','r')
+f:binary()
+f:longSize(8)
+tensor2 = f:readObject()
+f:close()
+print('Tensors are same: ',tensor:norm()==tensor2:norm())
+
+f = torch.DiskFile('tensor4.bin','w')
+f:binary()
+f:longSize(4)
+f:writeObject(tensor)
+f:close()
+f = torch.DiskFile('tensor4.bin','r')
+f:binary()
+f:longSize(4)
+tensor2 = f:readObject()
+f:close()
+print('Tensors are same: ',tensor:norm()==tensor2:norm())
+
+f = torch.MemoryFile()
+f:binary()
+f:longSize(8)
+f:writeObject(tensor)
+f:seek(1)
+tensor2 = f:readObject()
+f:close()
+print('Tensors are same: ',tensor:norm()==tensor2:norm())
+
+f = torch.MemoryFile()
+f:binary()
+f:longSize(4)
+f:writeObject(tensor)
+f:seek(1)
+tensor2 = f:readObject()
+f:close()
+print('Tensors are same: ',tensor:norm()==tensor2:norm())