SHOGUN  6.0.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Modules
MemoryMappedFile.h
Go to the documentation of this file.
1 /*
2  * This program is free software; you can redistribute it and/or modify
3  * it under the terms of the GNU General Public License as published by
4  * the Free Software Foundation; either version 3 of the License, or
5  * (at your option) any later version.
6  *
7  * Written (W) 2009 Soeren Sonnenburg
8  * Copyright (C) 2009 Fraunhofer Institute FIRST and Max-Planck-Society
9  */
10 
11 #ifndef __MEMORYMAPPEDFILE_H__
12 #define __MEMORYMAPPEDFILE_H__
13 
14 #include <shogun/lib/config.h>
15 
16 #include <shogun/io/SGIO.h>
17 #include <shogun/base/SGObject.h>
18 
19 #include <stdio.h>
20 #include <string.h>
21 #ifndef _MSC_VER
22 #include <sys/mman.h>
23 #include <unistd.h>
24 #endif
25 #include <sys/stat.h>
26 #include <sys/types.h>
27 #include <fcntl.h>
28 
29 namespace shogun
30 {
35 template <class T> class CMemoryMappedFile : public CSGObject
36 {
37  public:
40  {
41  SG_UNSTABLE("CMemoryMappedFile::CMemoryMappedFile()",
42  "\n");
43 
44  fd = 0;
45  length = 0;
46  address = NULL;
47  rw = 'r';
49 
50  set_generic<T>();
51  }
52 
66  CMemoryMappedFile(const char* fname, char flag='r', int64_t fsize=0)
67  : CSGObject()
68  {
69  REQUIRE(flag=='w' || flag=='r', "Only 'r' and 'w' flags are allowed")
70 
72  rw=flag;
73 
74 #ifdef _MSC_VER
75  DWORD open_flags = GENERIC_READ;
76  DWORD share_mode = FILE_SHARE_READ;
77  DWORD create_disp = OPEN_EXISTING;
78  DWORD mmap_prot = PAGE_READONLY;
79  DWORD mmap_flags = FILE_MAP_READ;
80  if (rw=='w')
81  {
82  open_flags |= GENERIC_WRITE;
83  share_mode |= FILE_SHARE_WRITE;
84  create_disp = OPEN_ALWAYS;
85  mmap_prot = PAGE_READWRITE;
86  mmap_flags = FILE_MAP_ALL_ACCESS;
87  }
88 
89  fd = CreateFile(fname, open_flags, share_mode, 0, create_disp, FILE_ATTRIBUTE_NORMAL, NULL);
90  if (rw=='w' && fsize)
91  {
92  LARGE_INTEGER desired_len;
93  desired_len.QuadPart = fsize;
94  uint8_t byte=0;
95  DWORD bytes_written;
96  if ((SetFilePointerEx(fd, desired_len, NULL, FILE_BEGIN) == 0) || (WriteFile(fd, &byte, 1, &bytes_written, NULL) == 0))
97  SG_ERROR("Error creating file of size %ld bytes\n", fsize)
98  }
99 
100  DWORD length = GetFileSize(fd, NULL);
101  if (length == INVALID_FILE_SIZE)
102  SG_ERROR("Error determining file size\n")
103 
104  mapping = CreateFileMapping(fd, 0, mmap_prot, 0, 0, 0);
105 
106  address = MapViewOfFile(mapping, mmap_flags, 0, 0, length);
107  if (address == NULL)
108  SG_ERROR("Error mapping file")
109 #else
110  int open_flags=O_RDONLY;
111  int mmap_prot=PROT_READ;
112  int mmap_flags=MAP_PRIVATE;
113 
114  if (rw=='w')
115  {
116  open_flags=O_RDWR | O_CREAT;
117  mmap_prot=PROT_READ|PROT_WRITE;
118  mmap_flags=MAP_SHARED;
119  }
120 
121  fd = open(fname, open_flags, S_IRWXU | S_IRWXG | S_IRWXO);
122  if (fd == -1)
123  SG_ERROR("Error opening file\n")
124 
125  if (rw=='w' && fsize)
126  {
127  uint8_t byte=0;
128  if (lseek(fd, fsize, SEEK_SET) != fsize || write(fd, &byte, 1) != 1)
129  SG_ERROR("Error creating file of size %ld bytes\n", fsize)
130  }
131 
132  struct stat sb;
133  if (fstat(fd, &sb) == -1)
134  SG_ERROR("Error determining file size\n")
135 
136  length = sb.st_size;
137  address = mmap(NULL, length, mmap_prot, mmap_flags, fd, 0);
138  if (address == MAP_FAILED)
139  SG_ERROR("Error mapping file")
140 #endif
141  set_generic<T>();
142  }
143 
146  {
147 #ifdef _MSC_VER
148  UnmapViewOfFile(address);
149  CloseHandle(mapping);
150  if (rw=='w' && last_written_byte)
151  {
152  LARGE_INTEGER desired_len;
153  desired_len.QuadPart = last_written_byte;
154  if ((SetFilePointerEx(fd, desired_len, NULL, FILE_BEGIN) == 0) || (SetEndOfFile(fd) == 0)) {
155  CloseHandle(fd);
156  SG_ERROR("Error Truncating file to %ld bytes\n", last_written_byte)
157  }
158  }
159  CloseHandle(fd);
160 #else
161  munmap(address, length);
162  if (rw=='w' && last_written_byte && ftruncate(fd, last_written_byte) == -1)
163 
164  {
165  close(fd);
166  SG_ERROR("Error Truncating file to %ld bytes\n", last_written_byte)
167  }
168  close(fd);
169 #endif
170  }
171 
181  inline T* get_map()
182  {
183  return (T*) address;
184  }
185 
190  uint64_t get_length()
191  {
192  return length/sizeof(T);
193  }
194 
199  uint64_t get_size()
200  {
201  return length;
202  }
203 
215  char* get_line(uint64_t& len, uint64_t& offs)
216  {
217  char* s = (char*) address;
218  for (uint64_t i=offs; i<length; i++)
219  {
220  if (s[i] == '\n')
221  {
222  char* line=&s[offs];
223  len=i-offs;
224  offs=i+1;
225  return line;
226  }
227  }
228 
229  len=0;
230  offs=length;
231  return NULL;
232  }
233 
244  void write_line(const char* line, uint64_t len, uint64_t& offs)
245  {
246  char* s = ((char*) address) + offs;
247  if (len+1+offs > length)
248  SG_ERROR("Writing beyond size of file\n")
249 
250  for (uint64_t i=0; i<len; i++)
251  s[i] = line[i];
252 
253  s[len]='\n';
254  offs+=length+1;
255  last_written_byte=offs-1;
256  }
257 
269  inline void set_truncate_size(uint64_t sz=0)
270  {
272  }
273 
278  int32_t get_num_lines()
279  {
280  char* s = (char*) address;
281  int32_t linecount=0;
282  for (uint64_t i=0; i<length; i++)
283  {
284  if (s[i] == '\n')
285  linecount++;
286  }
287 
288  return linecount;
289  }
290 
298  inline T operator[](uint64_t index) const
299  {
300  return ((T*)address)[index];
301  }
302 
310  inline T operator[](int32_t index) const
311  {
312  return ((T*)address)[index];
313  }
314 
316  virtual const char* get_name() const { return "MemoryMappedFile"; }
317 
318  protected:
320 #ifdef _MSC_VER
321  HANDLE fd;
322  HANDLE mapping;
323 #else
324  int fd;
325 #endif
326 
327  uint64_t length;
329  void* address;
331  char rw;
332 
335 };
336 }
337 #endif
T operator[](uint64_t index) const
#define SG_ERROR(...)
Definition: SGIO.h:128
#define REQUIRE(x,...)
Definition: SGIO.h:205
virtual const char * get_name() const
char * get_line(uint64_t &len, uint64_t &offs)
Class SGObject is the base class of all shogun objects.
Definition: SGObject.h:125
void write_line(const char *line, uint64_t len, uint64_t &offs)
T operator[](int32_t index) const
all of classes and functions are contained in the shogun namespace
Definition: class_list.h:18
#define SG_UNSTABLE(func,...)
Definition: SGIO.h:131
CMemoryMappedFile(const char *fname, char flag='r', int64_t fsize=0)
void set_truncate_size(uint64_t sz=0)

SHOGUN Machine Learning Toolbox - Documentation