SHOGUN  6.0.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Modules
IOBuffer.cpp
Go to the documentation of this file.
1 /*
2  Copyright (c) 2009 Yahoo! Inc. All rights reserved. The copyrights
3  embodied in the content of this file are licensed under the BSD
4  (revised) open source license.
5 
6  Copyright (c) 2011 Berlin Institute of Technology and Max-Planck-Society.
7 
8  This program is free software; you can redistribute it and/or modify
9  it under the terms of the GNU General Public License as published by
10  the Free Software Foundation; either version 3 of the License, or
11  (at your option) any later version.
12 
13  Shogun adjustments (w) 2011 Shashwat Lal Das
14 */
15 
16 #include <string.h>
17 #include <fcntl.h>
18 #include <stdio.h>
19 #ifdef _WIN32
20 #include <io.h>
21 #else
22 #include <unistd.h>
23 #endif
24 
25 #include <shogun/io/IOBuffer.h>
26 #include <shogun/io/SGIO.h>
27 #include <shogun/lib/v_array.h>
28 
29 using namespace shogun;
30 
32 {
33  init();
34 }
35 
37 {
38  init();
39  working_file = fd;
40 }
41 
43 {
44 }
45 
47 {
48  size_t s = 1 << 16;
49  space.reserve(s);
51  working_file=-1;
52 }
53 
54 void CIOBuffer::use_file(int fd)
55 {
56  working_file = fd;
57 }
58 
59 int CIOBuffer::open_file(const char* name, char flag)
60 {
61  int ret=1;
62  switch(flag)
63  {
64  case 'r':
65  working_file = open(name, O_RDONLY|O_LARGEFILE);
66  break;
67 
68  case 'w':
69  working_file = open(name, O_CREAT|O_TRUNC|O_WRONLY, 0666);
70  break;
71 
72  default:
73  SG_ERROR("Unknown file operation. Something other than 'r'/'w' specified.\n")
74  ret = 0;
75  }
76  return ret;
77 }
78 
80 {
81  lseek(working_file, 0, SEEK_SET);
83  space.end = space.begin;
84 }
85 
86 void CIOBuffer::set(char *p)
87 {
88  space.end = p;
89 }
90 
91 ssize_t CIOBuffer::read_file(void* buf, size_t nbytes)
92 {
93  return read(working_file, buf, nbytes);
94 }
95 
97 {
98  if (space.end_array - endloaded == 0)
99  {
100  size_t offset = endloaded - space.begin;
102  endloaded = space.begin+offset;
103  }
104  ssize_t num_read = read_file(endloaded, space.end_array - endloaded);
105  if (num_read >= 0)
106  {
107  endloaded = endloaded+num_read;
108  return num_read;
109  }
110  else
111  return 0;
112 }
113 
114 ssize_t CIOBuffer::write_file(const void* buf, size_t nbytes)
115 {
116  return write(working_file, buf, nbytes);
117 }
118 
120 {
121  if (working_file>=0)
122  {
123  if (write_file(space.begin, space.index()) != (int) space.index())
124  SG_ERROR("Error, failed to write example!\n")
125  }
126  space.end = space.begin;
127 #ifdef _WIN32
128  _commit(working_file);
129 #else
130  fsync(working_file);
131 #endif
132 }
133 
135 {
136  if (working_file < 0)
137  return false;
138  else
139  {
140  int r = close(working_file);
141  if (r < 0)
142  SG_ERROR("Error closing the file!\n")
143  return true;
144  }
145 }
146 
147 ssize_t CIOBuffer::readto(char* &pointer, char terminal)
148 {
149 //Return a pointer to the bytes before the terminal. Must be less
150 //than the buffer size.
151  pointer = space.end;
152  while (pointer != endloaded && *pointer != terminal)
153  pointer++;
154  if (pointer != endloaded)
155  {
156  size_t n = pointer - space.end;
157  space.end = pointer+1;
158  pointer -= n;
159  return n;
160  }
161  else
162  {
163  if (endloaded == space.end_array)
164  {
165  size_t left = endloaded - space.end;
166  memmove(space.begin, space.end, left);
167  space.end = space.begin;
168  endloaded = space.begin+left;
169  pointer = endloaded;
170  }
171  if (fill() > 0)// more bytes are read.
172  return readto(pointer,terminal);
173  else //no more bytes to read, return nothing.
174  return 0;
175  }
176 }
177 
178 void CIOBuffer::buf_write(char* &pointer, int n)
179 {
180  if (space.end + n <= space.end_array)
181  {
182  pointer = space.end;
183  space.end += n;
184  }
185  else // Time to dump the file
186  {
187  if (space.end != space.begin)
188  flush();
189  else // Array is short, so increase size.
190  {
193  }
194  buf_write(pointer,n);
195  }
196 }
197 
198 unsigned int CIOBuffer::buf_read(char* &pointer, int n)
199 {
200  // Return a pointer to the next n bytes.
201  // n must be smaller than the maximum size.
202  if (space.end + n <= endloaded)
203  {
204  pointer = space.end;
205  space.end += n;
206  return n;
207  }
208  else // out of bytes, so refill.
209  {
210  if (space.end != space.begin) //There exists room to shift.
211  {
212  // Out of buffer so swap to beginning.
213  int left = endloaded - space.end;
214  memmove(space.begin, space.end, left);
215  space.end = space.begin;
216  endloaded = space.begin+left;
217  }
218  if (fill() > 0)
219  return buf_read(pointer,n);// more bytes are read.
220  else
221  {
222  // No more bytes to read, return all that we have left.
223  pointer = space.end;
224  space.end = endloaded;
225  return endloaded - pointer;
226  }
227  }
228 }
int working_file
file descriptor
Definition: IOBuffer.h:192
void reserve(size_t length)
Definition: v_array.h:199
T * end
Pointer to last set element in the array.
Definition: v_array.h:160
T * begin
Pointer to first element of the array.
Definition: v_array.h:157
virtual void use_file(int fd)
Definition: IOBuffer.cpp:54
#define SG_ERROR(...)
Definition: SGIO.h:128
virtual void reset_file()
Definition: IOBuffer.cpp:79
virtual bool close_file()
Definition: IOBuffer.cpp:134
#define O_LARGEFILE
Definition: IOBuffer.h:26
ssize_t readto(char *&pointer, char terminal)
Definition: IOBuffer.cpp:147
v_array< char > space
buffer space
Definition: IOBuffer.h:184
virtual ssize_t write_file(const void *buf, size_t nbytes)
Definition: IOBuffer.cpp:114
T * end_array
Pointer to end of array, based on memory reserved.
Definition: v_array.h:163
virtual void flush()
Definition: IOBuffer.cpp:119
virtual int open_file(const char *name, char flag='r')
Definition: IOBuffer.cpp:59
char * endloaded
end of loaded values
Definition: IOBuffer.h:189
virtual ssize_t read_file(void *buf, size_t nbytes)
Definition: IOBuffer.cpp:91
unsigned int buf_read(char *&pointer, int n)
Definition: IOBuffer.cpp:198
void buf_write(char *&pointer, int n)
Definition: IOBuffer.cpp:178
all of classes and functions are contained in the shogun namespace
Definition: class_list.h:18
void set(char *p)
Definition: IOBuffer.cpp:86

SHOGUN Machine Learning Toolbox - Documentation