SHOGUN  3.2.1
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
IOBuffer.cpp
Go to the documentation of this file.
1 /*
2  Copyright (c) 2009 Yahoo! Inc. All rights reserved. The copyrights
3  embodied in the content of this file are licensed under the BSD
4  (revised) open source license.
5 
6  Copyright (c) 2011 Berlin Institute of Technology and Max-Planck-Society.
7 
8  This program is free software; you can redistribute it and/or modify
9  it under the terms of the GNU General Public License as published by
10  the Free Software Foundation; either version 3 of the License, or
11  (at your option) any later version.
12 
13  Shogun adjustments (w) 2011 Shashwat Lal Das
14 */
15 
16 #include <string.h>
17 #include <fcntl.h>
18 #include <stdio.h>
19 #include <unistd.h>
20 
21 #include <shogun/io/IOBuffer.h>
22 #include <shogun/io/SGIO.h>
23 #include <shogun/lib/v_array.h>
24 
25 using namespace shogun;
26 
28 {
29  init();
30 }
31 
33 {
34  init();
35  working_file = fd;
36 }
37 
39 {
40 }
41 
43 {
44  size_t s = 1 << 16;
45  space.reserve(s);
47  working_file=-1;
48 }
49 
50 void CIOBuffer::use_file(int fd)
51 {
52  working_file = fd;
53 }
54 
55 int CIOBuffer::open_file(const char* name, char flag)
56 {
57  int ret=1;
58  switch(flag)
59  {
60  case 'r':
61  working_file = open(name, O_RDONLY|O_LARGEFILE);
62  break;
63 
64  case 'w':
65  working_file = open(name, O_CREAT|O_TRUNC|O_WRONLY, 0666);
66  break;
67 
68  default:
69  SG_ERROR("Unknown file operation. Something other than 'r'/'w' specified.\n")
70  ret = 0;
71  }
72  return ret;
73 }
74 
76 {
77  lseek(working_file, 0, SEEK_SET);
79  space.end = space.begin;
80 }
81 
82 void CIOBuffer::set(char *p)
83 {
84  space.end = p;
85 }
86 
87 ssize_t CIOBuffer::read_file(void* buf, size_t nbytes)
88 {
89  return read(working_file, buf, nbytes);
90 }
91 
93 {
94  if (space.end_array - endloaded == 0)
95  {
96  size_t offset = endloaded - space.begin;
98  endloaded = space.begin+offset;
99  }
100  ssize_t num_read = read_file(endloaded, space.end_array - endloaded);
101  if (num_read >= 0)
102  {
103  endloaded = endloaded+num_read;
104  return num_read;
105  }
106  else
107  return 0;
108 }
109 
110 ssize_t CIOBuffer::write_file(const void* buf, size_t nbytes)
111 {
112  return write(working_file, buf, nbytes);
113 }
114 
116 {
117  if (working_file>=0)
118  {
119  if (write_file(space.begin, space.index()) != (int) space.index())
120  SG_ERROR("Error, failed to write example!\n")
121  }
122  space.end = space.begin;
123  fsync(working_file);
124 }
125 
127 {
128  if (working_file < 0)
129  return false;
130  else
131  {
132  int r = close(working_file);
133  if (r < 0)
134  SG_ERROR("Error closing the file!\n")
135  return true;
136  }
137 }
138 
139 ssize_t CIOBuffer::readto(char* &pointer, char terminal)
140 {
141 //Return a pointer to the bytes before the terminal. Must be less
142 //than the buffer size.
143  pointer = space.end;
144  while (pointer != endloaded && *pointer != terminal)
145  pointer++;
146  if (pointer != endloaded)
147  {
148  size_t n = pointer - space.end;
149  space.end = pointer+1;
150  pointer -= n;
151  return n;
152  }
153  else
154  {
155  if (endloaded == space.end_array)
156  {
157  size_t left = endloaded - space.end;
158  memmove(space.begin, space.end, left);
159  space.end = space.begin;
160  endloaded = space.begin+left;
161  pointer = endloaded;
162  }
163  if (fill() > 0)// more bytes are read.
164  return readto(pointer,terminal);
165  else //no more bytes to read, return nothing.
166  return 0;
167  }
168 }
169 
170 void CIOBuffer::buf_write(char* &pointer, int n)
171 {
172  if (space.end + n <= space.end_array)
173  {
174  pointer = space.end;
175  space.end += n;
176  }
177  else // Time to dump the file
178  {
179  if (space.end != space.begin)
180  flush();
181  else // Array is short, so increase size.
182  {
185  }
186  buf_write(pointer,n);
187  }
188 }
189 
190 unsigned int CIOBuffer::buf_read(char* &pointer, int n)
191 {
192  // Return a pointer to the next n bytes.
193  // n must be smaller than the maximum size.
194  if (space.end + n <= endloaded)
195  {
196  pointer = space.end;
197  space.end += n;
198  return n;
199  }
200  else // out of bytes, so refill.
201  {
202  if (space.end != space.begin) //There exists room to shift.
203  {
204  // Out of buffer so swap to beginning.
205  int left = endloaded - space.end;
206  memmove(space.begin, space.end, left);
207  space.end = space.begin;
208  endloaded = space.begin+left;
209  }
210  if (fill() > 0)
211  return buf_read(pointer,n);// more bytes are read.
212  else
213  {
214  // No more bytes to read, return all that we have left.
215  pointer = space.end;
216  space.end = endloaded;
217  return endloaded - pointer;
218  }
219  }
220 }

SHOGUN Machine Learning Toolbox - Documentation