SHOGUN  3.2.1
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
ProtobufFile.h
Go to the documentation of this file.
1 /*
2  * This program is free software; you can redistribute it and/or modify
3  * it under the terms of the GNU General Public License as published by
4  * the Free Software Foundation; either version 3 of the License, or
5  * (at your option) any later version.
6  *
7  * Written (W) 2013 Evgeniy Andreev (gsomix)
8  */
9 
10 #ifdef HAVE_PROTOBUF
11 
12 #ifndef __PROTOBUFFILE_H__
13 #define __PROTOBUFFILE_H__
14 
15 #include <shogun/lib/config.h>
16 
17 #include <shogun/io/File.h>
18 
19 #include <google/protobuf/message.h>
20 
21 #include <shogun/io/protobuf/ShogunVersion.pb.h>
22 #include <shogun/io/protobuf/Headers.pb.h>
23 #include <shogun/io/protobuf/Chunks.pb.h>
24 
25 namespace shogun
26 {
27 
40 class CProtobufFile : public CFile
41 {
42 public:
44  CProtobufFile();
45 
51  CProtobufFile(FILE* f, const char* name=NULL);
52 
59  CProtobufFile(const char* fname, char rw='r', const char* name=NULL);
60 
62  virtual ~CProtobufFile();
63 
71  virtual void get_vector(int8_t*& vector, int32_t& len);
72  virtual void get_vector(uint8_t*& vector, int32_t& len);
73  virtual void get_vector(char*& vector, int32_t& len);
74  virtual void get_vector(int32_t*& vector, int32_t& len);
75  virtual void get_vector(uint32_t*& vector, int32_t& len);
76  virtual void get_vector(float64_t*& vector, int32_t& len);
77  virtual void get_vector(float32_t*& vector, int32_t& len);
78  virtual void get_vector(floatmax_t*& vector, int32_t& len);
79  virtual void get_vector(int16_t*& vector, int32_t& len);
80  virtual void get_vector(uint16_t*& vector, int32_t& len);
81  virtual void get_vector(int64_t*& vector, int32_t& len);
82  virtual void get_vector(uint64_t*& vector, int32_t& len);
84 
93  virtual void get_matrix(
94  uint8_t*& matrix, int32_t& num_feat, int32_t& num_vec);
95  virtual void get_matrix(
96  int8_t*& matrix, int32_t& num_feat, int32_t& num_vec);
97  virtual void get_matrix(
98  char*& matrix, int32_t& num_feat, int32_t& num_vec);
99  virtual void get_matrix(
100  int32_t*& matrix, int32_t& num_feat, int32_t& num_vec);
101  virtual void get_matrix(
102  uint32_t*& matrix, int32_t& num_feat, int32_t& num_vec);
103  virtual void get_matrix(
104  int64_t*& matrix, int32_t& num_feat, int32_t& num_vec);
105  virtual void get_matrix(
106  uint64_t*& matrix, int32_t& num_feat, int32_t& num_vec);
107  virtual void get_matrix(
108  float32_t*& matrix, int32_t& num_feat, int32_t& num_vec);
109  virtual void get_matrix(
110  float64_t*& matrix, int32_t& num_feat, int32_t& num_vec);
111  virtual void get_matrix(
112  floatmax_t*& matrix, int32_t& num_feat, int32_t& num_vec);
113  virtual void get_matrix(
114  int16_t*& matrix, int32_t& num_feat, int32_t& num_vec);
115  virtual void get_matrix(
116  uint16_t*& matrix, int32_t& num_feat, int32_t& num_vec);
118 
127  virtual void get_ndarray(
128  uint8_t*& array, int32_t*& dims, int32_t& num_dims);
129  virtual void get_ndarray(
130  char*& array, int32_t*& dims, int32_t& num_dims);
131  virtual void get_ndarray(
132  int32_t*& array, int32_t*& dims, int32_t& num_dims);
133  virtual void get_ndarray(
134  float32_t*& array, int32_t*& dims, int32_t& num_dims);
135  virtual void get_ndarray(
136  float64_t*& array, int32_t*& dims, int32_t& num_dims);
137  virtual void get_ndarray(
138  int16_t*& array, int32_t*& dims, int32_t& num_dims);
139  virtual void get_ndarray(
140  uint16_t*& array, int32_t*& dims, int32_t& num_dims);
142 
151  virtual void get_sparse_matrix(
152  SGSparseVector<bool>*& matrix, int32_t& num_feat, int32_t& num_vec);
153  virtual void get_sparse_matrix(
154  SGSparseVector<uint8_t>*& matrix, int32_t& num_feat, int32_t& num_vec);
155  virtual void get_sparse_matrix(
156  SGSparseVector<int8_t>*& matrix, int32_t& num_feat, int32_t& num_vec);
157  virtual void get_sparse_matrix(
158  SGSparseVector<char>*& matrix, int32_t& num_feat, int32_t& num_vec);
159  virtual void get_sparse_matrix(
160  SGSparseVector<int32_t>*& matrix, int32_t& num_feat, int32_t& num_vec);
161  virtual void get_sparse_matrix(
162  SGSparseVector<uint32_t>*& matrix, int32_t& num_feat, int32_t& num_vec);
163  virtual void get_sparse_matrix(
164  SGSparseVector<int64_t>*& matrix, int32_t& num_feat, int32_t& num_vec);
165  virtual void get_sparse_matrix(
166  SGSparseVector<uint64_t>*& matrix, int32_t& num_feat, int32_t& num_vec);
167  virtual void get_sparse_matrix(
168  SGSparseVector<int16_t>*& matrix, int32_t& num_feat, int32_t& num_vec);
169  virtual void get_sparse_matrix(
170  SGSparseVector<uint16_t>*& matrix, int32_t& num_feat, int32_t& num_vec);
171  virtual void get_sparse_matrix(
172  SGSparseVector<float32_t>*& matrix, int32_t& num_feat, int32_t& num_vec);
173  virtual void get_sparse_matrix(
174  SGSparseVector<float64_t>*& matrix, int32_t& num_feat, int32_t& num_vec);
175  virtual void get_sparse_matrix(
176  SGSparseVector<floatmax_t>*& matrix, int32_t& num_feat, int32_t& num_vec);
178 
187  virtual void get_string_list(
188  SGString<uint8_t>*& strings, int32_t& num_str,
189  int32_t& max_string_len);
190  virtual void get_string_list(
191  SGString<int8_t>*& strings, int32_t& num_str,
192  int32_t& max_string_len);
193  virtual void get_string_list(
194  SGString<char>*& strings, int32_t& num_str,
195  int32_t& max_string_len);
196  virtual void get_string_list(
197  SGString<int32_t>*& strings, int32_t& num_str,
198  int32_t& max_string_len);
199  virtual void get_string_list(
200  SGString<uint32_t>*& strings, int32_t& num_str,
201  int32_t& max_string_len);
202  virtual void get_string_list(
203  SGString<int16_t>*& strings, int32_t& num_str,
204  int32_t& max_string_len);
205  virtual void get_string_list(
206  SGString<uint16_t>*& strings, int32_t& num_str,
207  int32_t& max_string_len);
208  virtual void get_string_list(
209  SGString<int64_t>*& strings, int32_t& num_str,
210  int32_t& max_string_len);
211  virtual void get_string_list(
212  SGString<uint64_t>*& strings, int32_t& num_str,
213  int32_t& max_string_len);
214  virtual void get_string_list(
215  SGString<float32_t>*& strings, int32_t& num_str,
216  int32_t& max_string_len);
217  virtual void get_string_list(
218  SGString<float64_t>*& strings, int32_t& num_str,
219  int32_t& max_string_len);
220  virtual void get_string_list(
221  SGString<floatmax_t>*& strings, int32_t& num_str,
222  int32_t& max_string_len);
224 
226  /*virtual void get_vector(void*& vector, int32_t& len, DataType& dtype);*/
227 
235  virtual void set_vector(const int8_t* vector, int32_t len);
236  virtual void set_vector(const uint8_t* vector, int32_t len);
237  virtual void set_vector(const char* vector, int32_t len);
238  virtual void set_vector(const int32_t* vector, int32_t len);
239  virtual void set_vector(const uint32_t* vector, int32_t len);
240  virtual void set_vector(const float32_t* vector, int32_t len);
241  virtual void set_vector(const float64_t* vector, int32_t len);
242  virtual void set_vector(const floatmax_t* vector, int32_t len);
243  virtual void set_vector(const int16_t* vector, int32_t len);
244  virtual void set_vector(const uint16_t* vector, int32_t len);
245  virtual void set_vector(const int64_t* vector, int32_t len);
246  virtual void set_vector(const uint64_t* vector, int32_t len);
248 
256  virtual void set_matrix(
257  const uint8_t* matrix, int32_t num_feat, int32_t num_vec);
258  virtual void set_matrix(
259  const int8_t* matrix, int32_t num_feat, int32_t num_vec);
260  virtual void set_matrix(
261  const char* matrix, int32_t num_feat, int32_t num_vec);
262  virtual void set_matrix(
263  const int32_t* matrix, int32_t num_feat, int32_t num_vec);
264  virtual void set_matrix(
265  const uint32_t* matrix, int32_t num_feat, int32_t num_vec);
266  virtual void set_matrix(
267  const int64_t* matrix, int32_t num_feat, int32_t num_vec);
268  virtual void set_matrix(
269  const uint64_t* matrix, int32_t num_feat, int32_t num_vec);
270  virtual void set_matrix(
271  const float32_t* matrix, int32_t num_feat, int32_t num_vec);
272  virtual void set_matrix(
273  const float64_t* matrix, int32_t num_feat, int32_t num_vec);
274  virtual void set_matrix(
275  const floatmax_t* matrix, int32_t num_feat, int32_t num_vec);
276  virtual void set_matrix(
277  const int16_t* matrix, int32_t num_feat, int32_t num_vec);
278  virtual void set_matrix(
279  const uint16_t* matrix, int32_t num_feat, int32_t num_vec);
281 
289  virtual void set_sparse_matrix(
290  const SGSparseVector<bool>* matrix, int32_t num_feat, int32_t num_vec);
291  virtual void set_sparse_matrix(
292  const SGSparseVector<uint8_t>* matrix, int32_t num_feat, int32_t num_vec);
293  virtual void set_sparse_matrix(
294  const SGSparseVector<int8_t>* matrix, int32_t num_feat, int32_t num_vec);
295  virtual void set_sparse_matrix(
296  const SGSparseVector<char>* matrix, int32_t num_feat, int32_t num_vec);
297  virtual void set_sparse_matrix(
298  const SGSparseVector<int32_t>* matrix, int32_t num_feat, int32_t num_vec);
299  virtual void set_sparse_matrix(
300  const SGSparseVector<uint32_t>* matrix, int32_t num_feat, int32_t num_vec);
301  virtual void set_sparse_matrix(
302  const SGSparseVector<int64_t>* matrix, int32_t num_feat, int32_t num_vec);
303  virtual void set_sparse_matrix(
304  const SGSparseVector<uint64_t>* matrix, int32_t num_feat, int32_t num_vec);
305  virtual void set_sparse_matrix(
306  const SGSparseVector<int16_t>* matrix, int32_t num_feat, int32_t num_vec);
307  virtual void set_sparse_matrix(
308  const SGSparseVector<uint16_t>* matrix, int32_t num_feat, int32_t num_vec);
309  virtual void set_sparse_matrix(
310  const SGSparseVector<float32_t>* matrix, int32_t num_feat, int32_t num_vec);
311  virtual void set_sparse_matrix(
312  const SGSparseVector<float64_t>* matrix, int32_t num_feat, int32_t num_vec);
313  virtual void set_sparse_matrix(
314  const SGSparseVector<floatmax_t>* matrix, int32_t num_feat, int32_t num_vec);
316 
325  virtual void set_string_list(
326  const SGString<uint8_t>* strings, int32_t num_str);
327  virtual void set_string_list(
328  const SGString<int8_t>* strings, int32_t num_str);
329  virtual void set_string_list(
330  const SGString<char>* strings, int32_t num_str);
331  virtual void set_string_list(
332  const SGString<int32_t>* strings, int32_t num_str);
333  virtual void set_string_list(
334  const SGString<uint32_t>* strings, int32_t num_str);
335  virtual void set_string_list(
336  const SGString<int16_t>* strings, int32_t num_str);
337  virtual void set_string_list(
338  const SGString<uint16_t>* strings, int32_t num_str);
339  virtual void set_string_list(
340  const SGString<int64_t>* strings, int32_t num_str);
341  virtual void set_string_list(
342  const SGString<uint64_t>* strings, int32_t num_str);
343  virtual void set_string_list(
344  const SGString<float32_t>* strings, int32_t num_str);
345  virtual void set_string_list(
346  const SGString<float64_t>* strings, int32_t num_str);
347  virtual void set_string_list(
348  const SGString<floatmax_t>* strings, int32_t num_str);
350 
351  virtual const char* get_name() const { return "ProtobufFile"; }
352 
353 private:
355  void init();
356 
358  void write_big_endian_uint(uint32_t number, uint8_t* array, uint32_t size);
359 
361  uint32_t read_big_endian_uint(uint8_t* array, uint32_t size);
362 
364  int32_t compute_num_messages(uint64_t len, int32_t sizeof_type) const;
365 
367  void read_and_validate_global_header(ShogunVersion_SGDataType type);
368 
370  void write_global_header(ShogunVersion_SGDataType type);
371 
374  VectorHeader read_vector_header();
375  MatrixHeader read_matrix_header();
376  SparseMatrixHeader read_sparse_matrix_header();
377  StringListHeader read_string_list_header();
379 
382  void write_vector_header(int32_t len, int32_t num_messages);
383  void write_matrix_header(int32_t num_feat, int32_t num_vec, int32_t num_messages);
385 
390  void write_sparse_matrix_header(
391  const SGSparseVector<bool>* matrix, int32_t num_feat, int32_t num_vec);
392  void write_sparse_matrix_header(
393  const SGSparseVector<uint8_t>* matrix, int32_t num_feat, int32_t num_vec);
394  void write_sparse_matrix_header(
395  const SGSparseVector<int8_t>* matrix, int32_t num_feat, int32_t num_vec);
396  void write_sparse_matrix_header(
397  const SGSparseVector<char>* matrix, int32_t num_feat, int32_t num_vec);
398  void write_sparse_matrix_header(
399  const SGSparseVector<int32_t>* matrix, int32_t num_feat, int32_t num_vec);
400  void write_sparse_matrix_header(
401  const SGSparseVector<uint32_t>* matrix, int32_t num_feat, int32_t num_vec);
402  void write_sparse_matrix_header(
403  const SGSparseVector<float64_t>* matrix, int32_t num_feat, int32_t num_vec);
404  void write_sparse_matrix_header(
405  const SGSparseVector<float32_t>* matrix, int32_t num_feat, int32_t num_vec);
406  void write_sparse_matrix_header(
407  const SGSparseVector<floatmax_t>* matrix, int32_t num_feat, int32_t num_vec);
408  void write_sparse_matrix_header(
409  const SGSparseVector<int16_t>* matrix, int32_t num_feat, int32_t num_vec);
410  void write_sparse_matrix_header(
411  const SGSparseVector<uint16_t>* matrix, int32_t num_feat, int32_t num_vec);
412  void write_sparse_matrix_header(
413  const SGSparseVector<int64_t>* matrix, int32_t num_feat, int32_t num_vec);
414  void write_sparse_matrix_header(
415  const SGSparseVector<uint64_t>* matrix, int32_t num_feat, int32_t num_vec);
417 
422  void write_string_list_header(
423  const SGString<uint8_t>* strings, int32_t num_str);
424  void write_string_list_header(
425  const SGString<int8_t>* strings, int32_t num_str);
426  void write_string_list_header(
427  const SGString<char>* strings, int32_t num_str);
428  void write_string_list_header(
429  const SGString<int32_t>* strings, int32_t num_str);
430  void write_string_list_header(
431  const SGString<uint32_t>* strings, int32_t num_str);
432  void write_string_list_header(
433  const SGString<float64_t>* strings, int32_t num_str);
434  void write_string_list_header(
435  const SGString<float32_t>* strings, int32_t num_str);
436  void write_string_list_header(
437  const SGString<floatmax_t>* strings, int32_t num_str);
438  void write_string_list_header(
439  const SGString<int16_t>* strings, int32_t num_str);
440  void write_string_list_header(
441  const SGString<uint16_t>* strings, int32_t num_str);
442  void write_string_list_header(
443  const SGString<int64_t>* strings, int32_t num_str);
444  void write_string_list_header(
445  const SGString<uint64_t>* strings, int32_t num_str);
447 
449  void read_message(google::protobuf::Message& message);
450 
452  void write_message(const google::protobuf::Message& message);
453 
459  void read_memory_block(uint8_t*& vector, uint64_t len, int32_t num_messages);
460  void read_memory_block(int8_t*& vector, uint64_t len, int32_t num_messages);
461  void read_memory_block(char*& vector, uint64_t len, int32_t num_messages);
462  void read_memory_block(int32_t*& vector, uint64_t len, int32_t num_messages);
463  void read_memory_block(uint32_t*& vector, uint64_t len, int32_t num_messages);
464  void read_memory_block(float64_t*& vector, uint64_t len, int32_t num_messages);
465  void read_memory_block(float32_t*& vector, uint64_t len, int32_t num_messages);
466  void read_memory_block(floatmax_t*& vector, uint64_t len, int32_t num_messages);
467  void read_memory_block(int16_t*& vector, uint64_t len, int32_t num_messages);
468  void read_memory_block(uint16_t*& vector, uint64_t len, int32_t num_messages);
469  void read_memory_block(int64_t*& vector, uint64_t len, int32_t num_messages);
470  void read_memory_block(uint64_t*& vector, uint64_t len, int32_t num_messages);
472 
478  void write_memory_block(const int8_t* vector, uint64_t len, int32_t num_messages);
479  void write_memory_block(const uint8_t* vector, uint64_t len, int32_t num_messages);
480  void write_memory_block(const char* vector, uint64_t len, int32_t num_messages);
481  void write_memory_block(const int32_t* vector, uint64_t len, int32_t num_messages);
482  void write_memory_block(const uint32_t* vector, uint64_t len, int32_t num_messages);
483  void write_memory_block(const float32_t* vector, uint64_t len, int32_t num_messages);
484  void write_memory_block(const float64_t* vector, uint64_t len, int32_t num_messages);
485  void write_memory_block(const floatmax_t* vector, uint64_t len, int32_t num_messages);
486  void write_memory_block(const int16_t* vector, uint64_t len, int32_t num_messages);
487  void write_memory_block(const uint16_t* vector, uint64_t len, int32_t num_messages);
488  void write_memory_block(const int64_t* vector, uint64_t len, int32_t num_messages);
489  void write_memory_block(const uint64_t* vector, uint64_t len, int32_t num_messages);
491 
493  void read_sparse_matrix(SGSparseVector<bool>*& matrix,
494  const SparseMatrixHeader& data_header);
495  void read_sparse_matrix(SGSparseVector<uint8_t>*& matrix,
496  const SparseMatrixHeader& data_header);
497  void read_sparse_matrix(SGSparseVector<int8_t>*& matrix,
498  const SparseMatrixHeader& data_header);
499  void read_sparse_matrix(SGSparseVector<char>*& matrix,
500  const SparseMatrixHeader& data_header);
501  void read_sparse_matrix(SGSparseVector<int32_t>*& matrix,
502  const SparseMatrixHeader& data_header);
503  void read_sparse_matrix(SGSparseVector<uint32_t>*& matrix,
504  const SparseMatrixHeader& data_header);
505  void read_sparse_matrix(SGSparseVector<int16_t>*& matrix,
506  const SparseMatrixHeader& data_header);
507  void read_sparse_matrix(SGSparseVector<uint16_t>*& matrix,
508  const SparseMatrixHeader& data_header);
509  void read_sparse_matrix(SGSparseVector<int64_t>*& matrix,
510  const SparseMatrixHeader& data_header);
511  void read_sparse_matrix(SGSparseVector<uint64_t>*& matrix,
512  const SparseMatrixHeader& data_header);
513  void read_sparse_matrix(SGSparseVector<float32_t>*& matrix,
514  const SparseMatrixHeader& data_header);
515  void read_sparse_matrix(SGSparseVector<float64_t>*& matrix,
516  const SparseMatrixHeader& data_header);
517  void read_sparse_matrix(SGSparseVector<floatmax_t>*& matrix,
518  const SparseMatrixHeader& data_header);
520 
524  void write_sparse_matrix(
525  const SGSparseVector<bool>* matrix, int32_t num_vec);
526  void write_sparse_matrix(
527  const SGSparseVector<uint8_t>* matrix, int32_t num_vec);
528  void write_sparse_matrix(
529  const SGSparseVector<int8_t>* matrix, int32_t num_vec);
530  void write_sparse_matrix(
531  const SGSparseVector<char>* matrix, int32_t num_vec);
532  void write_sparse_matrix(
533  const SGSparseVector<int32_t>* matrix, int32_t num_vec);
534  void write_sparse_matrix(
535  const SGSparseVector<uint32_t>* matrix, int32_t num_vec);
536  void write_sparse_matrix(
537  const SGSparseVector<int16_t>* matrix, int32_t num_vec);
538  void write_sparse_matrix(
539  const SGSparseVector<uint16_t>* matrix, int32_t num_vec);
540  void write_sparse_matrix(
541  const SGSparseVector<int64_t>* matrix, int32_t num_vec);
542  void write_sparse_matrix(
543  const SGSparseVector<uint64_t>* matrix, int32_t num_vec);
544  void write_sparse_matrix(
545  const SGSparseVector<float32_t>* matrix, int32_t num_vec);
546  void write_sparse_matrix(
547  const SGSparseVector<float64_t>* matrix, int32_t num_vec);
548  void write_sparse_matrix(
549  const SGSparseVector<floatmax_t>* matrix, int32_t num_vec);
551 
553  void read_string_list(SGString<uint8_t>*& strings,
554  const StringListHeader& data_header);
555  void read_string_list(SGString<int8_t>*& strings,
556  const StringListHeader& data_header);
557  void read_string_list(SGString<char>*& strings,
558  const StringListHeader& data_header);
559  void read_string_list(SGString<int32_t>*& strings,
560  const StringListHeader& data_header);
561  void read_string_list(SGString<uint32_t>*& strings,
562  const StringListHeader& data_header);
563  void read_string_list(SGString<int16_t>*& strings,
564  const StringListHeader& data_header);
565  void read_string_list(SGString<uint16_t>*& strings,
566  const StringListHeader& data_header);
567  void read_string_list(SGString<int64_t>*& strings,
568  const StringListHeader& data_header);
569  void read_string_list(SGString<uint64_t>*& strings,
570  const StringListHeader& data_header);
571  void read_string_list(SGString<float32_t>*& strings,
572  const StringListHeader& data_header);
573  void read_string_list(SGString<float64_t>*& strings,
574  const StringListHeader& data_header);
575  void read_string_list(SGString<floatmax_t>*& strings,
576  const StringListHeader& data_header);
578 
582  void write_string_list(
583  const SGString<uint8_t>* strings, int32_t num_str);
584  void write_string_list(
585  const SGString<int8_t>* strings, int32_t num_str);
586  void write_string_list(
587  const SGString<char>* strings, int32_t num_str);
588  void write_string_list(
589  const SGString<int32_t>* strings, int32_t num_str);
590  void write_string_list(
591  const SGString<uint32_t>* strings, int32_t num_str);
592  void write_string_list(
593  const SGString<int16_t>* strings, int32_t num_str);
594  void write_string_list(
595  const SGString<uint16_t>* strings, int32_t num_str);
596  void write_string_list(
597  const SGString<int64_t>* strings, int32_t num_str);
598  void write_string_list(
599  const SGString<uint64_t>* strings, int32_t num_str);
600  void write_string_list(
601  const SGString<float32_t>* strings, int32_t num_str);
602  void write_string_list(
603  const SGString<float64_t>* strings, int32_t num_str);
604  void write_string_list(
605  const SGString<floatmax_t>* strings, int32_t num_str);
607 
608 private:
610  int32_t version;
611 
613  int32_t message_size;
614 
616  uint8_t* buffer;
617 
619  uint8_t uint_buffer[4];
620 };
621 
622 }
623 
624 #endif
626 #endif

SHOGUN Machine Learning Toolbox - Documentation