SHOGUN  3.2.1
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
ProtobufFile.h
Go to the documentation of this file.
1 /*
2  * This program is free software; you can redistribute it and/or modify
3  * it under the terms of the GNU General Public License as published by
4  * the Free Software Foundation; either version 3 of the License, or
5  * (at your option) any later version.
6  *
7  * Written (W) 2013 Evgeniy Andreev (gsomix)
8  */
9 
10 #ifdef HAVE_PROTOBUF
11 
12 #ifndef __PROTOBUFFILE_H__
13 #define __PROTOBUFFILE_H__
14 
15 #include <shogun/lib/config.h>
16 
17 #include <shogun/io/File.h>
18 
19 #include <google/protobuf/message.h>
20 
21 #include <shogun/io/protobuf/ShogunVersion.pb.h>
22 #include <shogun/io/protobuf/Headers.pb.h>
23 #include <shogun/io/protobuf/Chunks.pb.h>
24 
25 namespace shogun
26 {
27 
40 class CProtobufFile : public CFile
41 {
42 public:
44  CProtobufFile();
45 
51  CProtobufFile(FILE* f, const char* name=NULL);
52 
59  CProtobufFile(const char* fname, char rw='r', const char* name=NULL);
60 
62  virtual ~CProtobufFile();
63 
64 #ifndef SWIG // SWIG should skip this
65 
72  virtual void get_vector(int8_t*& vector, int32_t& len);
73  virtual void get_vector(uint8_t*& vector, int32_t& len);
74  virtual void get_vector(char*& vector, int32_t& len);
75  virtual void get_vector(int32_t*& vector, int32_t& len);
76  virtual void get_vector(uint32_t*& vector, int32_t& len);
77  virtual void get_vector(float64_t*& vector, int32_t& len);
78  virtual void get_vector(float32_t*& vector, int32_t& len);
79  virtual void get_vector(floatmax_t*& vector, int32_t& len);
80  virtual void get_vector(int16_t*& vector, int32_t& len);
81  virtual void get_vector(uint16_t*& vector, int32_t& len);
82  virtual void get_vector(int64_t*& vector, int32_t& len);
83  virtual void get_vector(uint64_t*& vector, int32_t& len);
85 
94  virtual void get_matrix(
95  uint8_t*& matrix, int32_t& num_feat, int32_t& num_vec);
96  virtual void get_matrix(
97  int8_t*& matrix, int32_t& num_feat, int32_t& num_vec);
98  virtual void get_matrix(
99  char*& matrix, int32_t& num_feat, int32_t& num_vec);
100  virtual void get_matrix(
101  int32_t*& matrix, int32_t& num_feat, int32_t& num_vec);
102  virtual void get_matrix(
103  uint32_t*& matrix, int32_t& num_feat, int32_t& num_vec);
104  virtual void get_matrix(
105  int64_t*& matrix, int32_t& num_feat, int32_t& num_vec);
106  virtual void get_matrix(
107  uint64_t*& matrix, int32_t& num_feat, int32_t& num_vec);
108  virtual void get_matrix(
109  float32_t*& matrix, int32_t& num_feat, int32_t& num_vec);
110  virtual void get_matrix(
111  float64_t*& matrix, int32_t& num_feat, int32_t& num_vec);
112  virtual void get_matrix(
113  floatmax_t*& matrix, int32_t& num_feat, int32_t& num_vec);
114  virtual void get_matrix(
115  int16_t*& matrix, int32_t& num_feat, int32_t& num_vec);
116  virtual void get_matrix(
117  uint16_t*& matrix, int32_t& num_feat, int32_t& num_vec);
119 
128  virtual void get_ndarray(
129  uint8_t*& array, int32_t*& dims, int32_t& num_dims);
130  virtual void get_ndarray(
131  char*& array, int32_t*& dims, int32_t& num_dims);
132  virtual void get_ndarray(
133  int32_t*& array, int32_t*& dims, int32_t& num_dims);
134  virtual void get_ndarray(
135  float32_t*& array, int32_t*& dims, int32_t& num_dims);
136  virtual void get_ndarray(
137  float64_t*& array, int32_t*& dims, int32_t& num_dims);
138  virtual void get_ndarray(
139  int16_t*& array, int32_t*& dims, int32_t& num_dims);
140  virtual void get_ndarray(
141  uint16_t*& array, int32_t*& dims, int32_t& num_dims);
143 
152  virtual void get_sparse_matrix(
153  SGSparseVector<bool>*& matrix, int32_t& num_feat, int32_t& num_vec);
154  virtual void get_sparse_matrix(
155  SGSparseVector<uint8_t>*& matrix, int32_t& num_feat, int32_t& num_vec);
156  virtual void get_sparse_matrix(
157  SGSparseVector<int8_t>*& matrix, int32_t& num_feat, int32_t& num_vec);
158  virtual void get_sparse_matrix(
159  SGSparseVector<char>*& matrix, int32_t& num_feat, int32_t& num_vec);
160  virtual void get_sparse_matrix(
161  SGSparseVector<int32_t>*& matrix, int32_t& num_feat, int32_t& num_vec);
162  virtual void get_sparse_matrix(
163  SGSparseVector<uint32_t>*& matrix, int32_t& num_feat, int32_t& num_vec);
164  virtual void get_sparse_matrix(
165  SGSparseVector<int64_t>*& matrix, int32_t& num_feat, int32_t& num_vec);
166  virtual void get_sparse_matrix(
167  SGSparseVector<uint64_t>*& matrix, int32_t& num_feat, int32_t& num_vec);
168  virtual void get_sparse_matrix(
169  SGSparseVector<int16_t>*& matrix, int32_t& num_feat, int32_t& num_vec);
170  virtual void get_sparse_matrix(
171  SGSparseVector<uint16_t>*& matrix, int32_t& num_feat, int32_t& num_vec);
172  virtual void get_sparse_matrix(
173  SGSparseVector<float32_t>*& matrix, int32_t& num_feat, int32_t& num_vec);
174  virtual void get_sparse_matrix(
175  SGSparseVector<float64_t>*& matrix, int32_t& num_feat, int32_t& num_vec);
176  virtual void get_sparse_matrix(
177  SGSparseVector<floatmax_t>*& matrix, int32_t& num_feat, int32_t& num_vec);
179 
188  virtual void get_string_list(
189  SGString<uint8_t>*& strings, int32_t& num_str,
190  int32_t& max_string_len);
191  virtual void get_string_list(
192  SGString<int8_t>*& strings, int32_t& num_str,
193  int32_t& max_string_len);
194  virtual void get_string_list(
195  SGString<char>*& strings, int32_t& num_str,
196  int32_t& max_string_len);
197  virtual void get_string_list(
198  SGString<int32_t>*& strings, int32_t& num_str,
199  int32_t& max_string_len);
200  virtual void get_string_list(
201  SGString<uint32_t>*& strings, int32_t& num_str,
202  int32_t& max_string_len);
203  virtual void get_string_list(
204  SGString<int16_t>*& strings, int32_t& num_str,
205  int32_t& max_string_len);
206  virtual void get_string_list(
207  SGString<uint16_t>*& strings, int32_t& num_str,
208  int32_t& max_string_len);
209  virtual void get_string_list(
210  SGString<int64_t>*& strings, int32_t& num_str,
211  int32_t& max_string_len);
212  virtual void get_string_list(
213  SGString<uint64_t>*& strings, int32_t& num_str,
214  int32_t& max_string_len);
215  virtual void get_string_list(
216  SGString<float32_t>*& strings, int32_t& num_str,
217  int32_t& max_string_len);
218  virtual void get_string_list(
219  SGString<float64_t>*& strings, int32_t& num_str,
220  int32_t& max_string_len);
221  virtual void get_string_list(
222  SGString<floatmax_t>*& strings, int32_t& num_str,
223  int32_t& max_string_len);
225 
227  /*virtual void get_vector(void*& vector, int32_t& len, DataType& dtype);*/
228 
236  virtual void set_vector(const int8_t* vector, int32_t len);
237  virtual void set_vector(const uint8_t* vector, int32_t len);
238  virtual void set_vector(const char* vector, int32_t len);
239  virtual void set_vector(const int32_t* vector, int32_t len);
240  virtual void set_vector(const uint32_t* vector, int32_t len);
241  virtual void set_vector(const float32_t* vector, int32_t len);
242  virtual void set_vector(const float64_t* vector, int32_t len);
243  virtual void set_vector(const floatmax_t* vector, int32_t len);
244  virtual void set_vector(const int16_t* vector, int32_t len);
245  virtual void set_vector(const uint16_t* vector, int32_t len);
246  virtual void set_vector(const int64_t* vector, int32_t len);
247  virtual void set_vector(const uint64_t* vector, int32_t len);
249 
257  virtual void set_matrix(
258  const uint8_t* matrix, int32_t num_feat, int32_t num_vec);
259  virtual void set_matrix(
260  const int8_t* matrix, int32_t num_feat, int32_t num_vec);
261  virtual void set_matrix(
262  const char* matrix, int32_t num_feat, int32_t num_vec);
263  virtual void set_matrix(
264  const int32_t* matrix, int32_t num_feat, int32_t num_vec);
265  virtual void set_matrix(
266  const uint32_t* matrix, int32_t num_feat, int32_t num_vec);
267  virtual void set_matrix(
268  const int64_t* matrix, int32_t num_feat, int32_t num_vec);
269  virtual void set_matrix(
270  const uint64_t* matrix, int32_t num_feat, int32_t num_vec);
271  virtual void set_matrix(
272  const float32_t* matrix, int32_t num_feat, int32_t num_vec);
273  virtual void set_matrix(
274  const float64_t* matrix, int32_t num_feat, int32_t num_vec);
275  virtual void set_matrix(
276  const floatmax_t* matrix, int32_t num_feat, int32_t num_vec);
277  virtual void set_matrix(
278  const int16_t* matrix, int32_t num_feat, int32_t num_vec);
279  virtual void set_matrix(
280  const uint16_t* matrix, int32_t num_feat, int32_t num_vec);
282 
290  virtual void set_sparse_matrix(
291  const SGSparseVector<bool>* matrix, int32_t num_feat, int32_t num_vec);
292  virtual void set_sparse_matrix(
293  const SGSparseVector<uint8_t>* matrix, int32_t num_feat, int32_t num_vec);
294  virtual void set_sparse_matrix(
295  const SGSparseVector<int8_t>* matrix, int32_t num_feat, int32_t num_vec);
296  virtual void set_sparse_matrix(
297  const SGSparseVector<char>* matrix, int32_t num_feat, int32_t num_vec);
298  virtual void set_sparse_matrix(
299  const SGSparseVector<int32_t>* matrix, int32_t num_feat, int32_t num_vec);
300  virtual void set_sparse_matrix(
301  const SGSparseVector<uint32_t>* matrix, int32_t num_feat, int32_t num_vec);
302  virtual void set_sparse_matrix(
303  const SGSparseVector<int64_t>* matrix, int32_t num_feat, int32_t num_vec);
304  virtual void set_sparse_matrix(
305  const SGSparseVector<uint64_t>* matrix, int32_t num_feat, int32_t num_vec);
306  virtual void set_sparse_matrix(
307  const SGSparseVector<int16_t>* matrix, int32_t num_feat, int32_t num_vec);
308  virtual void set_sparse_matrix(
309  const SGSparseVector<uint16_t>* matrix, int32_t num_feat, int32_t num_vec);
310  virtual void set_sparse_matrix(
311  const SGSparseVector<float32_t>* matrix, int32_t num_feat, int32_t num_vec);
312  virtual void set_sparse_matrix(
313  const SGSparseVector<float64_t>* matrix, int32_t num_feat, int32_t num_vec);
314  virtual void set_sparse_matrix(
315  const SGSparseVector<floatmax_t>* matrix, int32_t num_feat, int32_t num_vec);
317 
326  virtual void set_string_list(
327  const SGString<uint8_t>* strings, int32_t num_str);
328  virtual void set_string_list(
329  const SGString<int8_t>* strings, int32_t num_str);
330  virtual void set_string_list(
331  const SGString<char>* strings, int32_t num_str);
332  virtual void set_string_list(
333  const SGString<int32_t>* strings, int32_t num_str);
334  virtual void set_string_list(
335  const SGString<uint32_t>* strings, int32_t num_str);
336  virtual void set_string_list(
337  const SGString<int16_t>* strings, int32_t num_str);
338  virtual void set_string_list(
339  const SGString<uint16_t>* strings, int32_t num_str);
340  virtual void set_string_list(
341  const SGString<int64_t>* strings, int32_t num_str);
342  virtual void set_string_list(
343  const SGString<uint64_t>* strings, int32_t num_str);
344  virtual void set_string_list(
345  const SGString<float32_t>* strings, int32_t num_str);
346  virtual void set_string_list(
347  const SGString<float64_t>* strings, int32_t num_str);
348  virtual void set_string_list(
349  const SGString<floatmax_t>* strings, int32_t num_str);
351 #endif // #ifndef SWIG // SWIG should skip this
352 
353  virtual const char* get_name() const { return "ProtobufFile"; }
354 
355 private:
357  void init();
358 
360  void write_big_endian_uint(uint32_t number, uint8_t* array, uint32_t size);
361 
363  uint32_t read_big_endian_uint(uint8_t* array, uint32_t size);
364 
366  int32_t compute_num_messages(uint64_t len, int32_t sizeof_type) const;
367 
369  void read_and_validate_global_header(ShogunVersion_SGDataType type);
370 
372  void write_global_header(ShogunVersion_SGDataType type);
373 
376  VectorHeader read_vector_header();
377  MatrixHeader read_matrix_header();
378  SparseMatrixHeader read_sparse_matrix_header();
379  StringListHeader read_string_list_header();
381 
384  void write_vector_header(int32_t len, int32_t num_messages);
385  void write_matrix_header(int32_t num_feat, int32_t num_vec, int32_t num_messages);
387 
392  void write_sparse_matrix_header(
393  const SGSparseVector<bool>* matrix, int32_t num_feat, int32_t num_vec);
394  void write_sparse_matrix_header(
395  const SGSparseVector<uint8_t>* matrix, int32_t num_feat, int32_t num_vec);
396  void write_sparse_matrix_header(
397  const SGSparseVector<int8_t>* matrix, int32_t num_feat, int32_t num_vec);
398  void write_sparse_matrix_header(
399  const SGSparseVector<char>* matrix, int32_t num_feat, int32_t num_vec);
400  void write_sparse_matrix_header(
401  const SGSparseVector<int32_t>* matrix, int32_t num_feat, int32_t num_vec);
402  void write_sparse_matrix_header(
403  const SGSparseVector<uint32_t>* matrix, int32_t num_feat, int32_t num_vec);
404  void write_sparse_matrix_header(
405  const SGSparseVector<float64_t>* matrix, int32_t num_feat, int32_t num_vec);
406  void write_sparse_matrix_header(
407  const SGSparseVector<float32_t>* matrix, int32_t num_feat, int32_t num_vec);
408  void write_sparse_matrix_header(
409  const SGSparseVector<floatmax_t>* matrix, int32_t num_feat, int32_t num_vec);
410  void write_sparse_matrix_header(
411  const SGSparseVector<int16_t>* matrix, int32_t num_feat, int32_t num_vec);
412  void write_sparse_matrix_header(
413  const SGSparseVector<uint16_t>* matrix, int32_t num_feat, int32_t num_vec);
414  void write_sparse_matrix_header(
415  const SGSparseVector<int64_t>* matrix, int32_t num_feat, int32_t num_vec);
416  void write_sparse_matrix_header(
417  const SGSparseVector<uint64_t>* matrix, int32_t num_feat, int32_t num_vec);
419 
424  void write_string_list_header(
425  const SGString<uint8_t>* strings, int32_t num_str);
426  void write_string_list_header(
427  const SGString<int8_t>* strings, int32_t num_str);
428  void write_string_list_header(
429  const SGString<char>* strings, int32_t num_str);
430  void write_string_list_header(
431  const SGString<int32_t>* strings, int32_t num_str);
432  void write_string_list_header(
433  const SGString<uint32_t>* strings, int32_t num_str);
434  void write_string_list_header(
435  const SGString<float64_t>* strings, int32_t num_str);
436  void write_string_list_header(
437  const SGString<float32_t>* strings, int32_t num_str);
438  void write_string_list_header(
439  const SGString<floatmax_t>* strings, int32_t num_str);
440  void write_string_list_header(
441  const SGString<int16_t>* strings, int32_t num_str);
442  void write_string_list_header(
443  const SGString<uint16_t>* strings, int32_t num_str);
444  void write_string_list_header(
445  const SGString<int64_t>* strings, int32_t num_str);
446  void write_string_list_header(
447  const SGString<uint64_t>* strings, int32_t num_str);
449 
451  void read_message(google::protobuf::Message& message);
452 
454  void write_message(const google::protobuf::Message& message);
455 
461  void read_memory_block(uint8_t*& vector, uint64_t len, int32_t num_messages);
462  void read_memory_block(int8_t*& vector, uint64_t len, int32_t num_messages);
463  void read_memory_block(char*& vector, uint64_t len, int32_t num_messages);
464  void read_memory_block(int32_t*& vector, uint64_t len, int32_t num_messages);
465  void read_memory_block(uint32_t*& vector, uint64_t len, int32_t num_messages);
466  void read_memory_block(float64_t*& vector, uint64_t len, int32_t num_messages);
467  void read_memory_block(float32_t*& vector, uint64_t len, int32_t num_messages);
468  void read_memory_block(floatmax_t*& vector, uint64_t len, int32_t num_messages);
469  void read_memory_block(int16_t*& vector, uint64_t len, int32_t num_messages);
470  void read_memory_block(uint16_t*& vector, uint64_t len, int32_t num_messages);
471  void read_memory_block(int64_t*& vector, uint64_t len, int32_t num_messages);
472  void read_memory_block(uint64_t*& vector, uint64_t len, int32_t num_messages);
474 
480  void write_memory_block(const int8_t* vector, uint64_t len, int32_t num_messages);
481  void write_memory_block(const uint8_t* vector, uint64_t len, int32_t num_messages);
482  void write_memory_block(const char* vector, uint64_t len, int32_t num_messages);
483  void write_memory_block(const int32_t* vector, uint64_t len, int32_t num_messages);
484  void write_memory_block(const uint32_t* vector, uint64_t len, int32_t num_messages);
485  void write_memory_block(const float32_t* vector, uint64_t len, int32_t num_messages);
486  void write_memory_block(const float64_t* vector, uint64_t len, int32_t num_messages);
487  void write_memory_block(const floatmax_t* vector, uint64_t len, int32_t num_messages);
488  void write_memory_block(const int16_t* vector, uint64_t len, int32_t num_messages);
489  void write_memory_block(const uint16_t* vector, uint64_t len, int32_t num_messages);
490  void write_memory_block(const int64_t* vector, uint64_t len, int32_t num_messages);
491  void write_memory_block(const uint64_t* vector, uint64_t len, int32_t num_messages);
493 
495  void read_sparse_matrix(SGSparseVector<bool>*& matrix,
496  const SparseMatrixHeader& data_header);
497  void read_sparse_matrix(SGSparseVector<uint8_t>*& matrix,
498  const SparseMatrixHeader& data_header);
499  void read_sparse_matrix(SGSparseVector<int8_t>*& matrix,
500  const SparseMatrixHeader& data_header);
501  void read_sparse_matrix(SGSparseVector<char>*& matrix,
502  const SparseMatrixHeader& data_header);
503  void read_sparse_matrix(SGSparseVector<int32_t>*& matrix,
504  const SparseMatrixHeader& data_header);
505  void read_sparse_matrix(SGSparseVector<uint32_t>*& matrix,
506  const SparseMatrixHeader& data_header);
507  void read_sparse_matrix(SGSparseVector<int16_t>*& matrix,
508  const SparseMatrixHeader& data_header);
509  void read_sparse_matrix(SGSparseVector<uint16_t>*& matrix,
510  const SparseMatrixHeader& data_header);
511  void read_sparse_matrix(SGSparseVector<int64_t>*& matrix,
512  const SparseMatrixHeader& data_header);
513  void read_sparse_matrix(SGSparseVector<uint64_t>*& matrix,
514  const SparseMatrixHeader& data_header);
515  void read_sparse_matrix(SGSparseVector<float32_t>*& matrix,
516  const SparseMatrixHeader& data_header);
517  void read_sparse_matrix(SGSparseVector<float64_t>*& matrix,
518  const SparseMatrixHeader& data_header);
519  void read_sparse_matrix(SGSparseVector<floatmax_t>*& matrix,
520  const SparseMatrixHeader& data_header);
522 
526  void write_sparse_matrix(
527  const SGSparseVector<bool>* matrix, int32_t num_vec);
528  void write_sparse_matrix(
529  const SGSparseVector<uint8_t>* matrix, int32_t num_vec);
530  void write_sparse_matrix(
531  const SGSparseVector<int8_t>* matrix, int32_t num_vec);
532  void write_sparse_matrix(
533  const SGSparseVector<char>* matrix, int32_t num_vec);
534  void write_sparse_matrix(
535  const SGSparseVector<int32_t>* matrix, int32_t num_vec);
536  void write_sparse_matrix(
537  const SGSparseVector<uint32_t>* matrix, int32_t num_vec);
538  void write_sparse_matrix(
539  const SGSparseVector<int16_t>* matrix, int32_t num_vec);
540  void write_sparse_matrix(
541  const SGSparseVector<uint16_t>* matrix, int32_t num_vec);
542  void write_sparse_matrix(
543  const SGSparseVector<int64_t>* matrix, int32_t num_vec);
544  void write_sparse_matrix(
545  const SGSparseVector<uint64_t>* matrix, int32_t num_vec);
546  void write_sparse_matrix(
547  const SGSparseVector<float32_t>* matrix, int32_t num_vec);
548  void write_sparse_matrix(
549  const SGSparseVector<float64_t>* matrix, int32_t num_vec);
550  void write_sparse_matrix(
551  const SGSparseVector<floatmax_t>* matrix, int32_t num_vec);
553 
555  void read_string_list(SGString<uint8_t>*& strings,
556  const StringListHeader& data_header);
557  void read_string_list(SGString<int8_t>*& strings,
558  const StringListHeader& data_header);
559  void read_string_list(SGString<char>*& strings,
560  const StringListHeader& data_header);
561  void read_string_list(SGString<int32_t>*& strings,
562  const StringListHeader& data_header);
563  void read_string_list(SGString<uint32_t>*& strings,
564  const StringListHeader& data_header);
565  void read_string_list(SGString<int16_t>*& strings,
566  const StringListHeader& data_header);
567  void read_string_list(SGString<uint16_t>*& strings,
568  const StringListHeader& data_header);
569  void read_string_list(SGString<int64_t>*& strings,
570  const StringListHeader& data_header);
571  void read_string_list(SGString<uint64_t>*& strings,
572  const StringListHeader& data_header);
573  void read_string_list(SGString<float32_t>*& strings,
574  const StringListHeader& data_header);
575  void read_string_list(SGString<float64_t>*& strings,
576  const StringListHeader& data_header);
577  void read_string_list(SGString<floatmax_t>*& strings,
578  const StringListHeader& data_header);
580 
584  void write_string_list(
585  const SGString<uint8_t>* strings, int32_t num_str);
586  void write_string_list(
587  const SGString<int8_t>* strings, int32_t num_str);
588  void write_string_list(
589  const SGString<char>* strings, int32_t num_str);
590  void write_string_list(
591  const SGString<int32_t>* strings, int32_t num_str);
592  void write_string_list(
593  const SGString<uint32_t>* strings, int32_t num_str);
594  void write_string_list(
595  const SGString<int16_t>* strings, int32_t num_str);
596  void write_string_list(
597  const SGString<uint16_t>* strings, int32_t num_str);
598  void write_string_list(
599  const SGString<int64_t>* strings, int32_t num_str);
600  void write_string_list(
601  const SGString<uint64_t>* strings, int32_t num_str);
602  void write_string_list(
603  const SGString<float32_t>* strings, int32_t num_str);
604  void write_string_list(
605  const SGString<float64_t>* strings, int32_t num_str);
606  void write_string_list(
607  const SGString<floatmax_t>* strings, int32_t num_str);
609 
610 private:
612  int32_t version;
613 
615  int32_t message_size;
616 
618  uint8_t* buffer;
619 
621  uint8_t uint_buffer[4];
622 };
623 
624 }
625 
626 #endif
628 #endif

SHOGUN Machine Learning Toolbox - Documentation