SHOGUN  3.2.1
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
ProtobufFile.cpp
Go to the documentation of this file.
1 /*
2  * This program is free software; you can redistribute it and/or modify
3  * it under the terms of the GNU General Public License as published by
4  * the Free Software Foundation; either version 3 of the License, or
5  * (at your option) any later version.
6  *
7  * Written (W) 2013 Evgeniy Andreev (gsomix)
8  */
9 #ifdef HAVE_PROTOBUF
10 
11 #include <shogun/io/ProtobufFile.h>
12 #include <shogun/io/SGIO.h>
13 
14 #include <shogun/lib/SGVector.h>
15 #include <shogun/lib/SGMatrix.h>
17 #include <shogun/lib/SGString.h>
18 
19 using namespace shogun;
20 
22 {
23  init();
24 }
25 
26 CProtobufFile::CProtobufFile(FILE* f, const char* name) :
27  CFile(f, name)
28 {
29  init();
30 }
31 
32 CProtobufFile::CProtobufFile(const char* fname, char rw, const char* name) :
33  CFile(fname, rw, name)
34 {
35  init();
36 }
37 
39 {
40  SG_FREE(buffer);
41 }
42 
43 void CProtobufFile::init()
44 {
45  version=1;
46  message_size=1024*1024;
47 
48  buffer=SG_MALLOC(uint8_t, message_size*sizeof(uint32_t));
49 }
50 
51 #define GET_VECTOR(sg_type) \
52 void CProtobufFile::get_vector(sg_type*& vector, int32_t& len) \
53 { \
54  read_and_validate_global_header(ShogunVersion::VECTOR); \
55  VectorHeader data_header=read_vector_header(); \
56  len=data_header.len(); \
57  read_memory_block(vector, len, data_header.num_messages()); \
58 }
59 
60 GET_VECTOR(int8_t)
61 GET_VECTOR(uint8_t)
62 GET_VECTOR(char)
63 GET_VECTOR(int32_t)
64 GET_VECTOR(uint32_t)
68 GET_VECTOR(int16_t)
69 GET_VECTOR(uint16_t)
70 GET_VECTOR(int64_t)
71 GET_VECTOR(uint64_t)
72 #undef GET_VECTOR
73 
74 #define GET_MATRIX(read_func, sg_type) \
75 void CProtobufFile::get_matrix(sg_type*& matrix, int32_t& num_feat, int32_t& num_vec) \
76 { \
77  read_and_validate_global_header(ShogunVersion::MATRIX); \
78  MatrixHeader data_header=read_matrix_header(); \
79  num_feat=data_header.num_cols(); \
80  num_vec=data_header.num_rows(); \
81  read_memory_block(matrix, num_feat*num_vec, data_header.num_messages()); \
82 }
83 
84 GET_MATRIX(read_char, int8_t)
85 GET_MATRIX(read_byte, uint8_t)
86 GET_MATRIX(read_char, char)
87 GET_MATRIX(read_int, int32_t)
88 GET_MATRIX(read_uint, uint32_t)
89 GET_MATRIX(read_short_real, float32_t)
90 GET_MATRIX(read_real, float64_t)
91 GET_MATRIX(read_long_real, floatmax_t)
92 GET_MATRIX(read_short, int16_t)
93 GET_MATRIX(read_word, uint16_t)
94 GET_MATRIX(read_long, int64_t)
95 GET_MATRIX(read_ulong, uint64_t)
96 #undef GET_MATRIX
97 
98 #define GET_NDARRAY(read_func, sg_type) \
99 void CProtobufFile::get_ndarray(sg_type*& array, int32_t*& dims, int32_t& num_dims) \
100 { \
101  SG_NOTIMPLEMENTED \
102 }
103 
104 GET_NDARRAY(read_byte, uint8_t)
105 GET_NDARRAY(read_char, char)
106 GET_NDARRAY(read_int, int32_t)
107 GET_NDARRAY(read_short_real, float32_t)
108 GET_NDARRAY(read_real, float64_t)
109 GET_NDARRAY(read_short, int16_t)
110 GET_NDARRAY(read_word, uint16_t)
111 #undef GET_NDARRAY
112 
113 #define GET_SPARSE_MATRIX(sg_type) \
114 void CProtobufFile::get_sparse_matrix( \
115  SGSparseVector<sg_type>*& matrix, int32_t& num_feat, int32_t& num_vec) \
116 { \
117  read_and_validate_global_header(ShogunVersion::SPARSE_MATRIX); \
118  SparseMatrixHeader data_header=read_sparse_matrix_header(); \
119  num_feat=data_header.num_features(); \
120  num_vec=data_header.num_vectors(); \
121  read_sparse_matrix(matrix, data_header); \
122 }
123 
124 GET_SPARSE_MATRIX(bool)
125 GET_SPARSE_MATRIX(int8_t)
126 GET_SPARSE_MATRIX(uint8_t)
127 GET_SPARSE_MATRIX(char)
128 GET_SPARSE_MATRIX(int32_t)
129 GET_SPARSE_MATRIX(uint32_t)
133 GET_SPARSE_MATRIX(int16_t)
134 GET_SPARSE_MATRIX(uint16_t)
135 GET_SPARSE_MATRIX(int64_t)
136 GET_SPARSE_MATRIX(uint64_t)
137 #undef GET_SPARSE_MATRIX
138 
139 #define SET_VECTOR(sg_type) \
140 void CProtobufFile::set_vector(const sg_type* vector, int32_t len) \
141 { \
142  int32_t num_messages=compute_num_messages(len, sizeof(sg_type)); \
143  write_global_header(ShogunVersion::VECTOR); \
144  write_vector_header(len, num_messages); \
145  write_memory_block(vector, len, num_messages); \
146 }
147 
148 SET_VECTOR(int8_t)
149 SET_VECTOR(uint8_t)
150 SET_VECTOR(char)
151 SET_VECTOR(int32_t)
152 SET_VECTOR(uint32_t)
153 SET_VECTOR(int64_t)
154 SET_VECTOR(uint64_t)
158 SET_VECTOR(int16_t)
159 SET_VECTOR(uint16_t)
160 #undef SET_VECTOR
161 
162 #define SET_MATRIX(sg_type) \
163 void CProtobufFile::set_matrix(const sg_type* matrix, int32_t num_feat, int32_t num_vec) \
164 { \
165  int32_t num_messages=compute_num_messages(num_feat*num_vec, sizeof(sg_type)); \
166  write_global_header(ShogunVersion::MATRIX); \
167  write_matrix_header(num_feat, num_vec, num_messages); \
168  write_memory_block(matrix, num_feat*num_vec, num_messages); \
169 }
170 
171 SET_MATRIX(int8_t)
172 SET_MATRIX(uint8_t)
173 SET_MATRIX(char)
174 SET_MATRIX(int32_t)
175 SET_MATRIX(uint32_t)
176 SET_MATRIX(int64_t)
177 SET_MATRIX(uint64_t)
181 SET_MATRIX(int16_t)
182 SET_MATRIX(uint16_t)
183 #undef SET_MATRIX
184 
185 #define SET_SPARSE_MATRIX(sg_type) \
186 void CProtobufFile::set_sparse_matrix( \
187  const SGSparseVector<sg_type>* matrix, int32_t num_feat, int32_t num_vec) \
188 { \
189  write_global_header(ShogunVersion::SPARSE_MATRIX); \
190  write_sparse_matrix_header(matrix, num_feat, num_vec); \
191  write_sparse_matrix(matrix, num_vec); \
192 }
193 
194 SET_SPARSE_MATRIX(bool)
195 SET_SPARSE_MATRIX(int8_t)
196 SET_SPARSE_MATRIX(uint8_t)
197 SET_SPARSE_MATRIX(char)
198 SET_SPARSE_MATRIX(int32_t)
199 SET_SPARSE_MATRIX(uint32_t)
200 SET_SPARSE_MATRIX(int64_t)
201 SET_SPARSE_MATRIX(uint64_t)
205 SET_SPARSE_MATRIX(int16_t)
206 SET_SPARSE_MATRIX(uint16_t)
207 #undef SET_SPARSE_MATRIX
208 
209 #define GET_STRING_LIST(sg_type) \
210 void CProtobufFile::get_string_list( \
211  SGString<sg_type>*& strings, int32_t& num_str, \
212  int32_t& max_string_len) \
213 { \
214  read_and_validate_global_header(ShogunVersion::STRING_LIST); \
215  StringListHeader data_header=read_string_list_header(); \
216  num_str=data_header.num_str(); \
217  max_string_len=data_header.max_string_len(); \
218  read_string_list(strings, data_header); \
219 }
220 
221 GET_STRING_LIST(int8_t)
222 GET_STRING_LIST(uint8_t)
223 GET_STRING_LIST(char)
224 GET_STRING_LIST(int32_t)
225 GET_STRING_LIST(uint32_t)
226 GET_STRING_LIST(int64_t)
227 GET_STRING_LIST(uint64_t)
231 GET_STRING_LIST(int16_t)
232 GET_STRING_LIST(uint16_t)
233 #undef GET_STRING_LIST
234 
235 #define SET_STRING_LIST(sg_type) \
236 void CProtobufFile::set_string_list( \
237  const SGString<sg_type>* strings, int32_t num_str) \
238 { \
239  write_global_header(ShogunVersion::STRING_LIST); \
240  write_string_list_header(strings, num_str); \
241  write_string_list(strings, num_str); \
242 }
243 
244 SET_STRING_LIST(int8_t)
245 SET_STRING_LIST(uint8_t)
246 SET_STRING_LIST(char)
247 SET_STRING_LIST(int32_t)
248 SET_STRING_LIST(uint32_t)
249 SET_STRING_LIST(int64_t)
250 SET_STRING_LIST(uint64_t)
254 SET_STRING_LIST(int16_t)
255 SET_STRING_LIST(uint16_t)
256 #undef SET_STRING_LIST
257 
258 void CProtobufFile::write_big_endian_uint(uint32_t number, uint8_t* array, uint32_t size)
259 {
260  if (size<4)
261  SG_ERROR("array is too small to write\n");
262 
263  array[0]=(number>>24)&0xffu;
264  array[1]=(number>>16)&0xffu;
265  array[2]=(number>>8)&0xffu;
266  array[3]=number&0xffu;
267 }
268 
269 uint32_t CProtobufFile::read_big_endian_uint(uint8_t* array, uint32_t size)
270 {
271  if (size<4)
272  SG_ERROR("array is too small to read\n");
273 
274  return (array[0]<<24) | (array[1]<<16) | (array[2]<<8) | array[3];
275 }
276 
277 int32_t CProtobufFile::compute_num_messages(uint64_t len, int32_t sizeof_type) const
278 {
279  uint32_t elements_in_message=message_size/sizeof_type;
280  uint32_t num_messages=len/elements_in_message;
281  if (len % elements_in_message > 0)
282  num_messages++;
283 
284  return num_messages;
285 }
286 
287 void CProtobufFile::read_and_validate_global_header(ShogunVersion_SGDataType type)
288 {
289  ShogunVersion header;
290  read_message(header);
291  REQUIRE(header.version()==version, "wrong version\n")
292  REQUIRE(header.data_type()==type, "wrong type\n")
293 }
294 
295 void CProtobufFile::write_global_header(ShogunVersion_SGDataType type)
296 {
297  ShogunVersion header;
298  header.set_version(version);
299  header.set_data_type(type);
300  write_message(header);
301 }
302 
303 VectorHeader CProtobufFile::read_vector_header()
304 {
305  VectorHeader data_header;
306  read_message(data_header);
307 
308  return data_header;
309 }
310 
311 SparseMatrixHeader CProtobufFile::read_sparse_matrix_header()
312 {
313  SparseMatrixHeader data_header;
314  read_message(data_header);
315 
316  return data_header;
317 }
318 
319 MatrixHeader CProtobufFile::read_matrix_header()
320 {
321  MatrixHeader data_header;
322  read_message(data_header);
323 
324  return data_header;
325 }
326 
327 StringListHeader CProtobufFile::read_string_list_header()
328 {
329  StringListHeader data_header;
330  read_message(data_header);
331 
332  return data_header;
333 }
334 
335 void CProtobufFile::write_vector_header(int32_t len, int32_t num_messages)
336 {
337  VectorHeader data_header;
338  data_header.set_len(len);
339  data_header.set_num_messages(num_messages);
340  write_message(data_header);
341 }
342 
343 void CProtobufFile::write_matrix_header(int32_t num_feat, int32_t num_vec, int32_t num_messages)
344 {
345  MatrixHeader data_header;
346  data_header.set_num_cols(num_feat);
347  data_header.set_num_rows(num_vec);
348  data_header.set_num_messages(num_messages);
349  write_message(data_header);
350 }
351 
352 #define WRITE_SPARSE_MATRIX_HEADER(sg_type) \
353 void CProtobufFile::write_sparse_matrix_header( \
354  const SGSparseVector<sg_type>* matrix, int32_t num_feat, int32_t num_vec) \
355 { \
356  SparseMatrixHeader data_header; \
357  data_header.set_num_features(num_feat); \
358  data_header.set_num_vectors(num_vec); \
359  for (int32_t i=0; i<num_vec; i++) \
360  { \
361  data_header.add_num_feat_entries(matrix[i].num_feat_entries); \
362  } \
363  \
364  write_message(data_header); \
365 }
366 
380 #undef WRITE_SPARSE_MATRIX_HEADER
381 
382 #define WRITE_STRING_LIST_HEADER(sg_type) \
383 void CProtobufFile::write_string_list_header(const SGString<sg_type>* strings, int32_t num_str) \
384 { \
385  int32_t max_string_len=0; \
386  StringListHeader data_header; \
387  data_header.set_num_str(num_str); \
388  for (int32_t i=0; i<num_str; i++) \
389  { \
390  data_header.add_str_len(strings[i].slen); \
391  if (strings[i].slen>max_string_len) \
392  max_string_len=strings[i].slen; \
393  } \
394  data_header.set_max_string_len(max_string_len); \
395  write_message(data_header); \
396 }
397 
402 WRITE_STRING_LIST_HEADER(uint32_t)
404 WRITE_STRING_LIST_HEADER(uint64_t)
409 WRITE_STRING_LIST_HEADER(uint16_t)
410 #undef WRITE_STRING_LIST_HEADER
411 
412 void CProtobufFile::read_message(google::protobuf::Message& message)
413 {
414  uint32_t bytes_read=0;
415  uint32_t msg_size=0;
416 
417  // read size of message
418  bytes_read=fread(uint_buffer, sizeof(char), sizeof(uint32_t), file);
419  REQUIRE(bytes_read==sizeof(uint32_t), "IO error\n");
420  msg_size=read_big_endian_uint(uint_buffer, sizeof(uint32_t));
421  REQUIRE(msg_size>0, "message size should be more than zero\n");
422 
423  // read message
424  bytes_read=fread(buffer, sizeof(char), msg_size, file);
425  REQUIRE(bytes_read==msg_size, "IO error\n");
426 
427  // try to parse message from read data
428  REQUIRE(message.ParseFromArray(buffer, msg_size), "cannot parse header\n");
429 }
430 
431 void CProtobufFile::write_message(const google::protobuf::Message& message)
432 {
433  uint32_t bytes_write=0;
434  uint32_t msg_size=message.ByteSize();
435 
436  // write size of message
437  write_big_endian_uint(msg_size, uint_buffer, sizeof(uint32_t));
438  bytes_write=fwrite(uint_buffer, sizeof(char), sizeof(uint32_t), file);
439  REQUIRE(bytes_write==sizeof(uint32_t), "IO error\n");
440 
441  // write serialized message
442  message.SerializeToArray(buffer, msg_size);
443  bytes_write=fwrite(buffer, sizeof(char), msg_size, file);
444  REQUIRE(bytes_write==msg_size, "IO error\n");
445 }
446 
447 #define READ_MEMORY_BLOCK(chunk_type, sg_type) \
448 void CProtobufFile::read_memory_block(sg_type*& vector, uint64_t len, int32_t num_messages) \
449 { \
450  vector=SG_MALLOC(sg_type, len); \
451  \
452  chunk_type chunk; \
453  int32_t elements_in_message=message_size/sizeof(sg_type); \
454  for (int32_t i=0; i<num_messages; i++) \
455  { \
456  read_message(chunk); \
457  \
458  int32_t num_elements_to_read=0; \
459  if ((len-(i+1)*elements_in_message)<=0) \
460  num_elements_to_read=len-i*elements_in_message; \
461  else \
462  num_elements_to_read=elements_in_message; \
463  \
464  for (int32_t j=0; j<num_elements_to_read; j++) \
465  vector[j+i*elements_in_message]=chunk.data(j); \
466  } \
467 }
468 
469 READ_MEMORY_BLOCK(Int32Chunk, int8_t)
470 READ_MEMORY_BLOCK(UInt32Chunk, uint8_t)
471 READ_MEMORY_BLOCK(UInt32Chunk, char)
472 READ_MEMORY_BLOCK(Int32Chunk, int32_t)
473 READ_MEMORY_BLOCK(UInt32Chunk, uint32_t)
474 READ_MEMORY_BLOCK(Float32Chunk, float32_t)
475 READ_MEMORY_BLOCK(Float64Chunk, float64_t)
476 READ_MEMORY_BLOCK(Float64Chunk, floatmax_t)
477 READ_MEMORY_BLOCK(Int32Chunk, int16_t)
478 READ_MEMORY_BLOCK(UInt32Chunk, uint16_t)
479 READ_MEMORY_BLOCK(Int64Chunk, int64_t)
480 READ_MEMORY_BLOCK(UInt64Chunk, uint64_t)
481 #undef READ_MEMORY_BLOCK
482 
483 #define WRITE_MEMORY_BLOCK(chunk_type, sg_type) \
484 void CProtobufFile::write_memory_block(const sg_type* vector, uint64_t len, int32_t num_messages) \
485 { \
486  chunk_type chunk; \
487  int32_t elements_in_message=message_size/sizeof(sg_type); \
488  for (int32_t i=0; i<num_messages; i++) \
489  { \
490  \
491  int32_t num_elements_to_write=0; \
492  if ((len-(i+1)*elements_in_message)<=0) \
493  num_elements_to_write=len-i*elements_in_message; \
494  else \
495  num_elements_to_write=elements_in_message; \
496  \
497  for (int32_t j=0; j<num_elements_to_write; j++) \
498  chunk.add_data(vector[j+i*elements_in_message]); \
499  \
500  write_message(chunk); \
501  chunk.Clear(); \
502  } \
503 }
504 
505 WRITE_MEMORY_BLOCK(Int32Chunk, int8_t)
506 WRITE_MEMORY_BLOCK(UInt32Chunk, uint8_t)
507 WRITE_MEMORY_BLOCK(UInt32Chunk, char)
508 WRITE_MEMORY_BLOCK(Int32Chunk, int32_t)
509 WRITE_MEMORY_BLOCK(UInt64Chunk, uint32_t)
510 WRITE_MEMORY_BLOCK(Int64Chunk, int64_t)
511 WRITE_MEMORY_BLOCK(UInt64Chunk, uint64_t)
512 WRITE_MEMORY_BLOCK(Float32Chunk, float32_t)
513 WRITE_MEMORY_BLOCK(Float64Chunk, float64_t)
514 WRITE_MEMORY_BLOCK(Float64Chunk, floatmax_t)
515 WRITE_MEMORY_BLOCK(Int32Chunk, int16_t)
516 WRITE_MEMORY_BLOCK(UInt32Chunk, uint16_t)
517 #undef WRITE_MEMORY_BLOCK
518 
519 #define READ_SPARSE_MATRIX(chunk_type, sg_type) \
520 void CProtobufFile::read_sparse_matrix( \
521  SGSparseVector<sg_type>*& matrix, const SparseMatrixHeader& data_header) \
522 { \
523  matrix=SG_MALLOC(SGSparseVector<sg_type>, data_header.num_vectors()); \
524  \
525  UInt64Chunk feat_index_chunk; \
526  chunk_type entry_chunk; \
527  read_message(feat_index_chunk); \
528  read_message(entry_chunk); \
529  \
530  int32_t elements_in_message=message_size/sizeof(sg_type); \
531  int32_t buffer_counter=0; \
532  for (uint32_t i=0; i<data_header.num_vectors(); i++) \
533  { \
534  matrix[i]=SGSparseVector<sg_type>(data_header.num_feat_entries(i)); \
535  for (int32_t j=0; j<matrix[i].num_feat_entries; j++) \
536  { \
537  matrix[i].features[j].feat_index=feat_index_chunk.data(buffer_counter); \
538  matrix[i].features[j].entry=entry_chunk.data(buffer_counter); \
539  buffer_counter++; \
540  \
541  if (buffer_counter==elements_in_message) \
542  { \
543  read_message(feat_index_chunk); \
544  read_message(entry_chunk); \
545  buffer_counter=0; \
546  } \
547  } \
548  } \
549 }
550 
551 READ_SPARSE_MATRIX(BoolChunk, bool)
552 READ_SPARSE_MATRIX(Int32Chunk, int8_t)
553 READ_SPARSE_MATRIX(UInt32Chunk, uint8_t)
554 READ_SPARSE_MATRIX(UInt32Chunk, char)
555 READ_SPARSE_MATRIX(Int32Chunk, int32_t)
556 READ_SPARSE_MATRIX(UInt32Chunk, uint32_t)
557 READ_SPARSE_MATRIX(Float32Chunk, float32_t)
558 READ_SPARSE_MATRIX(Float64Chunk, float64_t)
559 READ_SPARSE_MATRIX(Float64Chunk, floatmax_t)
560 READ_SPARSE_MATRIX(Int32Chunk, int16_t)
561 READ_SPARSE_MATRIX(UInt32Chunk, uint16_t)
562 READ_SPARSE_MATRIX(Int64Chunk, int64_t)
563 READ_SPARSE_MATRIX(UInt64Chunk, uint64_t)
564 #undef READ_SPARSE_MATRIX
565 
566 #define WRITE_SPARSE_MATRIX(chunk_type, sg_type) \
567 void CProtobufFile::write_sparse_matrix( \
568  const SGSparseVector<sg_type>* matrix, int32_t num_vec) \
569 { \
570  UInt64Chunk feat_index_chunk; \
571  chunk_type entry_chunk; \
572  int32_t elements_in_message=message_size/sizeof(sg_type); \
573  int32_t buffer_counter=0; \
574  for (int32_t i=0; i<num_vec; i++) \
575  { \
576  for (int32_t j=0; j<matrix[i].num_feat_entries; j++) \
577  { \
578  feat_index_chunk.add_data(matrix[i].features[j].feat_index); \
579  entry_chunk.add_data(matrix[i].features[j].entry); \
580  buffer_counter++; \
581  \
582  if (buffer_counter==elements_in_message) \
583  { \
584  write_message(feat_index_chunk); \
585  write_message(entry_chunk); \
586  feat_index_chunk.Clear(); \
587  entry_chunk.Clear(); \
588  buffer_counter=0; \
589  } \
590  } \
591  } \
592  \
593  if (buffer_counter!=0) \
594  { \
595  write_message(feat_index_chunk); \
596  write_message(entry_chunk); \
597  } \
598 }
599 
600 WRITE_SPARSE_MATRIX(BoolChunk, bool)
601 WRITE_SPARSE_MATRIX(Int32Chunk, int8_t)
602 WRITE_SPARSE_MATRIX(UInt32Chunk, uint8_t)
603 WRITE_SPARSE_MATRIX(UInt32Chunk, char)
604 WRITE_SPARSE_MATRIX(Int32Chunk, int32_t)
605 WRITE_SPARSE_MATRIX(UInt64Chunk, uint32_t)
606 WRITE_SPARSE_MATRIX(Int64Chunk, int64_t)
607 WRITE_SPARSE_MATRIX(UInt64Chunk, uint64_t)
608 WRITE_SPARSE_MATRIX(Float32Chunk, float32_t)
609 WRITE_SPARSE_MATRIX(Float64Chunk, float64_t)
610 WRITE_SPARSE_MATRIX(Float64Chunk, floatmax_t)
611 WRITE_SPARSE_MATRIX(Int32Chunk, int16_t)
612 WRITE_SPARSE_MATRIX(UInt32Chunk, uint16_t)
613 #undef WRITE_SPARSE_MATRIX
614 
615 #define READ_STRING_LIST(chunk_type, sg_type) \
616 void CProtobufFile::read_string_list( \
617  SGString<sg_type>*& strings, const StringListHeader& data_header) \
618 { \
619  strings=SG_MALLOC(SGString<sg_type>, data_header.num_str()); \
620  \
621  chunk_type chunk; \
622  read_message(chunk); \
623  int32_t elements_in_message=message_size/sizeof(sg_type); \
624  int32_t buffer_counter=0; \
625  for (uint32_t i=0; i<data_header.num_str(); i++) \
626  { \
627  strings[i]=SGString<sg_type>(data_header.str_len(i)); \
628  for (int32_t j=0; j<strings[i].slen; j++) \
629  { \
630  strings[i].string[j]=chunk.data(buffer_counter); \
631  buffer_counter++; \
632  \
633  if (buffer_counter==elements_in_message) \
634  { \
635  read_message(chunk); \
636  buffer_counter=0; \
637  } \
638  } \
639  } \
640 }
641 
642 READ_STRING_LIST(Int32Chunk, int8_t)
643 READ_STRING_LIST(UInt32Chunk, uint8_t)
644 READ_STRING_LIST(UInt32Chunk, char)
645 READ_STRING_LIST(Int32Chunk, int32_t)
646 READ_STRING_LIST(UInt32Chunk, uint32_t)
647 READ_STRING_LIST(Float32Chunk, float32_t)
648 READ_STRING_LIST(Float64Chunk, float64_t)
649 READ_STRING_LIST(Float64Chunk, floatmax_t)
650 READ_STRING_LIST(Int32Chunk, int16_t)
651 READ_STRING_LIST(UInt32Chunk, uint16_t)
652 READ_STRING_LIST(Int64Chunk, int64_t)
653 READ_STRING_LIST(UInt64Chunk, uint64_t)
654 #undef READ_STRING_LIST
655 
656 #define WRITE_STRING_LIST(chunk_type, sg_type) \
657 void CProtobufFile::write_string_list( \
658  const SGString<sg_type>* strings, int32_t num_str) \
659 { \
660  chunk_type chunk; \
661  int32_t elements_in_message=message_size/sizeof(sg_type); \
662  int32_t buffer_counter=0; \
663  for (int32_t i=0; i<num_str; i++) \
664  { \
665  for (int32_t j=0; j<strings[i].slen; j++) \
666  { \
667  chunk.add_data(strings[i].string[j]); \
668  buffer_counter++; \
669  \
670  if (buffer_counter==elements_in_message) \
671  { \
672  write_message(chunk); \
673  chunk.Clear(); \
674  buffer_counter=0; \
675  } \
676  } \
677  } \
678  \
679  if (buffer_counter!=0) \
680  write_message(chunk); \
681 }
682 
683 WRITE_STRING_LIST(Int32Chunk, int8_t)
684 WRITE_STRING_LIST(UInt32Chunk, uint8_t)
685 WRITE_STRING_LIST(UInt32Chunk, char)
686 WRITE_STRING_LIST(Int32Chunk, int32_t)
687 WRITE_STRING_LIST(UInt64Chunk, uint32_t)
688 WRITE_STRING_LIST(Int64Chunk, int64_t)
689 WRITE_STRING_LIST(UInt64Chunk, uint64_t)
690 WRITE_STRING_LIST(Float32Chunk, float32_t)
691 WRITE_STRING_LIST(Float64Chunk, float64_t)
692 WRITE_STRING_LIST(Float64Chunk, floatmax_t)
693 WRITE_STRING_LIST(Int32Chunk, int16_t)
694 WRITE_STRING_LIST(UInt32Chunk, uint16_t)
695 #undef WRITE_STRING_LIST
696 
697 #endif /* HAVE_PROTOBUF */

SHOGUN Machine Learning Toolbox - Documentation