SHOGUN  4.1.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
NeuralNetwork.cpp
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2014, Shogun Toolbox Foundation
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions are met:
7 
8  * 1. Redistributions of source code must retain the above copyright notice,
9  * this list of conditions and the following disclaimer.
10  *
11  * 2. Redistributions in binary form must reproduce the above copyright notice,
12  * this list of conditions and the following disclaimer in the documentation
13  * and/or other materials provided with the distribution.
14  *
15  * 3. Neither the name of the copyright holder nor the names of its
16  * contributors may be used to endorse or promote products derived from this
17  * software without specific prior written permission.
18 
19  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
23  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29  * POSSIBILITY OF SUCH DAMAGE.
30  *
31  * Written (W) 2014 Khaled Nasr
32  */
33 
40 
41 using namespace shogun;
42 
44 : CMachine()
45 {
46  init();
47 }
48 
50 {
51  init();
52  set_layers(layers);
53 }
54 
56 {
57  REQUIRE(layers, "Layers should not be NULL")
58 
60  SG_REF(layers);
61  m_layers = layers;
62 
66 
67  m_num_inputs = 0;
68  for (int32_t i=0; i<m_num_layers; i++)
69  {
70  if (get_layer(i)->is_input())
72  }
73 }
74 
75 void CNeuralNetwork::connect(int32_t i, int32_t j)
76 {
77  REQUIRE("i<j", "i(%i) must be less that j(%i)\n", i, j);
78  m_adj_matrix(i,j) = true;
79 }
80 
82 {
84  for (int32_t i=1; i<m_num_layers; i++)
85  m_adj_matrix(i-1, i) = true;
86 }
87 
88 void CNeuralNetwork::disconnect(int32_t i, int32_t j)
89 {
90  m_adj_matrix(i,j) = false;
91 }
92 
94 {
96 }
97 
99 {
100  for (int32_t j=0; j<m_num_layers; j++)
101  {
102  if (!get_layer(j)->is_input())
103  {
104  int32_t num_inputs = 0;
105  for (int32_t i=0; i<m_num_layers; i++)
106  num_inputs += m_adj_matrix(i,j);
107 
108  SGVector<int32_t> input_indices(num_inputs);
109 
110  int32_t k = 0;
111  for (int i=0; i<m_num_layers; i++)
112  {
113  if (m_adj_matrix(i,j))
114  {
115  input_indices[k] = i;
116  k++;
117  }
118  }
119 
120  get_layer(j)->initialize(m_layers, input_indices);
121  }
122  }
123 
125 
127  m_index_offsets[0] = 0;
128  for (int32_t i=1; i<m_num_layers; i++)
129  {
131  m_total_num_parameters += get_layer(i)->get_num_parameters();
132  }
133 
136 
137  m_params.zero();
138  m_param_regularizable.set_const(true);
139 
140  for (int32_t i=0; i<m_num_layers; i++)
141  {
142  SGVector<float64_t> layer_param = get_section(m_params, i);
143  SGVector<bool> layer_param_regularizable =
144  get_section(m_param_regularizable, i);
145 
146  get_layer(i)->initialize_parameters(layer_param,
147  layer_param_regularizable, sigma);
148 
150  }
151 }
152 
154 {
156 }
157 
159 {
160  SGMatrix<float64_t> output_activations = forward_propagate(data);
162 
163  for (int32_t i=0; i<m_batch_size; i++)
164  {
165  if (get_num_outputs()==1)
166  {
167  if (output_activations[i]>0.5) labels->set_label(i, 1);
168  else labels->set_label(i, -1);
169 
170  labels->set_value(output_activations[i], i);
171  }
172  else if (get_num_outputs()==2)
173  {
174  float64_t v1 = output_activations[2*i];
175  float64_t v2 = output_activations[2*i+1];
176  if (v1>v2)
177  labels->set_label(i, 1);
178  else labels->set_label(i, -1);
179 
180  labels->set_value(v2/(v1+v2), i);
181  }
182  }
183 
184  return labels;
185 }
186 
188 {
189  SGMatrix<float64_t> output_activations = forward_propagate(data);
190  SGVector<float64_t> labels_vec(m_batch_size);
191 
192  for (int32_t i=0; i<m_batch_size; i++)
193  labels_vec[i] = output_activations[i];
194 
195  return new CRegressionLabels(labels_vec);
196 }
197 
198 
200 {
201  SGMatrix<float64_t> output_activations = forward_propagate(data);
202  SGVector<float64_t> labels_vec(m_batch_size);
203 
204  for (int32_t i=0; i<m_batch_size; i++)
205  {
206  labels_vec[i] = CMath::arg_max(
207  output_activations.matrix+i*get_num_outputs(), 1, get_num_outputs());
208  }
209 
210  CMulticlassLabels* labels = new CMulticlassLabels(labels_vec);
211 
213  for (int32_t i=0; i<m_batch_size; i++)
214  {
216  output_activations.matrix, get_num_outputs(), i*get_num_outputs()));
217  }
218 
219  return labels;
220 }
221 
224 {
225  SGMatrix<float64_t> output_activations = forward_propagate(data);
226  return new CDenseFeatures<float64_t>(output_activations);
227 }
228 
230 {
232  "Maximum number of epochs (%i) must be >= 0\n", max_num_epochs);
233 
236 
237  for (int32_t i=0; i<m_num_layers-1; i++)
238  {
239  get_layer(i)->dropout_prop =
241  }
242  get_layer(m_num_layers-1)->dropout_prop = 0.0;
243 
244  m_is_training = true;
245  for (int32_t i=0; i<m_num_layers; i++)
246  get_layer(i)->is_training = true;
247 
248  bool result = false;
250  result = train_gradient_descent(inputs, targets);
252  result = train_lbfgs(inputs, targets);
253 
254  for (int32_t i=0; i<m_num_layers; i++)
255  get_layer(i)->is_training = false;
256  m_is_training = false;
257 
258  return result;
259 }
260 
262  SGMatrix<float64_t> targets)
263 {
265  "Gradient descent learning rate (%f) must be > 0\n", gd_learning_rate);
266  REQUIRE(gd_momentum>=0,
267  "Gradient descent momentum (%f) must be > 0\n", gd_momentum);
268 
269  int32_t training_set_size = inputs.num_cols;
270  if (gd_mini_batch_size==0) gd_mini_batch_size = training_set_size;
272 
273  int32_t n_param = get_num_parameters();
274  SGVector<float64_t> gradients(n_param);
275 
276  // needed for momentum
277  SGVector<float64_t> param_updates(n_param);
278  param_updates.zero();
279 
280  float64_t error_last_time = -1.0, error = -1.0;
281 
283  if (c==-1.0)
284  c = 0.99*(float64_t)gd_mini_batch_size/training_set_size + 1e-2;
285 
286  bool continue_training = true;
287  float64_t alpha = gd_learning_rate;
288 
289  for (int32_t i=0; continue_training; i++)
290  {
291  if (max_num_epochs!=0)
292  if (i>=max_num_epochs) break;
293 
294  for (int32_t j=0; j < training_set_size; j += gd_mini_batch_size)
295  {
296  alpha = gd_learning_rate_decay*alpha;
297 
298  if (j+gd_mini_batch_size>training_set_size)
299  j = training_set_size-gd_mini_batch_size;
300 
301  SGMatrix<float64_t> targets_batch(targets.matrix+j*get_num_outputs(),
302  get_num_outputs(), gd_mini_batch_size, false);
303 
304  SGMatrix<float64_t> inputs_batch(inputs.matrix+j*m_num_inputs,
305  m_num_inputs, gd_mini_batch_size, false);
306 
307  for (int32_t k=0; k<n_param; k++)
308  m_params[k] += gd_momentum*param_updates[k];
309 
310  float64_t e = compute_gradients(inputs_batch, targets_batch, gradients);
311 
312  // filter the errors
313  if (error==-1.0)
314  error = e;
315  else
316  error = (1.0-c) * error + c*e;
317 
318  for (int32_t k=0; k<n_param; k++)
319  {
320  param_updates[k] = gd_momentum*param_updates[k]
321  -alpha*gradients[k];
322 
323  m_params[k] -= alpha*gradients[k];
324  }
325 
326  if (error_last_time!=-1.0)
327  {
328  float64_t error_change = (error_last_time-error)/error;
329  if (error_change< epsilon && error_change>=0)
330  {
331  SG_INFO("Gradient Descent Optimization Converged\n");
332  continue_training = false;
333  break;
334  }
335 
336  SG_INFO("Epoch %i: Error = %f\n",i, error);
337  }
338  error_last_time = error;
339  }
340  }
341 
342  return true;
343 }
344 
346  const SGMatrix<float64_t> targets)
347 {
348  int32_t training_set_size = inputs.num_cols;
349  set_batch_size(training_set_size);
350 
351  lbfgs_parameter_t lbfgs_param;
352  lbfgs_parameter_init(&lbfgs_param);
353  lbfgs_param.max_iterations = max_num_epochs;
354  lbfgs_param.epsilon = 0;
355  lbfgs_param.past = 1;
356  lbfgs_param.delta = epsilon;
357 
358  m_lbfgs_temp_inputs = &inputs;
359  m_lbfgs_temp_targets = &targets;
360 
361  int32_t result = lbfgs(m_total_num_parameters,
362  m_params,
363  NULL,
364  &CNeuralNetwork::lbfgs_evaluate,
365  &CNeuralNetwork::lbfgs_progress,
366  this,
367  &lbfgs_param);
368 
369  m_lbfgs_temp_inputs = NULL;
370  m_lbfgs_temp_targets = NULL;
371 
372  if (result==LBFGS_SUCCESS || 1)
373  {
374  SG_INFO("L-BFGS Optimization Converged\n");
375  }
376  else if (result==LBFGSERR_MAXIMUMITERATION)
377  {
378  SG_INFO("L-BFGS Max Number of Epochs reached\n");
379  }
380  else
381  {
382  SG_INFO("L-BFGS optimization ended with return code %i\n",result);
383  }
384  return true;
385 }
386 
387 float64_t CNeuralNetwork::lbfgs_evaluate(void* userdata,
388  const float64_t* W,
389  float64_t* grad,
390  const int32_t n,
391  const float64_t step)
392 {
393  CNeuralNetwork* network = static_cast<CNeuralNetwork*>(userdata);
394 
395  SGVector<float64_t> grad_vector(grad, network->get_num_parameters(), false);
396 
397  return network->compute_gradients(*network->m_lbfgs_temp_inputs,
398  *network->m_lbfgs_temp_targets, grad_vector);
399 }
400 
401 int CNeuralNetwork::lbfgs_progress(void* instance,
402  const float64_t* x,
403  const float64_t* g,
404  const float64_t fx,
405  const float64_t xnorm,
406  const float64_t gnorm,
407  const float64_t step,
408  int n, int k, int ls)
409 {
410  SG_SINFO("Epoch %i: Error = %f\n",k, fx);
411  return 0;
412 }
413 
415 {
418  return forward_propagate(inputs, j);
419 }
420 
422  SGMatrix<float64_t> inputs, int32_t j)
423 {
424  if (j==-1)
425  j = m_num_layers-1;
426 
427  for (int32_t i=0; i<=j; i++)
428  {
429  CNeuralLayer* layer = get_layer(i);
430 
431  if (layer->is_input())
432  layer->compute_activations(inputs);
433  else
434  layer->compute_activations(get_section(m_params, i), m_layers);
435 
436  layer->dropout_activations();
437  }
438 
439  return get_layer(j)->get_activations();
440 }
441 
443  SGMatrix<float64_t> targets, SGVector<float64_t> gradients)
444 {
445  forward_propagate(inputs);
446 
447  for (int32_t i=0; i<m_num_layers; i++)
449 
450  for (int32_t i=m_num_layers-1; i>=0; i--)
451  {
452  if (i==m_num_layers-1)
453  get_layer(i)->compute_gradients(get_section(m_params,i), targets,
454  m_layers, get_section(gradients,i));
455  else
456  get_layer(i)->compute_gradients(get_section(m_params,i),
457  SGMatrix<float64_t>(), m_layers, get_section(gradients,i));
458  }
459 
460  // L2 regularization
461  if (l2_coefficient != 0.0)
462  {
463  for (int32_t i=0; i<m_total_num_parameters; i++)
464  {
465  if (m_param_regularizable[i])
466  gradients[i] += l2_coefficient*m_params[i];
467  }
468  }
469 
470  // L1 regularization
471  if (l1_coefficient != 0.0)
472  {
473  for (int32_t i=0; i<m_total_num_parameters; i++)
474  {
475  if (m_param_regularizable[i])
476  gradients[i] +=
477  l1_coefficient*CMath::sign<float64_t>(m_params[i]);
478  }
479  }
480 
481  // max-norm regularization
482  if (max_norm != -1.0)
483  {
484  for (int32_t i=0; i<m_num_layers; i++)
485  {
486  SGVector<float64_t> layer_params = get_section(m_params,i);
487  get_layer(i)->enforce_max_norm(layer_params, max_norm);
488  }
489  }
490 
491  return compute_error(targets);
492 }
493 
495 {
496  float64_t error = get_layer(m_num_layers-1)->compute_error(targets);
497 
498  // L2 regularization
499  if (l2_coefficient != 0.0)
500  {
501  for (int32_t i=0; i<m_total_num_parameters; i++)
502  {
503  if (m_param_regularizable[i])
504  error += 0.5*l2_coefficient*m_params[i]*m_params[i];
505  }
506  }
507 
508  // L1 regularization
509  if (l1_coefficient != 0.0)
510  {
511  for (int32_t i=0; i<m_total_num_parameters; i++)
512  {
513  if (m_param_regularizable[i])
514  error += l1_coefficient*CMath::abs(m_params[i]);
515  }
516  }
517 
518  return error;
519 }
520 
522  SGMatrix<float64_t> targets)
523 {
524  forward_propagate(inputs);
525  return compute_error(targets);
526 }
527 
528 
530 {
531  // some random inputs and ouputs
534 
535  for (int32_t i=0; i<x.num_rows; i++)
536  x[i] = CMath::random(0.0,1.0);
537 
538  // the outputs are set up in the form of a probability distribution (in case
539  // that is required by the output layer, i.e softmax)
540  for (int32_t i=0; i<y.num_rows; i++)
541  y[i] = CMath::random(0.0,1.0);
542 
544  for (int32_t i=0; i<y.num_rows; i++)
545  y[i] /= y_sum;
546 
547  set_batch_size(1);
548 
549  // numerically compute gradients
550  SGVector<float64_t> gradients_numerical(m_total_num_parameters);
551 
552  for (int32_t i=0; i<m_total_num_parameters; i++)
553  {
554  float64_t c =
555  CMath::max<float64_t>(CMath::abs(approx_epsilon*m_params[i]),s);
556 
557  m_params[i] += c;
558  float64_t error_plus = compute_error(x,y);
559  m_params[i] -= 2*c;
560  float64_t error_minus = compute_error(x,y);
561  m_params[i] += c;
562 
563  gradients_numerical[i] = (error_plus-error_minus)/(2*c);
564  }
565 
566  // compute gradients using backpropagation
567  SGVector<float64_t> gradients_backprop(m_total_num_parameters);
568  compute_gradients(x, y, gradients_backprop);
569 
570  float64_t sum = 0.0;
571  for (int32_t i=0; i<m_total_num_parameters; i++)
572  {
573  sum += CMath::abs(gradients_backprop[i]-gradients_numerical[i]);
574  }
575 
576  return sum/m_total_num_parameters;
577 }
578 
579 void CNeuralNetwork::set_batch_size(int32_t batch_size)
580 {
581  if (batch_size!=m_batch_size)
582  {
583  m_batch_size = batch_size;
584  for (int32_t i=0; i<m_num_layers; i++)
586  }
587 }
588 
590 {
591  REQUIRE(features != NULL, "Invalid (NULL) feature pointer\n");
592  REQUIRE(features->get_feature_type() == F_DREAL,
593  "Feature type must be F_DREAL\n");
594  REQUIRE(features->get_feature_class() == C_DENSE,
595  "Feature class must be C_DENSE\n");
596 
598  REQUIRE(inputs->get_num_features()==m_num_inputs,
599  "Number of features (%i) must match the network's number of inputs "
600  "(%i)\n", inputs->get_num_features(), get_num_inputs());
601 
602  return inputs->get_feature_matrix();
603 }
604 
606 {
607  REQUIRE(labs != NULL, "Invalid (NULL) labels pointer\n");
608 
610  targets.zero();
611 
612  if (labs->get_label_type() == LT_MULTICLASS)
613  {
614  CMulticlassLabels* labels_mc = (CMulticlassLabels*) labs;
615  REQUIRE(labels_mc->get_num_classes()==get_num_outputs(),
616  "Number of classes (%i) must match the network's number of "
617  "outputs (%i)\n", labels_mc->get_num_classes(), get_num_outputs());
618 
619  for (int32_t i=0; i<labels_mc->get_num_labels(); i++)
620  targets[((int32_t)labels_mc->get_label(i))+ i*get_num_outputs()]
621  = 1.0;
622  }
623  else if (labs->get_label_type() == LT_BINARY)
624  {
625  CBinaryLabels* labels_bin = (CBinaryLabels*) labs;
626  if (get_num_outputs()==1)
627  {
628  for (int32_t i=0; i<labels_bin->get_num_labels(); i++)
629  targets[i] = (labels_bin->get_label(i)==1);
630  }
631  else if (get_num_outputs()==2)
632  {
633  for (int32_t i=0; i<labels_bin->get_num_labels(); i++)
634  {
635  targets[i*2] = (labels_bin->get_label(i)==1);
636  targets[i*2+1] = (labels_bin->get_label(i)==-1);
637  }
638  }
639  }
640  else if (labs->get_label_type() == LT_REGRESSION)
641  {
642  CRegressionLabels* labels_reg = (CRegressionLabels*) labs;
643  for (int32_t i=0; i<labels_reg->get_num_labels(); i++)
644  targets[i] = labels_reg->get_label(i);
645  }
646 
647  return targets;
648 }
649 
651 {
652  // problem type depends on the type of labels given to the network
653  // if no labels are given yet, just return PT_MULTICLASS
654  if (m_labels==NULL)
655  return PT_MULTICLASS;
656 
658  return PT_BINARY;
659  else if (m_labels->get_label_type() == LT_REGRESSION)
660  return PT_REGRESSION;
661  else return PT_MULTICLASS;
662 }
663 
665 {
666  return (lab->get_label_type() == LT_MULTICLASS ||
667  lab->get_label_type() == LT_BINARY ||
668  lab->get_label_type() == LT_REGRESSION);
669 }
670 
672 {
673  if (lab->get_label_type() == LT_BINARY)
674  {
675  REQUIRE(get_num_outputs() <= 2, "Cannot use %s in a neural network "
676  "with more that 2 output neurons\n", lab->get_name());
677  }
678  else if (lab->get_label_type() == LT_REGRESSION)
679  {
680  REQUIRE(get_num_outputs() == 1, "Cannot use %s in a neural network "
681  "with more that 1 output neuron\n", lab->get_name());
682  }
683 
685 }
686 
688 {
689  REQUIRE(i<m_num_layers && i >= 0, "Layer index (%i) out of range\n", i);
690 
691  int32_t n = get_layer(i)->get_num_parameters();
693 
694  memcpy(p->vector, get_section(m_params, i), n*sizeof(float64_t));
695  return p;
696 }
697 
699 {
700  CNeuralLayer* layer = (CNeuralLayer*)m_layers->element(i);
701  // needed because m_layers->element(i) increases the reference count of
702  // layer i
703  SG_UNREF(layer);
704  return layer;
705 }
706 
707 template <class T>
708 SGVector<T> CNeuralNetwork::get_section(SGVector<T> v, int32_t i)
709 {
710  return SGVector<T>(v.vector+m_index_offsets[i],
711  get_layer(i)->get_num_parameters(), false);
712 }
713 
715 {
717 }
718 
720 {
721  SG_REF(m_layers);
722  return m_layers;
723 }
724 
725 void CNeuralNetwork::init()
726 {
728  dropout_hidden = 0.0;
729  dropout_input = 0.0;
730  max_norm = -1.0;
731  l2_coefficient = 0.0;
732  l1_coefficient = 0.0;
733  gd_mini_batch_size = 0;
734  max_num_epochs = 0;
735  gd_learning_rate = 0.1;
737  gd_momentum = 0.9;
738  gd_error_damping_coeff = -1.0;
739  epsilon = 1.0e-5;
740  m_num_inputs = 0;
741  m_num_layers = 0;
742  m_layers = NULL;
744  m_batch_size = 1;
745  m_lbfgs_temp_inputs = NULL;
746  m_lbfgs_temp_targets = NULL;
747  m_is_training = false;
748 
749  SG_ADD((machine_int_t*)&optimization_method, "optimization_method",
750  "Optimization Method", MS_NOT_AVAILABLE);
751  SG_ADD(&gd_mini_batch_size, "gd_mini_batch_size",
752  "Gradient Descent Mini-batch size", MS_NOT_AVAILABLE);
753  SG_ADD(&max_num_epochs, "max_num_epochs",
754  "Max number of Epochs", MS_NOT_AVAILABLE);
755  SG_ADD(&gd_learning_rate, "gd_learning_rate",
756  "Gradient descent learning rate", MS_NOT_AVAILABLE);
757  SG_ADD(&gd_learning_rate_decay, "gd_learning_rate_decay",
758  "Gradient descent learning rate decay", MS_NOT_AVAILABLE);
759  SG_ADD(&gd_momentum, "gd_momentum",
760  "Gradient Descent Momentum", MS_NOT_AVAILABLE);
761  SG_ADD(&gd_error_damping_coeff, "gd_error_damping_coeff",
762  "Gradient Descent Error Damping Coeff", MS_NOT_AVAILABLE);
763  SG_ADD(&epsilon, "epsilon",
764  "Epsilon", MS_NOT_AVAILABLE);
765  SG_ADD(&m_num_inputs, "num_inputs",
766  "Number of Inputs", MS_NOT_AVAILABLE);
767  SG_ADD(&m_num_layers, "num_layers",
768  "Number of Layers", MS_NOT_AVAILABLE);
769  SG_ADD(&m_adj_matrix, "adj_matrix",
770  "Adjacency Matrix", MS_NOT_AVAILABLE);
771  SG_ADD(&l2_coefficient, "l2_coefficient",
772  "L2 regularization coeff", MS_NOT_AVAILABLE);
773  SG_ADD(&l1_coefficient, "l1_coefficient",
774  "L1 regularization coeff", MS_NOT_AVAILABLE);
775  SG_ADD(&dropout_hidden, "dropout_hidden",
776  "Hidden neuron dropout probability", MS_NOT_AVAILABLE);
777  SG_ADD(&dropout_input, "dropout_input",
778  "Input neuron dropout probability", MS_NOT_AVAILABLE);
779  SG_ADD(&max_norm, "max_norm",
780  "Max Norm", MS_NOT_AVAILABLE);
781  SG_ADD(&m_total_num_parameters, "total_num_parameters",
782  "Total number of parameters", MS_NOT_AVAILABLE);
783  SG_ADD(&m_index_offsets, "index_offsets",
784  "Index Offsets", MS_NOT_AVAILABLE);
785  SG_ADD(&m_params, "params",
786  "Parameters", MS_NOT_AVAILABLE);
787  SG_ADD(&m_param_regularizable, "param_regularizable",
788  "Parameter Regularizable", MS_NOT_AVAILABLE);
789  SG_ADD((CSGObject**)&m_layers, "layers",
790  "DynamicObjectArray of NeuralNetwork objects",
792  SG_ADD(&m_is_training, "is_training",
793  "is_training", MS_NOT_AVAILABLE);
794 }

SHOGUN Machine Learning Toolbox - Documentation