Adapted layer_activations to be string or list

Author: RandomDefaultUser

examples/advanced/ex01_checkpoint_training.py

@@ -22,7 +22,7 @@ def initial_setup():
     parameters.data.data_splitting_type = "by_snapshot"
     parameters.data.input_rescaling_type = "feature-wise-standard"
     parameters.data.output_rescaling_type = "minmax"
-    parameters.network.layer_activations = ["ReLU"]
+    parameters.network.layer_activations = "ReLU"
     parameters.running.max_number_epochs = 9
     parameters.running.mini_batch_size = 8
     parameters.running.learning_rate = 0.00001

examples/advanced/ex02_shuffle_data.py

@@ -23,7 +23,7 @@
 parameters.verbosity = 1
 parameters.data.input_rescaling_type = "feature-wise-standard"
 parameters.data.output_rescaling_type = "minmax"
-parameters.network.layer_activations = ["ReLU"]
+parameters.network.layer_activations = "ReLU"
 
 # No real training, just showing how shuffling directly before training works.
 parameters.running.max_number_epochs = 5

examples/advanced/ex03_tensor_board.py

@@ -17,7 +17,7 @@
 parameters.targets.ldos_gridsize = 11
 parameters.targets.ldos_gridspacing_ev = 2.5
 parameters.targets.ldos_gridoffset_ev = -5
-parameters.network.layer_activations = ["ReLU"]
+parameters.network.layer_activations = "ReLU"
 parameters.running.max_number_epochs = 100
 parameters.running.mini_batch_size = 40
 parameters.running.learning_rate = 0.001

examples/basic/ex01_train_network.py

@@ -22,7 +22,7 @@
 parameters.data.input_rescaling_type = "feature-wise-standard"
 parameters.data.output_rescaling_type = "minmax"
 # Specify the used activation function.
-parameters.network.layer_activations = ["ReLU"]
+parameters.network.layer_activations = "ReLU"
 # Specify the training parameters.
 # These may be determined via hyperparameter tuning.
 parameters.running.max_number_epochs = 100

mala/common/parameters.py

@@ -344,16 +344,17 @@ class ParametersNetwork(ParametersBase):
         network. Please note that the input layer is included therein.
         Default: [10,10,0]
 
-    layer_activations : list
+    layer_activations : list or str
         A list of strings detailing the activation functions to be used
-        by the neural network. If the dimension of layer_activations is
-        smaller than the dimension of layer_sizes-1, than the first entry
-        is used for all layers.
+        by the neural network. If a single string is supplied, then this
+        activation function is used for all layers (including the output layer,
+        i.e., an output activation is used!). Otherwise, the activation
+        functions are added layer by layer.
         Currently supported activation functions are:
 
-            - Sigmoid (default)
+            - Sigmoid
             - ReLU
-            - LeakyReLU
+            - LeakyReLU (default)
 
     loss_function_type : string
         Loss function for the neural network
@@ -388,7 +389,7 @@ def __init__(self):
         super(ParametersNetwork, self).__init__()
         self.nn_type = "feed-forward"
         self.layer_sizes = [10, 10, 10]
-        self.layer_activations = ["Sigmoid"]
+        self.layer_activations = "LeakyReLU"
         self.loss_function_type = "mse"
 
         # for LSTM/Gru

mala/network/network.py

@@ -231,10 +231,8 @@ def __init__(self, params):
         # We should NOT modify the list itself. This would break the
         # hyperparameter algorithms.
         use_only_one_activation_type = False
-        if len(self.params.layer_activations) == 1:
+        if type(self.params.layer_activations) == str:
             use_only_one_activation_type = True
-        elif len(self.params.layer_activations) < self.number_of_layers:
-            raise Exception("Not enough activation layers provided.")
         elif len(self.params.layer_activations) > self.number_of_layers:
             printout(
                 "Too many activation layers provided. The last",
@@ -246,7 +244,7 @@ def __init__(self, params):
             )
 
         # Add the layers.
-        # As this is a feedforward layer we always add linear layers, and then
+        # As this is a feedforward NN we always add linear layers, and then
         # an activation function
         for i in range(0, self.number_of_layers):
             self.layers.append(
@@ -261,7 +259,7 @@ def __init__(self, params):
                 if use_only_one_activation_type:
                     self.layers.append(
                         self._activation_mappings[
-                            self.params.layer_activations[0]
+                            self.params.layer_activations
                         ]()
                     )
                 else:
@@ -272,6 +270,9 @@ def __init__(self, params):
                     )
             except KeyError:
                 raise Exception("Invalid activation type seleceted.")
+            except IndexError:
+                # Layer without activation
+                pass
 
         # Once everything is done, we can move the Network on the target
         # device.

test/all_lazy_loading_test.py

@@ -34,7 +34,7 @@ def test_scaling(self):
         test_parameters.data.data_splitting_type = "by_snapshot"
         test_parameters.descriptors.bispectrum_twojmax = 11
         test_parameters.targets.ldos_gridsize = 10
-        test_parameters.network.layer_activations = ["LeakyReLU"]
+        test_parameters.network.layer_activations = "LeakyReLU"
         test_parameters.running.max_number_epochs = 3
         test_parameters.running.mini_batch_size = 512
         test_parameters.running.learning_rate = 0.00001
@@ -256,7 +256,7 @@ def _train_lazy_loading(prefetching):
         test_parameters.data.data_splitting_type = "by_snapshot"
         test_parameters.data.input_rescaling_type = "feature-wise-standard"
         test_parameters.data.output_rescaling_type = "minmax"
-        test_parameters.network.layer_activations = ["ReLU"]
+        test_parameters.network.layer_activations = "ReLU"
         test_parameters.manual_seed = 1234
         test_parameters.running.max_number_epochs = 100
         test_parameters.running.mini_batch_size = 40

test/basic_gpu_test.py

@@ -85,7 +85,7 @@ def __run(use_gpu):
         test_parameters.data.output_rescaling_type = "minmax"
 
         # Specify the used activation function.
-        test_parameters.network.layer_activations = ["ReLU"]
+        test_parameters.network.layer_activations = "ReLU"
 
         # Specify the training parameters.
         test_parameters.running.max_number_epochs = 100

test/checkpoint_training_test.py

@@ -140,7 +140,7 @@ def __original_setup(
         test_parameters.data.output_rescaling_type = "minmax"
 
         # Specify the used activation function.
-        test_parameters.network.layer_activations = ["ReLU"]
+        test_parameters.network.layer_activations = "ReLU"
 
         # Specify the training parameters.
         test_parameters.running.max_number_epochs = maxepochs

test/complete_interfaces_test.py

@@ -185,7 +185,7 @@ def test_ase_calculator(self):
         test_parameters.data.data_splitting_type = "by_snapshot"
         test_parameters.data.input_rescaling_type = "feature-wise-standard"
         test_parameters.data.output_rescaling_type = "minmax"
-        test_parameters.network.layer_activations = ["ReLU"]
+        test_parameters.network.layer_activations = "ReLU"
         test_parameters.running.max_number_epochs = 100
         test_parameters.running.mini_batch_size = 40
         test_parameters.running.learning_rate = 0.00001