diff --git a/contrib_docs/structure.py b/contrib_docs/structure.py index 13dc4a634..b767ae481 100644 --- a/contrib_docs/structure.py +++ b/contrib_docs/structure.py @@ -1,93 +1,93 @@ -from keras_contrib import layers -from keras_contrib.layers import advanced_activations -from keras_contrib import initializers -from keras_contrib import optimizers -from keras_contrib import callbacks -from keras_contrib import losses -from keras_contrib import backend -from keras_contrib import constraints - - -EXCLUDE = { - 'Optimizer', - 'TFOptimizer', - 'Wrapper', - 'get_session', - 'set_session', - 'CallbackList', - 'serialize', - 'deserialize', - 'get', - 'set_image_dim_ordering', - 'normalize_data_format', - 'image_dim_ordering', - 'get_variable_shape', - 'Constraint' -} - - -# For each class to document, it is possible to: -# 1) Document only the class: [classA, classB, ...] -# 2) Document all its methods: [classA, (classB, "*")] -# 3) Choose which methods to document (methods listed as strings): -# [classA, (classB, ["method1", "method2", ...]), ...] -# 4) Choose which methods to document (methods listed as qualified names): -# [classA, (classB, [module.classB.method1, module.classB.method2, ...]), ...] -PAGES = [ - { - 'page': 'layers/core.md', - 'classes': [ - layers.CosineDense, - ], - }, - { - 'page': 'layers/convolutional.md', - 'classes': [ - layers.CosineConv2D, - layers.SubPixelUpscaling, - ], - }, - { - 'page': 'layers/normalization.md', - 'classes': [ - layers.InstanceNormalization, - layers.GroupNormalization - ], - }, - { - 'page': 'layers/advanced-activations.md', - 'all_module_classes': [advanced_activations], - }, - { - 'page': 'layers/crf.md', - 'classes': [ - layers.CRF, - ] - }, - { - 'page': 'losses.md', - 'all_module_functions': [losses], - }, - { - 'page': 'initializers.md', - 'all_module_classes': [initializers], - }, - { - 'page': 'optimizers.md', - 'all_module_classes': [optimizers], - }, - { - 'page': 'callbacks.md', - 'all_module_classes': [callbacks], - }, - { - 'page': 'backend.md', - 'all_module_functions': [backend], - }, - { - 'page': 'constraints.md', - 'all_module_classes': [constraints], - }, -] - -ROOT = 'http://keras.io/' +from keras_contrib import layers +from keras_contrib.layers import advanced_activations +from keras_contrib import initializers +from keras_contrib import optimizers +from keras_contrib import callbacks +from keras_contrib import losses +from keras_contrib import backend +from keras_contrib import constraints + + +EXCLUDE = { + 'Optimizer', + 'TFOptimizer', + 'Wrapper', + 'get_session', + 'set_session', + 'CallbackList', + 'serialize', + 'deserialize', + 'get', + 'set_image_dim_ordering', + 'normalize_data_format', + 'image_dim_ordering', + 'get_variable_shape', + 'Constraint' +} + + +# For each class to document, it is possible to: +# 1) Document only the class: [classA, classB, ...] +# 2) Document all its methods: [classA, (classB, "*")] +# 3) Choose which methods to document (methods listed as strings): +# [classA, (classB, ["method1", "method2", ...]), ...] +# 4) Choose which methods to document (methods listed as qualified names): +# [classA, (classB, [module.classB.method1, module.classB.method2, ...]), ...] +PAGES = [ + { + 'page': 'layers/core.md', + 'classes': [ + layers.CosineDense, + ], + }, + { + 'page': 'layers/convolutional.md', + 'classes': [ + layers.CosineConv2D, + layers.SubPixelUpscaling, + ], + }, + { + 'page': 'layers/normalization.md', + 'classes': [ + layers.InstanceNormalization, + layers.GroupNormalization + ], + }, + { + 'page': 'layers/advanced-activations.md', + 'all_module_classes': [advanced_activations], + }, + { + 'page': 'layers/crf.md', + 'classes': [ + layers.CRF, + ] + }, + { + 'page': 'losses.md', + 'all_module_functions': [losses], + }, + { + 'page': 'initializers.md', + 'all_module_classes': [initializers], + }, + { + 'page': 'optimizers.md', + 'all_module_classes': [optimizers], + }, + { + 'page': 'callbacks.md', + 'all_module_classes': [callbacks], + }, + { + 'page': 'backend.md', + 'all_module_functions': [backend], + }, + { + 'page': 'constraints.md', + 'all_module_classes': [constraints], + }, +] + +ROOT = 'http://keras.io/' diff --git a/keras_contrib/backend/numpy_backend.py b/keras_contrib/backend/numpy_backend.py index ea51ec3ab..3f86fd0bc 100644 --- a/keras_contrib/backend/numpy_backend.py +++ b/keras_contrib/backend/numpy_backend.py @@ -1,18 +1,18 @@ -import numpy as np -from keras import backend as K - - -def extract_image_patches(X, ksizes, strides, - padding='valid', - data_format='channels_first'): - raise NotImplementedError - - -def depth_to_space(input, scale, data_format=None): - raise NotImplementedError - - -def moments(x, axes, shift=None, keep_dims=False): - mean_batch = np.mean(x, axis=tuple(axes), keepdims=keep_dims) - var_batch = np.var(x, axis=tuple(axes), keepdims=keep_dims) - return mean_batch, var_batch +import numpy as np +from keras import backend as K + + +def extract_image_patches(X, ksizes, strides, + padding='valid', + data_format='channels_first'): + raise NotImplementedError + + +def depth_to_space(input, scale, data_format=None): + raise NotImplementedError + + +def moments(x, axes, shift=None, keep_dims=False): + mean_batch = np.mean(x, axis=tuple(axes), keepdims=keep_dims) + var_batch = np.var(x, axis=tuple(axes), keepdims=keep_dims) + return mean_batch, var_batch diff --git a/keras_contrib/layers/advanced_activations.py b/keras_contrib/layers/advanced_activations.py index 58f62f67b..a699c02a2 100644 --- a/keras_contrib/layers/advanced_activations.py +++ b/keras_contrib/layers/advanced_activations.py @@ -6,6 +6,115 @@ from keras import backend as K +class PTReLU(Layer): + """Parametric Tan Hyperbolic Linear Unit. + It follows: + `f(x) = x for x > 0`, + `f(x) = alphas * tanh(betas * x) for x <= 0`, + where `alphas` & `betas` are non-negative learned arrays with the same shape as x. + # Input shape + Arbitrary. Use the keyword argument `input_shape` + (tuple of integers, does not include the samples axis) + when using this layer as the first layer in a model. + # Output shape + Same shape as the input. + # Arguments + alphas_initializer: initialization function for the alpha variable weights. + betas_initializer: initialization function for the beta variable weights. + weights: initial weights, as a list of a single Numpy array. + shared_axes: the axes along which to share learnable + parameters for the activation function. + For example, if the incoming feature maps + are from a 2D convolution + with output shape `(batch, height, width, channels)`, + and you wish to share parameters across space + so that each filter only has one set of parameters, + set `shared_axes=[1, 2]`. + # References + - [Parametric Tan Hyperbolic Linear Unit Activation for Deep Neural Networks] + (http://openaccess.thecvf.com/content_ICCV_2017_workshops/papers/w18/Duggal_P-TELU_Parametric_Tan_ICCV_2017_paper.pdf) + """ + + def __init__(self, alpha_initializer='ones', + alpha_regularizer=None, + alpha_constraint=constraints.non_neg(), + beta_initializer='ones', + beta_regularizer=None, + beta_constraint=constraints.non_neg(), + shared_axes=None, + **kwargs): + super(PTReLU, self).__init__(**kwargs) + self.supports_masking = True + self.alpha_initializer = initializers.get(alpha_initializer) + self.alpha_regularizer = regularizers.get(alpha_regularizer) + self.alpha_constraint = constraints.get(alpha_constraint) + self.beta_initializer = initializers.get(beta_initializer) + self.beta_regularizer = regularizers.get(beta_regularizer) + self.beta_constraint = constraints.get(beta_constraint) + if shared_axes is None: + self.shared_axes = None + elif not isinstance(shared_axes, (list, tuple)): + self.shared_axes = [shared_axes] + else: + self.shared_axes = list(shared_axes) + + def build(self, input_shape): + param_shape = list(input_shape[1:]) + self.param_broadcast = [False] * len(param_shape) + if self.shared_axes is not None: + for i in self.shared_axes: + param_shape[i - 1] = 1 + self.param_broadcast[i - 1] = True + + param_shape = tuple(param_shape) + # Initialised as ones to emulate the default TReLU + self.alpha = self.add_weight(param_shape, + name='alpha', + initializer=self.alpha_initializer, + regularizer=self.alpha_regularizer, + constraint=self.alpha_constraint) + self.beta = self.add_weight(param_shape, + name='beta', + initializer=self.beta_initializer, + regularizer=self.beta_regularizer, + constraint=self.beta_constraint) + + # Set input spec + axes = {} + if self.shared_axes: + for i in range(1, len(input_shape)): + if i not in self.shared_axes: + axes[i] = input_shape[i] + self.input_spec = InputSpec(ndim=len(input_shape), axes=axes) + self.built = True + + def call(self, x, mask=None): + pos = K.relu(x) + if K.backend() == 'theano': + neg = (K.pattern_broadcast(self.alpha, self.param_broadcast) * + K.tanh((K.pattern_broadcast(self.beta, self.param_broadcast) * + (x - K.abs(x)) * 0.5))) + else: + neg = self.alpha * K.tanh(self.beta * (-K.relu(-x))) + return neg + pos + + def get_config(self): + config = { + 'alpha_initializer': initializers.serialize(self.alpha_initializer), + 'alpha_regularizer': regularizers.serialize(self.alpha_regularizer), + 'alpha_constraint': constraints.serialize(self.alpha_constraint), + 'beta_initializer': initializers.serialize(self.beta_initializer), + 'beta_regularizer': regularizers.serialize(self.beta_regularizer), + 'beta_constraint': constraints.serialize(self.beta_constraint), + 'shared_axes': self.shared_axes + } + base_config = super(PTReLU, self).get_config() + return dict(list(base_config.items()) + list(config.items())) + + def compute_output_shape(self, input_shape): + return input_shape + + class PELU(Layer): """Parametric Exponential Linear Unit. diff --git a/keras_contrib/utils/conv_utils.py b/keras_contrib/utils/conv_utils.py index a8cb5f9c8..8be9400eb 100644 --- a/keras_contrib/utils/conv_utils.py +++ b/keras_contrib/utils/conv_utils.py @@ -1,66 +1,66 @@ -import keras.backend as K - - -def conv_output_length(input_length, filter_size, - padding, stride, dilation=1): - """Determines output length of a convolution given input length. - - Copy of the function of keras-team/keras because it's not in the public API - So we can't use the function in keras-team/keras to test tf.keras - - # Arguments - input_length: integer. - filter_size: integer. - padding: one of `"same"`, `"valid"`, `"full"`. - stride: integer. - dilation: dilation rate, integer. - - # Returns - The output length (integer). - """ - if input_length is None: - return None - assert padding in {'same', 'valid', 'full', 'causal'} - dilated_filter_size = filter_size + (filter_size - 1) * (dilation - 1) - if padding == 'same': - output_length = input_length - elif padding == 'valid': - output_length = input_length - dilated_filter_size + 1 - elif padding == 'causal': - output_length = input_length - elif padding == 'full': - output_length = input_length + dilated_filter_size - 1 - return (output_length + stride - 1) // stride - - -def normalize_data_format(value): - """Checks that the value correspond to a valid data format. - - Copy of the function in keras-team/keras because it's not public API. - - # Arguments - value: String or None. `'channels_first'` or `'channels_last'`. - - # Returns - A string, either `'channels_first'` or `'channels_last'` - - # Example - ```python - >>> from keras import backend as K - >>> K.normalize_data_format(None) - 'channels_first' - >>> K.normalize_data_format('channels_last') - 'channels_last' - ``` - - # Raises - ValueError: if `value` or the global `data_format` invalid. - """ - if value is None: - value = K.image_data_format() - data_format = value.lower() - if data_format not in {'channels_first', 'channels_last'}: - raise ValueError('The `data_format` argument must be one of ' - '"channels_first", "channels_last". Received: ' + - str(value)) - return data_format +import keras.backend as K + + +def conv_output_length(input_length, filter_size, + padding, stride, dilation=1): + """Determines output length of a convolution given input length. + + Copy of the function of keras-team/keras because it's not in the public API + So we can't use the function in keras-team/keras to test tf.keras + + # Arguments + input_length: integer. + filter_size: integer. + padding: one of `"same"`, `"valid"`, `"full"`. + stride: integer. + dilation: dilation rate, integer. + + # Returns + The output length (integer). + """ + if input_length is None: + return None + assert padding in {'same', 'valid', 'full', 'causal'} + dilated_filter_size = filter_size + (filter_size - 1) * (dilation - 1) + if padding == 'same': + output_length = input_length + elif padding == 'valid': + output_length = input_length - dilated_filter_size + 1 + elif padding == 'causal': + output_length = input_length + elif padding == 'full': + output_length = input_length + dilated_filter_size - 1 + return (output_length + stride - 1) // stride + + +def normalize_data_format(value): + """Checks that the value correspond to a valid data format. + + Copy of the function in keras-team/keras because it's not public API. + + # Arguments + value: String or None. `'channels_first'` or `'channels_last'`. + + # Returns + A string, either `'channels_first'` or `'channels_last'` + + # Example + ```python + >>> from keras import backend as K + >>> K.normalize_data_format(None) + 'channels_first' + >>> K.normalize_data_format('channels_last') + 'channels_last' + ``` + + # Raises + ValueError: if `value` or the global `data_format` invalid. + """ + if value is None: + value = K.image_data_format() + data_format = value.lower() + if data_format not in {'channels_first', 'channels_last'}: + raise ValueError('The `data_format` argument must be one of ' + '"channels_first", "channels_last". Received: ' + + str(value)) + return data_format diff --git a/tests/keras_contrib/layers/test_advanced_activations.py b/tests/keras_contrib/layers/test_advanced_activations.py index 8529e0b85..78c932122 100644 --- a/tests/keras_contrib/layers/test_advanced_activations.py +++ b/tests/keras_contrib/layers/test_advanced_activations.py @@ -3,6 +3,12 @@ from keras_contrib.layers import advanced_activations +@pytest.mark.parametrize('kwargs', [{}, {'shared_axes': 1}]) +def test_ptrelu(kwargs): + layer_test(advanced_activations.PTReLU, kwargs=kwargs, + input_shape=(2, 3, 4)) + + @pytest.mark.parametrize('kwargs', [{}, {'shared_axes': 1}]) def test_pelu(kwargs): layer_test(advanced_activations.PELU, kwargs=kwargs,