From 16b381f1ecb323da7d03c004599fdf0473c4a133 Mon Sep 17 00:00:00 2001 From: Tarun S Paparaju Date: Wed, 13 Feb 2019 23:34:19 +0530 Subject: [PATCH 1/6] Create isrlu.py --- keras_contrib/layers/advanced_activations/isrlu.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 keras_contrib/layers/advanced_activations/isrlu.py diff --git a/keras_contrib/layers/advanced_activations/isrlu.py b/keras_contrib/layers/advanced_activations/isrlu.py new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/keras_contrib/layers/advanced_activations/isrlu.py @@ -0,0 +1 @@ + From b938d0d56b041de6157270404f5b7de0848c3acf Mon Sep 17 00:00:00 2001 From: Tarun S Paparaju Date: Wed, 13 Feb 2019 23:34:48 +0530 Subject: [PATCH 2/6] Add ISRLU --- .../layers/advanced_activations/isrlu.py | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/keras_contrib/layers/advanced_activations/isrlu.py b/keras_contrib/layers/advanced_activations/isrlu.py index 8b1378917..7ead43860 100644 --- a/keras_contrib/layers/advanced_activations/isrlu.py +++ b/keras_contrib/layers/advanced_activations/isrlu.py @@ -1 +1,71 @@ +# -*- coding: utf-8 -*- +from keras import backend as K +from keras.layers import Layer + +class ISRLU(Layer): + """Inverse Square Root Linear Unit + See: https://arxiv.org/pdf/1710.09967.pdf by AI Perf + Reference: https://en.wikipedia.org/wiki/Activation_function + Inverse Square Root Linear activation f(α, x): + x >= 0: x + x < 0: x / sqrt(1 + α * x^2) + # Input shape + Arbitrary. Use the keyword argument `input_shape` + (tuple of integers, does not include the samples axis) + when using this layer as the first layer in a model. + # Output shape + Same shape as the input. + # Arguments + alpha: Value of the alpha weights (float) + NOTE : This function can become unstable for + negative values of α (it may return + NaNs). In particular, this happens when + α < 0 and x < -1/sqrt(α) or x > 1/sqrt(α). + If this happens, try limiting the magnitude + of α below a certain threshold, such that + 1 + α * x^2 is always positive. + Alternatively, you can normalize the inputs + into fixed ranges before passing them to ISRLU. + Adjust the value of α based on your specific + dataset and use-case. + # Example + model = Sequential() + model.add(Dense(5, input_shape=(15,)) + model.add(ISRLU(alpha=-0.3)) + """ + def __init__(self, + alpha=0.1, + **kwargs): + + super(ISRLU, self).__init__(**kwargs) + self.alpha = alpha + + def alpha_initializer(self, input_shape): + return self.alpha * K.ones(input_shape) + + def build(self, input_shape): + new_input_shape = input_shape[1:] + self.alphas = self.add_weight(shape=new_input_shape, + name='{}_alphas'.format(self.name), + initializer=self.alpha_initializer, + trainable=False) + self.build = True + + def call(self, x): + def inverse_square(x): + return x / K.sqrt(1 + self.alphas * K.square(x)) + + def identity(x): + return x + + return K.switch(K.less(x, K.zeros_like(x)), inverse_square(x), identity(x)) + + def compute_output_shape(self, input_shape): + return input_shape + + def get_config(self): + config = {'alpha': self.alpha} + base_config = super(ISRLU, self).get_config() + base_config['trainable'] = False + return dict(list(base_config.items()) + list(config.items())) From 2e5026a8577fa4a50acc9f6ea9f0632dcd8fa966 Mon Sep 17 00:00:00 2001 From: Tarun S Paparaju Date: Wed, 13 Feb 2019 23:35:22 +0530 Subject: [PATCH 3/6] Add import --- keras_contrib/layers/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/keras_contrib/layers/__init__.py b/keras_contrib/layers/__init__.py index bc460c6e6..3968c09b7 100644 --- a/keras_contrib/layers/__init__.py +++ b/keras_contrib/layers/__init__.py @@ -1,5 +1,6 @@ from __future__ import absolute_import +from .advanced_activations.isrlu import ISRLU from .advanced_activations.pelu import PELU from .advanced_activations.srelu import SReLU from .advanced_activations.swish import Swish From e3268916133534d6124e786606bbdf7d5264a3b1 Mon Sep 17 00:00:00 2001 From: Tarun S Paparaju Date: Wed, 13 Feb 2019 23:35:54 +0530 Subject: [PATCH 4/6] Create test_isrlu.py --- tests/keras_contrib/layers/advanced_activations/test_isrlu.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 tests/keras_contrib/layers/advanced_activations/test_isrlu.py diff --git a/tests/keras_contrib/layers/advanced_activations/test_isrlu.py b/tests/keras_contrib/layers/advanced_activations/test_isrlu.py new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/tests/keras_contrib/layers/advanced_activations/test_isrlu.py @@ -0,0 +1 @@ + From ece43b32f8d75d390d6b1cb8dd989c7735c75eec Mon Sep 17 00:00:00 2001 From: Tarun S Paparaju Date: Wed, 13 Feb 2019 23:36:16 +0530 Subject: [PATCH 5/6] Add test --- .../layers/advanced_activations/test_isrlu.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/keras_contrib/layers/advanced_activations/test_isrlu.py b/tests/keras_contrib/layers/advanced_activations/test_isrlu.py index 8b1378917..ffbad40bb 100644 --- a/tests/keras_contrib/layers/advanced_activations/test_isrlu.py +++ b/tests/keras_contrib/layers/advanced_activations/test_isrlu.py @@ -1 +1,15 @@ +# -*- coding: utf-8 -*- +import pytest +from keras_contrib.utils.test_utils import layer_test +from keras_contrib.layers import ISRLU + +@pytest.mark.parametrize('alpha', [0.2, 0.3, -0.01]) +def test_isrlu(alpha): + layer_test(ISRLU, + kwargs={'alpha': alpha}, + input_shape=(2, 3, 4)) + + +if __name__ == '__main__': + pytest.main([__file__]) From 2f170221f2136164cb2cbf19412e80fc8ef1ede4 Mon Sep 17 00:00:00 2001 From: Tarun S Paparaju Date: Wed, 13 Feb 2019 23:39:27 +0530 Subject: [PATCH 6/6] Update CODEOWNERS --- CODEOWNERS | 1 + 1 file changed, 1 insertion(+) diff --git a/CODEOWNERS b/CODEOWNERS index afa98df95..cd1b1b2c9 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -26,6 +26,7 @@ keras_contrib/callbacks/snapshot.py @titu1994 # layers +keras_contrib/layers/advanced_activations/isrlu.py @SriRangaTarun keras_contrib/layers/advanced_activations/sinerelu.py @wilderrodrigues keras_contrib/layers/advanced_activations/swish.py @gabrieldemarmiesse keras_contrib/layers/convolutional/subpixelupscaling.py @titu1994