From f2d3bd141523a3f3fd0bba97d88f61a5310f6a21 Mon Sep 17 00:00:00 2001 From: Parcollet Titouan Date: Thu, 22 Mar 2018 16:29:15 -0400 Subject: [PATCH] Initial commit with clean code --- DeepComplexNetworks.egg-info/PKG-INFO | 64 + LICENSE.md | 0 README | 0 README.md | 53 + build/lib.linux-x86_64-2.7/complexnn/__init__.py | 28 + build/lib.linux-x86_64-2.7/complexnn/bn.py | 1038 +++++++++++++ build/lib.linux-x86_64-2.7/complexnn/conv.py | 1796 ++++++++++++++++++++++ build/lib.linux-x86_64-2.7/complexnn/dense.py | 230 +++ build/lib.linux-x86_64-2.7/complexnn/fft.py | 172 +++ build/lib.linux-x86_64-2.7/complexnn/init.py | 449 ++++++ build/lib.linux-x86_64-2.7/complexnn/norm.py | 271 ++++ build/lib.linux-x86_64-2.7/complexnn/pool.py | 153 ++ build/lib.linux-x86_64-2.7/complexnn/utils.py | 196 +++ build/lib.linux-x86_64-2.7/musicnet/__init__.py | 0 build/lib.linux-x86_64-2.7/musicnet/callbacks.py | 107 ++ build/lib.linux-x86_64-2.7/musicnet/dataset.py | 217 +++ build/scripts-2.7/resample.py | 51 + build/scripts-2.7/run.py | 251 +++ build/scripts-2.7/train.py | 142 ++ build/scripts-2.7/training.py | 524 +++++++ complexnn/__init__.py | 18 + complexnn/conv.py | 827 ++++++++++ complexnn/dense.py | 241 +++ complexnn/init.py | 105 ++ complexnn/utils.py | 117 ++ datasets/__init__.py | 0 datasets/schemes.py | 53 + datasets/schemes.pyc | Bin 0 -> 2272 bytes datasets/test_timit.ipynb | 83 + datasets/timit.py | 54 + datasets/transformers.py | 350 +++++ datasets/transformers.pyc | Bin 0 -> 14289 bytes datasets/utils.py | 93 ++ scripts/models_timit.py | 191 +++ scripts/run.py | 222 +++ scripts/training.py | 470 ++++++ setup.py | 18 + 37 files changed, 8584 insertions(+) create mode 100644 DeepComplexNetworks.egg-info/PKG-INFO create mode 100644 LICENSE.md delete mode 100644 README create mode 100644 README.md create mode 100644 build/lib.linux-x86_64-2.7/complexnn/__init__.py create mode 100644 build/lib.linux-x86_64-2.7/complexnn/bn.py create mode 100644 build/lib.linux-x86_64-2.7/complexnn/conv.py create mode 100644 build/lib.linux-x86_64-2.7/complexnn/dense.py create mode 100644 build/lib.linux-x86_64-2.7/complexnn/fft.py create mode 100644 build/lib.linux-x86_64-2.7/complexnn/init.py create mode 100644 build/lib.linux-x86_64-2.7/complexnn/norm.py create mode 100644 build/lib.linux-x86_64-2.7/complexnn/pool.py create mode 100644 build/lib.linux-x86_64-2.7/complexnn/utils.py create mode 100644 build/lib.linux-x86_64-2.7/musicnet/__init__.py create mode 100644 build/lib.linux-x86_64-2.7/musicnet/callbacks.py create mode 100644 build/lib.linux-x86_64-2.7/musicnet/dataset.py create mode 100755 build/scripts-2.7/resample.py create mode 100755 build/scripts-2.7/run.py create mode 100755 build/scripts-2.7/train.py create mode 100755 build/scripts-2.7/training.py create mode 100644 complexnn/__init__.py create mode 100644 complexnn/conv.py create mode 100644 complexnn/dense.py create mode 100644 complexnn/init.py create mode 100644 complexnn/utils.py create mode 100644 datasets/__init__.py create mode 100644 datasets/schemes.py create mode 100644 datasets/schemes.pyc create mode 100644 datasets/test_timit.ipynb create mode 100644 datasets/timit.py create mode 100644 datasets/transformers.py create mode 100644 datasets/transformers.pyc create mode 100644 datasets/utils.py create mode 100644 scripts/models_timit.py create mode 100644 scripts/run.py create mode 100644 scripts/training.py create mode 100755 setup.py diff --git a/DeepComplexNetworks.egg-info/PKG-INFO b/DeepComplexNetworks.egg-info/PKG-INFO new file mode 100644 index 0000000..60853b1 --- /dev/null +++ b/DeepComplexNetworks.egg-info/PKG-INFO @@ -0,0 +1,64 @@ +Metadata-Version: 1.0 +Name: DeepComplexNetworks +Version: 1 +Summary: UNKNOWN +Home-page: UNKNOWN +Author: UNKNOWN +Author-email: UNKNOWN +License: MIT +Description-Content-Type: UNKNOWN +Description: Deep Quaternionary Convolutional Neural Networks + ===================== + + This repository contains code which reproduces experiments presented in + the paper ... + + Requirements + ------------ + + Install requirements for experiments with pip: + ``` + pip install numpy tensorflow-gpu keras + ``` + Depending on your Python installation you might want to use anaconda or other tools. + + + Installation + ------------ + + ``` + python setup.py install + ``` + + Experiments + ----------- + + 1. Get help: + + ``` + python scripts/run.py train --help + ``` + + 2. Run models: + + ``` + python scripts/run.py train -w WORKDIR --model {real,complex,quaternion} --seg{chiheb,parcollet} --sf STARTFILTER --nb NUMBEROFBLOCKSPERSTAGE + ``` + + Other arguments may be added as well; Refer to run.py train --help for + + - Optimizer settings + - Dropout rate + - Clipping + - ... + + Citation + -------- + + Please cite our work as + + ``` + + ``` + +Platform: UNKNOWN diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 0000000..e69de29 diff --git a/README b/README deleted file mode 100644 index e69de29..0000000 diff --git a/README.md b/README.md new file mode 100644 index 0000000..5923a8d --- /dev/null +++ b/README.md @@ -0,0 +1,53 @@ +Deep Quaternionary Convolutional Neural Networks +===================== + +This repository contains code which reproduces experiments presented in +the paper ... + +Requirements +------------ + +Install requirements for experiments with pip: +``` +pip install numpy tensorflow-gpu keras +``` +Depending on your Python installation you might want to use anaconda or other tools. + + +Installation +------------ + +``` +python setup.py install +``` + +Experiments +----------- + +1. Get help: + + ``` + python scripts/run.py train --help + ``` + +2. Run models: + + ``` + python scripts/run.py train -w WORKDIR --model {real,complex,quaternion} --seg{chiheb,parcollet} --sf STARTFILTER --nb NUMBEROFBLOCKSPERSTAGE + ``` + + Other arguments may be added as well; Refer to run.py train --help for + + - Optimizer settings + - Dropout rate + - Clipping + - ... + +Citation +-------- + +Please cite our work as + +``` + +``` diff --git a/build/lib.linux-x86_64-2.7/complexnn/__init__.py b/build/lib.linux-x86_64-2.7/complexnn/__init__.py new file mode 100644 index 0000000..d4d14cf --- /dev/null +++ b/build/lib.linux-x86_64-2.7/complexnn/__init__.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Contributors: Titouan Parcollet +# Authors: Olexa Bilaniuk +# +# What this module includes by default: +import bn, conv, dense, fft, init, norm, pool + +from .bn import ComplexBatchNormalization as ComplexBN +from .bn import QuaternionBatchNormalization as QuaternionBN +from .conv import (ComplexConv, + ComplexConv1D, + ComplexConv2D, + ComplexConv3D, + QuaternionConv, + QuaternionConv1D, + QuaternionConv2D, + QuaternionConv3D, + WeightNorm_Conv) +from .dense import ComplexDense +from .fft import fft, ifft, fft2, ifft2, FFT, IFFT, FFT2, IFFT2 +from .init import (ComplexIndependentFilters, IndependentFilters, + ComplexInit, SqrtInit, QuaternionInit, QuaternionIndependentFilters) +from .norm import LayerNormalization, ComplexLayerNorm +from .pool import SpectralPooling1D, SpectralPooling2D +from .utils import (get_realpart, get_imagpart, getpart_output_shape, + GetImag, GetReal, GetAbs, getpart_quaternion_output_shape, get_rpart, get_ipart, get_jpart, get_kpart, GetR, GetI,GetJ,GetK) diff --git a/build/lib.linux-x86_64-2.7/complexnn/bn.py b/build/lib.linux-x86_64-2.7/complexnn/bn.py new file mode 100644 index 0000000..93ed802 --- /dev/null +++ b/build/lib.linux-x86_64-2.7/complexnn/bn.py @@ -0,0 +1,1038 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Contributors: Titouan Parcollet +# Authors: Chiheb Trabelsi, Olexa Bilaniuk +# +# Note: The implementation of complex Batchnorm is based on +# the Keras implementation of batch Normalization +# available here: +# https://github.com/fchollet/keras/blob/master/keras/layers/normalization.py + +import numpy as np +from keras.layers import Layer, InputSpec +from keras import initializers, regularizers, constraints +import keras.backend as K +import tensorflow as tf +import theano as th + +def sqrt_init(shape, dtype=None): + value = (1 / K.sqrt(K.constant(16))) * K.ones(shape) + return value + + +def sanitizedInitGet(init): + if init in ["sqrt_init"]: + return sqrt_init + else: + return initializers.get(init) +def sanitizedInitSer(init): + if init in [sqrt_init]: + return "sqrt_init" + else: + return initializers.serialize(init) + +##################################################################### +# Quaternion Implementations # +##################################################################### + +class QuaternionBatchNormalization(Layer): + """Quaternion version of the real domain + Batch normalization layer (Ioffe and Szegedy, 2014). + Normalize the activations of the previous complex layer at each batch, + i.e. applies a transformation that maintains the mean of a complex unit + close to the null vector, the 2 by 2 covariance matrix of a complex unit close to identity + and the 2 by 2 relation matrix, also called pseudo-covariance, close to the + null matrix. + # Arguments + axis: Integer, the axis that should be normalized + (typically the features axis). + For instance, after a `Conv2D` layer with + `data_format="channels_first"`, + set `axis=2` in `ComplexBatchNormalization`. + momentum: Momentum for the moving statistics related to the real and + imaginary parts. + epsilon: Small float added to each of the variances related to the + real and imaginary parts in order to avoid dividing by zero. + center: If True, add offset of `beta` to complex normalized tensor. + If False, `beta` is ignored. + (beta is formed by real_beta and imag_beta) + scale: If True, multiply by the `gamma` matrix. + If False, `gamma` is not used. + beta_initializer: Initializer for the real_beta and the imag_beta weight. + gamma_diag_initializer: Initializer for the diagonal elements of the gamma matrix. + which are the variances of the real part and the imaginary part. + gamma_off_initializer: Initializer for the off-diagonal elements of the gamma matrix. + moving_mean_initializer: Initializer for the moving means. + moving_variance_initializer: Initializer for the moving variances. + moving_covariance_initializer: Initializer for the moving covariance of + the real and imaginary parts. + beta_regularizer: Optional regularizer for the beta weights. + gamma_regularizer: Optional regularizer for the gamma weights. + beta_constraint: Optional constraint for the beta weights. + gamma_constraint: Optional constraint for the gamma weights. + # Input shape + Arbitrary. Use the keyword argument `input_shape` + (tuple of integers, does not include the samples axis) + when using this layer as the first layer in a model. + # Output shape + Same shape as input. + # References + - [Batch Normalization: Accelerating Deep Network Training by Reducing Internal Covariate Shift](https://arxiv.org/abs/1502.03167) + """ + + def __init__(self, + axis=-1, + momentum=0.9, + epsilon=1e-4, + center=True, + scale=True, + beta_initializer='zeros', + gamma_diag_initializer='sqrt_init', + gamma_off_initializer='zeros', + moving_mean_initializer='zeros', + moving_variance_initializer='sqrt_init', + moving_covariance_initializer='zeros', + beta_regularizer=None, + gamma_diag_regularizer=None, + gamma_off_regularizer=None, + beta_constraint=None, + gamma_diag_constraint=None, + gamma_off_constraint=None, + **kwargs): + super(QuaternionBatchNormalization, self).__init__(**kwargs) + self.supports_masking = True + self.axis = axis + self.momentum = momentum + self.epsilon = epsilon + self.center = center + self.scale = scale + self.beta_initializer = sanitizedInitGet(beta_initializer) + self.gamma_diag_initializer = sanitizedInitGet(gamma_diag_initializer) + self.gamma_off_initializer = sanitizedInitGet(gamma_off_initializer) + self.moving_mean_initializer = sanitizedInitGet(moving_mean_initializer) + self.moving_variance_initializer = sanitizedInitGet(moving_variance_initializer) + self.moving_covariance_initializer = sanitizedInitGet(moving_covariance_initializer) + self.beta_regularizer = regularizers.get(beta_regularizer) + self.gamma_diag_regularizer = regularizers.get(gamma_diag_regularizer) + self.gamma_off_regularizer = regularizers.get(gamma_off_regularizer) + self.beta_constraint = constraints .get(beta_constraint) + self.gamma_diag_constraint = constraints .get(gamma_diag_constraint) + self.gamma_off_constraint = constraints .get(gamma_off_constraint) + + def build(self, input_shape): + + ndim = len(input_shape) + + dim = input_shape[self.axis] + if dim is None: + raise ValueError('Axis ' + str(self.axis) + ' of ' + 'input tensor should have a defined dimension ' + 'but the layer received an input with shape ' + + str(input_shape) + '.') + self.input_spec = InputSpec(ndim=len(input_shape), + axes={self.axis: dim}) + + param_shape = (input_shape[self.axis] // 4,) + + if self.scale: + self.gamma_rr = self.add_weight(shape=param_shape, + name='gamma_rr', + initializer=self.gamma_diag_initializer, + regularizer=self.gamma_diag_regularizer, + constraint=self.gamma_diag_constraint) + self.gamma_ri = self.add_weight(shape=param_shape, + name='gamma_ri', + initializer=self.gamma_diag_initializer, + regularizer=self.gamma_diag_regularizer, + constraint=self.gamma_diag_constraint) + self.gamma_rj = self.add_weight(shape=param_shape, + name='gamma_rj', + initializer=self.gamma_diag_initializer, + regularizer=self.gamma_diag_regularizer, + constraint=self.gamma_diag_constraint) + self.gamma_rk = self.add_weight(shape=param_shape, + name='gamma_rk', + initializer=self.gamma_diag_initializer, + regularizer=self.gamma_diag_regularizer, + constraint=self.gamma_diag_constraint) + self.gamma_ii = self.add_weight(shape=param_shape, + name='gamma_ii', + initializer=self.gamma_off_initializer, + regularizer=self.gamma_off_regularizer, + constraint=self.gamma_off_constraint) + self.gamma_ij = self.add_weight(shape=param_shape, + name='gamma_ij', + initializer=self.gamma_off_initializer, + regularizer=self.gamma_off_regularizer, + constraint=self.gamma_off_constraint) + self.gamma_ik = self.add_weight(shape=param_shape, + name='gamma_ik', + initializer=self.gamma_off_initializer, + regularizer=self.gamma_off_regularizer, + constraint=self.gamma_off_constraint) + self.gamma_jj = self.add_weight(shape=param_shape, + name='gamma_jj', + initializer=self.gamma_off_initializer, + regularizer=self.gamma_off_regularizer, + constraint=self.gamma_off_constraint) + self.gamma_jk = self.add_weight(shape=param_shape, + name='gamma_jk', + initializer=self.gamma_off_initializer, + regularizer=self.gamma_off_regularizer, + constraint=self.gamma_off_constraint) + self.gamma_kk = self.add_weight(shape=param_shape, + name='gamma_kk', + initializer=self.gamma_off_initializer, + regularizer=self.gamma_off_regularizer, + constraint=self.gamma_off_constraint) + self.moving_Vrr = self.add_weight(shape=param_shape, + initializer=self.moving_variance_initializer, + name='moving_Vrr', + trainable=False) + self.moving_Vri = self.add_weight(shape=param_shape, + initializer=self.moving_variance_initializer, + name='moving_Vri', + trainable=False) + self.moving_Vrj = self.add_weight(shape=param_shape, + initializer=self.moving_variance_initializer, + name='moving_Vrj', + trainable=False) + self.moving_Vrk = self.add_weight(shape=param_shape, + initializer=self.moving_variance_initializer, + name='moving_Vrk', + trainable=False) + self.moving_Vii = self.add_weight(shape=param_shape, + initializer=self.moving_covariance_initializer, + name='moving_Vii', + trainable=False) + self.moving_Vij = self.add_weight(shape=param_shape, + initializer=self.moving_covariance_initializer, + name='moving_Vij', + trainable=False) + self.moving_Vik = self.add_weight(shape=param_shape, + initializer=self.moving_covariance_initializer, + name='moving_Vik', + trainable=False) + self.moving_Vjj = self.add_weight(shape=param_shape, + initializer=self.moving_covariance_initializer, + name='moving_Vjj', + trainable=False) + self.moving_Vjk = self.add_weight(shape=param_shape, + initializer=self.moving_covariance_initializer, + name='moving_Vjk', + trainable=False) + self.moving_Vkk = self.add_weight(shape=param_shape, + initializer=self.moving_covariance_initializer, + name='moving_Vkk', + trainable=False) + else: + self.gamma_rr = None + self.gamma_ri = None + self.gamma_rj = None + self.gamma_rk = None + self.gamma_ii = None + self.gamma_ij = None + self.gamma_ik = None + self.gamma_jj = None + self.gamma_jk = None + self.gamma_kk = None + self.moving_Vrr = None + self.moving_Vri = None + self.moving_Vrj = None + self.moving_Vrk = None + self.moving_Vii = None + self.moving_Vij = None + self.moving_Vik = None + self.moving_Vjj = None + self.moving_Vjk = None + self.moving_Vkk = None + + if self.center: + self.beta = self.add_weight(shape=(input_shape[self.axis],), + name='beta', + initializer=self.beta_initializer, + regularizer=self.beta_regularizer, + constraint=self.beta_constraint) + self.moving_mean = self.add_weight(shape=(input_shape[self.axis],), + initializer=self.moving_mean_initializer, + name='moving_mean', + trainable=False) + else: + self.beta = None + self.moving_mean = None + + self.built = True + + def call(self, inputs, training=None): + input_shape = K.int_shape(inputs) + ndim = len(input_shape) + reduction_axes = list(range(ndim)) + del reduction_axes[self.axis] + input_dim = input_shape[self.axis] // 4 + mu = K.mean(inputs, axis=reduction_axes) + broadcast_mu_shape = [1] * len(input_shape) + broadcast_mu_shape[self.axis] = input_shape[self.axis] + broadcast_mu = K.reshape(mu, broadcast_mu_shape) + if self.center: + input_centred = inputs - broadcast_mu + else: + input_centred = inputs + centred_squared = input_centred ** 2 + start_i = input_dim + start_j = input_dim*2 + start_k = input_dim*3 + if (self.axis == 1 and ndim != 3) or ndim == 2: + centred_squared_r = centred_squared[:, :input_dim] + centred_squared_i = centred_squared[:, input_dim:input_dim*2] + centred_squared_j = centred_squared[:, input_dim*2:input_dim*3] + centred_squared_k = centred_squared[:, input_dim*3:] + centred_r = input_centred[:, :input_dim] + centred_i = input_centred[:, input_dim:input_dim*2] + centred_j = input_centred[:, input_dim*2:input_dim*3] + centred_k = input_centred[:, input_dim*3:] + elif ndim == 3: + centred_squared_r = centred_squared[:, :, :input_dim] + centred_squared_i = centred_squared[:, :, input_dim:input_dim*2] + centred_squared_j = centred_squared[:, :, input_dim*2:input_dim*3] + centred_squared_k = centred_squared[:, :, input_dim*3:] + centred_r = input_centred[:, :, :input_dim] + centred_i = input_centred[:, :, input_dim:input_dim*2] + centred_j = input_centred[:, :, input_dim*2:input_dim*3] + centred_k = input_centred[:, :, input_dim*3:] + elif self.axis == -1 and ndim == 4: + centred_squared_r = centred_squared[:, :, :, :input_dim] + centred_squared_i = centred_squared[:, :, :, input_dim:input_dim*2] + centred_squared_j = centred_squared[:, :, :, input_dim*2:input_dim*3] + centred_squared_k = centred_squared[:, :, :, input_dim*3:] + centred_r = input_centred[:, :, :, :input_dim] + centred_i = input_centred[:, :, :, input_dim:input_dim*2] + centred_j = input_centred[:, :, :, input_dim*2:input_dim*3] + centred_k = input_centred[:, :, :, input_dim*3:] + elif self.axis == -1 and ndim == 5: + centred_squared_r = centred_squared[:, :, :, :, :input_dim] + centred_squared_i = centred_squared[:, :, :, :, input_dim:input_dim*2] + centred_squared_j = centred_squared[:, :, :, :, input_dim*2:input_dim*3] + centred_squared_k = centred_squared[:, :, :, :, input_dim*3:] + centred_r = input_centred[:, :, :, :, :input_dim] + centred_i = input_centred[:, :, :, :, input_dim:input_dim*2] + centred_j = input_centred[:, :, :, :, input_dim*2:input_dim*3] + centred_k = input_centred[:, :, :, :, input_dim*3:] + else: + raise ValueError( + 'Incorrect Batchnorm combination of axis and dimensions. axis should be either 1 or -1. ' + 'axis: ' + str(self.axis) + '; ndim: ' + str(ndim) + '.' + ) + if self.scale: + # #Variances: + Vrr = K.mean( + centred_squared_r, + axis=reduction_axes + ) + self.epsilon + + Vii = K.mean( + centred_squared_i, + axis=reduction_axes + ) + self.epsilon + Vjj = K.mean( + centred_squared_j, + axis=reduction_axes + ) + self.epsilon + Vkk = K.mean( + centred_squared_k, + axis=reduction_axes + ) + self.epsilon + + #Co-Variances: + Vri = K.mean( + centred_r * centred_i, + axis=reduction_axes, + ) + Vrj = K.mean( + centred_r * centred_j, + axis=reduction_axes, + ) + Vrk = K.mean( + centred_r * centred_k, + axis=reduction_axes, + ) + Vij = K.mean( + centred_i * centred_j, + axis=reduction_axes, + ) + Vik = K.mean( + centred_i * centred_k, + axis=reduction_axes, + ) + Vjk = K.mean( + centred_j * centred_k, + axis=reduction_axes, + ) + elif self.center: + Vrr = None + Vri = None + Vrj = None + Vrk = None + Vii = None + Vij = None + Vik = None + Vjj = None + Vjk = None + Vkk = None + else: + raise ValueError('Error. Both scale and center in batchnorm are set to False.') + + + input_bn = QuaternionBN( input_centred, Vrr, Vri, Vrj, Vrk, Vii, Vij, Vik, Vjj, Vjk, Vkk, + self.beta, self.gamma_rr, self.gamma_ri, self.gamma_rj, self.gamma_rk, + self.gamma_ii, self.gamma_ij, self.gamma_ik, self.gamma_jj, self.gamma_jk, self.gamma_kk, + self.scale, self.center, axis=self.axis) + if training in {0, False}: + return input_bn + else: + update_list = [] + if self.center: + update_list.append(K.moving_average_update(self.moving_mean, mu, self.momentum)) + if self.scale: + update_list.append(K.moving_average_update(self.moving_Vrr, Vrr, self.momentum)) + update_list.append(K.moving_average_update(self.moving_Vri, Vri, self.momentum)) + update_list.append(K.moving_average_update(self.moving_Vrk, Vrk, self.momentum)) + update_list.append(K.moving_average_update(self.moving_Vrj, Vrj, self.momentum)) + update_list.append(K.moving_average_update(self.moving_Vii, Vii, self.momentum)) + update_list.append(K.moving_average_update(self.moving_Vij, Vij, self.momentum)) + update_list.append(K.moving_average_update(self.moving_Vik, Vik, self.momentum)) + update_list.append(K.moving_average_update(self.moving_Vjj, Vjj, self.momentum)) + update_list.append(K.moving_average_update(self.moving_Vjk, Vjk, self.momentum)) + update_list.append(K.moving_average_update(self.moving_Vkk, Vkk, self.momentum)) + self.add_update(update_list, inputs) + + def normalize_inference(): + if self.center: + inference_centred = inputs - K.reshape(self.moving_mean, broadcast_mu_shape) + else: + inference_centred = inputs + return QuaternionBN( + inference_centred, self.moving_Vrr, self.moving_Vri, self.moving_Vrj, self.moving_Vrk, + self.moving_Vii, self.moving_Vij,self.moving_Vik, self.moving_Vjj, self.moving_Vjk, self.moving_Vkk, + self.beta, self.gamma_rr, self.gamma_ri, self.gamma_rj, self.gamma_rk, self.gamma_ii, self.gamma_ij, + self.gamma_ik, self.gamma_jj, self.gamma_jk, self.gamma_kk, self.scale, self.center, axis=self.axis) + + ## Pick the normalized form corresponding to the training phase. + return K.in_train_phase(input_bn,normalize_inference,training=training) + + def get_config(self): + config = { + 'axis': self.axis, + 'momentum': self.momentum, + 'epsilon': self.epsilon, + 'center': self.center, + 'scale': self.scale, + 'beta_initializer': sanitizedInitSer(self.beta_initializer), + 'gamma_diag_initializer': sanitizedInitSer(self.gamma_diag_initializer), + 'gamma_off_initializer': sanitizedInitSer(self.gamma_off_initializer), + 'moving_mean_initializer': sanitizedInitSer(self.moving_mean_initializer), + 'moving_variance_initializer': sanitizedInitSer(self.moving_variance_initializer), + 'moving_covariance_initializer': sanitizedInitSer(self.moving_covariance_initializer), + 'beta_regularizer': regularizers.serialize(self.beta_regularizer), + 'gamma_diag_regularizer': regularizers.serialize(self.gamma_diag_regularizer), + 'gamma_off_regularizer': regularizers.serialize(self.gamma_off_regularizer), + 'beta_constraint': constraints .serialize(self.beta_constraint), + 'gamma_diag_constraint': constraints .serialize(self.gamma_diag_constraint), + 'gamma_off_constraint': constraints .serialize(self.gamma_off_constraint), + } + base_config = super(QuaternionBatchNormalization, self).get_config() + return dict(list(base_config.items()) + list(config.items())) + + +def quaternion_standardization(input_centred, Vrr, Vri, Vrj, Vrk, Vii, Vij, Vik, Vjj, Vjk, Vkk, + layernorm=False, axis=-1): + + ndim = K.ndim(input_centred) + input_dim = K.shape(input_centred)[axis] // 4 + variances_broadcast = [1] * ndim + variances_broadcast[axis] = input_dim + if layernorm: + variances_broadcast[0] = K.shape(input_centred)[0] + + # We require the covariance matrix's inverse square root. That first requires + # square rooting, followed by inversion (I do this in that order because during + # the computation of square root we compute the determinant we'll need for + # inversion as well). + + row1 = tf.stack([Vrr, Vri, Vrj, Vrk],1) + row2 = tf.stack([Vri, Vii, Vij, Vik],1) + row3 = tf.stack([Vrj, Vij, Vjj, Vjk],1) + row4 = tf.stack([Vrk, Vik, Vjk, Vkk],1) + covMat = tf.stack([row1,row2,row3,row4], 2) + + invMat = tf.matrix_inverse(covMat, adjoint=False, name="Inverse_matrix") + W = tf.linalg.cholesky(invMat, "Square_root_matrix") + W = K.reshape(W, [1,1,1,tf.size(W)]) + + #print(W.shape) + + cat_W_4_r, cat_W_4_i, cat_W_4_j, cat_W_4_k = tf.split(W, num_or_size_splits=4, axis=3) + + if (axis == 1 and ndim != 3) or ndim == 2: + centred_r = input_centred[:, :input_dim] + centred_i = input_centred[:, input_dim:input_dim*2] + centred_j = input_centred[:, input_dim*2:input_dim*3] + centred_k = input_centred[:, input_dim*3:] + elif ndim == 3: + centred_r = input_centred[:, :, :input_dim] + centred_i = input_centred[:, :, input_dim:input_dim*2] + centred_j = input_centred[:, :, input_dim*2:input_dim*3] + centred_k = input_centred[:, :, input_dim*3:] + elif axis == -1 and ndim == 4: + centred_r = input_centred[:, :, :, :input_dim] + centred_i = input_centred[:, :, :, input_dim:input_dim*2] + centred_j = input_centred[:, :, :, input_dim*2:input_dim*3] + centred_k = input_centred[:, :, :, input_dim*3:] + elif axis == -1 and ndim == 5: + centred_r = input_centred[:, :, :, :, :input_dim] + centred_i = input_centred[:, :, :, :, input_dim:input_dim*2] + centred_j = input_centred[:, :, :, :, input_dim*2:input_dim*3] + centred_k = input_centred[:, :, :, :, input_dim*3:] + else: + raise ValueError( + 'Incorrect Batchnorm combination of axis and dimensions. axis should be either 1 or -1. ' + 'axis: ' + str(self.axis) + '; ndim: ' + str(ndim) + '.' + ) + + input1 = K.concatenate([centred_r, centred_r, centred_r, centred_r], axis=axis) + input2 = K.concatenate([centred_i, centred_i, centred_i, centred_i], axis=axis) + input3 = K.concatenate([centred_j, centred_j, centred_j, centred_j], axis=axis) + input4 = K.concatenate([centred_k, centred_k, centred_k, centred_k], axis=axis) + output = cat_W_4_r * input1 + cat_W_4_i * input2 + cat_W_4_j * input3 + cat_W_4_k * input4 + + # Wrr * x_real_centered | Wii * x_imag_centered + # + Wri * x_imag_centered | Wri * x_real_centered + # ----------------------------------------------- + # = output + return output + + + +def QuaternionBN(input_centred, Vrr, Vri, Vrj, Vrk, Vii, Vij, Vik, Vjj, Vjk, Vkk, beta, + gamma_rr, gamma_ri, gamma_rj, gamma_rk, gamma_ii, gamma_ij, gamma_ik, gamma_jj, gamma_jk, gamma_kk, + scale=True, center=True, layernorm=False, axis=-1): + + ndim = K.ndim(input_centred) + input_dim = K.shape(input_centred)[axis] // 4 + if scale: + gamma_broadcast_shape = [1] * ndim + gamma_broadcast_shape[axis] = input_dim + if center: + broadcast_beta_shape = [1] * ndim + broadcast_beta_shape[axis] = input_dim * 4 + if scale: + standardized_output = quaternion_standardization( + input_centred, Vrr, Vri, Vrj, Vrk, Vii, Vij, Vik, Vjj, Vjk, Vkk, + layernorm, + axis=axis + ) + + # Now we perform th scaling and Shifting of the normalized x using + # the scaling parameter + # [ gamma_rr gamma_ri ] + # Gamma = [ gamma_ri gamma_ii ] + # and the shifting parameter + # Beta = [beta_real beta_imag].T + # where: + # x_real_BN = gamma_rr * x_real_normed + gamma_ri * x_imag_normed + beta_real + # x_imag_BN = gamma_ri * x_real_normed + gamma_ii * x_imag_normed + beta_imag + + broadcast_gamma_rr = K.reshape(gamma_rr, gamma_broadcast_shape) + broadcast_gamma_ri = K.reshape(gamma_ri, gamma_broadcast_shape) + broadcast_gamma_rj = K.reshape(gamma_rj, gamma_broadcast_shape) + broadcast_gamma_rk = K.reshape(gamma_rk, gamma_broadcast_shape) + broadcast_gamma_ii = K.reshape(gamma_ii, gamma_broadcast_shape) + broadcast_gamma_ij = K.reshape(gamma_ij, gamma_broadcast_shape) + broadcast_gamma_ik = K.reshape(gamma_ik, gamma_broadcast_shape) + broadcast_gamma_jj = K.reshape(gamma_jj, gamma_broadcast_shape) + broadcast_gamma_jk = K.reshape(gamma_jk, gamma_broadcast_shape) + broadcast_gamma_kk = K.reshape(gamma_kk, gamma_broadcast_shape) + + cat_gamma_4_r = K.concatenate([broadcast_gamma_rr, broadcast_gamma_ri, broadcast_gamma_rj, broadcast_gamma_rk], axis=axis) + cat_gamma_4_i = K.concatenate([broadcast_gamma_ri, broadcast_gamma_ii, broadcast_gamma_ij, broadcast_gamma_ik], axis=axis) + cat_gamma_4_j = K.concatenate([broadcast_gamma_rj, broadcast_gamma_ij, broadcast_gamma_jj, broadcast_gamma_jk], axis=axis) + cat_gamma_4_k = K.concatenate([broadcast_gamma_rk, broadcast_gamma_ik, broadcast_gamma_jk, broadcast_gamma_kk], axis=axis) + if (axis == 1 and ndim != 3) or ndim == 2: + centred_r = standardized_output[:, :input_dim] + centred_i = standardized_output[:, input_dim:input_dim*2] + centred_j = standardized_output[:, input_dim*2:input_dim*3] + centred_k = standardized_output[:, input_dim*3:] + elif ndim == 3: + centred_r = standardized_output[:, :, :input_dim] + centred_i = standardized_output[:, :, input_dim:input_dim*2] + centred_j = standardized_output[:, :, input_dim*2:input_dim*3] + centred_k = standardized_output[:, :, input_dim*3:] + elif axis == -1 and ndim == 4: + centred_r = standardized_output[:, :, :, :input_dim] + centred_i = standardized_output[:, :, :, input_dim:input_dim*2] + centred_j = standardized_output[:, :, :, input_dim*2:input_dim*3] + centred_k = standardized_output[:, :, :, input_dim*3:] + elif axis == -1 and ndim == 5: + centred_r = standardized_output[:, :, :, :, :input_dim] + centred_i = standardized_output[:, :, :, :, input_dim:input_dim*2] + centred_j = standardized_output[:, :, :, :, input_dim*2:input_dim*3] + centred_k = standardized_output[:, :, :, :, input_dim*3:] + else: + raise ValueError( + 'Incorrect Batchnorm combination of axis and dimensions. axis should be either 1 or -1. ' + 'axis: ' + str(self.axis) + '; ndim: ' + str(ndim) + '.' + ) + + input1 = K.concatenate([centred_r, centred_r, centred_r, centred_r], axis=axis) + input2 = K.concatenate([centred_i, centred_i, centred_i, centred_i], axis=axis) + input3 = K.concatenate([centred_j, centred_j, centred_j, centred_j], axis=axis) + input4 = K.concatenate([centred_k, centred_k, centred_k, centred_k], axis=axis) + + if center: + broadcast_beta = K.reshape(beta, broadcast_beta_shape) + return cat_gamma_4_r * input1 + cat_gamma_4_i * input2 + cat_gamma_4_j * input3 + cat_gamma_4_k * input4 + broadcast_beta + else: + return cat_gamma_4_r * input1 + cat_gamma_4_i * input2 + cat_gamma_4_j * input3 + cat_gamma_4_k * input4 + else: + if center: + broadcast_beta = K.reshape(beta, broadcast_beta_shape) + return input_centred + broadcast_beta + else: + return input_centred + + + + +##################################################################### +# Complex Implementations # +##################################################################### + + +def complex_standardization(input_centred, Vrr, Vii, Vri, + layernorm=False, axis=-1): + + ndim = K.ndim(input_centred) + input_dim = K.shape(input_centred)[axis] // 2 + variances_broadcast = [1] * ndim + variances_broadcast[axis] = input_dim + if layernorm: + variances_broadcast[0] = K.shape(input_centred)[0] + + # We require the covariance matrix's inverse square root. That first requires + # square rooting, followed by inversion (I do this in that order because during + # the computation of square root we compute the determinant we'll need for + # inversion as well). + + # tau = Vrr + Vii = Trace. Guaranteed >= 0 because SPD + tau = Vrr + Vii + # delta = (Vrr * Vii) - (Vri ** 2) = Determinant. Guaranteed >= 0 because SPD + delta = (Vrr * Vii) - (Vri ** 2) + + s = K.sqrt(delta) # Determinant of square root matrix + t = K.sqrt(tau + 2 * s) + + #test = np.array([[Vrr, Vri],[Vri, Vii]]) + #testulu = np.linalg.cholesky(test) + # The square root matrix could now be explicitly formed as + # [ Vrr+s Vri ] + # (1/t) [ Vir Vii+s ] + # https://en.wikipedia.org/wiki/Square_root_of_a_2_by_2_matrix + # but we don't need to do this immediately since we can also simultaneously + # invert. We can do this because we've already computed the determinant of + # the square root matrix, and can thus invert it using the analytical + # solution for 2x2 matrices + # [ A B ] [ D -B ] + # inv( [ C D ] ) = (1/det) [ -C A ] + # http://mathworld.wolfram.com/MatrixInverse.html + # Thus giving us + # [ Vii+s -Vri ] + # (1/s)(1/t)[ -Vir Vrr+s ] + # So we proceed as follows: + + inverse_st = 1.0 / (s * t) + Wrr = (Vii + s) * inverse_st + Wii = (Vrr + s) * inverse_st + Wri = -Vri * inverse_st + + # And we have computed the inverse square root matrix W = sqrt(V)! + # Normalization. We multiply, x_normalized = W.x. + + # The returned result will be a complex standardized input + # where the real and imaginary parts are obtained as follows: + # x_real_normed = Wrr * x_real_centred + Wri * x_imag_centred + # x_imag_normed = Wri * x_real_centred + Wii * x_imag_centred + + broadcast_Wrr = K.reshape(Wrr, variances_broadcast) + broadcast_Wri = K.reshape(Wri, variances_broadcast) + broadcast_Wii = K.reshape(Wii, variances_broadcast) + + cat_W_4_real = K.concatenate([broadcast_Wrr, broadcast_Wii], axis=axis) + cat_W_4_imag = K.concatenate([broadcast_Wri, broadcast_Wri], axis=axis) + if (axis == 1 and ndim != 3) or ndim == 2: + centred_real = input_centred[:, :input_dim] + centred_imag = input_centred[:, input_dim:] + elif ndim == 3: + centred_real = input_centred[:, :, :input_dim] + centred_imag = input_centred[:, :, input_dim:] + elif axis == -1 and ndim == 4: + centred_real = input_centred[:, :, :, :input_dim] + centred_imag = input_centred[:, :, :, input_dim:] + elif axis == -1 and ndim == 5: + centred_real = input_centred[:, :, :, :, :input_dim] + centred_imag = input_centred[:, :, :, :, input_dim:] + else: + raise ValueError( + 'Incorrect Batchnorm combination of axis and dimensions. axis should be either 1 or -1. ' + 'axis: ' + str(self.axis) + '; ndim: ' + str(ndim) + '.' + ) + rolled_input = K.concatenate([centred_imag, centred_real], axis=axis) + + output = cat_W_4_real * input_centred + cat_W_4_imag * rolled_input + + # Wrr * x_real_centered | Wii * x_imag_centered + # + Wri * x_imag_centered | Wri * x_real_centered + # ----------------------------------------------- + # = output + return output + + +def ComplexBN(input_centred, Vrr, Vii, Vri, beta, + gamma_rr, gamma_ri, gamma_ii, scale=True, + center=True, layernorm=False, axis=-1): + ndim = K.ndim(input_centred) + input_dim = K.shape(input_centred)[axis] // 2 + + if scale: + gamma_broadcast_shape = [1] * ndim + gamma_broadcast_shape[axis] = input_dim + if center: + broadcast_beta_shape = [1] * ndim + broadcast_beta_shape[axis] = input_dim * 2 + + if scale: + standardized_output = complex_standardization( + input_centred, Vrr, Vii, Vri, + layernorm, + axis=axis + ) + + # Now we perform th scaling and Shifting of the normalized x using + # the scaling parameter + # [ gamma_rr gamma_ri ] + # Gamma = [ gamma_ri gamma_ii ] + # and the shifting parameter + # Beta = [beta_real beta_imag].T + # where: + # x_real_BN = gamma_rr * x_real_normed + gamma_ri * x_imag_normed + beta_real + # x_imag_BN = gamma_ri * x_real_normed + gamma_ii * x_imag_normed + beta_imag + + broadcast_gamma_rr = K.reshape(gamma_rr, gamma_broadcast_shape) + broadcast_gamma_ri = K.reshape(gamma_ri, gamma_broadcast_shape) + broadcast_gamma_ii = K.reshape(gamma_ii, gamma_broadcast_shape) + + + cat_gamma_4_real = K.concatenate([broadcast_gamma_rr, broadcast_gamma_ii], axis=axis) + cat_gamma_4_imag = K.concatenate([broadcast_gamma_ri, broadcast_gamma_ri], axis=axis) + if (axis == 1 and ndim != 3) or ndim == 2: + centred_real = standardized_output[:, :input_dim] + centred_imag = standardized_output[:, input_dim:] + elif ndim == 3: + centred_real = standardized_output[:, :, :input_dim] + centred_imag = standardized_output[:, :, input_dim:] + elif axis == -1 and ndim == 4: + centred_real = standardized_output[:, :, :, :input_dim] + centred_imag = standardized_output[:, :, :, input_dim:] + elif axis == -1 and ndim == 5: + centred_real = standardized_output[:, :, :, :, :input_dim] + centred_imag = standardized_output[:, :, :, :, input_dim:] + else: + raise ValueError( + 'Incorrect Batchnorm combination of axis and dimensions. axis should be either 1 or -1. ' + 'axis: ' + str(self.axis) + '; ndim: ' + str(ndim) + '.' + ) + rolled_standardized_output = K.concatenate([centred_imag, centred_real], axis=axis) + if center: + broadcast_beta = K.reshape(beta, broadcast_beta_shape) + return cat_gamma_4_real * standardized_output + cat_gamma_4_imag * rolled_standardized_output + broadcast_beta + else: + return cat_gamma_4_real * standardized_output + cat_gamma_4_imag * rolled_standardized_output + else: + if center: + broadcast_beta = K.reshape(beta, broadcast_beta_shape) + return input_centred + broadcast_beta + else: + return input_centred + + +class ComplexBatchNormalization(Layer): + """Complex version of the real domain + Batch normalization layer (Ioffe and Szegedy, 2014). + Normalize the activations of the previous complex layer at each batch, + i.e. applies a transformation that maintains the mean of a complex unit + close to the null vector, the 2 by 2 covariance matrix of a complex unit close to identity + and the 2 by 2 relation matrix, also called pseudo-covariance, close to the + null matrix. + # Arguments + axis: Integer, the axis that should be normalized + (typically the features axis). + For instance, after a `Conv2D` layer with + `data_format="channels_first"`, + set `axis=2` in `ComplexBatchNormalization`. + momentum: Momentum for the moving statistics related to the real and + imaginary parts. + epsilon: Small float added to each of the variances related to the + real and imaginary parts in order to avoid dividing by zero. + center: If True, add offset of `beta` to complex normalized tensor. + If False, `beta` is ignored. + (beta is formed by real_beta and imag_beta) + scale: If True, multiply by the `gamma` matrix. + If False, `gamma` is not used. + beta_initializer: Initializer for the real_beta and the imag_beta weight. + gamma_diag_initializer: Initializer for the diagonal elements of the gamma matrix. + which are the variances of the real part and the imaginary part. + gamma_off_initializer: Initializer for the off-diagonal elements of the gamma matrix. + moving_mean_initializer: Initializer for the moving means. + moving_variance_initializer: Initializer for the moving variances. + moving_covariance_initializer: Initializer for the moving covariance of + the real and imaginary parts. + beta_regularizer: Optional regularizer for the beta weights. + gamma_regularizer: Optional regularizer for the gamma weights. + beta_constraint: Optional constraint for the beta weights. + gamma_constraint: Optional constraint for the gamma weights. + # Input shape + Arbitrary. Use the keyword argument `input_shape` + (tuple of integers, does not include the samples axis) + when using this layer as the first layer in a model. + # Output shape + Same shape as input. + # References + - [Batch Normalization: Accelerating Deep Network Training by Reducing Internal Covariate Shift](https://arxiv.org/abs/1502.03167) + """ + + def __init__(self, + axis=-1, + momentum=0.9, + epsilon=1e-4, + center=True, + scale=True, + beta_initializer='zeros', + gamma_diag_initializer='sqrt_init', + gamma_off_initializer='zeros', + moving_mean_initializer='zeros', + moving_variance_initializer='sqrt_init', + moving_covariance_initializer='zeros', + beta_regularizer=None, + gamma_diag_regularizer=None, + gamma_off_regularizer=None, + beta_constraint=None, + gamma_diag_constraint=None, + gamma_off_constraint=None, + **kwargs): + super(ComplexBatchNormalization, self).__init__(**kwargs) + self.supports_masking = True + self.axis = axis + self.momentum = momentum + self.epsilon = epsilon + self.center = center + self.scale = scale + self.beta_initializer = sanitizedInitGet(beta_initializer) + self.gamma_diag_initializer = sanitizedInitGet(gamma_diag_initializer) + self.gamma_off_initializer = sanitizedInitGet(gamma_off_initializer) + self.moving_mean_initializer = sanitizedInitGet(moving_mean_initializer) + self.moving_variance_initializer = sanitizedInitGet(moving_variance_initializer) + self.moving_covariance_initializer = sanitizedInitGet(moving_covariance_initializer) + self.beta_regularizer = regularizers.get(beta_regularizer) + self.gamma_diag_regularizer = regularizers.get(gamma_diag_regularizer) + self.gamma_off_regularizer = regularizers.get(gamma_off_regularizer) + self.beta_constraint = constraints .get(beta_constraint) + self.gamma_diag_constraint = constraints .get(gamma_diag_constraint) + self.gamma_off_constraint = constraints .get(gamma_off_constraint) + + def build(self, input_shape): + + ndim = len(input_shape) + + dim = input_shape[self.axis] + if dim is None: + raise ValueError('Axis ' + str(self.axis) + ' of ' + 'input tensor should have a defined dimension ' + 'but the layer received an input with shape ' + + str(input_shape) + '.') + self.input_spec = InputSpec(ndim=len(input_shape), + axes={self.axis: dim}) + + param_shape = (input_shape[self.axis] // 2,) + + if self.scale: + self.gamma_rr = self.add_weight(shape=param_shape, + name='gamma_rr', + initializer=self.gamma_diag_initializer, + regularizer=self.gamma_diag_regularizer, + constraint=self.gamma_diag_constraint) + self.gamma_ii = self.add_weight(shape=param_shape, + name='gamma_ii', + initializer=self.gamma_diag_initializer, + regularizer=self.gamma_diag_regularizer, + constraint=self.gamma_diag_constraint) + self.gamma_ri = self.add_weight(shape=param_shape, + name='gamma_ri', + initializer=self.gamma_off_initializer, + regularizer=self.gamma_off_regularizer, + constraint=self.gamma_off_constraint) + self.moving_Vrr = self.add_weight(shape=param_shape, + initializer=self.moving_variance_initializer, + name='moving_Vrr', + trainable=False) + self.moving_Vii = self.add_weight(shape=param_shape, + initializer=self.moving_variance_initializer, + name='moving_Vii', + trainable=False) + self.moving_Vri = self.add_weight(shape=param_shape, + initializer=self.moving_covariance_initializer, + name='moving_Vri', + trainable=False) + else: + self.gamma_rr = None + self.gamma_ii = None + self.gamma_ri = None + self.moving_Vrr = None + self.moving_Vii = None + self.moving_Vri = None + + if self.center: + self.beta = self.add_weight(shape=(input_shape[self.axis],), + name='beta', + initializer=self.beta_initializer, + regularizer=self.beta_regularizer, + constraint=self.beta_constraint) + self.moving_mean = self.add_weight(shape=(input_shape[self.axis],), + initializer=self.moving_mean_initializer, + name='moving_mean', + trainable=False) + else: + self.beta = None + self.moving_mean = None + + self.built = True + + def call(self, inputs, training=None): + input_shape = K.int_shape(inputs) + ndim = len(input_shape) + reduction_axes = list(range(ndim)) + del reduction_axes[self.axis] + input_dim = input_shape[self.axis] // 2 + mu = K.mean(inputs, axis=reduction_axes) + broadcast_mu_shape = [1] * len(input_shape) + broadcast_mu_shape[self.axis] = input_shape[self.axis] + broadcast_mu = K.reshape(mu, broadcast_mu_shape) + if self.center: + input_centred = inputs - broadcast_mu + else: + input_centred = inputs + centred_squared = input_centred ** 2 + if (self.axis == 1 and ndim != 3) or ndim == 2: + centred_squared_real = centred_squared[:, :input_dim] + centred_squared_imag = centred_squared[:, input_dim:] + centred_real = input_centred[:, :input_dim] + centred_imag = input_centred[:, input_dim:] + elif ndim == 3: + centred_squared_real = centred_squared[:, :, :input_dim] + centred_squared_imag = centred_squared[:, :, input_dim:] + centred_real = input_centred[:, :, :input_dim] + centred_imag = input_centred[:, :, input_dim:] + elif self.axis == -1 and ndim == 4: + centred_squared_real = centred_squared[:, :, :, :input_dim] + centred_squared_imag = centred_squared[:, :, :, input_dim:] + centred_real = input_centred[:, :, :, :input_dim] + centred_imag = input_centred[:, :, :, input_dim:] + elif self.axis == -1 and ndim == 5: + centred_squared_real = centred_squared[:, :, :, :, :input_dim] + centred_squared_imag = centred_squared[:, :, :, :, input_dim:] + centred_real = input_centred[:, :, :, :, :input_dim] + centred_imag = input_centred[:, :, :, :, input_dim:] + else: + raise ValueError( + 'Incorrect Batchnorm combination of axis and dimensions. axis should be either 1 or -1. ' + 'axis: ' + str(self.axis) + '; ndim: ' + str(ndim) + '.' + ) + if self.scale: + Vrr = K.mean( + centred_squared_real, + axis=reduction_axes + ) + self.epsilon + Vii = K.mean( + centred_squared_imag, + axis=reduction_axes + ) + self.epsilon + # Vri contains the real and imaginary covariance for each feature map. + Vri = K.mean( + centred_real * centred_imag, + axis=reduction_axes, + ) + elif self.center: + Vrr = None + Vii = None + Vri = None + else: + raise ValueError('Error. Both scale and center in batchnorm are set to False.') + + + input_bn = ComplexBN( + input_centred, Vrr, Vii, Vri, + self.beta, self.gamma_rr, self.gamma_ri, + self.gamma_ii, self.scale, self.center, + axis=self.axis + ) + + if training in {0, False}: + return input_bn + else: + update_list = [] + if self.center: + update_list.append(K.moving_average_update(self.moving_mean, mu, self.momentum)) + if self.scale: + update_list.append(K.moving_average_update(self.moving_Vrr, Vrr, self.momentum)) + update_list.append(K.moving_average_update(self.moving_Vii, Vii, self.momentum)) + update_list.append(K.moving_average_update(self.moving_Vri, Vri, self.momentum)) + self.add_update(update_list, inputs) + + def normalize_inference(): + if self.center: + inference_centred = inputs - K.reshape(self.moving_mean, broadcast_mu_shape) + else: + inference_centred = inputs + + return ComplexBN( + inference_centred, self.moving_Vrr, self.moving_Vii, + self.moving_Vri, self.beta, self.gamma_rr, self.gamma_ri, + self.gamma_ii, self.scale, self.center, axis=self.axis + ) + + # Pick the normalized form corresponding to the training phase. + return K.in_train_phase(input_bn, + normalize_inference, + training=training) + + def get_config(self): + config = { + 'axis': self.axis, + 'momentum': self.momentum, + 'epsilon': self.epsilon, + 'center': self.center, + 'scale': self.scale, + 'beta_initializer': sanitizedInitSer(self.beta_initializer), + 'gamma_diag_initializer': sanitizedInitSer(self.gamma_diag_initializer), + 'gamma_off_initializer': sanitizedInitSer(self.gamma_off_initializer), + 'moving_mean_initializer': sanitizedInitSer(self.moving_mean_initializer), + 'moving_variance_initializer': sanitizedInitSer(self.moving_variance_initializer), + 'moving_covariance_initializer': sanitizedInitSer(self.moving_covariance_initializer), + 'beta_regularizer': regularizers.serialize(self.beta_regularizer), + 'gamma_diag_regularizer': regularizers.serialize(self.gamma_diag_regularizer), + 'gamma_off_regularizer': regularizers.serialize(self.gamma_off_regularizer), + 'beta_constraint': constraints .serialize(self.beta_constraint), + 'gamma_diag_constraint': constraints .serialize(self.gamma_diag_constraint), + 'gamma_off_constraint': constraints .serialize(self.gamma_off_constraint), + } + base_config = super(ComplexBatchNormalization, self).get_config() + return dict(list(base_config.items()) + list(config.items())) + diff --git a/build/lib.linux-x86_64-2.7/complexnn/conv.py b/build/lib.linux-x86_64-2.7/complexnn/conv.py new file mode 100644 index 0000000..1009c3e --- /dev/null +++ b/build/lib.linux-x86_64-2.7/complexnn/conv.py @@ -0,0 +1,1796 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Contributors : Titouan Parcollet +# Initial Authors: Chiheb Trabelsi + +from keras import backend as K +from keras import activations, initializers, regularizers, constraints +from keras.layers import Lambda, Layer, InputSpec, Convolution1D, Convolution2D, add, multiply, Activation, Input, concatenate +from keras.layers.convolutional import _Conv +from keras.layers.merge import _Merge +from keras.layers.recurrent import Recurrent +from keras.utils import conv_utils +from keras.models import Model +import numpy as np +from theano.sandbox.rng_mrg import MRG_RandomStreams as RandomStreams +from .fft import fft, ifft, fft2, ifft2 +from .bn import ComplexBN as complex_normalization +from .bn import sqrt_init +from .init import * +from .norm import LayerNormalization, ComplexLayerNorm +import sys + + +##################################################################### +# Quaternion Implementations # +##################################################################### + +class QuaternionConv(Layer): + """Abstract nD quaternion convolution layer. + This layer creates a quaternion convolution kernel that is convolved + with the layer input to produce a tensor of outputs. + If `use_bias` is True, a bias vector is created and added to the outputs. + Finally, if `activation` is not `None`, + it is applied to the outputs as well. + # Arguments + rank: An integer, the rank of the convolution, + e.g. "2" for 2D convolution. + filters: Integer, the dimensionality of the output space, i.e, + the number of quaternion feature maps. It is also the effective number + of feature maps for each of the real and imaginary parts. + (i.e. the number of quaternion filters in the convolution) + The total effective number of filters is 2 x filters. + kernel_size: An integer or tuple/list of n integers, specifying the + dimensions of the convolution window. + strides: An integer or tuple/list of n integers, + spfying the strides of the convolution. + Specifying any stride value != 1 is incompatible with specifying + any `dilation_rate` value != 1. + padding: One of `"valid"` or `"same"` (case-insensitive). + data_format: A string, + one of `channels_last` (default) or `channels_first`. + The ordering of the dimensions in the inputs. + `channels_last` corresponds to inputs with shape + `(batch, ..., channels)` while `channels_first` corresponds to + inputs with shape `(batch, channels, ...)`. + It defaults to the `image_data_format` value found in your + Keras config file at `~/.keras/keras.json`. + If you never set it, then it will be "channels_last". + dilation_rate: An integer or tuple/list of n integers, specifying + the dilation rate to use for dilated convolution. + Currently, specifying any `dilation_rate` value != 1 is + incompatible with specifying any `strides` value != 1. + activation: Activation function to use + (see keras.activations). + If you don't specify anything, no activation is applied + (ie. "linear" activation: `a(x) = x`). + use_bias: Boolean, whether the layer uses a bias vector. + normalize_weight: Boolean, whether the layer normalizes its quaternion + weights before convolving the quaternion input. + The quaternion normalization performed is similar to the one + for the batchnorm. Each of the quaternion kernels are centred and multiplied by + the inverse square root of covariance matrix. + Then, a quaternion multiplication is perfromed as the normalized weights are + multiplied by the quaternion scaling factor gamma. + kernel_initializer: Initializer for the quaternion `kernel` weights matrix. + By default it is 'quaternion'. The 'quaternion_independent' + and the usual initializers could also be used. + (see keras.initializers and init.py). + bias_initializer: Initializer for the bias vector + (see keras.initializers). + kernel_regularizer: Regularizer function applied to + the `kernel` weights matrix + (see keras.regularizers). + bias_regularizer: Regularizer function applied to the bias vector + (see keras.regularizers). + activity_regularizer: Regularizer function applied to + the output of the layer (its "activation"). + (see keras.regularizers). + kernel_constraint: Constraint function applied to the kernel matrix + (see keras.constraints). + bias_constraint: Constraint function applied to the bias vector + (see keras.constraints). + spectral_parametrization: Whether or not to use a spectral + parametrization of the parameters. + """ + + def __init__(self, rank, + filters, + kernel_size, + strides=1, + padding='valid', + data_format=None, + dilation_rate=1, + activation=None, + use_bias=True, + normalize_weight=False, + kernel_initializer='quaternion', + bias_initializer='zeros', + gamma_diag_initializer=sqrt_init, + gamma_off_initializer='zeros', + kernel_regularizer=None, + bias_regularizer=None, + gamma_diag_regularizer=None, + gamma_off_regularizer=None, + activity_regularizer=None, + kernel_constraint=None, + bias_constraint=None, + gamma_diag_constraint=None, + gamma_off_constraint=None, + init_criterion='he', + seed=None, + spectral_parametrization=False, + epsilon=1e-7, + **kwargs): + super(QuaternionConv, self).__init__(**kwargs) + self.rank = rank + self.filters = filters + self.kernel_size = conv_utils.normalize_tuple(kernel_size, rank, 'kernel_size') + self.strides = conv_utils.normalize_tuple(strides, rank, 'strides') + self.padding = conv_utils.normalize_padding(padding) + self.data_format = 'channels_last' if rank == 1 else conv_utils.normalize_data_format(data_format) + self.dilation_rate = conv_utils.normalize_tuple(dilation_rate, rank, 'dilation_rate') + self.activation = activations.get(activation) + self.use_bias = use_bias + self.normalize_weight = normalize_weight + self.init_criterion = init_criterion + self.spectral_parametrization = spectral_parametrization + self.epsilon = epsilon + self.kernel_initializer = sanitizedInitGet(kernel_initializer) + self.bias_initializer = sanitizedInitGet(bias_initializer) + self.gamma_diag_initializer = sanitizedInitGet(gamma_diag_initializer) + self.gamma_off_initializer = sanitizedInitGet(gamma_off_initializer) + self.kernel_regularizer = regularizers.get(kernel_regularizer) + self.bias_regularizer = regularizers.get(bias_regularizer) + self.gamma_diag_regularizer = regularizers.get(gamma_diag_regularizer) + self.gamma_off_regularizer = regularizers.get(gamma_off_regularizer) + self.activity_regularizer = regularizers.get(activity_regularizer) + self.kernel_constraint = constraints.get(kernel_constraint) + self.bias_constraint = constraints.get(bias_constraint) + self.gamma_diag_constraint = constraints.get(gamma_diag_constraint) + self.gamma_off_constraint = constraints.get(gamma_off_constraint) + if seed is None: + self.seed = np.random.randint(1, 10e6) + else: + self.seed = seed + self.input_spec = InputSpec(ndim=self.rank + 2) + + def build(self, input_shape): + + if self.data_format == 'channels_first': + channel_axis = 1 + else: + channel_axis = -1 + + if input_shape[channel_axis] is None: + raise ValueError('The channel dimension of the inputs ' + 'should be defined. Found `None`.') + + input_dim = input_shape[channel_axis] // 4 + self.kernel_shape = self.kernel_size + (input_dim , self.filters) + # The kernel shape here is a complex kernel shape: + # nb of complex feature maps = input_dim; + # nb of output complex feature maps = self.filters; + # imaginary kernel size = real kernel size + # = self.kernel_size + # = complex kernel size + if self.kernel_initializer in {'quaternion', 'quaternion_independent'}: + kls = {'quaternion': QuaternionInit, + 'quaternion_independent': QuaternionIndependentFilters}[self.kernel_initializer] + kern_init = kls( + kernel_size=self.kernel_size, + input_dim=input_dim, + weight_dim=self.rank, + nb_filters=self.filters, + criterion=self.init_criterion + ) + else: + kern_init = self.kernel_initializer + + self.kernel = self.add_weight( + self.kernel_shape, + initializer=kern_init, + name='kernel', + regularizer=self.kernel_regularizer, + constraint=self.kernel_constraint + ) + + # Don't understand the purpose of this block + if self.normalize_weight: + gamma_shape = (input_dim * self.filters,) + self.gamma_rr = self.add_weight( + shape=gamma_shape, + name='gamma_rr', + initializer=self.gamma_diag_initializer, + regularizer=self.gamma_diag_regularizer, + constraint=self.gamma_diag_constraint + ) + + self.gamma_ri = self.add_weight( + shape=gamma_shape, + name='gamma_ri', + initializer=self.gamma_diag_initializer, + regularizer=self.gamma_diag_regularizer, + constraint=self.gamma_diag_constraint + ) + self.gamma_rj = self.add_weight( + shape=gamma_shape, + name='gamma_rj', + initializer=self.gamma_diag_initializer, + regularizer=self.gamma_diag_regularizer, + constraint=self.gamma_diag_constraint + ) + self.gamma_rk = self.add_weight( + shape=gamma_shape, + name='gamma_rk', + initializer=self.gamma_off_initializer, + regularizer=self.gamma_off_regularizer, + constraint=self.gamma_off_constraint + ) + self.gamma_ii = self.add_weight( + shape=gamma_shape, + name='gamma_ii', + initializer=self.gamma_diag_initializer, + regularizer=self.gamma_diag_regularizer, + constraint=self.gamma_diag_constraint + ) + + self.gamma_ij = self.add_weight( + shape=gamma_shape, + name='gamma_ij', + initializer=self.gamma_diag_initializer, + regularizer=self.gamma_diag_regularizer, + constraint=self.gamma_diag_constraint + ) + self.gamma_ik = self.add_weight( + shape=gamma_shape, + name='gamma_ik', + initializer=self.gamma_diag_initializer, + regularizer=self.gamma_diag_regularizer, + constraint=self.gamma_diag_constraint + ) + self.gamma_jj = self.add_weight( + shape=gamma_shape, + name='gamma_jj', + initializer=self.gamma_off_initializer, + regularizer=self.gamma_off_regularizer, + constraint=self.gamma_off_constraint + ) + self.gamma_jk = self.add_weight( + shape=gamma_shape, + name='gamma_jk', + initializer=self.gamma_diag_initializer, + regularizer=self.gamma_diag_regularizer, + constraint=self.gamma_diag_constraint + ) + self.gamma_kk = self.add_weight( + shape=gamma_shape, + name='gamma_kk', + initializer=self.gamma_off_initializer, + regularizer=self.gamma_off_regularizer, + constraint=self.gamma_off_constraint + ) + else: + self.gamma_rr = None + self.gamma_ri = None + self.gamma_rj = None + self.gamma_rk = None + self.gamma_ii = None + self.gamma_ij = None + self.gamma_ik = None + self.gamma_jj = None + self.gamma_jk = None + self.gamma_kk = None + + #End of non understanded block + + if self.use_bias: + bias_shape = (4 * self.filters,) + self.bias = self.add_weight( + bias_shape, + initializer=self.bias_initializer, + name='bias', + regularizer=self.bias_regularizer, + constraint=self.bias_constraint + ) + + else: + self.bias = None + + # Set input spec. + self.input_spec = InputSpec(ndim=self.rank + 2, + axes={channel_axis: input_dim * 4}) + self.built = True + + def call(self, inputs): + channel_axis = 1 if self.data_format == 'channels_first' else -1 + input_dim = K.shape(inputs)[channel_axis] // 4 + index2 = self.filters*2 + index3 = self.filters*3 + if self.rank == 1: + f_r = self.kernel[:, :, :self.filters] + f_i = self.kernel[:, :, self.filters:index2] + f_j = self.kernel[:, :, index2:index3] + f_k = self.kernel[:, :, index3:] + elif self.rank == 2: + f_r = self.kernel[:, :, :, :self.filters] + f_i = self.kernel[:, :, :, self.filters:index2] + f_j = self.kernel[:, :, :, index2:index3] + f_k = self.kernel[:, :, :, index3:] + elif self.rank == 3: + f_r = self.kernel[:, :, :, :, :self.filters] + f_i = self.kernel[:, :, :, :, self.filters:index2] + f_j = self.kernel[:, :, :, :, index2:index3] + f_k = self.kernel[:, :, :, :, index3:] + + convArgs = {"strides": self.strides[0] if self.rank == 1 else self.strides, + "padding": self.padding, + "data_format": self.data_format, + "dilation_rate": self.dilation_rate[0] if self.rank == 1 else self.dilation_rate} + convFunc = {1: K.conv1d, + 2: K.conv2d, + 3: K.conv3d}[self.rank] + + # processing if the weights are assumed to be represented in the spectral domain + # Do we conserve this for quaternions ? Currently no + + if self.spectral_parametrization: + print("Quaternion spectral weights parametrization not implemented yet, aborting.") + sys.exit(1) + if self.rank == 1: + f_r = K.permute_dimensions(f_r, (2,1,0)) + f_i = K.permute_dimensions(f_i, (2,1,0)) + f = K.concatenate([f_r, f_i], axis=0) + fshape = K.shape(f) + f = K.reshape(f, (fshape[0] * fshape[1], fshape[2])) + f = ifft(f) + f = K.reshape(f, fshape) + f_r = f[:fshape[0]//2] + f_i = f[fshape[0]//2:] + f_r = K.permute_dimensions(f_r, (2,1,0)) + f_i = K.permute_dimensions(f_i, (2,1,0)) + elif self.rank == 2: + f_r = K.permute_dimensions(f_r, (3,2,0,1)) + f_i = K.permute_dimensions(f_i, (3,2,0,1)) + f = K.concatenate([f_r, f_i], axis=0) + fshape = K.shape(f) + f = K.reshape(f, (fshape[0] * fshape[1], fshape[2], fshape[3])) + f = ifft2(f) + f = K.reshape(f, fshape) + f_r = f[:fshape[0]//2] + f_i = f[fshape[0]//2:] + f_r = K.permute_dimensions(f_r, (2,3,1,0)) + f_i = K.permute_dimensions(f_i, (2,3,1,0)) + + # In case of weight normalization, real and imaginary weights are normalized + + if self.normalize_weight: + + print("Quaternion weights normalization not implemented yet, aborting.") + sys.exit(1) + ker_shape = self.kernel_shape + nb_kernels = ker_shape[-2] * ker_shape[-1] + kernel_shape_4_norm = (np.prod(self.kernel_size), nb_kernels) + reshaped_f_r = K.reshape(f_r, kernel_shape_4_norm) + reshaped_f_i = K.reshape(f_i, kernel_shape_4_norm) + reduction_axes = list(range(2)) + del reduction_axes[-1] + mu_real = K.mean(reshaped_f_r, axis=reduction_axes) + mu_imag = K.mean(reshaped_f_i, axis=reduction_axes) + + broadcast_mu_shape = [1] * 2 + broadcast_mu_shape[-1] = nb_kernels + broadcast_mu_real = K.reshape(mu_real, broadcast_mu_shape) + broadcast_mu_imag = K.reshape(mu_imag, broadcast_mu_shape) + reshaped_f_r_centred = reshaped_f_r - broadcast_mu_real + reshaped_f_i_centred = reshaped_f_i - broadcast_mu_imag + Vrr = K.mean(reshaped_f_r_centred ** 2, axis=reduction_axes) + self.epsilon + Vii = K.mean(reshaped_f_i_centred ** 2, axis=reduction_axes) + self.epsilon + Vri = K.mean(reshaped_f_r_centred * reshaped_f_i_centred, + axis=reduction_axes) + self.epsilon + + normalized_weight = complex_normalization( + K.concatenate([reshaped_f_r, reshaped_f_i], axis=-1), + Vrr, Vii, Vri, + beta = None, + gamma_rr = self.gamma_rr, + gamma_ri = self.gamma_ri, + gamma_ii = self.gamma_ii, + scale=True, + center=False, + axis=-1 + ) + + normalized_real = normalized_weight[:, :nb_kernels] + normalized_imag = normalized_weight[:, nb_kernels:] + f_r = K.reshape(normalized_real, self.kernel_shape) + f_i = K.reshape(normalized_imag, self.kernel_shape) + + # + # Performing quaternion convolution + # + + f_r._keras_shape = self.kernel_shape + f_i._keras_shape = self.kernel_shape + f_j._keras_shape = self.kernel_shape + f_k._keras_shape = self.kernel_shape + + cat_kernels_4_r = K.concatenate([f_r, -f_i, -f_j, -f_k], axis=-2) + cat_kernels_4_i = K.concatenate([f_i, f_r, -f_k, f_j], axis=-2) + cat_kernels_4_j = K.concatenate([f_j, f_k, f_r, -f_i], axis=-2) + cat_kernels_4_k = K.concatenate([f_k, -f_j, f_i, f_r], axis=-2) + + cat_kernels_4_quaternion = K.concatenate([cat_kernels_4_r, cat_kernels_4_i, cat_kernels_4_j, cat_kernels_4_k], axis=-1) + cat_kernels_4_quaternion._keras_shape = self.kernel_size + (4 * input_dim, 4 * self.filters) + + output = convFunc(inputs, cat_kernels_4_quaternion, **convArgs) + + if self.use_bias: + output = K.bias_add( + output, + self.bias, + data_format=self.data_format + ) + + if self.activation is not None: + output = self.activation(output) + + return output + + def compute_output_shape(self, input_shape): + if self.data_format == 'channels_last': + space = input_shape[1:-1] + new_space = [] + for i in range(len(space)): + new_dim = conv_utils.conv_output_length( + space[i], + self.kernel_size[i], + padding=self.padding, + stride=self.strides[i], + dilation=self.dilation_rate[i] + ) + new_space.append(new_dim) + return (input_shape[0],) + tuple(new_space) + (4 * self.filters,) + if self.data_format == 'channels_first': + space = input_shape[2:] + new_space = [] + for i in range(len(space)): + new_dim = conv_utils.conv_output_length( + space[i], + self.kernel_size[i], + padding=self.padding, + stride=self.strides[i], + dilation=self.dilation_rate[i]) + new_space.append(new_dim) + return (input_shape[0],) + (4 * self.filters,) + tuple(new_space) + + def get_config(self): + config = { + 'rank': self.rank, + 'filters': self.filters, + 'kernel_size': self.kernel_size, + 'strides': self.strides, + 'padding': self.padding, + 'data_format': self.data_format, + 'dilation_rate': self.dilation_rate, + 'activation': activations.serialize(self.activation), + 'use_bias': self.use_bias, + 'normalize_weight': self.normalize_weight, + 'kernel_initializer': sanitizedInitSer(self.kernel_initializer), + 'bias_initializer': sanitizedInitSer(self.bias_initializer), + 'gamma_diag_initializer': sanitizedInitSer(self.gamma_diag_initializer), + 'gamma_off_initializer': sanitizedInitSer(self.gamma_off_initializer), + 'kernel_regularizer': regularizers.serialize(self.kernel_regularizer), + 'bias_regularizer': regularizers.serialize(self.bias_regularizer), + 'gamma_diag_regularizer': regularizers.serialize(self.gamma_diag_regularizer), + 'gamma_off_regularizer': regularizers.serialize(self.gamma_off_regularizer), + 'activity_regularizer': regularizers.serialize(self.activity_regularizer), + 'kernel_constraint': constraints.serialize(self.kernel_constraint), + 'bias_constraint': constraints.serialize(self.bias_constraint), + 'gamma_diag_constraint': constraints.serialize(self.gamma_diag_constraint), + 'gamma_off_constraint': constraints.serialize(self.gamma_off_constraint), + 'init_criterion': self.init_criterion, + 'spectral_parametrization': self.spectral_parametrization, + } + base_config = super(QuaternionConv, self).get_config() + return dict(list(base_config.items()) + list(config.items())) + + + +class QuaternionConv1D(QuaternionConv): + """1D quaternion convolution layer. + This layer creates a quaternion convolution kernel that is convolved + with a quaternion input layer over a single quaternion spatial (or temporal) dimension + to produce a quaternion output tensor. + If `use_bias` is True, a bias vector is created and added to the quaternion output. + Finally, if `activation` is not `None`, + it is applied each of the real and imaginary parts of the output. + When using this layer as the first layer in a model, + provide an `input_shape` argument + (tuple of integers or `None`, e.g. + `(10, 128)` for sequences of 10 vectors of 128-dimensional vectors, + or `(None, 128)` for variable-length sequences of 128-dimensional vectors. + # Arguments + filters: Integer, the dimensionality of the output space, i.e, + the number of quaternion feature maps. It is also the effective number + of feature maps for each of the real and imaginary parts. + (i.e. the number of quaternion filters in the convolution) + The total effective number of filters is 2 x filters. + kernel_size: An integer or tuple/list of n integers, specifying the + dimensions of the convolution window. + strides: An integer or tuple/list of a single integer, + specifying the stride length of the convolution. + Specifying any stride value != 1 is incompatible with specifying + any `dilation_rate` value != 1. + padding: One of `"valid"`, `"causal"` or `"same"` (case-insensitive). + `"causal"` results in causal (dilated) convolutions, e.g. output[t] + does not depend on input[t+1:]. Useful when modeling temporal data + where the model should not violate the temporal order. + See [WaveNet: A Generative Model for Raw Audio, section 2.1](https://arxiv.org/abs/1609.03499). + dilation_rate: an integer or tuple/list of a single integer, specifying + the dilation rate to use for dilated convolution. + Currently, specifying any `dilation_rate` value != 1 is + incompatible with specifying any `strides` value != 1. + activation: Activation function to use + (see keras.activations). + If you don't specify anything, no activation is applied + (ie. "linear" activation: `a(x) = x`). + use_bias: Boolean, whether the layer uses a bias vector. + normalize_weight: Boolean, whether the layer normalizes its quaternion + weights before convolving the quaternion input. + The quaternion normalization performed is similar to the one + for the batchnorm. Each of the quaternion kernels are centred and multiplied by + the inverse square root of covariance matrix. + Then, a quaternion multiplication is perfromed as the normalized weights are + multiplied by the quaternion scaling factor gamma. + kernel_initializer: Initializer for the quaternion `kernel` weights matrix. + By default it is 'quaternion'. The 'quaternion_independent' + and the usual initializers could also be used. + (see keras.initializers and init.py). + bias_initializer: Initializer for the bias vector + (see keras.initializers). + kernel_regularizer: Regularizer function applied to + the `kernel` weights matrix + (see keras.regularizers). + bias_regularizer: Regularizer function applied to the bias vector + (see keras.regularizers). + activity_regularizer: Regularizer function applied to + the output of the layer (its "activation"). + (see keras.regularizers). + kernel_constraint: Constraint function applied to the kernel matrix + (see keras.constraints). + bias_constraint: Constraint function applied to the bias vector + (see keras.constraints). + spectral_parametrization: Whether or not to use a spectral + parametrization of the parameters. + # Input shape + 3D tensor with shape: `(batch_size, steps, input_dim)` + # Output shape + 3D tensor with shape: `(batch_size, new_steps, 2 x filters)` + `steps` value might have changed due to padding or strides. + """ + + def __init__(self, filters, + kernel_size, + strides=1, + padding='valid', + dilation_rate=1, + activation=None, + use_bias=True, + kernel_initializer='quaternion', + bias_initializer='zeros', + kernel_regularizer=None, + bias_regularizer=None, + activity_regularizer=None, + kernel_constraint=None, + bias_constraint=None, + seed=None, + init_criterion='he', + spectral_parametrization=False, + **kwargs): + super(QuaternionConv1D, self).__init__( + rank=1, + filters=filters, + kernel_size=kernel_size, + strides=strides, + padding=padding, + data_format='channels_last', + dilation_rate=dilation_rate, + activation=activation, + use_bias=use_bias, + kernel_initializer=kernel_initializer, + bias_initializer=bias_initializer, + kernel_regularizer=kernel_regularizer, + bias_regularizer=bias_regularizer, + activity_regularizer=activity_regularizer, + kernel_constraint=kernel_constraint, + bias_constraint=bias_constraint, + init_criterion=init_criterion, + spectral_parametrization=spectral_parametrization, + **kwargs) + + def get_config(self): + config = super(QuaternionConv1D, self).get_config() + config.pop('rank') + config.pop('data_format') + return config + + +class QuaternionConv2D(QuaternionConv): + """2D Quaternion convolution layer (e.g. spatial convolution over images). + This layer creates a quaternion convolution kernel that is convolved + with a quaternion input layer to produce a quaternion output tensor. If `use_bias` + is True, a quaternion bias vector is created and added to the outputs. + Finally, if `activation` is not `None`, it is applied to both the + real and imaginary parts of the output. + When using this layer as the first layer in a model, + provide the keyword argument `input_shape` + (tuple of integers, does not include the sample axis), + e.g. `input_shape=(128, 128, 3)` for 128x128 RGB pictures + in `data_format="channels_last"`. + # Arguments + filters: Integer, the dimensionality of the quaternion output space + (i.e, the number quaternion feature maps in the convolution). + The total effective number of filters or feature maps is 2 x filters. + kernel_size: An integer or tuple/list of 2 integers, specifying the + width and height of the 2D convolution window. + Can be a single integer to specify the same value for + all spatial dimensions. + strides: An integer or tuple/list of 2 integers, + specifying the strides of the convolution along the width and height. + Can be a single integer to specify the same value for + all spatial dimensions. + Specifying any stride value != 1 is incompatible with specifying + any `dilation_rate` value != 1. + padding: one of `"valid"` or `"same"` (case-insensitive). + data_format: A string, + one of `channels_last` (default) or `channels_first`. + The ordering of the dimensions in the inputs. + `channels_last` corresponds to inputs with shape + `(batch, height, width, channels)` while `channels_first` + corresponds to inputs with shape + `(batch, channels, height, width)`. + It defaults to the `image_data_format` value found in your + Keras config file at `~/.keras/keras.json`. + If you never set it, then it will be "channels_last". + dilation_rate: an integer or tuple/list of 2 integers, specifying + the dilation rate to use for dilated convolution. + Can be a single integer to specify the same value for + all spatial dimensions. + Currently, specifying any `dilation_rate` value != 1 is + incompatible with specifying any stride value != 1. + activation: Activation function to use + (see keras.activations). + If you don't specify anything, no activation is applied + (ie. "linear" activation: `a(x) = x`). + use_bias: Boolean, whether the layer uses a bias vector. + normalize_weight: Boolean, whether the layer normalizes its quaternion + weights before convolving the quaternion input. + The quaternion normalization performed is similar to the one + for the batchnorm. Each of the quaternion kernels are centred and multiplied by + the inverse square root of covariance matrix. + Then, a quaternion multiplication is perfromed as the normalized weights are + multiplied by the quaternion scaling factor gamma. + kernel_initializer: Initializer for the quaternion `kernel` weights matrix. + By default it is 'quaternion'. The 'quaternion_independent' + and the usual initializers could also be used. + (see keras.initializers and init.py). + bias_initializer: Initializer for the bias vector + (see keras.initializers). + kernel_regularizer: Regularizer function applied to + the `kernel` weights matrix + (see keras.regularizers). + bias_regularizer: Regularizer function applied to the bias vector + (see keras.regularizers). + activity_regularizer: Regularizer function applied to + the output of the layer (its "activation"). + (see keras.regularizers). + kernel_constraint: Constraint function applied to the kernel matrix + (see keras.constraints). + bias_constraint: Constraint function applied to the bias vector + (see keras.constraints). + spectral_parametrization: Whether or not to use a spectral + parametrization of the parameters. + # Input shape + 4D tensor with shape: + `(samples, channels, rows, cols)` if data_format='channels_first' + or 4D tensor with shape: + `(samples, rows, cols, channels)` if data_format='channels_last'. + # Output shape + 4D tensor with shape: + `(samples, 2 x filters, new_rows, new_cols)` if data_format='channels_first' + or 4D tensor with shape: + `(samples, new_rows, new_cols, 2 x filters)` if data_format='channels_last'. + `rows` and `cols` values might have changed due to padding. + """ + + def __init__(self, filters, + kernel_size, + strides=(1, 1), + padding='valid', + data_format=None, + dilation_rate=(1, 1), + activation=None, + use_bias=True, + kernel_initializer='quaternion', + bias_initializer='zeros', + kernel_regularizer=None, + bias_regularizer=None, + activity_regularizer=None, + kernel_constraint=None, + bias_constraint=None, + seed=None, + init_criterion='he', + spectral_parametrization=False, + **kwargs): + super(QuaternionConv2D, self).__init__( + rank=2, + filters=filters, + kernel_size=kernel_size, + strides=strides, + padding=padding, + data_format=data_format, + dilation_rate=dilation_rate, + activation=activation, + use_bias=use_bias, + kernel_initializer=kernel_initializer, + bias_initializer=bias_initializer, + kernel_regularizer=kernel_regularizer, + bias_regularizer=bias_regularizer, + activity_regularizer=activity_regularizer, + kernel_constraint=kernel_constraint, + bias_constraint=bias_constraint, + init_criterion=init_criterion, + spectral_parametrization=spectral_parametrization, + **kwargs) + + def get_config(self): + config = super(QuaternionConv2D, self).get_config() + config.pop('rank') + return config + + +class QuaternionConv3D(QuaternionConv): + """3D convolution layer (e.g. spatial convolution over volumes). + This layer creates a quaternion convolution kernel that is convolved + with a quaternion layer input to produce a quaternion output tensor. + If `use_bias` is True, + a quaternion bias vector is created and added to the outputs. Finally, if + `activation` is not `None`, it is applied to each of the real and imaginary + parts of the output. + When using this layer as the first layer in a model, + provide the keyword argument `input_shape` + (tuple of integers, does not include the sample axis), + e.g. `input_shape=(2, 128, 128, 128, 3)` for 128x128x128 volumes + with 3 channels, + in `data_format="channels_last"`. + # Arguments + filters: Integer, the dimensionality of the quaternion output space + (i.e, the number quaternion feature maps in the convolution). + The total effective number of filters or feature maps is 2 x filters. + kernel_size: An integer or tuple/list of 3 integers, specifying the + width and height of the 3D convolution window. + Can be a single integer to specify the same value for + all spatial dimensions. + strides: An integer or tuple/list of 3 integers, + specifying the strides of the convolution along each spatial dimension. + Can be a single integer to specify the same value for + all spatial dimensions. + Specifying any stride value != 1 is incompatible with specifying + any `dilation_rate` value != 1. + padding: one of `"valid"` or `"same"` (case-insensitive). + data_format: A string, + one of `channels_last` (default) or `channels_first`. + The ordering of the dimensions in the inputs. + `channels_last` corresponds to inputs with shape + `(batch, spatial_dim1, spatial_dim2, spatial_dim3, channels)` + while `channels_first` corresponds to inputs with shape + `(batch, channels, spatial_dim1, spatial_dim2, spatial_dim3)`. + It defaults to the `image_data_format` value found in your + Keras config file at `~/.keras/keras.json`. + If you never set it, then it will be "channels_last". + dilation_rate: an integer or tuple/list of 3 integers, specifying + the dilation rate to use for dilated convolution. + Can be a single integer to specify the same value for + all spatial dimensions. + Currently, specifying any `dilation_rate` value != 1 is + incompatible with specifying any stride value != 1. + activation: Activation function to use + (see keras.activations). + If you don't specify anything, no activation is applied + (ie. "linear" activation: `a(x) = x`). + use_bias: Boolean, whether the layer uses a bias vector. + normalize_weight: Boolean, whether the layer normalizes its quaternion + weights before convolving the quaternion input. + The quaternion normalization performed is similar to the one + for the batchnorm. Each of the quaternion kernels are centred and multiplied by + the inverse square root of covariance matrix. + Then, a quaternion multiplication is perfromed as the normalized weights are + multiplied by the quaternion scaling factor gamma. + kernel_initializer: Initializer for the quaternion `kernel` weights matrix. + By default it is 'quaternion'. The 'quaternion_independent' + and the usual initializers could also be used. + (see keras.initializers and init.py). + bias_initializer: Initializer for the bias vector + (see keras.initializers). + kernel_regularizer: Regularizer function applied to + the `kernel` weights matrix + (see keras.regularizers). + bias_regularizer: Regularizer function applied to the bias vector + (see keras.regularizers). + activity_regularizer: Regularizer function applied to + the output of the layer (its "activation"). + (see keras.regularizers). + kernel_constraint: Constraint function applied to the kernel matrix + (see keras.constraints). + bias_constraint: Constraint function applied to the bias vector + (see keras.constraints). + spectral_parametrization: Whether or not to use a spectral + parametrization of the parameters. + # Input shape + 5D tensor with shape: + `(samples, channels, conv_dim1, conv_dim2, conv_dim3)` if data_format='channels_first' + or 5D tensor with shape: + `(samples, conv_dim1, conv_dim2, conv_dim3, channels)` if data_format='channels_last'. + # Output shape + 5D tensor with shape: + `(samples, 2 x filters, new_conv_dim1, new_conv_dim2, new_conv_dim3)` if data_format='channels_first' + or 5D tensor with shape: + `(samples, new_conv_dim1, new_conv_dim2, new_conv_dim3, 2 x filters)` if data_format='channels_last'. + `new_conv_dim1`, `new_conv_dim2` and `new_conv_dim3` values might have changed due to padding. + """ + + def __init__(self, filters, + kernel_size, + strides=(1, 1, 1), + padding='valid', + data_format=None, + dilation_rate=(1, 1, 1), + activation=None, + use_bias=True, + kernel_initializer='quaternion', + bias_initializer='zeros', + kernel_regularizer=None, + bias_regularizer=None, + activity_regularizer=None, + kernel_constraint=None, + bias_constraint=None, + seed=None, + init_criterion='he', + spectral_parametrization=False, + **kwargs): + super(QuaternionConv3D, self).__init__( + rank=3, + filters=filters, + kernel_size=kernel_size, + strides=strides, + padding=padding, + data_format=data_format, + dilation_rate=dilation_rate, + activation=activation, + use_bias=use_bias, + kernel_initializer=kernel_initializer, + bias_initializer=bias_initializer, + kernel_regularizer=kernel_regularizer, + bias_regularizer=bias_regularizer, + activity_regularizer=activity_regularizer, + kernel_constraint=kernel_constraint, + bias_constraint=bias_constraint, + init_criterion=init_criterion, + spectral_parametrization=spectral_parametrization, + **kwargs) + + def get_config(self): + config = super(QuaternionConv3D, self).get_config() + config.pop('rank') + return config + +def sanitizedInitGet(init): + if init in ["sqrt_init"]: + return sqrt_init + elif init in ["complex", "complex_independent", + "glorot_complex", "he_complex", + "quaternion", "quaternion_independent"]: + return init + else: + return initializers.get(init) +def sanitizedInitSer(init): + if init in [sqrt_init]: + return "sqrt_init" + elif init == "complex" or isinstance(init, ComplexInit): + return "complex" + elif init == "complex_independent" or isinstance(init, ComplexIndependentFilters): + return "complex_independent" + elif init == "quaternion" or isinstance(init, QuaternionInit): + return "quaternion" + elif init == "quaternion_independent" or isinstance(init, QuaternionIndependentFilters): + return "quaternion_independent" + else: + return initializers.serialize(init) + + +##################################################################### +# Complex Implementations # +##################################################################### + + + + +class ComplexConv(Layer): + """Abstract nD complex convolution layer. + This layer creates a complex convolution kernel that is convolved + with the layer input to produce a tensor of outputs. + If `use_bias` is True, a bias vector is created and added to the outputs. + Finally, if `activation` is not `None`, + it is applied to the outputs as well. + # Arguments + rank: An integer, the rank of the convolution, + e.g. "2" for 2D convolution. + filters: Integer, the dimensionality of the output space, i.e, + the number of complex feature maps. It is also the effective number + of feature maps for each of the real and imaginary parts. + (i.e. the number of complex filters in the convolution) + The total effective number of filters is 2 x filters. + kernel_size: An integer or tuple/list of n integers, specifying the + dimensions of the convolution window. + strides: An integer or tuple/list of n integers, + spfying the strides of the convolution. + Specifying any stride value != 1 is incompatible with specifying + any `dilation_rate` value != 1. + padding: One of `"valid"` or `"same"` (case-insensitive). + data_format: A string, + one of `channels_last` (default) or `channels_first`. + The ordering of the dimensions in the inputs. + `channels_last` corresponds to inputs with shape + `(batch, ..., channels)` while `channels_first` corresponds to + inputs with shape `(batch, channels, ...)`. + It defaults to the `image_data_format` value found in your + Keras config file at `~/.keras/keras.json`. + If you never set it, then it will be "channels_last". + dilation_rate: An integer or tuple/list of n integers, specifying + the dilation rate to use for dilated convolution. + Currently, specifying any `dilation_rate` value != 1 is + incompatible with specifying any `strides` value != 1. + activation: Activation function to use + (see keras.activations). + If you don't specify anything, no activation is applied + (ie. "linear" activation: `a(x) = x`). + use_bias: Boolean, whether the layer uses a bias vector. + normalize_weight: Boolean, whether the layer normalizes its complex + weights before convolving the complex input. + The complex normalization performed is similar to the one + for the batchnorm. Each of the complex kernels are centred and multiplied by + the inverse square root of covariance matrix. + Then, a complex multiplication is perfromed as the normalized weights are + multiplied by the complex scaling factor gamma. + kernel_initializer: Initializer for the complex `kernel` weights matrix. + By default it is 'complex'. The 'complex_independent' + and the usual initializers could also be used. + (see keras.initializers and init.py). + bias_initializer: Initializer for the bias vector + (see keras.initializers). + kernel_regularizer: Regularizer function applied to + the `kernel` weights matrix + (see keras.regularizers). + bias_regularizer: Regularizer function applied to the bias vector + (see keras.regularizers). + activity_regularizer: Regularizer function applied to + the output of the layer (its "activation"). + (see keras.regularizers). + kernel_constraint: Constraint function applied to the kernel matrix + (see keras.constraints). + bias_constraint: Constraint function applied to the bias vector + (see keras.constraints). + spectral_parametrization: Whether or not to use a spectral + parametrization of the parameters. + """ + + def __init__(self, rank, + filters, + kernel_size, + strides=1, + padding='valid', + data_format=None, + dilation_rate=1, + activation=None, + use_bias=True, + normalize_weight=False, + kernel_initializer='complex', + bias_initializer='zeros', + gamma_diag_initializer=sqrt_init, + gamma_off_initializer='zeros', + kernel_regularizer=None, + bias_regularizer=None, + gamma_diag_regularizer=None, + gamma_off_regularizer=None, + activity_regularizer=None, + kernel_constraint=None, + bias_constraint=None, + gamma_diag_constraint=None, + gamma_off_constraint=None, + init_criterion='he', + seed=None, + spectral_parametrization=False, + epsilon=1e-7, + **kwargs): + super(ComplexConv, self).__init__(**kwargs) + self.rank = rank + self.filters = filters + self.kernel_size = conv_utils.normalize_tuple(kernel_size, rank, 'kernel_size') + self.strides = conv_utils.normalize_tuple(strides, rank, 'strides') + self.padding = conv_utils.normalize_padding(padding) + self.data_format = 'channels_last' if rank == 1 else conv_utils.normalize_data_format(data_format) + self.dilation_rate = conv_utils.normalize_tuple(dilation_rate, rank, 'dilation_rate') + self.activation = activations.get(activation) + self.use_bias = use_bias + self.normalize_weight = normalize_weight + self.init_criterion = init_criterion + self.spectral_parametrization = spectral_parametrization + self.epsilon = epsilon + self.kernel_initializer = sanitizedInitGet(kernel_initializer) + self.bias_initializer = sanitizedInitGet(bias_initializer) + self.gamma_diag_initializer = sanitizedInitGet(gamma_diag_initializer) + self.gamma_off_initializer = sanitizedInitGet(gamma_off_initializer) + self.kernel_regularizer = regularizers.get(kernel_regularizer) + self.bias_regularizer = regularizers.get(bias_regularizer) + self.gamma_diag_regularizer = regularizers.get(gamma_diag_regularizer) + self.gamma_off_regularizer = regularizers.get(gamma_off_regularizer) + self.activity_regularizer = regularizers.get(activity_regularizer) + self.kernel_constraint = constraints.get(kernel_constraint) + self.bias_constraint = constraints.get(bias_constraint) + self.gamma_diag_constraint = constraints.get(gamma_diag_constraint) + self.gamma_off_constraint = constraints.get(gamma_off_constraint) + if seed is None: + self.seed = np.random.randint(1, 10e6) + else: + self.seed = seed + self.input_spec = InputSpec(ndim=self.rank + 2) + + def build(self, input_shape): + + if self.data_format == 'channels_first': + channel_axis = 1 + else: + channel_axis = -1 + if input_shape[channel_axis] is None: + raise ValueError('The channel dimension of the inputs ' + 'should be defined. Found `None`.') + input_dim = input_shape[channel_axis] // 2 + self.kernel_shape = self.kernel_size + (input_dim , self.filters) + # The kernel shape here is a complex kernel shape: + # nb of complex feature maps = input_dim; + # nb of output complex feature maps = self.filters; + # imaginary kernel size = real kernel size + # = self.kernel_size + # = complex kernel size + if self.kernel_initializer in {'complex', 'complex_independent'}: + kls = {'complex': ComplexInit, + 'complex_independent': ComplexIndependentFilters}[self.kernel_initializer] + kern_init = kls( + kernel_size=self.kernel_size, + input_dim=input_dim, + weight_dim=self.rank, + nb_filters=self.filters, + criterion=self.init_criterion + ) + else: + kern_init = self.kernel_initializer + + self.kernel = self.add_weight( + self.kernel_shape, + initializer=kern_init, + name='kernel', + regularizer=self.kernel_regularizer, + constraint=self.kernel_constraint + ) + + if self.normalize_weight: + gamma_shape = (input_dim * self.filters,) + self.gamma_rr = self.add_weight( + shape=gamma_shape, + name='gamma_rr', + initializer=self.gamma_diag_initializer, + regularizer=self.gamma_diag_regularizer, + constraint=self.gamma_diag_constraint + ) + self.gamma_ii = self.add_weight( + shape=gamma_shape, + name='gamma_ii', + initializer=self.gamma_diag_initializer, + regularizer=self.gamma_diag_regularizer, + constraint=self.gamma_diag_constraint + ) + self.gamma_ri = self.add_weight( + shape=gamma_shape, + name='gamma_ri', + initializer=self.gamma_off_initializer, + regularizer=self.gamma_off_regularizer, + constraint=self.gamma_off_constraint + ) + else: + self.gamma_rr = None + self.gamma_ii = None + self.gamma_ri = None + + if self.use_bias: + bias_shape = (2 * self.filters,) + self.bias = self.add_weight( + bias_shape, + initializer=self.bias_initializer, + name='bias', + regularizer=self.bias_regularizer, + constraint=self.bias_constraint + ) + + else: + self.bias = None + + # Set input spec. + self.input_spec = InputSpec(ndim=self.rank + 2, + axes={channel_axis: input_dim * 2}) + self.built = True + + def call(self, inputs): + channel_axis = 1 if self.data_format == 'channels_first' else -1 + input_dim = K.shape(inputs)[channel_axis] // 2 + if self.rank == 1: + f_real = self.kernel[:, :, :self.filters] + f_imag = self.kernel[:, :, self.filters:] + elif self.rank == 2: + f_real = self.kernel[:, :, :, :self.filters] + f_imag = self.kernel[:, :, :, self.filters:] + elif self.rank == 3: + f_real = self.kernel[:, :, :, :, :self.filters] + f_imag = self.kernel[:, :, :, :, self.filters:] + + convArgs = {"strides": self.strides[0] if self.rank == 1 else self.strides, + "padding": self.padding, + "data_format": self.data_format, + "dilation_rate": self.dilation_rate[0] if self.rank == 1 else self.dilation_rate} + convFunc = {1: K.conv1d, + 2: K.conv2d, + 3: K.conv3d}[self.rank] + + # processing if the weights are assumed to be represented in the spectral domain + + if self.spectral_parametrization: + if self.rank == 1: + f_real = K.permute_dimensions(f_real, (2,1,0)) + f_imag = K.permute_dimensions(f_imag, (2,1,0)) + f = K.concatenate([f_real, f_imag], axis=0) + fshape = K.shape(f) + f = K.reshape(f, (fshape[0] * fshape[1], fshape[2])) + f = ifft(f) + f = K.reshape(f, fshape) + f_real = f[:fshape[0]//2] + f_imag = f[fshape[0]//2:] + f_real = K.permute_dimensions(f_real, (2,1,0)) + f_imag = K.permute_dimensions(f_imag, (2,1,0)) + elif self.rank == 2: + f_real = K.permute_dimensions(f_real, (3,2,0,1)) + f_imag = K.permute_dimensions(f_imag, (3,2,0,1)) + f = K.concatenate([f_real, f_imag], axis=0) + fshape = K.shape(f) + f = K.reshape(f, (fshape[0] * fshape[1], fshape[2], fshape[3])) + f = ifft2(f) + f = K.reshape(f, fshape) + f_real = f[:fshape[0]//2] + f_imag = f[fshape[0]//2:] + f_real = K.permute_dimensions(f_real, (2,3,1,0)) + f_imag = K.permute_dimensions(f_imag, (2,3,1,0)) + + # In case of weight normalization, real and imaginary weights are normalized + + if self.normalize_weight: + ker_shape = self.kernel_shape + nb_kernels = ker_shape[-2] * ker_shape[-1] + kernel_shape_4_norm = (np.prod(self.kernel_size), nb_kernels) + reshaped_f_real = K.reshape(f_real, kernel_shape_4_norm) + reshaped_f_imag = K.reshape(f_imag, kernel_shape_4_norm) + reduction_axes = list(range(2)) + del reduction_axes[-1] + mu_real = K.mean(reshaped_f_real, axis=reduction_axes) + mu_imag = K.mean(reshaped_f_imag, axis=reduction_axes) + + broadcast_mu_shape = [1] * 2 + broadcast_mu_shape[-1] = nb_kernels + broadcast_mu_real = K.reshape(mu_real, broadcast_mu_shape) + broadcast_mu_imag = K.reshape(mu_imag, broadcast_mu_shape) + reshaped_f_real_centred = reshaped_f_real - broadcast_mu_real + reshaped_f_imag_centred = reshaped_f_imag - broadcast_mu_imag + Vrr = K.mean(reshaped_f_real_centred ** 2, axis=reduction_axes) + self.epsilon + Vii = K.mean(reshaped_f_imag_centred ** 2, axis=reduction_axes) + self.epsilon + Vri = K.mean(reshaped_f_real_centred * reshaped_f_imag_centred, + axis=reduction_axes) + self.epsilon + + normalized_weight = complex_normalization( + K.concatenate([reshaped_f_real, reshaped_f_imag], axis=-1), + Vrr, Vii, Vri, + beta = None, + gamma_rr = self.gamma_rr, + gamma_ri = self.gamma_ri, + gamma_ii = self.gamma_ii, + scale=True, + center=False, + axis=-1 + ) + + normalized_real = normalized_weight[:, :nb_kernels] + normalized_imag = normalized_weight[:, nb_kernels:] + f_real = K.reshape(normalized_real, self.kernel_shape) + f_imag = K.reshape(normalized_imag, self.kernel_shape) + + # Performing complex convolution + + f_real._keras_shape = self.kernel_shape + f_imag._keras_shape = self.kernel_shape + + cat_kernels_4_real = K.concatenate([f_real, -f_imag], axis=-2) + cat_kernels_4_imag = K.concatenate([f_imag, f_real], axis=-2) + cat_kernels_4_complex = K.concatenate([cat_kernels_4_real, cat_kernels_4_imag], axis=-1) + cat_kernels_4_complex._keras_shape = self.kernel_size + (2 * input_dim, 2 * self.filters) + + output = convFunc(inputs, cat_kernels_4_complex, **convArgs) + + if self.use_bias: + output = K.bias_add( + output, + self.bias, + data_format=self.data_format + ) + + if self.activation is not None: + output = self.activation(output) + + return output + + def compute_output_shape(self, input_shape): + if self.data_format == 'channels_last': + space = input_shape[1:-1] + new_space = [] + for i in range(len(space)): + new_dim = conv_utils.conv_output_length( + space[i], + self.kernel_size[i], + padding=self.padding, + stride=self.strides[i], + dilation=self.dilation_rate[i] + ) + new_space.append(new_dim) + return (input_shape[0],) + tuple(new_space) + (2 * self.filters,) + if self.data_format == 'channels_first': + space = input_shape[2:] + new_space = [] + for i in range(len(space)): + new_dim = conv_utils.conv_output_length( + space[i], + self.kernel_size[i], + padding=self.padding, + stride=self.strides[i], + dilation=self.dilation_rate[i]) + new_space.append(new_dim) + return (input_shape[0],) + (2 * self.filters,) + tuple(new_space) + + def get_config(self): + config = { + 'rank': self.rank, + 'filters': self.filters, + 'kernel_size': self.kernel_size, + 'strides': self.strides, + 'padding': self.padding, + 'data_format': self.data_format, + 'dilation_rate': self.dilation_rate, + 'activation': activations.serialize(self.activation), + 'use_bias': self.use_bias, + 'normalize_weight': self.normalize_weight, + 'kernel_initializer': sanitizedInitSer(self.kernel_initializer), + 'bias_initializer': sanitizedInitSer(self.bias_initializer), + 'gamma_diag_initializer': sanitizedInitSer(self.gamma_diag_initializer), + 'gamma_off_initializer': sanitizedInitSer(self.gamma_off_initializer), + 'kernel_regularizer': regularizers.serialize(self.kernel_regularizer), + 'bias_regularizer': regularizers.serialize(self.bias_regularizer), + 'gamma_diag_regularizer': regularizers.serialize(self.gamma_diag_regularizer), + 'gamma_off_regularizer': regularizers.serialize(self.gamma_off_regularizer), + 'activity_regularizer': regularizers.serialize(self.activity_regularizer), + 'kernel_constraint': constraints.serialize(self.kernel_constraint), + 'bias_constraint': constraints.serialize(self.bias_constraint), + 'gamma_diag_constraint': constraints.serialize(self.gamma_diag_constraint), + 'gamma_off_constraint': constraints.serialize(self.gamma_off_constraint), + 'init_criterion': self.init_criterion, + 'spectral_parametrization': self.spectral_parametrization, + } + base_config = super(ComplexConv, self).get_config() + return dict(list(base_config.items()) + list(config.items())) + + +class ComplexConv1D(ComplexConv): + """1D complex convolution layer. + This layer creates a complex convolution kernel that is convolved + with a complex input layer over a single complex spatial (or temporal) dimension + to produce a complex output tensor. + If `use_bias` is True, a bias vector is created and added to the complex output. + Finally, if `activation` is not `None`, + it is applied each of the real and imaginary parts of the output. + When using this layer as the first layer in a model, + provide an `input_shape` argument + (tuple of integers or `None`, e.g. + `(10, 128)` for sequences of 10 vectors of 128-dimensional vectors, + or `(None, 128)` for variable-length sequences of 128-dimensional vectors. + # Arguments + filters: Integer, the dimensionality of the output space, i.e, + the number of complex feature maps. It is also the effective number + of feature maps for each of the real and imaginary parts. + (i.e. the number of complex filters in the convolution) + The total effective number of filters is 2 x filters. + kernel_size: An integer or tuple/list of n integers, specifying the + dimensions of the convolution window. + strides: An integer or tuple/list of a single integer, + specifying the stride length of the convolution. + Specifying any stride value != 1 is incompatible with specifying + any `dilation_rate` value != 1. + padding: One of `"valid"`, `"causal"` or `"same"` (case-insensitive). + `"causal"` results in causal (dilated) convolutions, e.g. output[t] + does not depend on input[t+1:]. Useful when modeling temporal data + where the model should not violate the temporal order. + See [WaveNet: A Generative Model for Raw Audio, section 2.1](https://arxiv.org/abs/1609.03499). + dilation_rate: an integer or tuple/list of a single integer, specifying + the dilation rate to use for dilated convolution. + Currently, specifying any `dilation_rate` value != 1 is + incompatible with specifying any `strides` value != 1. + activation: Activation function to use + (see keras.activations). + If you don't specify anything, no activation is applied + (ie. "linear" activation: `a(x) = x`). + use_bias: Boolean, whether the layer uses a bias vector. + normalize_weight: Boolean, whether the layer normalizes its complex + weights before convolving the complex input. + The complex normalization performed is similar to the one + for the batchnorm. Each of the complex kernels are centred and multiplied by + the inverse square root of covariance matrix. + Then, a complex multiplication is perfromed as the normalized weights are + multiplied by the complex scaling factor gamma. + kernel_initializer: Initializer for the complex `kernel` weights matrix. + By default it is 'complex'. The 'complex_independent' + and the usual initializers could also be used. + (see keras.initializers and init.py). + bias_initializer: Initializer for the bias vector + (see keras.initializers). + kernel_regularizer: Regularizer function applied to + the `kernel` weights matrix + (see keras.regularizers). + bias_regularizer: Regularizer function applied to the bias vector + (see keras.regularizers). + activity_regularizer: Regularizer function applied to + the output of the layer (its "activation"). + (see keras.regularizers). + kernel_constraint: Constraint function applied to the kernel matrix + (see keras.constraints). + bias_constraint: Constraint function applied to the bias vector + (see keras.constraints). + spectral_parametrization: Whether or not to use a spectral + parametrization of the parameters. + # Input shape + 3D tensor with shape: `(batch_size, steps, input_dim)` + # Output shape + 3D tensor with shape: `(batch_size, new_steps, 2 x filters)` + `steps` value might have changed due to padding or strides. + """ + + def __init__(self, filters, + kernel_size, + strides=1, + padding='valid', + dilation_rate=1, + activation=None, + use_bias=True, + kernel_initializer='complex', + bias_initializer='zeros', + kernel_regularizer=None, + bias_regularizer=None, + activity_regularizer=None, + kernel_constraint=None, + bias_constraint=None, + seed=None, + init_criterion='he', + spectral_parametrization=False, + **kwargs): + super(ComplexConv1D, self).__init__( + rank=1, + filters=filters, + kernel_size=kernel_size, + strides=strides, + padding=padding, + data_format='channels_last', + dilation_rate=dilation_rate, + activation=activation, + use_bias=use_bias, + kernel_initializer=kernel_initializer, + bias_initializer=bias_initializer, + kernel_regularizer=kernel_regularizer, + bias_regularizer=bias_regularizer, + activity_regularizer=activity_regularizer, + kernel_constraint=kernel_constraint, + bias_constraint=bias_constraint, + init_criterion=init_criterion, + spectral_parametrization=spectral_parametrization, + **kwargs) + + def get_config(self): + config = super(ComplexConv1D, self).get_config() + config.pop('rank') + config.pop('data_format') + return config + + +class ComplexConv2D(ComplexConv): + """2D Complex convolution layer (e.g. spatial convolution over images). + This layer creates a complex convolution kernel that is convolved + with a complex input layer to produce a complex output tensor. If `use_bias` + is True, a complex bias vector is created and added to the outputs. + Finally, if `activation` is not `None`, it is applied to both the + real and imaginary parts of the output. + When using this layer as the first layer in a model, + provide the keyword argument `input_shape` + (tuple of integers, does not include the sample axis), + e.g. `input_shape=(128, 128, 3)` for 128x128 RGB pictures + in `data_format="channels_last"`. + # Arguments + filters: Integer, the dimensionality of the complex output space + (i.e, the number complex feature maps in the convolution). + The total effective number of filters or feature maps is 2 x filters. + kernel_size: An integer or tuple/list of 2 integers, specifying the + width and height of the 2D convolution window. + Can be a single integer to specify the same value for + all spatial dimensions. + strides: An integer or tuple/list of 2 integers, + specifying the strides of the convolution along the width and height. + Can be a single integer to specify the same value for + all spatial dimensions. + Specifying any stride value != 1 is incompatible with specifying + any `dilation_rate` value != 1. + padding: one of `"valid"` or `"same"` (case-insensitive). + data_format: A string, + one of `channels_last` (default) or `channels_first`. + The ordering of the dimensions in the inputs. + `channels_last` corresponds to inputs with shape + `(batch, height, width, channels)` while `channels_first` + corresponds to inputs with shape + `(batch, channels, height, width)`. + It defaults to the `image_data_format` value found in your + Keras config file at `~/.keras/keras.json`. + If you never set it, then it will be "channels_last". + dilation_rate: an integer or tuple/list of 2 integers, specifying + the dilation rate to use for dilated convolution. + Can be a single integer to specify the same value for + all spatial dimensions. + Currently, specifying any `dilation_rate` value != 1 is + incompatible with specifying any stride value != 1. + activation: Activation function to use + (see keras.activations). + If you don't specify anything, no activation is applied + (ie. "linear" activation: `a(x) = x`). + use_bias: Boolean, whether the layer uses a bias vector. + normalize_weight: Boolean, whether the layer normalizes its complex + weights before convolving the complex input. + The complex normalization performed is similar to the one + for the batchnorm. Each of the complex kernels are centred and multiplied by + the inverse square root of covariance matrix. + Then, a complex multiplication is perfromed as the normalized weights are + multiplied by the complex scaling factor gamma. + kernel_initializer: Initializer for the complex `kernel` weights matrix. + By default it is 'complex'. The 'complex_independent' + and the usual initializers could also be used. + (see keras.initializers and init.py). + bias_initializer: Initializer for the bias vector + (see keras.initializers). + kernel_regularizer: Regularizer function applied to + the `kernel` weights matrix + (see keras.regularizers). + bias_regularizer: Regularizer function applied to the bias vector + (see keras.regularizers). + activity_regularizer: Regularizer function applied to + the output of the layer (its "activation"). + (see keras.regularizers). + kernel_constraint: Constraint function applied to the kernel matrix + (see keras.constraints). + bias_constraint: Constraint function applied to the bias vector + (see keras.constraints). + spectral_parametrization: Whether or not to use a spectral + parametrization of the parameters. + # Input shape + 4D tensor with shape: + `(samples, channels, rows, cols)` if data_format='channels_first' + or 4D tensor with shape: + `(samples, rows, cols, channels)` if data_format='channels_last'. + # Output shape + 4D tensor with shape: + `(samples, 2 x filters, new_rows, new_cols)` if data_format='channels_first' + or 4D tensor with shape: + `(samples, new_rows, new_cols, 2 x filters)` if data_format='channels_last'. + `rows` and `cols` values might have changed due to padding. + """ + + def __init__(self, filters, + kernel_size, + strides=(1, 1), + padding='valid', + data_format=None, + dilation_rate=(1, 1), + activation=None, + use_bias=True, + kernel_initializer='complex', + bias_initializer='zeros', + kernel_regularizer=None, + bias_regularizer=None, + activity_regularizer=None, + kernel_constraint=None, + bias_constraint=None, + seed=None, + init_criterion='he', + spectral_parametrization=False, + **kwargs): + super(ComplexConv2D, self).__init__( + rank=2, + filters=filters, + kernel_size=kernel_size, + strides=strides, + padding=padding, + data_format=data_format, + dilation_rate=dilation_rate, + activation=activation, + use_bias=use_bias, + kernel_initializer=kernel_initializer, + bias_initializer=bias_initializer, + kernel_regularizer=kernel_regularizer, + bias_regularizer=bias_regularizer, + activity_regularizer=activity_regularizer, + kernel_constraint=kernel_constraint, + bias_constraint=bias_constraint, + init_criterion=init_criterion, + spectral_parametrization=spectral_parametrization, + **kwargs) + + def get_config(self): + config = super(ComplexConv2D, self).get_config() + config.pop('rank') + return config + + +class ComplexConv3D(ComplexConv): + """3D convolution layer (e.g. spatial convolution over volumes). + This layer creates a complex convolution kernel that is convolved + with a complex layer input to produce a complex output tensor. + If `use_bias` is True, + a complex bias vector is created and added to the outputs. Finally, if + `activation` is not `None`, it is applied to each of the real and imaginary + parts of the output. + When using this layer as the first layer in a model, + provide the keyword argument `input_shape` + (tuple of integers, does not include the sample axis), + e.g. `input_shape=(2, 128, 128, 128, 3)` for 128x128x128 volumes + with 3 channels, + in `data_format="channels_last"`. + # Arguments + filters: Integer, the dimensionality of the complex output space + (i.e, the number complex feature maps in the convolution). + The total effective number of filters or feature maps is 2 x filters. + kernel_size: An integer or tuple/list of 3 integers, specifying the + width and height of the 3D convolution window. + Can be a single integer to specify the same value for + all spatial dimensions. + strides: An integer or tuple/list of 3 integers, + specifying the strides of the convolution along each spatial dimension. + Can be a single integer to specify the same value for + all spatial dimensions. + Specifying any stride value != 1 is incompatible with specifying + any `dilation_rate` value != 1. + padding: one of `"valid"` or `"same"` (case-insensitive). + data_format: A string, + one of `channels_last` (default) or `channels_first`. + The ordering of the dimensions in the inputs. + `channels_last` corresponds to inputs with shape + `(batch, spatial_dim1, spatial_dim2, spatial_dim3, channels)` + while `channels_first` corresponds to inputs with shape + `(batch, channels, spatial_dim1, spatial_dim2, spatial_dim3)`. + It defaults to the `image_data_format` value found in your + Keras config file at `~/.keras/keras.json`. + If you never set it, then it will be "channels_last". + dilation_rate: an integer or tuple/list of 3 integers, specifying + the dilation rate to use for dilated convolution. + Can be a single integer to specify the same value for + all spatial dimensions. + Currently, specifying any `dilation_rate` value != 1 is + incompatible with specifying any stride value != 1. + activation: Activation function to use + (see keras.activations). + If you don't specify anything, no activation is applied + (ie. "linear" activation: `a(x) = x`). + use_bias: Boolean, whether the layer uses a bias vector. + normalize_weight: Boolean, whether the layer normalizes its complex + weights before convolving the complex input. + The complex normalization performed is similar to the one + for the batchnorm. Each of the complex kernels are centred and multiplied by + the inverse square root of covariance matrix. + Then, a complex multiplication is perfromed as the normalized weights are + multiplied by the complex scaling factor gamma. + kernel_initializer: Initializer for the complex `kernel` weights matrix. + By default it is 'complex'. The 'complex_independent' + and the usual initializers could also be used. + (see keras.initializers and init.py). + bias_initializer: Initializer for the bias vector + (see keras.initializers). + kernel_regularizer: Regularizer function applied to + the `kernel` weights matrix + (see keras.regularizers). + bias_regularizer: Regularizer function applied to the bias vector + (see keras.regularizers). + activity_regularizer: Regularizer function applied to + the output of the layer (its "activation"). + (see keras.regularizers). + kernel_constraint: Constraint function applied to the kernel matrix + (see keras.constraints). + bias_constraint: Constraint function applied to the bias vector + (see keras.constraints). + spectral_parametrization: Whether or not to use a spectral + parametrization of the parameters. + # Input shape + 5D tensor with shape: + `(samples, channels, conv_dim1, conv_dim2, conv_dim3)` if data_format='channels_first' + or 5D tensor with shape: + `(samples, conv_dim1, conv_dim2, conv_dim3, channels)` if data_format='channels_last'. + # Output shape + 5D tensor with shape: + `(samples, 2 x filters, new_conv_dim1, new_conv_dim2, new_conv_dim3)` if data_format='channels_first' + or 5D tensor with shape: + `(samples, new_conv_dim1, new_conv_dim2, new_conv_dim3, 2 x filters)` if data_format='channels_last'. + `new_conv_dim1`, `new_conv_dim2` and `new_conv_dim3` values might have changed due to padding. + """ + + def __init__(self, filters, + kernel_size, + strides=(1, 1, 1), + padding='valid', + data_format=None, + dilation_rate=(1, 1, 1), + activation=None, + use_bias=True, + kernel_initializer='complex', + bias_initializer='zeros', + kernel_regularizer=None, + bias_regularizer=None, + activity_regularizer=None, + kernel_constraint=None, + bias_constraint=None, + seed=None, + init_criterion='he', + spectral_parametrization=False, + **kwargs): + super(ComplexConv3D, self).__init__( + rank=3, + filters=filters, + kernel_size=kernel_size, + strides=strides, + padding=padding, + data_format=data_format, + dilation_rate=dilation_rate, + activation=activation, + use_bias=use_bias, + kernel_initializer=kernel_initializer, + bias_initializer=bias_initializer, + kernel_regularizer=kernel_regularizer, + bias_regularizer=bias_regularizer, + activity_regularizer=activity_regularizer, + kernel_constraint=kernel_constraint, + bias_constraint=bias_constraint, + init_criterion=init_criterion, + spectral_parametrization=spectral_parametrization, + **kwargs) + + def get_config(self): + config = super(ComplexConv3D, self).get_config() + config.pop('rank') + return config + + +class WeightNorm_Conv(_Conv): + # Real-valued Convolutional Layer that normalizes its weights + # before convolving the input. + # The weight Normalization performed the one + # described in the following paper: + # Weight Normalization: A Simple Reparameterization to Accelerate Training of Deep Neural Networks + # (see https://arxiv.org/abs/1602.07868) + + def __init__(self, + gamma_initializer='ones', + gamma_regularizer=None, + gamma_constraint=None, + epsilon=1e-07, + **kwargs): + super(WeightNorm_Conv, self).__init__(**kwargs) + if self.rank == 1: + self.data_format = 'channels_last' + self.gamma_initializer = sanitizedInitGet(gamma_initializer) + self.gamma_regularizer = regularizers.get(gamma_regularizer) + self.gamma_constraint = constraints.get(gamma_constraint) + self.epsilon = epsilon + + def build(self, input_shape): + super(WeightNorm_Conv, self).build(input_shape) + if self.data_format == 'channels_first': + channel_axis = 1 + else: + channel_axis = -1 + if input_shape[channel_axis] is None: + raise ValueError('The channel dimension of the inputs ' + 'should be defined. Found `None`.') + input_dim = input_shape[channel_axis] + gamma_shape = (input_dim * self.filters,) + self.gamma = self.add_weight( + shape=gamma_shape, + name='gamma', + initializer=self.gamma_initializer, + regularizer=self.gamma_regularizer, + constraint=self.gamma_constraint + ) + + def call(self, inputs): + input_shape = K.shape(inputs) + if self.data_format == 'channels_first': + channel_axis = 1 + else: + channel_axis = -1 + if input_shape[channel_axis] is None: + raise ValueError('The channel dimension of the inputs ' + 'should be defined. Found `None`.') + input_dim = input_shape[channel_axis] + ker_shape = self.kernel_size + (input_dim, self.filters) + nb_kernels = ker_shape[-2] * ker_shape[-1] + kernel_shape_4_norm = (np.prod(self.kernel_size), nb_kernels) + reshaped_kernel = K.reshape(self.kernel, kernel_shape_4_norm) + normalized_weight = K.l2_normalize(reshaped_kernel, axis=0, epsilon=self.epsilon) + normalized_weight = K.reshape(self.gamma, (1, ker_shape[-2] * ker_shape[-1])) * normalized_weight + shaped_kernel = K.reshape(normalized_weight, ker_shape) + shaped_kernel._keras_shape = ker_shape + + convArgs = {"strides": self.strides[0] if self.rank == 1 else self.strides, + "padding": self.padding, + "data_format": self.data_format, + "dilation_rate": self.dilation_rate[0] if self.rank == 1 else self.dilation_rate} + convFunc = {1: K.conv1d, + 2: K.conv2d, + 3: K.conv3d}[self.rank] + output = convFunc(inputs, shaped_kernel, **convArgs) + + if self.use_bias: + output = K.bias_add( + output, + self.bias, + data_format=self.data_format + ) + + if self.activation is not None: + output = self.activation(output) + + return output + + def get_config(self): + config = { + 'gamma_initializer': sanitizedInitSer(self.gamma_initializer), + 'gamma_regularizer': regularizers.serialize(self.gamma_regularizer), + 'gamma_constraint': constraints.serialize(self.gamma_constraint), + 'epsilon': self.epsilon + } + base_config = super(WeightNorm_Conv, self).get_config() + return dict(list(base_config.items()) + list(config.items())) + + + +# Aliases +QuaternionConvolution1D = QuaternionConv1D +QuaternionConvolution2D = QuaternionConv2D +QuaternionConvolution3D = QuaternionConv3D + +ComplexConvolution1D = ComplexConv1D +ComplexConvolution2D = ComplexConv2D +ComplexConvolution3D = ComplexConv3D diff --git a/build/lib.linux-x86_64-2.7/complexnn/dense.py b/build/lib.linux-x86_64-2.7/complexnn/dense.py new file mode 100644 index 0000000..2fab2fc --- /dev/null +++ b/build/lib.linux-x86_64-2.7/complexnn/dense.py @@ -0,0 +1,230 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# +# Authors: Chiheb Trabelsi +# + +from keras import backend as K +import sys; sys.path.append('.') +from keras import backend as K +from keras import activations, initializers, regularizers, constraints +from keras.layers import Layer, InputSpec +import numpy as np +from theano.sandbox.rng_mrg import MRG_RandomStreams as RandomStreams + + +class ComplexDense(Layer): + """Regular complex densely-connected NN layer. + `Dense` implements the operation: + `real_preact = dot(real_input, real_kernel) - dot(imag_input, imag_kernel)` + `imag_preact = dot(real_input, imag_kernel) + dot(imag_input, real_kernel)` + `output = activation(K.concatenate([real_preact, imag_preact]) + bias)` + where `activation` is the element-wise activation function + passed as the `activation` argument, `kernel` is a weights matrix + created by the layer, and `bias` is a bias vector created by the layer + (only applicable if `use_bias` is `True`). + Note: if the input to the layer has a rank greater than 2, then + AN ERROR MESSAGE IS PRINTED. + # Arguments + units: Positive integer, dimensionality of each of the real part + and the imaginary part. It is actualy the number of complex units. + activation: Activation function to use + (see keras.activations). + If you don't specify anything, no activation is applied + (ie. "linear" activation: `a(x) = x`). + use_bias: Boolean, whether the layer uses a bias vector. + kernel_initializer: Initializer for the complex `kernel` weights matrix. + By default it is 'complex'. + and the usual initializers could also be used. + (see keras.initializers and init.py). + bias_initializer: Initializer for the bias vector + (see keras.initializers). + kernel_regularizer: Regularizer function applied to + the `kernel` weights matrix + (see keras.regularizers). + bias_regularizer: Regularizer function applied to the bias vector + (see keras.regularizers). + activity_regularizer: Regularizer function applied to + the output of the layer (its "activation"). + (see keras.regularizers). + kernel_constraint: Constraint function applied to the kernel matrix + (see keras.constraints). + bias_constraint: Constraint function applied to the bias vector + (see keras.constraints). + # Input shape + a 2D input with shape `(batch_size, input_dim)`. + # Output shape + For a 2D input with shape `(batch_size, input_dim)`, + the output would have shape `(batch_size, units)`. + """ + + def __init__(self, units, + activation=None, + use_bias=True, + init_criterion='he', + kernel_initializer='complex', + bias_initializer='zeros', + kernel_regularizer=None, + bias_regularizer=None, + activity_regularizer=None, + kernel_constraint=None, + bias_constraint=None, + seed=None, + **kwargs): + if 'input_shape' not in kwargs and 'input_dim' in kwargs: + kwargs['input_shape'] = (kwargs.pop('input_dim'),) + super(ComplexDense, self).__init__(**kwargs) + self.units = units + self.activation = activations.get(activation) + self.use_bias = use_bias + self.init_criterion = init_criterion + if kernel_initializer in {'complex'}: + self.kernel_initializer = kernel_initializer + else: + self.kernel_initializer = initializers.get(kernel_initializer) + self.bias_initializer = initializers.get(bias_initializer) + self.kernel_regularizer = regularizers.get(kernel_regularizer) + self.bias_regularizer = regularizers.get(bias_regularizer) + self.activity_regularizer = regularizers.get(activity_regularizer) + self.kernel_constraint = constraints.get(kernel_constraint) + self.bias_constraint = constraints.get(bias_constraint) + if seed is None: + self.seed = np.random.randint(1, 10e6) + else: + self.seed = seed + self.input_spec = InputSpec(ndim=2) + self.supports_masking = True + + def build(self, input_shape): + assert len(input_shape) == 2 + assert input_shape[-1] % 2 == 0 + input_dim = input_shape[-1] // 2 + data_format = K.image_data_format() + kernel_shape = (input_dim, self.units) + fan_in, fan_out = initializers._compute_fans( + kernel_shape, + data_format=data_format + ) + if self.init_criterion == 'he': + s = K.sqrt(1. / fan_in) + elif self.init_criterion == 'glorot': + s = K.sqrt(1. / (fan_in + fan_out)) + rng = RandomStreams(seed=self.seed) + + # Equivalent initialization using amplitude phase representation: + """modulus = rng.rayleigh(scale=s, size=kernel_shape) + phase = rng.uniform(low=-np.pi, high=np.pi, size=kernel_shape) + def init_w_real(shape, dtype=None): + return modulus * K.cos(phase) + def init_w_imag(shape, dtype=None): + return modulus * K.sin(phase)""" + + # Initialization using euclidean representation: + def init_w_real(shape, dtype=None): + return rng.normal( + size=kernel_shape, + avg=0, + std=s, + dtype=dtype + ) + def init_w_imag(shape, dtype=None): + return rng.normal( + size=kernel_shape, + avg=0, + std=s, + dtype=dtype + ) + if self.kernel_initializer in {'complex'}: + real_init = init_w_real + imag_init = init_w_imag + else: + real_init = self.kernel_initializer + imag_init = self.kernel_initializer + + self.real_kernel = self.add_weight( + shape=kernel_shape, + initializer=real_init, + name='real_kernel', + regularizer=self.kernel_regularizer, + constraint=self.kernel_constraint + ) + self.imag_kernel = self.add_weight( + shape=kernel_shape, + initializer=imag_init, + name='imag_kernel', + regularizer=self.kernel_regularizer, + constraint=self.kernel_constraint + ) + + if self.use_bias: + self.bias = self.add_weight( + shape=(2 * self.units,), + initializer=self.bias_initializer, + name='bias', + regularizer=self.bias_regularizer, + constraint=self.bias_constraint + ) + else: + self.bias = None + + self.input_spec = InputSpec(ndim=2, axes={-1: 2 * input_dim}) + self.built = True + + def call(self, inputs): + input_shape = K.shape(inputs) + input_dim = input_shape[-1] // 2 + real_input = inputs[:, :input_dim] + imag_input = inputs[:, input_dim:] + + cat_kernels_4_real = K.concatenate( + [self.real_kernel, -self.imag_kernel], + axis=-1 + ) + cat_kernels_4_imag = K.concatenate( + [self.imag_kernel, self.real_kernel], + axis=-1 + ) + cat_kernels_4_complex = K.concatenate( + [cat_kernels_4_real, cat_kernels_4_imag], + axis=0 + ) + + output = K.dot(inputs, cat_kernels_4_complex) + + if self.use_bias: + output = K.bias_add(output, self.bias) + if self.activation is not None: + output = self.activation(output) + + return output + + def compute_output_shape(self, input_shape): + assert input_shape and len(input_shape) == 2 + assert input_shape[-1] + output_shape = list(input_shape) + output_shape[-1] = 2 * self.units + return tuple(output_shape) + + def get_config(self): + if self.kernel_initializer in {'complex'}: + ki = self.kernel_initializer + else: + ki = initializers.serialize(self.kernel_initializer) + config = { + 'units': self.units, + 'activation': activations.serialize(self.activation), + 'use_bias': self.use_bias, + 'init_criterion': self.init_criterion, + 'kernel_initializer': ki, + 'bias_initializer': initializers.serialize(self.bias_initializer), + 'kernel_regularizer': regularizers.serialize(self.kernel_regularizer), + 'bias_regularizer': regularizers.serialize(self.bias_regularizer), + 'activity_regularizer': regularizers.serialize(self.activity_regularizer), + 'kernel_constraint': constraints.serialize(self.kernel_constraint), + 'bias_constraint': constraints.serialize(self.bias_constraint), + 'seed': self.seed, + } + base_config = super(ComplexDense, self).get_config() + return dict(list(base_config.items()) + list(config.items())) + diff --git a/build/lib.linux-x86_64-2.7/complexnn/fft.py b/build/lib.linux-x86_64-2.7/complexnn/fft.py new file mode 100644 index 0000000..df5d4b7 --- /dev/null +++ b/build/lib.linux-x86_64-2.7/complexnn/fft.py @@ -0,0 +1,172 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# +# Authors: Olexa Bilaniuk +# + +import keras.backend as KB +import keras.engine as KE +import keras.layers as KL +import keras.optimizers as KO +import theano as T +import theano.ifelse as TI +import theano.tensor as TT +import theano.tensor.fft as TTF +import numpy as np + + +# +# FFT functions: +# +# fft(): Batched 1-D FFT (Input: (Batch, TimeSamples)) +# ifft(): Batched 1-D IFFT (Input: (Batch, FreqSamples)) +# fft2(): Batched 2-D FFT (Input: (Batch, TimeSamplesH, TimeSamplesW)) +# ifft2(): Batched 2-D IFFT (Input: (Batch, FreqSamplesH, FreqSamplesW)) +# + +def fft(z): + B = z.shape[0]//2 + L = z.shape[1] + C = TT.as_tensor_variable(np.asarray([[[1,-1]]], dtype=T.config.floatX)) + Zr, Zi = TTF.rfft(z[:B], norm="ortho"), TTF.rfft(z[B:], norm="ortho") + isOdd = TT.eq(L%2, 1) + Zr = TI.ifelse(isOdd, TT.concatenate([Zr, C*Zr[:,1: ][:,::-1]], axis=1), + TT.concatenate([Zr, C*Zr[:,1:-1][:,::-1]], axis=1)) + Zi = TI.ifelse(isOdd, TT.concatenate([Zi, C*Zi[:,1: ][:,::-1]], axis=1), + TT.concatenate([Zi, C*Zi[:,1:-1][:,::-1]], axis=1)) + Zi = (C*Zi)[:,:,::-1] # Zi * i + Z = Zr+Zi + return TT.concatenate([Z[:,:,0], Z[:,:,1]], axis=0) +def ifft(z): + B = z.shape[0]//2 + L = z.shape[1] + C = TT.as_tensor_variable(np.asarray([[[1,-1]]], dtype=T.config.floatX)) + Zr, Zi = TTF.rfft(z[:B], norm="ortho"), TTF.rfft(z[B:]*-1, norm="ortho") + isOdd = TT.eq(L%2, 1) + Zr = TI.ifelse(isOdd, TT.concatenate([Zr, C*Zr[:,1: ][:,::-1]], axis=1), + TT.concatenate([Zr, C*Zr[:,1:-1][:,::-1]], axis=1)) + Zi = TI.ifelse(isOdd, TT.concatenate([Zi, C*Zi[:,1: ][:,::-1]], axis=1), + TT.concatenate([Zi, C*Zi[:,1:-1][:,::-1]], axis=1)) + Zi = (C*Zi)[:,:,::-1] # Zi * i + Z = Zr+Zi + return TT.concatenate([Z[:,:,0], Z[:,:,1]*-1], axis=0) +def fft2(x): + tt = x + tt = KB.reshape(tt, (x.shape[0] *x.shape[1], x.shape[2])) + tf = fft(tt) + tf = KB.reshape(tf, (x.shape[0], x.shape[1], x.shape[2])) + tf = KB.permute_dimensions(tf, (0, 2, 1)) + tf = KB.reshape(tf, (x.shape[0] *x.shape[2], x.shape[1])) + ff = fft(tf) + ff = KB.reshape(ff, (x.shape[0], x.shape[2], x.shape[1])) + ff = KB.permute_dimensions(ff, (0, 2, 1)) + return ff +def ifft2(x): + ff = x + ff = KB.permute_dimensions(ff, (0, 2, 1)) + ff = KB.reshape(ff, (x.shape[0] *x.shape[2], x.shape[1])) + tf = ifft(ff) + tf = KB.reshape(tf, (x.shape[0], x.shape[2], x.shape[1])) + tf = KB.permute_dimensions(tf, (0, 2, 1)) + tf = KB.reshape(tf, (x.shape[0] *x.shape[1], x.shape[2])) + tt = ifft(tf) + tt = KB.reshape(tt, (x.shape[0], x.shape[1], x.shape[2])) + return tt + +# +# FFT Layers: +# +# FFT: Batched 1-D FFT (Input: (Batch, FeatureMaps, TimeSamples)) +# IFFT: Batched 1-D IFFT (Input: (Batch, FeatureMaps, FreqSamples)) +# FFT2: Batched 2-D FFT (Input: (Batch, FeatureMaps, TimeSamplesH, TimeSamplesW)) +# IFFT2: Batched 2-D IFFT (Input: (Batch, FeatureMaps, FreqSamplesH, FreqSamplesW)) +# + +class FFT(KL.Layer): + def call(self, x, mask=None): + a = KB.permute_dimensions(x, (1,0,2)) + a = KB.reshape(a, (x.shape[1] *x.shape[0], x.shape[2])) + a = fft(a) + a = KB.reshape(a, (x.shape[1], x.shape[0], x.shape[2])) + return KB.permute_dimensions(a, (1,0,2)) +class IFFT(KL.Layer): + def call(self, x, mask=None): + a = KB.permute_dimensions(x, (1,0,2)) + a = KB.reshape(a, (x.shape[1] *x.shape[0], x.shape[2])) + a = ifft(a) + a = KB.reshape(a, (x.shape[1], x.shape[0], x.shape[2])) + return KB.permute_dimensions(a, (1,0,2)) +class FFT2(KL.Layer): + def call(self, x, mask=None): + a = KB.permute_dimensions(x, (1,0,2,3)) + a = KB.reshape(a, (x.shape[1] *x.shape[0], x.shape[2], x.shape[3])) + a = fft2(a) + a = KB.reshape(a, (x.shape[1], x.shape[0], x.shape[2], x.shape[3])) + return KB.permute_dimensions(a, (1,0,2,3)) +class IFFT2(KL.Layer): + def call(self, x, mask=None): + a = KB.permute_dimensions(x, (1,0,2,3)) + a = KB.reshape(a, (x.shape[1] *x.shape[0], x.shape[2], x.shape[3])) + a = ifft2(a) + a = KB.reshape(a, (x.shape[1], x.shape[0], x.shape[2], x.shape[3])) + return KB.permute_dimensions(a, (1,0,2,3)) + + + +# +# Tests +# +# Note: The IFFT is the conjugate of the FFT of the conjugate. +# +# np.fft.ifft(x) == np.conj(np.fft.fft(np.conj(x))) +# + +if __name__ == "__main__": + # Numpy + np.random.seed(1) + L = 19 + r = np.random.normal(0.8, size=(L,)) + i = np.random.normal(0.8, size=(L,)) + x = r+i*1j + R = np.fft.rfft(r, norm="ortho") + I = np.fft.rfft(i, norm="ortho") + X = np.fft.fft (x, norm="ortho") + + if L&1: + R = np.concatenate([R, np.conj(R[1: ][::-1])]) + I = np.concatenate([I, np.conj(I[1: ][::-1])]) + else: + R = np.concatenate([R, np.conj(R[1:-1][::-1])]) + I = np.concatenate([I, np.conj(I[1:-1][::-1])]) + Y = R+I*1j + print np.allclose(X, Y) + + + # Theano + z = TT.dmatrix() + f = T.function([z], ifft(fft(z))) + v = np.concatenate([np.real(x)[np.newaxis,:], np.imag(x)[np.newaxis,:]], axis=0) + print v + print f(v) + print np.allclose(v, f(v)) + + + # Keras + x = i = KL.Input(shape=(128, 32,32)) + x = IFFT2()(x) + model = KE.Model([i],[x]) + + loss = "mse" + opt = KO.Adam() + + model.compile(opt, loss) + model._make_train_function() + model._make_predict_function() + model._make_test_function() + + v = np.random.normal(size=(13,128,32,32)) + #print v + V = model.predict(v) + #print V + print V.shape diff --git a/build/lib.linux-x86_64-2.7/complexnn/init.py b/build/lib.linux-x86_64-2.7/complexnn/init.py new file mode 100644 index 0000000..59d4717 --- /dev/null +++ b/build/lib.linux-x86_64-2.7/complexnn/init.py @@ -0,0 +1,449 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Contributors: Titouan Parcollet +# Authors: Chiheb Trabelsi + +import numpy as np +from numpy.random import RandomState +from random import gauss +import keras.backend as K +from keras import initializers +from keras.initializers import Initializer +from keras.utils.generic_utils import (serialize_keras_object, + deserialize_keras_object) + + +##################################################################### +# Quaternion Implementations # +##################################################################### + +class QuaternionIndependentFilters(Initializer): + # This initialization constructs quaternion-valued kernels + # that are independent as much as possible from each other + # while respecting either the He or the Glorot criterion. + def __init__(self, kernel_size, input_dim, + weight_dim, nb_filters=None, + criterion='glorot', seed=None): + + # `weight_dim` is used as a parameter for sanity check + # as we should not pass an integer as kernel_size when + # the weight dimension is >= 2. + # nb_filters == 0 if weights are not convolutional (matrix instead of filters) + # then in such a case, weight_dim = 2. + # (in case of 2D input): + # nb_filters == None and len(kernel_size) == 2 and_weight_dim == 2 + # conv1D: len(kernel_size) == 1 and weight_dim == 1 + # conv2D: len(kernel_size) == 2 and weight_dim == 2 + # conv3d: len(kernel_size) == 3 and weight_dim == 3 + + assert len(kernel_size) == weight_dim and weight_dim in {0, 1, 2, 3} + self.nb_filters = nb_filters + self.kernel_size = kernel_size + self.input_dim = input_dim + self.weight_dim = weight_dim + self.criterion = criterion + self.seed = 1337 if seed is None else seed + + def __call__(self, shape, dtype=None): + + if self.nb_filters is not None: + num_rows = self.nb_filters * self.input_dim + num_cols = np.prod(self.kernel_size) + else: + # in case it is the kernel is a matrix and not a filter + # which is the case of 2D input (No feature maps). + num_rows = self.input_dim + num_cols = self.kernel_size[-1] + + flat_shape = (int(num_rows), int(num_cols)) + rng = RandomState(self.seed) + r = rng.uniform(size=flat_shape) + i = rng.uniform(size=flat_shape) + z = r + 1j * i + u, _, v = np.linalg.svd(z) + unitary_z = np.dot(u, np.dot(np.eye(int(num_rows), int(num_cols)), np.conjugate(v).T)) + real_unitary = unitary_z.real + imag_unitary = unitary_z.imag + if self.nb_filters is not None: + indep_real = np.reshape(real_unitary, (num_rows,) + tuple(self.kernel_size)) + indep_imag = np.reshape(imag_unitary, (num_rows,) + tuple(self.kernel_size)) + fan_in, fan_out = initializers._compute_fans( + tuple(self.kernel_size) + (int(self.input_dim), self.nb_filters) + ) + else: + indep_real = real_unitary + indep_imag = imag_unitary + fan_in, fan_out = (int(self.input_dim), self.kernel_size[-1]) + + if self.criterion == 'glorot': + desired_var = 1. / (fan_in + fan_out) + elif self.criterion == 'he': + desired_var = 1. / (fan_in) + else: + raise ValueError('Invalid criterion: ' + self.criterion) + + multip_real = np.sqrt(desired_var / np.var(indep_real)) + multip_imag = np.sqrt(desired_var / np.var(indep_imag)) + scaled_real = multip_real * indep_real + scaled_imag = multip_imag * indep_imag + + if self.weight_dim == 2 and self.nb_filters is None: + weight_real = scaled_real + weight_imag = scaled_imag + else: + kernel_shape = tuple(self.kernel_size) + (int(self.input_dim), self.nb_filters) + if self.weight_dim == 1: + transpose_shape = (1, 0) + elif self.weight_dim == 2 and self.nb_filters is not None: + transpose_shape = (1, 2, 0) + elif self.weight_dim == 3 and self.nb_filters is not None: + transpose_shape = (1, 2, 3, 0) + + weight_real = np.transpose(scaled_real, transpose_shape) + weight_imag = np.transpose(scaled_imag, transpose_shape) + weight_real = np.reshape(weight_real, kernel_shape) + weight_imag = np.reshape(weight_imag, kernel_shape) + weight = np.concatenate([weight_real, weight_imag], axis=-1) + + return weight + + def get_config(self): + return {'nb_filters': self.nb_filters, + 'kernel_size': self.kernel_size, + 'input_dim': self.input_dim, + 'weight_dim': self.weight_dim, + 'criterion': self.criterion, + 'seed': self.seed} + + +class QuaternionInit(Initializer): + # The standard complex initialization using + # either the He or the Glorot criterion. + def __init__(self, kernel_size, input_dim, + weight_dim, nb_filters=None, + criterion='glorot', seed=None): + + # `weight_dim` is used as a parameter for sanity check + # as we should not pass an integer as kernel_size when + # the weight dimension is >= 2. + # nb_filters == 0 if weights are not convolutional (matrix instead of filters) + # then in such a case, weight_dim = 2. + # (in case of 2D input): + # nb_filters == None and len(kernel_size) == 2 and_weight_dim == 2 + # conv1D: len(kernel_size) == 1 and weight_dim == 1 + # conv2D: len(kernel_size) == 2 and weight_dim == 2 + # conv3d: len(kernel_size) == 3 and weight_dim == 3 + + assert len(kernel_size) == weight_dim and weight_dim in {0, 1, 2, 3} + self.nb_filters = nb_filters + self.kernel_size = kernel_size + self.input_dim = input_dim + self.weight_dim = weight_dim + self.criterion = criterion + self.seed = 1337 if seed is None else seed + + def __call__(self, shape, dtype=None): + + if self.nb_filters is not None: + kernel_shape = tuple(self.kernel_size) + (int(self.input_dim), self.nb_filters) + else: + kernel_shape = (int(self.input_dim), self.kernel_size[-1]) + + fan_in, fan_out = initializers._compute_fans( + tuple(self.kernel_size) + (self.input_dim, self.nb_filters) + ) + + # Quaternion operations start here + + if self.criterion == 'glorot': + s = 1. / np.sqrt(2*(fan_in + fan_out)) + elif self.criterion == 'he': + s = 1. / np.sqrt(2*fan_in) + else: + raise ValueError('Invalid criterion: ' + self.criterion) + + #Generating randoms and purely imaginary quaternions : + number_of_weights = np.prod(kernel_shape) + v_i = np.random.uniform(0.0,1.0,number_of_weights) + v_j = np.random.uniform(0.0,1.0,number_of_weights) + v_k = np.random.uniform(0.0,1.0,number_of_weights) + #Make these purely imaginary quaternions unitary + for i in range(0, number_of_weights): + norm = np.sqrt(v_i[i]**2 + v_j[i]**2 + v_k[i]**2) + v_i[i]/= norm + v_j[i]/= norm + v_k[i]/= norm + v_i = v_i.reshape(kernel_shape) + v_j = v_j.reshape(kernel_shape) + v_k = v_k.reshape(kernel_shape) + + rng = RandomState(self.seed) + modulus = rng.rayleigh(scale=s, size=kernel_shape) + phase = rng.uniform(low=-np.pi, high=np.pi, size=kernel_shape) + + weight_r = modulus * np.cos(phase) + weight_i = modulus * v_i*np.sin(phase) + weight_j = modulus * v_j*np.sin(phase) + weight_k = modulus * v_k*np.sin(phase) + weight = np.concatenate([weight_r, weight_i, weight_j, weight_k], axis=-1) + + return weight + + +##################################################################### +# Complex Implementations # +##################################################################### + + +class IndependentFilters(Initializer): + # This initialization constructs real-valued kernels + # that are independent as much as possible from each other + # while respecting either the He or the Glorot criterion. + def __init__(self, kernel_size, input_dim, + weight_dim, nb_filters=None, + criterion='glorot', seed=None): + + # `weight_dim` is used as a parameter for sanity check + # as we should not pass an integer as kernel_size when + # the weight dimension is >= 2. + # nb_filters == 0 if weights are not convolutional (matrix instead of filters) + # then in such a case, weight_dim = 2. + # (in case of 2D input): + # nb_filters == None and len(kernel_size) == 2 and_weight_dim == 2 + # conv1D: len(kernel_size) == 1 and weight_dim == 1 + # conv2D: len(kernel_size) == 2 and weight_dim == 2 + # conv3d: len(kernel_size) == 3 and weight_dim == 3 + + assert len(kernel_size) == weight_dim and weight_dim in {0, 1, 2, 3} + self.nb_filters = nb_filters + self.kernel_size = kernel_size + self.input_dim = input_dim + self.weight_dim = weight_dim + self.criterion = criterion + self.seed = 1337 if seed is None else seed + + def __call__(self, shape, dtype=None): + + if self.nb_filters is not None: + num_rows = self.nb_filters * self.input_dim + num_cols = np.prod(self.kernel_size) + else: + # in case it is the kernel is a matrix and not a filter + # which is the case of 2D input (No feature maps). + num_rows = self.input_dim + num_cols = self.kernel_size[-1] + + flat_shape = (num_rows, num_cols) + rng = RandomState(self.seed) + x = rng.uniform(size=flat_shape) + u, _, v = np.linalg.svd(x) + orthogonal_x = np.dot(u, np.dot(np.eye(num_rows, num_cols), v.T)) + if self.nb_filters is not None: + independent_filters = np.reshape(orthogonal_x, (num_rows,) + tuple(self.kernel_size)) + fan_in, fan_out = initializers._compute_fans( + tuple(self.kernel_size) + (self.input_dim, self.nb_filters) + ) + else: + independent_filters = orthogonal_x + fan_in, fan_out = (self.input_dim, self.kernel_size[-1]) + + if self.criterion == 'glorot': + desired_var = 2. / (fan_in + fan_out) + elif self.criterion == 'he': + desired_var = 2. / fan_in + else: + raise ValueError('Invalid criterion: ' + self.criterion) + + multip_constant = np.sqrt (desired_var / np.var(independent_filters)) + scaled_indep = multip_constant * independent_filters + + if self.weight_dim == 2 and self.nb_filters is None: + weight_real = scaled_real + weight_imag = scaled_imag + else: + kernel_shape = tuple(self.kernel_size) + (self.input_dim, self.nb_filters) + if self.weight_dim == 1: + transpose_shape = (1, 0) + elif self.weight_dim == 2 and self.nb_filters is not None: + transpose_shape = (1, 2, 0) + elif self.weight_dim == 3 and self.nb_filters is not None: + transpose_shape = (1, 2, 3, 0) + weight = np.transpose(scaled_indep, transpose_shape) + weight = np.reshape(weight, kernel_shape) + + return weight + + def get_config(self): + return {'nb_filters': self.nb_filters, + 'kernel_size': self.kernel_size, + 'input_dim': self.input_dim, + 'weight_dim': self.weight_dim, + 'criterion': self.criterion, + 'seed': self.seed} + + +class ComplexIndependentFilters(Initializer): + # This initialization constructs complex-valued kernels + # that are independent as much as possible from each other + # while respecting either the He or the Glorot criterion. + def __init__(self, kernel_size, input_dim, + weight_dim, nb_filters=None, + criterion='glorot', seed=None): + + # `weight_dim` is used as a parameter for sanity check + # as we should not pass an integer as kernel_size when + # the weight dimension is >= 2. + # nb_filters == 0 if weights are not convolutional (matrix instead of filters) + # then in such a case, weight_dim = 2. + # (in case of 2D input): + # nb_filters == None and len(kernel_size) == 2 and_weight_dim == 2 + # conv1D: len(kernel_size) == 1 and weight_dim == 1 + # conv2D: len(kernel_size) == 2 and weight_dim == 2 + # conv3d: len(kernel_size) == 3 and weight_dim == 3 + + assert len(kernel_size) == weight_dim and weight_dim in {0, 1, 2, 3} + self.nb_filters = nb_filters + self.kernel_size = kernel_size + self.input_dim = input_dim + self.weight_dim = weight_dim + self.criterion = criterion + self.seed = 1337 if seed is None else seed + + def __call__(self, shape, dtype=None): + + if self.nb_filters is not None: + num_rows = self.nb_filters * self.input_dim + num_cols = np.prod(self.kernel_size) + else: + # in case it is the kernel is a matrix and not a filter + # which is the case of 2D input (No feature maps). + num_rows = self.input_dim + num_cols = self.kernel_size[-1] + + flat_shape = (int(num_rows), int(num_cols)) + rng = RandomState(self.seed) + r = rng.uniform(size=flat_shape) + i = rng.uniform(size=flat_shape) + z = r + 1j * i + u, _, v = np.linalg.svd(z) + unitary_z = np.dot(u, np.dot(np.eye(int(num_rows), int(num_cols)), np.conjugate(v).T)) + real_unitary = unitary_z.real + imag_unitary = unitary_z.imag + if self.nb_filters is not None: + indep_real = np.reshape(real_unitary, (num_rows,) + tuple(self.kernel_size)) + indep_imag = np.reshape(imag_unitary, (num_rows,) + tuple(self.kernel_size)) + fan_in, fan_out = initializers._compute_fans( + tuple(self.kernel_size) + (int(self.input_dim), self.nb_filters) + ) + else: + indep_real = real_unitary + indep_imag = imag_unitary + fan_in, fan_out = (int(self.input_dim), self.kernel_size[-1]) + + if self.criterion == 'glorot': + desired_var = 1. / (fan_in + fan_out) + elif self.criterion == 'he': + desired_var = 1. / (fan_in) + else: + raise ValueError('Invalid criterion: ' + self.criterion) + + multip_real = np.sqrt(desired_var / np.var(indep_real)) + multip_imag = np.sqrt(desired_var / np.var(indep_imag)) + scaled_real = multip_real * indep_real + scaled_imag = multip_imag * indep_imag + + if self.weight_dim == 2 and self.nb_filters is None: + weight_real = scaled_real + weight_imag = scaled_imag + else: + kernel_shape = tuple(self.kernel_size) + (int(self.input_dim), self.nb_filters) + if self.weight_dim == 1: + transpose_shape = (1, 0) + elif self.weight_dim == 2 and self.nb_filters is not None: + transpose_shape = (1, 2, 0) + elif self.weight_dim == 3 and self.nb_filters is not None: + transpose_shape = (1, 2, 3, 0) + + weight_real = np.transpose(scaled_real, transpose_shape) + weight_imag = np.transpose(scaled_imag, transpose_shape) + weight_real = np.reshape(weight_real, kernel_shape) + weight_imag = np.reshape(weight_imag, kernel_shape) + weight = np.concatenate([weight_real, weight_imag], axis=-1) + + return weight + + def get_config(self): + return {'nb_filters': self.nb_filters, + 'kernel_size': self.kernel_size, + 'input_dim': self.input_dim, + 'weight_dim': self.weight_dim, + 'criterion': self.criterion, + 'seed': self.seed} + + +class ComplexInit(Initializer): + # The standard complex initialization using + # either the He or the Glorot criterion. + def __init__(self, kernel_size, input_dim, + weight_dim, nb_filters=None, + criterion='glorot', seed=None): + + # `weight_dim` is used as a parameter for sanity check + # as we should not pass an integer as kernel_size when + # the weight dimension is >= 2. + # nb_filters == 0 if weights are not convolutional (matrix instead of filters) + # then in such a case, weight_dim = 2. + # (in case of 2D input): + # nb_filters == None and len(kernel_size) == 2 and_weight_dim == 2 + # conv1D: len(kernel_size) == 1 and weight_dim == 1 + # conv2D: len(kernel_size) == 2 and weight_dim == 2 + # conv3d: len(kernel_size) == 3 and weight_dim == 3 + + assert len(kernel_size) == weight_dim and weight_dim in {0, 1, 2, 3} + self.nb_filters = nb_filters + self.kernel_size = kernel_size + self.input_dim = input_dim + self.weight_dim = weight_dim + self.criterion = criterion + self.seed = 1337 if seed is None else seed + + def __call__(self, shape, dtype=None): + + if self.nb_filters is not None: + kernel_shape = tuple(self.kernel_size) + (int(self.input_dim), self.nb_filters) + else: + kernel_shape = (int(self.input_dim), self.kernel_size[-1]) + + fan_in, fan_out = initializers._compute_fans( + tuple(self.kernel_size) + (self.input_dim, self.nb_filters) + ) + + if self.criterion == 'glorot': + s = 1. / (fan_in + fan_out) + elif self.criterion == 'he': + s = 1. / fan_in + else: + raise ValueError('Invalid criterion: ' + self.criterion) + rng = RandomState(self.seed) + modulus = rng.rayleigh(scale=s, size=kernel_shape) + phase = rng.uniform(low=-np.pi, high=np.pi, size=kernel_shape) + weight_real = modulus * np.cos(phase) + weight_imag = modulus * np.sin(phase) + #print (weight_real.shape) + weight = np.concatenate([weight_real, weight_imag], axis=-1) + + return weight + + +class SqrtInit(Initializer): + def __call__(self, shape, dtype=None): + return K.constant(1 / K.sqrt(2), shape=shape, dtype=dtype) + + +# Aliases: +sqrt_init = SqrtInit +independent_filters = IndependentFilters +quaternion_independent_filters = QuaternionIndependentFilters +complex_init = ComplexInit +quaternion_init = QuaternionInit diff --git a/build/lib.linux-x86_64-2.7/complexnn/norm.py b/build/lib.linux-x86_64-2.7/complexnn/norm.py new file mode 100644 index 0000000..34d67db --- /dev/null +++ b/build/lib.linux-x86_64-2.7/complexnn/norm.py @@ -0,0 +1,271 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# +# Authors: Chiheb Trabelsi + +# +# Implementation of Layer Normalization and Complex Layer Normalization +# + +import numpy as np +from keras.layers import Layer, InputSpec +from keras import initializers, regularizers, constraints +import keras.backend as K +from .bn import ComplexBN as complex_normalization +from .bn import sqrt_init + +def layernorm(x, axis, epsilon, gamma, beta): + # assert self.built, 'Layer must be built before being called' + input_shape = K.shape(x) + reduction_axes = list(range(K.ndim(x))) + del reduction_axes[axis] + del reduction_axes[0] + broadcast_shape = [1] * K.ndim(x) + broadcast_shape[axis] = input_shape[axis] + broadcast_shape[0] = K.shape(x)[0] + + # Perform normalization: centering and reduction + + mean = K.mean(x, axis=reduction_axes) + broadcast_mean = K.reshape(mean, broadcast_shape) + x_centred = x - broadcast_mean + variance = K.mean(x_centred ** 2, axis=reduction_axes) + epsilon + broadcast_variance = K.reshape(variance, broadcast_shape) + + x_normed = x_centred / K.sqrt(broadcast_variance) + + # Perform scaling and shifting + + broadcast_shape_params = [1] * K.ndim(x) + broadcast_shape_params[axis] = K.shape(x)[axis] + broadcast_gamma = K.reshape(gamma, broadcast_shape_params) + broadcast_beta = K.reshape(beta, broadcast_shape_params) + + x_LN = broadcast_gamma * x_normed + broadcast_beta + + return x_LN + +class LayerNormalization(Layer): + + def __init__(self, + epsilon=1e-4, + axis=-1, + beta_init='zeros', + gamma_init='ones', + gamma_regularizer=None, + beta_regularizer=None, + **kwargs): + + self.supports_masking = True + self.beta_init = initializers.get(beta_init) + self.gamma_init = initializers.get(gamma_init) + self.epsilon = epsilon + self.axis = axis + self.gamma_regularizer = regularizers.get(gamma_regularizer) + self.beta_regularizer = regularizers.get(beta_regularizer) + + super(LayerNormalization, self).__init__(**kwargs) + + def build(self, input_shape): + self.input_spec = InputSpec(ndim=len(input_shape), + axes={self.axis: input_shape[self.axis]}) + shape = (input_shape[self.axis],) + + self.gamma = self.add_weight(shape, + initializer=self.gamma_init, + regularizer=self.gamma_regularizer, + name='{}_gamma'.format(self.name)) + self.beta = self.add_weight(shape, + initializer=self.beta_init, + regularizer=self.beta_regularizer, + name='{}_beta'.format(self.name)) + + self.built = True + + def call(self, x, mask=None): + assert self.built, 'Layer must be built before being called' + return layernorm(x, self.axis, self.epsilon, self.gamma, self.beta) + + def get_config(self): + config = {'epsilon': self.epsilon, + 'axis': self.axis, + 'gamma_regularizer': self.gamma_regularizer.get_config() if self.gamma_regularizer else None, + 'beta_regularizer': self.beta_regularizer.get_config() if self.beta_regularizer else None + } + base_config = super(LayerNormalization, self).get_config() + return dict(list(base_config.items()) + list(config.items())) + + +class ComplexLayerNorm(Layer): + def __init__(self, + epsilon=1e-4, + axis=-1, + center=True, + scale=True, + beta_initializer='zeros', + gamma_diag_initializer=sqrt_init, + gamma_off_initializer='zeros', + beta_regularizer=None, + gamma_diag_regularizer=None, + gamma_off_regularizer=None, + beta_constraint=None, + gamma_diag_constraint=None, + gamma_off_constraint=None, + **kwargs): + + self.supports_masking = True + self.epsilon = epsilon + self.axis = axis + self.center = center + self.scale = scale + self.beta_initializer = initializers.get(beta_initializer) + self.gamma_diag_initializer = initializers.get(gamma_diag_initializer) + self.gamma_off_initializer = initializers.get(gamma_off_initializer) + self.beta_regularizer = regularizers.get(beta_regularizer) + self.gamma_diag_regularizer = regularizers.get(gamma_diag_regularizer) + self.gamma_off_regularizer = regularizers.get(gamma_off_regularizer) + self.beta_constraint = constraints.get(beta_constraint) + self.gamma_diag_constraint = constraints.get(gamma_diag_constraint) + self.gamma_off_constraint = constraints.get(gamma_off_constraint) + super(ComplexLayerNorm, self).__init__(**kwargs) + + def build(self, input_shape): + + ndim = len(input_shape) + dim = input_shape[self.axis] + if dim is None: + raise ValueError('Axis ' + str(self.axis) + ' of ' + 'input tensor should have a defined dimension ' + 'but the layer received an input with shape ' + + str(input_shape) + '.') + self.input_spec = InputSpec(ndim=len(input_shape), + axes={self.axis: dim}) + + gamma_shape = (input_shape[self.axis] // 2,) + if self.scale: + self.gamma_rr = self.add_weight( + shape=gamma_shape, + name='gamma_rr', + initializer=self.gamma_diag_initializer, + regularizer=self.gamma_diag_regularizer, + constraint=self.gamma_diag_constraint + ) + self.gamma_ii = self.add_weight( + shape=gamma_shape, + name='gamma_ii', + initializer=self.gamma_diag_initializer, + regularizer=self.gamma_diag_regularizer, + constraint=self.gamma_diag_constraint + ) + self.gamma_ri = self.add_weight( + shape=gamma_shape, + name='gamma_ri', + initializer=self.gamma_off_initializer, + regularizer=self.gamma_off_regularizer, + constraint=self.gamma_off_constraint + ) + else: + self.gamma_rr = None + self.gamma_ii = None + self.gamma_ri = None + + if self.center: + self.beta = self.add_weight(shape=(input_shape[self.axis],), + name='beta', + initializer=self.beta_initializer, + regularizer=self.beta_regularizer, + constraint=self.beta_constraint) + else: + self.beta = None + + self.built = True + + def call(self, inputs): + input_shape = K.shape(inputs) + ndim = K.ndim(inputs) + reduction_axes = list(range(ndim)) + del reduction_axes[self.axis] + del reduction_axes[0] + input_dim = input_shape[self.axis] // 2 + mu = K.mean(inputs, axis=reduction_axes) + broadcast_mu_shape = [1] * ndim + broadcast_mu_shape[self.axis] = input_shape[self.axis] + broadcast_mu_shape[0] = K.shape(inputs)[0] + broadcast_mu = K.reshape(mu, broadcast_mu_shape) + if self.center: + input_centred = inputs - broadcast_mu + else: + input_centred = inputs + centred_squared = input_centred ** 2 + if (self.axis == 1 and ndim != 3) or ndim == 2: + centred_squared_real = centred_squared[:, :input_dim] + centred_squared_imag = centred_squared[:, input_dim:] + centred_real = input_centred[:, :input_dim] + centred_imag = input_centred[:, input_dim:] + elif ndim == 3: + centred_squared_real = centred_squared[:, :, :input_dim] + centred_squared_imag = centred_squared[:, :, input_dim:] + centred_real = input_centred[:, :, :input_dim] + centred_imag = input_centred[:, :, input_dim:] + elif self.axis == -1 and ndim == 4: + centred_squared_real = centred_squared[:, :, :, :input_dim] + centred_squared_imag = centred_squared[:, :, :, input_dim:] + centred_real = input_centred[:, :, :, :input_dim] + centred_imag = input_centred[:, :, :, input_dim:] + elif self.axis == -1 and ndim == 5: + centred_squared_real = centred_squared[:, :, :, :, :input_dim] + centred_squared_imag = centred_squared[:, :, :, :, input_dim:] + centred_real = input_centred[:, :, :, :, :input_dim] + centred_imag = input_centred[:, :, :, :, input_dim:] + else: + raise ValueError( + 'Incorrect Layernorm combination of axis and dimensions. axis should be either 1 or -1. ' + 'axis: ' + str(self.axis) + '; ndim: ' + str(ndim) + '.' + ) + if self.scale: + Vrr = K.mean( + centred_squared_real, + axis=reduction_axes + ) + self.epsilon + Vii = K.mean( + centred_squared_imag, + axis=reduction_axes + ) + self.epsilon + # Vri contains the real and imaginary covariance for each feature map. + Vri = K.mean( + centred_real * centred_imag, + axis=reduction_axes, + ) + elif self.center: + Vrr = None + Vii = None + Vri = None + else: + raise ValueError('Error. Both scale and center in batchnorm are set to False.') + + return complex_normalization( + input_centred, Vrr, Vii, Vri, + self.beta, self.gamma_rr, self.gamma_ri, + self.gamma_ii, self.scale, self.center, + layernorm=True, axis=self.axis + ) + + def get_config(self): + config = { + 'axis': self.axis, + 'epsilon': self.epsilon, + 'center': self.center, + 'scale': self.scale, + 'beta_initializer': initializers.serialize(self.beta_initializer), + 'gamma_diag_initializer': initializers.serialize(self.gamma_diag_initializer), + 'gamma_off_initializer': initializers.serialize(self.gamma_off_initializer), + 'beta_regularizer': regularizers.serialize(self.beta_regularizer), + 'gamma_diag_regularizer': regularizers.serialize(self.gamma_diag_regularizer), + 'gamma_off_regularizer': regularizers.serialize(self.gamma_off_regularizer), + 'beta_constraint': constraints.serialize(self.beta_constraint), + 'gamma_diag_constraint': constraints.serialize(self.gamma_diag_constraint), + 'gamma_off_constraint': constraints.serialize(self.gamma_off_constraint), + } + base_config = super(ComplexLayerNorm, self).get_config() + return dict(list(base_config.items()) + list(config.items())) diff --git a/build/lib.linux-x86_64-2.7/complexnn/pool.py b/build/lib.linux-x86_64-2.7/complexnn/pool.py new file mode 100644 index 0000000..e976e54 --- /dev/null +++ b/build/lib.linux-x86_64-2.7/complexnn/pool.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# +# Authors: Olexa Bilaniuk +# + +import keras.backend as KB +import keras.engine as KE +import keras.layers as KL +import keras.optimizers as KO +import theano as T +import theano.ifelse as TI +import theano.tensor as TT +import theano.tensor.fft as TTF +import numpy as np + + +# +# Spectral Pooling Layer +# + +class SpectralPooling1D(KL.Layer): + def __init__(self, topf=(0,)): + super(SpectralPooling1D, self).__init__() + if "topf" in kwargs: + self.topf = (int (kwargs["topf" ][0]),) + self.topf = (self.topf[0]//2,) + elif "gamma" in kwargs: + self.gamma = (float(kwargs["gamma"][0]),) + self.gamma = (self.gamma[0]/2,) + else: + raise RuntimeError("Must provide either topf= or gamma= !") + def call(self, x, mask=None): + xshape = x._keras_shape + if hasattr(self, "topf"): + topf = self.topf + else: + if KB.image_data_format() == "channels_first": + topf = (int(self.gamma[0]*xshape[2]),) + else: + topf = (int(self.gamma[0]*xshape[1]),) + + if KB.image_data_format() == "channels_first": + if topf[0] > 0 and xshape[2] >= 2*topf[0]: + mask = [1]*(topf[0] ) +\ + [0]*(xshape[2] - 2*topf[0]) +\ + [1]*(topf[0] ) + mask = [[mask]] + mask = np.asarray(mask, dtype=KB.floatx()).transpose((0,1,2)) + mask = KB.constant(mask) + x *= mask + else: + if topf[0] > 0 and xshape[1] >= 2*topf[0]: + mask = [1]*(topf[0] ) +\ + [0]*(xshape[1] - 2*topf[0]) +\ + [1]*(topf[0] ) + mask = [[mask]] + mask = np.asarray(mask, dtype=KB.floatx()).transpose((0,2,1)) + mask = KB.constant(mask) + x *= mask + + return x +class SpectralPooling2D(KL.Layer): + def __init__(self, **kwargs): + super(SpectralPooling2D, self).__init__() + if "topf" in kwargs: + self.topf = (int (kwargs["topf" ][0]), int (kwargs["topf" ][1])) + self.topf = (self.topf[0]//2, self.topf[1]//2) + elif "gamma" in kwargs: + self.gamma = (float(kwargs["gamma"][0]), float(kwargs["gamma"][1])) + self.gamma = (self.gamma[0]/2, self.gamma[1]/2) + else: + raise RuntimeError("Must provide either topf= or gamma= !") + def call(self, x, mask=None): + xshape = x._keras_shape + if hasattr(self, "topf"): + topf = self.topf + else: + if KB.image_data_format() == "channels_first": + topf = (int(self.gamma[0]*xshape[2]), int(self.gamma[1]*xshape[3])) + else: + topf = (int(self.gamma[0]*xshape[1]), int(self.gamma[1]*xshape[2])) + + if KB.image_data_format() == "channels_first": + if topf[0] > 0 and xshape[2] >= 2*topf[0]: + mask = [1]*(topf[0] ) +\ + [0]*(xshape[2] - 2*topf[0]) +\ + [1]*(topf[0] ) + mask = [[[mask]]] + mask = np.asarray(mask, dtype=KB.floatx()).transpose((0,1,3,2)) + mask = KB.constant(mask) + x *= mask + if topf[1] > 0 and xshape[3] >= 2*topf[1]: + mask = [1]*(topf[1] ) +\ + [0]*(xshape[3] - 2*topf[1]) +\ + [1]*(topf[1] ) + mask = [[[mask]]] + mask = np.asarray(mask, dtype=KB.floatx()).transpose((0,1,2,3)) + mask = KB.constant(mask) + x *= mask + else: + if topf[0] > 0 and xshape[1] >= 2*topf[0]: + mask = [1]*(topf[0] ) +\ + [0]*(xshape[1] - 2*topf[0]) +\ + [1]*(topf[0] ) + mask = [[[mask]]] + mask = np.asarray(mask, dtype=KB.floatx()).transpose((0,3,1,2)) + mask = KB.constant(mask) + x *= mask + if topf[1] > 0 and xshape[2] >= 2*topf[1]: + mask = [1]*(topf[1] ) +\ + [0]*(xshape[2] - 2*topf[1]) +\ + [1]*(topf[1] ) + mask = [[[mask]]] + mask = np.asarray(mask, dtype=KB.floatx()).transpose((0,1,3,2)) + mask = KB.constant(mask) + x *= mask + + return x + + +if __name__ == "__main__": + import cv2, sys + import __main__ as SP + import fft as CF + + # Build Model + x = i = KL.Input(shape=(6,512,512)) + f = CF.FFT2()(x) + p = SP.SpectralPooling2D(gamma=[0.15,0.15])(f) + o = CF.IFFT2()(p) + + model = KE.Model([i], [f,p,o]) + model.compile("sgd", "mse") + + # Use it + img = cv2.imread(sys.argv[1]) + imgBatch = img[np.newaxis,...].transpose((0,3,1,2)) + imgBatch = np.concatenate([imgBatch, np.zeros_like(imgBatch)], axis=1) + f,p,o = model.predict(imgBatch) + ffted = np.sqrt(np.sum(f[:,:3]**2 + f[:,3:]**2, axis=1)) + ffted = ffted .transpose((1,2,0))/255 + pooled = np.sqrt(np.sum(p[:,:3]**2 + p[:,3:]**2, axis=1)) + pooled = pooled.transpose((1,2,0))/255 + filtered = np.clip(o,0,255).transpose((0,2,3,1))[0,:,:,:3].astype("uint8") + + # Display it + cv2.imshow("Original", img) + cv2.imshow("FFT", ffted) + cv2.imshow("Pooled", pooled) + cv2.imshow("Filtered", filtered) + cv2.waitKey(0) diff --git a/build/lib.linux-x86_64-2.7/complexnn/utils.py b/build/lib.linux-x86_64-2.7/complexnn/utils.py new file mode 100644 index 0000000..4604980 --- /dev/null +++ b/build/lib.linux-x86_64-2.7/complexnn/utils.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Contributors: Titouan Parcollet +# Authors: Dmitriy Serdyuk, Olexa Bilaniuk, Chiheb Trabelsi + +import keras.backend as K +from keras.layers import Layer, Lambda + +################ +# Quaternions # +################ + +def get_rpart(x): + image_format = K.image_data_format() + ndim = K.ndim(x) + input_shape = K.shape(x) + + if (image_format == 'channels_first' and ndim != 3) or ndim == 2: + input_dim = input_shape[1] // 4 + return x[:, :input_dim] + + input_dim = input_shape[-1] // 4 + if ndim == 3: + return x[:, :, :input_dim] + elif ndim == 4: + return x[:, :, :, :input_dim] + elif ndim == 5: + return x[:, :, :, :, :input_dim] + +def get_ipart(x): + image_format = K.image_data_format() + ndim = K.ndim(x) + input_shape = K.shape(x) + + if (image_format == 'channels_first' and ndim != 3) or ndim == 2: + input_dim = input_shape[1] // 4 + return x[:, input_dim:input_dim*2] + + input_dim = input_shape[-1] // 4 + if ndim == 3: + return x[:, :, input_dim:input_dim*2] + elif ndim == 4: + return x[:, :, :, input_dim:input_dim*2] + elif ndim == 5: + return x[:, :, :, :, input_dim:input_dim*2] + +def get_jpart(x): + image_format = K.image_data_format() + ndim = K.ndim(x) + input_shape = K.shape(x) + + if (image_format == 'channels_first' and ndim != 3) or ndim == 2: + input_dim = input_shape[1] // 4 + return x[:, input_dim*2:input_dim*3] + + input_dim = input_shape[-1] // 4 + if ndim == 3: + return x[:, :, input_dim*2:input_dim*3] + elif ndim == 4: + return x[:, :, :, input_dim*2:input_dim*3] + elif ndim == 5: + return x[:, :, :, :, input_dim*2:input_dim*3] + +def get_kpart(x): + image_format = K.image_data_format() + ndim = K.ndim(x) + input_shape = K.shape(x) + + if (image_format == 'channels_first' and ndim != 3) or ndim == 2: + input_dim = input_shape[1] // 4 + return x[:, input_dim*3:] + + input_dim = input_shape[-1] // 4 + if ndim == 3: + return x[:, :, input_dim*3:] + elif ndim == 4: + return x[:, :, :, input_dim*3:] + elif ndim == 5: + return x[:, :, :, :, input_dim*3:] + +class GetR(Layer): + def call(self, inputs): + return get_rpart(inputs) + def compute_output_shape(self, input_shape): + return getpart_quaternion_output_shape(input_shape) +class GetI(Layer): + def call(self, inputs): + return get_ipart(inputs) + def compute_output_shape(self, input_shape): + return getpart_quaternion_output_shape(input_shape) +class GetJ(Layer): + def call(self, inputs): + return get_jpart(inputs) + def compute_output_shape(self, input_shape): + return getpart_quaternion_output_shape(input_shape) +class GetK(Layer): + def call(self, inputs): + return get_kpart(inputs) + def compute_output_shape(self, input_shape): + return getpart_quaternion_output_shape(input_shape) + +def getpart_quaternion_output_shape(input_shape): + returned_shape = list(input_shape[:]) + image_format = K.image_data_format() + ndim = len(returned_shape) + + if (image_format == 'channels_first' and ndim != 3) or ndim == 2: + axis = 1 + else: + axis = -1 + + returned_shape[axis] = returned_shape[axis] // 4 + + return tuple(returned_shape) + + +# +# GetReal/GetImag Lambda layer Implementation +# + + +def get_realpart(x): + image_format = K.image_data_format() + ndim = K.ndim(x) + input_shape = K.shape(x) + + if (image_format == 'channels_first' and ndim != 3) or ndim == 2: + input_dim = input_shape[1] // 2 + return x[:, :input_dim] + + input_dim = input_shape[-1] // 2 + if ndim == 3: + return x[:, :, :input_dim] + elif ndim == 4: + return x[:, :, :, :input_dim] + elif ndim == 5: + return x[:, :, :, :, :input_dim] + + +def get_imagpart(x): + image_format = K.image_data_format() + ndim = K.ndim(x) + input_shape = K.shape(x) + + if (image_format == 'channels_first' and ndim != 3) or ndim == 2: + input_dim = input_shape[1] // 2 + return x[:, input_dim:] + + input_dim = input_shape[-1] // 2 + if ndim == 3: + return x[:, :, input_dim:] + elif ndim == 4: + return x[:, :, :, input_dim:] + elif ndim == 5: + return x[:, :, :, :, input_dim:] + + +def get_abs(x): + real = get_realpart(x) + imag = get_imagpart(x) + + return K.sqrt(real * real + imag * imag) + + +def getpart_output_shape(input_shape): + returned_shape = list(input_shape[:]) + image_format = K.image_data_format() + ndim = len(returned_shape) + + if (image_format == 'channels_first' and ndim != 3) or ndim == 2: + axis = 1 + else: + axis = -1 + + returned_shape[axis] = returned_shape[axis] // 2 + + return tuple(returned_shape) + + +class GetReal(Layer): + def call(self, inputs): + return get_realpart(inputs) + def compute_output_shape(self, input_shape): + return getpart_output_shape(input_shape) +class GetImag(Layer): + def call(self, inputs): + return get_imagpart(inputs) + def compute_output_shape(self, input_shape): + return getpart_output_shape(input_shape) +class GetAbs(Layer): + def call(self, inputs): + return get_abs(inputs) + def compute_output_shape(self, input_shape): + return getpart_output_shape(input_shape) + diff --git a/build/lib.linux-x86_64-2.7/musicnet/__init__.py b/build/lib.linux-x86_64-2.7/musicnet/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/build/lib.linux-x86_64-2.7/musicnet/callbacks.py b/build/lib.linux-x86_64-2.7/musicnet/callbacks.py new file mode 100644 index 0000000..5e9fbba --- /dev/null +++ b/build/lib.linux-x86_64-2.7/musicnet/callbacks.py @@ -0,0 +1,107 @@ +# -*- coding: utf-8 -*- + +# +# Authors: Dmitriy Serdyuk + +import os +import numpy as np +import h5py + +from time import time +from os import path +from sklearn.metrics import ( + precision_recall_curve, average_precision_score, log_loss) + +import keras +from keras.callbacks import Callback, ModelCheckpoint, LearningRateScheduler + + +class SaveLastModel(Callback): + def __init__(self, workdir, period=10, name=''): + self.name = name + self.workdir = workdir + self.chkptsdir = path.join(self.workdir, "chkpts") + + if not path.isdir(self.chkptsdir): + os.mkdir(self.chkptsdir) + + self.period_of_epochs = period + self.link_filename = path.join(self.chkptsdir, + "model_checkpoint.hdf5") + + def on_epoch_end(self, epoch, logs=None): + if (epoch + 1) % self.period_of_epochs == 0: + # Filenames + base_hdf5_filename = "model{}_checkpoint{:06d}.hdf5".format( + self.name, epoch + 1) + base_yaml_filename = "model{}_checkpoint{:06d}.yaml".format( + self.name, epoch + 1) + hdf5_filename = path.join(self.chkptsdir, base_hdf5_filename) + yaml_filename = path.join(self.chkptsdir, base_yaml_filename) + + # YAML + yaml_model = self.model.to_yaml() + with open(yaml_filename, "w") as yaml_file: + yaml_file.write(yaml_model) + + # HDF5 + keras.models.save_model(self.model, hdf5_filename) + with h5py.File(hdf5_filename, "r+") as f: + f.require_dataset("initialEpoch", (), "uint64", True)[...] = int(epoch+1) + f.flush() + + # Symlink to new HDF5 file, then atomically rename and replace. + os.symlink(base_hdf5_filename, self.link_filename + ".rename") + os.rename (self.link_filename + ".rename", + self.link_filename) + + +class Performance(Callback): + def __init__(self, logger): + self.logger = logger + + def on_epoch_begin(self, epoch, logs=None): + self.timestamps = [] + + def on_epoch_end(self, epoch, logs=None): + t = np.asarray(self.timestamps, dtype=np.float64) + train_function_time = float(np.mean(t[ :,1] - t[:,0])) + load_data_time = float(np.mean(t[1:,0] - t[:-1, 1])) + self.logger.log( + {'epoch': epoch, 'train_function_time': train_function_time}) + self.logger.log({'epoch': epoch, 'load_data_time': load_data_time}) + + def on_batch_begin(self, epoch, logs=None): + self.timestamps += [[time(), time()]] + + def on_batch_end(self, epoch, logs=None): + self.timestamps[-1][-1] = time() + + +class Validation(Callback): + def __init__(self, x, y, name, logger): + self.x = x + self.y = y + self.name = name + self.logger = logger + + def evaluate(self): + pr = self.model.predict(self.x) + average_precision = average_precision_score( + self.y.flatten(), pr.flatten()) + loss = log_loss(self.y.flatten(), pr.flatten()) + return average_precision, loss + + def on_train_begin(self, logs=None): + average_precision, loss = self.evaluate() + self.logger.log( + {'epoch': 0, self.name + "_avg_precision": average_precision, + self.name + "_loss": loss}) + + def on_epoch_end(self, epoch, logs=None): + average_precision, loss = self.evaluate() + self.logger.log( + {'epoch': epoch + 1, + self.name + "_avg_precision": average_precision, + self.name + "_loss": loss}) + diff --git a/build/lib.linux-x86_64-2.7/musicnet/dataset.py b/build/lib.linux-x86_64-2.7/musicnet/dataset.py new file mode 100644 index 0000000..cc0e497 --- /dev/null +++ b/build/lib.linux-x86_64-2.7/musicnet/dataset.py @@ -0,0 +1,217 @@ +# -*- coding: utf-8 -*- + +# +# Authors: Dmitriy Serdyuk + +import itertools +import numpy + +from six.moves import range +from itertools import chain +from scipy import fft +from scipy.signal import stft + + +FS = 44100 # samples/second +DEFAULT_WINDOW_SIZE = 2048 # fourier window size +OUTPUT_SIZE = 128 # number of distinct notes +STRIDE = 512 # samples between windows +WPS = FS / float(512) # windows/second + + +class MusicNet(object): + def __init__(self, filename, in_memory=True, window_size=4096, + output_size=84, feature_size=1024, sample_freq=11000, + complex_=False, fourier=False, stft=False, fast_load=False, + rng=None, seed=123): + if not in_memory: + raise NotImplementedError + self.filename = filename + + self.window_size = window_size + self.output_size = output_size + self.feature_size = feature_size + self.sample_freq = sample_freq + self.complex_ = complex_ + self.fourier = fourier + self.stft = stft + self.fast_load = fast_load + + if rng is not None: + self.rng = rng + else: + self.rng = numpy.random.RandomState(seed) + + self._train_data = {} + self._valid_data = {} + self._test_data = {} + self._loaded = False + + self._eval_sets = {} + + def splits(self): + with open(self.filename, 'rb') as f: + # This should be fast + all_inds = numpy.load(f).keys() + test_inds = ['2303', '2382', '1819'] + valid_inds = ['2131', '2384', '1792', + '2514', '2567', '1876'] + train_inds = [ind for ind in all_inds + if ind not in test_inds and ind not in test_inds] + return train_inds, valid_inds, test_inds + + @classmethod + def note_to_class(cls, note): + return note - 21 + + @property + def train_data(self): + if self._train_data == {}: + self.load() + return self._train_data + + @property + def valid_data(self): + if self._valid_data == {}: + self.load() + return self._valid_data + + @property + def test_data(self): + if self._test_data == {}: + self.load() + return self._test_data + + def load(self, filename=None, reload=False): + if filename is None: + filename = self.filename + if self._loaded and not reload: + return + + with open(filename, 'rb') as f: + train_inds, valid_inds, test_inds = self.splits() + data_file = numpy.load(f) + if self.fast_load: + train_inds = train_inds[:6] + train_data = {} + for ind in chain(train_inds, valid_inds, test_inds): + train_data[ind] = data_file[ind] + else: + train_data = dict(data_file) + + # test set + test_data = {} + for ind in test_inds: + if ind in train_data: + test_data[ind] = train_data.pop(ind) + + # valid set + valid_data = {} + for ind in valid_inds: + valid_data[ind] = train_data.pop(ind) + + self._train_data = train_data + self._valid_data = valid_data + self._test_data = test_data + + def construct_eval_set(self, data, step=128): + n_files = len(data) + pos_per_file = 7500 + features = numpy.empty([n_files * pos_per_file, self.window_size]) + outputs = numpy.zeros([n_files * pos_per_file, self.output_size]) + + features_ind = 0 + labels_ind = 1 + + for i, ind in enumerate(data): + print(ind) + audio = data[ind][features_ind] + + for j in range(pos_per_file): + if j % 1000 == 0: + print(j) + # start from one second to give us some wiggle room for larger + # segments + index = self.sample_freq + j * step + features[pos_per_file * i + j] = audio[index: + index + self.window_size] + + # label stuff that's on in the center of the window + s = int((index + self.window_size / 2)) + for label in data[ind][labels_ind][s]: + note = label.data[1] + outputs[pos_per_file * i + j, self.note_to_class(note)] = 1 + return features, outputs + + @property + def feature_dim(self): + dummy_features = numpy.zeros((1, self.window_size, 1)) + dummy_output = numpy.zeros((1, self.output_size)) + dummy_features, _ = self.aggregate_raw_batch( + dummy_features, dummy_output) + return dummy_features.shape[1:] + + def aggregate_raw_batch(self, features, output): + """Aggregate batch. + + All post processing goes here. + + Parameters: + ----------- + features : 3D float tensor + Input tensor + output : 2D integer tensor + Output classes + + """ + channels = 2 if self.complex_ else 1 + features_out = numpy.zeros( + [features.shape[0], self.window_size, channels]) + if self.fourier: + if self.complex_: + data = fft(features, axis=1) + features_out[:, :, 0] = numpy.real(data[:, :, 0]) + features_out[:, :, 1] = numpy.imag(data[:, :, 0]) + else: + data = numpy.abs(fft(features, axis=1)) + features_out = data + elif self.stft: + _, _, data = stft(features, nperseg=120, noverlap=60, axis=1) + length = data.shape[1] + n_feats = data.shape[3] + if self.complex_: + features_out = numpy.zeros( + [len(self.train_data), length, n_feats * 2]) + features_out[:, :, :n_feats] = numpy.real(data) + features_out[:, :, n_feats:] = numpy.imag(data) + else: + features_out = numpy.abs(data[:, :, 0, :]) + else: + features_out = features + return features_out, output + + def train_iterator(self): + features = numpy.zeros([len(self.train_data), self.window_size]) + + while True: + output = numpy.zeros([len(self.train_data), self.output_size]) + for j, ind in enumerate(self.train_data): + s = self.rng.randint( + self.window_size / 2, + len(self.train_data[ind][0]) - self.window_size / 2) + data = self.train_data[ind][0][s - self.window_size / 2: + s + self.window_size / 2] + features[j, :] = data + for label in self.train_data[ind][1][s]: + note = label.data[1] + output[j, self.note_to_class(note)] = 1 + yield self.aggregate_raw_batch(features[:, :, None], output) + + def eval_set(self, set_name): + if not self._eval_sets: + for name in ['valid', 'test']: + data = self.valid_data if name == 'valid' else self.test_data + x, y = self.construct_eval_set(data) + x, y = self.aggregate_raw_batch(x[:, :, None], y) + self._eval_sets[name] = (x, y) + return self._eval_sets[set_name] diff --git a/build/scripts-2.7/resample.py b/build/scripts-2.7/resample.py new file mode 100755 index 0000000..b96cf19 --- /dev/null +++ b/build/scripts-2.7/resample.py @@ -0,0 +1,51 @@ +#!/users/parcollet/.pyenv/versions/2.7.13/bin/python +# -*- coding: utf-8 -*- + +# +# Authors: Dmitriy Serdyuk + +from __future__ import print_function +import numpy +import argparse + +from intervaltree import Interval, IntervalTree +from resampy import resample + + +def resample_musicnet(file_in, file_out, frame_rate, frame_rate_out): + ratio = frame_rate_out / float(frame_rate) + print('.. resampling {} ({}Hz) into {} ({}Hz)'.format( + file_in, frame_rate, file_out, frame_rate_out)) + print('.. sampling with ratio {}'.format(ratio)) + + resampled_data = {} + with open(file_in, 'rb') as f_in: + data_in = numpy.load(file_in) + n_files = len(data_in.keys()) + for i, key in enumerate(data_in): + print('.. aggregating {} ({} / {})'.format(key, i, n_files)) + data = data_in[key] + data[0] = resample(data[0], frame_rate, frame_rate_out) + resampled_intervals = [] + for interval in data[1]: + resampled_begin = int(interval.begin * ratio) + resampled_end = int(interval.end * ratio) + resampled_interval = Interval( + resampled_begin, resampled_end, interval.data) + resampled_intervals.append(resampled_interval) + data[1] = IntervalTree(resampled_intervals) + resampled_data[key] = data + + print('.. saving output') + with open(file_out, 'wb') as f_out: + numpy.savez(f_out, **resampled_data) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('file_in') + parser.add_argument('file_out') + parser.add_argument('frame_rate', type=int) + parser.add_argument('frame_rate_out', type=int) + + resample_musicnet(**parser.parse_args().__dict__) diff --git a/build/scripts-2.7/run.py b/build/scripts-2.7/run.py new file mode 100755 index 0000000..764d959 --- /dev/null +++ b/build/scripts-2.7/run.py @@ -0,0 +1,251 @@ +#!/u/parcollt/anaconda2/bin/python +# -*- coding: utf-8 -*- +# Contributors: Titouan Parcollet +# Authors: Olexa Bilaniuk + +# Imports. +import sys; sys.path += [".", ".."] +import argparse as Ap +import logging as L +import numpy as np +import os, pdb, sys +import time +import tensorflow as tf +from keras.backend.tensorflow_backend import set_session +__version__ = "0.0.0" + + + +# +# Message Formatter +# + +class MsgFormatter(L.Formatter): + """Message Formatter + + Formats messages with time format YYYY-MM-DD HH:MM:SS.mmm TZ + """ + + def formatTime(self, record, datefmt): + t = record.created + timeFrac = abs(t-long(t)) + timeStruct = time.localtime(record.created) + timeString = "" + timeString += time.strftime("%F %T", timeStruct) + timeString += "{:.3f} ".format(timeFrac)[1:] + timeString += time.strftime("%Z", timeStruct) + return timeString + + + +############################################################################################################# +############################## Subcommands ################################## +############################################################################################################# + +class Subcommand(object): + name = None + + @classmethod + def addArgParser(cls, subp, *args, **kwargs): + argp = subp.add_parser(cls.name, usage=cls.__doc__, *args, **kwargs) + cls.addArgs(argp) + argp.set_defaults(__subcmdfn__=cls.run) + return argp + + @classmethod + def addArgs(cls, argp): + pass + + @classmethod + def run(cls, d): + pass + + +class Screw(Subcommand): + """Screw around with me in Screw(Subcommand).""" + name = "screw" + + @classmethod + def run(cls, d): + print(cls.__doc__) + + +class Train(Subcommand): + name = "train" + + LOGLEVELS = {"none":L.NOTSET, "debug": L.DEBUG, "info": L.INFO, + "warn":L.WARN, "err": L.ERROR, "crit": L.CRITICAL} + + + @classmethod + def addArgs(cls, argp): + argp.add_argument("-d", "--datadir", default=".", type=str, + help="Path to datasets directory.") + argp.add_argument("-w", "--workdir", default=".", type=str, + help="Path to the workspace directory for this experiment.") + argp.add_argument("-l", "--loglevel", default="info", type=str, + choices=cls.LOGLEVELS.keys(), + help="Logging severity level.") + argp.add_argument("-s", "--seed", default=0xe4223644e98b8e64, type=long, + help="Seed for PRNGs.") + argp.add_argument("--summary", action="store_true", + help="""Print a summary of the network.""") + argp.add_argument("--batchnorm", default=0, type=int, + help="0 = No batchNorm; 1 = BatchNorm") + argp.add_argument("--dataset", default="cifar10", type=str, + choices=["timit","decoda","cifar10", "cifar100", "svhn"], + help="Dataset Selection.") + argp.add_argument("--model", default="real", type=str, + choices=["complex","quaternion", "real"], + help="Dataset Selection.") + argp.add_argument("--dropout", default=0, type=float, + help="Dropout probability.") + argp.add_argument("-n", "--num-epochs", default=1000, type=int, + help="Number of epochs") + argp.add_argument("-b", "--batch-size", default=64, type=int, + help="Batch Size") + argp.add_argument("--start-filter", "--sf", default=11, type=int, + help="Number of feature maps in starting stage") + argp.add_argument("--num-blocks", "--nb", default=10, type=int, + help="Number of filters in initial block") + argp.add_argument("--spectral-param", action="store_true", + help="""Use spectral parametrization.""") + argp.add_argument("--spectral-pool-gamma", default=0.50, type=float, + help="""Use spectral pooling, preserving a fraction gamma of frequencies""") + argp.add_argument("--spectral-pool-scheme", default="none", type=str, + choices=["none", "stagemiddle", "proj", "nodownsample"], + help="""Spectral pooling scheme""") + argp.add_argument("--act", default="relu", type=str, + choices=["relu"], + help="Activation.") + argp.add_argument("--aact", default="modrelu", type=str, + choices=["modrelu"], + help="Advanced Activation.") + argp.add_argument("--no-validation", action="store_true", + help="Do not create a separate validation set.") + argp.add_argument("--comp-init", default='complex', type=str, + help="Initializer for the complex kernel.") + argp.add_argument("--quat-init", default='quaternion', type=str, + help="Initializer for the quaternion kernel.") + optp = argp.add_argument_group("Optimizers", "Tunables for all optimizers") + optp.add_argument("--optimizer", "--opt", default="nag", type=str, + choices=["sgd", "nag", "adam", "rmsprop"], + help="Optimizer selection.") + optp.add_argument("--clipnorm", "--cn", default=1.0, type=float, + help="The norm of the gradient will be clipped at this magnitude.") + optp.add_argument("--clipval", "--cv", default=1.0, type=float, + help="The values of the gradients will be individually clipped at this magnitude.") + optp.add_argument("--l1", default=0, type=float, + help="L1 penalty.") + optp.add_argument("--l2", default=0, type=float, + help="L2 penalty.") + optp.add_argument("--lr", default=1e-4, type=float, + help="Master learning rate for optimizers.") + optp.add_argument("--momentum", "--mom", default=0.9, type=float, + help="Momentum for optimizers supporting momentum.") + optp.add_argument("--decay", default=0, type=float, + help="Learning rate decay for optimizers.") + optp.add_argument("--schedule", default="default", type=str, + help="Learning rate schedule") + optp = argp.add_argument_group("Adam", "Tunables for Adam optimizer") + optp.add_argument("--beta1", default=0.9, type=float, + help="Beta1 for Adam.") + optp.add_argument("--beta2", default=0.999, type=float, + help="Beta2 for Adam.") + optp.add_argument("--device", default="0", type=str, + help="CUDA Device, starting at 0.") + optp.add_argument("--gpus", default=1, type=int, + help="Number of GPUs to be used, starting at 1") + optp.add_argument("--memory", default=1.0, type=float, + help="Memory to be allocated on the selected device, only for tensorflow backend, from 0 to 1") + optp.add_argument("--save-prefix", default="", type=str, + help="Save prefix for resuming and saving best model") + optp.add_argument("--seg", default="chiheb", type=str, + choices=["chiheb", "parcollet"], help="Segmentation to be use on quaternions, \ + following NIPS Deep Complex Networks or SLT Quaternion Neural Networks") + optp.add_argument("--output-type", default="real", type=str, + choices=["quaternion", "real"], + help="Type of the dense output layer") + + @classmethod + def run(cls, d): + if not os.path.isdir(d.workdir): + os.mkdir(d.workdir) + + logDir = os.path.join(d.workdir, "logs") + if not os.path.isdir(logDir): + os.mkdir(logDir) + + logFormatter = MsgFormatter ("[%(asctime)s ~~ %(levelname)-8s] %(message)s") + + stdoutLogSHandler = L.StreamHandler(sys.stdout) + stdoutLogSHandler .setLevel (cls.LOGLEVELS[d.loglevel]) + stdoutLogSHandler .setFormatter (logFormatter) + defltLogger = L.getLogger () + defltLogger .setLevel (cls.LOGLEVELS[d.loglevel]) + defltLogger .addHandler (stdoutLogSHandler) + + trainLogFilename = os.path.join(d.workdir, "logs", "train.txt") + trainLogFHandler = L.FileHandler (trainLogFilename, "a", "UTF-8", delay=True) + trainLogFHandler .setLevel (cls.LOGLEVELS[d.loglevel]) + trainLogFHandler .setFormatter (logFormatter) + trainLogger = L.getLogger ("train") + trainLogger .setLevel (cls.LOGLEVELS[d.loglevel]) + trainLogger .addHandler (trainLogFHandler) + + entryLogFilename = os.path.join(d.workdir, "logs", "entry.txt") + entryLogFHandler = L.FileHandler (entryLogFilename, "a", "UTF-8", delay=True) + entryLogFHandler .setLevel (cls.LOGLEVELS[d.loglevel]) + entryLogFHandler .setFormatter (logFormatter) + entryLogger = L.getLogger ("entry") + entryLogger .setLevel (cls.LOGLEVELS[d.loglevel]) + entryLogger .addHandler (entryLogFHandler) + + np.random.seed(d.seed % 2**32) + + import training;training.train(d) + + + + +############################################################################################################# +############################## Argument Parsers ################################# +############################################################################################################# + +def getArgParser(prog): + argp = Ap.ArgumentParser(prog = prog, + usage = None, + description = None, + epilog = None, + version = __version__) + subp = argp.add_subparsers() + argp.set_defaults(argp=argp) + argp.set_defaults(subp=subp) + + # Add global args to argp here? + # ... + + + # Add subcommands + for v in globals().itervalues(): + if(isinstance(v, type) and + issubclass(v, Subcommand) and + v != Subcommand): + v.addArgParser(subp) + + # Return argument parser. + return argp + + + +############################################################################################################# +############################## Main ################################## +############################################################################################################# + +def main(argv): + sys.setrecursionlimit(10000) + d = getArgParser(argv[0]).parse_args(argv[1:]) + return d.__subcmdfn__(d) +if __name__ == "__main__": + main(sys.argv) + diff --git a/build/scripts-2.7/train.py b/build/scripts-2.7/train.py new file mode 100755 index 0000000..c78c4fc --- /dev/null +++ b/build/scripts-2.7/train.py @@ -0,0 +1,142 @@ +#!/users/parcollet/.pyenv/versions/2.7.13/bin/python +# -*- coding: utf-8 -*- + +# +# Authors: Dmitriy Serdyuk + +from __future__ import print_function +import numpy +import numpy as np +from os import path +import argparse +import mimir + +import keras + +import musicnet.models.complex +from musicnet.callbacks import ( + SaveLastModel, Performance, Validation, LearningRateScheduler) +from musicnet.dataset import MusicNet +from musicnet import models + + +# input dimensions +d = 16384 / 4 +window_size = d +# number of notes +m = 84 +step = 512 +step = step / 4 + + +def schedule(epoch): + if epoch >= 0 and epoch < 10: + lrate = 1e-3 + if epoch == 0: + print('\ncurrent learning rate value is ' + str(lrate)) + elif epoch >= 10 and epoch < 100: + lrate = 1e-4 + if epoch == 10: + print('\ncurrent learning rate value is ' + str(lrate)) + elif epoch >= 100 and epoch < 120: + lrate = 5e-5 + if epoch == 100: + print('\ncurrent learning rate value is ' + str(lrate)) + elif epoch >= 120 and epoch < 150: + lrate = 1e-5 + if epoch == 120: + print('\ncurrent learning rate value is ' + str(lrate)) + elif epoch >= 150: + lrate = 1e-6 + if epoch == 150: + print('\ncurrent learning rate value is ' + str(lrate)) + return lrate + + +def get_model(model, feature_dim): + if model.startswith('complex'): + complex_ = True + model = model.split('_')[1] + else: + complex_ = False + if complex_: + model_module = models.complex + print('.. complex network') + else: + model_module = models + if model == 'mlp': + print('.. using MLP') + return model_module.get_mlp(window_size=numpy.prod(feature_dim)) + elif model == 'shallow_convnet': + print('.. using shallow convnet') + return model_module.get_shallow_convnet(window_size=feature_dim[0], + channels=feature_dim[1]) + elif model == 'deep_convnet': + print('.. using deep convnet') + return model_module.get_deep_convnet(window_size=feature_dim[0], + channels=feature_dim[1]) + else: + raise ValueError + + +def main(model_name, in_memory, complex_, model, local_data, epochs, fourier, + stft, fast_load): + rng = numpy.random.RandomState(123) + + # Warning: the full dataset is over 40GB. Make sure you have enough RAM! + # This can take a few minutes to load + if in_memory: + print('.. loading train data') + dataset = MusicNet(local_data, complex_=complex_, fourier=fourier, + stft=stft, rng=rng, fast_load=fast_load) + dataset.load() + print('.. train data loaded') + Xvalid, Yvalid = dataset.eval_set('valid') + Xtest, Ytest = dataset.eval_set('test') + else: + raise ValueError + + print(".. building model") + model = get_model(model, dataset.feature_dim) + + model.summary() + print(".. parameters: {:03.2f}M".format(model.count_params() / 1000000.)) + + if in_memory: + pass + # do nothing + else: + raise ValueError + + logger = mimir.Logger( + filename='models/log_{}.jsonl.gz'.format(model_name)) + + it = dataset.train_iterator() + + callbacks = [Validation(Xvalid, Yvalid, 'valid', logger), + Validation(Xtest, Ytest, 'test', logger), + SaveLastModel("./models/", 1, name=model), + Performance(logger), + LearningRateScheduler(schedule)] + + print('.. start training') + model.fit_generator( + it, steps_per_epoch=1000, epochs=epochs, + callbacks=callbacks, workers=1) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('model_name') + parser.add_argument('--in-memory', action='store_true', default=False) + parser.add_argument('--complex', dest='complex_', action='store_true', + default=False) + parser.add_argument('--model', default='shallow_convnet') + parser.add_argument('--epochs', default=200, type=int) + parser.add_argument('--fourier', action='store_true', default=False) + parser.add_argument('--stft', action='store_true', default=False) + parser.add_argument('--fast-load', action='store_true', default=False) + parser.add_argument( + '--local-data', + default="/Tmp/serdyuk/data/musicnet_11khz.npz") + main(**parser.parse_args().__dict__) diff --git a/build/scripts-2.7/training.py b/build/scripts-2.7/training.py new file mode 100755 index 0000000..2af6601 --- /dev/null +++ b/build/scripts-2.7/training.py @@ -0,0 +1,524 @@ +#!/u/parcollt/anaconda2/bin/python +# -*- coding: utf-8 -*- +# Authors: Parcollet Titouan, + +# Imports +import editdistance +import h5py +import datasets.timit +from datasets.timit import Timit +from datasets.utils import construct_conv_stream, phone_to_phoneme_dict +import complexnn +from complexnn import * +import h5py as H +import keras +from keras.callbacks import Callback, ModelCheckpoint, LearningRateScheduler +from keras.datasets import cifar10, cifar100 +from keras.initializers import Orthogonal +from keras.layers import Layer, Dropout, AveragePooling1D, AveragePooling2D, AveragePooling3D, add, Add, concatenate, Concatenate, Input, Flatten, Dense, Convolution2D, BatchNormalization, Activation, Reshape, ConvLSTM2D, Conv2D, Lambda +from keras.models import Model, load_model, save_model +from keras.optimizers import SGD, Adam, RMSprop +from keras.preprocessing.image import ImageDataGenerator +from keras.regularizers import l2 +from keras.utils.np_utils import to_categorical +import keras.backend as K +import keras.models as KM +from keras.utils.training_utils import multi_gpu_model +import logging as L +import numpy as np +import os, pdb, socket, sys, time +import theano as T +from keras.backend.tensorflow_backend import set_session +from models_cifar import getCifarResnetModel2D +from models_timit import getTimitResnetModel2D,ctc_lambda_func +from models_decoda import * +import tensorflow as tf +import itertools +import random + + +# +# Generator wrapper for timit +# + +def timitGenerator(stream): + while True: + #dataset = Timit(stream) + #seed = random.randint(0,100) + #rng=np.random.RandomState(seed) + #data_stream_dev = construct_conv_stream(dataset, rng, 200, 10000, quaternion=False) + for data in stream.get_epoch_iterator(): + yield data +# +# Custom metrics +# +class EditDistance(Callback): + def __init__(self, func, dataset, quaternion, save_prefix): + self.func = func + if(dataset in ['train','test','dev']): + self.dataset_type = dataset + self.save_prefix = save_prefix + self.dataset = Timit(str(dataset)) + self.full_phonemes_dict = self.dataset.get_phoneme_dict() + self.ind_phonemes_dict = self.dataset.get_phoneme_ind_dict() + self.rng = np.random.RandomState(123) + self.data_stream = construct_conv_stream(self.dataset, self.rng, 200, 10000,quaternion) + else: + raise ValueError("Unknown dataset for edit distance "+dataset) + + def labels_to_text(self,labels): + ret = [] + for c in labels: + if c == len(self.full_phonemes_dict) - 2: + ret.append("") + else: + c_ = self.full_phonemes_dict[c + 1] + ret.append(phone_to_phoneme_dict.get(c_, c_)) + ret = [k for k, g in itertools.groupby(ret)] + return list(filter(lambda c: c != "", ret)) + + def decode_batch(self, out, mask): + ret = [] + for j in range(out.shape[0]): + out_best = list(np.argmax(out[j], 1))[:int(mask[j])] + out_best = [k for k, g in itertools.groupby(out_best)] + # map from 61-d to 39-d + outstr = self.labels_to_text(out_best) + ret.append(outstr) + return ret + + def on_epoch_end(self, epoch, logs={}): + mean_norm_ed = 0. + num = 0 + for data in self.data_stream.get_epoch_iterator(): + x, y = data + y_pred = self.func([x[0]])[0] + decoded_y_pred = self.decode_batch(y_pred, x[1]) + decoded_gt = [] + for i in range(x[2].shape[0]): + decoded_gt.append(self.labels_to_text(x[2][i][:int(x[3][i])])) + num += len(decoded_y_pred) + for i, (_pred, _gt) in enumerate(zip(decoded_y_pred, decoded_gt)): + #print _pred + #print _gt + #print "######" + edit_dist = editdistance.eval(_pred, _gt) + mean_norm_ed += float(edit_dist) / x[3][i] + mean_norm_ed = mean_norm_ed / num + + # Dump To File Logs at every epoch for clusters sbatch + f=open(str(self.save_prefix)+"_"+str(self.dataset_type)+"_PER.txt",'ab') + np.savetxt(f,mean_norm_ed) + f.close() + L.getLogger("train").info("PER on "+str(self.dataset_type)+" : "+str(mean_norm_ed)+" at epoch "+str(epoch)) + +# +# Callbacks: +# + +class TrainLoss(Callback): + def __init__(self, savedir): + self.savedir = savedir + def on_epoch_end(self, epoch, logs={}): + f=open(str(self.savedir)+"_train_loss.txt",'ab') + f2=open(str(self.savedir)+"_dev_loss.txt",'ab') + value = float(logs['loss']) + np.savetxt(f,np.array([value])) + f.close() + value = float(logs['val_loss']) + np.savetxt(f2,np.array([value])) + f2.close() +# +# Print a newline after each epoch, because Keras doesn't. Grumble. +# + +class PrintNewlineAfterEpochCallback(Callback): + def on_epoch_end(self, epoch, logs={}): + sys.stdout.write("\n") +# +# Save checkpoints. +# + +class SaveLastModel(Callback): + def __init__(self, workdir, save_prefix, model_mono,period=10): + self.workdir = workdir + self.model_mono = model_mono + self.chkptsdir = os.path.join(self.workdir, "chkpts") + self.save_prefix = save_prefix + if not os.path.isdir(self.chkptsdir): + os.mkdir(self.chkptsdir) + self.period_of_epochs = period + self.linkFilename = os.path.join(self.chkptsdir, str(save_prefix)+"ModelChkpt.hdf5") + self.linkFilename_weight = os.path.join(self.chkptsdir, str(save_prefix)+"ModelChkpt_weight.hdf5") + + def on_epoch_end(self, epoch, logs={}): + if (epoch + 1) % self.period_of_epochs == 0: + # Filenames + baseHDF5Filename = str(self.save_prefix)+"ModelChkpt{:06d}.hdf5".format(epoch+1) + baseHDF5Filename_weight = str(self.save_prefix)+"ModelChkpt{:06d}_weight.hdf5".format(epoch+1) + baseYAMLFilename = str(self.save_prefix)+"ModelChkpt{:06d}.yaml".format(epoch+1) + hdf5Filename = os.path.join(self.chkptsdir, baseHDF5Filename) + hdf5Filename_weight = os.path.join(self.chkptsdir, baseHDF5Filename_weight) + yamlFilename = os.path.join(self.chkptsdir, baseYAMLFilename) + + # YAML + yamlModel = self.model_mono.to_yaml() + with open(yamlFilename, "w") as yamlFile: + yamlFile.write(yamlModel) + + # HDF5 + KM.save_model(self.model_mono, hdf5Filename) + self.model_mono.save_weights(hdf5Filename_weight) + with H.File(hdf5Filename, "r+") as f: + f.require_dataset("initialEpoch", (), "uint64", True)[...] = int(epoch+1) + f.flush() + with H.File(hdf5Filename_weight, "r+") as f: + f.require_dataset("initialEpoch", (), "uint64", True)[...] = int(epoch+1) + f.flush() + + + # Symlink to new HDF5 file, then atomically rename and replace. + os.symlink(baseHDF5Filename_weight, self.linkFilename_weight+".rename") + os.rename (self.linkFilename_weight+".rename", + self.linkFilename_weight) + + + # Symlink to new HDF5 file, then atomically rename and replace. + os.symlink(baseHDF5Filename, self.linkFilename+".rename") + os.rename (self.linkFilename+".rename", + self.linkFilename) + + # Print + L.getLogger("train").info("Saved checkpoint to {:s} at epoch {:5d}".format(hdf5Filename, epoch+1)) + +def q_normalize(x,size): + for line in range(0,len(x)): + for data in range(0,size/4): + norm = np.sqrt(pow(x[line][data][0],2)+ pow(x[line][data][1],2)+ \ + pow(x[line][data][2],2)+ pow(x[line][data][3],2)) + x[line][data][0] /= norm + x[line][data][1] /= norm + x[line][data][2] /= norm + x[line][data][3] /= norm + return x + + + +# +# Summarize environment variable. +# + +def summarizeEnvvar(var): + if var in os.environ: return var+"="+os.environ.get(var) + else: return var+" unset" + +# +# TRAINING PROCESS +# + +def train(d): + + # + + # + # Log important data about how we were invoked. + # + + L.getLogger("entry").info("INVOCATION: "+" ".join(sys.argv)) + L.getLogger("entry").info("HOSTNAME: "+socket.gethostname()) + L.getLogger("entry").info("PWD: "+os.getcwd()) + L.getLogger("entry").info("CUDA DEVICE: "+str(d.device)) + os.environ["CUDA_VISIBLE_DEVICES"]=str(d.device) + # + # Setup GPUs + # + config = tf.ConfigProto() + + # Don't pre-allocate memory; allocate as-needed + config.gpu_options.allow_growth = True + + # Only allow a total of half the GPU memory to be allocated + config.gpu_options.per_process_gpu_memory_fraction = d.memory + + # Create a session with the above options specified. + K.tensorflow_backend.set_session(tf.Session(config=config)) + + summary = "\n" + summary += "Environment:\n" + summary += summarizeEnvvar("THEANO_FLAGS")+"\n" + summary += "\n" + summary += "Software Versions:\n" + summary += "Theano: "+T.__version__+"\n" + summary += "Keras: "+keras.__version__+"\n" + summary += "\n" + summary += "Arguments:\n" + summary += "Path to Datasets: "+str(d.datadir)+"\n" + summary += "Number of GPUs: "+str(d.datadir)+"\n" + summary += "Path to Workspace: "+str(d.workdir)+"\n" + summary += "Model: "+str(d.model)+"\n" + summary += "Dataset: "+str(d.dataset)+"\n" + summary += "Number of Epochs: "+str(d.num_epochs)+"\n" + summary += "Batch Size: "+str(d.batch_size)+"\n" + summary += "Number of Start Filters: "+str(d.start_filter)+"\n" + summary += "Number of Blocks/Stage: "+str(d.num_blocks)+"\n" + summary += "Optimizer: "+str(d.optimizer)+"\n" + summary += "Learning Rate: "+str(d.lr)+"\n" + summary += "Learning Rate Decay: "+str(d.decay)+"\n" + summary += "Learning Rate Schedule: "+str(d.schedule)+"\n" + summary += "Clipping Norm: "+str(d.clipnorm)+"\n" + summary += "Clipping Value: "+str(d.clipval)+"\n" + summary += "Dropout Probability: "+str(d.dropout)+"\n" + if d.optimizer in ["adam"]: + summary += "Beta 1: "+str(d.beta1)+"\n" + summary += "Beta 2: "+str(d.beta2)+"\n" + else: + summary += "Momentum: "+str(d.momentum)+"\n" + summary += "Save Prefix: "+str(d.save_prefix)+"\n" + if d.model == "quaternion": + summary += "Quat. Segmentation: "+str(d.seg)+"\n" + L.getLogger("entry").info(summary[:-1]) + + # + # Load dataset + # + + L.getLogger("entry").info("Loading dataset {:s} ...".format(d.dataset)) + np.random.seed(d.seed % 2**32) + if d.dataset == "timit": + + # + # Create training data generator + # + dataset = Timit('train') + rng=np.random.RandomState(123) + if d.model =="quaternion": + data_stream_train = construct_conv_stream(dataset, rng, 200, 10000, quaternion=True) + else: + data_stream_train = construct_conv_stream(dataset, rng, 200, 10000, quaternion=False) + # + # Create dev data generator + # + dataset = Timit('dev') + rng=np.random.RandomState(123) + if d.model =="quaternion": + data_stream_dev = construct_conv_stream(dataset, rng, 200, 10000, quaternion=True) + else: + data_stream_dev = construct_conv_stream(dataset, rng, 200, 10000, quaternion=False) + + + L.getLogger("entry").info("Training set length: "+str(Timit('train').num_examples)) + L.getLogger("entry").info("Validation set length: "+str(Timit('dev').num_examples)) + L.getLogger("entry").info("Test set length: "+str(Timit('test').num_examples)) + L.getLogger("entry").info("Loaded dataset {:s}.".format(d.dataset)) + + + # Optimizer + if d.optimizer in ["sgd", "nag"]: + opt = SGD (lr = d.lr, + momentum = d.momentum, + decay = d.decay, + nesterov = (d.optimizer=="nag"), + clipnorm = d.clipnorm) + elif d.optimizer == "rmsprop": + opt = RMSProp(lr = d.lr, + decay = d.decay, + clipnorm = d.clipnorm) + elif d.optimizer == "adam": + opt = Adam (lr = d.lr, + beta_1 = d.beta1, + beta_2 = d.beta2, + decay = d.decay, + clipnorm = d.clipnorm) + else: + raise ValueError("Unknown optimizer "+d.optimizer) + + + # + # Initial Entry or Resume? + # + + initialEpoch = 0 + chkptFilename = os.path.join(d.workdir, "chkpts", str(d.save_prefix)+"ModelChkpt.hdf5") + chkptFilename_weight = os.path.join(d.workdir, "chkpts", str(d.save_prefix)+"ModelChkpt_weight.hdf5") + isResuming = os.path.isfile(chkptFilename) + isResuming_weight = os.path.isfile(chkptFilename_weight) + + #### HAVE TO BE EXTEND TO WORK WITH QUATERNION SAVES + if isResuming or isResuming_weight: + # Reload Model and Optimizer + if d.dataset == "timit": + L.getLogger("entry").info("Re-Creating the model from scratch.") + model_mono,test_func = getTimitResnetModel2D(d) + model_mono.load_weights(chkptFilename_weight) + with H.File(chkptFilename_weight, "r") as f: + initialEpoch = int(f["initialEpoch"][...]) + L.getLogger("entry").info("Training will restart at epoch {:5d}.".format(initialEpoch+1)) + L.getLogger("entry").info("Compilation Started.") + + else: + + L.getLogger("entry").info("Reloading a model from "+chkptFilename+" ...") + np.random.seed(d.seed % 2**32) + model = KM.load_model(chkptFilename, custom_objects={ + "ComplexConv2D": ComplexConv2D, + "QuaternionConv2D": QuaternionConv2D, + "QuaternionConv1D": QuaternionConv1D, + "ComplexBatchNormalization": ComplexBN, + "QuaternionBatchNormalization": QuaternionBN, + "GetReal": GetReal, + "GetImag": GetImag, + "GetIFirst": GetIFirst, + "GetJFirst": GetJFirst, + "GetKFirst": GetKFirst, + "GetRFirst": GetRFirst, + "ComplexConv2D": ComplexConv2D, + "ComplexBatchNormalization": ComplexBN, + }) + L.getLogger("entry").info("... reloading complete.") + with H.File(chkptFilename, "r") as f: + initialEpoch = int(f["initialEpoch"][...]) + L.getLogger("entry").info("Training will restart at epoch {:5d}.".format(initialEpoch+1)) + L.getLogger("entry").info("Compilation Started.") + else: + # Model + L.getLogger("entry").info("Creating new model from scratch.") + np.random.seed(d.seed % 2**32) + if d.dataset == "decoda": + model_mono = getModel1D(d) + #model = getResnetModel1D(d) + elif d.dataset == "timit": + model_mono,test_func = getTimitResnetModel2D(d) + else: + model_mono = getCifarResnetModel2D(d) + + L.getLogger("entry").info("Compilation Started.") + + # + # Multi GPU: Can only save the model_mono because of keras bug + # + + if d.gpus >1: + model = multi_gpu_model(model_mono, gpus=d.gpus) + else: + model = model_mono + + if d.dataset == "timit": + model.compile(opt, loss={'ctc': lambda y_true, y_pred: y_pred}) #,metrics=[ctc_accuracy]) + else: + model.compile(opt, 'categorical_crossentropy', metrics=['accuracy']) + + + print model.summary() + + # + # Precompile several backend functions + # + + if d.summary: + model.summary() + L.getLogger("entry").info("# of Parameters: {:10d}".format(model.count_params())) + L.getLogger("entry").info("Compiling Train Function...") + t =- time.time() + model._make_train_function() + t += time.time() + L.getLogger("entry").info(" {:10.3f}s".format(t)) + L.getLogger("entry").info("Compiling Predict Function...") + t =- time.time() + model._make_predict_function() + t += time.time() + L.getLogger("entry").info(" {:10.3f}s".format(t)) + L.getLogger("entry").info("Compiling Test Function...") + t =- time.time() + model._make_test_function() + t += time.time() + L.getLogger("entry").info(" {:10.3f}s".format(t)) + L.getLogger("entry").info("Compilation Ended.") + + # + # Create Callbacks + # + + newLineCb = PrintNewlineAfterEpochCallback() + + saveLastCb = SaveLastModel(d.workdir, d.save_prefix, model_mono,period=3) + #saveBestCb = SaveBestModel(d.save_prefix, d.workdir, d.output_type, model_mono) + #trainValHistCb = TrainValHistory(d.output_type) + + callbacks = [] + callbacks += [newLineCb] + if d.dataset == "timit": + if d.model=="quaternion": + quaternion = True + else: + quaternion = False + + savedir = d.workdir+"/LOGS/"+d.save_prefix + trainLoss = TrainLoss(savedir) + editDistValCb = EditDistance(test_func,'dev',quaternion, savedir) + editDistTestCb = EditDistance(test_func,'test',quaternion, savedir) + callbacks += [trainLoss] + callbacks += [editDistValCb] + callbacks += [editDistTestCb] + else: + testErrCb = TestErrorCallback((X_test, Y_test), d.output_type) + callbacks += [testErrCb] + callbacks += [trainValHistCb] + callbacks += [saveBestCb] + + callbacks += [newLineCb] + callbacks += [saveLastCb] + + # + # Enter training loop. + # + + L .getLogger("entry").info("**********************************************") + if isResuming: L.getLogger("entry").info("*** Reentering Training Loop @ Epoch {:5d} ***".format(initialEpoch+1)) + else: L.getLogger("entry").info("*** Entering Training Loop @ First Epoch ***") + L .getLogger("entry").info("**********************************************") + + if d.dataset == "timit": + + model.fit_generator(generator = timitGenerator(data_stream_train), + steps_per_epoch = 188, + epochs = d.num_epochs, + verbose = 1, + validation_data = timitGenerator(data_stream_dev), + validation_steps = 20, + callbacks = callbacks, + initial_epoch = initialEpoch) + + else: + + # + # Create training data generator + # + + datagen = ImageDataGenerator(height_shift_range = 0.125, + width_shift_range = 0.125, + horizontal_flip = True) + + + model.fit_generator(generator = datagen.flow(X_train, Y_train, batch_size=d.batch_size), + steps_per_epoch = (len(X_train)+d.batch_size-1) // d.batch_size, + epochs = d.num_epochs, + verbose = 1, + callbacks = callbacks, + validation_data = (X_val, Y_val), + initial_epoch = initialEpoch) + + # + # Dump histories. + # + + np.savetxt(os.path.join(d.workdir, 'test_loss.txt'), np.asarray(testErrCb.loss_history)) + np.savetxt(os.path.join(d.workdir, 'test_acc.txt'), np.asarray(testErrCb.acc_history)) + np.savetxt(os.path.join(d.workdir, 'train_loss.txt'), np.asarray(trainValHistCb.train_loss)) + np.savetxt(os.path.join(d.workdir, 'train_acc.txt'), np.asarray(trainValHistCb.train_acc)) + np.savetxt(os.path.join(d.workdir, 'val_loss.txt'), np.asarray(trainValHistCb.val_loss)) + np.savetxt(os.path.join(d.workdir, 'val_acc.txt'), np.asarray(trainValHistCb.val_acc)) + + # CIFAR-10: + # - Baseline + # - Baseline but with complex parametrization + # - Baseline but with spectral pooling diff --git a/complexnn/__init__.py b/complexnn/__init__.py new file mode 100644 index 0000000..ba25000 --- /dev/null +++ b/complexnn/__init__.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Contributors: Titouan Parcollet +# Authors: Olexa Bilaniuk +# +# What this module includes by default: +import conv, dense, fft, init, + +from .conv import (QuaternionConv, + QuaternionConv1D, + QuaternionConv2D, + QuaternionConv3D, + WeightNorm_Conv) +from .dense import QuaternionDense +from .init import (SqrtInit, QuaternionInit) +from .utils import (GetRFirst, GetIFirst, GetJFirst, GetKFirst, getpart_quaternion_output_shape, get_rpart_first, get_ipart_first, get_jpart_first, + get_kpart_first) diff --git a/complexnn/conv.py b/complexnn/conv.py new file mode 100644 index 0000000..31e0ca9 --- /dev/null +++ b/complexnn/conv.py @@ -0,0 +1,827 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Contributors : Titouan Parcollet +# Initial Authors: Chiheb Trabelsi + +from keras import backend as K +from keras import activations, initializers, regularizers, constraints +from keras.layers import Lambda, Layer, InputSpec, Convolution1D, Convolution2D, add, multiply, Activation, Input, concatenate +from keras.layers.convolutional import _Conv +from keras.layers.merge import _Merge +from keras.layers.recurrent import Recurrent +from keras.utils import conv_utils +from keras.models import Model +import numpy as np +from theano.sandbox.rng_mrg import MRG_RandomStreams as RandomStreams +from .bn import sqrt_init +from .init import * +import sys + + +##################################################################### +# Quaternion Implementations # +##################################################################### + +class QuaternionConv(Layer): + """Abstract nD quaternion convolution layer. + This layer creates a quaternion convolution kernel that is convolved + with the layer input to produce a tensor of outputs. + If `use_bias` is True, a bias vector is created and added to the outputs. + Finally, if `activation` is not `None`, + it is applied to the outputs as well. + # Arguments + rank: An integer, the rank of the convolution, + e.g. "2" for 2D convolution. + filters: Integer, the dimensionality of the output space, i.e, + the number of quaternion feature maps. It is also the effective number + of feature maps for each of the real and imaginary parts. + (i.e. the number of quaternion filters in the convolution) + The total effective number of filters is 2 x filters. + kernel_size: An integer or tuple/list of n integers, specifying the + dimensions of the convolution window. + strides: An integer or tuple/list of n integers, + spfying the strides of the convolution. + Specifying any stride value != 1 is incompatible with specifying + any `dilation_rate` value != 1. + padding: One of `"valid"` or `"same"` (case-insensitive). + data_format: A string, + one of `channels_last` (default) or `channels_first`. + The ordering of the dimensions in the inputs. + `channels_last` corresponds to inputs with shape + `(batch, ..., channels)` while `channels_first` corresponds to + inputs with shape `(batch, channels, ...)`. + It defaults to the `image_data_format` value found in your + Keras config file at `~/.keras/keras.json`. + If you never set it, then it will be "channels_last". + dilation_rate: An integer or tuple/list of n integers, specifying + the dilation rate to use for dilated convolution. + Currently, specifying any `dilation_rate` value != 1 is + incompatible with specifying any `strides` value != 1. + activation: Activation function to use + (see keras.activations). + If you don't specify anything, no activation is applied + (ie. "linear" activation: `a(x) = x`). + use_bias: Boolean, whether the layer uses a bias vector. + normalize_weight: Boolean, whether the layer normalizes its quaternion + weights before convolving the quaternion input. + The quaternion normalization performed is similar to the one + for the batchnorm. Each of the quaternion kernels are centred and multiplied by + the inverse square root of covariance matrix. + Then, a quaternion multiplication is perfromed as the normalized weights are + multiplied by the quaternion scaling factor gamma. + kernel_initializer: Initializer for the quaternion `kernel` weights matrix. + By default it is 'quaternion'. The 'quaternion_independent' + and the usual initializers could also be used. + (see keras.initializers and init.py). + bias_initializer: Initializer for the bias vector + (see keras.initializers). + kernel_regularizer: Regularizer function applied to + the `kernel` weights matrix + (see keras.regularizers). + bias_regularizer: Regularizer function applied to the bias vector + (see keras.regularizers). + activity_regularizer: Regularizer function applied to + the output of the layer (its "activation"). + (see keras.regularizers). + kernel_constraint: Constraint function applied to the kernel matrix + (see keras.constraints). + bias_constraint: Constraint function applied to the bias vector + (see keras.constraints). + spectral_parametrization: Whether or not to use a spectral + parametrization of the parameters. + """ + + def __init__(self, rank, + filters, + kernel_size, + strides=1, + padding='valid', + data_format=None, + dilation_rate=1, + activation=None, + use_bias=True, + normalize_weight=False, + kernel_initializer='quaternion', + bias_initializer='zeros', + gamma_diag_initializer=sqrt_init, + gamma_off_initializer='zeros', + kernel_regularizer=None, + bias_regularizer=None, + gamma_diag_regularizer=None, + gamma_off_regularizer=None, + activity_regularizer=None, + kernel_constraint=None, + bias_constraint=None, + gamma_diag_constraint=None, + gamma_off_constraint=None, + init_criterion='he', + seed=None, + spectral_parametrization=False, + epsilon=1e-7, + **kwargs): + super(QuaternionConv, self).__init__(**kwargs) + self.rank = rank + self.filters = filters + self.kernel_size = conv_utils.normalize_tuple(kernel_size, rank, 'kernel_size') + self.strides = conv_utils.normalize_tuple(strides, rank, 'strides') + self.padding = conv_utils.normalize_padding(padding) + self.data_format = conv_utils.normalize_data_format(data_format) + self.dilation_rate = conv_utils.normalize_tuple(dilation_rate, rank, 'dilation_rate') + self.activation = activations.get(activation) + self.use_bias = use_bias + self.normalize_weight = normalize_weight + self.init_criterion = init_criterion + self.spectral_parametrization = spectral_parametrization + self.epsilon = epsilon + self.kernel_initializer = sanitizedInitGet(kernel_initializer) + self.bias_initializer = sanitizedInitGet(bias_initializer) + self.gamma_diag_initializer = sanitizedInitGet(gamma_diag_initializer) + self.gamma_off_initializer = sanitizedInitGet(gamma_off_initializer) + self.kernel_regularizer = regularizers.get(kernel_regularizer) + self.bias_regularizer = regularizers.get(bias_regularizer) + self.gamma_diag_regularizer = regularizers.get(gamma_diag_regularizer) + self.gamma_off_regularizer = regularizers.get(gamma_off_regularizer) + self.activity_regularizer = regularizers.get(activity_regularizer) + self.kernel_constraint = constraints.get(kernel_constraint) + self.bias_constraint = constraints.get(bias_constraint) + self.gamma_diag_constraint = constraints.get(gamma_diag_constraint) + self.gamma_off_constraint = constraints.get(gamma_off_constraint) + if seed is None: + self.seed = np.random.randint(1, 10e6) + else: + self.seed = seed + self.input_spec = InputSpec(ndim=self.rank + 2) + + def build(self, input_shape): + + if self.data_format == 'channels_first': + channel_axis = 1 + else: + channel_axis = -1 + + if input_shape[channel_axis] is None: + raise ValueError('The channel dimension of the inputs ' + 'should be defined. Found `None`.') + + input_dim = input_shape[channel_axis] // 4 + self.kernel_shape = self.kernel_size + (input_dim , self.filters) + + if self.kernel_initializer in {'quaternion', 'quaternion_independent'}: + kls = {'quaternion': QuaternionInit, + 'quaternion_independent': QuaternionIndependentFilters}[self.kernel_initializer] + kern_init = kls( + kernel_size=self.kernel_size, + input_dim=input_dim, + weight_dim=self.rank, + nb_filters=self.filters, + criterion=self.init_criterion + ) + else: + kern_init = self.kernel_initializer + + self.kernel = self.add_weight( + self.kernel_shape, + initializer=kern_init, + name='kernel', + regularizer=self.kernel_regularizer, + constraint=self.kernel_constraint + ) + + if self.normalize_weight: + gamma_shape = (input_dim * self.filters,) + self.gamma_rr = self.add_weight( + shape=gamma_shape, + name='gamma_rr', + initializer=self.gamma_diag_initializer, + regularizer=self.gamma_diag_regularizer, + constraint=self.gamma_diag_constraint + ) + + self.gamma_ri = self.add_weight( + shape=gamma_shape, + name='gamma_ri', + initializer=self.gamma_off_initializer, + regularizer=self.gamma_off_regularizer, + constraint=self.gamma_off_constraint + ) + self.gamma_rj = self.add_weight( + shape=gamma_shape, + name='gamma_rj', + initializer=self.gamma_off_initializer, + regularizer=self.gamma_off_regularizer, + constraint=self.gamma_off_constraint + ) + self.gamma_rk = self.add_weight( + shape=gamma_shape, + name='gamma_rk', + initializer=self.gamma_off_initializer, + regularizer=self.gamma_off_regularizer, + constraint=self.gamma_off_constraint + ) + self.gamma_ii = self.add_weight( + shape=gamma_shape, + name='gamma_ii', + initializer=self.gamma_diag_initializer, + regularizer=self.gamma_diag_regularizer, + constraint=self.gamma_diag_constraint + ) + + self.gamma_ij = self.add_weight( + shape=gamma_shape, + name='gamma_ij', + initializer=self.gamma_off_initializer, + regularizer=self.gamma_off_regularizer, + constraint=self.gamma_off_constraint + ) + self.gamma_ik = self.add_weight( + shape=gamma_shape, + name='gamma_ik', + initializer=self.gamma_off_initializer, + regularizer=self.gamma_off_regularizer, + constraint=self.gamma_off_constraint + ) + self.gamma_jj = self.add_weight( + shape=gamma_shape, + name='gamma_jj', + initializer=self.gamma_diag_initializer, + regularizer=self.gamma_diag_regularizer, + constraint=self.gamma_diag_constraint + ) + self.gamma_jk = self.add_weight( + shape=gamma_shape, + name='gamma_jk', + initializer=self.gamma_diag_initializer, + regularizer=self.gamma_diag_regularizer, + constraint=self.gamma_diag_constraint + ) + self.gamma_kk = self.add_weight( + shape=gamma_shape, + name='gamma_kk', + initializer=self.gamma_off_initializer, + regularizer=self.gamma_off_regularizer, + constraint=self.gamma_off_constraint + ) + else: + self.gamma_rr = None + self.gamma_ri = None + self.gamma_rj = None + self.gamma_rk = None + self.gamma_ii = None + self.gamma_ij = None + self.gamma_ik = None + self.gamma_jj = None + self.gamma_jk = None + self.gamma_kk = None + + + if self.use_bias: + bias_shape = (4 * self.filters,) + self.bias = self.add_weight( + bias_shape, + initializer=self.bias_initializer, + name='bias', + regularizer=self.bias_regularizer, + constraint=self.bias_constraint + ) + + else: + self.bias = None + + # Set input spec. + self.input_spec = InputSpec(ndim=self.rank + 2, + axes={channel_axis: input_dim * 4}) + self.built = True + + def call(self, inputs): + channel_axis = 1 if self.data_format == 'channels_first' else -1 + input_dim = K.shape(inputs)[channel_axis] // 4 + index2 = self.filters*2 + index3 = self.filters*3 + if self.rank == 1: + f_r = self.kernel[:, :, :self.filters] + f_i = self.kernel[:, :, self.filters:index2] + f_j = self.kernel[:, :, index2:index3] + f_k = self.kernel[:, :, index3:] + elif self.rank == 2: + f_r = self.kernel[:, :, :, :self.filters] + f_i = self.kernel[:, :, :, self.filters:index2] + f_j = self.kernel[:, :, :, index2:index3] + f_k = self.kernel[:, :, :, index3:] + elif self.rank == 3: + f_r = self.kernel[:, :, :, :, :self.filters] + f_i = self.kernel[:, :, :, :, self.filters:index2] + f_j = self.kernel[:, :, :, :, index2:index3] + f_k = self.kernel[:, :, :, :, index3:] + + convArgs = {"strides": self.strides[0] if self.rank == 1 else self.strides, + "padding": self.padding, + "data_format": self.data_format, + "dilation_rate": self.dilation_rate[0] if self.rank == 1 else self.dilation_rate} + convFunc = {1: K.conv1d, + 2: K.conv2d, + 3: K.conv3d}[self.rank] + + + # + # Performing quaternion convolution + # + + f_r._keras_shape = self.kernel_shape + f_i._keras_shape = self.kernel_shape + f_j._keras_shape = self.kernel_shape + f_k._keras_shape = self.kernel_shape + + cat_kernels_4_r = K.concatenate([f_r, -f_i, -f_j, -f_k], axis=-2) + cat_kernels_4_i = K.concatenate([f_i, f_r, -f_k, f_j], axis=-2) + cat_kernels_4_j = K.concatenate([f_j, f_k, f_r, -f_i], axis=-2) + cat_kernels_4_k = K.concatenate([f_k, -f_j, f_i, f_r], axis=-2) + cat_kernels_4_quaternion = K.concatenate([cat_kernels_4_r, cat_kernels_4_i, cat_kernels_4_j, cat_kernels_4_k], axis=-1) + cat_kernels_4_quaternion._keras_shape = self.kernel_size + (4 * input_dim, 4 * self.filters) + + output = convFunc(inputs, cat_kernels_4_quaternion, **convArgs) + + if self.use_bias: + output = K.bias_add( + output, + self.bias, + data_format=self.data_format + ) + if self.activation is not None: + output = self.activation(output) + + return output + + def compute_output_shape(self, input_shape): + if self.data_format == 'channels_last': + space = input_shape[1:-1] + new_space = [] + for i in range(len(space)): + new_dim = conv_utils.conv_output_length( + space[i], + self.kernel_size[i], + padding=self.padding, + stride=self.strides[i], + dilation=self.dilation_rate[i] + ) + new_space.append(new_dim) + return (input_shape[0],) + tuple(new_space) + (4 * self.filters,) + if self.data_format == 'channels_first': + space = input_shape[2:] + new_space = [] + for i in range(len(space)): + new_dim = conv_utils.conv_output_length( + space[i], + self.kernel_size[i], + padding=self.padding, + stride=self.strides[i], + dilation=self.dilation_rate[i]) + new_space.append(new_dim) + return (input_shape[0],) + (4 * self.filters,) + tuple(new_space) + + def get_config(self): + config = { + 'rank': self.rank, + 'filters': self.filters, + 'kernel_size': self.kernel_size, + 'strides': self.strides, + 'padding': self.padding, + 'data_format': self.data_format, + 'dilation_rate': self.dilation_rate, + 'activation': activations.serialize(self.activation), + 'use_bias': self.use_bias, + 'normalize_weight': self.normalize_weight, + 'kernel_initializer': sanitizedInitSer(self.kernel_initializer), + 'bias_initializer': sanitizedInitSer(self.bias_initializer), + 'gamma_diag_initializer': sanitizedInitSer(self.gamma_diag_initializer), + 'gamma_off_initializer': sanitizedInitSer(self.gamma_off_initializer), + 'kernel_regularizer': regularizers.serialize(self.kernel_regularizer), + 'bias_regularizer': regularizers.serialize(self.bias_regularizer), + 'gamma_diag_regularizer': regularizers.serialize(self.gamma_diag_regularizer), + 'gamma_off_regularizer': regularizers.serialize(self.gamma_off_regularizer), + 'activity_regularizer': regularizers.serialize(self.activity_regularizer), + 'kernel_constraint': constraints.serialize(self.kernel_constraint), + 'bias_constraint': constraints.serialize(self.bias_constraint), + 'gamma_diag_constraint': constraints.serialize(self.gamma_diag_constraint), + 'gamma_off_constraint': constraints.serialize(self.gamma_off_constraint), + 'init_criterion': self.init_criterion, + 'spectral_parametrization': self.spectral_parametrization, + } + base_config = super(QuaternionConv, self).get_config() + return dict(list(base_config.items()) + list(config.items())) + + + +class QuaternionConv1D(QuaternionConv): + """1D quaternion convolution layer. + This layer creates a quaternion convolution kernel that is convolved + with a quaternion input layer over a single quaternion spatial (or temporal) dimension + to produce a quaternion output tensor. + If `use_bias` is True, a bias vector is created and added to the quaternion output. + Finally, if `activation` is not `None`, + it is applied each of the real and imaginary parts of the output. + When using this layer as the first layer in a model, + provide an `input_shape` argument + (tuple of integers or `None`, e.g. + `(10, 128)` for sequences of 10 vectors of 128-dimensional vectors, + or `(None, 128)` for variable-length sequences of 128-dimensional vectors. + # Arguments + filters: Integer, the dimensionality of the output space, i.e, + the number of quaternion feature maps. It is also the effective number + of feature maps for each of the real and imaginary parts. + (i.e. the number of quaternion filters in the convolution) + The total effective number of filters is 2 x filters. + kernel_size: An integer or tuple/list of n integers, specifying the + dimensions of the convolution window. + strides: An integer or tuple/list of a single integer, + specifying the stride length of the convolution. + Specifying any stride value != 1 is incompatible with specifying + any `dilation_rate` value != 1. + padding: One of `"valid"`, `"causal"` or `"same"` (case-insensitive). + `"causal"` results in causal (dilated) convolutions, e.g. output[t] + does not depend on input[t+1:]. Useful when modeling temporal data + where the model should not violate the temporal order. + See [WaveNet: A Generative Model for Raw Audio, section 2.1](https://arxiv.org/abs/1609.03499). + dilation_rate: an integer or tuple/list of a single integer, specifying + the dilation rate to use for dilated convolution. + Currently, specifying any `dilation_rate` value != 1 is + incompatible with specifying any `strides` value != 1. + activation: Activation function to use + (see keras.activations). + If you don't specify anything, no activation is applied + (ie. "linear" activation: `a(x) = x`). + use_bias: Boolean, whether the layer uses a bias vector. + normalize_weight: Boolean, whether the layer normalizes its quaternion + weights before convolving the quaternion input. + The quaternion normalization performed is similar to the one + for the batchnorm. Each of the quaternion kernels are centred and multiplied by + the inverse square root of covariance matrix. + Then, a quaternion multiplication is perfromed as the normalized weights are + multiplied by the quaternion scaling factor gamma. + kernel_initializer: Initializer for the quaternion `kernel` weights matrix. + By default it is 'quaternion'. The 'quaternion_independent' + and the usual initializers could also be used. + (see keras.initializers and init.py). + bias_initializer: Initializer for the bias vector + (see keras.initializers). + kernel_regularizer: Regularizer function applied to + the `kernel` weights matrix + (see keras.regularizers). + bias_regularizer: Regularizer function applied to the bias vector + (see keras.regularizers). + activity_regularizer: Regularizer function applied to + the output of the layer (its "activation"). + (see keras.regularizers). + kernel_constraint: Constraint function applied to the kernel matrix + (see keras.constraints). + bias_constraint: Constraint function applied to the bias vector + (see keras.constraints). + spectral_parametrization: Whether or not to use a spectral + parametrization of the parameters. + # Input shape + 3D tensor with shape: `(batch_size, steps, input_dim)` + # Output shape + 3D tensor with shape: `(batch_size, new_steps, 2 x filters)` + `steps` value might have changed due to padding or strides. + """ + + def __init__(self, filters, + kernel_size, + strides=1, + padding='valid', + dilation_rate=1, + activation=None, + use_bias=True, + kernel_initializer='quaternion', + bias_initializer='zeros', + kernel_regularizer=None, + bias_regularizer=None, + activity_regularizer=None, + kernel_constraint=None, + bias_constraint=None, + seed=None, + init_criterion='he', + spectral_parametrization=False, + **kwargs): + super(QuaternionConv1D, self).__init__( + rank=1, + filters=filters, + kernel_size=kernel_size, + strides=strides, + padding=padding, + data_format='channels_last', + dilation_rate=dilation_rate, + activation=activation, + use_bias=use_bias, + kernel_initializer=kernel_initializer, + bias_initializer=bias_initializer, + kernel_regularizer=kernel_regularizer, + bias_regularizer=bias_regularizer, + activity_regularizer=activity_regularizer, + kernel_constraint=kernel_constraint, + bias_constraint=bias_constraint, + init_criterion=init_criterion, + spectral_parametrization=spectral_parametrization, + **kwargs) + + def get_config(self): + config = super(QuaternionConv1D, self).get_config() + config.pop('rank') + config.pop('data_format') + return config + + +class QuaternionConv2D(QuaternionConv): + """2D Quaternion convolution layer (e.g. spatial convolution over images). + This layer creates a quaternion convolution kernel that is convolved + with a quaternion input layer to produce a quaternion output tensor. If `use_bias` + is True, a quaternion bias vector is created and added to the outputs. + Finally, if `activation` is not `None`, it is applied to both the + real and imaginary parts of the output. + When using this layer as the first layer in a model, + provide the keyword argument `input_shape` + (tuple of integers, does not include the sample axis), + e.g. `input_shape=(128, 128, 3)` for 128x128 RGB pictures + in `data_format="channels_last"`. + # Arguments + filters: Integer, the dimensionality of the quaternion output space + (i.e, the number quaternion feature maps in the convolution). + The total effective number of filters or feature maps is 2 x filters. + kernel_size: An integer or tuple/list of 2 integers, specifying the + width and height of the 2D convolution window. + Can be a single integer to specify the same value for + all spatial dimensions. + strides: An integer or tuple/list of 2 integers, + specifying the strides of the convolution along the width and height. + Can be a single integer to specify the same value for + all spatial dimensions. + Specifying any stride value != 1 is incompatible with specifying + any `dilation_rate` value != 1. + padding: one of `"valid"` or `"same"` (case-insensitive). + data_format: A string, + one of `channels_last` (default) or `channels_first`. + The ordering of the dimensions in the inputs. + `channels_last` corresponds to inputs with shape + `(batch, height, width, channels)` while `channels_first` + corresponds to inputs with shape + `(batch, channels, height, width)`. + It defaults to the `image_data_format` value found in your + Keras config file at `~/.keras/keras.json`. + If you never set it, then it will be "channels_last". + dilation_rate: an integer or tuple/list of 2 integers, specifying + the dilation rate to use for dilated convolution. + Can be a single integer to specify the same value for + all spatial dimensions. + Currently, specifying any `dilation_rate` value != 1 is + incompatible with specifying any stride value != 1. + activation: Activation function to use + (see keras.activations). + If you don't specify anything, no activation is applied + (ie. "linear" activation: `a(x) = x`). + use_bias: Boolean, whether the layer uses a bias vector. + normalize_weight: Boolean, whether the layer normalizes its quaternion + weights before convolving the quaternion input. + The quaternion normalization performed is similar to the one + for the batchnorm. Each of the quaternion kernels are centred and multiplied by + the inverse square root of covariance matrix. + Then, a quaternion multiplication is perfromed as the normalized weights are + multiplied by the quaternion scaling factor gamma. + kernel_initializer: Initializer for the quaternion `kernel` weights matrix. + By default it is 'quaternion'. The 'quaternion_independent' + and the usual initializers could also be used. + (see keras.initializers and init.py). + bias_initializer: Initializer for the bias vector + (see keras.initializers). + kernel_regularizer: Regularizer function applied to + the `kernel` weights matrix + (see keras.regularizers). + bias_regularizer: Regularizer function applied to the bias vector + (see keras.regularizers). + activity_regularizer: Regularizer function applied to + the output of the layer (its "activation"). + (see keras.regularizers). + kernel_constraint: Constraint function applied to the kernel matrix + (see keras.constraints). + bias_constraint: Constraint function applied to the bias vector + (see keras.constraints). + spectral_parametrization: Whether or not to use a spectral + parametrization of the parameters. + # Input shape + 4D tensor with shape: + `(samples, channels, rows, cols)` if data_format='channels_first' + or 4D tensor with shape: + `(samples, rows, cols, channels)` if data_format='channels_last'. + # Output shape + 4D tensor with shape: + `(samples, 2 x filters, new_rows, new_cols)` if data_format='channels_first' + or 4D tensor with shape: + `(samples, new_rows, new_cols, 2 x filters)` if data_format='channels_last'. + `rows` and `cols` values might have changed due to padding. + """ + + def __init__(self, filters, + kernel_size, + strides=(1, 1), + padding='valid', + data_format=None, + dilation_rate=(1, 1), + activation=None, + use_bias=True, + kernel_initializer='quaternion', + bias_initializer='zeros', + kernel_regularizer=None, + bias_regularizer=None, + activity_regularizer=None, + kernel_constraint=None, + bias_constraint=None, + seed=None, + init_criterion='he', + spectral_parametrization=False, + **kwargs): + super(QuaternionConv2D, self).__init__( + rank=2, + filters=filters, + kernel_size=kernel_size, + strides=strides, + padding=padding, + data_format=data_format, + dilation_rate=dilation_rate, + activation=activation, + use_bias=use_bias, + kernel_initializer=kernel_initializer, + bias_initializer=bias_initializer, + kernel_regularizer=kernel_regularizer, + bias_regularizer=bias_regularizer, + activity_regularizer=activity_regularizer, + kernel_constraint=kernel_constraint, + bias_constraint=bias_constraint, + init_criterion=init_criterion, + spectral_parametrization=spectral_parametrization, + **kwargs) + + def get_config(self): + config = super(QuaternionConv2D, self).get_config() + config.pop('rank') + return config + + +class QuaternionConv3D(QuaternionConv): + """3D convolution layer (e.g. spatial convolution over volumes). + This layer creates a quaternion convolution kernel that is convolved + with a quaternion layer input to produce a quaternion output tensor. + If `use_bias` is True, + a quaternion bias vector is created and added to the outputs. Finally, if + `activation` is not `None`, it is applied to each of the real and imaginary + parts of the output. + When using this layer as the first layer in a model, + provide the keyword argument `input_shape` + (tuple of integers, does not include the sample axis), + e.g. `input_shape=(2, 128, 128, 128, 3)` for 128x128x128 volumes + with 3 channels, + in `data_format="channels_last"`. + # Arguments + filters: Integer, the dimensionality of the quaternion output space + (i.e, the number quaternion feature maps in the convolution). + The total effective number of filters or feature maps is 2 x filters. + kernel_size: An integer or tuple/list of 3 integers, specifying the + width and height of the 3D convolution window. + Can be a single integer to specify the same value for + all spatial dimensions. + strides: An integer or tuple/list of 3 integers, + specifying the strides of the convolution along each spatial dimension. + Can be a single integer to specify the same value for + all spatial dimensions. + Specifying any stride value != 1 is incompatible with specifying + any `dilation_rate` value != 1. + padding: one of `"valid"` or `"same"` (case-insensitive). + data_format: A string, + one of `channels_last` (default) or `channels_first`. + The ordering of the dimensions in the inputs. + `channels_last` corresponds to inputs with shape + `(batch, spatial_dim1, spatial_dim2, spatial_dim3, channels)` + while `channels_first` corresponds to inputs with shape + `(batch, channels, spatial_dim1, spatial_dim2, spatial_dim3)`. + It defaults to the `image_data_format` value found in your + Keras config file at `~/.keras/keras.json`. + If you never set it, then it will be "channels_last". + dilation_rate: an integer or tuple/list of 3 integers, specifying + the dilation rate to use for dilated convolution. + Can be a single integer to specify the same value for + all spatial dimensions. + Currently, specifying any `dilation_rate` value != 1 is + incompatible with specifying any stride value != 1. + activation: Activation function to use + (see keras.activations). + If you don't specify anything, no activation is applied + (ie. "linear" activation: `a(x) = x`). + use_bias: Boolean, whether the layer uses a bias vector. + normalize_weight: Boolean, whether the layer normalizes its quaternion + weights before convolving the quaternion input. + The quaternion normalization performed is similar to the one + for the batchnorm. Each of the quaternion kernels are centred and multiplied by + the inverse square root of covariance matrix. + Then, a quaternion multiplication is perfromed as the normalized weights are + multiplied by the quaternion scaling factor gamma. + kernel_initializer: Initializer for the quaternion `kernel` weights matrix. + By default it is 'quaternion'. The 'quaternion_independent' + and the usual initializers could also be used. + (see keras.initializers and init.py). + bias_initializer: Initializer for the bias vector + (see keras.initializers). + kernel_regularizer: Regularizer function applied to + the `kernel` weights matrix + (see keras.regularizers). + bias_regularizer: Regularizer function applied to the bias vector + (see keras.regularizers). + activity_regularizer: Regularizer function applied to + the output of the layer (its "activation"). + (see keras.regularizers). + kernel_constraint: Constraint function applied to the kernel matrix + (see keras.constraints). + bias_constraint: Constraint function applied to the bias vector + (see keras.constraints). + spectral_parametrization: Whether or not to use a spectral + parametrization of the parameters. + # Input shape + 5D tensor with shape: + `(samples, channels, conv_dim1, conv_dim2, conv_dim3)` if data_format='channels_first' + or 5D tensor with shape: + `(samples, conv_dim1, conv_dim2, conv_dim3, channels)` if data_format='channels_last'. + # Output shape + 5D tensor with shape: + `(samples, 2 x filters, new_conv_dim1, new_conv_dim2, new_conv_dim3)` if data_format='channels_first' + or 5D tensor with shape: + `(samples, new_conv_dim1, new_conv_dim2, new_conv_dim3, 2 x filters)` if data_format='channels_last'. + `new_conv_dim1`, `new_conv_dim2` and `new_conv_dim3` values might have changed due to padding. + """ + + def __init__(self, filters, + kernel_size, + strides=(1, 1, 1), + padding='valid', + data_format=None, + dilation_rate=(1, 1, 1), + activation=None, + use_bias=True, + kernel_initializer='quaternion', + bias_initializer='zeros', + kernel_regularizer=None, + bias_regularizer=None, + activity_regularizer=None, + kernel_constraint=None, + bias_constraint=None, + seed=None, + init_criterion='he', + spectral_parametrization=False, + **kwargs): + super(QuaternionConv3D, self).__init__( + rank=3, + filters=filters, + kernel_size=kernel_size, + strides=strides, + padding=padding, + data_format=data_format, + dilation_rate=dilation_rate, + activation=activation, + use_bias=use_bias, + kernel_initializer=kernel_initializer, + bias_initializer=bias_initializer, + kernel_regularizer=kernel_regularizer, + bias_regularizer=bias_regularizer, + activity_regularizer=activity_regularizer, + kernel_constraint=kernel_constraint, + bias_constraint=bias_constraint, + init_criterion=init_criterion, + spectral_parametrization=spectral_parametrization, + **kwargs) + + def get_config(self): + config = super(QuaternionConv3D, self).get_config() + config.pop('rank') + return config + +def sanitizedInitGet(init): + if init in ["sqrt_init"]: + return sqrt_init + elif init in ["complex", "complex_independent", + "glorot_complex", "he_complex", + "quaternion", "quaternion_independent"]: + return init + else: + return initializers.get(init) + +def sanitizedInitSer(init): + if init in [sqrt_init]: + return "sqrt_init" + elif init == "quaternion" or isinstance(init, QuaternionInit): + return "quaternion" + elif init == "quaternion_independent" or isinstance(init, QuaternionIndependentFilters): + return "quaternion_independent" + else: + return initializers.serialize(init) + + +# Aliases +QuaternionConvolution1D = QuaternionConv1D +QuaternionConvolution2D = QuaternionConv2D +QuaternionConvolution3D = QuaternionConv3D diff --git a/complexnn/dense.py b/complexnn/dense.py new file mode 100644 index 0000000..ec53fe0 --- /dev/null +++ b/complexnn/dense.py @@ -0,0 +1,241 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# +# Authors: Titouan Parcollet +# + +from keras import backend as K +import sys; sys.path.append('.') +from keras import backend as K +from keras import activations, initializers, regularizers, constraints +from keras.layers import Layer, InputSpec +import numpy as np +from theano.sandbox.rng_mrg import MRG_RandomStreams as RandomStreams +from .bn import sqrt_init + +class QuaternionDense(Layer): + """Regular quaternion densely-connected NN layer. + `QuaternionDense` implements the Hamilton product operation: + where `activation` is the element-wise activation function + passed as the `activation` argument, `kernel` is a weights matrix + created by the layer, and `bias` is a bias vector created by the layer + (only applicable if `use_bias` is `True`). + Note: if the input to the layer has a rank greater than 2, then + AN ERROR MESSAGE IS PRINTED. + # Arguments + units: Positive integer, dimensionality of each of the real part + and the imaginary part. It is actualy the number of complex units. + activation: Activation function to use + (see keras.activations). + If you don't specify anything, no activation is applied + (ie. "linear" activation: `a(x) = x`). + use_bias: Boolean, whether the layer uses a bias vector. + kernel_initializer: Initializer for the complex `kernel` weights matrix. + By default it is 'quaternion'. + and the usual initializers could also be used. + (see keras.initializers and init.py). + bias_initializer: Initializer for the bias vector + (see keras.initializers). + kernel_regularizer: Regularizer function applied to + the `kernel` weights matrix + (see keras.regularizers). + bias_regularizer: Regularizer function applied to the bias vector + (see keras.regularizers). + activity_regularizer: Regularizer function applied to + the output of the layer (its "activation"). + (see keras.regularizers). + kernel_constraint: Constraint function applied to the kernel matrix + (see keras.constraints). + bias_constraint: Constraint function applied to the bias vector + (see keras.constraints). + # Input shape + a 2D input with shape `(batch_size, input_dim)`. + # Output shape + For a 2D input with shape `(batch_size, input_dim)`, + the output would have shape `(batch_size, units)`. + """ + + def __init__(self, units, + activation=None, + use_bias=True, + init_criterion='he', + kernel_initializer=sqrt_init, + bias_initializer='zeros', + kernel_regularizer=None, + bias_regularizer=None, + activity_regularizer=None, + kernel_constraint=None, + bias_constraint=None, + seed=None, + **kwargs): + if 'input_shape' not in kwargs and 'input_dim' in kwargs: + kwargs['input_shape'] = (kwargs.pop('input_dim'),) + super(QuaternionDense, self).__init__(**kwargs) + self.units = units + self.activation = activations.get(activation) + self.use_bias = use_bias + self.init_criterion = init_criterion + if kernel_initializer in {'complex'}: + self.kernel_initializer = kernel_initializer + else: + self.kernel_initializer = initializers.get(kernel_initializer) + self.bias_initializer = initializers.get(bias_initializer) + self.kernel_regularizer = regularizers.get(kernel_regularizer) + self.bias_regularizer = regularizers.get(bias_regularizer) + self.activity_regularizer = regularizers.get(activity_regularizer) + self.kernel_constraint = constraints.get(kernel_constraint) + self.bias_constraint = constraints.get(bias_constraint) + if seed is None: + self.seed = np.random.randint(1, 10e6) + else: + self.seed = seed + self.input_spec = InputSpec(ndim=2) + self.supports_masking = True + + def build(self, input_shape): + assert len(input_shape) == 2 + assert input_shape[-1] % 2 == 0 + input_dim = input_shape[-1] // 4 + data_format = K.image_data_format() + kernel_shape = (input_dim, self.units) + fan_in, fan_out = initializers._compute_fans( + kernel_shape, + data_format=data_format + ) + if self.init_criterion == 'he': + s = np.sqrt(1. / fan_in) + elif self.init_criterion == 'glorot': + s = np.sqrt(1. / (fan_in + fan_out)) + rng = RandomStreams(seed=self.seed) + + # Initialization using euclidean representation: + def init_w_real(shape, dtype=None): + return rng.normal( + size=kernel_shape, + avg=0, + std=s, + dtype=dtype + ) + def init_w_imag(shape, dtype=None): + return rng.normal( + size=kernel_shape, + avg=0, + std=s, + dtype=dtype + ) + if self.kernel_initializer in {'quaternion'}: + real_init = init_w_real + imag_init = init_w_imag + else: + real_init = self.kernel_initializer + imag_init = self.kernel_initializer + + self.r = self.add_weight( + shape=kernel_shape, + initializer=real_init, + name='r', + regularizer=self.kernel_regularizer, + constraint=self.kernel_constraint + ) + self.i = self.add_weight( + shape=kernel_shape, + initializer=imag_init, + name='i', + regularizer=self.kernel_regularizer, + constraint=self.kernel_constraint + ) + self.j = self.add_weight( + shape=kernel_shape, + initializer=imag_init, + name='j', + regularizer=self.kernel_regularizer, + constraint=self.kernel_constraint + ) + self.k = self.add_weight( + shape=kernel_shape, + initializer=imag_init, + name='k', + regularizer=self.kernel_regularizer, + constraint=self.kernel_constraint + ) + + if self.use_bias: + self.bias = self.add_weight( + shape=(4 * self.units,), + initializer=self.bias_initializer, + name='bias', + regularizer=self.bias_regularizer, + constraint=self.bias_constraint + ) + else: + self.bias = None + + self.input_spec = InputSpec(ndim=2, axes={-1: 4 * input_dim}) + self.built = True + + def call(self, inputs): + input_shape = K.shape(inputs) + input_dim = input_shape[-1] // 4 + + + # + # Concatenate to obtain Hamilton matrix + # + + cat_kernels_4_r = K.concatenate([self.r, -self.i, -self.j, -self.k], axis=-1) + cat_kernels_4_i = K.concatenate([self.i, self.r, -self.k, self.j], axis=-1) + cat_kernels_4_j = K.concatenate([self.j, self.k, self.r, -self.i], axis=-1) + cat_kernels_4_k = K.concatenate([self.k, -self.j, self.i, self.r], axis=-1) + cat_kernels_4_quaternion = K.concatenate([cat_kernels_4_r, cat_kernels_4_i, cat_kernels_4_j, cat_kernels_4_k], axis=0) + + # + # Perform inference + # + + output = K.dot(inputs, cat_kernels_4_quaternion) + + r_input = output[:, :self.units] + i_input = output[:, self.units:self.units*2] + j_input = output[:, self.units*2:self.units*3] + k_input = output[:, self.units*3:] + + + output = K.concatenate([r_input, i_input, j_input, k_input], axis = -1) + + if self.use_bias: + output = K.bias_add(output, self.bias) + if self.activation is not None: + output = self.activation(output) + + return output + + def compute_output_shape(self, input_shape): + assert input_shape and len(input_shape) == 2 + assert input_shape[-1] + output_shape = list(input_shape) + output_shape[-1] = self.units * 4 + return tuple(output_shape) + + def get_config(self): + if self.kernel_initializer in {'quaternion'}: + ki = self.kernel_initializer + else: + ki = initializers.serialize(self.kernel_initializer) + config = { + 'units': self.units, + 'activation': activations.serialize(self.activation), + 'use_bias': self.use_bias, + 'init_criterion': self.init_criterion, + 'kernel_initializer': ki, + 'bias_initializer': initializers.serialize(self.bias_initializer), + 'kernel_regularizer': regularizers.serialize(self.kernel_regularizer), + 'bias_regularizer': regularizers.serialize(self.bias_regularizer), + 'activity_regularizer': regularizers.serialize(self.activity_regularizer), + 'kernel_constraint': constraints.serialize(self.kernel_constraint), + 'bias_constraint': constraints.serialize(self.bias_constraint), + 'seed': self.seed, + } + base_config = super(QuaternionDense, self).get_config() + return dict(list(base_config.items()) + list(config.items())) + diff --git a/complexnn/init.py b/complexnn/init.py new file mode 100644 index 0000000..73619f4 --- /dev/null +++ b/complexnn/init.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Contributors: Titouan Parcollet +# Authors: Chiheb Trabelsi + +import numpy as np +from numpy.random import RandomState +from random import gauss +import keras.backend as K +from keras import initializers +from keras.initializers import Initializer +from keras.utils.generic_utils import (serialize_keras_object, + deserialize_keras_object) + + +##################################################################### +# Quaternion Implementations # +##################################################################### + + +class QuaternionInit(Initializer): + # The standard complex initialization using + # either the He or the Glorot criterion. + def __init__(self, kernel_size, input_dim, + weight_dim, nb_filters=None, + criterion='he', seed=None): + + # `weight_dim` is used as a parameter for sanity check + # as we should not pass an integer as kernel_size when + # the weight dimension is >= 2. + # nb_filters == 0 if weights are not convolutional (matrix instead of filters) + # then in such a case, weight_dim = 2. + # (in case of 2D input): + # nb_filters == None and len(kernel_size) == 2 and_weight_dim == 2 + # conv1D: len(kernel_size) == 1 and weight_dim == 1 + # conv2D: len(kernel_size) == 2 and weight_dim == 2 + # conv3d: len(kernel_size) == 3 and weight_dim == 3 + + assert len(kernel_size) == weight_dim and weight_dim in {0, 1, 2, 3} + self.nb_filters = nb_filters + self.kernel_size = kernel_size + self.input_dim = input_dim + self.weight_dim = weight_dim + self.criterion = criterion + self.seed = 1337 if seed is None else seed + + def __call__(self, shape, dtype=None): + + if self.nb_filters is not None: + kernel_shape = tuple(self.kernel_size) + (int(self.input_dim), self.nb_filters) + else: + kernel_shape = (int(self.input_dim), self.kernel_size[-1]) + + fan_in, fan_out = initializers._compute_fans( + tuple(self.kernel_size) + (self.input_dim, self.nb_filters) + ) + + # Quaternion operations start here + + if self.criterion == 'glorot': + s = 1. / np.sqrt(2*(fan_in + fan_out)) + elif self.criterion == 'he': + s = 1. / np.sqrt(2*fan_in) + else: + raise ValueError('Invalid criterion: ' + self.criterion) + + #Generating randoms and purely imaginary quaternions : + number_of_weights = np.prod(kernel_shape) + v_i = np.random.uniform(0.0,1.0,number_of_weights) + v_j = np.random.uniform(0.0,1.0,number_of_weights) + v_k = np.random.uniform(0.0,1.0,number_of_weights) + #Make these purely imaginary quaternions unitary + for i in range(0, number_of_weights): + norm = np.sqrt(v_i[i]**2 + v_j[i]**2 + v_k[i]**2)+0.0001 + v_i[i]/= norm + v_j[i]/= norm + v_k[i]/= norm + v_i = v_i.reshape(kernel_shape) + v_j = v_j.reshape(kernel_shape) + v_k = v_k.reshape(kernel_shape) + + rng = RandomState(self.seed) + modulus = rng.rayleigh(scale=s, size=kernel_shape) + phase = rng.uniform(low=-np.pi, high=np.pi, size=kernel_shape) + + weight_r = modulus * np.cos(phase) + weight_i = modulus * v_i*np.sin(phase) + weight_j = modulus * v_j*np.sin(phase) + weight_k = modulus * v_k*np.sin(phase) + weight = np.concatenate([weight_r, weight_i, weight_j, weight_k], axis=-1) + + return weight + + + +class SqrtInit(Initializer): + def __call__(self, shape, dtype=None): + return K.constant(1 / K.sqrt(2), shape=shape, dtype=dtype) + + +# Aliases: +sqrt_init = SqrtInit +quaternion_independent_filters = QuaternionIndependentFilters +quaternion_init = QuaternionInit diff --git a/complexnn/utils.py b/complexnn/utils.py new file mode 100644 index 0000000..45cc296 --- /dev/null +++ b/complexnn/utils.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Authors: Titouan Parcollet + + +import keras.backend as K +from keras.layers import Layer, Lambda + +###### +# Need to rewrite this part to have only one getter for each part +# +###################### +# Quaternions TIMIT # +###################### + +def get_rpart_first(x): + ndim = K.ndim(x) + input_shape = K.shape(x) + data_format = 'channels_first' + if (data_format == 'channels_first' and ndim != 3) or ndim == 2: + input_dim = input_shape[1] // 4 + return x[:, :input_dim] + + input_dim = input_shape[-1] // 4 + if ndim == 3: + return x[:, :, :input_dim] + elif ndim == 4: + return x[:, :, :, :input_dim] + elif ndim == 5: + return x[:, :, :, :, :input_dim] + +def get_ipart_first(x): + ndim = K.ndim(x) + input_shape = K.shape(x) + data_format = 'channels_first' + if (data_format == 'channels_first' and ndim != 3) or ndim == 2: + input_dim = input_shape[1] // 4 + return x[:, input_dim:input_dim*2] + + input_dim = input_shape[-1] // 4 + if ndim == 3: + return x[:, :, input_dim:input_dim*2] + elif ndim == 4: + return x[:, :, :, input_dim:input_dim*2] + elif ndim == 5: + return x[:, :, :, :, input_dim:input_dim*2] + +def get_jpart_first(x): + ndim = K.ndim(x) + input_shape = K.shape(x) + data_format = 'channels_first' + if (data_format == 'channels_first' and ndim != 3) or ndim == 2: + input_dim = input_shape[1] // 4 + return x[:, input_dim*2:input_dim*3] + + input_dim = input_shape[-1] // 4 + if ndim == 3: + return x[:, :, input_dim*2:input_dim*3] + elif ndim == 4: + return x[:, :, :, input_dim*2:input_dim*3] + elif ndim == 5: + return x[:, :, :, :, input_dim*2:input_dim*3] + +def get_kpart_first(x): + ndim = K.ndim(x) + input_shape = K.shape(x) + data_format = 'channels_first' + if (data_format == 'channels_first' and ndim != 3) or ndim == 2: + input_dim = input_shape[1] // 4 + return x[:, input_dim*3:] + + input_dim = input_shape[-1] // 4 + if ndim == 3: + return x[:, :, input_dim*3:] + elif ndim == 4: + return x[:, :, :, input_dim*3:] + elif ndim == 5: + return x[:, :, :, :, input_dim*3:] + +class GetRFirst(Layer): + def call(self, inputs): + return get_rpart_first(inputs) + def compute_output_shape(self, input_shape): + return getpart_quaternion_output_shape_first(input_shape) +class GetIFirst(Layer): + def call(self, inputs): + return get_ipart_first(inputs) + def compute_output_shape(self, input_shape): + return getpart_quaternion_output_shape_first(input_shape) +class GetJFirst(Layer): + def call(self, inputs): + return get_jpart_first(inputs) + def compute_output_shape(self, input_shape): + return getpart_quaternion_output_shape_first(input_shape) +class GetKFirst(Layer): + def call(self, inputs): + return get_kpart_first(inputs) + def compute_output_shape(self, input_shape): + return getpart_quaternion_output_shape_first(input_shape) + +def getpart_quaternion_output_shape_first(input_shape): + returned_shape = list(input_shape[:]) + image_format = K.image_data_format() + ndim = len(returned_shape) + + data_format = 'channels_first' + if (data_format == 'channels_first' and ndim != 3) or ndim == 2: + axis = 1 + else: + axis = -1 + + returned_shape[axis] = returned_shape[axis] // 4 + + return tuple(returned_shape) + + diff --git a/datasets/__init__.py b/datasets/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/datasets/schemes.py b/datasets/schemes.py new file mode 100644 index 0000000..f5c3144 --- /dev/null +++ b/datasets/schemes.py @@ -0,0 +1,53 @@ +import six +import math +import numpy +from picklable_itertools import imap +from picklable_itertools.extras import partition_all +from fuel.schemes import BatchScheme + + +class SequentialShuffledScheme(BatchScheme): + """Sequential batches iterator. + + Iterate over all the examples in a dataset of fixed size sequentially + in batches of a given size. + + Notes + ----- + The batch size isn't enforced, so the last batch could be smaller. + + """ + def __init__(self, num_examples, batch_size, rng): + self.num_examples = num_examples + self.batch_size = batch_size + self.rng = rng + + def get_request_iterator(self): + return SequentialShuffledIterator(self.num_examples, self.batch_size, + self.rng) + +class SequentialShuffledIterator(six.Iterator): + def __init__(self, num_examples, batch_size, rng): + self.num_examples = num_examples + self.batch_size = batch_size + self.rng = rng + self.batch_indexes = range(int(math.ceil(num_examples/ float(batch_size)))) + self.rng.shuffle(self.batch_indexes) + self.current = 0 + self.current_batch = 0 + + def __iter__(self): + self.rng.shuffle(self.batch_indexes) + return self + + def __next__(self): + if self.current >= self.num_examples: + raise StopIteration + current_index = self.batch_indexes[self.current_batch] + slice_ = slice(current_index * self.batch_size, + min(self.num_examples, + (current_index + 1) * self.batch_size)) + self.current += self.batch_size + self.current_batch += 1 + return slice_ + diff --git a/datasets/schemes.pyc b/datasets/schemes.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b408262b2186adff0a2a89d92c9b723fb6812dab GIT binary patch literal 2272 zcma)7&5j#I5Uw8otnFlBmEB|sC@>O883}NXTo6Lx#Kng(7kp{W&a`*O>6tOp?Z`pd z3mb`-;l()f0Pt1y*bXaL?Touz)m`0PKVQ}K{yLsK`{Iv9PTkYP|MwW~caQ= z=M^o~Za$(#Bz#1+M^&E|eNw!qM|MEfkQT!r_sJenHKN4`@_@{UmN;Zc?}%=QeqM~J zI3h!|n#jUvzwm(XpC@D3QvQu+mME~k&vKQmgLDjOowdI7WmD_S+A#e9(=RihU(NF? zQyKpeJh&n{406NzmQh0nIGFEMJ>ibi1o&jWK!z#uo7f?yq32*}0K(=y^mKKVZ1eK;9~POQgX>!a(c^Bk-6@K4uav{}Af+ zrqVmt{R9(YnPzuAhqSFP6TU<8?2KKeETsb`Ly!a`5ce3|^hFmr_rgi4-E{rNkFcV3 zS(jexGw_agk{aX9Im`P5bpoumU05P-Xvbb$xw|4Da$#AJnWQ|$md}8CGrl%aYvQxal=`VkfW>Ig&P0? z>GuO#4OtCd0b4hU9%F9FI|N9A zid!Zea&$*}IHRBPRUd!^IFD7LPTz<1zBbRH!kB&!)*K*A?Lf`GN=>DVzVsAC;C_G9 zvO%abv^TAvgYS@f8mf%X->3WED*`Je*`AaKS2 z2yWoZq3E0iOy2+$5nW5U_Tguk{wCsHg!;xwP=?rN`0Oyi2u|4Ul-q}7LDDq$%{r7a zJ|q&0^a&Is?ar3DkrpKQsLERUnZ)T|ITk#4_F1;*`($-j&t{8e^(Ak81VZW&vU;M9 zLC?W=#h>zrEfolh51i!m8OY%#;qCwt*E8=3HLk|vabIdeTyy0%J;9>5jaH3b%CK0M z`74`Upv7>mebZQ%Vwya+#Q#{F!kIR6DF<7+{&HjNv}@2QLsVLgxE_Lz{>g^j+YQ= self.max_frames: + return i + 1 + return len(self.num_frames) + + def get_data(self, request=None): + if not self.cache[self.cache.keys()[0]]: + self._cache() + data = [] + request = self.next_request() + for source_name in self.cache: + data.append(numpy.asarray(self.cache[source_name][:request])) + self.cache = OrderedDict([(name, dt[request:]) for name, dt + in self.cache.iteritems()]) + self.num_frames = self.num_frames[request:] + + return tuple(data) + + def get_epoch_iterator(self, **kwargs): + self.cache = OrderedDict([(name, []) for name in self.sources]) + self.num_frames = [] + return super(MaximumFrameCache, self).get_epoch_iterator(**kwargs) + + def _cache(self): + data = next(self.child_epoch_iterator) + indexes = range(len(data[0])) + self.rng.shuffle(indexes) + data = [[dt[i] for i in indexes] for dt in data] + self.cache = OrderedDict([(name, self.cache[name] + dt) for name, dt + in equizip(self.data_stream.sources, data)]) + self.num_frames.extend([x.shape[0] for x in data[0]]) + + +class Transpose(Transformer): + """Transpose axes of datastream. + """ + def __init__(self, datastream, axes_list): + super(Transpose, self).__init__(datastream) + self.axes_list = axes_list + self.produces_examples = False + + def get_data(self, request=None): + data = next(self.child_epoch_iterator) + transposed_data = [] + for axes, data in zip(self.axes_list, data): + transposed_data.append(numpy.transpose(data, axes)) + return transposed_data + + +class AddUniformAlignmentMask(Transformer): + """Adds an uniform alignment mask to the incoming batch. + + Parameters + ---------- + + """ + def __init__(self, data_stream): + super(AddUniformAlignmentMask, self).__init__(data_stream) + self.sources = self.data_stream.sources + ('alignment',) + + def get_data(self, request=None): + data = next(self.child_epoch_iterator) + sources = self.data_stream.sources + + x_idx = sources.index('x') + y_idx = sources.index('y') + x_mask_idx = sources.index('x_mask') + y_mask_idx = sources.index('y_mask') + + batch_size = data[x_idx].shape[1] + max_len_output = data[y_idx].shape[0] + max_len_input = data[x_idx].shape[0] + mask_shape = (max_len_output, batch_size, max_len_input) + alignment = numpy.zeros(mask_shape, dtype=config.floatX) + + for k in xrange(batch_size): + in_size = numpy.count_nonzero(data[x_mask_idx][:,k]) + out_size = numpy.count_nonzero(data[y_mask_idx][:,k]) + n = int(in_size/out_size) # Maybe clever way than int to do this + v = numpy.hstack([numpy.ones(n, dtype=config.floatX), + numpy.zeros(max_len_input - n, + dtype=config.floatX)]) + alignment[0,k] = v + for i in xrange(1, out_size): + alignment[i,k] = numpy.roll(v, i*n) + + # DEBUG + #plt.figure() + #plt.imshow(alignment[:,k,:], cmap='gray', interpolation='none') + #plt.show() + data = data + (alignment,) + + return data + + +class AlignmentPadding(Transformer): + def __init__(self, data_stream, alignment_source): + super(AlignmentPadding, self).__init__(data_stream) + self.alignment_source = alignment_source + + def get_data(self, request=None): + data = next(self.child_epoch_iterator) + data = OrderedDict(equizip(self.sources, data)) + + alignments = data[self.alignment_source] + + input_lengths = [alignment.shape[1] for alignment in alignments] + output_lengths = [alignment.shape[0] for alignment in alignments] + max_input_length = max(input_lengths) + max_output_length = max(output_lengths) + + batch_size = len(alignments) + + padded_alignments = numpy.zeros((max_output_length, batch_size, + max_input_length)) + padded_targets = numpy.zeros((batch_size, max_output_length)) + padded_targets_mask = numpy.zeros((batch_size, max_output_length)) + for i, alignment in enumerate(alignments): + out_size, inp_size = alignment.shape + padded_alignments[:out_size, i, :inp_size] = alignment + alignment_index = [list(align).index(1) + for align in alignment] + occurance = [alignment_index.count(j) + for j in set(alignment_index)] + alignment_target = data['phonemes'][i][alignment_index] + padded_targets[i, :out_size] = alignment_target + #get rid of start label + padded_targets[i, 0] = data['phonemes'][i, 1] + alignment_target_mask = sum([[occur]*num for occur, num in + zip(data['phonemes_mask'][i], + occurance)], []) + padded_targets_mask[i, :out_size] = alignment_target_mask + data[self.alignment_source] = padded_alignments + data['phonemes'] = padded_targets.astype('int') + data['phonemes_mask'] = padded_targets_mask + return data.values() + + +class Reshape(Transformer): + """Reshapes data in the stream according to shape source.""" + def __init__(self, data_source, shape_source, **kwargs): + super(Reshape, self).__init__(**kwargs) + self.data_source = data_source + self.shape_source = shape_source + self.sources = tuple(source for source in self.data_stream.sources + if source != shape_source) + + def get_data(self, request=None): + data = next(self.child_epoch_iterator) + data = OrderedDict(zip(self.data_stream.sources, data)) + shapes = data.pop(self.shape_source) + reshaped_data = [] + for dt, shape in zip(data[self.data_source], shapes): + reshaped_data.append(dt.reshape(shape)) + data[self.data_source] = reshaped_data + return data.values() + +class ConvReshape(Transformer): + def __init__(self, data_source, quaternion, **kwargs): + super(ConvReshape, self).__init__(**kwargs) + self.data_source = data_source + self.sources = tuple(source for source in self.data_stream.sources) + self.quaternion = quaternion + def get_data(self, request=None): + data = next(self.child_epoch_iterator) + data = OrderedDict(zip(self.data_stream.sources, data)) + #shapes = data.pop(self.shape_source) + reshaped_data = [] + if self.data_source in ['features', 'X']: + for dt in data[self.data_source]: + shape = (1, dt.shape[0], 3, dt.shape[1] / 3) + if self.quaternion: + empty_channel = numpy.zeros((1, shape[1], 1, shape[3])) + reshaped_data.append(numpy.concatenate([dt.reshape(shape), + empty_channel], + axis=2)) + else: + reshaped_data.append(dt.reshape(shape)) + else: + for dt in data[self.data_source]: + shape = (numpy.prod(dt.shape), 1) + reshaped_data.append(dt.reshape(shape)) + if len(reshaped_data) == 1: + reshaped_data = reshaped_data * 2 + data['phonemes'] = numpy.repeat(data['phonemes'], 2, axis=0) + data['features_mask'] = numpy.repeat(data['features_mask'], + 2, axis=1) + data['phonemes_mask'] = numpy.repeat(data['phonemes_mask'], + 2, axis=1) + data[self.data_source] = numpy.vstack(reshaped_data) + # remove the start label + data['phonemes'] = data['phonemes'][:, 1:] + # get the indice starts at 0 + data['phonemes'] = data['phonemes'] - 1. + # remove the end label + data['phonemes'][data['phonemes']==61] = 0 + # recover original padding + data['phonemes'][data['phonemes']==-1] = 0 + data['features_mask'] = numpy.sum(data['features_mask'], axis=0)[:, None] + data['phonemes_mask'] = numpy.sum(data['phonemes_mask'], axis=0)[:, None] - 2 + return data.values() + + +class DictRep(Transformer): + def __init__(self, data_source): + super(DictRep, self).__init__(data_source) + self.data_source = data_source + + def get_data(self, request=None): + data = next(self.child_epoch_iterator) + data = OrderedDict(zip(self.data_stream.sources, data)) + return (data.values(), numpy.zeros([data['phonemes_mask'].shape[0]])) + + +class Subsample(Transformer): + def __init__(self, data_stream, source, step): + super(Subsample, self).__init__(data_stream) + self.source = source + self.step = step + + def get_data(self, request=None): + data = next(self.child_epoch_iterator) + data = OrderedDict(equizip(self.sources, data)) + dt = data[self.source] + + indexes = ((slice(None, None, self.step),) + + (slice(None),) * (len(dt.shape) - 1)) + subsampled = dt[indexes] + data[self.source] = subsampled + return data.values() + + +class WindowFeatures(Transformer): + def __init__(self, data_stream, source, window_size): + super(WindowFeatures, self).__init__(data_stream) + self.source = source + self.window_size = window_size + + def get_data(self, request=None): + data = next(self.child_epoch_iterator) + data = OrderedDict(equizip(self.sources, data)) + feature_batch = data[self.source] + + windowed_features = [] + for features in feature_batch: + features_padded = features.copy() + + features_shifted = [features] + # shift forward + for i in xrange(self.window_size / 2): + feats = numpy.roll(features_padded, i + 1, axis=0) + feats[:i + 1, :] = 0 + features_shifted.append(feats) + features_padded = features.copy() + + # shift backward + for i in xrange(self.window_size / 2): + feats = numpy.roll(features_padded, -i - 1, axis=0) + feats[-i - 1:, :] = 0 + features_shifted.append(numpy.roll(features_padded, -i - 1, + axis=0)) + windowed_features.append(numpy.concatenate( + features_shifted, axis=1)) + data[self.source] = windowed_features + return data.values() + + +class Normalize(Transformer): + """Normalizes each features : x = (x - means)/stds""" + def __init__(self, data_stream, means, stds, over='features'): + super(Normalize, self).__init__(data_stream) + self.means = means + self.stds = stds + self.over = over + + self.produces_examples = False + + def get_data(self, request=None): + data = next(self.child_epoch_iterator) + data = OrderedDict(zip(self.data_stream.sources, data)) + for i in range(len(data[self.over])): + data[self.over][i] -= self.means + data[self.over][i] /= self.stds + return data.values() + + +def length_getter(dt): + def get_length(k): + return dt[k].shape[0] + return get_length + + +class SortByLegth(Transformer): + def __init__(self, data_stream, source='features'): + super(SortByLegth, self).__init__(data_stream) + self.source = source + + def get_data(self, request=None): + data = next(self.child_epoch_iterator) + data = OrderedDict(zip(self.data_stream.sources, data)) + dt = data[self.source] + indexes = sorted(range(len(dt)), key=length_getter(dt)) + for source in self.sources: + data[source] = [data[source][k] for k in indexes] + return data.values() diff --git a/datasets/transformers.pyc b/datasets/transformers.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c422c90f3699feeadf50a5179e4980e49f1064d4 GIT binary patch literal 14289 zcmc&*U2`1Ab?w<*EItU3AOL~@Nm(NUk|2u&Wl3>j3RXk{QsUTx?4cz}jF8oGW&kX( zm|e`wihu&RlAm}1HnY$%DE1PE7>{QI%ihWiw%>lDhHFvAFK46-IW@pIU9YVco!m3$!OfYCZH0FUZ z@7x_W!BE~hT(pjuU?gunQ?#Bj!D!w(RYL zf~maqLeV;Ig6X{VV$piu1T%T-Y|%Plf=jI2oixGa{PC6I@f3M>d*LxKwei3BSpZ{A zdIs4WaS+B~u+s3;RohxctKW*&8ynqT--+u{vfhe!!q_$rqEoorX?)OVr{__Ay}sAj z>Fm4~*LT9@y1y9~hgrhYTS&>v90Cf#edIViM+BEi8Y!VB(}qz(TlmEWq9<}{2G^}AO5sEV0t zc2n*Rw-KdkV08aM8B((uITwJ*%vY$Pe?;9v2gb#E*5Q#*#s zeTQ7B=X1H6ttJ=+dPDCejITXvJi2ccWH)7;?P zH{MHHo!AeP_qt0hy03!wZrBWW!U!`)J@a6C5w2wdh-=6_Nd9*a4l2x+(cO)HS>Tuc>*>y2g;qwa4)=%O$T zP);}FR%c^V)7-!zZ{BzhgK8AC_gvOnK_;ty1)@KYrDdu?16hh+#~c93N@lBM4jl6U z2zh{#F}1QuD`vy6yft7})+Wqh#T=+u{ z@nUs>fRU1lqysVXNK-n5tX6A8jkH!vM^NdCq=%YGrL^|kG*(PiPv~VPylELd<7h)2 zUaL2gP^i+5TR{isUCYmGfs12Xpdf6n3&6;%npH!OOL-!aT_On40{$9h$t9E)ud_R5 zPp+q3NJ|#m`yOG>MM};hGtN0@&>41~MH+T|?w%`iRxuxJC4L<`X7AfP1@c(TUs-#_ zyjwDd_yzxQ5MU=BpbAdo7qC|{7pLq=a8%iB514p5e{y?w+T6hgiwSQpkTt73ZPOqf zmIcEoh2UQp1M@txT`UCe#ROHFVWiF;ccgQopx4j`LW9}V6y+x%&bHEevu~EAqnITM z_tIKS-AdADP|w&`b*_}oIcLGfQOBnVAbXj_zQ03(JIr(PP7Ix$O`g`15*wVzP%yzv z%PBOG83vHE`z0mxhi1UAx^%sc10&uF^2PbD0>-$z+I7b6;kF=B6lhx65VCfb(g*{G+dd7CGNV%3iSct4q z_`Pv-f)IkWY5%7>g#I}Sy?}Buh0HjYoH3`0(;Ro6#NS_X#!FI*GMwqj|H2#C5k>o2)!UPWbcsKl30Rg zoZ($y%z*SkoZNzx4nPgc=3>V9@8*oh6AV9Nwnnw4rEz6#(j0>IRkH=p6-0CCpKu60 z?jxcuI1#`?_ftxTzu9O8$G9&U2gYuM-XywMSSH(&&CdFIGqgua@$t?vJ9@ZM7EGTj zOR`yShw}e)pxzT0BHd|3LAVzt-W=z+z)V6P`-=wxZCRM?r$*ahWbrDxGGQI1AxDA=p+g zRGs1B;gQj?;Q`oZW8{~&TgeITHW?D0=x$q&8(#JReQGX{e6e>fw+@EVQ6aQ zJ4G&2G4#Z$DPL@@H(FBX!D+9_GM;jd!8i@>n@_wX+q3XegjF&ua+Rh1L;N7ZwA4e zQG@r&o6W{XL=XP;da~`~k$!kUw~&%KWPJ9o^Z9WxO19m!C7-(y z`K_Hsv|%qqnXCQq3zt;_62(+TlGI#7;UP*Ofq`hy>7y2)L@?#hRV(p1UGlKKm+-)z zxU}O2s+RKwM9Sx|7|6EnL7a95Jiv9NgbHt*X)FGOW=f?t^4CDkW8zOs0h?TwVJXwm zf^DENgTrL25_Gyywp!Lx7!q5BCZID6mb79PrVsb|*~1|Q=0ON%kj^Y@-vh@v95s7Cmq!Mlj+XRi=5S11&uBu-1cu}HBg{TV7lg+6 z@N7?)!IQh3GjMMY&YFXB<^qR|Ylxj2t%%L^kde^_n#avd?}H7?o%7}ZzlaloFQQ^; z@0Uq6c!BwaCQq5e2`zC>eJ4286rNzz0V;OX?tSOZ?vDW}YDXOFBAp6Gc7||a>Xqv%f98f^|$(h+EC(0%g%kw#yQcPF{fs z*{d~zJ(c!VB4*kB6;->(qN*TAjX{Kz9t*FfxDnM_owVIaMPuI7h}gmz$*-Dx4XL7d zxt;kLb}BL&Q`ic55UJDq@;&<3#(9j~LP~gK#NJU{m-Mf6-r>J<&P-|2nZv(v=Q7d+ z(zLTs9V>kf_)MIsks5+TwF-Q$q!fz+mA_RF0^p>NHvJU9%!pzE%pcmgo(=86zo^i^ zAl7}-ahOBvLogE(4Qk+|bguvv^tSH)TP%@E(N~~)&E9&t&Vm(VTDShsgwy^Z(ouxu z)vZ9tSF!P92$^~MI8Hz!)ZJ>l)RN#yw6nf_8Sn%j8@+G^K!7vzv0tLwz5(C=_;X+t zeemC;TTYjMAcIntt12T%1Aw%Mi+ky?q0^5R0Ouf4lejgKw+l@HfNTVs+RnX=fYKKo zTp{g*D{D_V@8U`^)}@%$v*r*0g*_-P1bno>in9Wcp^R0RxS0wzds(On{2J$?_F91g zH?`m#p4U|dPh=R~3H4bYP=%)u7!ZcA61(AQrRgRu_|D3e0|kh~TreAm(@dKwuK!QGqTxsnFPa1V zVlT(m-2E4zF}Nx!rRxeyH<>X=FQ}D!{R(Rwicpw9Rd8)vIy>H1 z_?Q&d6PgGW->)|j<5{3O90O?~XQ7ITMNZlP;VMCQBL(Oage~|MO~e?2aBoKi{@8e= zulG8-+L$1q2tIFjJx$!Utq8?3>YCD&9Br~})CKW2ry{?xti^F!HBFe(rz)lfUASBd z-?!wgwR}X->X9EFLm#`W7^yB|B;q?M*XvGayNQziQE47i{17QwLPkq5QyMF|j*C#t zXbJ!D_X7US;>qJq8NnH*31_$y$uIy3q8JYes$y7UC%A0P=V zWiSRgT`1$}qo|+4PRyZN*oh-Ae@?iKQv9-+?5mb6E}y@osr!B_raht`uLWI;|BEs- zv=)?Y27;4sBRg(r%B0*uP z%TIWBT-pc`m2Jl_|Cj^FjjAa-auKZ&-9?i%qGS=T)Y%dqIAy%YnRPE5oWN>)hb47+ z3MDJTBvoyvP^R_%7M+tJWW0YUvQG9kduNT85M-e z9&~*XMJApGmAb=wC>Y^omt)fYcB^e!C8qImFE=u_3gXe*>IrALoQ2Y3rGUXMYX!3Y z_Kt6ZTy(}Q?@>5SAiR0>`3d_GYY88h0S;AXPO$JfxRR8uzu|gBmnv2o9#mL*xfQ+t zP;~edUL*yD%B^@avs08CHV@vwhgPEWj-W@IAqCdkJW;JFn+X$9oN9e0Q|pJSvKOPL z$1!~$&{2*3M=r;8tENX^Kj5rFYYEd){gn0H?cg;~95q^z;(0yl`r91aRaewb6&bGd z5A4^~)W>bKfSSrAP5lw*>|awi@a_gU2&IDSd)91MP5iHB@85IJ2^aO1wLhn?30IG* zi4%d7#tX8ozyU<%!n%E3Bi)RDnz^wYf;a-Z*bbR;0)gb|3a{w~}w%r%yI($IO_ z&iTm!p~I2#zn=B(?Uy~c6TN2kmXDD8BFUvA4JTGaqALi1vzdiM;G*^{sEPg?ucF}K z0*=&oyJX_Oa!>^kQ{WG08|NbUCS?|FB#*&F$2Ki>pAQ$$F_(Ql3@5i3Kk~`vBW5YW zk)aEmU{PSDA15N;nM>xu66^zII`s6TMNiS^jG4(=K$|UPU1srl47RlN$@Vd|b8R^J zn2zMzy*c`#M{8lle7S5dwlnwgu+CtTsKMqi@79zNRMUE`%Ew#BMntABiLl>YVpBo9MXagRIl|4q70-{Q0+EZF~)X>^C$T1Bo`?A+iOc#BuLjo=jm% zJ*lsPDN@TT9C4GG>;sK4KD_av$`Rk{-bY;&@-?{pd0B@zY-3N>Cs!!NTjLaRHBC|!c7 z8G)xd=1d|@!zZ0DO_XLzGtM>V%a7O}B}V=|$*LezfSEwW&ziij?W3q49@#CVq)&^( z{!A2>WsV~Yk6Ro$hh29PQk=*8NsD#4K4PhW@SY$pMWSzGp&9-QBecsRh)e3Ylqk#` zULviH)b`;(J;0_0>F!W95to2tgoyPlF2aB*sALwg_NWO8A1*Lhc8ai578NwKA}+Uu zlpec1_P_ag4E`S^xfa%yd9-&L9@2ku0Wwo}8oJZBJNNK865hW1sOaa@N)e}H;_jD` z`gv^zfW{H#FAV5qXM5Ep?qk%gIMLY+!nb@^NcT&AA6|^n znL>^$hExZ?`#RNUO2eug6W*yHtw&w=Ge9O4o_He^Z1fyu;$Ds*TjPMN$#B; zU$mF(qjQLspXn~$^}gZHVqXkFTX*|r?_Ha*Q#f$=Jc#y_I=U*aOfH-EUDU1Rkybv9 zYR~<<%qovjcQ3)A-@-S2%WR^cSUdFVEYa2SeutSfU54zaruVvlHN8br;L!>sLXWw3 z=mCg~B8GWO&Bh!R9W2_C&Bl5f26{oH5Fwu={fq8AR=Gma{Spca00K3tK#bASaOskB z1rFAEXS93?ukN3K%XJn3!Kp`%CsfesBX~m7xQaLT2ycFHf@ypTkHe#%pWx&_#9JYW zHUxoXeSso8zTP_>zOm)Ev^fj$0rH;vHFsgpy@s!1@nOU>*ON3z#DAZonJ5E1Z+&O} z5Eso}_F+zP;|jZiL&V1^uuLpuaR6)-=2im5sl6XEBZTPuL_%hB1N;86^*%m_mFeO2 zq34?onc+X;@IPkuCwz)zvwlKmFOlgK(mZ4~T=I$66(rK8E&l-uzW}(713(mkJ{FO2llw})G%b$#x8|9_l73eb5_fv+-yk|hUHNIqx z*X5Lylbbx|E`EmhtjlB}56Ak9E|la8pT}1&l#;J=r;DzWI?)@z_^4I{WzueSce;V+rZBV2^wm zemZI2(!5UcNvm7(dw%mi2yFG6Jc5gfE%@46}8;k>f9w6FjwyK;bkV zXqwyIZD*g|3}R%Vy?=p9|Js}BAit48yhtCRSg#ohj9;(`X9~qt)uCVV6)7$M3&6Aa zyJo(gosw??jfQWIV3=Mu$HD}d;@$9wKJ!Rc71zd>%_sWU$08kMl0p;-4j9I-tLHz$ zlZS@CyH2(&xhH|ffWBnen@7hx5GU{-OX^X}em8R-Pu4qObFue-A$a6+dI?#(;cqwV_waEkzhzEat!A=_ z7m;y2vG02&^Dl7r8_eEfw#$rB5bq;qfEMG?YFpzau9NJp 0: + hasher.update(buf) + buf = afile.read(blocksize) + return hasher.digest() + +def make_local_copy(filename): + local_name = os.path.join('/Tmp/', os.environ['USER'], + os.path.basename(filename)) + if (not os.path.isfile(local_name) or + file_hash(open(filename)) != file_hash(open(local_name))): + print '.. made local copy at', local_name + shutil.copy(filename, local_name) + return local_name + +def key(x): + return x[0].shape[0] + +def construct_conv_stream(dataset, rng, pool_size, maximum_frames, + quaternion=False, **kwargs): + """Construct data stream. + Parameters: + ----------- + dataset : Dataset + Dataset to use. + rng : numpy.random.RandomState + Random number generator. + pool_size : int + Pool size for TIMIT dataset. + maximum_frames : int + Maximum frames for TIMIT datset. + subsample : bool, optional + Subsample features. + """ + stream = DataStream( + dataset, + iteration_scheme=SequentialShuffledScheme(dataset.num_examples, + pool_size, rng)) + stream = Reshape('features', 'features_shapes', data_stream=stream) + means, stds = dataset.get_normalization_factors() + stream = Normalize(stream, means, stds) + stream.produces_examples = False + stream = Mapping(stream, + SortMapping(key=key)) + stream = MaximumFrameCache(max_frames=maximum_frames, data_stream=stream, + rng=rng) + stream = Padding(data_stream=stream, + mask_sources=['features', 'phonemes']) + stream = Transpose(stream, [(0, 1, 2), (1, 0), (0, 1), (1, 0)]) + stream = ConvReshape('features', + data_stream=stream, quaternion=quaternion) + stream = Transpose(stream, [(0, 2, 3, 1), (0, 1), (0, 1), (0, 1)]) + stream.produces_examples = False + stream = Cast(stream, 'int32', which_sources=('phonemes',)) + stream = ForceFloatX(stream) + return DictRep(stream) diff --git a/scripts/models_timit.py b/scripts/models_timit.py new file mode 100644 index 0000000..382fa15 --- /dev/null +++ b/scripts/models_timit.py @@ -0,0 +1,191 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Authors: Parcollet Titouan + +# Imports + +import complexnn +from complexnn import * +import keras +from keras.callbacks import Callback, ModelCheckpoint, LearningRateScheduler +from keras.datasets import cifar10, cifar100 +from keras.initializers import Orthogonal +from keras.layers import Layer, Dropout, AveragePooling1D, + AveragePooling2D, AveragePooling3D, add, Add, concatenate, + Concatenate, Input, Flatten, Dense, Convolution2D, BatchNormalization, + Activation, Reshape, ConvLSTM2D, Conv2D, Lambda, Permute, TimeDistributed, + SpatialDropout1D, PReLU +from keras.models import Model, load_model, save_model +from keras.optimizers import SGD, Adam, RMSprop +from keras.preprocessing.image import ImageDataGenerator +from keras.regularizers import l2 +from keras.utils.np_utils import to_categorical +import keras.backend as K +import keras.models as KM +from keras.utils.training_utils import multi_gpu_model +import logging as L +import numpy as np +import os, pdb, socket, sys, time +import theano as T +from keras.backend.tensorflow_backend import set_session +import tensorflow as tf + + +# +# CTC Loss +# + +def ctc_lambda_func(args): + y_pred, labels, input_length, label_length = args + return K.ctc_batch_cost(labels, y_pred, input_length, label_length) + +# +# Get ResNet Model +# + +def getTimitModel2D(d): + n = d.num_layers + sf = d.start_filter + dataset = d.dataset + activation = d.act + advanced_act = d.aact + drop_prob = d.dropout + inputShape = (3,41,None) + filsize = (3, 5) + channelAxis = 1 + + if d.aact != "none": + d.act = 'linear' + + convArgs = { + "activation": d.act, + "data_format": "channels_first", + "padding": "same", + "bias_initializer": "zeros", + "kernel_regularizer": l2(d.l2), + "kernel_initializer": "random_uniform", + } + denseArgs = { + "activation": d.act, + "kernel_regularizer": l2(d.l2), + "kernel_initializer": "random_uniform", + "bias_initializer": "zeros", + "use_bias": True + } + + if d.model == "quaternion": + convArgs.update({"kernel_initializer": d.quat_init}) + + # + # Input Layer & CTC Parameters for TIMIT + # + if d.model == "quaternion": + I = Input(shape=(4,41,None)) + else: + I = Input(shape=inputShape) + + labels = Input(name='the_labels', shape=[None], dtype='float32') + input_length = Input(name='input_length', shape=[1], dtype='int64') + label_length = Input(name='label_length', shape=[1], dtype='int64') + + # + # Input stage: + # + if d.model == "real": + O = Conv2D(sf, filsize, name='conv', use_bias=True, **convArgs)(I) + if d.aact == "prelu": + O = PReLU(shared_axes=[1,0])(O) + else: + O = QuaternionConv2D(sf, filsize, name='conv', use_bias=True, **convArgs)(I) + if d.aact == "prelu": + O = PReLU(shared_axes=[1,0])(O) + # + # Pooling + # + O = keras.layers.MaxPooling2D(pool_size=(1, 3), padding='same')(O) + + + # + # Stage 1 + # + for i in xrange(0,n/2): + if d.model=="real": + O = Conv2D(sf, filsize, name='conv'+str(i), use_bias=True,**convArgs)(O) + if d.aact == "prelu": + O = PReLU(shared_axes=[1,0])(O) + O = Dropout(d.dropout)(O) + else: + O = QuaternionConv2D(sf, filsize, name='conv'+str(i), use_bias=True, **convArgs)(O) + if d.aact == "prelu": + O = PReLU(shared_axes=[1,0])(O) + O = Dropout(d.dropout)(O) + + # + # Stage 2 + # + for i in xrange(0,n/2): + if d.model=="real": + O = Conv2D(sf*2, filsize, name='conv'+str(i+n/2), use_bias=True, **convArgs)(O) + if d.aact == "prelu": + O = PReLU(shared_axes=[1,0])(O) + O = Dropout(d.dropout)(O) + else: + O = QuaternionConv2D(sf*2, filsize, name='conv'+str(i+n/2), use_bias=True, **convArgs)(O) + if d.aact == "prelu": + O = PReLU(shared_axes=[1,0])(O) + O = Dropout(d.dropout)(O) + + # + # Permutation for CTC + # + + O = Permute((3,1,2))(O) + O = Lambda(lambda x: K.reshape(x, (K.shape(x)[0], K.shape(x)[1], + K.shape(x)[2] * K.shape(x)[3])), + output_shape=lambda x: (None, None, x[2] * x[3]))(O) + + # + # Dense + # + if d.model== "quaternion": + O = TimeDistributed( QuaternionDense(256, **denseArgs))(O) + if d.aact == "prelu": + O = PReLU(shared_axes=[1,0])(O) + O = Dropout(d.dropout)(O) + O = TimeDistributed( QuaternionDense(256, **denseArgs))(O) + if d.aact == "prelu": + O = PReLU(shared_axes=[1,0])(O) + O = Dropout(d.dropout)(O) + O = TimeDistributed( QuaternionDense(256, **denseArgs))(O) + if d.aact == "prelu": + O = PReLU(shared_axes=[1,0])(O) + else: + O = TimeDistributed( Dense(1024, **denseArgs))(O) + if d.aact == "prelu": + O = PReLU(shared_axes=[1,0])(O) + O = Dropout(d.dropout)(O) + O = TimeDistributed( Dense(1024, **denseArgs))(O) + if d.aact == "prelu": + O = PReLU(shared_axes=[1,0])(O) + O = Dropout(d.dropout)(O) + O = TimeDistributed( Dense(1024, **denseArgs))(O) + if d.aact == "prelu": + O = PReLU(shared_axes=[1,0])(O) + + pred = TimeDistributed( Dense(62, activation='softmax', kernel_regularizer=l2(d.l2), use_bias=True, bias_initializer="zeros", kernel_initializer='random_uniform' ))(O) + + # + # CTC For sequence labelling + # + O = Lambda(ctc_lambda_func, output_shape=(1,), name='ctc')([pred, labels,input_length,label_length]) + + # Return the model + if d.dataset == "timit": + # + # Creating a function for testing and validation purpose + # + val_function = K.function([I],[pred]) + return Model(inputs=[I, input_length, labels, label_length], outputs=O), val_function + else: + raise ValueError("Unknown dataset "+d.dataset) + diff --git a/scripts/run.py b/scripts/run.py new file mode 100644 index 0000000..c1f92fb --- /dev/null +++ b/scripts/run.py @@ -0,0 +1,222 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Authors: Titouan Parcollet + +# Imports. +import sys; sys.path += [".", ".."] +import argparse as Ap +import logging as L +import numpy as np +import os, pdb, sys +import time +import tensorflow as tf +from keras.backend.tensorflow_backend import set_session +__version__ = "1.0.0" + + + +# +# Message Formatter +# + +class MsgFormatter(L.Formatter): + """Message Formatter + + Formats messages with time format YYYY-MM-DD HH:MM:SS.mmm TZ + """ + + def formatTime(self, record, datefmt): + t = record.created + timeFrac = abs(t-long(t)) + timeStruct = time.localtime(record.created) + timeString = "" + timeString += time.strftime("%F %T", timeStruct) + timeString += "{:.3f} ".format(timeFrac)[1:] + timeString += time.strftime("%Z", timeStruct) + return timeString + + + +############################################################################################################# +############################## Subcommands ################################## +############################################################################################################# + +class Subcommand(object): + name = None + + @classmethod + def addArgParser(cls, subp, *args, **kwargs): + argp = subp.add_parser(cls.name, usage=cls.__doc__, *args, **kwargs) + cls.addArgs(argp) + argp.set_defaults(__subcmdfn__=cls.run) + return argp + + @classmethod + def addArgs(cls, argp): + pass + + @classmethod + def run(cls, d): + pass + + +class Screw(Subcommand): + """Screw around with me in Screw(Subcommand).""" + name = "screw" + + @classmethod + def run(cls, d): + print(cls.__doc__) + + +class Train(Subcommand): + name = "train" + + LOGLEVELS = {"none":L.NOTSET, "debug": L.DEBUG, "info": L.INFO, + "warn":L.WARN, "err": L.ERROR, "crit": L.CRITICAL} + + + @classmethod + def addArgs(cls, argp): + argp.add_argument("-d", "--datadir", default=".", type=str, + help="Path to datasets directory.") + argp.add_argument("-w", "--workdir", default=".", type=str, + help="Path to the workspace directory for this experiment.") + argp.add_argument("-l", "--loglevel", default="info", type=str, + choices=cls.LOGLEVELS.keys(), + help="Logging severity level.") + argp.add_argument("-s", "--seed", default=0xe4223644e98b8e64, type=long, + help="Seed for PRNGs.") + argp.add_argument("--summary", action="store_true", + help="""Print a summary of the network.""") + argp.add_argument("--model", default="real", type=str, + choices=["quaternion", "real"], + help="Dataset Selection.") + argp.add_argument("--dropout", default=0, type=float, + help="Dropout probability.") + argp.add_argument("-n", "--num-epochs", default=1000, type=int, + help="Number of epochs") + argp.add_argument("--start-filter", "--sf", default=11, type=int, + help="Number of feature maps in starting stage") + argp.add_argument("--num-layers", "--nl", default=10, type=int, + help="Number of layers") + argp.add_argument("--act", default="relu", type=str, + choices=["relu","sigmoid","tanh"], + help="Activation.") + argp.add_argument("--aact", default="none", type=str, + choices=["none","prelu"], + help="Advanced Activation.") + argp.add_argument("--quat-init", default='quaternion', type=str, + help="Initializer for the quaternion kernel.") + optp = argp.add_argument_group("Optimizers", "Tunables for all optimizers") + optp.add_argument("--optimizer", "--opt", default="nag", type=str, + choices=["sgd", "nag", "adam", "rmsprop"], + help="Optimizer selection.") + optp.add_argument("--clipnorm", "--cn", default=1.0, type=float, + help="The norm of the gradient will be clipped at this magnitude.") + optp.add_argument("--clipval", "--cv", default=1.0, type=float, + help="The values of the gradients will be individually clipped at this magnitude.") + optp.add_argument("--l2", default=0, type=float, + help="L2 penalty.") + optp.add_argument("--lr", default=1e-4, type=float, + help="Master learning rate for optimizers.") + optp.add_argument("--momentum", "--mom", default=0.9, type=float, + help="Momentum for optimizers supporting momentum.") + optp.add_argument("--decay", default=0, type=float, + help="Learning rate decay for optimizers.") + optp = argp.add_argument_group("Adam", "Tunables for Adam optimizer") + optp.add_argument("--beta1", default=0.9, type=float, + help="Beta1 for Adam.") + optp.add_argument("--beta2", default=0.999, type=float, + help="Beta2 for Adam.") + optp.add_argument("--device", default="0", type=str, + help="CUDA Device, starting at 0.") + optp.add_argument("--gpus", default=1, type=int, + help="Number of GPUs to be used, starting at 1") + optp.add_argument("--memory", default=1.0, type=float, + help="Memory to be allocated on the selected device, only for tensorflow backend, from 0 to 1") + optp.add_argument("--save-prefix", default="", type=str, + help="Save prefix for resuming and saving best model") + + @classmethod + def run(cls, d): + if not os.path.isdir(d.workdir): + os.mkdir(d.workdir) + + logDir = os.path.join(d.workdir, "logs") + if not os.path.isdir(logDir): + os.mkdir(logDir) + + logFormatter = MsgFormatter ("[%(asctime)s ~~ %(levelname)-8s] %(message)s") + + stdoutLogSHandler = L.StreamHandler(sys.stdout) + stdoutLogSHandler .setLevel (cls.LOGLEVELS[d.loglevel]) + stdoutLogSHandler .setFormatter (logFormatter) + defltLogger = L.getLogger () + defltLogger .setLevel (cls.LOGLEVELS[d.loglevel]) + defltLogger .addHandler (stdoutLogSHandler) + + trainLogFilename = os.path.join(d.workdir, "logs", "train.txt") + trainLogFHandler = L.FileHandler (trainLogFilename, "a", "UTF-8", delay=True) + trainLogFHandler .setLevel (cls.LOGLEVELS[d.loglevel]) + trainLogFHandler .setFormatter (logFormatter) + trainLogger = L.getLogger ("train") + trainLogger .setLevel (cls.LOGLEVELS[d.loglevel]) + trainLogger .addHandler (trainLogFHandler) + + entryLogFilename = os.path.join(d.workdir, "logs", "entry.txt") + entryLogFHandler = L.FileHandler (entryLogFilename, "a", "UTF-8", delay=True) + entryLogFHandler .setLevel (cls.LOGLEVELS[d.loglevel]) + entryLogFHandler .setFormatter (logFormatter) + entryLogger = L.getLogger ("entry") + entryLogger .setLevel (cls.LOGLEVELS[d.loglevel]) + entryLogger .addHandler (entryLogFHandler) + + np.random.seed(d.seed % 2**32) + + import training;training.train(d) + + + + +############################################################################################################# +############################## Argument Parsers ################################# +############################################################################################################# + +def getArgParser(prog): + argp = Ap.ArgumentParser(prog = prog, + usage = None, + description = None, + epilog = None, + version = __version__) + subp = argp.add_subparsers() + argp.set_defaults(argp=argp) + argp.set_defaults(subp=subp) + + # Add global args to argp here? + # ... + + + # Add subcommands + for v in globals().itervalues(): + if(isinstance(v, type) and + issubclass(v, Subcommand) and + v != Subcommand): + v.addArgParser(subp) + + # Return argument parser. + return argp + + + +############################################################################################################# +############################## Main ################################## +############################################################################################################# + +def main(argv): + sys.setrecursionlimit(10000) + d = getArgParser(argv[0]).parse_args(argv[1:]) + return d.__subcmdfn__(d) +if __name__ == "__main__": + main(sys.argv) + diff --git a/scripts/training.py b/scripts/training.py new file mode 100644 index 0000000..df3e8b2 --- /dev/null +++ b/scripts/training.py @@ -0,0 +1,470 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Authors: Parcollet Titouan + +# Imports +import editdistance +import h5py +import datasets.timit +from datasets.timit import Timit +from datasets.utils import construct_conv_stream, phone_to_phoneme_dict +import complexnn +from complexnn import * +import h5py as H +import keras +from keras.callbacks import Callback, ModelCheckpoint, LearningRateScheduler +from keras.initializers import Orthogonal +from keras.layers import Layer, Dropout, AveragePooling1D, AveragePooling2D, + AveragePooling3D, add, Add, concatenate, Concatenate, + Input, Flatten, Dense, Convolution2D, BatchNormalization, + Activation, Reshape, ConvLSTM2D, Conv2D, Lambda +from keras.models import Model, load_model, save_model +from keras.optimizers import SGD, Adam, RMSprop +from keras.regularizers import l2 +from keras.utils.np_utils import to_categorical +import keras.backend as K +import keras.models as KM +from keras.utils.training_utils import multi_gpu_model +import logging as L +import numpy as np +import os, pdb, socket, sys, time +import theano as T +from keras.backend.tensorflow_backend import set_session +from models_timit import getTimitResnetModel2D,ctc_lambda_func +import tensorflow as tf +import itertools +import random + + +# +# Generator wrapper for timit +# + +def timitGenerator(stream): + while True: + for data in stream.get_epoch_iterator(): + yield data + +# +# Custom metrics +# +class EditDistance(Callback): + def __init__(self, func, dataset, quaternion, save_prefix): + self.func = func + if(dataset in ['train','test','dev']): + self.dataset_type = dataset + self.save_prefix = save_prefix + self.dataset = Timit(str(dataset)) + self.full_phonemes_dict = self.dataset.get_phoneme_dict() + self.ind_phonemes_dict = self.dataset.get_phoneme_ind_dict() + self.rng = np.random.RandomState(123) + self.data_stream = construct_conv_stream(self.dataset, self.rng, 20, 1000,quaternion) + + else: + raise ValueError("Unknown dataset for edit distance "+dataset) + + def labels_to_text(self,labels): + ret = [] + for c in labels: + if c == len(self.full_phonemes_dict) - 2: + ret.append("") + else: + c_ = self.full_phonemes_dict[c + 1] + ret.append(phone_to_phoneme_dict.get(c_, c_)) + ret = [k for k, g in itertools.groupby(ret)] + return list(filter(lambda c: c != "", ret)) + + def decode_batch(self, out, mask): + ret = [] + for j in range(out.shape[0]): + out_best = list(np.argmax(out[j], 1))[:int(mask[j])] + out_best = [k for k, g in itertools.groupby(out_best)] + # map from 61-d to 39-d + out_str = self.labels_to_text(out_best) + ret.append(out_str) + return ret + + def on_epoch_end(self, epoch, logs={}): + mean_norm_ed = 0. + num = 0 + for data in self.data_stream.get_epoch_iterator(): + x, y = data + y_pred = self.func([x[0]])[0] + decoded_y_pred = self.decode_batch(y_pred, x[1]) + decoded_gt = [] + for i in range(x[2].shape[0]): + decoded_gt.append(self.labels_to_text(x[2][i][:int(x[3][i])])) + num += len(decoded_y_pred) + for i, (_pred, _gt) in enumerate(zip(decoded_y_pred, decoded_gt)): + edit_dist = editdistance.eval(_pred, _gt) + mean_norm_ed += float(edit_dist) / float(len(_gt)) + mean_norm_ed = mean_norm_ed / num + + # Dump To File Logs at every epoch for clusters sbatch + f=open(str(self.save_prefix)+"_"+str(self.dataset_type)+"_PER.txt",'ab') + mean = np.array([mean_norm_ed]) + np.savetxt(f,mean) + f.close() + L.getLogger("train").info("PER on "+str(self.dataset_type)+" : "+str(mean_norm_ed)+" at epoch "+str(epoch)) + +# +# Callbacks: +# + +class TrainLoss(Callback): + def __init__(self, savedir): + self.savedir = savedir + def on_epoch_end(self, epoch, logs={}): + f=open(str(self.savedir)+"_train_loss.txt",'ab') + f2=open(str(self.savedir)+"_dev_loss.txt",'ab') + value = float(logs['loss']) + np.savetxt(f,np.array([value])) + f.close() + value = float(logs['val_loss']) + np.savetxt(f2,np.array([value])) + f2.close() +# +# Print a newline after each epoch, because Keras doesn't. Grumble. +# + +class PrintNewlineAfterEpochCallback(Callback): + def on_epoch_end(self, epoch, logs={}): + sys.stdout.write("\n") +# +# Save checkpoints. +# + +class SaveLastModel(Callback): + def __init__(self, workdir, save_prefix, model_mono,period=10): + self.workdir = workdir + self.model_mono = model_mono + self.chkptsdir = os.path.join(self.workdir, "chkpts") + self.save_prefix = save_prefix + if not os.path.isdir(self.chkptsdir): + os.mkdir(self.chkptsdir) + self.period_of_epochs = period + self.linkFilename = os.path.join(self.chkptsdir, str(save_prefix)+"ModelChkpt.hdf5") + self.linkFilename_weight = os.path.join(self.chkptsdir, str(save_prefix)+"ModelChkpt_weight.hdf5") + + def on_epoch_end(self, epoch, logs={}): + if (epoch + 1) % self.period_of_epochs == 0: + + # Filenames + baseHDF5Filename = str(self.save_prefix)+"ModelChkpt{:06d}.hdf5".format(epoch+1) + baseHDF5Filename_weight = str(self.save_prefix)+"ModelChkpt{:06d}_weight.hdf5".format(epoch+1) + baseYAMLFilename = str(self.save_prefix)+"ModelChkpt{:06d}.yaml".format(epoch+1) + hdf5Filename = os.path.join(self.chkptsdir, baseHDF5Filename) + hdf5Filename_weight = os.path.join(self.chkptsdir, baseHDF5Filename_weight) + yamlFilename = os.path.join(self.chkptsdir, baseYAMLFilename) + + # YAML + yamlModel = self.model_mono.to_yaml() + with open(yamlFilename, "w") as yamlFile: + yamlFile.write(yamlModel) + + # HDF5 + KM.save_model(self.model_mono, hdf5Filename) + self.model_mono.save_weights(hdf5Filename_weight) + with H.File(hdf5Filename, "r+") as f: + f.require_dataset("initialEpoch", (), "uint64", True)[...] = int(epoch+1) + f.flush() + with H.File(hdf5Filename_weight, "r+") as f: + f.require_dataset("initialEpoch", (), "uint64", True)[...] = int(epoch+1) + f.flush() + + + # Symlink to new HDF5 file, then atomically rename and replace. + os.symlink(baseHDF5Filename_weight, self.linkFilename_weight+".rename") + os.rename (self.linkFilename_weight+".rename", + self.linkFilename_weight) + + + # Symlink to new HDF5 file, then atomically rename and replace. + os.symlink(baseHDF5Filename, self.linkFilename+".rename") + os.rename (self.linkFilename+".rename", + self.linkFilename) + + # Print + L.getLogger("train").info("Saved checkpoint to {:s} at epoch {:5d}".format(hdf5Filename, epoch+1)) + +# +# Summarize environment variable. +# + +def summarizeEnvvar(var): + if var in os.environ: return var+"="+os.environ.get(var) + else: return var+" unset" + +# +# TRAINING PROCESS +# + +def train(d): + + # + # + # Log important data about how we were invoked. + # + L.getLogger("entry").info("INVOCATION: "+" ".join(sys.argv)) + L.getLogger("entry").info("HOSTNAME: "+socket.gethostname()) + L.getLogger("entry").info("PWD: "+os.getcwd()) + L.getLogger("entry").info("CUDA DEVICE: "+str(d.device)) + os.environ["CUDA_VISIBLE_DEVICES"]=str(d.device) + + # + # Setup GPUs + # + config = tf.ConfigProto() + + # + # Don't pre-allocate memory; allocate as-needed + # + config.gpu_options.allow_growth = True + + # + # Only allow a total of half the GPU memory to be allocated + # + config.gpu_options.per_process_gpu_memory_fraction = d.memory + + # + # Create a session with the above options specified. + # + K.tensorflow_backend.set_session(tf.Session(config=config)) + + summary = "\n" + summary += "Environment:\n" + summary += summarizeEnvvar("THEANO_FLAGS")+"\n" + summary += "\n" + summary += "Software Versions:\n" + summary += "Theano: "+T.__version__+"\n" + summary += "Keras: "+keras.__version__+"\n" + summary += "\n" + summary += "Arguments:\n" + summary += "Path to Datasets: "+str(d.datadir)+"\n" + summary += "Number of GPUs: "+str(d.datadir)+"\n" + summary += "Path to Workspace: "+str(d.workdir)+"\n" + summary += "Model: "+str(d.model)+"\n" + summary += "Number of Epochs: "+str(d.num_epochs)+"\n" + summary += "Number of Start Filters: "+str(d.start_filter)+"\n" + summary += "Number of Layers: "+str(d.num_layers)+"\n" + summary += "Optimizer: "+str(d.optimizer)+"\n" + summary += "Learning Rate: "+str(d.lr)+"\n" + summary += "Learning Rate Decay: "+str(d.decay)+"\n" + summary += "Clipping Norm: "+str(d.clipnorm)+"\n" + summary += "Clipping Value: "+str(d.clipval)+"\n" + summary += "Dropout Probability: "+str(d.dropout)+"\n" + if d.optimizer in ["adam"]: + summary += "Beta 1: "+str(d.beta1)+"\n" + summary += "Beta 2: "+str(d.beta2)+"\n" + else: + summary += "Momentum: "+str(d.momentum)+"\n" + summary += "Save Prefix: "+str(d.save_prefix)+"\n" + L.getLogger("entry").info(summary[:-1]) + + # + # Load dataset + # + L.getLogger("entry").info("Loading dataset {:s} ...".format(d.dataset)) + np.random.seed(d.seed % 2**32) + + # + # Create training data generator + # + dataset = Timit('train') + rng=np.random.RandomState(123) + if d.model =="quaternion": + data_stream_train = construct_conv_stream(dataset, rng, 200, 1000, quaternion=True) + else: + data_stream_train = construct_conv_stream(dataset, rng, 200, 1000, quaternion=False) + + # + # Create dev data generator + # + dataset = Timit('dev') + rng=np.random.RandomState(123) + if d.model =="quaternion": + data_stream_dev = construct_conv_stream(dataset, rng, 200, 10000, quaternion=True) + else: + data_stream_dev = construct_conv_stream(dataset, rng, 200, 1000, quaternion=False) + + + L.getLogger("entry").info("Training set length: "+str(Timit('train').num_examples)) + L.getLogger("entry").info("Validation set length: "+str(Timit('dev').num_examples)) + L.getLogger("entry").info("Test set length: "+str(Timit('test').num_examples)) + L.getLogger("entry").info("Loaded dataset {:s}.".format(d.dataset)) + + # + # Optimizers + # + if d.optimizer in ["sgd", "nag"]: + opt = SGD (lr = d.lr, + momentum = d.momentum, + decay = d.decay, + nesterov = (d.optimizer=="nag"), + clipnorm = d.clipnorm) + elif d.optimizer == "rmsprop": + opt = RMSProp(lr = d.lr, + decay = d.decay, + clipnorm = d.clipnorm) + elif d.optimizer == "adam": + opt = Adam (lr = d.lr, + beta_1 = d.beta1, + beta_2 = d.beta2, + decay = d.decay, + clipnorm = d.clipnorm) + else: + raise ValueError("Unknown optimizer "+d.optimizer) + + + # + # Initial Entry or Resume ? + # + + initialEpoch = 0 + chkptFilename = os.path.join(d.workdir, "chkpts", str(d.save_prefix)+"ModelChkpt.hdf5") + chkptFilename_weight = os.path.join(d.workdir, "chkpts", str(d.save_prefix)+"ModelChkpt_weight.hdf5") + isResuming = os.path.isfile(chkptFilename) + isResuming_weight = os.path.isfile(chkptFilename_weight) + + if isResuming or isResuming_weight: + + # Reload Model and Optimizer + if d.dataset == "timit": + L.getLogger("entry").info("Re-Creating the model from scratch.") + model_mono,test_func = getTimitResnetModel2D(d) + model_mono.load_weights(chkptFilename_weight) + with H.File(chkptFilename_weight, "r") as f: + initialEpoch = int(f["initialEpoch"][...]) + L.getLogger("entry").info("Training will restart at epoch {:5d}.".format(initialEpoch+1)) + L.getLogger("entry").info("Compilation Started.") + + else: + + L.getLogger("entry").info("Reloading a model from "+chkptFilename+" ...") + np.random.seed(d.seed % 2**32) + model = KM.load_model(chkptFilename, custom_objects={ + "QuaternionConv2D": QuaternionConv2D, + "QuaternionConv1D": QuaternionConv1D, + "GetIFirst": GetIFirst, + "GetJFirst": GetJFirst, + "GetKFirst": GetKFirst, + "GetRFirst": GetRFirst, + }) + L.getLogger("entry").info("... reloading complete.") + with H.File(chkptFilename, "r") as f: + initialEpoch = int(f["initialEpoch"][...]) + L.getLogger("entry").info("Training will restart at epoch {:5d}.".format(initialEpoch+1)) + L.getLogger("entry").info("Compilation Started.") + else: + model_mono,test_func = getTimitModel2D(d) + + L.getLogger("entry").info("Compilation Started.") + + # + # Multi GPU: Can only save the model_mono because of keras bug + # + if d.gpus >1: + model = multi_gpu_model(model_mono, gpus=d.gpus) + else: + model = model_mono + + # + # Compile with CTC koss function + # + model.compile(opt, loss={'ctc': lambda y_true, y_pred: y_pred}) + + + # + # Precompile several backend functions + # + if d.summary: + model.summary() + L.getLogger("entry").info("# of Parameters: {:10d}".format(model.count_params())) + L.getLogger("entry").info("Compiling Train Function...") + t =- time.time() + model._make_train_function() + t += time.time() + L.getLogger("entry").info(" {:10.3f}s".format(t)) + L.getLogger("entry").info("Compiling Predict Function...") + t =- time.time() + model._make_predict_function() + t += time.time() + L.getLogger("entry").info(" {:10.3f}s".format(t)) + L.getLogger("entry").info("Compiling Test Function...") + t =- time.time() + model._make_test_function() + t += time.time() + L.getLogger("entry").info(" {:10.3f}s".format(t)) + L.getLogger("entry").info("Compilation Ended.") + + # + # Create Callbacks + # + newLineCb = PrintNewlineAfterEpochCallback() + saveLastCb = SaveLastModel(d.workdir, d.save_prefix, model_mono, period=10) + + + callbacks = [] + + # + # End of line for better looking + # + callbacks += [newLineCb] + if d.model=="quaternion": + quaternion = True + else: + quaternion = False + + savedir = d.workdir+"/LOGS/"+d.save_prefix + + # + # Save the Train loss + # + trainLoss = TrainLoss(savedir) + + # + # Compute accuracies and save + # + editDistValCb = EditDistance(test_func,'dev',quaternion, savedir) + editDistTestCb = EditDistance(test_func,'test',quaternion, savedir) + callbacks += [trainLoss] + callbacks += [editDistValCb] + callbacks += [editDistTestCb] + + callbacks += [newLineCb] + + # + # Save the model + # + callbacks += [saveLastCb] + + # + # Enter training loop. + # + L .getLogger("entry").info("**********************************************") + if isResuming: L.getLogger("entry").info("*** Reentering Training Loop @ Epoch {:5d} ***".format(initialEpoch+1)) + else: L.getLogger("entry").info("*** Entering Training Loop @ First Epoch ***") + L .getLogger("entry").info("**********************************************") + + + # + # TRAIN + # + + ######## + # Make sure to give the right number of mini_batch size + # needed to complete ONE epoch (according to your data generator) + ######## + + epochs_train = 1144 + epochs_dev = 121 + + model.fit_generator(generator = timitGenerator(data_stream_train), + steps_per_epoch = epochs_train, + epochs = d.num_epochs, + verbose = 1, + validation_data = timitGenerator(data_stream_dev), + validation_steps = epochs_dev, + callbacks = callbacks, + initial_epoch = initialEpoch) + diff --git a/setup.py b/setup.py new file mode 100755 index 0000000..abed74a --- /dev/null +++ b/setup.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python +from setuptools import setup + +with open('README.md') as f: + DESCRIPTION = f.read() + + +setup( + name='Quaternion Convolutional Neural Networks for automatic speech recognition', + version='1', + license='MIT', + long_description=DESCRIPTION, + packages=['complexnn'], + scripts=['scripts/run.py', 'scripts/training.py'], + install_requires=[ + "numpy", "scipy", "sklearn", "Theano", "keras", "kerosene", "tensorflow-gpu"] + +) -- 1.8.2.3