Source code for epynn.rnn.backward

# EpyNN/epynn/rnn/
# Related third party imports
import numpy as np

[docs]def initialize_backward(layer, dX): """Backward cache initialization. :param layer: An instance of RNN layer. :type layer: :class:`epynn.rnn.models.RNN` :param dX: Output of backward propagation from next layer. :type dX: :class:`numpy.ndarray` :return: Input of backward propagation for current layer. :rtype: :class:`numpy.ndarray` :return: Next hidden state initialized with zeros. :rtype: :class:`numpy.ndarray` """ if layer.sequences: dA = dX # Full length sequence elif not layer.sequences: dA = np.zeros(layer.fs['h']) # Empty full length sequence dA[:, -1] = dX # Assign to last index cache_keys = ['dh_', 'dh', 'dhn'] layer.bc.update({k: np.zeros(layer.fs['h']) for k in cache_keys}) layer.bc['dA'] = dA layer.bc['dX'] = np.zeros(layer.fs['X']) # To previous layer dh = layer.bc['dh'][:, 0] # To previous step return dA, dh
[docs]def rnn_backward(layer, dX): """Backward propagate error gradients to previous layer. """ # (1) Initialize cache and hidden state gradient dA, dh = initialize_backward(layer, dX) # Reverse iteration over sequence steps for s in reversed(range(layer.d['s'])): # (2s) Slice sequence (m, s, u) w.r.t step dA = layer.bc['dA'][:, s] # dL/dA # (3s) Gradient of the loss w.r.t. next hidden state dhn = layer.bc['dhn'][:, s] = dh # dL/dhn # (4s) Gradient of the loss w.r.t hidden state h_ dh_ = layer.bc['dh_'][:, s] = ( (dA + dhn) * layer.activate(layer.fc['h_'][:, s], deriv=True) ) # dL/dh_ - To parameters gradients # (5s) Gradient of the loss w.r.t hidden state h dh = layer.bc['dh'][:, s] = (, layer.p['V'].T) ) # dL/dh - To previous step # (6s) Gradient of the loss w.r.t X dX = layer.bc['dX'][:, s] = (, layer.p['U'].T) ) # dL/dX - To previous layer dX = layer.bc['dX'] return dX # To previous layer