// nnet3/nnet-convolutional-component.h
  
  // Copyright      2017  Johns Hopkins University (author: Daniel Povey)
  
  // See ../../COPYING for clarification regarding multiple authors
  //
  // Licensed under the Apache License, Version 2.0 (the "License");
  // you may not use this file except in compliance with the License.
  // You may obtain a copy of the License at
  //
  //  http://www.apache.org/licenses/LICENSE-2.0
  //
  // THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  // KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
  // WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
  // MERCHANTABLITY OR NON-INFRINGEMENT.
  // See the Apache 2 License for the specific language governing permissions and
  // limitations under the License.
  
  #ifndef KALDI_NNET3_NNET_CONVOLUTIONAL_COMPONENT_H_
  #define KALDI_NNET3_NNET_CONVOLUTIONAL_COMPONENT_H_
  
  #include "nnet3/nnet-common.h"
  #include "nnet3/nnet-component-itf.h"
  #include "nnet3/natural-gradient-online.h"
  #include "nnet3/convolution.h"
  #include <iostream>
  
  namespace kaldi {
  namespace nnet3 {
  
  /// @file  nnet-convolutional-component.h
  ///
  /// This file can be viewed as 'overflow' from nnet-general-component.h.
  /// It contains a number of components which implement some kind of
  /// convolution.
  
  
  /**
     TimeHeightConvolutionComponent implements 2-dimensional convolution where one
     of the dimensions of convolution (which traditionally would be called the
     width axis) is identified with time (i.e. the 't' component of Indexes).  For
     a deeper understanding of how this works, please see convolution.h.
  
     The following are the parameters accepted on the config line, with examples
     of their values.
  
  
     Parameters inherited from UpdatableComponent (see comment above declaration of
     UpdadableComponent in nnet-component-itf.h for details):
         learning-rate, learning-rate-factor, max-change
  
     Convolution-related parameters:
  
       num-filters-in   E.g. num-filters-in=32.  Number of input filters (the
                        number of separate versions of the input image).  The
                        filter-dim has stride 1 in the input and output vectors,
                        i.e. we order the input as (all-filters-for-height=0,
                        all-filters-for-height=1, etc.)
       num-filters-out  E.g. num-filters-out=64. The number of output filters (the
                        number of separate versions of the output image).  As with
                        the input, the filter-dim has stride 1.
       height-in        E.g. height-in=40.  The height of the input image.  The
                        width is not specified the the model level, as it's
                        identified with "t" and is called the time axis; the width
                        is determined by how many "t" values were available at the
                        input of the network, and how many were requested at the
                        output.
       height-out       E.g. height-out=40.  The height of the output image.  Will
                        normally be <= (the input height divided by
                        height-subsample-out).
       height-subsample-out E.g. height-subsample-out=2 (defaults to 1).
                        Subsampling factor on the height axis, e.g. you might set
                        this to 2 if you are doing subsampling on this layer,
                        which would involve discarding every other height
                        increment at the output.  There is no corresponding config
                        for the time dimension, as time subsampling is determined
                        by which 't' values you request at the output, together
                        with the values of 'time-offsets' at different layers of
                        the network.
       height-offsets   E.g. height-offsets=-1,0,1 The set of height offsets that
                        contribute to each output pixel: with the values -1,0,1,
                        height 10 at the output would see data from heights
                        9,10,11 at the input.  These values will normally be
                        consecutive.  Negative values imply zero-padding on the
                        bottom of the image, since output-height 0 is always
                        defined.  Zero-padding at the top of the image is
                        determined in a similar way (e.g. if height-in==height-out
                        and height-offsets=-1,0,1, then there is 1 pixel of
                        padding at the top and bottom of the image).
       time-offsets     E.g. time-offsets=-1,0,1 The time offsets that we require
                        at the input to produce a given output; these are
                        comparable to the offsets used in TDNNs.  Note that the
                        time axis is always numbered using an absolute scheme, so
                        that if there is subsampling on the time axis, then later
                        in the network you'll see time-offsets like "-2,0,2" or
                        "-4,0,4".  Subsampling on the time axis is not explicitly
                        specified but is implicit based on tracking dependencies.
       offsets          Setting 'offsets' is an alternative to setting both
                        height-offsets and time-offsets, that is useful for
                        configurations with less regularity.  It is a semicolon-
                        separated list of pairs (time-offset,height-offset) that
                        might look like: -1,1;-1,0;-1,1;0,1;....;1,1
       required-time-offsets E.g. required-time-offsets=0 (defaults to the same
                        value as time-offsets).  This is a set of time offsets,
                        which if specified must be a nonempty subset of
                        time-offsets; it determines whether zero-padding on the
                        time axis is allowed in cases where there is insufficient
                        input.  If not specified it defaults to the same as
                        'time-offsets', meaning there is no zero-padding on the
                        time axis.  Note: for speech tasks we tend to pad on the
                        time axis with repeats of the first or last frame, rather
                        than zero; and this is handled while preparing the data
                        and not by the core components of the nnet3 framework.  So
                        for speech tasks we wouldn't normally set this value.
       max-memory-mb    Maximum amount of temporary memory, in megabytes, that may
                        be used as temporary matrices in the convolution computation.
                        default=200.0.
  
     Initialization parameters:
        param-stddev    Standard deviation of the linear parameters of the
                        convolution.  Defaults to sqrt(1.0 / (num-filters-in *
                        num-height-offsets * num-time-offsets)), e.g.
                        sqrt(1.0/(64*3*3)) for a 3x3 kernel with 64 input
                        filters; this value will ensure that the output has
                        unit stddev if the input has unit stddev.
        bias-stddev     Standard deviation of bias terms.  default=0.0.
        init-unit       Defaults to false.  If true, it is required that
                        num-filters-in equal num-filters-out and there should
                        exist a (height, time) offset in the model equal to (0,
                        0).  We will initialize the parameter matrix to be
                        equivalent to the identity transform.  In this case,
                        param-stddev is ignored.
  
  
     Natural-gradient related options are below; you won't normally have to
     set these.
  
        use-natural-gradient e.g. use-natural-gradient=false (defaults to true).
                         You can set this to false to disable the natural gradient
                         updates (you won't normally want to do this).
        rank-out        Rank used in low-rank-plus-unit estimate of the Fisher-matrix
                        factor that has the dimension (num-rows of the parameter
                        space), which equals num-filters-out.  It
                        defaults to the minimum of 80, or half of the number of
                        output filters.
        rank-in         Rank used in low-rank-plus-unit estimate of the Fisher
                        matrix factor which has the dimension (num-cols of the
                        parameter matrix), which has the dimension
                        (num-input-filters * number of time-offsets * number of
                        height-offsets + 1), e.g. num-input-filters * 3 * 3 + 1
                        for a 3x3 kernel (the +1 is for the bias term).
                        It defaults to the minimum of 80, or half the
                        num-rows of the parameter matrix.  [note: I'm considering
                        decreasing this default to e.g. 40 or 20].
        num-minibatches-history
                        This is used setting the 'num_samples_history'
                        configuration value of the natural gradient object.
                        There is no concept of samples (frames) in the
                        application of natural gradient to the convnet, because
                        we do it all on the rows and columns of the derivative.
                        default=4.0.  A larger value means the Fisher matrix is
                        averaged over more minibatches (it's an exponential-decay
                        thing).
        alpha-out       Constant that determines how much we smooth the
                        Fisher-matrix factors with the unit matrix, for the
                        space of dimension num-filters-out.  default=4.0.
        alpha-in        Constant that determines how much we smooth the
                        Fisher-matrix factors with the unit matrix, for the
                        space of dimension (num-filters-in * num-time-offsets *
                        num-height-offsets + 1).  default=4.0.
  
  
     Example of a 3x3 kernel with no subsampling, and with zero-padding on both
     the the height and time axis, and where there has previously been no
     subsampling on the time axis:
  
       num-filters-in=32 num-filters-out=64 height-in=28 height-out=28 \
         height-subsample-out=1 height-offsets=-1,0,1 time-offsets=-1,0,1 \
         required-time-offsets=0
  
     Example of a 3x3 kernel with no subsampling, without zero-padding on
     either axis, and where there has *previously* been 2-fold subsampling
     on the time axis:
  
       num-filters-in=32 num-filters-out=64 height-in=20 height-out=18 \
         height-subsample-out=1 height-offsets=0,1,2 time-offsets=0,2,4
  
     [note: above, the choice to have the time-offsets start at zero rather than
     be centered is just a choice: it assumes that at the output of the network
     you would want to request indexes with t=0, while at the input the t values
     start from zero.]
  
     Example of a 3x3 kernel with subsampling on the height axis,
     without zero-padding on either axis, and where there has
     previously been 2-fold subsampling on the time axis:
  
       num-filters-in=32 num-filters-out=64 height-in=20 height-out=9 \
         height-subsample-out=2 height-offsets=0,1,2 time-offsets=0,2,4
  
    [note: subsampling on the time axis is not expressed in the layer itself:
    any time you increase the distance between time-offsets, like changing
    them from 0,1,2 to 0,2,4, you are effectively subsampling the previous
    layer-- assuming you only request the output at one time value or at
    multiples of the total subsampling factor.]
  
    Example of a 1x1 kernel:
  
       num-filters-in=64 num-filters-out=64 height-in=20 height-out=20 \
         height-subsample-out=1 height-offsets=0 time-offsets=0
   */
  class TimeHeightConvolutionComponent: public UpdatableComponent {
   public:
  
    // The use of this constructor should only precede InitFromConfig()
    TimeHeightConvolutionComponent();
  
    // Copy constructor
    TimeHeightConvolutionComponent(const TimeHeightConvolutionComponent &other);
  
    virtual int32 InputDim() const;
    virtual int32 OutputDim() const;
  
    virtual std::string Info() const;
    virtual void InitFromConfig(ConfigLine *cfl);
    virtual std::string Type() const { return "TimeHeightConvolutionComponent"; }
    virtual int32 Properties() const {
      return kUpdatableComponent|kReordersIndexes|kBackpropAdds|
          kBackpropNeedsInput|kInputContiguous|kOutputContiguous;
    }
    virtual void* Propagate(const ComponentPrecomputedIndexes *indexes,
                           const CuMatrixBase<BaseFloat> &in,
                           CuMatrixBase<BaseFloat> *out) const;
    virtual void Backprop(const std::string &debug_info,
                          const ComponentPrecomputedIndexes *indexes,
                          const CuMatrixBase<BaseFloat> &in_value,
                          const CuMatrixBase<BaseFloat> &out_value,
                          const CuMatrixBase<BaseFloat> &out_deriv,
                          void *memo,
                          Component *to_update,
                          CuMatrixBase<BaseFloat> *in_deriv) const;
  
    virtual void Read(std::istream &is, bool binary);
    virtual void Write(std::ostream &os, bool binary) const;
    virtual Component* Copy() const {
      return new TimeHeightConvolutionComponent(*this);
    }
  
  
    // Some functions that are only to be reimplemented for GeneralComponents.
  
    // This ReorderIndexes function may insert 'blank' indexes (indexes with
    // t == kNoTime) as well as reordering the indexes.  This is allowed
    // behavior of ReorderIndexes functions.
    virtual void ReorderIndexes(std::vector<Index> *input_indexes,
                                std::vector<Index> *output_indexes) const;
  
    virtual void GetInputIndexes(const MiscComputationInfo &misc_info,
                                 const Index &output_index,
                                 std::vector<Index> *desired_indexes) const;
  
    // This function returns true if at least one of the input indexes used to
    // compute this output index is computable.
    virtual bool IsComputable(const MiscComputationInfo &misc_info,
                              const Index &output_index,
                              const IndexSet &input_index_set,
                              std::vector<Index> *used_inputs) const;
  
    virtual ComponentPrecomputedIndexes* PrecomputeIndexes(
        const MiscComputationInfo &misc_info,
        const std::vector<Index> &input_indexes,
        const std::vector<Index> &output_indexes,
        bool need_backprop) const;
  
    // Some functions from base-class UpdatableComponent.
    virtual void Scale(BaseFloat scale);
    virtual void Add(BaseFloat alpha, const Component &other);
    virtual void PerturbParams(BaseFloat stddev);
    virtual BaseFloat DotProduct(const UpdatableComponent &other) const;
    virtual int32 NumParameters() const;
    virtual void Vectorize(VectorBase<BaseFloat> *params) const;
    virtual void UnVectorize(const VectorBase<BaseFloat> &params);
    virtual void FreezeNaturalGradient(bool freeze);
  
  
    class PrecomputedIndexes: public ComponentPrecomputedIndexes {
     public:
      PrecomputedIndexes() { }
      PrecomputedIndexes(const PrecomputedIndexes &other):
          computation(other.computation) { }
      virtual PrecomputedIndexes *Copy() const;
      virtual void Write(std::ostream &os, bool binary) const;
      virtual void Read(std::istream &os, bool binary);
      virtual std::string Type() const {
        return "TimeHeightConvolutionComponentPrecomputedIndexes";
      }
      virtual ~PrecomputedIndexes() { }
  
      time_height_convolution::ConvolutionComputation computation;
    };
  
    void ScaleLinearParams(BaseFloat alpha) { linear_params_.Scale(alpha); }
  
    void ConsolidateMemory();
   private:
  
    void Check() const;
  
    // computes derived parameters required_time_offsets_ and all_time_offsets_.
    void ComputeDerived();
  
    // Function that updates linear_params_ and bias_params_, which
    // uses the natural gradient code.
    void UpdateNaturalGradient(
        const PrecomputedIndexes &indexes,
        const CuMatrixBase<BaseFloat> &in_value,
        const CuMatrixBase<BaseFloat> &out_deriv);
  
    // Function that updates linear_params_ and bias_params_, which
    // does not use the natural gradient code.
    void UpdateSimple(
        const PrecomputedIndexes &indexes,
        const CuMatrixBase<BaseFloat> &in_value,
        const CuMatrixBase<BaseFloat> &out_deriv);
  
    // Function called to initialize linear_params_ if init-unit=true in the config
    // line.
    void InitUnit();
  
    time_height_convolution::ConvolutionModel model_;
  
    // all_time_offsets_ is a copy of the corresponding variable in
    // model, stored as a vector instead of as a set for efficiency.
    std::vector<int32> all_time_offsets_;
    // time_offset_required_ is a vector with the same dimension as
    // 'all_time_offsets_', which is true if the corresponding time-offset
    // is a member of model_.required_time_offsets_.
    std::vector<bool> time_offset_required_;
  
    // the linear parameters of the convolution.
    // dimension is model_.ParamRows() by model.ParamCols(),
    // which equals num-filters-out by
    // (num-filters-in * patch-rows * patch-cols),
    // a.k.a.
    // (num-filters-in * num-time-offsets * num-height-offset).
    CuMatrix<BaseFloat> linear_params_;
    // the bias parameters of the convolution, dimension is
    // model_.num_filters_out.
    CuVector<BaseFloat> bias_params_;
  
  
    // Maximum amount of temporary memory in megabytes that is allowed to be used
    // in the convolution computation.  (this is per computation, but it's
    // released immediately after it's used, so it doesn't matter how many there
    // are).
    BaseFloat max_memory_mb_;
  
    // Controls whether or not the natural-gradient is used.
    // Note: even if this is true, if is_gradient_ (from the
    // UpdatableComponent base class) is true, we'll do the 'simple'
    // update that doesn't include natural gradient.
    bool use_natural_gradient_;
  
    // Preconditioner for the input space, of dimension linear_params_.NumCols() +
    // 1 (the 1 is for the bias).  As with other natural-gradient objects, it's
    // not stored with the model on disk but is reinitialized each time we start
    // up.
    OnlineNaturalGradient preconditioner_in_;
  
    // Preconditioner for the output space, of dimension
    // linear_params_.NumRows().
    OnlineNaturalGradient preconditioner_out_;
  };
  
  
  
  /**
     TdnnComponent is a more memory-efficient alternative to manually splicing
     several frames of input and then using a NaturalGradientAffineComponent or
     a LinearComponent.  It does the splicing of the input itself, using
     mechanisms similar to what TimeHeightConvolutionComponent uses.  The
     implementation is in nnet-tdnn-component.cc
  
     Parameters inherited from UpdatableComponent (see comment above declaration of
     UpdadableComponent in nnet-component-itf.h for details):
         learning-rate, learning-rate-factor, max-change
  
     Important parameters:
  
       input-dim         The input feature dimension (before splicing).
  
       output-dim        The output feature dimension
  
       time-offsets     E.g. time-offsets=-1,0,1 or time-offsets=-3,0,3.
                        The time offsets that we require at the input to produce a given output.
                        comparable to the offsets used in TDNNs.  They
                        must be unique (no repeats).
       use-bias         Defaults to true, but set to false if you want this to
                        be linear rather than affine in its input.
  
  
    Extra parameters:
      orthonormal-constraint=0.0 If you set this to 1.0, then the linear_params_
                        matrix will be (approximately) constrained during training
                        to have orthonormal rows (or columns, whichever is
                        fewer).. it turns out the real name for this is a
                        "semi-orthogonal" matrix.  You can choose a positive
                        nonzero value different than 1.0 to have a scaled
                        semi-orthgonal matrix, i.e. with singular values at the
                        selected value (e.g. 0.5, or 2.0).  This is not enforced
                        inside the component itself; you have to call
                        ConstrainOrthonormal() from the training code to do this.
                        All this component does is return the
                        OrthonormalConstraint() value.  If you set this to a
                        negative value, it's like saying "for any value", i.e. it
                        will constrain the parameter matrix to be closer to "any
                        alpha" times a semi-orthogonal matrix, without changing
                        its overall norm.
  
  
     Initialization parameters:
        param-stddev    Standard deviation of the linear parameters of the
                        convolution.  Defaults to
                        sqrt(1.0 / (input-dim * the number of time-offsets))
        bias-stddev     Standard deviation of bias terms.  default=0.0.
                        You should not set this if you set use-bias=false.
  
  
     Natural-gradient related options are below; you won't normally have to
     set these as the defaults are reasonable.
  
        use-natural-gradient e.g. use-natural-gradient=false (defaults to true).
                         You can set this to false to disable the natural gradient
                         updates (you won't normally want to do this).
        rank-out         Rank used in low-rank-plus-unit estimate of the Fisher-matrix
                         factor that has the dimension (num-rows of linear_params_),
                         which equals output_dim.  It
                         defaults to the minimum of 80, or half of the output dim.
        rank-in          Rank used in low-rank-plus-unit estimate of the Fisher
                         matrix factor which has the dimension (num-cols of the
                         parameter matrix), which is input-dim times the number of
                         time offsets.  It defaults to the minimum of 20, or half the
                         num-rows of the parameter matrix.
        num-samples-history
                        This becomes the 'num_samples_history'
                        configuration value of the natural gradient objects.  The
                        default value is 2000.0.
  
   */
  class TdnnComponent: public UpdatableComponent {
   public:
  
    // The use of this constructor should only precede InitFromConfig()
    TdnnComponent();
  
    // Copy constructor
    TdnnComponent(const TdnnComponent &other);
  
    virtual int32 InputDim() const {
      return linear_params_.NumCols() / static_cast<int32>(time_offsets_.size());
    }
    virtual int32 OutputDim() const { return linear_params_.NumRows(); }
  
    virtual std::string Info() const;
    virtual void InitFromConfig(ConfigLine *cfl);
    virtual std::string Type() const { return "TdnnComponent"; }
    virtual int32 Properties() const {
      return kUpdatableComponent|kReordersIndexes|kBackpropAdds|
          (bias_params_.Dim() == 0 ? kPropagateAdds : 0)|
          kBackpropNeedsInput;
    }
    virtual void* Propagate(const ComponentPrecomputedIndexes *indexes,
                           const CuMatrixBase<BaseFloat> &in,
                           CuMatrixBase<BaseFloat> *out) const;
    virtual void Backprop(const std::string &debug_info,
                          const ComponentPrecomputedIndexes *indexes,
                          const CuMatrixBase<BaseFloat> &in_value,
                          const CuMatrixBase<BaseFloat> &out_value,
                          const CuMatrixBase<BaseFloat> &out_deriv,
                          void *memo,
                          Component *to_update,
                          CuMatrixBase<BaseFloat> *in_deriv) const;
  
    virtual void Read(std::istream &is, bool binary);
    virtual void Write(std::ostream &os, bool binary) const;
    virtual Component* Copy() const {
      return new TdnnComponent(*this);
    }
  
  
    // Some functions that are only to be reimplemented for GeneralComponents.
  
    // This ReorderIndexes function may insert 'blank' indexes (indexes with
    // t == kNoTime) as well as reordering the indexes.  This is allowed
    // behavior of ReorderIndexes functions.
    virtual void ReorderIndexes(std::vector<Index> *input_indexes,
                                std::vector<Index> *output_indexes) const;
  
    virtual void GetInputIndexes(const MiscComputationInfo &misc_info,
                                 const Index &output_index,
                                 std::vector<Index> *desired_indexes) const;
  
    // This function returns true if at least one of the input indexes used to
    // compute this output index is computable.
    virtual bool IsComputable(const MiscComputationInfo &misc_info,
                              const Index &output_index,
                              const IndexSet &input_index_set,
                              std::vector<Index> *used_inputs) const;
  
    virtual ComponentPrecomputedIndexes* PrecomputeIndexes(
        const MiscComputationInfo &misc_info,
        const std::vector<Index> &input_indexes,
        const std::vector<Index> &output_indexes,
        bool need_backprop) const;
  
    // Some functions from base-class UpdatableComponent.
    virtual void Scale(BaseFloat scale);
    virtual void Add(BaseFloat alpha, const Component &other);
    virtual void PerturbParams(BaseFloat stddev);
    virtual BaseFloat DotProduct(const UpdatableComponent &other) const;
    virtual int32 NumParameters() const;
    virtual void Vectorize(VectorBase<BaseFloat> *params) const;
    virtual void UnVectorize(const VectorBase<BaseFloat> &params);
    virtual void FreezeNaturalGradient(bool freeze);
  
  
    class PrecomputedIndexes: public ComponentPrecomputedIndexes {
     public:
      PrecomputedIndexes() { }
      PrecomputedIndexes(const PrecomputedIndexes &other):
          row_stride(other.row_stride), row_offsets(other.row_offsets) { }
      virtual PrecomputedIndexes *Copy() const;
      virtual void Write(std::ostream &os, bool binary) const;
      virtual void Read(std::istream &os, bool binary);
      virtual std::string Type() const {
        return "TdnnComponentPrecomputedIndexes";
      }
      virtual ~PrecomputedIndexes() { }
  
  
      // input_row_stride is the stride (in number of rows) we have to take in the
      // input matrix each time we form a sub-matrix that will be part of the
      // input to the tdnn operation.  Normally this will be 1, but it may be,
      // for example, 3 in layers where we do subsampling.
      int32 row_stride;
  
      // 'row_offsets' is of the same dimension as time_offsets_.  Each element
      // describes the row offset (in the input matrix) of a sub-matrix, and each.
      // We will append together these sub-matrices (row-wise) to be the input to
      // the affine or linear transform.
      std::vector<int32> row_offsets;
    };
  
    CuMatrixBase<BaseFloat> &LinearParams() { return linear_params_; }
  
    // This allows you to resize the vector in order to add a bias where
    // there previously was none-- obviously this should be done carefully.
    CuVector<BaseFloat> &BiasParams() { return bias_params_; }
  
    BaseFloat OrthonormalConstraint() const { return orthonormal_constraint_; }
  
    void ConsolidateMemory();
   private:
  
    // This static function is a utility function that extracts a CuSubMatrix
    // representing a subset of rows of 'input_matrix'.
    // The numpy syntax would be:
    //   return input_matrix[row_offset:row_stride:num_output_rows*row_stride,:]
    static CuSubMatrix<BaseFloat> GetInputPart(
        const CuMatrixBase<BaseFloat> &input_matrix,
        int32 num_output_rows,
        int32 row_stride,
        int32 row_offset);
  
    // see the definition for more explanation.
    static void ModifyComputationIo(time_height_convolution::ConvolutionComputationIo *io);
  
    void Check() const;
  
    // Function that updates linear_params_, and bias_params_ if present, which
    // uses the natural gradient code.
    void UpdateNaturalGradient(
        const PrecomputedIndexes &indexes,
        const CuMatrixBase<BaseFloat> &in_value,
        const CuMatrixBase<BaseFloat> &out_deriv);
  
    // Function that updates linear_params_, and bias_params_ if present, which
    // does not use the natural gradient code.
    void UpdateSimple(
        const PrecomputedIndexes &indexes,
        const CuMatrixBase<BaseFloat> &in_value,
        const CuMatrixBase<BaseFloat> &out_deriv);
  
  
  
  
    // time_offsets_ is the list of time-offsets of the input that
    // we append together; it will typically be (-1,0,1) or (-3,0,3).
    std::vector<int32> time_offsets_;
  
    // the linear parameters of the network; its NumRows() is the output
    // dim, and its NumCols() equals the input dim times time_offsets_.size().
    CuMatrix<BaseFloat> linear_params_;
  
    // the bias parameters if this is an affine transform, or the empty vector if
    // this is a linear operation (i.e. use-bias == false in the config).
    CuVector<BaseFloat> bias_params_;
  
    // If nonzero, this controls how we apply an orthonormal constraint to the
    // parameter matrix; see docs for ConstrainOrthonormal() in nnet-utils.h.
    // This class just returns the value via the OrthonormalConstraint() function;
    // it doesn't actually do anything with it directly.
    BaseFloat orthonormal_constraint_;
  
    // Controls whether or not the natural-gradient is used.  Note: even if this
    // is true, if is_gradient_ (from the UpdatableComponent base class) is true,
    // we'll do the 'simple' update that doesn't include natural gradient.
    bool use_natural_gradient_;
  
    // Preconditioner for the input space, of dimension linear_params_.NumCols() +
    // 1 (the 1 is for the bias).  As with other natural-gradient objects, it's
    // not stored with the model on disk but is reinitialized each time we start
    // up.
    OnlineNaturalGradient preconditioner_in_;
  
    // Preconditioner for the output space, of dimension
    // linear_params_.NumRows().
    OnlineNaturalGradient preconditioner_out_;
  };
  
  
  
  
  
  
  
  } // namespace nnet3
  } // namespace kaldi
  
  
  #endif