Lock the weight when updating cache

/**

• @note This [[Adagrad]] hyperparameter is usually used before global [[LearningRate]].
• e.g. Adagrad with FixedLearningRate, not FixedLearningRate with Adagrad */ trait Adagrad extends com.thoughtworks.deeplearning.plugins.INDArrayWeights {

import org.nd4j.linalg.api.ndarray.INDArray import org.nd4j.linalg.factory.Nd4j import org.nd4j.linalg.ops.transforms.Transforms

/** The hyperparmaeter eps, which should be configured in Factory.newInstance() */ def eps: Double

trait INDArrayWeightApi extends super.INDArrayWeightApi { this: INDArrayWeight => /** The cache state injected to INDArrayWeight by this Adagrad plug-in */ var cache: Option[INDArray] = None }

override type INDArrayWeight <: INDArrayWeightApi with Weight

trait INDArrayOptimizerApi extends super.INDArrayOptimizerApi { this: INDArrayOptimizer => private lazy val delta0: INDArray = { // The original delta computed by previous plug-ins val superDelta = super.delta

  import org.nd4s.Implicits._
  import weight._
  val newCache = weight.synchronized {
    val newCache = weight.cache.getOrElse(Nd4j.zeros(superDelta.shape: _*)) + superDelta * superDelta
    weight.cache = Some(newCache)
    newCache
  }
  superDelta / (Transforms.sqrt(newCache) + eps)
}

/** The computation of delta for injected to `INDArrayOptimizer` by this `Adagrad` plug-in */
override def delta = delta0

} override type INDArrayOptimizer <: INDArrayOptimizerApi with Optimizer }