May
2016
Stochastic Neural Networks with Monotonic Activation Functions
Authors:
Abstract:
We propose a Laplace approximation that creates a stochastic unit from any smooth monotonic activation function, using only Gaussian noise. This paper investigates the application of this stochastic approximation in training a family of Restricted Boltzmann Machines (RBM) that are closely linked to Bregman divergences. This family, that we call exponential family RBM (Exp-RBM), is a subset of the exponential family Harmoniums that expresses family members through a choice of smooth monotonic non-linearity for each neuron. Using contrastive divergence along with our Gaussian approximation, we show that Exp-RBM can learn useful representations using novel stochastic units.
copied = false, 2000);
">
@conference{Ravanbakhsh-2016-119753,
author = {S. Ravanbakhsh And B. Poczos And J. Schneider And D. Schuurmans And R. Greiner},
title = {Stochastic Neural Networks with Monotonic Activation Functions},
booktitle = {Proceedings of 19th International Conference on Artificial Intelligence and Statistics (AISTATS '16)},
year = {2016},
month = {May},
volume = {51},
pages = {809 - 818},
}
author = {S. Ravanbakhsh And B. Poczos And J. Schneider And D. Schuurmans And R. Greiner},
title = {Stochastic Neural Networks with Monotonic Activation Functions},
booktitle = {Proceedings of 19th International Conference on Artificial Intelligence and Statistics (AISTATS '16)},
year = {2016},
month = {May},
volume = {51},
pages = {809 - 818},
}