Restricted Boltzmann Machines

Sigmoid / binary

P (h_{j} = 1 | v) = s i g m o i d (b_{j}^{h} + \sum_{i} v_{i} w_{i, j})

P (v_{i} = 1 | h) = s i g m o i d (b_{i}^{v} + \sum_{j} h_{j} w_{i, j})

E (v, h) = - \sum_{i} b_{i}^{v} v_{i} - \sum_{j} b_{j}^{h} h_{j} - \sum_{i} \sum_{j} v_{i} h_{j} w_{i, j}

P (v, h) = \frac{1}{Z} e^{- E (v, h)}

Z = \sum_{v^{'}} \sum_{h^{'}} e^{- E (v^{'}, h^{'})}

P (v) = \frac{1}{Z} \sum_{h^{'}} e^{- E (v, h^{'})} = \frac{1}{Z} e^{- F (v)}

F (v) = - l n (\sum_{h^{'}} e^{- E (v, h^{'})}) = - \sum_{i} b_{i}^{v} v_{i} - \sum_{j} s o f t p l u s (b_{j}^{h} + \sum_{i} v_{i} w_{i, j})

\frac{\partial \log F (v)}{\partial w_{i, j}} = ⟨ v_{i} h_{j} ⟩_{d a t a} - ⟨ v_{i} h_{j} ⟩_{m o d e l}

\nabla w_{i, j} = ϵ (⟨ v_{i} h_{j} ⟩_{d a t a} - ⟨ v_{i} h_{j} ⟩_{m o d e l})

\nabla b_{i}^{v} = ϵ (⟨ v_{i} ⟩_{d a t a} - ⟨ v_{i} ⟩_{m o d e l})

\nabla b_{j}^{h} = ϵ (⟨ h_{j} ⟩_{d a t a} - ⟨ h_{j} ⟩_{m o d e l})

P (v_{i_{0}, i_{1}} = 1 | h) = \frac{e^{b_{i_{0}, i_{1}}^{v} + \sum_{j} h_{j} w_{i_{0}, i_{1}, j}}}{\sum_{{i_{1}}^{'}} e^{b_{i_{0}, {i_{1}}^{'}}^{v} + \sum_{j} h_{j} w_{i_{0}, {i_{1}}^{'}, j}}}

E (v, h) = - \sum_{i_{0}, i_{1}} b_{i_{0}, i_{1}}^{v} v_{i_{0}, i_{1}} - \sum_{j} b_{j}^{h} h_{j} - \sum_{i_{0}, i_{1}} \sum_{j} v_{i_{0}, i_{1}} h_{j} w_{i_{0}, i_{1}, j}

E (v, h) = - \sum_{i_{0}, i_{1}} b_{i_{0}, i_{1}}^{v} v_{i_{0}, i_{1}} - \sum_{j_{0}, j_{1}} b_{j_{0}, j_{1}}^{h} h_{j_{0}, j_{1}} - \sum_{i_{0}, i_{1}} \sum_{j_{0}, j_{1}} v_{i_{0}, i_{1}} h_{j_{0}, j_{1}} w_{i_{0}, i_{1}, j_{1}, j_{0}}

E (v, h) = \sum_{i} \frac{(v_{i} - b_{i}^{v})^{2}}{2 σ_{i}^{2}} - \sum_{j} b_{j}^{h} h_{j} - \sum_{i} \sum_{j} \frac{v_{i}}{σ_{i}^{2}} h_{j} w_{i, j}

P (v_{i} | h) = N (v_{i} | b_{i}^{v} + \sum_{j} h_{j} w_{i, j}, σ_{i}^{2})

P (h_{j} = 1 | x, y) = s i g m o i d (b_{j}^{h} + \sum_{i^{x}} x_{i^{x}} w_{i^{x}, j}^{x, h} + \sum_{i_{0}^{y}, i_{1}^{y}, j} y_{i_{0}^{y}, i_{1}^{y}} w_{i_{0}^{y}, i_{1}^{y}, j}^{y_{0}, y_{1}, h})

E (x, y, h) = - \sum_{j} b_{j}^{h} h_{j} - \sum_{i^{x}} b_{i^{x}}^{x} x_{i^{x}} - \sum_{i_{0}^{y}, i_{1}^{y}} b_{i_{0}^{y}, i_{1}^{y}}^{y} y_{i_{0}^{y}, i_{1}^{y}} - \sum_{i^{x}, j} x_{i^{x}} h_{j} w_{i^{x}, j}^{x, h} - \sum_{i_{0}^{y}, i_{1}^{y}, j} y_{i_{0}^{y}, i_{1}^{y}} h_{j} w_{i_{0}^{y}, i_{1}^{y}, j}^{y_{0}, y_{1}, h}

P (x, y, h) = \frac{1}{Z} e^{- E (x, y, h)}

P (x, y) = \frac{1}{Z} \sum_{h^{'}} e^{- E (x, y, h^{'})} = \frac{1}{Z} e^{- F (x, y)}

F (x, y) = - \sum_{i^{x}} b_{i^{x}}^{x} x_{i^{x}} - \sum_{i_{0}^{y}, i_{1}^{y}} b_{i_{0}^{y}, i_{1}^{y}}^{y} y_{i_{0}^{y}, i_{1}^{y}} - \sum_{j} s o f t p l u s (b_{j}^{h} + \sum_{i^{x}} x_{i^{x}} w_{i^{x}, j}^{x, h} + \sum_{i_{0}^{y}, i_{1}^{y}} y_{i_{0}^{y}, i_{1}^{y}} w_{i_{0}^{y}, i_{1}^{y}, j}^{y_{0}, y_{1}, h})

P (y_{i_{0}, i_{1}} = 1 | h) = \frac{e^{b_{i_{0}, i_{1}}^{y} + \sum_{j} h_{j} w_{i_{0}, i_{1}, j}^{y_{0}, y_{1}, h}}}{\sum_{{i_{1}}^{'}} e^{b_{i_{0}, {i_{1}}^{'}}^{y} + \sum_{j} h_{j} w_{i_{0}, {i_{1}}^{'}, j}^{y_{0}, y_{1}, h}}}

P (y_{i_{0}, i_{1}} = 1 | x) = \frac{\sum_{h^{'}} e^{- E (x, y, h^{'})}}{\sum_{y^{'}} \sum_{h^{'}} e^{- E (x, y^{'}, h^{'})}} = \frac{e^{b_{i_{0}, i_{1}}^{y} + \sum_{j} s o f t p l u s (b_{j}^{h} + \sum_{i^{x}} x_{i^{x}} w_{i^{x}, j}^{x, h} + w_{i_{0}^{y}, i_{1}^{y}, j}^{y_{0}, y_{1}, h})}}{\sum_{y^{'}} e^{b_{i_{0}, i_{1}}^{y} + \sum_{j} s o f t p l u s (b_{j}^{h} + \sum_{i^{x}} x_{i^{x}} w_{i^{x}, j}^{x, h} + w_{i_{0}^{y}, i_{1}^{y}, j}^{y_{0}, y_{1}, h})}} = \frac{e^{- F (y_{i_{0}, i_{1}} | x)}}{\sum_{i_{1}^{^{'}}} e^{- F (y_{i_{0}, i_{1}^{^{'}}} | x)}}

F (y_{i_{0}, i_{1}} | x) = - b_{i_{0}, i_{1}}^{y} - \sum_{j} s o f t p l u s (b_{j}^{h} + \sum_{i^{x}} x_{i^{x}} w_{i^{x}, j}^{x, h} + w_{i_{0}^{y}, i_{1}^{y}, j}^{y_{0}, y_{1}, h})