Feature: Spectral Normalization Layer

[oovm]

Seems to does not need to implementation when forward

const POWER_ITERATION = 1
const EPSILON = 1e-12

let weight: TensorT<C_out,C_in,K,K>
let bias: TensorT<C_out>
let u: TensorT<C_out>
let v: TensorT<C_in * K^2>

function spectral_norm() {
  w_mat = reshape(weight, [weight.shape[0], -1])
  _u = u
  _v = None
  for _ in range(POWER_ITERATION):
    _v = L2Normalization(dot(_u, w_mat))
    _u = L2Normalization(dot(_v, w_mat.T))
  sigma = sum(dot(_u, w_mat) * _v)
  return weight / sqrt(sigma + EPSILON)
}

function spectral_norm_forward() {
  // x shape is batch_size x in_channels x height x width
  return Convolution(x,
    weight = spectral_norm(weight),
    bias = bias
  )
}

[oovm]

mat = FlattenLayer[-2][weight];
{v, u} = {v/Norm[v.Transpose[mat], 2], u/Norm[u.mat, 2]};
newWeight = weight/Sqrt[Tr[v*(u.mat)]^2 + 1*^-12]