RidgeRegression_SmoothedL1TV

[JinpengLI]

When I use RidgeRegression_SmoothedL1TV, sometimes beta become NaN. Maybe it need to add a warning when one of "l, k, g" is zero.

>>> import numpy as np
>>> import scipy.sparse as sparse
>>> import parsimony.estimators as estimators
>>> import parsimony.algorithms as algorithms
>>> import parsimony.tv
>>> shape = (4, 4, 1)
>>> num_samples = 10
>>> num_ft = shape[0] * shape[1] * shape[2]
>>> X = np.random.random((num_samples, num_ft))
>>> y = np.random.randint(0, 2, (num_samples, 1))
>>> spams_X =np.asarray([[  5.19856444e-01,   1.98565067e-02,  -1.06309916e-01,
...           5.30122459e-01,   3.69935878e-01,   3.76416443e-01,
...          -2.75350802e-01,  -2.88576527e-01,   1.81536696e-01,
...           4.86466087e-01,   2.98503534e-01,  -5.42962362e-02,
...          -3.39623886e-01,  -4.17748409e-01,  -5.73760251e-01,
...          -3.11696224e-01],
...        [ -3.94913310e-01,   3.19148830e-01,   6.43585845e-01,
...           5.49464971e-02,  -1.10775436e-01,  -2.75163767e-02,
...          -2.02034877e-01,  -3.29979686e-01,  -5.12041453e-01,
...           5.81334752e-01,   1.67907704e-01,   2.46442195e-01,
...          -4.55725709e-01,  -3.27178329e-01,  -4.26616325e-01,
...           1.68637091e-01],
...        [ -3.72266759e-02,  -7.10474145e-03,   1.42787906e-01,
...           1.91548868e-01,   3.69253836e-01,   4.73065994e-01,
...           1.11353590e-01,   6.10831910e-02,   2.08244357e-01,
...           2.50066524e-02,   3.24586006e-01,  -4.76911867e-01,
...           1.69032466e-01,   3.03174816e-01,   2.52657616e-01,
...          -5.37322535e-01],
...        [  4.52838558e-01,  -1.27114912e-01,  -3.35896474e-01,
...          -2.32936688e-01,  -3.59032968e-01,  -2.78976102e-01,
...           1.46243199e-01,  -3.96277749e-01,   3.34488449e-01,
...          -3.61910011e-01,  -3.26741577e-01,   3.30131899e-01,
...           3.56461502e-01,  -3.87681439e-02,   4.23534702e-01,
...           1.53836289e-04],
...        [  2.18819443e-01,   5.63504650e-01,  -4.43473156e-01,
...           5.02363938e-01,  -3.65050875e-01,   1.36012048e-01,
...          -5.12909327e-01,   1.39278314e-01,   2.66383330e-01,
...          -3.09367330e-01,  -4.04753248e-01,   2.33583752e-01,
...           1.78987067e-01,  -5.88413655e-02,   1.75993568e-01,
...           4.32252962e-01],
...        [ -2.27411474e-01,  -1.27272384e-01,  -1.25057541e-01,
...          -2.36513812e-01,  -2.23241600e-02,  -4.49169611e-01,
...           2.16310732e-01,   2.06231913e-01,  -3.16904786e-01,
...           2.74856854e-02,  -8.78854596e-02,   3.15211150e-01,
...          -5.20870987e-01,  -2.71470677e-01,  -2.50429675e-01,
...           3.40254560e-01],
...        [ -2.03663680e-01,  -3.41453268e-01,  -1.47585127e-01,
...          -4.35952948e-01,   4.62036983e-01,  -1.06573080e-01,
...           2.01262221e-01,  -9.27743079e-02,  -1.88653825e-01,
...           1.77845390e-02,   3.62285704e-01,  -4.30052620e-01,
...          -4.62360048e-02,   3.16073577e-02,  -7.48430152e-02,
...           3.02609967e-01],
...        [ -4.50619036e-01,  -2.52575914e-01,   3.07222868e-01,
...          -3.55573390e-01,  -4.76314038e-01,   8.21010644e-02,
...           1.32610158e-05,   5.92303921e-01,   1.75775869e-01,
...           1.30670614e-01,  -4.62587543e-01,  -1.14250158e-01,
...           2.48787410e-01,  -1.82025241e-01,  -3.78943512e-02,
...          -1.80102707e-01],
...        [  1.49841248e-01,   4.04475484e-01,   2.67640128e-01,
...          -1.36276496e-02,   2.89758941e-02,  -4.87710672e-01,
...          -3.16551882e-01,   3.85866923e-01,  -4.62137557e-01,
...          -2.71754334e-01,   3.34082498e-01,   3.26301249e-01,
...           1.71913626e-02,   6.41452619e-01,   1.71683064e-01,
...           1.53373837e-01],
...        [ -2.75215161e-02,  -4.51464252e-01,  -2.02914532e-01,
...          -4.37727385e-03,   1.03294886e-01,   2.82350293e-01,
...           6.31663884e-01,  -2.77155992e-01,   3.13308920e-01,
...          -3.25716655e-01,  -2.05397620e-01,  -3.76159362e-01,
...           3.91996780e-01,   3.19797373e-01,   3.39674666e-01,
...          -3.68160788e-01]])
>>> spams_Y =np.asarray([[-0.20701967],
...        [ 0.48304589],
...        [-0.20701967],
...        [ 0.48304589],
...        [-0.20701967],
...        [ 0.48304589],
...        [-0.20701967],
...        [-0.20701967],
...        [-0.20701967],
...        [-0.20701967]])
>>> Atv, n_compacts = parsimony.tv.A_from_shape(shape)
>>> Al1 = sparse.eye(num_ft, num_ft)
>>> k = 0.05  # ridge regression coefficient
>>> l = 0  # l1 coefficient
>>> g = 0  # tv coefficient
>>> rr = estimators.RidgeRegression_SmoothedL1TV(
...             k, l, g,
...             Atv=Atv, Al1=Al1,
...             algorithm=algorithms.ExcessiveGapMethod(max_iter=1000))
>>> rr.fit(spams_X, spams_Y)
/home/jinpeng/github/pylearn-structured/parsimony/algorithms.py:239: RuntimeWarning: invalid value encountered in divide
  v /= np.sqrt(np.sum(v ** 2.0))
print rr.beta

k = 0  # ridge regression coefficient
<parsimony.estimators.RidgeRegression_SmoothedL1TV object at 0x216d550>
>>> [[ nan]
 [ nan]
 [ nan]
 [ nan]
 [ nan]
 [ nan]
 [ nan]
 [ nan]
 [ nan]
 [ nan]
 [ nan]
 [ nan]
 [ nan]
 [ nan]
 [ nan]
 [ nan]]
>>> >>> >>> l = 0.05  # l1 coefficient
>>> g = 0  # tv coefficient
>>> rr = estimators.RidgeRegression_SmoothedL1TV(
...             k, l, g,
...             Atv=Atv, Al1=Al1,
...             algorithm=algorithms.ExcessiveGapMethod(max_iter=1000))
>>> rr.fit(spams_X, spams_Y)
print rr.beta

<parsimony.estimators.RidgeRegression_SmoothedL1TV object at 0x216d750>
>>> [[ nan]
 [ nan]
 [ nan]
 [ nan]
 [ nan]
 [ nan]
 [ nan]
 [ nan]
 [ nan]
 [ nan]
 [ nan]
 [ nan]
 [ nan]
 [ nan]
 [ nan]
 [ nan]]
>>> >>> 
>>> k = 0.05  # ridge regression coefficient
>>> l = 0.05  # l1 coefficient
>>> g = 0  # tv coefficient
>>> rr = estimators.RidgeRegression_SmoothedL1TV(
...             k, l, g,
...             Atv=Atv, Al1=Al1,
...             algorithm=algorithms.ExcessiveGapMethod(max_iter=1000))
>>> rr.fit(spams_X, spams_Y)
print rr.beta

<parsimony.estimators.RidgeRegression_SmoothedL1TV object at 0x1cc9e10>
>>> [[ nan]
 [ nan]
 [ nan]
 [ nan]
 [ nan]
 [ nan]
 [ nan]
 [ nan]
 [ nan]
 [ nan]
 [ nan]
 [ nan]
 [ nan]
 [ nan]
 [ nan]
 [ nan]]
>>> >>> 
>>> k = 0.05  # ridge regression coefficient
>>> l = 0.05  # l1 coefficient
>>> g = 0.05  # tv coefficient
>>> rr = estimators.RidgeRegression_SmoothedL1TV(
...             k, l, g,
...             Atv=Atv, Al1=Al1,
...             algorithm=algorithms.ExcessiveGapMethod(max_iter=1000))
>>> rr.fit(spams_X, spams_Y)
print rr.beta
<parsimony.estimators.RidgeRegression_SmoothedL1TV object at 0x1cc9c90>
>>> [[ -3.06795144e-02]
 [ -1.15708593e-02]
 [ -1.09255970e-05]
 [ -2.38825088e-01]
 [ -6.74746378e-02]
 [ -5.50823531e-02]
 [  1.91713527e-05]
 [ -2.38855356e-01]
 [ -6.92107632e-02]
 [ -3.09792738e-02]
 [ -1.06060363e-02]
 [  2.68985405e-01]
 [ -1.37646869e-01]
 [ -1.37638461e-01]
 [  6.50395154e-06]
 [  2.09415248e-05]]
>>> k = 0.0  # ridge regression coefficient
>>> l = 0.0  # l1 coefficient
>>> g = 0.05  # tv coefficient
>>> rr = estimators.RidgeRegression_SmoothedL1TV(
...             k, l, g,
...             Atv=Atv, Al1=Al1,
...             algorithm=algorithms.ExcessiveGapMethod(max_iter=1000))
>>> rr.fit(spams_X, spams_Y)
print rr.beta
<parsimony.estimators.RidgeRegression_SmoothedL1TV object at 0x216d390>
>>> [[ nan]
 [ nan]
 [ nan]
 [ nan]
 [ nan]
 [ nan]
 [ nan]
 [ nan]
 [ nan]
 [ nan]
 [ nan]
 [ nan]
 [ nan]
 [ nan]
 [ nan]
 [ nan]]

[JinpengLI]

I added a warning message.

[tomlof]

We can solve for this in the code. I'll take care of it.

[tomlof]

I have changed it such that it only warns if k, the ridge parameter, is too small.

I have made it such that the code handles the parameters for L1 and TV. Now I set it to consts.TOLERANCE when it is smaller than that. This may not be the desired behaviour, however. We could ignore that part of the code if both the L1 and the TV parameters are too small.

What do you think?