Adding different activations

[duhd1993]

It would be nice to include something like this

$ diff pyrenn.py pyrenn-original.py
3c3
< def CreateNN(nn,dIn=[0],dIntern=[],dOut=[], activation='tanh'):
---
> def CreateNN(nn,dIn=[0],dIntern=[],dOut=[]):
43,48d42
<   if activation == "tanh":
<       net['activation'] = tanh
<   elif activation == "softplus":
<       net['activation'] = softplus
<   elif activation == "relu":
<       net['activation'] = relu
51,68d44
< def tanh(n, a=None, deriv=False):
<   if deriv:
<       return 1 - a**2
<   else:
<       return np.tanh(n)
<
< def softplus(n, a=None, deriv=False):
<   if deriv:
<       return 1 / (1 + np.exp(-n))
<   else:
<       return np.log(1 + np.exp(n))
<
< def relu(n, a=None, deriv=False):
<   if deriv:
<       return np.ones(n.shape) * (n > 0)
<   else:
<       return n * (n > 0)
<
288d263
<   activation = net['activation']
321c296
<               a[q,m] = activation(n[q,m])
---
>               a[q,m] = np.tanh(n[q,m])
388d362
<   activation = net['activation']
442c416
<                       + np.dot(np.dot(S[q,u,l],LW[l,m,0]),np.diag(activation(n[q,m],a[q,m],deriv=True)))
---
>                       + np.dot(np.dot(S[q,u,l],LW[l,m,0]),np.diag(1-(np.tanh(n[q,m]))**2))
454c428
<                   S[q,m,m] = np.diag(activation(n[q,m],a[q,m],deriv=True)) #Sensitivity Matrix S[m,m]
---
>                   S[q,m,m] = np.diag(1-(np.tanh(n[q,m]))**2) #Sensitivity Matrix S[m,m]
612c586
<                       + np.dot(np.dot(S[q,u,l],LW[l,m,0]),np.diag(activation(n[q,m],a[q,m],deriv=True)))
---
>                       + np.dot(np.dot(S[q,u,l],LW[l,m,0]),np.diag(1-(np.tanh(n[q,m]))**2))
622c596
<                   S[q,m,m] = np.diag(activation(n[q,m],a[q,m],deriv=True)) #Sensitivity Matrix S[m,m]
---
>                   S[q,m,m] = np.diag(1-(np.tanh(n[q,m]))**2) #Sensitivity Matrix S[m,m]

[gopal-m]

Please remove me from your list.

On Aug 10, 2019, at 8:38 AM, Haodong DU notifications@github.com wrote:

It would be nice to include something like this $ diff pyrenn.py pyrenn-original.py 3c3 < def CreateNN(nn,dIn=[0],dIntern=[],dOut=[], activation='tanh'):

def CreateNN(nn,dIn=[0],dIntern=[],dOut=[]): 43,48d42 < if activation == "tanh": < net['activation'] = tanh < elif activation == "softplus": < net['activation'] = softplus < elif activation == "relu": < net['activation'] == relu 51,68d44 < def tanh(n, a=None, deriv=False): < if deriv: < return 1 - a*2 < else: < return np.tanh(n) < < def softplus(n, a=None, deriv=False): < if deriv: < return 1 / (1 + np.exp(-n)) < else: < return np.log(1 + np.exp(n)) < < def relu(n, a=None, deriv=False): < if deriv: < return np.ones(n.shape) (n > 0) < else: < return n * (n > 0) < 288d263 < activation = net['activation'] 321c296 < a[q,m] = activation(n[q,m])

      a[q,m] = np.tanh(n[q,m])

388d362 < activation = net['activation'] 442c416 < + np.dot(np.dot(S[q,u,l],LW[l,m,0]),np.diag(activation(n[q,m],a[q,m],deriv=True)))

              + np.dot(np.dot(S[q,u,l],LW[l,m,0]),np.diag(1-(np.tanh(n[q,m]))**2))

454c428 < S[q,m,m] = np.diag(activation(n[q,m],a[q,m],deriv=True)) #Sensitivity Matrix S[m,m]

          S[q,m,m] = np.diag(1-(np.tanh(n[q,m]))**2) #Sensitivity Matrix S[m,m]

612c586 < + np.dot(np.dot(S[q,u,l],LW[l,m,0]),np.diag(activation(n[q,m],a[q,m],deriv=True)))

              + np.dot(np.dot(S[q,u,l],LW[l,m,0]),np.diag(1-(np.tanh(n[q,m]))**2))

622c596 < S[q,m,m] = np.diag(activation(n[q,m],a[q,m],deriv=True)) #Sensitivity Matrix S[m,m]

          S[q,m,m] = np.diag(1-(np.tanh(n[q,m]))**2) #Sensitivity Matrix S[m,m]

— You are receiving this because you are subscribed to this thread. Reply to this email directly, view it on GitHub https://github.com/yabata/pyrenn/issues/9?email_source=notifications&email_token=ACL3YZ5JFQ7Z5J6FKJ2YQ2DQD2ZCRA5CNFSM4IKZL7E2YY3PNVWWK3TUL52HS4DFUVEXG43VMWVGG33NNVSW45C7NFSM4HEQ7F4A, or mute the thread https://github.com/notifications/unsubscribe-auth/ACL3YZ4NYOI2SOJ477SENKTQD2ZCRANCNFSM4IKZL7EQ.

[duhd1993]

Please remove me from your list. … On Aug 10, 2019, at 8:38 AM, Haodong DU @.> wrote: It would be nice to include something like this $ diff pyrenn.py pyrenn-original.py 3c3 < def CreateNN(nn,dIn=[0],dIntern=[],dOut=[], activation='tanh'): def CreateNN(nn,dIn=[0],dIntern=[],dOut=[]): 43,48d42 < if activation == "tanh": < net['activation'] = tanh < elif activation == "softplus": < net['activation'] = softplus < elif activation == "relu": < net['activation'] == relu 51,68d44 < def tanh(n, a=None, deriv=False): < if deriv: < return 1 - a2 < else: < return np.tanh(n) < < def softplus(n, a=None, deriv=False): < if deriv: < return 1 / (1 + np.exp(-n)) < else: < return np.log(1 + np.exp(n)) < < def relu(n, a=None, deriv=False): < if deriv: < return np.ones(n.shape) (n > 0) < else: < return n * (n > 0) < 288d263 < activation = net['activation'] 321c296 < a[q,m] = activation(n[q,m]) a[q,m] = np.tanh(n[q,m]) 388d362 < activation = net['activation'] 442c416 < + np.dot(np.dot(S[q,u,l],LW[l,m,0]),np.diag(activation(n[q,m],a[q,m],deriv=True))) + np.dot(np.dot(S[q,u,l],LW[l,m,0]),np.diag(1-(np.tanh(n[q,m]))2)) 454c428 < S[q,m,m] = np.diag(activation(n[q,m],a[q,m],deriv=True)) #Sensitivity Matrix S[m,m] S[q,m,m] = np.diag(1-(np.tanh(n[q,m]))2) #Sensitivity Matrix S[m,m] 612c586 < + np.dot(np.dot(S[q,u,l],LW[l,m,0]),np.diag(activation(n[q,m],a[q,m],deriv=True))) + np.dot(np.dot(S[q,u,l],LW[l,m,0]),np.diag(1-(np.tanh(n[q,m]))2)) 622c596 < S[q,m,m] = np.diag(activation(n[q,m],a[q,m],deriv=True)) #Sensitivity Matrix S[m,m] S[q,m,m] = np.diag(1-(np.tanh(n[q,m]))2) #Sensitivity Matrix S[m,m] — You are receiving this because you are subscribed to this thread. Reply to this email directly, view it on GitHub <#9?email_source=notifications&email_token=ACL3YZ5JFQ7Z5J6FKJ2YQ2DQD2ZCRA5CNFSM4IKZL7E2YY3PNVWWK3TUL52HS4DFUVEXG43VMWVGG33NNVSW45C7NFSM4HEQ7F4A>, or mute the thread https://github.com/notifications/unsubscribe-auth/ACL3YZ4NYOI2SOJ477SENKTQD2ZCRANCNFSM4IKZL7EQ.

This is not controlled by the rep owner. You should be able to find a unsubscribe link in your email or you can change general settings in https://github.com/settings/notifications