bitdom8
commented
1 year ago When we try to add this attention model layer :
model.add(tf.layers.dense({
activation: "attention1",
units: 5,
}))ValueError: Unknown activation: attention1. This may be due to one of the following reasons:
This is the code:
import "@tensorflow/tfjs-node"
import * as tf from '@tensorflow/tfjs';
import bstockstrainingdata from "./bstockstrainingdata.json" assert { type: "json" };
let iris = bstockstrainingdata
let irisTesting = [{"{id}":1,"{Ticker}":"ASELS","{Description}":"ASELSAN","{Price}":"62.150002","{Change %}":"5.69728411","{Change}":"3.350003","{1-Month High}":"63.299999","{1-Month Low}":"45.939999","{Oscillators Rating}":"Neutral","{Average Directional Index (14)}":"53.36832786","{Volume}":"174768398","{Volume*Price}":"10861856285.236795","{Awesome Oscillator}":"7.44514723","{Average True Range (14)}":"2.76631841","{Commodity Channel Index (20)}":"99.56932835","{MACD Level (12, 26)}":"3.890496","{MACD Signal (12, 26)}":"4.27058564","{Momentum (10)}":"4.25","{Relative Strength Index (14)}":"72.01021313","{Stochastic %K (14, 3, 3)}":"69.30224861","{Stochastic %D (14, 3, 3)}":"67.94386538","{Aroon Up (14)}":"100","{Aroon Down (14)}":"0","{Bull Bear Power}":"6.45791654","{Parabolic SAR}":"56.150002","{Simple Moving Average (10)}":"59.2300004","{Moving Averages Rating}":"Strong Buy","{Pivot Fibonacci P}":"43.30666567","{Ichimoku Conversion Line (9, 26, 52, 26)}":"59.7250005","{Pattern}":"","{Pivot Camarilla R1}":"50.06733343","{Pivot Camarilla P}":"43.30666567","{Pivot Camarilla R3}":"53.16200028","{Pivot Camarilla S2}":"45.42533315","{Pivot Camarilla S3}":"43.87799973","{Pivot Fibonacci R1}":"49.75482605","{Volatility Week}":"4.03141615","{Volatility}":"7.5821401","{Relative Volume}":"2.11003232","{Technical Rating}":"Buy","{YTD Performance}":"196.23451859","{Exponential Moving Average (20)}":"56.17065518","{Exponential Moving Average (50)}":"48.58961817","{Exponential Moving Average (200)}":"34.06279706","{Change 1M, %}":"28.09151278"},
]
// convert/setup our data
const trainingData = tf.tensor2d(iris.map(item => [
Number(item["{1-Month High}"]), Number(item["{Aroon Down (14)}"]), Number(item["{Bull Bear Power}"]), Number(item["{Exponential Moving Average (20)}"]),
Number(item["{1-Month Low}"]), Number(item["{Average Directional Index (14)}"]), Number(item["{Awesome Oscillator}"]), Number(item["{Average True Range (14)}"]),
Number(item["{Commodity Channel Index (20)}"]), Number(item["{MACD Level (12, 26)}"]), Number(item["{MACD Signal (12, 26)}"]), Number(item["{Parabolic SAR}"]),
Number(item["{Simple Moving Average (10)}"]), Number(item["{Volatility Week}"]), Number(item["{Volatility}"]), Number(item["{Ichimoku Conversion Line (9, 26, 52, 26)}"]),
]))
const outputData = tf.tensor2d(iris.map(item => [
item["{Moving Averages Rating}"] === "Strong Buy" && item["{Technical Rating}"] === "Strong Buy" ? 1 : 0,
item["{Moving Averages Rating}"] === "Strong Buy" ? 1 : 0,
item["{Moving Averages Rating}"] === "Buy" ? 1 : 0,
item["{Moving Averages Rating}"] === "Sell" ? 1 : 0,
item["{Moving Averages Rating}"] === "Strong Sell" ? 1 : 0
]))
const testingData = tf.tensor2d(irisTesting.map(item => [
Number(item["{1-Month High}"]), Number(item["{Aroon Down (14)}"]), Number(item["{Bull Bear Power}"]), Number(item["{Exponential Moving Average (20)}"]),
Number(item["{1-Month Low}"]), Number(item["{Average Directional Index (14)}"]), Number(item["{Awesome Oscillator}"]), Number(item["{Average True Range (14)}"]),
Number(item["{Commodity Channel Index (20)}"]), Number(item["{MACD Level (12, 26)}"]), Number(item["{MACD Signal (12, 26)}"]), Number(item["{Parabolic SAR}"]),
Number(item["{Simple Moving Average (10)}"]), Number(item["{Volatility Week}"]), Number(item["{Volatility}"]), Number(item["{Relative Volume}"]),
]))
// build neural network
const model = tf.sequential()
model.add(tf.layers.dense({
inputShape: [16],
activation: "sigmoid",
units: 17,
}))
model.add(tf.layers.dense({
inputShape: [17],
activation: "sigmoid",
units: 16,
}))
model.add(tf.layers.dense({
activation: "sigmoid",
units: 5,
}))
//sss
async function loadAttentionModel(){
// await log('building new attention model...');
let queries = tf.input({name: 'queries', shape: [INPUTSIZE, dModel]}); // 0, 1024, 64
let keys = tf.input({name: 'keys', shape: [INPUTSIZE, dModel]}); // 0, 1024, 64
let values = tf.input({name: 'values', shape: [INPUTSIZE, dModel]}); // 0, 1024, 64
let keysTransposed = tf.layers.permute({dims: [2, 1]}).apply(keys); // 0, 64, 1024
let valuesTransposed = tf.layers.permute({dims: [2, 1]}).apply(values); // 0, 64, 1024
let proj = tf.input({shape: [dModel, INPUTSIZE]}); // 0, 64, 1024
let projOutput = tf.layers.dense({units: dModel, kernelInitializer: 'glorotNormal', biasInitializer: 'glorotNormal'}).apply(proj); // 0, 64, 64
let projModel = tf.model({inputs: proj, outputs: projOutput});
let eProj = projModel.apply(keysTransposed); // 0, 64, 64
let fProj = projModel.apply(valuesTransposed); // 0, 64, 64
class lambdaLayer extends tf.layers.Layer {
constructor(config) {
super(config);
if (config.name === undefined) {
config.name = ((+new Date) * Math.random()).toString(36); //random name from timestamp in case name hasn't been set
}
this.name = config.name;
this.lambdaFunction = config.lambdaFunction;
this.lambdaOutputShape = config.lambdaOutputShape;
}
call(input) {
return tf.tidy(() => {
let result = null;
eval(this.lambdaFunction);
return result;
});
}
computeOutputShape(inputShape) {
if (this.lambdaOutputShape === undefined) { //if no outputshape provided, try to set as inputshape
return inputShape[0];
} else {
return this.lambdaOutputShape;
}
}
getConfig() {
const config = super.getConfig();
Object.assign(config, {
lambdaFunction: this.lambdaFunction,
lambdaOutputShape: this.lambdaOutputShape
});
return config;
}
static get className() {
return 'lambdaLayer';
}
}
tf.serialization.registerClass(lambdaLayer);
let attention1 = new lambdaLayer({name: 'attention1', lambdaFunction: `
let qk = tf.matMul(input[0], input[1]); // 0, 1024, 64
let pBar = tf.div(qk, tf.sqrt(tf.scalar(input[0].shape[2]))); // 0, 1024, 64
let causalMask = tf.transpose(tf.linalg.bandPart(tf.ones([input[0].shape[2], input[0].shape[1]]), 0, -1)); // 1024, 64
let infinityMask = tf.sub(causalMask, tf.scalar(1)); // 1024, 64
infinityMask = tf.mul(infinityMask, tf.scalar(1e16)); //this should be preprocessed // 1024, 64
pBar = tf.mul(pBar, causalMask); // 0, 1024, 64
pBar = tf.add(pBar, infinityMask); // 0, 1024, 64
pBar = tf.softmax(pBar, 2); // 0, 1024, 64
result = tf.matMul(pBar, input[2], false, true);
`, lambdaOutputShape: [queries.shape]}).apply([queries, eProj, fProj]); // 0, 1024, 64
attention1 = tf.layers.dropout({rate: 0.1}).apply(attention1); // 0, 1024, 64
let attention2 = new lambdaLayer({name: 'attention2', lambdaFunction: `
result = tf.matMul(input[0], input[1], false, true);
`, lambdaOutputShape: [values.shape]}).apply([attention1, fProj]); // 0, 1024, 64
let attention_model = tf.model({inputs: [queries, keys, values], outputs: attention2});
attention_model.compile({optimizer: OPTIMIZER, loss: LOSS});
attention_model.summary();
}
loadAttentionModel()
//sss
model.add(tf.layers.dense({
activation: "attention1",
units: 5,
}))
model.compile({
loss: "meanSquaredError",
optimizer: tf.train.adam(.01),
})
// train/fit our network
const startTime = Date.now()
model.fit(trainingData, outputData, {epochs: 800})
.then(async (history) => {
// console.log(history)
// const saveResult = await model.save('/mnt/c/Users/Administrator/Documents/Client')
await model.save('file://./static/model-1a');
model.predict(testingData).print()
})
// test network
Thanks for great repo but using svelte with this is really hard. Cad we make it for nodejs or bunjs at least?