Open aheissenberger opened 2 years ago
Ok - I understand the problem - you check for DM_
and then you only add the functions before the code:
function DM_getPanelFile(i) {
return '/XXXX/DataStationProjects/.my-projec.results'+{"0":"a73a309d-b7b7-47eb-a8b4-f56404c36be2","1":"066250cd-0972-4fad-86af-3f6bf91be317","Untitled panel #1":"066250cd-0972-4fad-86af-3f6bf91be317","Untitled panel #2":"a73a309d-b7b7-47eb-a8b4-f56404c36be2"}[i];
}
function DM_getPanel(i) {
const fs = require('fs');
return JSON.parse(fs.readFileSync('/XXXX/DataStationProjects/.my-projec.results'+{"0":"a73a309d-b7b7-47eb-a8b4-f56404c36be2","1":"066250cd-0972-4fad-86af-3f6bf91be317","Untitled panel #1":"066250cd-0972-4fad-86af-3f6bf91be317","Untitled panel #2":"a73a309d-b7b7-47eb-a8b4-f56404c36be2"}[i]));
}
function DM_setPanel(v) {
const fs = require('fs');
const fd = fs.openSync('/XXXX/DataStationProjects/.my-projec.results066250cd-0972-4fad-86af-3f6bf91be317', 'w');
if (Array.isArray(v)) {
fs.writeSync(fd, '[');
for (let i = 0; i < v.length; i++) {
const row = v[i];
let rowJSON = JSON.stringify(row);
if (i < v.length - 1) {
rowJSON += ',';
}
fs.writeSync(fd, rowJSON);
}
fs.writeSync(fd, ']');
} else {
fs.writeSync(fd, JSON.stringify(v));
}
}
import {readFile} from "node:fs/promises"
let transform = DM_getPanel(0);
const content = await readFile('/XXXX/test.txt',{encoding:'utf8'})
I suggest to change your approach:
const fs = require('fs');
ES modules: import fs from 'node:fs'
Streaming could be improved by switching from your current JSON format to JSON Streaming.
import fs from 'node:fs'
/* BEGIN: user code */
import {readFile} from "node:fs/promises"
let transform = DM_getPanel(0);
const content = await readFile('/XXXX/test.txt',{encoding:'utf8'})
/* END: user code */
function DM_getPanelFile(i) {
return '/XXXX/DataStationProjects/.my-projec.results'+{"0":"a73a309d-b7b7-47eb-a8b4-f56404c36be2","1":"066250cd-0972-4fad-86af-3f6bf91be317","Untitled panel #1":"066250cd-0972-4fad-86af-3f6bf91be317","Untitled panel #2":"a73a309d-b7b7-47eb-a8b4-f56404c36be2"}[i];
}
function DM_getPanel(i) {
const fs = require('fs');
return JSON.parse(fs.readFileSync('/XXXX/DataStationProjects/.my-projec.results'+{"0":"a73a309d-b7b7-47eb-a8b4-f56404c36be2","1":"066250cd-0972-4fad-86af-3f6bf91be317","Untitled panel #1":"066250cd-0972-4fad-86af-3f6bf91be317","Untitled panel #2":"a73a309d-b7b7-47eb-a8b4-f56404c36be2"}[i]));
}
function DM_setPanel(v) {
const fs = require('fs');
const fd = fs.openSync('/XXXX/DataStationProjects/.my-projec.results066250cd-0972-4fad-86af-3f6bf91be317', 'w');
if (Array.isArray(v)) {
fs.writeSync(fd, '[');
for (let i = 0; i < v.length; i++) {
const row = v[i];
let rowJSON = JSON.stringify(row);
if (i < v.length - 1) {
rowJSON += ',';
}
fs.writeSync(fd, rowJSON);
}
fs.writeSync(fd, ']');
} else {
fs.writeSync(fd, JSON.stringify(v));
}
}
You can't hardcode those DM_* functions by the way. They are generated with information for the specific function project/page/panel.
Yes I'd like to switch away from the current JSON format on disk.
only add the import statement before the user code: CommonJS: const fs = require('fs'); ES modules: import fs from 'node:fs'
The reason I wanted to put the imports inside the functions is so that they couldn't possibly conflict with user code. If the user made their own variable fs
the code would crash and they wouldn't know why.
add the code for the function after the user code
Why does it make a difference if it's before or after? Some other languages wouldn't allow this too since most languages require things to be declared before use.
You can place this functions in a library and provide the variable parts e.g. command line parameter, or ENV variable. I would write them to a temp file and provide the path to the temp file as a parameter. Node and many other runtimes support preloading of libraries.
I moved the function after the user script to make it easy to add other imports without getting a conflict as imports need to be before any other code.
I have been tinkering with a simple function which will parse a NDJSON Format (or something where each object is in in one line to a stream. This allows to use a simple TransferStream to handle huge amounts of data without using a lot of memory.
This works with bash helper (code at end of this note):
but when I add the functions to import/export the Data it fails:
it fails with
require is not defined in ES module scope
- I think you userequire
in your wrapper or macro:I used this bash script with
experimental-loader
to forces node to load file as module: