Sqlite3 error with node-pre-gyp

[jsgv]

Hi,

I am attempting to use the this to add sqlite3 support for my Electron app. When I try to include the package, I get warnings followed by errors. I am using Webpack as well. I need to use sqlite3 since Gun with file.js does not trigger .on( updates.

Thanks.

// store.js
var Gun = require('gun')
require('sqlite.gun')

var gun = Gun({
    file: false,
    sqlite: {
        file: "gun.sqlite3"
    }
})
//...

// webpack.base.conf.js
var path = require('path')
var webpack = require('webpack')
var config = require('../config')
var cssLoaders = require('./css-loaders')
var projectRoot = path.resolve(__dirname, '../')

module.exports = {

    output: {
        path: config.build.outputRoot,
        filename: '[name].js'
    },

    target: 'node',
    node: {
        __filename: false,
        __dirname: false
    },
    resolve: {
        extensions: ['', '.js', '.vue'],
        fallback: [path.join(__dirname, '../node_modules')],
        alias: {
            app: path.resolve(__dirname, '../app'),
            'vue': 'vue/dist/vue.common.js'
        }
    },
    resolveLoader: {        
        fallback: [
            path.join(__dirname, '../node_modules')
        ]
    },
    module: {
        preLoaders: [
            {
                test: /\.vue$/,
                loader: 'eslint',
                include: projectRoot,
                exclude: /node_modules/
            },
            {
                test: /\.js$/,
                loader: 'eslint',
                include: projectRoot,
                exclude: /vue-devtools|node_modules/
            }
        ],
        loaders: [
            {
                test: /\.vue$/,
                loader: 'vue-loader',
                options: {
                    // postcss: [
                    //   require('autoprefixer')({
                    //     browsers: ['last 3 versions']
                    //   })
                    // ],
                    loaders: {
                        scss: 'style-loader!css-loader!sass-loader',
                        less: 'style-loader!css-loader!less-loader'
                    }
                }
            },            
            {
                test: /\.js$/,
                loader: 'babel',
                include: projectRoot,
                exclude: /vue-devtools|node_modules/
            },
            {
                test: /\.json$/,
                loader: 'json'
            },
            {
                test: /\.html$/,
                loader: 'vue-html'
            },
            // {
            //   test: /\.scss$/,
            //   loader: 'sass-loader'
            // },
            // {
            //   test: /\.(png|jpe?g|gif|svg|woff2?|eot|ttf|otf)(\?.*)?$/,
            //   loader: 'url',
            //   query: {
            //     limit: 10000,
            //     name: path.join(config.build.assetsSubDirectory, '[name].[ext]').replace('\\', '/')
            //   }
            // }

            {
                test: /\.(png|jpe?g|gif|svg)(\?.*)?$/,
                loader: 'url',
                query: {
                    limit: 10000,
                    name: path.join(config.build.assetsSubDirectory, '[name].[ext]')
                }
            },
            {
                test: /\.(woff2?|eot|ttf|otf)(\?.*)?$/,
                loader: 'url',
                query: {
                    limit: 10000,
                    name: path.join(config.build.assetsSubDirectory, 'fonts/[name].[ext]')
                }
            }
        ]
    },
    plugins: [
        new webpack.ExternalsPlugin('commonjs2', [
            'desktop-capturer',
            'electron',
            'ipc',
            'ipc-renderer',
            'native-image',
            'remote',
            'web-frame',
            'clipboard',
            'crash-reporter',
            'screen',
            'shell'
        ])
    ],
    vue: {
        // loaders: cssLoaders()
        loaders: {
            scss: 'style-loader!css-loader!sass-loader',
            less: 'style-loader!css-loader!less-loader'
        }
    },
    eslint: {
        formatter: require('eslint-friendly-formatter')
    },
    externals:  {}
}

Warnings:

WARNING in ./~/sqlite3/lib/sqlite3.js
Critical dependencies:
4:14-35 the request of a dependency is an expression
 @ ./~/sqlite3/lib/sqlite3.js 4:14-35

WARNING in ./~/sqlite3/lib/binding/electron-v1.6-darwin-x64/node_sqlite3.node
Module parse failed: /Volumes/MacintoshHD/myapp/node_modules/sqlite3/lib/binding/electron-v1.6-darwin-x64/node_sqlite3.node Unexpected character '�' (1:0)
You may need an appropriate loader to handle this file type.
SyntaxError: Unexpected character '�' (1:0)
    at Parser.pp$4.raise (/Volumes/MacintoshHD/myapp/node_modules/webpack/node_modules/acorn/dist/acorn.js:2221:15)
    at Parser.pp$7.getTokenFromCode (/Volumes/MacintoshHD/myapp/node_modules/webpack/node_modules/acorn/dist/acorn.js:2756:10)
    at Parser.pp$7.readToken (/Volumes/MacintoshHD/myapp/node_modules/webpack/node_modules/acorn/dist/acorn.js:2477:17)
    at Parser.pp$7.nextToken (/Volumes/MacintoshHD/myapp/node_modules/webpack/node_modules/acorn/dist/acorn.js:2468:15)
    at Parser.parse (/Volumes/MacintoshHD/myapp/node_modules/webpack/node_modules/acorn/dist/acorn.js:515:10)
    at Object.parse (/Volumes/MacintoshHD/myapp/node_modules/webpack/node_modules/acorn/dist/acorn.js:3098:39)
    at Parser.parse (/Volumes/MacintoshHD/myapp/node_modules/webpack/lib/Parser.js:902:15)
    at DependenciesBlock.<anonymous> (/Volumes/MacintoshHD/myapp/node_modules/webpack/lib/NormalModule.js:104:16)
    at DependenciesBlock.onModuleBuild (/Volumes/MacintoshHD/myapp/node_modules/webpack-core/lib/NormalModuleMixin.js:310:10)
    at nextLoader (/Volumes/MacintoshHD/myapp/node_modules/webpack-core/lib/NormalModuleMixin.js:275:25)
    at /Volumes/MacintoshHD/myapp/node_modules/webpack-core/lib/NormalModuleMixin.js:259:5
    at Storage.finished (/Volumes/MacintoshHD/myapp/node_modules/enhanced-resolve/lib/CachedInputFileSystem.js:38:16)
    at /Volumes/MacintoshHD/myapp/node_modules/graceful-fs/graceful-fs.js:78:16
    at FSReqWrap.readFileAfterClose [as oncomplete] (fs.js:445:3)
 @ ./~/sqlite3/lib ^\.\/.*$

WARNING in ./~/sqlite3/lib/binding/node-v48-darwin-x64/node_sqlite3.node
Module parse failed: /Volumes/MacintoshHD/myapp/node_modules/sqlite3/lib/binding/node-v48-darwin-x64/node_sqlite3.node Unexpected character '�' (1:0)
You may need an appropriate loader to handle this file type.
SyntaxError: Unexpected character '�' (1:0)
    at Parser.pp$4.raise (/Volumes/MacintoshHD/myapp/node_modules/webpack/node_modules/acorn/dist/acorn.js:2221:15)
    at Parser.pp$7.getTokenFromCode (/Volumes/MacintoshHD/myapp/node_modules/webpack/node_modules/acorn/dist/acorn.js:2756:10)
    at Parser.pp$7.readToken (/Volumes/MacintoshHD/myapp/node_modules/webpack/node_modules/acorn/dist/acorn.

dev:server  js:2477:17)

//
// more warnings...
//

Error:

ERROR in ./~/sqlite3/~/node-pre-gyp/lib/node-pre-gyp.js
Module not found: Error: Cannot resolve 'file' or 'directory' ../package in /Volumes/MacintoshHD/myapp/node_modules/sqlite3/node_modules/node-pre-gyp/lib
 @ ./~/sqlite3/~/node-pre-gyp/lib/node-pre-gyp.js 60:16-37

ERROR in ./~/sqlite3/~/node-pre-gyp/~/rc/index.js
Module parse failed: /Volumes/MacintoshHD/myapp/node_modules/sqlite3/node_modules/node-pre-gyp/node_modules/rc/index.js Unexpected character '#' (1:0)
You may need an appropriate loader to handle this file type.
SyntaxError: Unexpected character '#' (1:0)
    at Parser.pp$4.raise (/Volumes/MacintoshHD/myapp/node_modules/webpack/node_modules/acorn/dist/acorn.js:2221:15)
    at Parser.pp$7.getTokenFromCode (/Volumes/MacintoshHD/myapp/node_modules/webpack/node_modules/acorn/dist/acorn.js:2756:10)
    at Parser.pp$7.readToken (/Volumes/MacintoshHD/myapp/node_modules/webpack/node_modules/acorn/dist/acorn.js:2477:17)
    at Parser.pp$7.nextToken (/Volumes/MacintoshHD/myapp/node_modules/webpack/node_modules/acorn/dist/acorn.js:2468:15)
    at Parser.parse (/Volumes/MacintoshHD/myapp/node_modules/webpack/node_modules/acorn/dist/acorn.js:515:10)
    at Object.parse (/Volumes/MacintoshHD/myapp/node_modules/webpack/node_modules/acorn/dist/acorn.js:3098:39)
    at Parser.parse (/Volumes/MacintoshHD/myapp/node_modules/webpack/lib/Parser.js:902:15)
    at DependenciesBlock.<anonymous> (/Volumes/MacintoshHD/myapp/node_modules/webpack/lib/NormalModule.js:104:16)
    at DependenciesBlock.onModuleBuild (/Volumes/MacintoshHD/myapp/node_modules/webpack-core/lib/NormalModuleMixin.js:310:10)
    at nextLoader (/Volumes/MacintoshHD/myapp/node_modules/webpack-core/lib/NormalModuleMixin.js:275:25)
    at /Volumes/MacintoshHD/myapp/node_modules/webpack-core/lib/NormalModuleMixin.js:259:5
    at Storage.finished (/Volumes/MacintoshHD/myapp/node_modules/enhanced-resolve/lib/CachedInputFileSystem.js:38:16)
    at /Volumes/MacintoshHD/myapp/node_modules/graceful-fs/graceful-fs.js:78:16
    at FSReqWrap.readFileAfterClose [as oncomplete] (fs.js:445:3)
 @ ./~/sqlite3/~/node-pre-gyp/lib/info.js 11:13-26

ERROR in ./~/node-gyp/lib/node-gyp.js
Module not found: Error: Cannot resolve 'file' or 'directory' ../package in /Volumes/MacintoshHD/myapp/node_modules/node-gyp/lib
 @ ./~/node-gyp/lib/node-gyp.js 67:16-37

[amark]

file.js isn't triggering for on? That needs to be reported (could you open a new issue with example code to replicate? Thanks!).

I'm not too familiar with debugging linking errors like this, but my wild guess is that NPM is looking in the wrong directories perhaps because there is a mix install of sudo and non-sudo? I'm currently out of town so I'll have to look at this later.

In the meantime @d3x0r has written another SQLite adapter - we still need to load test it (this one didn't perform very well) but I bet his does: https://www.npmjs.com/package/gun.db see if it is any easier to install. If not, let me know and we'll get this fixed. :)

[jsgv]

Thanks @amark for the quick reply!

Looks like it's an issue with webpack. I'm not a webpack expert so it's taking me some while to figure out a solution. Note, I'll create an issue with code for the file.js and .on( issue in a bit also.

My app is using Electron and webpack.

I tried gun.db, but that also has an issue of it's own:

var Gun = require('gun')
require('gun.db')

var gun = Gun({
    file: false,
    db: {
        file: "gun.db"
    }
})

WARNING in ./~/sack.vfs/build/Release/sack_vfs.node
Module parse failed: /Volumes/MacintoshHD/myapp/node_modules/sack.vfs/build/Release/sack_vfs.node Unexpected character '�' (1:0)
You may need an appropriate loader to handle this file type.
SyntaxError: Unexpected character '�' (1:0)
    at Parser.pp$4.raise (/Volumes/MacintoshHD/myapp/node_modules/webpack/node_modules/acorn/dist/acorn.js:2221:15)
    at Parser.pp$7.getTokenFromCode (/Volumes/MacintoshHD/myapp/node_modules/webpack/node_modules/acorn/dist/acorn.js:2756:10)
    at Parser.pp$7.readToken (/Volumes/MacintoshHD/myapp/node_modules/webpack/node_modules/acorn/dist/acorn.js:2477:17)
    at Parser.pp$7.nextToken (/Volumes/MacintoshHD/myapp/node_modules/webpack/node_modules/acorn/dist/acorn.js:2468:15)
    at Parser.parse (/Volumes/MacintoshHD/myapp/node_modules/webpack/node_modules/acorn/dist/acorn.js:515:10)
    at Object.parse (/Volumes/MacintoshHD/myapp/node_modules/webpack/node_modules/acorn/dist/acorn.js:3098:39)
    at Parser.parse (/Volumes/MacintoshHD/myapp/node_modules/webpack/lib/Parser.js:902:15)
    at DependenciesBlock.<anonymous> (/Volumes/MacintoshHD/myapp/node_modules/webpack/lib/NormalModule.js:104:16)
    at DependenciesBlock.onModuleBuild (/Volumes/MacintoshHD/myapp/node_modules/webpack-core/lib/NormalModuleMixin.js:310:10)
    at nextLoader (/Volumes/MacintoshHD/myapp/node_modules/webpack-core/lib/NormalModuleMixin.js:275:25)
    at /Volumes/MacintoshHD/myapp/node_modules/webpack-core/lib/NormalModuleMixin.js:259:5
    at Storage.finished (/Volumes/MacintoshHD/myapp/node_modules/enhanced-resolve/lib/CachedInputFileSystem.js:38:16)
    at /Volumes/MacintoshHD/myapp/node_modules/graceful-fs/graceful-fs.js:78:16
    at FSReqWrap.readFileAfterClose [as oncomplete] (fs.js:445:3)
 @ ./~/sack.vfs/vfs_module.js 6:51-93

WARNING in ./~/sack.vfs/build/Debug/sack_vfs.node
Module parse failed: /Volumes/MacintoshHD/myapp/node_modules/sack.vfs/build/Debug/sack_vfs.node Unexpected character '�' (1:2)
You may need an appropriate loader to handle this file type.
SyntaxError: Unexpected character '�' (1:2)
    at Parser.pp$4.raise (/Volumes/MacintoshHD/myapp/node_modules/webpack/node_modules/acorn/dist/acorn.js:2221:15)

dev:server  at Parser.pp$7.getTokenFromCode (/Volumes/MacintoshHD/myapp/node_modules/webpack/node_modules/acorn/dist/acorn.js:2756:10)
    at Parser.pp$7.readToken (/Volumes/MacintoshHD/myapp/node_modules/webpack/node_modules/acorn/dist/acorn.js:2477:17)
    at Parser.pp$7.nextToken (/Volumes/MacintoshHD/myapp/node_modules/webpack/node_modules/acorn/dist/acorn.js:2468:15)
    at Parser.pp$7.next (/Volumes/MacintoshHD/myapp/node_modules/webpack/node_modules/acorn/dist/acorn.js:2413:10)
    at Parser.pp$3.parseIdent (/Volumes/MacintoshHD/myapp/node_modules/webpack/node_modules/acorn/dist/acorn.js:2191:10)
    at Parser.pp$3.parseExprAtom (/Volumes/MacintoshHD/myapp/node_modules/webpack/node_modules/acorn/dist/acorn.js:1774:21)
    at Parser.pp$3.parseExprSubscripts (/Volumes/MacintoshHD/myapp/node_modules/webpack/node_modules/acorn/dist/acorn.js:1715:21)
    at Parser.pp$3.parseMaybeUnary (/Volumes/MacintoshHD/myapp/node_modules/webpack/node_modules/acorn/dist/acorn.js:1692:19)
    at Parser.pp$3.parseExprOps (/Volumes/MacintoshHD/myapp/node_modules/webpack/node_modules/acorn/dist/acorn.js:1637:21)
    at Parser.pp$3.parseMaybeConditional (/Volumes/MacintoshHD/myapp/node_modules/webpack/node_modules/acorn/dist/acorn.js:1620:21)
    at Parser.pp$3.parseMaybeAssign (/Volumes/MacintoshHD/myapp/node_modules/webpack/node_modules/acorn/dist/acorn.js:1597:21)
    at Parser.pp$3.parseExpression (/Volumes/MacintoshHD/myapp/node_modules/webpack/node_modules/acorn/dist/acorn.js:1573:21)
    at Parser.pp$1.parseStatement (/Volumes/MacintoshHD/myapp/node_modules/webpack/node_modules/acorn/dist/acorn.js:727:47)
    at Parser.pp$1.parseTopLevel (/Volumes/MacintoshHD/myapp/node_modules/webpack/node_modules/acorn/dist/acorn.js:638:25)
    at Parser.parse (/Volumes/MacintoshHD/myapp/node_modules/webpack/node_modules/acorn/dist/acorn.js:516:17)
 @ ./~/sack.vfs/vfs_module.js 5:8-48

WARNING in ./~/ws/lib/Validation.js
Module not found: Error: Cannot resolve module 'utf-8-validate' in /Volumes/MacintoshHD/myapp/node_modules/ws/lib
 @ ./~/ws/lib/Validation.js 10:22-47

WARNING in ./~/ws/lib/BufferUtil.js
Module not found: Error: Cannot resolve module 'bufferutil' in /Volumes/MacintoshHD/myapp/node_modules/ws/lib
 @ ./~/ws/lib/BufferUtil.js 35:21-42

[jsgv]

And console in Electron dev tools:

[d3x0r]

looks like sack.vfs isn't built. (it first reads build/debug, then build/release, if both fail it throws an exception 'Error: cannot find module "./build/Release/sack_vfs.node"

being a binary module it might not work to package it.... I see you're using Electron, so you don't really NEED to package it?

[jsgv]

How could I go about not packaging it, but still accessing it from the Electron app? Thanks for the help.

[d3x0r]

electron supports node's require() which reads from node_modules as it is; it doesn't have to be 'taught' to require().... which is what packaging does; makes it browser friendly... although electron IS a browser it's more like Node in this case? Might not matter, I saw the bug report with the build error which might be fixed... but still I don't really know how a binary .so/.dll is going to be wrapped in a monolithic HTML such that electron can still use it?

[swhgoon]

works well under NwJs:)

[jsgv]

@amark Here is the example project with .on( not triggering. :)

https://github.com/jesusgalvan/electron-gun

[amark]

Both errors have acorn parse errors, which yeah seems to be Webpack trying to... run node-gyp code? Which shouldn't happen, cause node gyp code I think (correct me if I'm wrong) used to build binaries in advance so that way it won't have to be done at run time. More specifically, it doesn't seem to like SQLite3. I don't know if I'll be able to help you here :/ (maybe try another tool, other than Webpack, or don't use Webpack? If that is an option?), you should be able to tell Webpack (IDK how) to ignore certain files.

@jesusgalvan thanks for the link - mind giving me a quick walk through to save me time? Which files should I be looking at?

A quick guess, I saw that https://github.com/jesusgalvan/electron-gun/blob/master/app/store.js#L2 does not list any peers to sync with. So this may be your problem? (If you run the gun server, you'll do Gun('http://localhost:8080/gun') or whatever the port is. Both browsers will share that peer and then get the updates.) Or am I missing something?

[jsgv]

@amark Thanks! Updates are triggering when I use a Gun server and working as intended. Unfortunately, I won't always have a Gun server that both windows can connect to. If I kill the server, updates are no longer triggered. Any advice on how to keep the sync-ness without a Gun server connection?

[amark]

@jesusgalvan WebRTC. However WebRTC has a big problem: It needs a bootstrapping server. Which sucks. Unfortunately this is a problem with the state-of-the-art with the web :( not gun (I'm hoping our momentum will help change these problems).

If you are /only/ wanting sync between 2 tabs open in the SAME browser on the SAME machine, then localStorage actually has a notification system to update other tabs. However, this obviously won't work across devices / other users / browsers / etc.

Does this make sense? I want serverless sync, GUN is designed with P2P so it can do that. But current web technology still requires some servers unfortunately.

Could you zoom out and explain what your goal is, the app you are trying to build? Understanding what you want to accomplish at the high level will help guide coming up with new ideas/solutions.

[jsgv]

@amark Thanks for the guidance and help!

For my app, I am creating 2 browser windows and displaying them each on their own external monitors. What happens on browser window 1, needs to sync/update on browser window 2.

Currently using Vuex and Gun to help sync data. Works well if connected to a server. But my main goal is offline support.

[amark]

Sorry, have been gone in SF... sorry for the delay.

Great illustration! Yes :) that should be possible with localStorage's event listener - it needs to be coded (and I can't promise that I'll be able to do it / or if I can, if it is soon - more than happy to show/teach you how to do it. It shouldn't be very hard), but again big fat warning: This will only sync between 2 tabs on the same device. It won't be able to sync between a tab+incognito, or between different browser vendors, or between different devices (those all require WebRTC, and WebRTC requires a bootstrapping/relay server unfortunately). But YES it will work like your illustration shows.

Would you wanna help out with this?

[amark]

Back from SF! @jesusgalvan

[jsgv]

@amark No worries! Thanks for the help. I was able to get what I wanted with Gun as it is. I had tried to use Vuex as a wrapper for Gun, but it might not have been the best way to use it in some parts of my app.

I just need the db to be synchronized in the app amongst the browser windows, which I was able to do (since the browser windows share the same local storage) and running a Gun server as the Electron app is also running. Love the project, would also be open to help wherever I could. :)

[amark]

@jesusgalvan wonderful to hear! Yes, let's continue the discussion of localStorage's notification on http://gitter.im/amark/gun ! And then is everything good, for us to close this issue?

[jsgv]

Yes :)