Open haouarihk opened 5 days ago
here's the test file:
import assert from 'assert';
import config from './config.ts';
import {
randomCouchString,
createRxDatabase
} from '../../plugins/core/index.mjs';
import {
replicateWebRTC,
getConnectionHandlerSimplePeer,
SimplePeerWrtc
} from '../../plugins/replication-webrtc/index.mjs';
import {
isFastMode,
isDeno,
isNode
} from '../../plugins/test-utils/index.mjs';
import { waitUntil } from 'async-test-util';
import { wrappedAttachmentsCompressionStorage } from '../../plugins/attachments-compression/index.mjs';
describe('large-document-webrtc.test.ts', function () {
this.timeout(1000 * 60); // 1 minute
if (
!config.storage.hasReplication ||
!config.storage.hasPersistence ||
!config.storage.hasAttachments ||
isDeno
) {
return;
}
let wrtc: SimplePeerWrtc;
let webSocketConstructor: WebSocket;
const signalingServerUrl = 'ws://localhost:18006';
before(async () => {
if (isNode) {
const wrtcModule = await import('node-datachannel/polyfill');
wrtc = wrtcModule.default as any;
const wsModule = await import('ws');
webSocketConstructor = wsModule.WebSocket as any;
}
});
it('should replicate large attachments over WebRTC', async function () {
if (isFastMode()) {
return;
}
const createCollection = async (name: string) => {
const db = await createRxDatabase({
name: randomCouchString(10),
storage: wrappedAttachmentsCompressionStorage({
storage: config.storage.getStorage()
})
});
const collections = await db.addCollections({
[name]: {
schema: {
version: 0,
type: 'object',
primaryKey: 'id',
properties: {
id: {
type: 'string'
},
name: {
type: 'string'
}
},
attachments: {
compression: 'gzip'
}
}
}
});
return collections[name];
};
const c1 = await createCollection('humans1');
const c2 = await createCollection('humans2');
const topic = randomCouchString(10);
const replicationStates = await Promise.all([c1, c2].map(collection =>
replicateWebRTC({
collection,
topic,
connectionHandlerCreator: getConnectionHandlerSimplePeer({
signalingServerUrl,
wrtc,
webSocketConstructor
}),
pull: {},
push: {}
})
));
// Create a large attachment (1GB)
const largeData = new Uint8Array(1 * 1024 * 1024 * 1024);
for (let i = 0; i < largeData.length; i++) {
largeData[i] = Math.floor(Math.random() * 256);
}
const doc = await c1.insert({
id: 'doc1',
name: 'Large Document'
});
await doc.putAttachment({
id: 'large-attachment',
data: Buffer.from(largeData),
type: 'application/octet-stream'
});
// Wait for replication
await waitUntil(async () => {
const docInC2 = await c2.findOne('doc1').exec();
return docInC2 && docInC2.allAttachments().length > 0;
});
const replicatedDoc = await c2.findOne('doc1').exec();
assert.ok(replicatedDoc, 'Document should be replicated');
const attachment = replicatedDoc.getAttachment('large-attachment');
assert.ok(attachment, 'Attachment should be replicated');
const attachmentData = await attachment.getData();
const replicatedData = new Uint8Array(await attachmentData.arrayBuffer());
assert.strictEqual(replicatedData.length, largeData.length, 'Attachment size should match');
assert.deepStrictEqual(replicatedData, largeData, 'Attachment content should match');
// Clean up
await Promise.all(replicationStates.map(state => state.cancel()));
await c1.database.destroy();
await c2.database.destroy();
});
});
Have you found a bug?
WebRTC in browsers is only limited to 16kb, however this limit is not present in nodejs environment.
Did you make a unit test?
As far as i can tell, there are no browser based unit test in this project, but i made the test anyway for future implementation.
Do you have a feature request?
The solution to this problem is to send large contents in chunks, rather than sending them all as one blob. this could also open us for another event listener for "syncing".
if i missed anything, be sure to let me know.