Add download function

[tlover-code]

I'm having trouble downloading files with JavaScript code; the download speed is only 3 MB/s. I hope you can add the function Switch.download(url, pathStore, bigFile).

import { formatBytes, getFileNameFromUrl } from "./utilities";

const REPORT_INTERVAL_MS = 1000; 
const MB = 1024 * 1024; 
const WRITE_INTERVAL_MS = 3000; 
const FAT32_FILE_SIZE_LIMIT = 4194303 * 1024; 
interface DownloadResult {
    filename: string;
    path: string;
    size: number;
}

export function Download(url: string, path: string, filename?: string, callback_status?: MyCallbackType, debug?: boolean, signal?: AbortSignal): Promise<DownloadResult> {
    return new Promise(async (resolve, reject) => {
        try {
            const response = await fetch(url, { signal });
            if (!response.ok) {
                throw new Error(response.status.toString());
            }

            const contentLength = +response.headers.get("Content-Length")!;
            const fileName = filename || getFileNameFromUrl(url);
            const shouldChunk = contentLength > FAT32_FILE_SIZE_LIMIT;
            const fileOptions = shouldChunk ? { bigFile: true } : {};
            const fileHandle = await Switch.file(`${path}${fileName}`, fileOptions);
            const writer = fileHandle.writable.getWriter();

            let downloadedBytes = 0;
            let startTime = Date.now();
            let lastReportTime = startTime;
            let lastDownloadedBytes = 0;
            let lastWriteTime = startTime;

            const reader = response.body?.getReader();
            const buffer: Uint8Array[] = [];

            const reportProgress = () => {
                const currentTime = Date.now();
                const timeDiff = (currentTime - lastReportTime) / 1000; 
                const downloadedDiff = downloadedBytes - lastDownloadedBytes;
                const speedMBps = (downloadedDiff / MB) / timeDiff;
                lastReportTime = currentTime;
                lastDownloadedBytes = downloadedBytes;

                if (callback_status) {
                    callback_status({
                        state: "download",
                        percent: Math.floor((downloadedBytes / contentLength) * 100),
                        msg: `${formatBytes(downloadedBytes)}/${formatBytes(contentLength)} - ${speedMBps.toFixed(1)} MB/s`
                    });
                }
            };

            // Interval to log progress every second
            const intervalId = setInterval(reportProgress, REPORT_INTERVAL_MS);

            while (true) {
                const { done, value } = await reader!.read();
                if (done) break;

                buffer.push(value); 
                downloadedBytes += value.length;

                const currentTime = Date.now();
                if (currentTime - lastWriteTime >= WRITE_INTERVAL_MS) {
                    const combinedBuffer = concatUint8Arrays(buffer);
                    await writer.write(combinedBuffer);
                    buffer.length = 0;
                    lastWriteTime = currentTime; 
                }
            }

            if (buffer.length > 0) {
                const combinedBuffer = concatUint8Arrays(buffer);
                await writer.write(combinedBuffer);
            }

            clearInterval(intervalId); // Stop reporting when done
            await writer.close();

            if (callback_status) {
                callback_status({
                    state: "download",
                    percent: 100,
                    msg: `Download completed`
                });
            }

            resolve({ filename: fileName, path, size: contentLength });

        } catch (error: any) {
            if (callback_status) {
                callback_status({
                    state: "error",
                    percent: 0,
                    msg: `Download failed: ${error.message}`
                });
            }
            reject(error);
        }
    });
}

// Helper function to combine multiple Uint8Array chunks into one large Uint8Array
function concatUint8Arrays(arrays: Uint8Array[]): Uint8Array {
    const totalLength = arrays.reduce((sum, arr) => sum + arr.length, 0);
    const result = new Uint8Array(totalLength);
    let offset = 0;
    for (const arr of arrays) {
        result.set(arr, offset);
        offset += arr.length;
    }
    return result;
}

export function startDownload(url: string, path: string, callback_status?: MyCallbackType, filename?: string, debug?: boolean): { controller: AbortController, promise: Promise<DownloadResult> } {
    const controller = new AbortController();
    const signal = controller.signal;
    const promise = Download(url, path, filename, callback_status, debug, signal);
    return { controller, promise };
}

[TooTallNate]

A Switch.download() function wouldn't make sense for the core nx.js runtime, since it would not need to rely on any native APIs. It would be better as an external module.

Can you elaborate on what kind of error/issue you are seeing though?

[tlover-code]

The code above cannot increase the loading speed beyond 3MB/s because it needs to write to the memory card every 3 seconds to avoid memory overflow. I tried writing every 100MB, but the download speed remained unchanged. However, when using the following code, the download speed reached 8MB/s. I noticed that the code below does not have the step of writing to the memory card.

export async function fetchProgress(
  url: string,
  { onProgress, onDownloadStart }: FetchOptions
) {
  // Step 1: start the fetch and obtain a reader
  const response = await fetch(url);

  if (!response.ok) {
    throw new Error(response.status.toString());
  }

  const reader = response.body!.getReader();

  // Step 2: get total length
  const contentLength = +response.headers.get("Content-Length")!;

  // Step 3: read the data
  let receivedLength = 0; // received that many bytes at the moment

  let last = { time: 0, value: 0 };
  let counter = 0;
  let speed = 0;

  const stream = new ReadableStream({
    start(controller) {
      onDownloadStart();
      return pump();
      function pump(): Promise<
        ReadableStreamReadResult<Uint8Array> | undefined
      > {
        return reader.read().then(({ done, value }) => {
          // When no more data needs to be consumed, close the stream
          if (done) {
            controller.close();
            return;
          }
          // Enqueue the next data chunk into our target stream
          controller.enqueue(value);

          receivedLength += value.length;

          const progress = receivedLength / (contentLength / 100) / 100;

          const current = { time: Date.now(), value: receivedLength };

          if (counter % 50 === 0) {
            if (last.time) {
              const time = current.time - last.time;
              const val = current.value - last.value;

              speed = byteToMB(val / (time / 1000));
            }

            last = { ...current };
          }

          onProgress({
            progress: isFinite(progress) ? progress : 0,
            receivedLength,
            contentLength,
            speed,
          });

          counter += 1;
          return pump();
        });
      }
    },
  });

  return new Response(stream);
}