Refactor Load Model / OPFS / file logic

[nickcastel50]

Algorithm for fetch-model-then-sync-with-git:

rsp = ghuc.get(path, file ? If-None-Match: file.etag)
if (rsp.is304)
    display(cachedFile)
else
    display(rsp.file)
    hash = ghapi.getCommit(path)
    save...

Detailed web worker implementation:

// Function to fetch from GHUC
async function fetchFromGHUC(url, etag = null) {
    const headers = etag ? { 'If-None-Match': etag } : {};
    const response = await fetch(url, { headers });
    return response;
}

// Function to write temporary file to OPFS (Origin Private File System)
async function writeTemporaryFileToOPFS(path, data) {
    // Implementation for writing file to OPFS
}

// Function to parse commit hash and temporary file path from cached response
function parseCommitHashFromCachedResponse(response) {
    // Implementation for parsing commit hash and temp file path
}

// Function to query commit hash file
async function queryCommitHashFile(path) {
    // Implementation for querying commit hash file
}

// Function to post message to main thread
function postMessageToMainThread(message) {
    postMessage(message);
}

// Function to display model
function displayModel(fileHandle) {
    postMessageToMainThread({ type: 'displayModel', fileHandle });
}

// Function to handle the success case after fetching from GHUC
async function handleSuccessResponse(tempFilePath, finalFilePath) {
    let fileHandle = { path: tempFilePath }; // Assuming file handle is the path here
    displayModel(fileHandle);

    try {
        let commitHash = await getLatestCommitHash();
        await cacheFetchedGHUCResponse(commitHash, tempFilePath);
        await copyTempFileTo(finalFilePath, commitHash);

        let updatedFileHandle = { path: finalFilePath }; // Updated file handle
        postMessageToMainThread({ type: 'updatedFileHandle', fileHandle: updatedFileHandle });
    } catch (error) {
        console.error("Error getting latest commit hash:", error);
        // Implement retry logic or fallback here
    }
}

// Function to handle the not modified case
async function handleNotModifiedResponse(response, tempFilePath, finalFilePath, url) {
    let { commitHash, tempFilePath: tempFilePathFromCache } = parseCommitHashFromCachedResponse(response);

    if (tempFilePathExists(tempFilePathFromCache)) {
        await deleteTempFileAndFinalFile(tempFilePathFromCache);
        let commitHashFileExists = await queryCommitHashFile(commitHash);

        if (commitHashFileExists) {
            displayModel({ path: finalFilePath });
        } else {
            let newResponse = await fetchFromGHUC(url, '');
            await processGHUCResponse(newResponse, tempFilePath, finalFilePath, url);
        }
    }
}

// Function to process GHUC response
async function processGHUCResponse(response, tempFilePath, finalFilePath, url) {
    if (response.status === 200) {
        await writeTemporaryFileToOPFS(tempFilePath, await response.text());
        await handleSuccessResponse(tempFilePath, finalFilePath);
    } else if (response.status === 304) {
        await handleNotModifiedResponse(response, tempFilePath, finalFilePath, url);
    } else {
        // Handle other statuses or errors
        console.error("Unexpected response status:", response.status);
    }
}

// Main function to handle the logic
async function main(url, tempFilePath, finalFilePath) {
    let response = await fetchFromGHUC(url);
    await processGHUCResponse(response, tempFilePath, finalFilePath, url);
}

// Function to get the latest commit hash
async function getLatestCommitHash() {
    // Implementation for getting the latest commit hash
    // This should include handling for GH rate limits, timeout, max retry, etc.
}

// Function to cache fetched GHUC response
async function cacheFetchedGHUCResponse(commitHash, tempFilePath) {
    // Implementation for caching GHUC response
}

// Function to copy temporary file to final location
async function copyTempFileTo(finalFilePath, commitHash) {
    // Implementation for copying temp file to final location
}

// Function to update store file handle
function updateStoreFileHandle(finalFilePath) {
    // Implementation for updating store file handle to point to the final file
}

// Function to delete temporary file and final file
async function deleteTempFileAndFinalFile(tempFilePath) {
    // Implementation for deleting temp and final file
}

// Function to check if temporary file path exists
function tempFilePathExists(tempFilePath) {
    // Implementation to check if temp file path exists
}

// Event listener for messages from the main thread
onmessage = function(event) {
    const { url, tempFilePath, finalFilePath } = event.data;
    main(url, tempFilePath, finalFilePath);
}

Algorithm flow diagram: https://excalidraw.com/#json=4j16ZKcQ1hsP18ves_QUy,xa0c24_6BwN1mqpF2EdGnA

Current issues:

It seems there definitely are some limitations importing other js code into the web workers. Two things we needed were the Cache (src/net/github/Cache.js) and potentially OctoKit to get the commit hash asynchronously. What I ended up doing for Cache.js was rewriting it using the UMD (universal module definition pattern). This allows it to be imported in other files in ES6 and also used in CommonJS. It isn't pretty, but it works:

/**
 * This module implements an etag caching system for network requests
 * using the browser Cache API.
 * This module was rewritten to support ES6 + CommonJS for Web-Workers
 * using the UMD (Universal Module Definition) pattern.
 * 
 * Usage:
 *    1. ES6 Main Thread:
 *      import {checkCache} from './Cache'
 *      ...
 *      checkCache("test")
 *    2. CommonJS (Web Worker):
 *      importScripts('./Cache.js');
 *      ...
 *      CacheModule.checkCache("test")
 */

(function(root, factory) {
  // eslint-disable-next-line no-undef
  if (typeof define === 'function' && define.amd) {
    // AMD
    // eslint-disable-next-line no-undef
    define([], factory)
  } else if (typeof module === 'object' && module.exports) {
    // CommonJS
    module.exports = factory()
  } else {
    // Browser globals
    root.CacheModule = factory()
  }
  // eslint-disable-next-line no-invalid-this
}(typeof self !== 'undefined' ? self : this, function() {
  // http request etag cache
  let httpCache = null

  const httpCacheApiAvailable = (typeof caches !== 'undefined')

  /**
   * Retrieves the HTTP cache, opening it if it doesn't already exist.
   *
   * @return {Promise<Cache | object>} The HTTP cache object.
   */
  async function getCache() {
    if (!httpCache) {
      httpCache = await openCache()
    }
    return httpCache
  }

  /**
   * Opens the HTTP cache if the Cache API is available.
   *
   * @return {Promise<Cache | object>} A Cache object if the Cache API is available, otherwise an empty object.
   */
  async function openCache() {
    if (httpCacheApiAvailable) {
      return await caches.open('bldrs-github-api-cache')
    }
    return {}
  }

  /**
   * Converts a cached response to an Octokit response format.
   *
   * @param {Response|null} cachedResponse The cached response to convert.
   * @return {Promise<object | null>} A structured object mimicking an Octokit response, or null if the response is invalid.
   */
  async function convertToOctokitResponse(cachedResponse) {
    if (!cachedResponse) {
      return null
    }

    const data = await cachedResponse.json()
    const headers = cachedResponse.headers
    const status = cachedResponse.status

    const octokitResponse = {
      data: data,
      status: status,
      headers: {},
      url: cachedResponse.url,
    }

    headers.forEach((value, key) => {
      octokitResponse.headers[key] = value
    })

    return octokitResponse
  }

  /**
   * Checks the cache for a specific key and converts the response to an Octokit response format.
   *
   * @param {string} key The key to search for in the cache.
   * @return {Promise<object | null>} The cached response in Octokit format, or null if not found or an error occurs.
   */
  async function checkCache(key) {
    try {
      if (httpCacheApiAvailable) {
        const _httpCache = await getCache()
        const response = await _httpCache.match(key)
        return await convertToOctokitResponse(response)
      } else {
        return httpCache[key]
      }
    } catch (error) {
      return null
    }
  }

  /**
   * Updates the cache entry for a given key with the response received.
   * The cache will only be updated if the response headers contain an ETag.
   *
   * @param {string} key The cache key associated with the request.
   * @param {object} response The HTTP response object from Octokit which includes headers and data.
   */
  async function updateCache(key, response) {
    if (response.headers.etag) {
      const _httpCache = await getCache()
      if (httpCacheApiAvailable) {
        const body = JSON.stringify(response.data)
        const wrappedResponse = new Response(body)
        wrappedResponse.headers.set('etag', response.headers.etag)
        _httpCache.put(key, wrappedResponse)
      } else {
        _httpCache[key] = {
          response: response,
        }
      }
    }
  }

  // Export the functions
  return {
    getCache,
    convertToOctokitResponse,
    checkCache,
    updateCache,
  }
}))

The more pressing issue is OctoKit. A similar thing might be able to be done, but not sure if it is worth it for a single call. Might be a better pattern to post back to the main thread to run the OctoKit request in a callback.

[nickcastel50]

@pablo-mayrgundter