Confusion around the endpoint organization / fields

[NotAud]

Question Am I missing something in the way data is tied together in the API? It seems a bit unorganized in how different collections and items are tied together and at some points just requires guess-work.

I see "IDs" in multiple places i.e "id": "Titanic" in the rap endpoint, but it doesn't really seem in most cases that there's a way to tie that ID to the actual item from the collection. Imo, I would assume in something like the rap endpoint you'd have collection_id and item_id which you could relate to some collection and item (in this case something like collection->potions -> potions->titanic_xp_potion.

Or in other places there's keys like "pt" and "sh", which yeah can be assumed after some looking around, but I feel like we should just get a straight name for these fields. Another example is category, collection, and configName, which the collections all have, but seem to be formatted and mean different things based on that collection?

Not to just sound like I'm complaining, but a lot of it does seem very ambiguous. I assume we're just being given what's already directly used for petsim itself, but is it possible for some of these things to change (at least some more clear naming conventions) assuming the endpoints we are given are specifically for public consumption and aren't used internally?

[chickenputty]

You are correct, it is a lot of different internal designs combined together.

Best I can do right now is try to address your questions.

For looking up pet's rap & exists, here is a snippet from the Discord bot for pet lookups:

    const item = await fetchData(['Pets'], name, false, true);
    if (!item) {
        interaction.editReply("Pet does not exist!");
        return;
    }

    let rap = await getRap();
    const rapItem = rap.filter((rapItem) => rapItem.category === 'Pet' && rapItem.configData.id === item.configName);

    let exists = await getExists();
    const existsItem = exists.filter((existsItem) => existsItem.category === 'Pet' && existsItem.configData.id === item.configName);

    const mergedData = mergeData(rapItem, existsItem);

    const embed = titleEmbed(item.configData.name, item.configData.thumbnail);

    let descriptionLines = [];

    if (item.category == 'Uncategorized') {
        if (item.configData.exclusiveLevel) {
            descriptionLines.push(`**Companion:** ${item.configData.exclusiveLevel}`);
        }
        else if (item.configData.cachedPower && item.configData.cachedPower[0]) {
            descriptionLines.push(`**Area:** 🌍 ${Math.round(item.configData.fromZoneNumber)}`);
            descriptionLines.push(`**Power:** ⚡ ${item.configData.cachedPower[0].toLocaleString()} ${formatNumberContext(item.configData.cachedPower[0])}`);
        }
        else {
            descriptionLines.push(`✨ **Special** ✨`);
        }
    }
    else if (item.category == 'Huge' || item.category == 'Titanic') {
        descriptionLines.push(`**Status:** 🥚 ${item.configData.indexObtainable ? 'Hatchable' : 'Not Hatchable'}`);
    }

    const fields = [];
    let totalExists = 0;

    function addField(category, dataItem) {
        const { exists, rap } = dataItem;
        const { id } = dataItem.configData;
        const field = [];
        field.push(`⭐ ${hideExistsItems.includes(id) ? '?' : exists.toLocaleString()} exists`);
        field.push(`💎 ${rap.toLocaleString()}` + ((fields.length + 1) % 3 != 0 ? generateNumberPadding(Math.max(rap, 1)) : ``));
        fields.push({ name: `**${category}**`, value: field.join('\n'), inline: true });
        totalExists += exists;
    }

    mergedData.forEach(dataItem => {
        const { pt, sh } = dataItem.configData;
        if (!pt && !sh) {
            addField('Regular', dataItem);
        } else if (pt === 1 && !sh) {
            addField('Golden', dataItem);
        } else if (pt === 2 && !sh) {
            addField('Rainbow', dataItem);
        } else if (!pt && sh) {
            addField('Shiny', dataItem);
        } else if (pt === 1 && sh) {
            addField('Shiny Golden', dataItem);
        } else if (pt === 2 && sh) {
            addField('Shiny Rainbow', dataItem);
        }
    });

    if (totalExists > 0)
        descriptionLines.push(`**Total Exists:** ⭐ ${totalExists.toLocaleString()} exists`);

    embed.addFields(fields);

    embed.setDescription(descriptionLines.join('\n'));

And some of those utility functions:

// Remaps Collection category to a RAP category
function remapCollectionToRapCategory(collection) {
    // Manual first before
    switch (collection) {
        case 'Booths':
            return 'Booth';
        case 'Boxes':
            return 'Box';
        case 'Charms':
            return 'Charm';
        case 'Eggs':
            return 'Egg';
        case 'Enchants':
            return 'Enchant';
        case 'Fruits':
            return 'Fruit';
        case 'Hoverboards':
            return 'Hoverboard';
        case 'Lootboxes':
            return 'Lootbox';
        case 'MiscItems':
            return 'Misc';
        case 'Pets':
            return 'Pet';
        case 'Potions':
            return 'Potion';
        case 'Seeds':
            return 'Seed';
        case 'Ultimates':
            return 'Ultimate';
    }
    // Fallback methods to covert plural to singular
    if (collection.endsWith('es')) {
        return collection.slice(0, -2);
    } else if (collection.endsWith('s')) {
        return collection.slice(0, -1);
    }
    return null;
}

async function fetchData(collectionNames, byName, fetchRap, fetchExists) {
    const itemsPromises = collectionNames.map(async (collectionName) => {
        return await getCollectionByName(collectionName);
    });
    let items = (await Promise.all(itemsPromises)).flat();

    // If searching for one item, find just that item
    if (byName) {
        items = items.filter(item => {
            const displayName = (item.configData?.DisplayName || item.configData?.Title || item.configData?.ZoneName || item.configData?.name || item.configData?.Name)?.toLowerCase();
            let extra = '';
            if (item.collection == 'Booths') extra = 'booth';
            if (item.collection == 'Eggs') extra = 'egg';
            if (item.collection == 'Hoverboards') extra = 'hoverboard';
            if (item.category == 'Machine Eggs') extra = 'egg 1'; // uhh need to add 'egg 2', etc. somehow
            if (item.category.includes('Merch Series')) extra = 'gift';
            return displayName === byName.toLowerCase() || displayName === byName.toLowerCase() + ' ' + extra
        });
    }

    // Remove blacklisted items
    items = items.filter(item => {
        const configName = item.configName.includes(' | ') ? item.configName.split(' | ')[1] : item.configName;
        return !blacklistedItems.includes(configName);
    });

    // Inject rap and exists into the item
    if (fetchRap || fetchExists) {
        let rap = fetchRap ? await getRap() : null;
        let exists = fetchExists ? await getExists() : null;

        items.forEach(item => {
            let configName = item.configName.includes(' | ') ? item.configName.split(' | ')[1] : item.configName;
            let collection = remapCollectionToRapCategory(item.collection) || item.collection;

            if (fetchRap) {
                const rapItem = rap.find((rapItem) => {
                    const configDataKeys = Object.keys(rapItem.configData);
                    return configDataKeys.length === 1 && configDataKeys[0] === 'id' && // grab normal only (these need custom lookups after data reformatted [pet, potion, enchant, charm])
                        (!collection || rapItem.category === collection) && // compare category, if it exists
                        rapItem.configData.id === configName // compare name
                });
                item.rap = (rapItem && rapItem.value) || 0;
            }

            if (fetchExists) {
                const existsItem = exists.find((existsItem) => {
                    const configDataKeys = Object.keys(existsItem.configData);
                    return configDataKeys.length === 1 && configDataKeys[0] === 'id' && // grab normal only (these need custom lookups after data reformatted [pet, potion, enchant, charm])
                        (!collection || existsItem.category === collection) && // compare category, if it exists
                        existsItem.configData.id === configName && // compare name
                        !hideExistsItems.includes(existsItem.configData.id); // exclude if is in hideExistsItems
                });
                item.exists = (existsItem && existsItem.value) || 0;
            }
        });
    }

    return byName ? items[0] : items;
}

function mergeData(rapItem, existsItem) {
    const mergedData = [];

    // Create a map to store rap items by category and configData
    const rapMap = new Map();
    rapItem.forEach(item => {
        const key = `${item.category}_${JSON.stringify(item.configData)}`;
        rapMap.set(key, item);
    });

    // Loop through exists items and merge them with rap items if they exist
    existsItem.forEach(item => {
        const key = `${item.category}_${JSON.stringify(item.configData)}`;
        const rapItem = rapMap.get(key);

        if (rapItem) {
            mergedData.push({
                category: item.category,
                configData: item.configData,
                exists: item.value,
                rap: rapItem.value
            });
        } else {
            mergedData.push({
                category: item.category,
                configData: item.configData,
                exists: item.value,
                rap: 0
            });
        }
    });

    return mergedData;
}

And my API wrapper:

// Function to get collection data by name
async function getCollectionByName(collection, configName, category) {
    return fetchAndCache(`collection/${collection}`, { configName, category });
}

// Function to check existence
async function getExists(category) {
    return fetchAndCache('exists', { category });
}

// Function to get RAP data
async function getRap() {
    return fetchAndCache('rap');
}

// Function to make a GET request and handle the response
async function getEndpoint(endpoint, params = {}) {
    console.log(`${BASE_URL}/${endpoint}`);
    try {
        const response = await axios.get(`${BASE_URL}/${endpoint}`, { params, headers: { 'token': token } });
        if (response.data.status === 'ok') {
            return response.data.data;
        } else {
            throw new Error('API returned an error status');
        }
    } catch (error) {
        throw error;
    }
}

// Reusable function to fetch and cache data from the API
async function fetchAndCache(endpoint, params = {}) {
    const cacheKey = `${endpoint}_${JSON.stringify(params)}`;
    const cachedData = getCachedData(cacheKey);
    if (cachedData && isCacheFresh(cachedData.timestamp)) {
        return cachedData.data;
    } else {
        const data = await getEndpoint(endpoint, params);
        storeDataInCache(cacheKey, data);
        return data;
    }
}

Let me know what you think.

[NotAud]

All good, I appreciate the snippets, will reference the util's specifically when I start implementing. Thank you.