Data Points and relevant APIs

[chrlschwb]

1. Social Media
   • Number of followers of https://twitter.com/JoystreamDAO/ (twitter API) <1s

     
     const Twit = require('twit');

const T = new Twit({ consumer_key: '...', consumer_secret: '...', access_token: '...', access_token_secret: '...', timeout_ms: 60*1000, // optional HTTP request timeout to apply to all requests. });

T.get('users/show', { screen_name: 'username' }, function(err, data, response) { console.log(data.followers_count); });

- Number of tweets on `https://twitter.com/JoystreamDAO/`                  (twitter API)  <1s

const Twit = require('twit');

const T = new Twit({ consumer_key: '...', consumer_secret: '...', access_token: '...', access_token_secret: '...', timeout_ms: 60*1000, // optional HTTP request timeout to apply to all requests. });

T.get('users/show', { screen_name: 'username' }, function(err, data, response) { console.log(data.statuses_count); });

- Number of followers of `https://www.youtube.com/@joystream8627`       (Youtube API)  <1s
- Number of videos on `https://www.youtube.com/@joystream8627`          (Youtube API)  <1s

const { google } = require('googleapis'); const OAuth2 = google.auth.OAuth2;

const CLIENT_ID = 'YOUR_CLIENT_ID'; const CLIENT_SECRET = 'YOUR_CLIENT_SECRET'; const REDIRECT_URI = 'YOUR_REDIRECT_URI'; const REFRESH_TOKEN = 'YOUR_REFRESH_TOKEN';

const oauth2Client = new OAuth2( CLIENT_ID, CLIENT_SECRET, REDIRECT_URI );

oauth2Client.setCredentials({ refresh_token: REFRESH_TOKEN });

const youtube = google.youtube({ version: 'v3', auth: oauth2Client });

const channelId = 'YOUR_CHANNEL_ID';

youtube.channels.list({ part: ['snippet', 'tatistics'], id: channelId }, (err, res) => { if (err) return console.log('The API returned an error: ' err);

const channel = res.data.items[0]; const statistics = channel.statistics;

console.log(Total views: ${statistics.viewCount}); console.log(Total subscribers: ${statistics.subscriberCount}); console.log(Total videos: ${statistics.videoCount}); });

- Number of followers of `https://www.reddit.com/r/joystream_dao/`        ( reddit api ) <1s
- Number of posts on `https://www.reddit.com/r/joystream_dao/`             ( reddit api )

const Reddit = require('reddit');

const reddit = new Reddit({ userAgent: 'YOUR_USER_AGENT', clientId: 'YOUR_CLIENT_ID', clientSecret: 'YOUR_CLIENT_SECRET', refreshToken: 'YOUR_REFRESH_TOKEN' });

reddit.auth().then(() => { const username = 'YOUR_USERNAME';

reddit.get(/user/${username}/about.json).then(response => { const data = response.data;

console.log(`Username: ${data.name}`);
console.log(`Link karma: ${data.link_karma}`);
console.log(`Comment karma: ${data.comment_karma}`);
console.log(`Total posts: ${data.total_posts}`);

}); });

- Number of followers of `https://coinmarketcap.com/community/profile/JoystreamDAO/`           (CMC API) 1s
- Number of posts on `https://coinmarketcap.com/community/profile/JoystreamDAO/`                (CMC API)

const axios = require('axios');

const API_KEY = 'YOUR_API_KEY'; const COMMUNITY_ID = '1';

axios.get(https://pro-api.coinmarketcap.com/v1/cryptocurrency/quotes/latest?id=${COMMUNITY_ID}&convert=USD, { headers: { 'X-CMC_PRO_API_KEY': API_KEY } }).then(response => { const data = response.data; const community = data.data[COMMUNITY_ID]; const followers = community.quote.USD.followers; const posts = community.quote.posts;

console.log(JoystreamDAO community has ${followers} followers); }).catch(error => { console.log(error); });

- Number of followers on `https://debank.com/official-account/111686/stream`      (debank api) 1s
- Number of posts on `https://debank.com/official-account/111686/stream`            (debank api)

const DeBank = require('debank.js');

const debank = new DeBank({ apiEndpoint: 'https://api.debank.com/api/v1', apiKey: 'YOUR_API_KEY' });

const username = 'YOUR_USERNAME';

debank.getUser(username).then(user => { console.log(Username: ${user.username}); console.log(Followers: ${user.followers}); });

- Number of users on `https://t.me/JoyStreamOfficial`                                      (telegram API) 1s
- Number of messages on `https://t.me/JoyStreamOfficial`                               (telegram API)

const TelegramBot = require('node-telegram-bot-api');

const token = 'YOUR_API_TOKEN'; const bot = new TelegramBot(token, { polling: true });

const channel = 'JoyStreamOfficial'; let messages;

bot.getChatMembersCount(channel).then(count => { console.log(Number of users: ${count}); });

- Number of users on `https://discord.gg/joystream-811216481340751934`             ( discord api ) <1s

const Discord = require('discord.js'); const client = new Discord.Client();

client.on('ready', () => { console.log(Logged in as ${client.user.tag}!); });

client.on('message', msg => { if (msg.content === '!membercount') { const userCount = msg.guild.members.cache.filter(member => !member.user.bot).size; msg.reply(Server user count: ${userCount}); } });

client.login('your-token-goes-here');

- Number of messages on `https://discord.gg/joystream-811216481340751934`      (discord api) <1s 

client.on('message', async msg => { if (msg.content === '!messagecount') { let totalMessageCount = 0;

for (let channel of msg.guild.channels.cache.values()) {
  if (channel.type === 'text') {
    await channel.messages.fetch().then(messages => {
      totalMessageCount += messages.size;
    }).catch(console.error);
  }
}

msg.reply(`Total message count: ${totalMessageCount}`);

} });

client.login('your-token-goes-here');

- Number of contributors on `https://github.com/Joystream`                        <= github
- Number of commits on `https://github.com/Joystream`                              <= github
- Number of new releases on `https://github.com/Joystream`                       <= github

const axios = require('axios');

const token = 'YOUR_ACCESS_TOKEN'; const owner = 'Joystream'; const repo = 'joystream';

axios.get(https://api.github.com/repos/${owner}/${repo}/contributors, { headers: { 'Authorization': Bearer ${token} } }).then(response => { const contributors = response.data;

console.log(Number of contributors: ${contributors.length}); });

2. Token Info
- price                                                  (CMC)  1s
- volume                                              (CMC)

const getPrice = async () => { try { const { data: { data }, } = await axios.get( "https://pro-api.coinmarketcap.com/v2/cryptocurrency/quotes/latest?slug=joystream&convert=USD", { // body: JSON.stringify(params), headers: { Accepts: "application/json", "X-CMC_PRO_API_KEY": API_KEY, }, } );

return { price: data[JOYSTREAM_CMC_TOKEN_ID].quote.USD.price };

} catch (e) { // In case there are problems with the API, we just return 0.

return { price: 0 };

} };

- circulating supply                                         (RPC)  6s

async calculateCirculatingSupply() { const VESTING_STRING_HEX = "0x76657374696e6720";

const accounts = [];
const amounts: BN[] = [];
const lockData = await this.api.query.balances.locks.entries();

for (let [storageKey, palletBalances] of lockData) {
  let vested = new BN(0);
  for (let palletBalance of palletBalances) {
    if (
      palletBalance.id.toString() === VESTING_STRING_HEX &&
      palletBalance.amount.toBn().gt(vested)
    ) {
      vested = palletBalance.amount.toBn();
    }
  }

  if (vested.gt(new BN(0))) {
    accounts.push(storageKey.args[0].toString());
    amounts.push(vested);
  }
}

const intAccs = await this.api.query.system.account.multi(accounts);

const total = intAccs.reduce((accumulator, val, index) => {
  return accumulator.add(BN.min(amounts[index], BN.min(val.data.free, val.data.miscFrozen)));
}, new BN(0));

const totalSupply = await this.totalIssuanceInJOY();

return totalSupply - this.toJOY(total);

}

- total supply                                       (RPC)  6s

async totalIssuanceInJOY(blockHash?: Hash): Promise { const issuanceInHAPI = blockHash === undefined ? await this.api.query.balances.totalIssuance() : await this.api.query.balances.totalIssuance.at(blockHash);

return this.toJOY(issuanceInHAPI)

}

- total minting                                     (QN)
- minting for worker reward                (QN)
- minting for spending proposal         (QN)
- minting for validator payout             (QN)
- minting for creator payout                (QN)
- tokens staked on validators             (RPC)
- APR for staking on validators           (RPC)

const apr = rewardHistory.length && !stakingInfo.total.toBn().isZero() ? last(rewardHistory) .eraReward.toBn() .muln(ERAS_PER_YEAR) .mul(validatorInfo.commission.toBn()) .div(stakingInfo.total.toBn()) .divn(10 ** 7) // Convert from Perbill to Percent .toNumber() : 0

from Pioneer  `packages/ui/src/validators/hooks/useValidatorsList.tsx`

3. Traction
- Number of channels   (QN) 2s

channels(limit:1000000000,orderBy:createdAt_DESC,where:{totalVideosCreated_gt:0}){ id }

- Number of videos       (QN) 1s

videos(limit:1,orderBy:createdAt_DESC){ id }

- Number of comments    (QN) 2s

comments(limit:1000000000){ id }

- Number of reactions      (QN) 3s

videoReactions(limit:1000000000){ id
} commentReactions(limit:1000000000){ id }

- Number of video NFTs      (QN) 1s

nftIssuedEvents(limit:1000000000){ id }

- Number of video NFT sales        (QN) 5s
- total price of sold video NFTs     (QN)

nftBoughtEvents(limit:1000000){ id price } auctions(limit:1000000,where:{isCompleted_eq:true}){ id topBid{ amount } }

- YPP payout                          (data extraction from hubspot)
- Number of blocks                (RPC)  6s

chain.getHeader

- Number of transactions       (new subsquid required)
- Number of extrinsics           (new subsquid required)  

4. Team
- data of council members  (QN) 1s

councilMembers(limit:1000){ member{ handle metadata{ externalResources{ type value } }

} accumulatedReward electedInCouncilId }

- data of leads      (QN) 2s

workingGroups{ id leader{ membership{ handle externalResources{ type value } }

} }

[chrlschwb]

Schema for DB all data will be stored in four fields type, value, epoch, date_time Type: the name of the data Value: the value of the data at the given time Epoch: incrementally increasing number Date_Time: the time at which value was stored in the db

Schema for dashboard API Single API endpoint to fetch all datapoints for a given period like this /dashboard-api/2023-09-01T00:00:00/2023-09-02T00:00:00

Response would be like

[
{
   {
    type: telegram_follower,
    value: 100,
    epoch: 1,
    date_time: 2023-09-01T00:00:00
   },
   {
   type: telegram_posts,
   value: 50,
   epoch: 1,
   date_time: 2023-09-01T00:00:00
   }
   ...
},
{
   {
    type: telegram_follower,
    value: 100,
   epoch: 2,
    date_time: 2023-09-01T00:01:00
   },
   {
   type: telegram_posts,
   value: 50,
   epoch: 2,
   date_time: 2023-09-01T00:01:00
   }
   ...
}
]

[bedeho]

• Good job on mapping out the API calls, looks solid.
• This data model design does not correspond to requirements that all data is written in epochs atomically, with a single value for all time series. You have to at least address this point, or explain why you have not wanted to do so, or something, silently ignoring requirements does not work, because I will not always have time to review carefully, so I may not realize you have gone another path without this being drawn attention to carefully.
• I actually don't mind if you want to go for this model now, we can see how far it takes us, its simpler.
• I also notice that this is a very unstructured model, where there is not predefined set of types, or enforcement of data type for a given type of time series value. I don't know if this will end up well, but I'm happy to see where we end up.

Question

• How will we manage all of these API keys?
• Did you actually test all of these calls, or is this just snippets?
• There is no rate-limiting information here.

[chrlschwb]

• API keys should be managed by the person who hosts this service
• I tested the calls
• I will try to come up with rate-limit information further along the development

[bedeho]

I do not believe this is a complete proposal, for example I do not see anything for "daily active accounts" ?

Please, again, go through the full brief and make sure you address all data points, otherwise we will incurr costly delays.

[bedeho]

Schema for dashboard API

Please read the requirements, this proposal does not even begin to satisfy it.