Add duplicates column to generated CSV

[miketaylr]

Context: https://docs.google.com/spreadsheets/d/1lSbM-gPtyyuUf_gO3GGrZKfZ_FP50XZ6N9xmNlouqpU/edit#gid=1488805010 column H contains a value for "duplicate" bugs.

What this should represent is "bugs for the given domain on webcompat.com that live in the Duplicate milestone and appear as a "see also" bug for an open core Bugzilla bug".

It seems a little messy to get this data, but @adamopenweb did it once!

[adamopenweb]

After double checking, it doesn't appear to be limited at 500 bugs. @miketaylr maybe worth double checking my Bugzilla query below to make sure it fits our need.

Mind my awful coding, this might help:

var topSites = ["google.com","mail.google.com","docs.google.com","youtube.com","facebook.com","amazon.com","reddit.com","wikipedia.org","yahoo.com","mail.yahoo.com","twitter.com","instagram.com","linkedin.com","ebay.com","netflix.com","twitch.tv","outlook.live.com","office.live.com","pornhub.com","imgur.com","paypal.com","bing.com","pinterest.com","tumblr.com","imdb.com","wikia.com","livejasmin.com","apple.com","support.microsoft.com","github.com","stackoverflow.com","dropbox.com","xvideos.com","blogspot.com","adobe.com","msn.com","indeed.com","wordpress.com","open.spotify.com","chaturbate.com","xhamster.com","soundcloud.com","bbc.com","discordapp.com","vk.com","vimeo.com","xnxx.com","deviantart.com","yandex.ru","web.whatsapp.com","aliexpress.com","txxx.com","dailymotion.com","espn.com","craigslist.org","chase.com","cnn.com","instructure.com","nytimes.com","salesforce.com","yelp.com","walmart.com","hulu.com","bankofamerica.com","wellsfargo.com","zillow.com","quora.com","weather.com","breitbart.com","foxnews.com","etsy.com","xfinity.com","quizlet.com","stackexchange.com","washingtonpost.com","www.nlm.nih.gov","usps.com","bestbuy.com","capitalone.com","homedepot.com","godaddy.com","target.com","dailymail.co.uk","outbrain.com","slickdeals.net","intuit.com","mail.aol.com","glassdoor.com","blackboard.com","nfl.com","vice.com","tmall.com","baidu.com","forbes.com","patch.com","myshopify.com","businessinsider.com","huffingtonpost.com","okta.com","buzzfeed.com","theguardian.com","americanexpress.com","tripadvisor.com","youporn.com","upornia.com","ikea.com","ebay-kleinanzeigen.de","mail.ru","web.de","gmx.net","spiegel.de","t-online.de","bild.de","chip.de","otto.de","mobile.de","orange.fr","leboncoin.fr","free.fr","cdiscount.com","labanquepostale.fr","lemonde.fr","pole-emploi.fr","lefigaro.fr","sfr.fr","allocine.fr","kijiji.ca","td.com","royalbank.com","cbc.ca","theweathernetwork.com","narcity.com","rbcroyalbank.com","scotiabank.com","canada.ca","utoronto.ca","ladbible.com","hellomagazine.com","www.gov.uk","bt.com","gumtree.com","rightmove.co.uk","sportbible.com","lloydsbank.co.uk","telegraph.co.uk","asos.com","theatlantic.com","bloomberg.com","wired.com","theringer.com","newyorker.com","qz.com","npr.org","vox.com","theverge.com","hbr.org","fastcompany.com","medium.com","economist.com","aeon.co","gq.com","lifehacker.com","vulture.com","arstechnica.com","theoutline.com","zeit.de","sueddeutsche.de","tagesspiegel.de","heise.de","spektrum.de","wiwo.de","flipkart.com","porn555.com","onlinesbi.com","hotstar.com","indiatimes.com","irctc.co.in","hdfcbank.com","cricbuzz.com","ndtv.com","icicibank.com","tribunnews.com","detik.com","bukalapak.com","tokopedia.com","liputan6.com","kompas.com","sindonews.com","kumparan.com","kaskus.co.id","grid.id","rakuten.co.jp","nicovideo.jp","fc2.com","kakaku.com","ameblo.jp","livedoor.jp","dmm.co.jp","blog.jp","goo.ne.jp","abs-cbn.com","inquirer.net","gmanetwork.com","lazada.com.ph","rappler.com","shopee.ph","yts.am","kissanime.ru","slideshare.net","newsprofin.com","pantip.com","movie2free.com","line.me","lazada.co.th","sanook.com","kapook.com","shopee.co.th","anime-sugoi.com","037hd.com","dek-d.com","sina.com.cn","weibo.com","360.cn","login.tmall.com","pages.tmall.com","csdn.net","alipay.com","detail.tmall.com","bilibili.com","xinhuanet.com"];

// {code snipped for readability}

    // Loop through each Bugzilla report in search of Webcompat bugs
    _.forEach(report.see_also, function(seeAlsoUrl, i) {
      if (seeAlsoUrl.indexOf("webcompat.com") != -1 || seeAlsoUrl.indexOf("github.com/webcompat/web-bugs") != -1) {
        seeAlsoReports.push(seeAlsoUrl);

        //Get the issue number
        var reportNumber = seeAlsoUrl.substr(seeAlsoUrl.indexOf("/issues/") + 8);
        if (reportNumber.indexOf("#") != -1){
          var split = reportNumber.split("#");
          reportNumber = split[0];
        }

        // Check the webcompat report if it's a duplicate
        _.forEach(issuesList, function(issue, i) {
          if (issue.number == reportNumber) {
            if (issue.milestone.title == "duplicate"){
              var title = issue.title.toLowerCase();

              //Loop through top 200 sites and see if there's a match
              _.forEach(topSites, function(site, i) {

                if (title.indexOf(site) != -1 ){

Bugzilla query: https://bugzilla.mozilla.org/rest/bug?j_top=OR&list_id=14615419&o1=substring&o2=substring&v1=web-bugs&v2=webcompat.com&f1=see_also&resolution=---&classification=Client%20Software&classification=Developer%20Infrastructure&classification=Components&classification=Server%20Software&classification=Other&query_format=advanced&f2=see_also

Sample Bugzilla data dump: http://adamstevenson.ca/mozilla/files/issuesBugzilla.json

[miketaylr]

Awesome, thanks @adamopenweb

[miketaylr]

weee.