search

nodezoo / nodezoo-npm

nodezoo-npm
http://github.com/nodezoo/tao
MIT License
11 stars 39 forks source link

Sanitise Data leaving npm #31

Closed matt-oc closed 8 years ago

matt-oc commented 8 years ago

The data leaving the Npm module (and travis and github) needs to be decided upon before it leaves. The problem is we have alot of data for each module and the data for each query may be different. Below is an example dump from the Npm module (It returns every version but just the first and last are shown here for ease of readability):

Searching for Lodash:

{
  "_id": "lodash",
  "_rev": "1495-000035283f01c5c63d8ee8a2f45b37d9",
  "name": "lodash",
  "description": "Lodash modular utilities.",
  "dist-tags": {
    "latest": "4.8.2"
  },
  "versions": {
    "0.1.0": {
      "name": "lodash",
      "version": "0.1.0",
      "description": "A drop-in replacement for Underscore.js that delivers up to 8x performance improvements, bug fixes, and additional features.",
      "homepage": "https:\/\/github.com\/bestiejs\/lodash",
      "main": "lodash",
      "keywords": [
        "browser",
        "client",
        "functional",
        "performance",
        "server",
        "speed",
        "util"
      ],
      "licenses": [
        {
          "type": "MIT",
          "url": "http:\/\/mths.be\/mit"
        }
      ],
      "author": {
        "name": "John-David Dalton",
        "email": "john@fusejs.com",
        "url": "http:\/\/allyoucanleet.com\/"
      },
      "bugs": {
        "url": "https:\/\/github.com\/bestiejs\/lodash\/issues"
      },
      "repository": {
        "type": "git",
        "url": "git:\/\/github.com\/bestiejs\/lodash.git"
      },
      "engines": [
        "node",
        "rhino"
      ],
      "directories": {
        "doc": ".\/doc",
        "test": ".\/test"
      },
      "_npmUser": {
        "name": "jdalton",
        "email": "john@fusejs.com"
      },
      "_id": "lodash@0.1.0",
      "dependencies": {

      },
      "devDependencies": {

      },
      "optionalDependencies": {

      },
      "_engineSupported": true,
      "_npmVersion": "1.1.0-3",
      "_nodeVersion": "v0.6.11",
      "_defaultsLoaded": true,
      "dist": {
        "shasum": "392617f69a947e40cec7848d85fcc3dd29d74bc5",
        "tarball": "https:\/\/registry.npmjs.org\/lodash\/-\/lodash-0.1.0.tgz"
      },
      "maintainers": [
        {
          "name": "jdalton",
          "email": "john@fusejs.com"
        }
      ],
      "deprecated": "lodash@<3.0.0 is no longer maintained. Upgrade to lodash@^4.0.0."
    },
    "4.8.2": {
      "name": "lodash",
      "version": "4.8.2",
      "description": "Lodash modular utilities.",
      "homepage": "https:\/\/lodash.com\/",
      "icon": "https:\/\/lodash.com\/icon.svg",
      "license": "MIT",
      "main": "lodash.js",
      "keywords": [
        "modules",
        "stdlib",
        "util"
      ],
      "author": {
        "name": "John-David Dalton",
        "email": "john.david.dalton@gmail.com",
        "url": "http:\/\/allyoucanleet.com\/"
      },
      "contributors": [
        {
          "name": "John-David Dalton",
          "email": "john.david.dalton@gmail.com",
          "url": "http:\/\/allyoucanleet.com\/"
        },
        {
          "name": "Blaine Bublitz",
          "email": "blaine.bublitz@gmail.com",
          "url": "https:\/\/github.com\/phated"
        },
        {
          "name": "Mathias Bynens",
          "email": "mathias@qiwi.be",
          "url": "https:\/\/mathiasbynens.be\/"
        }
      ],
      "repository": {
        "type": "git",
        "url": "git+https:\/\/github.com\/lodash\/lodash.git"
      },
      "scripts": {
        "test": "echo \"See https:\/\/travis-ci.org\/lodash\/lodash-cli for testing details.\""
      },
      "bugs": {
        "url": "https:\/\/github.com\/lodash\/lodash\/issues"
      },
      "_id": "lodash@4.8.2",
      "_shasum": "478ad7ff648c3c71a2f6108e032c5c0cc40747df",
      "_from": ".",
      "_npmVersion": "2.15.2",
      "_nodeVersion": "5.9.1",
      "_npmUser": {
        "name": "jdalton",
        "email": "john.david.dalton@gmail.com"
      },
      "dist": {
        "shasum": "478ad7ff648c3c71a2f6108e032c5c0cc40747df",
        "tarball": "https:\/\/registry.npmjs.org\/lodash\/-\/lodash-4.8.2.tgz"
      },
      "maintainers": [
        {
          "name": "jdalton",
          "email": "john.david.dalton@gmail.com"
        },
        {
          "name": "jridgewell",
          "email": "justin+npm@ridgewell.name"
        },
        {
          "name": "mathias",
          "email": "mathias@qiwi.be"
        },
        {
          "name": "phated",
          "email": "blaine.bublitz@gmail.com"
        }
      ],
      "_npmOperationalInternal": {
        "host": "packages-12-west.internal.npmjs.com",
        "tmp": "tmp\/lodash-4.8.2.tgz_1459822516091_0.773343451321125"
      },
      "directories": {

      }
    }
  },
  "readme": "# lodash v4.8.2\n\nThe [Lodash](https:\/\/lodash.com\/) library exported as [Node.js](https:\/\/nodejs.org\/) modules.\n\n## Installation\n\nUsing npm:\n```bash\n$ {sudo -H} npm i -g npm\n$ npm i --save lodash\n```\n\nIn Node.js:\n```js\n\/\/ Load the full build.\nvar _ = require('lodash');\n\/\/ Load the core build.\nvar _ = require('lodash\/core');\n\/\/ Load the fp build for immutable auto-curried iteratee-first data-last methods.\nvar fp = require('lodash\/fp');\n\n\/\/ Load a method category.\nvar array = require('lodash\/array');\nvar object = require('lodash\/fp\/object');\n\n\/\/ Load a single method for smaller builds with browserify\/rollup\/webpack.\nvar chunk = require('lodash\/chunk');\nvar extend = require('lodash\/fp\/extend');\n```\n\nSee the [package source](https:\/\/github.com\/lodash\/lodash\/tree\/4.8.2-npm) for more details.\n\n**Note:**<br>\nDon\u00e2\u20ac\u2122t assign values to the [special variable](http:\/\/nodejs.org\/api\/repl.html#repl_repl_features) `_` when in the REPL.<br>\nInstall [n_](https:\/\/www.npmjs.com\/package\/n_) for a REPL that includes `lodash` by default.\n\n## Support\n\nTested in Chrome 48-49, Firefox 44-45, IE 9-11, Edge 13, Safari 8-9, Node.js 0.10, 0.12, 4, & 5, & PhantomJS 1.9.8.<br>\nAutomated [browser](https:\/\/saucelabs.com\/u\/lodash) & [CI](https:\/\/travis-ci.org\/lodash\/lodash\/) test runs are available.\n",
  "maintainers": [
    {
      "name": "jdalton",
      "email": "john.david.dalton@gmail.com"
    },
    {
      "name": "jridgewell",
      "email": "justin+npm@ridgewell.name"
    },
    {
      "name": "mathias",
      "email": "mathias@qiwi.be"
    },
    {
      "name": "phated",
      "email": "blaine.bublitz@gmail.com"
    }
  ],
  "time": {
    "modified": "2016-04-05T02:15:16.661Z",
    "created": "2012-04-23T16:37:11.912Z",
    "0.1.0": "2012-04-23T16:37:12.603Z",
    "0.2.0": "2012-05-22T04:06:24.044Z",
    "0.2.1": "2012-05-24T21:53:08.449Z",
    "0.2.2": "2012-05-30T07:56:26.644Z",
    "0.3.0": "2012-06-06T20:01:49.669Z",
    "0.3.1": "2012-06-11T04:12:51.792Z",
    "0.3.2": "2012-06-14T19:19:49.846Z",
    "0.4.0": "2012-07-11T17:14:20.142Z",
    "0.4.1": "2012-07-12T04:56:31.883Z",
    "0.4.2": "2012-07-16T18:49:41.162Z",
    "0.5.0-rc.1": "2012-08-07T15:08:27.331Z",
    "0.5.0": "2012-08-17T20:13:07.054Z",
    "0.5.1": "2012-08-18T20:15:42.131Z",
    "0.5.2": "2012-08-22T16:22:03.757Z",
    "0.6.0": "2012-08-28T16:01:09.459Z",
    "0.6.1": "2012-08-30T08:01:38.808Z",
    "0.7.0": "2012-09-11T16:24:07.425Z",
    "0.8.0": "2012-10-02T06:49:38.116Z",
    "0.8.1": "2012-10-04T08:53:29.540Z",
    "0.8.2": "2012-10-10T07:51:31.600Z",
    "0.9.0": "2013-08-31T04:46:20.474Z",
    "0.9.1": "2013-08-31T04:49:15.754Z",
    "0.9.2": "2013-08-31T04:52:21.307Z",
    "0.10.0": "2013-08-31T04:56:09.871Z",
    "1.0.0-rc.1": "2013-08-31T05:00:28.060Z",
    "1.0.0-rc.2": "2013-08-31T05:05:31.147Z",
    "1.0.0-rc.3": "2013-08-31T05:08:51.705Z",
    "1.0.0": "2013-08-31T05:11:42.645Z",
    "1.0.1": "2013-08-31T05:16:47.715Z",
    "1.1.0": "2013-09-04T14:23:45.728Z",
    "1.1.1": "2013-09-04T14:24:07.907Z",
    "1.2.0": "2013-09-04T14:24:34.140Z",
    "1.2.1": "2013-09-04T14:24:58.381Z",
    "1.3.0": "2013-09-04T14:25:19.793Z",
    "1.3.1": "2013-09-04T14:25:40.429Z",
    "2.0.0": "2013-09-14T04:22:28.159Z",
    "2.1.0": "2013-09-23T05:57:42.595Z",
    "2.2.0": "2013-09-29T21:52:47.266Z",
    "2.2.1": "2013-10-03T18:29:30.163Z",
    "2.3.0": "2013-11-11T17:30:27.058Z",
    "2.4.0": "2013-11-26T19:40:30.164Z",
    "2.4.1": "2013-12-03T16:51:12.879Z",
    "3.0.0": "2015-01-26T15:09:31.198Z",
    "3.0.1": "2015-01-30T09:33:51.621Z",
    "3.1.0": "2015-02-03T16:53:35.795Z",
    "3.2.0": "2015-02-12T17:01:18.403Z",
    "3.3.0": "2015-02-20T17:08:28.864Z",
    "3.3.1": "2015-02-24T16:02:47.458Z",
    "3.4.0": "2015-03-06T16:44:06.018Z",
    "3.5.0": "2015-03-09T05:01:51.264Z",
    "3.6.0": "2015-03-25T15:36:29.983Z",
    "1.0.2": "2015-03-30T15:58:01.337Z",
    "3.7.0": "2015-04-16T15:47:35.770Z",
    "2.4.2": "2015-04-26T21:04:49.443Z",
    "3.8.0": "2015-05-01T15:45:44.760Z",
    "3.9.0": "2015-05-19T18:26:55.450Z",
    "3.9.1": "2015-05-19T21:00:20.625Z",
    "3.9.2": "2015-05-24T20:57:57.973Z",
    "3.9.3": "2015-05-26T01:47:44.058Z",
    "3.10.0": "2015-06-30T15:13:28.602Z",
    "3.10.1": "2015-08-04T06:05:06.887Z",
    "4.0.0": "2016-01-12T23:13:20.539Z",
    "4.0.1": "2016-01-25T16:06:17.924Z",
    "4.1.0": "2016-01-29T16:33:24.543Z",
    "4.2.0": "2016-02-02T08:50:17.287Z",
    "4.2.1": "2016-02-03T16:00:16.046Z",
    "4.3.0": "2016-02-08T08:57:19.880Z",
    "4.4.0": "2016-02-16T07:10:16.856Z",
    "4.5.0": "2016-02-17T08:39:42.533Z",
    "4.5.1": "2016-02-22T06:42:24.244Z",
    "4.6.0": "2016-03-02T03:24:37.179Z",
    "4.6.1": "2016-03-02T18:09:40.696Z",
    "4.7.0": "2016-03-31T15:46:33.373Z",
    "4.8.0": "2016-04-04T14:54:33.612Z",
    "4.8.1": "2016-04-04T15:43:49.109Z",
    "4.8.2": "2016-04-05T02:15:16.661Z"
  },
  "author": {
    "name": "John-David Dalton",
    "email": "john.david.dalton@gmail.com",
    "url": "http:\/\/allyoucanleet.com\/"
  },
  "repository": {
    "type": "git",
    "url": "git+https:\/\/github.com\/lodash\/lodash.git"
  },
  "users": {
    "285858315": true,
    "306766053": true,
    "326060588": true,
    "sjonnet": true,
    "fgribreau": true,
    "sjonnet19": true,
    "pid": true,
    "jswartwood": true,
    "alexindigo": true,
    "klaemo": true,
    "Scryptonite": true,
    .........etc
  },
  "readmeFilename": "README.md",
  "homepage": "https:\/\/lodash.com\/",
  "keywords": [
    "modules",
    "stdlib",
    "util"
  ],
  "contributors": [
    {
      "name": "John-David Dalton",
      "email": "john.david.dalton@gmail.com",
      "url": "http:\/\/allyoucanleet.com\/"
    },
    {
      "name": "Blaine Bublitz",
      "email": "blaine.bublitz@gmail.com",
      "url": "https:\/\/github.com\/phated"
    },
    {
      "name": "Mathias Bynens",
      "email": "mathias@qiwi.be",
      "url": "https:\/\/mathiasbynens.be\/"
    }
  ],
  "bugs": {
    "url": "https:\/\/github.com\/lodash\/lodash\/issues"
  },
  "license": "MIT",
  "_attachments": {

  }
}

Proposed object:

{
"name":
"url":
"id":
"description":
"latest_version":
"release_count":
"dependencies":
"author":
"licence":
"maintainers":
}
matt-oc commented 8 years ago

Initial proposed object now implemented

mihaidma commented 8 years ago

@matt-oc may I close this?

mihaidma commented 8 years ago

seems solved, will close it