search

CartoDB / Windshaft-cartodb

Windshaft tailored for CARTO
BSD 3-Clause "New" or "Revised" License
72 stars 59 forks source link

Create log-collector utilility #1174

Closed dgaubert closed 4 years ago

dgaubert commented 4 years ago
dgaubert commented 4 years ago

Demo:

diff --git a/test/acceptance/analysis/regressions-test.js b/test/acceptance/analysis/regressions-test.js
index d576515..32ce0a2 100644
--- a/test/acceptance/analysis/regressions-test.js
+++ b/test/acceptance/analysis/regressions-test.js
@@ -6,7 +6,7 @@ var assert = require('../../support/assert');
 var TestClient = require('../../support/test-client');

 describe('analysis-layers regressions', function () {
-    it('should return a complete list of nodes from analysis', function (done) {
+    it.only('should return a complete list of nodes from analysis', function (done) {
         var mapConfig = {
             version: '1.5.0',
             layers: [
$ LOG_LEVEL=info npm t
{"level":30,"time":1591606469078,"id":"09e01799-0086-4632-a5cc-e411367a1054","request":{"method":"POST","url":"/api/v1/map?api_key=1234","headers":{"host":"localhost","content-type":"application/json","content-length":"1286","connection":"close"},"remoteAddress":"127.0.0.1","remotePort":55426}}
{"level":40,"time":1591606469213,"id":"09e01799-0086-4632-a5cc-e411367a1054","error":[{"type":"Error","message":"Limit for number of rows is not defined","stack":"Error: Limit for number of rows is not defined\n    at Object.limitNumberOfRows (/vagrant/repos/camshaft/lib/node/limits.js:11:21)\n    at BaseNode.Node.checkLimits (/vagrant/repos/camshaft/lib/node/node.js:537:12)\n    at /vagrant/repos/camshaft/lib/limits/checker.js:12:18\n    at /vagrant/repos/camshaft/node_modules/async/lib/async.js:181:20\n    at iterate (/vagrant/repos/camshaft/node_modules/async/lib/async.js:262:13)\n    at Object.async.forEachOfSeries.async.eachOfSeries (/vagrant/repos/camshaft/node_modules/async/lib/async.js:281:9)\n    at Object.async.forEachSeries.async.eachSeries (/vagrant/repos/camshaft/node_modules/async/lib/async.js:214:22)\n    at module.exports (/vagrant/repos/camshaft/lib/limits/checker.js:9:11)\n    at analysis$checkLimits (/vagrant/repos/camshaft/lib/analysis.js:91:17)\n    at fn (/vagrant/repos/camshaft/node_modules/async/lib/async.js:746:34)\n    at /vagrant/repos/camshaft/node_modules/async/lib/async.js:1213:16\n    at /vagrant/repos/camshaft/node_modules/async/lib/async.js:166:37\n    at /vagrant/repos/camshaft/node_modules/async/lib/async.js:706:43\n    at /vagrant/repos/camshaft/node_modules/async/lib/async.js:167:37\n    at Immediate._onImmediate (/vagrant/repos/camshaft/node_modules/async/lib/async.js:1206:34)\n    at processImmediate (internal/timers.js:456:21)","node":{"id":"a23b8472f12aa0c4caea27c8c7a39b49d85e1b77","type":"source","query":"SELECT * FROM populated_places_simple_reduced"}}]}
{"level":40,"time":1591606469214,"id":"09e01799-0086-4632-a5cc-e411367a1054","error":[{"type":"Error","message":"Limit for number of rows is not defined","stack":"Error: Limit for number of rows is not defined\n    at Object.limitNumberOfRows (/vagrant/repos/camshaft/lib/node/limits.js:11:21)\n    at BaseNode.Node.checkLimits (/vagrant/repos/camshaft/lib/node/node.js:537:12)\n    at /vagrant/repos/camshaft/lib/limits/checker.js:12:18\n    at /vagrant/repos/camshaft/node_modules/async/lib/async.js:181:20\n    at Immediate.iterate [as _onImmediate] (/vagrant/repos/camshaft/node_modules/async/lib/async.js:262:13)\n    at processImmediate (internal/timers.js:456:21)","node":{"id":"78eb828244807ecfd0974380bfae341052ec22e4","type":"kmeans","source":"a23b8472f12aa0c4caea27c8c7a39b49d85e1b77","clusters":5}}]}
{"level":40,"time":1591606469215,"id":"09e01799-0086-4632-a5cc-e411367a1054","error":[{"type":"Error","message":"Limit for number of rows is not defined","stack":"Error: Limit for number of rows is not defined\n    at Object.limitNumberOfRows (/vagrant/repos/camshaft/lib/node/limits.js:11:21)\n    at BaseNode.Node.checkLimits (/vagrant/repos/camshaft/lib/node/node.js:537:12)\n    at /vagrant/repos/camshaft/lib/limits/checker.js:12:18\n    at /vagrant/repos/camshaft/node_modules/async/lib/async.js:181:20\n    at Immediate.iterate [as _onImmediate] (/vagrant/repos/camshaft/node_modules/async/lib/async.js:262:13)\n    at processImmediate (internal/timers.js:456:21)","node":{"id":"393c4b15cc047c09f9d52bc9742577df5d8230ba","type":"centroid","source":"78eb828244807ecfd0974380bfae341052ec22e4","category_column":"cluster_no","aggregation":"count","aggregation_column":null}}]}
{"level":40,"time":1591606469217,"id":"09e01799-0086-4632-a5cc-e411367a1054","error":[{"type":"Error","message":"Limit for number of rows is not defined","stack":"Error: Limit for number of rows is not defined\n    at Object.limitNumberOfRows (/vagrant/repos/camshaft/lib/node/limits.js:11:21)\n    at BaseNode.Node.checkLimits (/vagrant/repos/camshaft/lib/node/node.js:537:12)\n    at /vagrant/repos/camshaft/lib/limits/checker.js:12:18\n    at /vagrant/repos/camshaft/node_modules/async/lib/async.js:181:20\n    at iterate (/vagrant/repos/camshaft/node_modules/async/lib/async.js:262:13)\n    at /vagrant/repos/camshaft/node_modules/async/lib/async.js:274:29\n    at /vagrant/repos/camshaft/node_modules/async/lib/async.js:44:16\n    at /vagrant/repos/camshaft/lib/limits/checker.js:19:24\n    at /vagrant/repos/camshaft/lib/node/limits.js:52:16\n    at /vagrant/repos/camshaft/lib/limits/estimates.js:42:16\n    at /vagrant/repos/camshaft/lib/postgresql/query-runner.js:37:16\n    at /vagrant/repos/camshaft/node_modules/cartodb-psql/lib/psql.js:359:24\n    at Result.<anonymous> (/vagrant/repos/camshaft/node_modules/cartodb-psql/lib/psql.js:321:24)\n    at Result.Query.handleReadyForQuery (/vagrant/repos/camshaft/node_modules/pg/lib/query.js:144:10)\n    at Connection.<anonymous> (/vagrant/repos/camshaft/node_modules/pg/lib/client.js:184:19)\n    at Connection.emit (events.js:310:20)\n    at Socket.<anonymous> (/vagrant/repos/camshaft/node_modules/pg/lib/connection.js:135:12)\n    at Socket.emit (events.js:310:20)\n    at addChunk (_stream_readable.js:286:12)\n    at readableAddChunk (_stream_readable.js:268:9)\n    at Socket.Readable.push (_stream_readable.js:209:10)\n    at TCP.onStreamRead (internal/stream_base_commons.js:186:23)","node":{"id":"98bef5f58b99419de07698ecbc42e67c0087d0e5","type":"point-in-polygon","points_source":"a23b8472f12aa0c4caea27c8c7a39b49d85e1b77","polygons_source":"2e4f095732f5191d2bf0539e830e2aa541c38f4e"}}]}
{"level":30,"time":1591606469437,"id":"09e01799-0086-4632-a5cc-e411367a1054","analysis":{"id":"98bef5f58b99419de07698ecbc42e67c0087d0e5","user":"localhost","nodes":[{"id":"78eb828244807ecfd0974380bfae341052ec22e4","status":"pending","queued":true,"affectedTables":[],"outdated":true,"tags":[],"targetTable":"analysis_3c835a874c_78eb828244807ecfd0974380bfae341052ec22e4","params":{"type":"kmeans","source":"a23b8472f12aa0c4caea27c8c7a39b49d85e1b77","clusters":5}},{"id":"393c4b15cc047c09f9d52bc9742577df5d8230ba","status":"waiting","queued":false,"affectedTables":[],"outdated":true,"tags":[],"targetTable":"analysis_ae64186757_393c4b15cc047c09f9d52bc9742577df5d8230ba","params":{"type":"centroid","source":"78eb828244807ecfd0974380bfae341052ec22e4","category_column":"cluster_no","aggregation":"count","aggregation_column":null}},{"id":"2e4f095732f5191d2bf0539e830e2aa541c38f4e","status":"pending","queued":true,"affectedTables":[],"outdated":true,"tags":[],"targetTable":"analysis_2f13a3dbd7_2e4f095732f5191d2bf0539e830e2aa541c38f4e","params":{"type":"buffer","source":"393c4b15cc047c09f9d52bc9742577df5d8230ba","radius":200000,"isolines":null,"dissolved":null}},{"id":"98bef5f58b99419de07698ecbc42e67c0087d0e5","status":"waiting","queued":false,"affectedTables":[],"outdated":true,"tags":[],"targetTable":"analysis_2e94d3858c_98bef5f58b99419de07698ecbc42e67c0087d0e5","params":{"type":"point-in-polygon","points_source":"a23b8472f12aa0c4caea27c8c7a39b49d85e1b77","polygons_source":"2e4f095732f5191d2bf0539e830e2aa541c38f4e"}}]}}
{"level":30,"time":1591606470320,"id":"09e01799-0086-4632-a5cc-e411367a1054","response":{"statusCode":200,"headers":{"x-served-by-host":"cdb-dev","access-control-allow-origin":"*","access-control-allow-headers":"X-Requested-With, X-Prototype-Version, X-CSRF-Token, Authorization, Carto-Event, Carto-Event-Source, Carto-Event-Group-Id","vary":"Authorization","x-served-by-db-host":"localhost","x-layergroup-id":"6e44594cdbf5f2f0ca05296b89a3dea3:0","x-request-id":"09e01799-0086-4632-a5cc-e411367a1054","carto-user":"localhost","carto-user-id":"1","carto-map-id":"6e44594cdbf5f2f0ca05296b89a3dea3","carto-cache-buster":"0","content-type":"application/json; charset=utf-8","content-length":"5250"}}}
{"level":30,"time":1591606470320,"id":"09e01799-0086-4632-a5cc-e411367a1054","stats":{"response":1218,"createLayergroup":789,"mapSave":4,"validate":784,"layer-metadata":1,"total":1218}}
{"level":30,"time":1591606470323,"id":"09e01799-0086-4632-a5cc-e411367a1054","end":true}

=>

$ LOG_LEVEL=info npm t | node ./log-collector.js
{"level":"warn","time":1591606666387,"id":"2d1a9303-5fe5-4903-9c66-985083f25174","request":{"method":"POST","url":"/api/v1/map?api_key=1234","headers":{"host":"localhost","content-type":"application/json","content-length":"1286","connection":"close"},"remoteAddress":"127.0.0.1","remotePort":44598},"error":[{"type":"Error","message":"Limit for number of rows is not defined","stack":"Error: Limit for number of rows is not defined\n    at Object.limitNumberOfRows (/vagrant/repos/camshaft/lib/node/limits.js:11:21)\n    at BaseNode.Node.checkLimits (/vagrant/repos/camshaft/lib/node/node.js:537:12)\n    at /vagrant/repos/camshaft/lib/limits/checker.js:12:18\n    at /vagrant/repos/camshaft/node_modules/async/lib/async.js:181:20\n    at iterate (/vagrant/repos/camshaft/node_modules/async/lib/async.js:262:13)\n    at Object.async.forEachOfSeries.async.eachOfSeries (/vagrant/repos/camshaft/node_modules/async/lib/async.js:281:9)\n    at Object.async.forEachSeries.async.eachSeries (/vagrant/repos/camshaft/node_modules/async/lib/async.js:214:22)\n    at module.exports (/vagrant/repos/camshaft/lib/limits/checker.js:9:11)\n    at analysis$checkLimits (/vagrant/repos/camshaft/lib/analysis.js:91:17)\n    at fn (/vagrant/repos/camshaft/node_modules/async/lib/async.js:746:34)\n    at /vagrant/repos/camshaft/node_modules/async/lib/async.js:1213:16\n    at /vagrant/repos/camshaft/node_modules/async/lib/async.js:166:37\n    at /vagrant/repos/camshaft/node_modules/async/lib/async.js:706:43\n    at /vagrant/repos/camshaft/node_modules/async/lib/async.js:167:37\n    at Immediate._onImmediate (/vagrant/repos/camshaft/node_modules/async/lib/async.js:1206:34)\n    at processImmediate (internal/timers.js:456:21)","node":{"id":"a23b8472f12aa0c4caea27c8c7a39b49d85e1b77","type":"source","query":"SELECT * FROM populated_places_simple_reduced"}},{"type":"Error","message":"Limit for number of rows is not defined","stack":"Error: Limit for number of rows is not defined\n    at Object.limitNumberOfRows (/vagrant/repos/camshaft/lib/node/limits.js:11:21)\n    at BaseNode.Node.checkLimits (/vagrant/repos/camshaft/lib/node/node.js:537:12)\n    at /vagrant/repos/camshaft/lib/limits/checker.js:12:18\n    at /vagrant/repos/camshaft/node_modules/async/lib/async.js:181:20\n    at Immediate.iterate [as _onImmediate] (/vagrant/repos/camshaft/node_modules/async/lib/async.js:262:13)\n    at processImmediate (internal/timers.js:456:21)","node":{"id":"78eb828244807ecfd0974380bfae341052ec22e4","type":"kmeans","source":"a23b8472f12aa0c4caea27c8c7a39b49d85e1b77","clusters":5}},{"type":"Error","message":"Limit for number of rows is not defined","stack":"Error: Limit for number of rows is not defined\n    at Object.limitNumberOfRows (/vagrant/repos/camshaft/lib/node/limits.js:11:21)\n    at BaseNode.Node.checkLimits (/vagrant/repos/camshaft/lib/node/node.js:537:12)\n    at /vagrant/repos/camshaft/lib/limits/checker.js:12:18\n    at /vagrant/repos/camshaft/node_modules/async/lib/async.js:181:20\n    at Immediate.iterate [as _onImmediate] (/vagrant/repos/camshaft/node_modules/async/lib/async.js:262:13)\n    at processImmediate (internal/timers.js:456:21)","node":{"id":"393c4b15cc047c09f9d52bc9742577df5d8230ba","type":"centroid","source":"78eb828244807ecfd0974380bfae341052ec22e4","category_column":"cluster_no","aggregation":"count","aggregation_column":null}},{"type":"Error","message":"Limit for number of rows is not defined","stack":"Error: Limit for number of rows is not defined\n    at Object.limitNumberOfRows (/vagrant/repos/camshaft/lib/node/limits.js:11:21)\n    at BaseNode.Node.checkLimits (/vagrant/repos/camshaft/lib/node/node.js:537:12)\n    at /vagrant/repos/camshaft/lib/limits/checker.js:12:18\n    at /vagrant/repos/camshaft/node_modules/async/lib/async.js:181:20\n    at iterate (/vagrant/repos/camshaft/node_modules/async/lib/async.js:262:13)\n    at /vagrant/repos/camshaft/node_modules/async/lib/async.js:274:29\n    at /vagrant/repos/camshaft/node_modules/async/lib/async.js:44:16\n    at /vagrant/repos/camshaft/lib/limits/checker.js:19:24\n    at /vagrant/repos/camshaft/lib/node/limits.js:52:16\n    at /vagrant/repos/camshaft/lib/limits/estimates.js:42:16\n    at /vagrant/repos/camshaft/lib/postgresql/query-runner.js:37:16\n    at /vagrant/repos/camshaft/node_modules/cartodb-psql/lib/psql.js:359:24\n    at Result.<anonymous> (/vagrant/repos/camshaft/node_modules/cartodb-psql/lib/psql.js:321:24)\n    at Result.Query.handleReadyForQuery (/vagrant/repos/camshaft/node_modules/pg/lib/query.js:144:10)\n    at Connection.<anonymous> (/vagrant/repos/camshaft/node_modules/pg/lib/client.js:184:19)\n    at Connection.emit (events.js:310:20)\n    at Socket.<anonymous> (/vagrant/repos/camshaft/node_modules/pg/lib/connection.js:135:12)\n    at Socket.emit (events.js:310:20)\n    at addChunk (_stream_readable.js:286:12)\n    at readableAddChunk (_stream_readable.js:268:9)\n    at Socket.Readable.push (_stream_readable.js:209:10)\n    at TCP.onStreamRead (internal/stream_base_commons.js:186:23)","node":{"id":"98bef5f58b99419de07698ecbc42e67c0087d0e5","type":"point-in-polygon","points_source":"a23b8472f12aa0c4caea27c8c7a39b49d85e1b77","polygons_source":"2e4f095732f5191d2bf0539e830e2aa541c38f4e"}}],"analysis":{"id":"98bef5f58b99419de07698ecbc42e67c0087d0e5","user":"localhost","nodes":[{"id":"78eb828244807ecfd0974380bfae341052ec22e4","status":"pending","queued":true,"affectedTables":[],"outdated":true,"tags":[],"targetTable":"analysis_3c835a874c_78eb828244807ecfd0974380bfae341052ec22e4","params":{"type":"kmeans","source":"a23b8472f12aa0c4caea27c8c7a39b49d85e1b77","clusters":5}},{"id":"393c4b15cc047c09f9d52bc9742577df5d8230ba","status":"waiting","queued":false,"affectedTables":[],"outdated":true,"tags":[],"targetTable":"analysis_ae64186757_393c4b15cc047c09f9d52bc9742577df5d8230ba","params":{"type":"centroid","source":"78eb828244807ecfd0974380bfae341052ec22e4","category_column":"cluster_no","aggregation":"count","aggregation_column":null}},{"id":"2e4f095732f5191d2bf0539e830e2aa541c38f4e","status":"pending","queued":true,"affectedTables":[],"outdated":true,"tags":[],"targetTable":"analysis_2f13a3dbd7_2e4f095732f5191d2bf0539e830e2aa541c38f4e","params":{"type":"buffer","source":"393c4b15cc047c09f9d52bc9742577df5d8230ba","radius":200000,"isolines":null,"dissolved":null}},{"id":"98bef5f58b99419de07698ecbc42e67c0087d0e5","status":"waiting","queued":false,"affectedTables":[],"outdated":true,"tags":[],"targetTable":"analysis_2e94d3858c_98bef5f58b99419de07698ecbc42e67c0087d0e5","params":{"type":"point-in-polygon","points_source":"a23b8472f12aa0c4caea27c8c7a39b49d85e1b77","polygons_source":"2e4f095732f5191d2bf0539e830e2aa541c38f4e"}}]},"response":{"statusCode":200,"headers":{"x-served-by-host":"cdb-dev","access-control-allow-origin":"*","access-control-allow-headers":"X-Requested-With, X-Prototype-Version, X-CSRF-Token, Authorization, Carto-Event, Carto-Event-Source, Carto-Event-Group-Id","vary":"Authorization","x-served-by-db-host":"localhost","x-layergroup-id":"6e44594cdbf5f2f0ca05296b89a3dea3:0","x-request-id":"2d1a9303-5fe5-4903-9c66-985083f25174","carto-user":"localhost","carto-user-id":"1","carto-map-id":"6e44594cdbf5f2f0ca05296b89a3dea3","carto-cache-buster":"0","content-type":"application/json; charset=utf-8","content-length":"5250"}},"stats":{"response":959,"createLayergroup":591,"mapSave":1,"validate":590,"total":959}}

cc/ @oleurud

dgaubert commented 4 years ago

Sample of the new log output, LOG_LEVEL=info npm t | node ./log-collector.js: maps-api-revamp-log.json.zip