From 1ffb64083358e4a586b0433d5878eef7cd42ad9b Mon Sep 17 00:00:00 2001 From: Tyler Renelle Date: Tue, 14 Oct 2014 15:10:13 -0600 Subject: [PATCH] tmp(memleak): temporary memleak measure to check avg response-time every 3mins. If >650ms (high mem) or ==0 (blackout), restart --- config.json.example | 2 ++ src/middleware.js | 23 +++++++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/config.json.example b/config.json.example index 4078868cec..821bb91fc3 100644 --- a/config.json.example +++ b/config.json.example @@ -17,6 +17,8 @@ "STRIPE_API_KEY":"aaaabbbbccccddddeeeeffff00001111", "STRIPE_PUB_KEY":"22223333444455556666777788889999", "NEW_RELIC_LICENSE_KEY":"NEW_RELIC_LICENSE_KEY", + "NEW_RELIC_APPLICATION_ID":"NEW_RELIC_APPLICATION_ID", + "NEW_RELIC_API_KEY":"NEW_RELIC_API_KEY", "GA_ID": "GA_ID", "PAYPAL_USERNAME": "PAYPAL_USERNAME", "PAYPAL_PASSWORD": "PAYPAL_PASSWORD", diff --git a/src/middleware.js b/src/middleware.js index 409f829668..1c9b723b52 100644 --- a/src/middleware.js +++ b/src/middleware.js @@ -9,6 +9,8 @@ var domainMiddleware = require('domain-middleware'); var cluster = require('cluster'); var i18n = require('./i18n.js'); var shared = require('habitrpg-shared'); +var request = require('request'); +var moment = require('moment'); module.exports.apiThrottle = function(app) { if (nconf.get('NODE_ENV') !== 'production') return; @@ -29,6 +31,27 @@ module.exports.apiThrottle = function(app) { } module.exports.domainMiddleware = function(server,mongoose) { + if (nconf.get('NODE_ENV')=='production') { + var mins = 3; + + //// Detect memory leak as high memory usage on the system. Tried this on prod for a few days, seems we can get blacked out even *without* memory being maxed. + //setInterval(function(){ + // if (os.freemem() / os.totalmem() < 0.3) throw 'Memory leak'; + //}, mins*60*1000); + + setInterval(function(){ + // see https://docs.newrelic.com/docs/apm/apis/api-v2-examples/average-response-time-examples-api-v2, https://rpm.newrelic.com/api/explore/applications/data + request({ + //url: 'https://api.newrelic.com/v2/applications/APPLICATION_ID/metrics/data.json?names[]=HttpDispatcher&values[]=average_call_time&values[]=call_count&from=2014-10-14T2014-03-01T20:59:00+00:00:44:00+00:00&to=2014-10-14Tto=2014-03-01T21:59:00+00:00:14:00+00:00&summarize=true', + url: 'https://api.newrelic.com/v2/applications/'+nconf.get('NEW_RELIC_APPLICATION_ID')+'/metrics/data.json?names[]=HttpDispatcher&values[]=average_response_time&from='+moment().subtract({minutes:mins}).utc().format()+'&to='+moment().utc().format()+'&summarize=true', + headers: {'X-Api-Key': nconf.get('NEW_RELIC_API_KEY')} + }, function(err, response, body){ + var avgResponseTime = JSON.parse(body).metric_data.metrics[0].timeslices[0].values.average_response_time; + if (avgResponseTime > 650 || avgResponseTime == 0) throw 'Memory leak'; + }) + }, mins*60*1000); + } + return domainMiddleware({ server: { close:function(){