From 6a07ef6fb77a9025d20dd6232d53c3dfd59296de Mon Sep 17 00:00:00 2001 From: Eric Richardson Date: Mon, 23 Mar 2015 10:47:03 -0400 Subject: [PATCH 1/4] Initial Prometheus source metrics --- src/streammachine/master/index.coffee | 13 ++++-- src/streammachine/master/prometheus.coffee | 49 ++++++++++++++++++++++ 2 files changed, 58 insertions(+), 4 deletions(-) create mode 100644 src/streammachine/master/prometheus.coffee diff --git a/src/streammachine/master/index.coffee b/src/streammachine/master/index.coffee index 0fd9ef81..25267af7 100644 --- a/src/streammachine/master/index.coffee +++ b/src/streammachine/master/index.coffee @@ -1,7 +1,7 @@ -_ = require "underscore" -temp = require "temp" -net = require "net" -fs = require "fs" +_ = require "underscore" +temp = require "temp" +net = require "net" +fs = require "fs" express = require "express" Throttle = require "throttle" @@ -14,6 +14,7 @@ Alerts = require "../alerts" Analytics = require "./analytics" Monitoring = require "./monitoring" SlaveIO = require "./master_io" +Prometheus = require "./prometheus" RewindDumpRestore = require "../rewind/dump_restore" @@ -105,6 +106,10 @@ module.exports = class Master extends require("events").EventEmitter @monitoring = new Monitoring @, @log.child(module:"monitoring") + # -- Prometheus metrics -- # + + @prometheus = new Prometheus @ + #---------- once_configured: (cb) -> diff --git a/src/streammachine/master/prometheus.coffee b/src/streammachine/master/prometheus.coffee new file mode 100644 index 00000000..5674b1fe --- /dev/null +++ b/src/streammachine/master/prometheus.coffee @@ -0,0 +1,49 @@ +Prometheus = require "prometheus-client" + +module.exports = class PrometheusMaster + constructor: (@master) -> + @client = new Prometheus namespace:"streammachine", subsystem:"master" + + # -- Register our Metrics -- # + + @connected_sources = @client.newGauge + name: "stream_sources" + help: "Number of sources connected for this stream." + + @client.register(@connected_sources) + + @source_latency = @client.newGauge + name: "stream_source_latency" + help: "How many milliseconds is the source chunk ts behind our clock time?" + + @client.register(@source_latency) + + # -- Set our source loop -- # + + @_sourceInt = setInterval => + _process = (stream,idx,source) => + latency = + if source.last_ts + Number(new Date()) - Number(source.last_ts) + else + -1 + + @source_latency.set stream:stream, index:idx, latency + + for k,stream of @master.streams + _process(stream.key,idx,source) for source,idx in stream.sources + @connected_sources.set stream:stream.key, stream.sources.length + + for k,sg of @master.stream_groups + _process(sg._stream.key,idx,source) for source,idx in sg._stream.sources + @connected_sources.set stream:sg._stream.key, sg._stream.sources.length + + , 1000 + + # -- Attach metrics to the API -- # + + # FIXME: This should probably happen in the API class... + console.log "Attaching prometheus routing for /metrics" + @master.api.app.get "/metrics", (req,res) => + console.log "METRICS REQUEST" + @client.metricsFunc req,res \ No newline at end of file From dc47626972d4c2400a6fc25302ff464ad270876d Mon Sep 17 00:00:00 2001 From: Eric Richardson Date: Mon, 23 Mar 2015 11:49:38 -0400 Subject: [PATCH 2/4] Prometheus: Add slave count metric --- src/streammachine/master/prometheus.coffee | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/streammachine/master/prometheus.coffee b/src/streammachine/master/prometheus.coffee index 5674b1fe..6ae5a306 100644 --- a/src/streammachine/master/prometheus.coffee +++ b/src/streammachine/master/prometheus.coffee @@ -10,13 +10,13 @@ module.exports = class PrometheusMaster name: "stream_sources" help: "Number of sources connected for this stream." - @client.register(@connected_sources) - @source_latency = @client.newGauge name: "stream_source_latency" help: "How many milliseconds is the source chunk ts behind our clock time?" - @client.register(@source_latency) + @connected_slaves = @client.newGauge + name: "slaves" + help: "Number of slaves that are connected." # -- Set our source loop -- # @@ -40,10 +40,11 @@ module.exports = class PrometheusMaster , 1000 + @_slaveInt = setInterval => + @connected_slaves.set {}, Object.keys(@master.slaves?.slaves||{}).length + , 1000 + # -- Attach metrics to the API -- # # FIXME: This should probably happen in the API class... - console.log "Attaching prometheus routing for /metrics" - @master.api.app.get "/metrics", (req,res) => - console.log "METRICS REQUEST" - @client.metricsFunc req,res \ No newline at end of file + @master.api.app.get "/metrics", @client.metricsFunc \ No newline at end of file From d5879596a30589ec1743c85cbd6599b7e45ce466 Mon Sep 17 00:00:00 2001 From: Eric Richardson Date: Mon, 23 Mar 2015 14:55:46 -0400 Subject: [PATCH 3/4] Tweak prometheus client API usage --- package.json | 3 ++- src/streammachine/master/prometheus.coffee | 22 ++++++++++++++-------- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/package.json b/package.json index 4b737dae..a856687d 100644 --- a/package.json +++ b/package.json @@ -44,7 +44,8 @@ "throttle": "~1.0.0", "ipc-rpc": "~0.1.2", "binary-search": "~1.2.0", - "compression": "~1.4.0" + "compression": "~1.4.0", + "prometheus-client": "~0.1.0" }, "devDependencies": { "webkit-devtools-agent": "~0.3.1", diff --git a/src/streammachine/master/prometheus.coffee b/src/streammachine/master/prometheus.coffee index 6ae5a306..732803d5 100644 --- a/src/streammachine/master/prometheus.coffee +++ b/src/streammachine/master/prometheus.coffee @@ -2,21 +2,27 @@ Prometheus = require "prometheus-client" module.exports = class PrometheusMaster constructor: (@master) -> - @client = new Prometheus namespace:"streammachine", subsystem:"master" + @client = new Prometheus() # -- Register our Metrics -- # @connected_sources = @client.newGauge - name: "stream_sources" - help: "Number of sources connected for this stream." + namespace: "streammachine", + subsystem: "master" + name: "stream_sources" + help: "Number of sources connected for this stream." @source_latency = @client.newGauge - name: "stream_source_latency" - help: "How many milliseconds is the source chunk ts behind our clock time?" + namespace: "streammachine", + subsystem: "master" + name: "stream_source_latency" + help: "How many milliseconds is the source chunk ts behind our clock time?" @connected_slaves = @client.newGauge - name: "slaves" - help: "Number of slaves that are connected." + namespace: "streammachine", + subsystem: "master" + name: "slaves" + help: "Number of slaves that are connected." # -- Set our source loop -- # @@ -47,4 +53,4 @@ module.exports = class PrometheusMaster # -- Attach metrics to the API -- # # FIXME: This should probably happen in the API class... - @master.api.app.get "/metrics", @client.metricsFunc \ No newline at end of file + @master.api.app.get "/metrics", @client.metricsFunc() \ No newline at end of file From 8b93a43072eb66ffd5fb08df1c6d10ec54786253 Mon Sep 17 00:00:00 2001 From: Eric Richardson Date: Thu, 4 Jun 2015 10:51:29 -0400 Subject: [PATCH 4/4] Tweak prometheus metrics to make it optional --- src/streammachine/index.coffee | 3 ++- src/streammachine/master/index.coffee | 3 ++- src/streammachine/master/prometheus.coffee | 5 +++-- src/streammachine/modes/master.coffee | 4 +++- 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/streammachine/index.coffee b/src/streammachine/index.coffee index cd40d2b9..34464748 100644 --- a/src/streammachine/index.coffee +++ b/src/streammachine/index.coffee @@ -27,6 +27,7 @@ module.exports = class StreamMachine behind_proxy: false + prometheus: true + admin: require_auth: false - diff --git a/src/streammachine/master/index.coffee b/src/streammachine/master/index.coffee index da9281af..c0eebdf4 100644 --- a/src/streammachine/master/index.coffee +++ b/src/streammachine/master/index.coffee @@ -108,7 +108,8 @@ module.exports = class Master extends require("events").EventEmitter # -- Prometheus metrics -- # - @prometheus = new Prometheus @ + if @options.prometheus + @prometheus = new Prometheus @ #---------- diff --git a/src/streammachine/master/prometheus.coffee b/src/streammachine/master/prometheus.coffee index 732803d5..d14559e0 100644 --- a/src/streammachine/master/prometheus.coffee +++ b/src/streammachine/master/prometheus.coffee @@ -1,4 +1,5 @@ Prometheus = require "prometheus-client" +express = require "express" module.exports = class PrometheusMaster constructor: (@master) -> @@ -52,5 +53,5 @@ module.exports = class PrometheusMaster # -- Attach metrics to the API -- # - # FIXME: This should probably happen in the API class... - @master.api.app.get "/metrics", @client.metricsFunc() \ No newline at end of file + @app = express() + @app.get "/", @client.metricsFunc() diff --git a/src/streammachine/modes/master.coffee b/src/streammachine/modes/master.coffee index 2af2e8fb..fe246c14 100644 --- a/src/streammachine/modes/master.coffee +++ b/src/streammachine/modes/master.coffee @@ -32,6 +32,8 @@ module.exports = class MasterMode extends require("./base") @server.use "/s", @master.transport.app @server.use "/api", @master.api.app + @server.use "/metrics", @master.prometheus.app if @master.prometheus + if process.send? @_rpc = new RPC process, functions: OK: (msg,handle,cb) -> @@ -143,4 +145,4 @@ module.exports = class MasterMode extends require("./base") cb null aFunc() - #---------- \ No newline at end of file + #----------