From 9ff12cc02588cc2e5ad280dd7dfd31a821342bb8 Mon Sep 17 00:00:00 2001 From: Steve Ford Date: Thu, 17 Dec 2015 17:19:13 +0000 Subject: [PATCH 1/2] The addition of resend_attempts This prevents metrics being spooled indefinately and causing logstash to crash when grahite becomes unavailable. --- lib/logstash/outputs/graphite.rb | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/lib/logstash/outputs/graphite.rb b/lib/logstash/outputs/graphite.rb index bad2cb3..b6cce14 100644 --- a/lib/logstash/outputs/graphite.rb +++ b/lib/logstash/outputs/graphite.rb @@ -26,6 +26,9 @@ class LogStash::Outputs::Graphite < LogStash::Outputs::Base # Interval between reconnect attempts to Carbon. config :reconnect_interval, :validate => :number, :default => 2 + # Number of attempts to be made resending metrics before abandoning + config :resend_attempts, :validate => :number, :default => 3 + # Should metrics be resent on failure? config :resend_on_failure, :validate => :boolean, :default => false @@ -94,12 +97,14 @@ def register def connect # TODO(sissel): Test error cases. Catch exceptions. Find fortune and glory. Retire to yak farm. + numattempts = 0 begin + numattempts += 1 @socket = TCPSocket.new(@host, @port) rescue Errno::ECONNREFUSED => e @logger.warn("Connection refused to graphite server, sleeping...", :host => @host, :port => @port) sleep(@reconnect_interval) - retry + retry if numattempts < @resend_attempts end end @@ -130,13 +135,15 @@ def receive(event) # Catch exceptions like ECONNRESET and friends, reconnect on failure. # TODO(sissel): Test error cases. Catch exceptions. Find fortune and glory. + numattempts = 0 begin + numattempts += 1 @socket.puts(message) rescue Errno::EPIPE, Errno::ECONNRESET, IOError => e @logger.warn("Connection to graphite server died", :exception => e, :host => @host, :port => @port) sleep(@reconnect_interval) connect - retry if @resend_on_failure + retry if @resend_on_failure || numattempts < @resend_attempts end end end From 67a2d5a59c97b2f054611e2a3ad72ab9b4720fb3 Mon Sep 17 00:00:00 2001 From: Steve Ford Date: Fri, 18 Dec 2015 11:29:02 +0000 Subject: [PATCH 2/2] Bumping version number to capture resend_attempts addition --- CHANGELOG.md | 5 ++++- logstash-output-graphite.gemspec | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d437646..698e927 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 2.1.0 + - Preventing output blocking when the graphite server is down by introducing a resend_attempts counter. + ## 2.0.3 - Fixed empty/nil messages handling @@ -10,4 +13,4 @@ - Added support for sprintf in field formatting ## 1.0.1 - - Added support for nested hashes as values \ No newline at end of file + - Added support for nested hashes as values diff --git a/logstash-output-graphite.gemspec b/logstash-output-graphite.gemspec index c264592..1d5d8fa 100644 --- a/logstash-output-graphite.gemspec +++ b/logstash-output-graphite.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-output-graphite' - s.version = '2.0.3' + s.version = '2.1.0' s.licenses = ['Apache License (2.0)'] s.summary = "This output allows you to pull metrics from your logs and ship them to Graphite" s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"