[Backport to 7.x] Remove unnecessary sleep after exhausted retries (#218)

* Remove unnecessary sleep after exhausted retries

Also fixes up some issues with logging, where the batch size was calculated
incorrectly.

Author: robbavey

.travis.yml

@@ -7,9 +7,9 @@ matrix:
   - rvm: jruby-9.1.13.0
     env: LOGSTASH_BRANCH=master
   - rvm: jruby-9.1.13.0
-    env: LOGSTASH_BRANCH=6.x
+    env: LOGSTASH_BRANCH=6.7
   - rvm: jruby-9.1.13.0
-    env: LOGSTASH_BRANCH=6.0
+    env: LOGSTASH_BRANCH=7.0
   - rvm: jruby-1.7.27
     env: LOGSTASH_BRANCH=5.6
   fast_finish: true
@@ -18,3 +18,4 @@ before_script: ci/build.sh
 script: ci/run.sh
 after_script: ci/cleanup.sh
 jdk: oraclejdk8
+before_install: gem install bundler -v '< 2'

CHANGELOG.md

@@ -1,3 +1,6 @@
+## 7.3.2
+  - Fixed issue with unnecessary sleep after retries exhausted [#216](https://github.com/logstash-plugins/logstash-output-kafka/pull/216)
+
 ## 7.3.1
   - Added support for kafka property `ssl.endpoint.identification.algorithm` [#213](https://github.com/logstash-plugins/logstash-output-kafka/pull/213)
 

lib/logstash/outputs/kafka.rb

@@ -231,7 +231,7 @@ def multi_receive(events)
   end
 
   def retrying_send(batch)
-    remaining = @retries;
+    remaining = @retries
 
     while batch.any?
       if !remaining.nil?
@@ -281,13 +281,15 @@ def retrying_send(batch)
       break if failures.empty?
 
       # Otherwise, retry with any failed transmissions
-      batch = failures
-      delay = @retry_backoff_ms / 1000.0
-      logger.info("Sending batch to Kafka failed. Will retry after a delay.", :batch_size => batch.size,
-                  :failures => failures.size, :sleep => delay);
-      sleep(delay)
+      if remaining.nil? || remaining >= 0
+        delay = @retry_backoff_ms / 1000.0
+        logger.info("Sending batch to Kafka failed. Will retry after a delay.", :batch_size => batch.size,
+                                                                                :failures => failures.size,
+                                                                                :sleep => delay)
+        batch = failures
+        sleep(delay)
+      end
     end
-
   end
 
   def close

logstash-output-kafka.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-output-kafka'
-  s.version         = '7.3.1'
+  s.version         = '7.3.2'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Writes events to a Kafka topic"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

spec/unit/outputs/kafka_spec.rb

@@ -49,7 +49,7 @@
       kafka.register
       kafka.multi_receive([event])
     end
-    
+
     it 'should raise config error when truststore location is not set and ssl is enabled' do
       kafka = LogStash::Outputs::Kafka.new(simple_kafka_config.merge("security_protocol" => "SSL"))
       expect { kafka.register }.to raise_error(LogStash::ConfigurationError, /ssl_truststore_location must be set when SSL is enabled/)
@@ -120,6 +120,41 @@
       end
     end
 
+    context 'when retries is 0' do
+      let(:retries) { 0  }
+      let(:max_sends) { 1 }
+
+      it "should should only send once" do
+        expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send)
+                                                                                       .once
+                                                                                       .and_wrap_original do |m, *args|
+          # Always fail.
+          future = java.util.concurrent.FutureTask.new { raise "Failed" }
+          future.run
+          future
+        end
+        kafka = LogStash::Outputs::Kafka.new(simple_kafka_config.merge("retries" => retries))
+        kafka.register
+        kafka.multi_receive([event])
+      end
+
+      it 'should not sleep' do
+        expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send)
+                                                                                       .once
+                                                                                       .and_wrap_original do |m, *args|
+          # Always fail.
+          future = java.util.concurrent.FutureTask.new { raise "Failed" }
+          future.run
+          future
+        end
+
+        kafka = LogStash::Outputs::Kafka.new(simple_kafka_config.merge("retries" => retries))
+        expect(kafka).not_to receive(:sleep).with(anything)
+        kafka.register
+        kafka.multi_receive([event])
+      end
+    end
+
     context "and when retries is set by the user" do
       let(:retries) { (rand * 10).to_i }
       let(:max_sends) { retries + 1 }
@@ -137,6 +172,21 @@
         kafka.register
         kafka.multi_receive([event])
       end
+
+      it 'should only sleep retries number of times' do
+        expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send)
+                                                                                       .at_most(max_sends)
+                                                                                       .and_wrap_original do |m, *args|
+          # Always fail.
+          future = java.util.concurrent.FutureTask.new { raise "Failed" }
+          future.run
+          future
+        end
+        kafka = LogStash::Outputs::Kafka.new(simple_kafka_config.merge("retries" => retries))
+        expect(kafka).to receive(:sleep).exactly(retries).times
+        kafka.register
+        kafka.multi_receive([event])
+      end
     end
   end
 end