Ensure RSpecFlaky doesn't automatically update flaky examples

Previously, instantiating a RspecFlaky::FlakyExample object would
automatically update its first_flaky_at, last_flaky_at and
last_flaky_job.

That was wrong because we would overwrite every time the suite report
with this false data.

We now:

- Get the suite report and only read from it
- Write only the currently detected flaky examples in the report, so
  that the final report is only updated with flaky examples that were
  actually detected in each job. Before, job1 could overwrite the legit
  report from job2!
- Write the newly detected flaky examples by rejecting the already
  tracked flaky specs instead of using another hash.

Signed-off-by: Rémy Coutable <remy@rymai.me>

2 files changed, 48 insertions, 38 deletions

diff --git a/lib/rspec_flaky/flaky_example.rb b/lib/rspec_flaky/flaky_example.rb
index f81fb90e870..6be24014d89 100644
--- a/lib/rspec_flaky/flaky_example.rb
+++ b/lib/rspec_flaky/flaky_example.rb
@@ -9,24 +9,21 @@ module RspecFlaky
           line: example.line,
           description: example.description,
           last_attempts_count: example.attempts,
-          flaky_reports: 1)
+          flaky_reports: 0)
       else
         super
       end
     end
 
-    def first_flaky_at
-      self[:first_flaky_at] || Time.now
-    end
-
-    def last_flaky_at
-      Time.now
-    end
+    def update_flakiness!(last_attempts_count: nil)
+      self.first_flaky_at ||= Time.now
+      self.last_flaky_at = Time.now
+      self.flaky_reports += 1
+      self.last_attempts_count = last_attempts_count if last_attempts_count
 
-    def last_flaky_job
-      return unless ENV['CI_PROJECT_URL'] && ENV['CI_JOB_ID']
-
-      "#{ENV['CI_PROJECT_URL']}/-/jobs/#{ENV['CI_JOB_ID']}"
+      if ENV['CI_PROJECT_URL'] && ENV['CI_JOB_ID']
+        self.last_flaky_job = "#{ENV['CI_PROJECT_URL']}/-/jobs/#{ENV['CI_JOB_ID']}"
+      end
     end
 
     def to_h
diff --git a/lib/rspec_flaky/listener.rb b/lib/rspec_flaky/listener.rb
index ec2fbd9e36c..4752ebe6410 100644
--- a/lib/rspec_flaky/listener.rb
+++ b/lib/rspec_flaky/listener.rb
@@ -2,11 +2,15 @@ require 'json'
 
 module RspecFlaky
   class Listener
-    attr_reader :all_flaky_examples, :new_flaky_examples
-
-    def initialize
-      @new_flaky_examples = {}
-      @all_flaky_examples = init_all_flaky_examples
+    # - suite_flaky_examples: contains all the currently tracked flacky example
+    #   for the whole RSpec suite
+    # - flaky_examples: contains the examples detected as flaky during the
+    #   current RSpec run
+    attr_reader :suite_flaky_examples, :flaky_examples
+
+    def initialize(suite_flaky_examples_json = nil)
+      @flaky_examples = {}
+      @suite_flaky_examples = init_suite_flaky_examples(suite_flaky_examples_json)
     end
 
     def example_passed(notification)
@@ -14,24 +18,16 @@ module RspecFlaky
 
       return unless current_example.attempts > 1
 
-      flaky_example_hash = all_flaky_examples[current_example.uid]
-
-      all_flaky_examples[current_example.uid] =
-        if flaky_example_hash
-          FlakyExample.new(flaky_example_hash).tap do |ex|
-            ex.last_attempts_count = current_example.attempts
-            ex.flaky_reports += 1
-          end
-        else
-          FlakyExample.new(current_example).tap do |ex|
-            new_flaky_examples[current_example.uid] = ex
-          end
-        end
+      flaky_example = suite_flaky_examples.fetch(current_example.uid) { FlakyExample.new(current_example) }
+      flaky_example.update_flakiness!(last_attempts_count: current_example.attempts)
+
+      flaky_examples[current_example.uid] = flaky_example
     end
 
     def dump_summary(_)
-      write_report_file(all_flaky_examples, all_flaky_examples_report_path)
+      write_report_file(flaky_examples, flaky_examples_report_path)
 
+      new_flaky_examples = _new_flaky_examples
       if new_flaky_examples.any?
         Rails.logger.warn "\nNew flaky examples detected:\n"
         Rails.logger.warn JSON.pretty_generate(to_report(new_flaky_examples))
@@ -46,12 +42,24 @@ module RspecFlaky
 
     private
 
-    def init_all_flaky_examples
-      return {} unless File.exist?(all_flaky_examples_report_path)
+    def init_suite_flaky_examples(suite_flaky_examples_json = nil)
+      unless suite_flaky_examples_json
+        return {} unless File.exist?(suite_flaky_examples_report_path)
+
+        suite_flaky_examples_json = File.read(suite_flaky_examples_report_path)
+      end
+
+      suite_flaky_examples = JSON.parse(suite_flaky_examples_json)
 
-      all_flaky_examples = JSON.parse(File.read(all_flaky_examples_report_path))
+      Hash[(suite_flaky_examples || {}).map { |k, ex| [k, FlakyExample.new(ex)] }].freeze
+    end
+
+    def _new_flaky_examples
+      flaky_examples.reject { |uid, _| already_flaky?(uid) }
+    end
 
-      Hash[(all_flaky_examples || {}).map { |k, ex| [k, FlakyExample.new(ex)] }]
+    def already_flaky?(example_uid)
+      suite_flaky_examples.key?(example_uid)
     end
 
     def write_report_file(examples, file_path)
@@ -62,9 +70,14 @@ module RspecFlaky
       File.write(file_path, JSON.pretty_generate(to_report(examples)))
     end
 
-    def all_flaky_examples_report_path
-      @all_flaky_examples_report_path ||= ENV['ALL_FLAKY_RSPEC_REPORT_PATH'] ||
-        Rails.root.join("rspec_flaky/all-report.json")
+    def suite_flaky_examples_report_path
+      @suite_flaky_examples_report_path ||= ENV['SUITE_FLAKY_RSPEC_REPORT_PATH'] ||
+        Rails.root.join("rspec_flaky/suite-report.json")
+    end
+
+    def flaky_examples_report_path
+      @flaky_examples_report_path ||= ENV['FLAKY_RSPEC_REPORT_PATH'] ||
+        Rails.root.join("rspec_flaky/report.json")
     end
 
     def new_flaky_examples_report_path