Csv Import Status and Errors (#1102)

* add class Data to import for status/feedback

* add tests

* clean up

* standard fix

* use clearing naming and use with_index

* add upload limit concern

* update concern

app/controllers/concerns/attachment_manageable.rb

@@ -0,0 +1,8 @@
+module AttachmentManageable
+  def allow_only_one_attachment
+    return unless params[:files].is_a?(Array)
+
+    flash.now[:alert] = t(".upload_limit_exceed")
+    render turbo_stream: turbo_stream.replace("flash", partial: "layouts/shared/flash_messages")
+  end
+end

app/controllers/organizations/staff/external_form_upload_controller.rb

@@ -1,21 +1,25 @@
 module Organizations
   module Staff
     class ExternalFormUploadController < Organizations::BaseController
+      include AttachmentManageable
       layout "dashboard"
+      before_action :allow_only_one_attachment, only: [:create]
 
       def index
-        authorize! :external_form_upload,
-          context: {organization: Current.organization}
+        authorize! :external_form_upload, context: {organization: Current.organization}
       end
 
       def create
-        authorize! :external_form_upload,
-          context: {organization: Current.organization}
-        file = params[:files]
+        authorize! :external_form_upload, context: {organization: Current.organization}
 
         # Only processes single file upload
-        import_service = Organizations::Importers::GoogleCsvImportService.new(file)
-        import_service.call
+        import = Organizations::Importers::GoogleCsvImportService.new(params[:files]).call
+
+        if import.success?
+          # do something
+        else
+          # import.errors
+        end
       end
     end
   end

app/services/organizations/importers/google_csv_import_service.rb

@@ -3,41 +3,62 @@
 module Organizations
   module Importers
     class GoogleCsvImportService
+      Status = Data.define(:success?, :count, :no_match, :errors)
       def initialize(file)
         @file = file
         @organization = Current.organization
+        @count = 0
+        @no_match = []
+        @errors = []
       end
 
       def call
-        CSV.foreach(@file.to_path, headers: true, skip_blanks: true) do |row|
+        CSV.foreach(@file.to_path, headers: true, skip_blanks: true).with_index(1) do |row, index|
           # Using Google Form headers
           email = row["Email"].downcase
           csv_timestamp = Time.parse(row["Timestamp"])
 
           person = Person.find_by(email:, organization: @organization)
-          previous = FormSubmission.where(person:, csv_timestamp:)
-          next unless person && previous.empty?
+          previously_matched_form_submission = FormSubmission.where(person:, csv_timestamp:)
 
-          latest_form_submission = person.latest_form_submission
-
-          if latest_form_submission.form_answers.empty?
-            create_form_answers(latest_form_submission, row)
+          if person.nil?
+            @no_match << [index, email]
+          elsif previously_matched_form_submission.present?
+            next
           else
-            create_form_answers(FormSubmission.create!(person:, csv_timestamp:), row)
+            latest_form_submission = person.latest_form_submission
+            ActiveRecord::Base.transaction do
+              # This checks for the empty form submission that is added when a person is created
+              if latest_form_submission.csv_timestamp.nil? && latest_form_submission.form_answers.empty?
+                latest_form_submission.update!(csv_timestamp:)
+                create_form_answers(latest_form_submission, row)
+              else
+                # if the person submits a new/updated form,
+                # i.e. an additional row in the csv with the same email / different timestamp,
+                # a new form_submission will be created
+                create_form_answers(FormSubmission.create!(person:, csv_timestamp:), row)
+              end
+              @count += 1
+            end
           end
+        rescue => e
+          @errors << [index, e]
         end
+        Status.new(@errors.empty?, @count, @no_match, @errors)
       end
 
       private
 
       def create_form_answers(form_submission, row)
-        ActiveRecord::Base.transaction do
-          row.each do |col|
-            next if col[0] == "Email" || col[0] == "Timestamp"
+        row.each do |col|
+          next if col[0] == "Email" || col[0] == "Timestamp"
 
-            FormAnswer.create!(form_submission:,
-              question_snapshot: col[0], value: col[1])
-          end
+          answer = col[1].nil? ? "" : col[1]
+          FormAnswer.create!(
+            form_submission:,
+            question_snapshot: col[0],
+            value: answer
+          )
         end
       end
     end

config/locales/en.yml

@@ -609,6 +609,8 @@ en:
         update_activation:
           activated: "%{staff} was activated."
           deactivated: "%{staff} was deactivated."
+      external_form_upload:
+        upload_limit_exceed: "Upload limit exceeded: Only one file allowed." 
     adopter_fosterer:
       adopter_applications:
         create:

test/services/organizations/csv_import_service_test.rb

@@ -59,7 +59,7 @@ class CsvImportServiceTest < ActiveSupport::TestCase
         csv << @data
         csv << @data
       end
-      assert_difference "FormSubmission.count" do
+      assert_no_difference "FormSubmission.count" do
         Organizations::Importers::GoogleCsvImportService.new(@file).call
       end
     end
@@ -88,5 +88,27 @@ class CsvImportServiceTest < ActiveSupport::TestCase
         end
       end
     end
+
+    should "return summary of import when successful" do
+      CSV.open(@file.path, "ab") do |csv|
+        csv << @data
+      end
+      import = Organizations::Importers::GoogleCsvImportService.new(@file).call
+
+      assert import.success?
+      assert_equal 1, import.count
+      assert import.errors.empty?
+    end
+
+    should "return errors" do
+      @data[0] = "2024/13/27 10:24:05 AM AST"
+      CSV.open(@file.path, "ab") do |csv|
+        csv << @data
+      end
+      import = Organizations::Importers::GoogleCsvImportService.new(@file).call
+
+      refute import.success?
+      assert_equal "mon out of range", import.errors[0][1].message
+    end
   end
 end