From c0c5e93ff8c219f118759a751e3c02aeec0c1884 Mon Sep 17 00:00:00 2001
From: Hai Hieu VU <haihieu.vu@enroute.mobi>
Date: Wed, 6 Dec 2023 08:09:37 +0100
Subject: [PATCH 01/10] CHOUETTE-3161 share referential builder for all import
 type

---
 app/models/concerns/local_import_support.rb | 45 +++++++++++++++++----
 app/models/import/netex_generic.rb          | 42 -------------------
 2 files changed, 38 insertions(+), 49 deletions(-)

diff --git a/app/models/concerns/local_import_support.rb b/app/models/concerns/local_import_support.rb
index 8366aa9aa5..c6188c2fab 100644
--- a/app/models/concerns/local_import_support.rb
+++ b/app/models/concerns/local_import_support.rb
@@ -77,15 +77,46 @@ def import_resources(*resources)
     end
   end
 
-  def create_referential
-    Chouette::Benchmark.measure 'create_referential' do
-      self.referential ||= Referential.new(
-        name: referential_name,
-        organisation_id: workbench.organisation_id,
-        workbench_id: workbench.id,
-        metadatas: [referential_metadata],
+  def referential_builder
+    ReferentialBuilder.new(workbench, name: referential_name, metadata: referential_metadata)
+  end
+
+  # Create a Referential with given name and medata
+  class ReferentialBuilder
+    def initialize(workbench, name:, metadata:)
+      @workbench = workbench
+      @name = name
+      @metadata = metadata
+    end
+    attr_reader :workbench, :name, :metadata
+
+    delegate :organisation, to: :workbench
+
+    def create(&block)
+      block.call referential if valid?
+    end
+
+    def referential
+      @referential ||= workbench.referentials.create(
+        name: name,
+        organisation: organisation,
+        metadatas: [metadata],
         ready: false
       )
+    end
+
+    def valid?
+      @valid ||= referential.valid?
+    end
+
+    def overlapping_referential_ids
+      @overlapping_referential_ids ||= referential.overlapped_referential_ids
+    end
+  end
+
+  def create_referential
+    Chouette::Benchmark.measure 'create_referential' do
+      self.referential ||= referential_builder.referential
 
       begin
         self.referential.save!
diff --git a/app/models/import/netex_generic.rb b/app/models/import/netex_generic.rb
index 1024cb7ac4..2e5cf42d81 100644
--- a/app/models/import/netex_generic.rb
+++ b/app/models/import/netex_generic.rb
@@ -91,10 +91,6 @@ def within_referential(&block)
     # self.overlapping_referential_ids = referential_builder.overlapping_referential_ids
   end
 
-  def referential_builder
-    @referential_builder ||= ReferentialBuilder.new(workbench, name: name, metadata: referential_metadata)
-  end
-
   def referential_metadata
     return unless [imported_line_ids, netex_source.validity_period].all?(&:present?)
 
@@ -102,44 +98,6 @@ def referential_metadata
       ReferentialMetadata.new line_ids: imported_line_ids, periodes: [netex_source.validity_period]
   end
 
-  # Create a Referential with given name and medata
-  class ReferentialBuilder
-    def initialize(workbench, name:, metadata:)
-      @workbench = workbench
-      @name = name
-      @metadata = metadata
-    end
-    attr_reader :workbench, :name, :metadata
-
-    delegate :organisation, to: :workbench
-
-    def create(&block)
-      if valid?
-        Rails.logger.debug "Create imported Referential: #{referential.inspect}"
-        block.call referential
-      else
-        Rails.logger.debug "Can't created imported Referential: #{referential.inspect}"
-      end
-    end
-
-    def referential
-      @referential ||= workbench.referentials.create(
-        name: name,
-        organisation: organisation,
-        metadatas: [metadata],
-        ready: false
-      )
-    end
-
-    def valid?
-      @valid ||= referential.valid?
-    end
-
-    def overlapping_referential_ids
-      @overlapping_referential_ids ||= referential.overlapped_referential_ids
-    end
-  end
-
   # TODO: why the resource statuses are not checked automaticaly ??
   # See CHOUETTE-2747
   def update_import_status

From 367227012d1b22b2e8f639b1ad19f0cd82903b43 Mon Sep 17 00:00:00 2001
From: Hai Hieu VU <haihieu.vu@enroute.mobi>
Date: Wed, 3 Jan 2024 10:13:57 +0100
Subject: [PATCH 02/10] CHOUETTE-3161 use build to for message creation

---
 app/models/concerns/local_import_support.rb | 7 ++++++-
 app/models/import/netex_generic.rb          | 2 +-
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/app/models/concerns/local_import_support.rb b/app/models/concerns/local_import_support.rb
index c6188c2fab..4a309ab25d 100644
--- a/app/models/concerns/local_import_support.rb
+++ b/app/models/concerns/local_import_support.rb
@@ -93,7 +93,12 @@ def initialize(workbench, name:, metadata:)
     delegate :organisation, to: :workbench
 
     def create(&block)
-      block.call referential if valid?
+      if valid?
+        Rails.logger.debug "Create imported Referential: #{referential.inspect}"
+        block.call referential
+      else
+        Rails.logger.debug "Can't created imported Referential: #{referential.inspect}"
+      end
     end
 
     def referential
diff --git a/app/models/import/netex_generic.rb b/app/models/import/netex_generic.rb
index 2e5cf42d81..fd3728608f 100644
--- a/app/models/import/netex_generic.rb
+++ b/app/models/import/netex_generic.rb
@@ -86,7 +86,7 @@ def within_referential(&block)
     return if referential_builder.valid?
 
     # Create a global error message
-    messages.create criticity: :error, message_key: 'referential_creation_overlapping_existing_referential'
+    messages.build criticity: :error, message_key: 'referential_creation_overlapping_existing_referential'
     # Save overlapping referentials for user display
     # self.overlapping_referential_ids = referential_builder.overlapping_referential_ids
   end

From 5c299a97c82691757e52e4aeafe47ee39f5781d0 Mon Sep 17 00:00:00 2001
From: Luc <luc.donnet@enroute.mobi>
Date: Tue, 2 Jan 2024 17:39:07 +0100
Subject: [PATCH 03/10] CHOUETTE-3161 Uncomment overlapping_referential_ids
 update in Import::NetexGeneric

---
 app/models/import/netex_generic.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/models/import/netex_generic.rb b/app/models/import/netex_generic.rb
index fd3728608f..74400084bd 100644
--- a/app/models/import/netex_generic.rb
+++ b/app/models/import/netex_generic.rb
@@ -88,7 +88,7 @@ def within_referential(&block)
     # Create a global error message
     messages.build criticity: :error, message_key: 'referential_creation_overlapping_existing_referential'
     # Save overlapping referentials for user display
-    # self.overlapping_referential_ids = referential_builder.overlapping_referential_ids
+    self.overlapping_referential_ids = referential_builder.overlapping_referential_ids
   end
 
   def referential_metadata

From a80f08a496594f241a61b08388bbd80fceb5789b Mon Sep 17 00:00:00 2001
From: Hai Hieu VU <haihieu.vu@enroute.mobi>
Date: Wed, 3 Jan 2024 16:22:47 +0100
Subject: [PATCH 04/10] CHOUETTE-3161 cache for referential_builder

---
 app/models/concerns/local_import_support.rb | 2 +-
 app/models/import/netex_generic.rb          | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/models/concerns/local_import_support.rb b/app/models/concerns/local_import_support.rb
index 4a309ab25d..5cfe9f5928 100644
--- a/app/models/concerns/local_import_support.rb
+++ b/app/models/concerns/local_import_support.rb
@@ -78,7 +78,7 @@ def import_resources(*resources)
   end
 
   def referential_builder
-    ReferentialBuilder.new(workbench, name: referential_name, metadata: referential_metadata)
+    @referential_builder ||= ReferentialBuilder.new(workbench, name: referential_name, metadata: referential_metadata)
   end
 
   # Create a Referential with given name and medata
diff --git a/app/models/import/netex_generic.rb b/app/models/import/netex_generic.rb
index 74400084bd..f820d9d18e 100644
--- a/app/models/import/netex_generic.rb
+++ b/app/models/import/netex_generic.rb
@@ -86,7 +86,7 @@ def within_referential(&block)
     return if referential_builder.valid?
 
     # Create a global error message
-    messages.build criticity: :error, message_key: 'referential_creation_overlapping_existing_referential'
+    messages.create criticity: :error, message_key: 'referential_creation_overlapping_existing_referential'
     # Save overlapping referentials for user display
     self.overlapping_referential_ids = referential_builder.overlapping_referential_ids
   end

From a80a1ee3c1802aa36f157aaf67e94b7b09abbdec Mon Sep 17 00:00:00 2001
From: Hai Hieu VU <haihieu.vu@enroute.mobi>
Date: Wed, 3 Jan 2024 17:19:14 +0100
Subject: [PATCH 05/10] CHOUETTE-3161 update overlapped referential ids for
 GTFS and Neptune import

---
 app/models/concerns/local_import_support.rb | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/app/models/concerns/local_import_support.rb b/app/models/concerns/local_import_support.rb
index 5cfe9f5928..8a8d251ea5 100644
--- a/app/models/concerns/local_import_support.rb
+++ b/app/models/concerns/local_import_support.rb
@@ -123,14 +123,13 @@ def create_referential
     Chouette::Benchmark.measure 'create_referential' do
       self.referential ||= referential_builder.referential
 
-      begin
-        self.referential.save!
-      rescue ActiveRecord::RecordInvalid
-        # No double capture for Chouette::Safe
-        Rails.logger.error "Unable to create referential: #{self.referential.errors.messages}"
-        raise
-      end
       main_resource.update referential: referential if main_resource
+
+      return referential if referential_builder.valid?
+
+      # Create a global error message
+      self.messages.create criticity: :error, message_key: 'referential_creation_overlapping_existing_referential'
+      self.overlapping_referential_ids = referential_builder.overlapping_referential_ids
     end
   end
 

From ae6b1b48a7a00a4ad7f74abbd077caff3c5f6d3d Mon Sep 17 00:00:00 2001
From: Hai Hieu VU <haihieu.vu@enroute.mobi>
Date: Thu, 11 Jan 2024 12:28:41 +0100
Subject: [PATCH 06/10] CHOUETTE-3316 fix Netex generique import fails without
 error message

---
 app/models/concerns/local_import_support.rb | 5 +----
 app/models/import/netex_generic.rb          | 9 ++++++---
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/app/models/concerns/local_import_support.rb b/app/models/concerns/local_import_support.rb
index 8a8d251ea5..c66eabb526 100644
--- a/app/models/concerns/local_import_support.rb
+++ b/app/models/concerns/local_import_support.rb
@@ -38,8 +38,7 @@ def import
     update status: 'failed', ended_at: Time.now
     Chouette::Safe.capture "#{self.class.name} ##{id} failed", e
 
-    if (referential && overlapped_referential_ids = referential.overlapped_referential_ids).present?
-      overlapped = Referential.find overlapped_referential_ids.last
+    if overlapping_referential_ids.present?
       create_message(
         criticity: :error,
         message_attributes: {
@@ -127,8 +126,6 @@ def create_referential
 
       return referential if referential_builder.valid?
 
-      # Create a global error message
-      self.messages.create criticity: :error, message_key: 'referential_creation_overlapping_existing_referential'
       self.overlapping_referential_ids = referential_builder.overlapping_referential_ids
     end
   end
diff --git a/app/models/import/netex_generic.rb b/app/models/import/netex_generic.rb
index f820d9d18e..07ae36998a 100644
--- a/app/models/import/netex_generic.rb
+++ b/app/models/import/netex_generic.rb
@@ -85,10 +85,13 @@ def within_referential(&block)
 
     return if referential_builder.valid?
 
-    # Create a global error message
-    messages.create criticity: :error, message_key: 'referential_creation_overlapping_existing_referential'
-    # Save overlapping referentials for user display
+    create_message(
+      criticity: :error,
+      message_key: 'referential_creation_overlapping_existing_referential'
+    )
     self.overlapping_referential_ids = referential_builder.overlapping_referential_ids
+
+    raise 'Referential creation overlapping existing referential'
   end
 
   def referential_metadata

From aa1e1cdec86c1cc1bf7897bdb90e428bf98cb97c Mon Sep 17 00:00:00 2001
From: Guillaume Perot <guillaume.perot@enroute.mobi>
Date: Fri, 1 Mar 2024 17:29:54 +0100
Subject: [PATCH 07/10] CHOUETTE-3161 - add new import error message
 (overlaping datasets) for local import support

---
 app/models/concerns/local_import_support.rb | 7 +------
 app/models/import/netex_generic.rb          | 2 +-
 config/locales/import_messages.en.yml       | 2 +-
 config/locales/import_messages.fr.yml       | 2 +-
 4 files changed, 4 insertions(+), 9 deletions(-)

diff --git a/app/models/concerns/local_import_support.rb b/app/models/concerns/local_import_support.rb
index c66eabb526..a01f809b44 100644
--- a/app/models/concerns/local_import_support.rb
+++ b/app/models/concerns/local_import_support.rb
@@ -41,12 +41,7 @@ def import
     if overlapping_referential_ids.present?
       create_message(
         criticity: :error,
-        message_attributes: {
-          referential_name: referential.name,
-          overlapped_name: overlapped.name,
-          overlapped_url: Rails.application.routes.url_helpers.workbench_referential_path(workbench, overlapped)
-        },
-        message_key: 'referential_creation_overlapping_existing_referential'
+        message_key: 'referential_creation_overlapping_existing_referential_block'
       )
     else
       create_message criticity: :error, message_key: :full_text, message_attributes: { text: e.message }
diff --git a/app/models/import/netex_generic.rb b/app/models/import/netex_generic.rb
index 07ae36998a..ce9f0e5409 100644
--- a/app/models/import/netex_generic.rb
+++ b/app/models/import/netex_generic.rb
@@ -87,7 +87,7 @@ def within_referential(&block)
 
     create_message(
       criticity: :error,
-      message_key: 'referential_creation_overlapping_existing_referential'
+      message_key: 'referential_creation_overlapping_existing_referential_block'
     )
     self.overlapping_referential_ids = referential_builder.overlapping_referential_ids
 
diff --git a/config/locales/import_messages.en.yml b/config/locales/import_messages.en.yml
index 193b833cc5..6aaa6de806 100644
--- a/config/locales/import_messages.en.yml
+++ b/config/locales/import_messages.en.yml
@@ -11,7 +11,7 @@ en:
     multiple_companies_in_file: "The file %{source_filename} contains multiple companies, imported lines have therefore not be associated."
     referential_creation_missing_lines_in_files: "Le référentiel %{referential_name} has not been created because the file contains no offer data"
     referential_creation_missing_lines: "The referential %{referential_name} has not been created because no line has been found in the organization"
-    # referential_creation_overlapping_existing_referential: "You are trying to import a file that overlaps DataSets in edition (see Overlapping section for details)."
+    referential_creation_overlapping_existing_referential_block: "You are trying to import a file that overlaps DataSets in edition (see Overlapping section for details)."
     referential_creation_overlapping_existing_referential: "The referential %{referential_name} has not been created because another referential already exists for the same periods and lines: <a href='%{overlapped_url}'>%{overlapped_name}</a>"
     referential_creation: "The referential %{referential_name} has not been created because another referential with the same lines and periods already exists"
     trip_with_inconsistent_stop_times: The trip %{trip_id} has inconsistent stop times
diff --git a/config/locales/import_messages.fr.yml b/config/locales/import_messages.fr.yml
index 7a5a08bed2..a76a19d603 100644
--- a/config/locales/import_messages.fr.yml
+++ b/config/locales/import_messages.fr.yml
@@ -11,7 +11,7 @@ fr:
     multiple_companies_in_file: "Le fichier %{source_filename} contient plusieurs transporteurs, les lignes importées n'ont donc pas étées associées."
     referential_creation_missing_lines_in_files: "Le référentiel %{referential_name} n'a pas pu être créé car aucune donnée d'offre n'est présente dans le fichier"
     referential_creation_missing_lines: "Le référentiel %{referential_name} n'a pas pu être créé car aucune ligne n'appartient à l'organisation"
-    # referential_creation_overlapping_existing_referential: "Vous essayez d’importer un fichier qui chevauche des Jeux de Données en édition (consulter la section Chevauchement)."
+    referential_creation_overlapping_existing_referential_block: "Vous essayez d’importer un fichier qui chevauche des Jeux de Données en édition (consulter la section Chevauchement)."
     referential_creation_overlapping_existing_referential: "Le référentiel %{referential_name} n'a pas pu être créé car un référentiel existe déjà sur les mêmes périodes et lignes: <a href='%{overlapped_url}'>%{overlapped_name}</a>"
     referential_creation: "Le référentiel %{referential_name} n'a pas pu être créé."
     trip_with_inconsistent_stop_times: La course %{trip_id} a des horaires incohérents

From 0d9c40f84f1b6d472c00c380988299c2022b2c11 Mon Sep 17 00:00:00 2001
From: Alban Peignier <alban.peignier@enroute.mobi>
Date: Wed, 15 May 2024 11:03:27 +0200
Subject: [PATCH 08/10] CHOUETTE-3161 Use a custom error to interrupt (legacy)
 imports when Referential can't be created

---
 app/models/concerns/local_import_support.rb | 28 ++++++++++++---------
 1 file changed, 16 insertions(+), 12 deletions(-)

diff --git a/app/models/concerns/local_import_support.rb b/app/models/concerns/local_import_support.rb
index a01f809b44..cd12728db0 100644
--- a/app/models/concerns/local_import_support.rb
+++ b/app/models/concerns/local_import_support.rb
@@ -21,7 +21,6 @@ def import_type
   def import
     Chouette::Benchmark.measure "import_#{import_type}", id: id do
       update status: 'running', started_at: Time.now
-      @progress = 0
 
       ActiveRecord::Base.cache do
         import_without_status
@@ -29,23 +28,20 @@ def import
 
       processor.after([referential])
 
-      @progress = nil
       @status ||= 'successful'
       referential&.active!
       update status: @status, ended_at: Time.now
     end
+  rescue InvalidReferential
+    update status: 'failed', ended_at: Time.now
+    if overlapping_referential_ids.present?
+      create_message criticity: :error, message_key: 'referential_creation_overlapping_existing_referential_block'
+    end
   rescue StandardError => e
     update status: 'failed', ended_at: Time.now
     Chouette::Safe.capture "#{self.class.name} ##{id} failed", e
 
-    if overlapping_referential_ids.present?
-      create_message(
-        criticity: :error,
-        message_key: 'referential_creation_overlapping_existing_referential_block'
-      )
-    else
-      create_message criticity: :error, message_key: :full_text, message_attributes: { text: e.message }
-    end
+    create_message criticity: :error, message_key: :full_text, message_attributes: { text: e.message }
     referential&.failed!
   ensure
     main_resource&.save
@@ -113,18 +109,26 @@ def overlapping_referential_ids
     end
   end
 
+  # DEPRECATED Use #within_referential
   def create_referential
     Chouette::Benchmark.measure 'create_referential' do
       self.referential ||= referential_builder.referential
 
       main_resource.update referential: referential if main_resource
 
-      return referential if referential_builder.valid?
+      unless referential_builder.valid?
+        self.overlapping_referential_ids = referential_builder.overlapping_referential_ids
+        raise InvalidReferential
+      end
 
-      self.overlapping_referential_ids = referential_builder.overlapping_referential_ids
+      referential
     end
   end
 
+  # DEPRECATED Use #within_referential
+  class InvalidReferential < StandardError
+  end
+
   def referential_name
     name.presence || File.basename(local_file.to_s)
   end

From aa58264bec5d6429aceb14ac2cf9ee8f640f2ca5 Mon Sep 17 00:00:00 2001
From: Alban Peignier <alban.peignier@enroute.mobi>
Date: Wed, 15 May 2024 11:05:14 +0200
Subject: [PATCH 09/10] CHOUETTE-3161 Capitalize overlapping tip

---
 config/locales/imports.en.yml | 2 +-
 config/locales/imports.fr.yml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/config/locales/imports.en.yml b/config/locales/imports.en.yml
index de33522f2a..0991291bbc 100644
--- a/config/locales/imports.en.yml
+++ b/config/locales/imports.en.yml
@@ -38,7 +38,7 @@ en:
       parent: Parent
       status: State
       overlapping: Overlapping
-      overlapping_tips: you can archive or delete these DataSets before restarting a new import
+      overlapping_tips: You can archive or delete these DataSets before restarting a new import
       netex:
         table_title: "Status of analyzed files"
         table_explanation: "When calendriers.xml and/or commun.xml are not imported, then all lines file are not processed."
diff --git a/config/locales/imports.fr.yml b/config/locales/imports.fr.yml
index c12a29ff69..78d6d56b70 100644
--- a/config/locales/imports.fr.yml
+++ b/config/locales/imports.fr.yml
@@ -40,7 +40,7 @@ fr:
       status: État
       referential: Jeu de données
       overlapping: Chevauchement
-      overlapping_tips: vous pouvez archiver ou supprimer ces Jeux de Données avant de relancer le même import
+      overlapping_tips: Vous pouvez archiver ou supprimer ces Jeux de Données avant de relancer le même import
       netex:
         table_title: "État des fichiers analysés"
         table_explanation: "Dans le cas ou le(s) fichiers calendriers.xml et/ou commun.xml sont dans un état non importé, alors tous les fichiers lignes sont automatiquement dans un état non traité."

From 8b5e19ba1e1f5977c38037e5abab9b158dabd279 Mon Sep 17 00:00:00 2001
From: Alban Peignier <alban.peignier@enroute.mobi>
Date: Wed, 15 May 2024 11:05:47 +0200
Subject: [PATCH 10/10] CHOUETTE-3161 Avoid exception when Referential can't be
 created in NeTEx import

---
 app/models/import/netex_generic.rb | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/app/models/import/netex_generic.rb b/app/models/import/netex_generic.rb
index ce9f0e5409..d461ef5434 100644
--- a/app/models/import/netex_generic.rb
+++ b/app/models/import/netex_generic.rb
@@ -85,13 +85,12 @@ def within_referential(&block)
 
     return if referential_builder.valid?
 
-    create_message(
+    # create_message has a strange behavior in this context
+    messages.build(
       criticity: :error,
       message_key: 'referential_creation_overlapping_existing_referential_block'
     )
     self.overlapping_referential_ids = referential_builder.overlapping_referential_ids
-
-    raise 'Referential creation overlapping existing referential'
   end
 
   def referential_metadata