v0.52 Improve handling and look at how we can improve ActiveRecord config for PostgreSQL and avoid insufficient pool size

gossamer_forums.rb

@@ -1090,7 +1090,7 @@ class GossamerForumsImporter < ImportScripts::Base
     # Use CachedThreadPool for dynamic thread management
 ####    pool = Concurrent::CachedThreadPool.new
 ######    pool = Concurrent::FixedThreadPool.new(7)
-    pool = Concurrent::FixedThreadPool.new(20)
+    pool = Concurrent::FixedThreadPool.new(7)
 
     # Define the connection pool inside the method
 ######    mariadb_pool = ConnectionPool.new(size: 14, timeout: 100) do
@@ -1157,7 +1157,8 @@ class GossamerForumsImporter < ImportScripts::Base
       # Submit the import job for the current post_id to the thread pool
         pool.post do
 ####    puts "PP 22 --  #{post_id}"
-
+          
+          retries = 0
           begin
 #######            mariadb_pool.with do |mysql_client|
 
@@ -1209,7 +1210,6 @@ class GossamerForumsImporter < ImportScripts::Base
             when /MySQL client is not connected/, /This connection is in use by/
               puts "Lost MySQL, retrying for post ID #{post_id}..."
               # Add reconnection attempt again here... if it proves necessary?
-              retries ||= 0
               retries += 1
               if retries < 5
                 sleep(1)
@@ -1219,7 +1219,6 @@ class GossamerForumsImporter < ImportScripts::Base
               end
             when /could not obtain a connection from the pool/
               puts "Connection pool exhausted, retrying for post ID #{post_id}..."
-              retries ||= 0
               retries += 1
               if retries < 5
                 sleep(1)
@@ -1326,13 +1325,13 @@ class GossamerForumsImporter < ImportScripts::Base
     # Check if the topic has already been imported using the custom field 'original_gossamer_id'
     unless TopicCustomField.exists?(name: 'original_gossamer_id', value: row['post_id'])
       puts "TIJ EE post_id #{post_id}"
-      # ActiveRecord::Base.transaction do
+      ActiveRecord::Base.transaction do
         # Create the new topic in Discourse
         begin
           suffix = 1
           topic_created = false
     
-     ActiveRecord::Base.transaction do
+##     ActiveRecord::Base.transaction do
           while !topic_created
             begin
               puts "TIJ FF post_id #{post_id}"
@@ -1361,7 +1360,7 @@ class GossamerForumsImporter < ImportScripts::Base
               # puts e.backtrace.join("\n")  # Print the full stack trace
             end
           end
-     end
+##     end
 
           # Workaround... take a copy of topic.id 
           current_topic_id = topic.id
@@ -1389,7 +1388,7 @@ class GossamerForumsImporter < ImportScripts::Base
 
           puts "TIJ GG post_id #{post_id}"
  
-     ActiveRecord::Base.transaction do
+##     ActiveRecord::Base.transaction do
             # Create the initial post in the new topic
             post = Post.create!(
               topic_id: current_topic_id,
@@ -1402,7 +1401,7 @@ class GossamerForumsImporter < ImportScripts::Base
             )
             post.custom_fields['original_gossamer_id'] = row['post_id']
             post.save!
-     end
+##     end
     
           sqlite_mutex.synchronize do
             # Increment the post count for the topic and user
@@ -1412,10 +1411,10 @@ class GossamerForumsImporter < ImportScripts::Base
   
           puts "TIJ HH post_id #{post_id}"
 
-     ActiveRecord::Base.transaction do
+##     ActiveRecord::Base.transaction do
           # Handle any attachments associated with the post
           handle_post_attachments(row['post_id'], post, discourse_user_id, mysql_client)
-     end
+##     end
     
           # Create URL mappings for the new topic
           new_url = "https://new/t/#{topic.slug}/#{current_topic_id}"
@@ -1464,7 +1463,7 @@ class GossamerForumsImporter < ImportScripts::Base
               end
               puts "TIJ JJ post_id #{post_id}  reply post_id #{reply_row['post_id']}  reply_post_views #{reply_post_views || 0}  post_number #{post_number}  current_topic_id #{current_topic_id}  reply_post_views #{reply_post_views || 0}"
                 
-     ActiveRecord::Base.transaction do
+##     ActiveRecord::Base.transaction do
               # Create the reply post in the existing topic
               post = Post.create!(
                 topic_id: current_topic_id,
@@ -1477,7 +1476,7 @@ class GossamerForumsImporter < ImportScripts::Base
               )
               post.custom_fields['original_gossamer_id'] = reply_row['post_id']
               post.save!
-     end
+##     end
 
               puts "TIJ KK post_id #{post_id}"
               # Increment the post count for the topic and user
@@ -1494,10 +1493,10 @@ class GossamerForumsImporter < ImportScripts::Base
                 end
               end
     
-     ActiveRecord::Base.transaction do
+##     ActiveRecord::Base.transaction do
               # Handle any attachments associated with the reply
               handle_post_attachments(reply_row['post_id'], post, reply_user_id, mysql_client)
-     end
+##     end
     
 #             # Update the highest processed post_id in the database (thread-safe)
 #             update_highest_processed_post_id_thread_safe(reply_row['post_id'])
@@ -1516,7 +1515,7 @@ class GossamerForumsImporter < ImportScripts::Base
           puts "---> ERROR importing topic with post_id #{row['post_id']}: #{e.message}"
           raise ActiveRecord::Rollback
         end
-#      end
+      end
     else
       puts "Topic for post_id #{row['post_id']} already exists, skipping creation."
     end