v0.52 Improve handling and look at how we can improve ActiveRecord config for PostgreSQL and avoid insufficient pool size

gossamer_forums.rb

@@ -1,7 +1,7 @@
 # Federated Computer, Inc.
 # David Sainty <saint@federated.computer>  2024 A.D.
 # Gossamer Threads to Discourse -- Migration-Import Script
-# v0.51 Fix mysql bug
+# v0.52 Improve handling and look at how we can improve ActiveRecord config for PostgreSQL and avoid insufficient pool size.
 
 require 'mysql2'
 require 'open-uri'
@@ -30,18 +30,6 @@ class GossamerForumsImporter < ImportScripts::Base
     super
       begin
 
-#        # Database configuration for ActiveRecord
-#        # This is not used, except for pool size... issue with our Bitnami Discourse?
-#        ActiveRecord::Base.establish_connection(
-#          adapter: 'postgresql',
-#          database: 'slowtwitch',
-#          username: 'admin',
-#          password: "yxnh93Ybbz2Nm8#mp28zCVv",
-#          host: 'slowtwitch.northend.network',
-#          pool: 20, # Adjust based on concurrency needs
-#          timeout: 5000
-#        )
-
         # Initialize MySQL client to connect to Gossamer Forums database
         @mysql_client = Mysql2::Client.new(
           host: "slowtwitch.northend.network",
@@ -1087,21 +1075,29 @@ class GossamerForumsImporter < ImportScripts::Base
   # Get list of TOPICS / OP posts, i.e. post ids that have no parent / root id - SELECT post_id FROM gforum_Post WHERE post_root_id = 0;
   def threaded_topic_import
 
+    # Define the custom connection pool settings
+    custom_pool = ActiveRecord::ConnectionAdapters::ConnectionPool.new(
+      ActiveRecord::Base.connection_pool.spec.to_h.merge(pool: 40, timeout: 5000)
+    )
+    
+    # Register the custom connection pool under a unique identifier
+    ActiveRecord::Base.connection_handler.connection_pools['CustomPool'] = custom_pool
+
     # Use CachedThreadPool for dynamic thread management
 ####    pool = Concurrent::CachedThreadPool.new
 ######    pool = Concurrent::FixedThreadPool.new(7)
-    pool = Concurrent::FixedThreadPool.new(7)
+    pool = Concurrent::FixedThreadPool.new(20)
 
     # Define the connection pool inside the method
 ######    mariadb_pool = ConnectionPool.new(size: 14, timeout: 100) do
-    mariadb_pool = ConnectionPool.new(size: 40, timeout: 100) do
+####    mariadb_pool = ConnectionPool.new(size: 40, timeout: 100) do
-      Mysql2::Client.new(
+####      Mysql2::Client.new(
-        host: "slowtwitch.northend.network",
+####        host: "slowtwitch.northend.network",
-        username: "admin",
+####        username: "admin",
-        password: "yxnh93Ybbz2Nm8#mp28zCVv",
+####        password: "yxnh93Ybbz2Nm8#mp28zCVv",
-        database: "slowtwitch"
+####        database: "slowtwitch"
-      )
+####      )
-    end
+####    end
 
     # The query selects post_ids from gforum_Post where post_root_id is 0, meaning these posts are the topic starters (OPs).
     # Execute the query and fetch the result
@@ -1184,7 +1180,8 @@ class GossamerForumsImporter < ImportScripts::Base
             end
 
               # Use connection pooling for PostgreSQL and synchronize access to shared resources
-              ActiveRecord::Base.connection_pool.with_connection do
+              # ActiveRecord::Base.connection_pool.with_connection do
+              ActiveRecord::Base.connected_to(pool: 'CustomPool') do
                 post_status = fetch_post_status(post_id)
                 if post_status.nil? || post_status == 0
                   puts "Starting import for post_id #{post_id}"
@@ -1203,10 +1200,28 @@ class GossamerForumsImporter < ImportScripts::Base
             sqlite_mutex.synchronize do
               mark_post_as_failed(post_id)
             end
-            if e.message =~ /MySQL client is not connected/ || e.message =~ /This connection is in use by/
+            case e.message
-              sleep(1)
+            when /MySQL client is not connected/, /This connection is in use by/
-              puts "Reconnecting to MySQL for post ID #{post_id} due to connection loss..."
+              puts "Lost MySQL, retrying for post ID #{post_id}..."
-              retry
+              # Add reconnection attempt again here... if it proves necessary?
+              retries ||= 0
+              retries += 1
+              if retries < 5
+                sleep(1)
+                retry
+              else
+                puts "Max retries reached for post ID #{post_id}"
+              end
+            when /could not obtain a connection from the pool/
+              puts "Connection pool exhausted, retrying for post ID #{post_id}..."
+              retries ||= 0
+              retries += 1
+              if retries < 5
+                sleep(1)
+                retry
+              else
+                puts "Max retries reached for post ID #{post_id}"
+              end
             end
           ensure
             # Ensure the MariaDB connection is closed after processing
@@ -1306,15 +1321,19 @@ class GossamerForumsImporter < ImportScripts::Base
     # Check if the topic has already been imported using the custom field 'original_gossamer_id'
     unless TopicCustomField.exists?(name: 'original_gossamer_id', value: row['post_id'])
       puts "TIJ EE post_id #{post_id}"
-      ActiveRecord::Base.transaction do
+      # ActiveRecord::Base.transaction do
+##      ActiveRecord::Base.connected_to(pool: 'CustomPool') do
+##        ActiveRecord::Base.transaction do
         # Create the new topic in Discourse
         begin
           suffix = 1
           topic_created = false
     
+     ActiveRecord::Base.connected_to(pool: 'CustomPool') do
+     ActiveRecord::Base.transaction do
           while !topic_created
             begin
-    puts "TIJ FF post_id #{post_id}"
+              puts "TIJ FF post_id #{post_id}"
               puts "CREATE TOPIC unique_title #{unique_title} title #{title} discourse_user_id #{discourse_user_id} category_id #{discourse_category_id}"
               topic = Topic.create!(
                 title: unique_title,
@@ -1340,6 +1359,8 @@ class GossamerForumsImporter < ImportScripts::Base
               # puts e.backtrace.join("\n")  # Print the full stack trace
             end
           end
+     end
+     end
 
           # Workaround... take a copy of topic.id 
           current_topic_id = topic.id
@@ -1365,8 +1386,10 @@ class GossamerForumsImporter < ImportScripts::Base
             update_db_topic_post_numbers(current_topic_id, post_number)
           end
 
-     puts "TIJ GG post_id #{post_id}"
+          puts "TIJ GG post_id #{post_id}"
  
+     ActiveRecord::Base.connected_to(pool: 'CustomPool') do
+     ActiveRecord::Base.transaction do
             # Create the initial post in the new topic
             post = Post.create!(
               topic_id: current_topic_id,
@@ -1379,6 +1402,8 @@ class GossamerForumsImporter < ImportScripts::Base
             )
             post.custom_fields['original_gossamer_id'] = row['post_id']
             post.save!
+     end
+     end
     
           sqlite_mutex.synchronize do
             # Increment the post count for the topic and user
@@ -1386,9 +1411,14 @@ class GossamerForumsImporter < ImportScripts::Base
             update_db_user_post_count(discourse_user_id, fetch_db_user_post_count(discourse_user_id).to_i + 1)
           end
   
-      puts "TIJ HH post_id #{post_id}"
+          puts "TIJ HH post_id #{post_id}"
+
+     ActiveRecord::Base.connected_to(pool: 'CustomPool') do
+     ActiveRecord::Base.transaction do
           # Handle any attachments associated with the post
           handle_post_attachments(row['post_id'], post, discourse_user_id, mysql_client)
+     end
+     end
     
           # Create URL mappings for the new topic
           new_url = "https://new/t/#{topic.slug}/#{current_topic_id}"
@@ -1412,7 +1442,7 @@ class GossamerForumsImporter < ImportScripts::Base
                 puts "reply_user_id is NOW Former_User id #{reply_user_id} for reply post_id #{reply_row['post_id']}"
               end
     
-      puts "TIJ II post_id #{post_id}"
+              puts "TIJ II post_id #{post_id}"
               # Sanitize and prepare the reply message for Discourse
               sanitized_reply_message = sanitize_post_message(reply_row['post_message'])
     
@@ -1437,6 +1467,8 @@ class GossamerForumsImporter < ImportScripts::Base
               end
               puts "TIJ JJ post_id #{post_id}  reply post_id #{reply_row['post_id']}  reply_post_views #{reply_post_views || 0}  post_number #{post_number}  current_topic_id #{current_topic_id}  reply_post_views #{reply_post_views || 0}"
                 
+     ActiveRecord::Base.connected_to(pool: 'CustomPool') do
+     ActiveRecord::Base.transaction do
               # Create the reply post in the existing topic
               post = Post.create!(
                 topic_id: current_topic_id,
@@ -1449,8 +1481,10 @@ class GossamerForumsImporter < ImportScripts::Base
               )
               post.custom_fields['original_gossamer_id'] = reply_row['post_id']
               post.save!
+     end
+     end
 
-      puts "TIJ KK post_id #{post_id}"
+              puts "TIJ KK post_id #{post_id}"
               # Increment the post count for the topic and user
               sqlite_mutex.synchronize do
                 update_db_topic_post_count(current_topic_id, fetch_db_topic_post_count(current_topic_id).to_i + 1)
@@ -1465,8 +1499,12 @@ class GossamerForumsImporter < ImportScripts::Base
                 end
               end
     
+     ActiveRecord::Base.connected_to(pool: 'CustomPool') do
+     ActiveRecord::Base.transaction do
               # Handle any attachments associated with the reply
               handle_post_attachments(reply_row['post_id'], post, reply_user_id, mysql_client)
+     end
+     end
     
 #             # Update the highest processed post_id in the database (thread-safe)
 #             update_highest_processed_post_id_thread_safe(reply_row['post_id'])
@@ -1485,7 +1523,8 @@ class GossamerForumsImporter < ImportScripts::Base
           puts "Error importing topic with post_id #{row['post_id']}: #{e.message}"
           raise ActiveRecord::Rollback
         end
-      end
+#      end
+#      end
     else
       puts "Topic for post_id #{row['post_id']} already exists, skipping creation."
     end