From af3c46e1cfc307ba730927607ae70c63c00ec0dc Mon Sep 17 00:00:00 2001 From: saint Date: Mon, 19 Aug 2024 21:33:37 +1000 Subject: [PATCH] v0.52 Improve handling and look at how we can improve ActiveRecord config for PostgreSQL and avoid insufficient pool size --- gossamer_forums.rb | 109 ++++++++++++++++++++++++++++++--------------- 1 file changed, 74 insertions(+), 35 deletions(-) diff --git a/gossamer_forums.rb b/gossamer_forums.rb index 632a4b9..b14936f 100644 --- a/gossamer_forums.rb +++ b/gossamer_forums.rb @@ -1,7 +1,7 @@ # Federated Computer, Inc. # David Sainty 2024 A.D. # Gossamer Threads to Discourse -- Migration-Import Script -# v0.51 Fix mysql bug +# v0.52 Improve handling and look at how we can improve ActiveRecord config for PostgreSQL and avoid insufficient pool size. require 'mysql2' require 'open-uri' @@ -30,18 +30,6 @@ class GossamerForumsImporter < ImportScripts::Base super begin -# # Database configuration for ActiveRecord -# # This is not used, except for pool size... issue with our Bitnami Discourse? -# ActiveRecord::Base.establish_connection( -# adapter: 'postgresql', -# database: 'slowtwitch', -# username: 'admin', -# password: "yxnh93Ybbz2Nm8#mp28zCVv", -# host: 'slowtwitch.northend.network', -# pool: 20, # Adjust based on concurrency needs -# timeout: 5000 -# ) - # Initialize MySQL client to connect to Gossamer Forums database @mysql_client = Mysql2::Client.new( host: "slowtwitch.northend.network", @@ -1087,21 +1075,29 @@ class GossamerForumsImporter < ImportScripts::Base # Get list of TOPICS / OP posts, i.e. post ids that have no parent / root id - SELECT post_id FROM gforum_Post WHERE post_root_id = 0; def threaded_topic_import + # Define the custom connection pool settings + custom_pool = ActiveRecord::ConnectionAdapters::ConnectionPool.new( + ActiveRecord::Base.connection_pool.spec.to_h.merge(pool: 40, timeout: 5000) + ) + + # Register the custom connection pool under a unique identifier + ActiveRecord::Base.connection_handler.connection_pools['CustomPool'] = custom_pool + # Use CachedThreadPool for dynamic thread management #### pool = Concurrent::CachedThreadPool.new ###### pool = Concurrent::FixedThreadPool.new(7) - pool = Concurrent::FixedThreadPool.new(7) + pool = Concurrent::FixedThreadPool.new(20) # Define the connection pool inside the method ###### mariadb_pool = ConnectionPool.new(size: 14, timeout: 100) do - mariadb_pool = ConnectionPool.new(size: 40, timeout: 100) do - Mysql2::Client.new( - host: "slowtwitch.northend.network", - username: "admin", - password: "yxnh93Ybbz2Nm8#mp28zCVv", - database: "slowtwitch" - ) - end +#### mariadb_pool = ConnectionPool.new(size: 40, timeout: 100) do +#### Mysql2::Client.new( +#### host: "slowtwitch.northend.network", +#### username: "admin", +#### password: "yxnh93Ybbz2Nm8#mp28zCVv", +#### database: "slowtwitch" +#### ) +#### end # The query selects post_ids from gforum_Post where post_root_id is 0, meaning these posts are the topic starters (OPs). # Execute the query and fetch the result @@ -1184,7 +1180,8 @@ class GossamerForumsImporter < ImportScripts::Base end # Use connection pooling for PostgreSQL and synchronize access to shared resources - ActiveRecord::Base.connection_pool.with_connection do + # ActiveRecord::Base.connection_pool.with_connection do + ActiveRecord::Base.connected_to(pool: 'CustomPool') do post_status = fetch_post_status(post_id) if post_status.nil? || post_status == 0 puts "Starting import for post_id #{post_id}" @@ -1203,10 +1200,28 @@ class GossamerForumsImporter < ImportScripts::Base sqlite_mutex.synchronize do mark_post_as_failed(post_id) end - if e.message =~ /MySQL client is not connected/ || e.message =~ /This connection is in use by/ - sleep(1) - puts "Reconnecting to MySQL for post ID #{post_id} due to connection loss..." - retry + case e.message + when /MySQL client is not connected/, /This connection is in use by/ + puts "Lost MySQL, retrying for post ID #{post_id}..." + # Add reconnection attempt again here... if it proves necessary? + retries ||= 0 + retries += 1 + if retries < 5 + sleep(1) + retry + else + puts "Max retries reached for post ID #{post_id}" + end + when /could not obtain a connection from the pool/ + puts "Connection pool exhausted, retrying for post ID #{post_id}..." + retries ||= 0 + retries += 1 + if retries < 5 + sleep(1) + retry + else + puts "Max retries reached for post ID #{post_id}" + end end ensure # Ensure the MariaDB connection is closed after processing @@ -1306,15 +1321,19 @@ class GossamerForumsImporter < ImportScripts::Base # Check if the topic has already been imported using the custom field 'original_gossamer_id' unless TopicCustomField.exists?(name: 'original_gossamer_id', value: row['post_id']) puts "TIJ EE post_id #{post_id}" - ActiveRecord::Base.transaction do + # ActiveRecord::Base.transaction do +## ActiveRecord::Base.connected_to(pool: 'CustomPool') do +## ActiveRecord::Base.transaction do # Create the new topic in Discourse begin suffix = 1 topic_created = false + ActiveRecord::Base.connected_to(pool: 'CustomPool') do + ActiveRecord::Base.transaction do while !topic_created begin - puts "TIJ FF post_id #{post_id}" + puts "TIJ FF post_id #{post_id}" puts "CREATE TOPIC unique_title #{unique_title} title #{title} discourse_user_id #{discourse_user_id} category_id #{discourse_category_id}" topic = Topic.create!( title: unique_title, @@ -1340,6 +1359,8 @@ class GossamerForumsImporter < ImportScripts::Base # puts e.backtrace.join("\n") # Print the full stack trace end end + end + end # Workaround... take a copy of topic.id current_topic_id = topic.id @@ -1365,8 +1386,10 @@ class GossamerForumsImporter < ImportScripts::Base update_db_topic_post_numbers(current_topic_id, post_number) end - puts "TIJ GG post_id #{post_id}" + puts "TIJ GG post_id #{post_id}" + ActiveRecord::Base.connected_to(pool: 'CustomPool') do + ActiveRecord::Base.transaction do # Create the initial post in the new topic post = Post.create!( topic_id: current_topic_id, @@ -1379,6 +1402,8 @@ class GossamerForumsImporter < ImportScripts::Base ) post.custom_fields['original_gossamer_id'] = row['post_id'] post.save! + end + end sqlite_mutex.synchronize do # Increment the post count for the topic and user @@ -1386,9 +1411,14 @@ class GossamerForumsImporter < ImportScripts::Base update_db_user_post_count(discourse_user_id, fetch_db_user_post_count(discourse_user_id).to_i + 1) end - puts "TIJ HH post_id #{post_id}" + puts "TIJ HH post_id #{post_id}" + + ActiveRecord::Base.connected_to(pool: 'CustomPool') do + ActiveRecord::Base.transaction do # Handle any attachments associated with the post handle_post_attachments(row['post_id'], post, discourse_user_id, mysql_client) + end + end # Create URL mappings for the new topic new_url = "https://new/t/#{topic.slug}/#{current_topic_id}" @@ -1412,7 +1442,7 @@ class GossamerForumsImporter < ImportScripts::Base puts "reply_user_id is NOW Former_User id #{reply_user_id} for reply post_id #{reply_row['post_id']}" end - puts "TIJ II post_id #{post_id}" + puts "TIJ II post_id #{post_id}" # Sanitize and prepare the reply message for Discourse sanitized_reply_message = sanitize_post_message(reply_row['post_message']) @@ -1437,6 +1467,8 @@ class GossamerForumsImporter < ImportScripts::Base end puts "TIJ JJ post_id #{post_id} reply post_id #{reply_row['post_id']} reply_post_views #{reply_post_views || 0} post_number #{post_number} current_topic_id #{current_topic_id} reply_post_views #{reply_post_views || 0}" + ActiveRecord::Base.connected_to(pool: 'CustomPool') do + ActiveRecord::Base.transaction do # Create the reply post in the existing topic post = Post.create!( topic_id: current_topic_id, @@ -1449,8 +1481,10 @@ class GossamerForumsImporter < ImportScripts::Base ) post.custom_fields['original_gossamer_id'] = reply_row['post_id'] post.save! - - puts "TIJ KK post_id #{post_id}" + end + end + + puts "TIJ KK post_id #{post_id}" # Increment the post count for the topic and user sqlite_mutex.synchronize do update_db_topic_post_count(current_topic_id, fetch_db_topic_post_count(current_topic_id).to_i + 1) @@ -1465,8 +1499,12 @@ class GossamerForumsImporter < ImportScripts::Base end end + ActiveRecord::Base.connected_to(pool: 'CustomPool') do + ActiveRecord::Base.transaction do # Handle any attachments associated with the reply handle_post_attachments(reply_row['post_id'], post, reply_user_id, mysql_client) + end + end # # Update the highest processed post_id in the database (thread-safe) # update_highest_processed_post_id_thread_safe(reply_row['post_id']) @@ -1485,7 +1523,8 @@ class GossamerForumsImporter < ImportScripts::Base puts "Error importing topic with post_id #{row['post_id']}: #{e.message}" raise ActiveRecord::Rollback end - end +# end +# end else puts "Topic for post_id #{row['post_id']} already exists, skipping creation." end