diff --git a/gossamer_forums.rb b/gossamer_forums.rb index b003326..6925aa2 100644 --- a/gossamer_forums.rb +++ b/gossamer_forums.rb @@ -1,7 +1,7 @@ # Federated Computer, Inc. # David Sainty 2024 A.D. # Gossamer Threads to Discourse -- Migration-Import Script -# v0.48 Further attempts to address MariaDB craziness. +# v0.48.1 Further attempts to address MariaDB challenges and make things as foolproof as possible. require 'mysql2' require 'open-uri' @@ -297,7 +297,7 @@ class GossamerForumsImporter < ImportScripts::Base end # Check if post_id exists and its status - def post_status(post_id) + def fetch_post_status(post_id) result = @db.execute("SELECT status FROM topic_import_status WHERE post_id = ?", post_id).flatten.first result.nil? ? nil : result.to_i end @@ -1153,7 +1153,7 @@ class GossamerForumsImporter < ImportScripts::Base #### post_id = parent_post_ids[current_post_batch] # Fetch the post_id for the current post #### # Check if the post has already been processed or is incomplete -#### post_status = post_status(post_id) +#### post_status = fetch_post_status(post_id) # Submit the import job for the current post_id to the thread pool pool.post do @@ -1166,17 +1166,18 @@ class GossamerForumsImporter < ImportScripts::Base #### ) #### puts "PP 22 -- #{post_id}" - mariadb_pool.with do |mysql_client| - begin + begin + mariadb_pool.with do |mysql_client| # Ensure the connection is active, otherwise reconnect puts "PP 11 -- #{post_id} -- Checking MySQL connections status.." - mysql_client.ping || mysql_client = Mysql2::Client.new( - host: "slowtwitch.northend.network", - username: "admin", - password: "yxnh93Ybbz2Nm8#mp28zCVv", - database: "slowtwitch" - ) + mysql_client.ping +## || mysql_client = Mysql2::Client.new( +## host: "slowtwitch.northend.network", +## username: "admin", +## password: "yxnh93Ybbz2Nm8#mp28zCVv", +## database: "slowtwitch" +## ) #### puts " FIRST Checking MySQL connection status..." #### if mysql_client.query('SELECT 1').nil? #### puts " MySQL connection is not valid" @@ -1186,7 +1187,7 @@ class GossamerForumsImporter < ImportScripts::Base # Use connection pooling for PostgreSQL and synchronize access to shared resources ActiveRecord::Base.connection_pool.with_connection do - post_status = post_status(post_id) + post_status = fetch_post_status(post_id) if post_status.nil? || post_status == 0 puts "Starting import for post_id #{post_id}" topic_import_job(post_id, sqlite_mutex, mysql_client) # Import topic and its replies @@ -1197,16 +1198,18 @@ class GossamerForumsImporter < ImportScripts::Base puts "Skipping post_id #{post_id}, already processed." end end - rescue => e - puts "Error processing post ID #{post_id}: #{e.message}" - puts e.backtrace.join("\n") # Print the full stack trace - sqlite_mutex.synchronize do - mark_post_as_failed(post_id) - end - if e.message =~ /MySQL client is not connected/ - puts "Reconnecting to MySQL for post ID #{post_id} due to connection loss..." - retry - end + end + rescue => e + puts "Error processing post ID #{post_id}: #{e.message}" + puts e.backtrace.join("\n") # Print the full stack trace + sqlite_mutex.synchronize do + mark_post_as_failed(post_id) + end + if e.message =~ /MySQL client is not connected/ || e.message =~ /This connection is in use by/ + sleep(1) + puts "Reconnecting to MySQL for post ID #{post_id} due to connection loss..." + retry + end #### ensure #### # Ensure the MariaDB connection is closed after processing #### mysql_client.close if mysql_client @@ -1218,7 +1221,6 @@ class GossamerForumsImporter < ImportScripts::Base #### else #### puts " MySQL connection is valid" #### end - end end end end @@ -1262,12 +1264,13 @@ class GossamerForumsImporter < ImportScripts::Base puts " FIRST Checking MySQL connection status..." if mysql_client.query('SELECT 1').nil? puts " MySQL connection is not valid, TRY TO RECONNECT II" - mysql_client.ping || mysql_client = Mysql2::Client.new( - host: "slowtwitch.northend.network", - username: "admin", - password: "yxnh93Ybbz2Nm8#mp28zCVv", - database: "slowtwitch" - ) + mysql_client.ping +# || mysql_client = Mysql2::Client.new( +# host: "slowtwitch.northend.network", +# username: "admin", +# password: "yxnh93Ybbz2Nm8#mp28zCVv", +# database: "slowtwitch" +# ) else puts " MySQL connection is valid" end