From 468b8b725bc10b1424f635c12e60f736f66cbd69 Mon Sep 17 00:00:00 2001 From: saint Date: Sun, 18 Aug 2024 15:56:12 +1000 Subject: [PATCH] v0.47 Significant tweaks and workarounds to try to improve smooth operation. --- gossamer_forums.rb | 75 +++++++++++++++++++++++++++++++--------------- 1 file changed, 51 insertions(+), 24 deletions(-) diff --git a/gossamer_forums.rb b/gossamer_forums.rb index a405a5e..021472f 100644 --- a/gossamer_forums.rb +++ b/gossamer_forums.rb @@ -1,7 +1,7 @@ # Federated Computer, Inc. # David Sainty 2024 A.D. # Gossamer Threads to Discourse -- Migration-Import Script -# v0.46 Restore use of ConnectionPool, tweak threads, mysql pool size, timeout, add full stacktraces +# v0.47 Significant tweaks and workarounds to try to improve smooth operation. require 'mysql2' require 'open-uri' @@ -1154,7 +1154,14 @@ class GossamerForumsImporter < ImportScripts::Base mariadb_pool.with do |mysql_client| begin + # Ensure the connection is active, otherwise reconnect puts "PP 11 -- #{post_id} -- Checking MySQL connections status.." + mysql_client.ping || mysql_client = Mysql2::Client.new( + host: "slowtwitch.northend.network", + username: "admin", + password: "yxnh93Ybbz2Nm8#mp28zCVv", + database: "slowtwitch" + ) #### puts " FIRST Checking MySQL connection status..." #### if mysql_client.query('SELECT 1').nil? #### puts " MySQL connection is not valid" @@ -1181,6 +1188,10 @@ class GossamerForumsImporter < ImportScripts::Base sqlite_mutex.synchronize do mark_post_as_failed(post_id) end + if e.message =~ /MySQL client is not connected/ + puts "Reconnecting to MySQL for post ID #{post_id} due to connection loss..." + retry + end #### ensure #### # Ensure the MariaDB connection is closed after processing #### mysql_client.close if mysql_client @@ -1235,7 +1246,13 @@ class GossamerForumsImporter < ImportScripts::Base ##### ) puts " FIRST Checking MySQL connection status..." if mysql_client.query('SELECT 1').nil? - puts " MySQL connection is not valid" + puts " MySQL connection is not valid, TRY TO RECONNECT II" + mysql_client.ping || mysql_client = Mysql2::Client.new( + host: "slowtwitch.northend.network", + username: "admin", + password: "yxnh93Ybbz2Nm8#mp28zCVv", + database: "slowtwitch" + ) else puts " MySQL connection is valid" end @@ -1300,14 +1317,17 @@ class GossamerForumsImporter < ImportScripts::Base else raise e end - puts e.backtrace.join("\n") # Print the full stack trace + # puts e.backtrace.join("\n") # Print the full stack trace end end - + + # Workaround... take a copy of topic.id + current_topic_id = topic.id + sqlite_mutex.synchronize do # Update the database with the last post time and user for the topic - update_db_topic_last_post_time(topic.id, Time.at(row['post_time']).to_i) - update_db_topic_last_post_user(topic.id, discourse_user_id) + update_db_topic_last_post_time(current_topic_id, Time.at(row['post_time']).to_i) + update_db_topic_last_post_user(current_topic_id, discourse_user_id) # Increment the topic count for the user update_db_user_topic_count(discourse_user_id, fetch_db_user_topic_count(discourse_user_id).to_i + 1) @@ -1316,20 +1336,20 @@ class GossamerForumsImporter < ImportScripts::Base # Sanitize and prepare the post message for Discourse sanitized_post_message = sanitize_post_message(row['post_message']) - puts "CREATE TOPIC POST topic.id #{topic.id} discourse_user_id #{discourse_user_id}" + puts "CREATE TOPIC POST for current_topic_id #{current_topic_id} discourse_user_id #{discourse_user_id}" post_number = 0 # Increment the post count for the topic - post_number = fetch_db_topic_post_numbers(topic.id).to_i + 1 + post_number = fetch_db_topic_post_numbers(current_topic_id).to_i + 1 sqlite_mutex.synchronize do - update_db_topic_post_numbers(topic.id, post_number) + update_db_topic_post_numbers(current_topic_id, post_number) end puts "TIJ GG post_id #{post_id}" # Create the initial post in the new topic post = Post.create!( - topic_id: topic.id, + topic_id: current_topic_id, user_id: discourse_user_id, raw: sanitized_post_message, created_at: Time.at(row['post_time']), @@ -1342,7 +1362,7 @@ class GossamerForumsImporter < ImportScripts::Base sqlite_mutex.synchronize do # Increment the post count for the topic and user - update_db_topic_post_count(topic.id, fetch_db_topic_post_count(topic.id).to_i + 1) + update_db_topic_post_count(current_topic_id, fetch_db_topic_post_count(current_topic_id).to_i + 1) update_db_user_post_count(discourse_user_id, fetch_db_user_post_count(discourse_user_id).to_i + 1) end @@ -1351,8 +1371,10 @@ class GossamerForumsImporter < ImportScripts::Base handle_post_attachments(row['post_id'], post, discourse_user_id, mysql_client) # Create URL mappings for the new topic - new_url = "https://new/t/#{topic.slug}/#{topic.id}" - insert_url_mapping(row['post_id'], new_url, unique_title) + new_url = "https://new/t/#{topic.slug}/#{current_topic_id}" + sqlite_mutex.synchronize do + insert_url_mapping(row['post_id'], new_url, unique_title) + end # Fetch and import all replies to this topic replies = execute_query_concurrent("SELECT post_id, user_id_fk, post_message, post_time FROM gforum_Post WHERE post_root_id = #{post_id} ORDER BY post_time ASC", mysql_client) @@ -1367,25 +1389,30 @@ class GossamerForumsImporter < ImportScripts::Base # Sanitize and prepare the reply message for Discourse sanitized_reply_message = sanitize_post_message(reply_row['post_message']) - puts "CREATE REPLY in topic_id #{topic.id}" + puts "CREATE REPLY in current_topic_id #{current_topic_id} for reply post_id #{reply_row['post_id']}" ### def get_topic_id ### return topic.id ### end # Increment the post count for the topic - post_number = fetch_db_topic_post_numbers(topic.id).to_i + 1 + post_number = fetch_db_topic_post_numbers(current_topic_id).to_i + 1 sqlite_mutex.synchronize do - update_db_topic_post_numbers(topic.id, post_number) + update_db_topic_post_numbers(current_topic_id, post_number) ### update_db_topic_post_numbers(get_topic_id, post_number) end # Fetch the number of views the post has had reply_post_views = fetch_post_views(reply_row['post_id']) - - puts "TIJ JJ post_id #{post_id}" + + # crazy sanity check + if topic.nil? + puts "ERROR: Topic is nil for reply post_id #{reply_row['post_id']}, attempting to BYPASS anyway" + end + puts "TIJ JJ post_id #{post_id} reply post_id #{reply_row['post_id']} reply_post_views #{reply_post_views || 0} post_number #{post_number} current_topic_id #{current_topic_id} reply_post_views #{reply_post_views || 0}" + # Create the reply post in the existing topic post = Post.create!( - topic_id: topic.id, + topic_id: current_topic_id, user_id: reply_user_id, raw: sanitized_reply_message, created_at: Time.at(reply_row['post_time']), @@ -1398,13 +1425,13 @@ class GossamerForumsImporter < ImportScripts::Base puts "TIJ KK post_id #{post_id}" # Increment the post count for the topic and user - update_db_topic_post_count(topic.id, fetch_db_topic_post_count(topic.id).to_i + 1) + update_db_topic_post_count(current_topic_id, fetch_db_topic_post_count(current_topic_id).to_i + 1) update_db_user_post_count(reply_user_id, fetch_db_user_post_count(reply_user_id).to_i + 1) # Update last post time and user for the topic - if fetch_db_topic_last_post_time(topic.id).nil? || Time.at(reply_row['post_time']).to_i > fetch_db_topic_last_post_time(topic.id).to_i - update_db_topic_last_post_time(topic.id, Time.at(reply_row['post_time']).to_i) - update_db_topic_last_post_user(topic.id, reply_user_id) + if fetch_db_topic_last_post_time(current_topic_id).nil? || Time.at(reply_row['post_time']).to_i > fetch_db_topic_last_post_time(current_topic_id).to_i + update_db_topic_last_post_time(current_topic_id, Time.at(reply_row['post_time']).to_i) + update_db_topic_last_post_user(current_topic_id, reply_user_id) end # Handle any attachments associated with the reply @@ -1433,7 +1460,7 @@ class GossamerForumsImporter < ImportScripts::Base end puts " LAST Removing MySQL connection" - mysql_client.close # if mysql_client +##### mysql_client.close # if mysql_client end