saint/discourse-import_scripts
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

v0.47 Significant tweaks and workarounds to try to improve smooth operation.

Browse Source
This commit is contained in:
David Sainty 2024-08-18 15:56:12 +10:00
parent ddf59a09a5
commit 468b8b725b
1 changed files with 51 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
gossamer_forums.rb
View File

@ -1,7 +1,7 @@
# Federated Computer, Inc. # Federated Computer, Inc.
# David Sainty <saint@federated.computer> 2024 A.D. # David Sainty <saint@federated.computer> 2024 A.D.
# Gossamer Threads to Discourse -- Migration-Import Script # Gossamer Threads to Discourse -- Migration-Import Script
# v0.46 Restore use of ConnectionPool, tweak threads, mysql pool size, timeout, add full stacktraces # v0.47 Significant tweaks and workarounds to try to improve smooth operation.
require 'mysql2' require 'mysql2'
require 'open-uri' require 'open-uri'
@ -1154,7 +1154,14 @@ class GossamerForumsImporter < ImportScripts::Base
mariadb_pool.with do |mysql_client| mariadb_pool.with do |mysql_client|
begin begin
# Ensure the connection is active, otherwise reconnect
puts "PP 11 -- #{post_id} -- Checking MySQL connections status.." puts "PP 11 -- #{post_id} -- Checking MySQL connections status.."
mysql_client.ping || mysql_client = Mysql2::Client.new(
host: "slowtwitch.northend.network",
username: "admin",
password: "yxnh93Ybbz2Nm8#mp28zCVv",
database: "slowtwitch"
)
#### puts " FIRST Checking MySQL connection status..." #### puts " FIRST Checking MySQL connection status..."
#### if mysql_client.query('SELECT 1').nil? #### if mysql_client.query('SELECT 1').nil?
#### puts " MySQL connection is not valid" #### puts " MySQL connection is not valid"
@ -1181,6 +1188,10 @@ class GossamerForumsImporter < ImportScripts::Base
sqlite_mutex.synchronize do sqlite_mutex.synchronize do
mark_post_as_failed(post_id) mark_post_as_failed(post_id)
end end
if e.message =~ /MySQL client is not connected/
puts "Reconnecting to MySQL for post ID #{post_id} due to connection loss..."
retry
end
#### ensure #### ensure
#### # Ensure the MariaDB connection is closed after processing #### # Ensure the MariaDB connection is closed after processing
#### mysql_client.close if mysql_client #### mysql_client.close if mysql_client
@ -1235,7 +1246,13 @@ class GossamerForumsImporter < ImportScripts::Base
##### ) ##### )
puts " FIRST Checking MySQL connection status..." puts " FIRST Checking MySQL connection status..."
if mysql_client.query('SELECT 1').nil? if mysql_client.query('SELECT 1').nil?
puts " MySQL connection is not valid" puts " MySQL connection is not valid, TRY TO RECONNECT II"
mysql_client.ping || mysql_client = Mysql2::Client.new(
host: "slowtwitch.northend.network",
username: "admin",
password: "yxnh93Ybbz2Nm8#mp28zCVv",
database: "slowtwitch"
)
else else
puts " MySQL connection is valid" puts " MySQL connection is valid"
end end
@ -1300,14 +1317,17 @@ class GossamerForumsImporter < ImportScripts::Base
else else
raise e raise e
end end
puts e.backtrace.join("\n") # Print the full stack trace # puts e.backtrace.join("\n") # Print the full stack trace
end end
end end
# Workaround... take a copy of topic.id
current_topic_id = topic.id
sqlite_mutex.synchronize do sqlite_mutex.synchronize do
# Update the database with the last post time and user for the topic # Update the database with the last post time and user for the topic
update_db_topic_last_post_time(topic.id, Time.at(row['post_time']).to_i) update_db_topic_last_post_time(current_topic_id, Time.at(row['post_time']).to_i)
update_db_topic_last_post_user(topic.id, discourse_user_id) update_db_topic_last_post_user(current_topic_id, discourse_user_id)
# Increment the topic count for the user # Increment the topic count for the user
update_db_user_topic_count(discourse_user_id, fetch_db_user_topic_count(discourse_user_id).to_i + 1) update_db_user_topic_count(discourse_user_id, fetch_db_user_topic_count(discourse_user_id).to_i + 1)
@ -1316,20 +1336,20 @@ class GossamerForumsImporter < ImportScripts::Base
# Sanitize and prepare the post message for Discourse # Sanitize and prepare the post message for Discourse
sanitized_post_message = sanitize_post_message(row['post_message']) sanitized_post_message = sanitize_post_message(row['post_message'])
puts "CREATE TOPIC POST topic.id #{topic.id} discourse_user_id #{discourse_user_id}" puts "CREATE TOPIC POST for current_topic_id #{current_topic_id} discourse_user_id #{discourse_user_id}"
post_number = 0 post_number = 0
# Increment the post count for the topic # Increment the post count for the topic
post_number = fetch_db_topic_post_numbers(topic.id).to_i + 1 post_number = fetch_db_topic_post_numbers(current_topic_id).to_i + 1
sqlite_mutex.synchronize do sqlite_mutex.synchronize do
update_db_topic_post_numbers(topic.id, post_number) update_db_topic_post_numbers(current_topic_id, post_number)
end end
puts "TIJ GG post_id #{post_id}" puts "TIJ GG post_id #{post_id}"
# Create the initial post in the new topic # Create the initial post in the new topic
post = Post.create!( post = Post.create!(
topic_id: topic.id, topic_id: current_topic_id,
user_id: discourse_user_id, user_id: discourse_user_id,
raw: sanitized_post_message, raw: sanitized_post_message,
created_at: Time.at(row['post_time']), created_at: Time.at(row['post_time']),
@ -1342,7 +1362,7 @@ class GossamerForumsImporter < ImportScripts::Base
sqlite_mutex.synchronize do sqlite_mutex.synchronize do
# Increment the post count for the topic and user # Increment the post count for the topic and user
update_db_topic_post_count(topic.id, fetch_db_topic_post_count(topic.id).to_i + 1) update_db_topic_post_count(current_topic_id, fetch_db_topic_post_count(current_topic_id).to_i + 1)
update_db_user_post_count(discourse_user_id, fetch_db_user_post_count(discourse_user_id).to_i + 1) update_db_user_post_count(discourse_user_id, fetch_db_user_post_count(discourse_user_id).to_i + 1)
end end
@ -1351,8 +1371,10 @@ class GossamerForumsImporter < ImportScripts::Base
handle_post_attachments(row['post_id'], post, discourse_user_id, mysql_client) handle_post_attachments(row['post_id'], post, discourse_user_id, mysql_client)
# Create URL mappings for the new topic # Create URL mappings for the new topic
new_url = "https://new/t/#{topic.slug}/#{topic.id}" new_url = "https://new/t/#{topic.slug}/#{current_topic_id}"
sqlite_mutex.synchronize do
insert_url_mapping(row['post_id'], new_url, unique_title) insert_url_mapping(row['post_id'], new_url, unique_title)
end
# Fetch and import all replies to this topic # Fetch and import all replies to this topic
replies = execute_query_concurrent("SELECT post_id, user_id_fk, post_message, post_time FROM gforum_Post WHERE post_root_id = #{post_id} ORDER BY post_time ASC", mysql_client) replies = execute_query_concurrent("SELECT post_id, user_id_fk, post_message, post_time FROM gforum_Post WHERE post_root_id = #{post_id} ORDER BY post_time ASC", mysql_client)
@ -1367,25 +1389,30 @@ class GossamerForumsImporter < ImportScripts::Base
# Sanitize and prepare the reply message for Discourse # Sanitize and prepare the reply message for Discourse
sanitized_reply_message = sanitize_post_message(reply_row['post_message']) sanitized_reply_message = sanitize_post_message(reply_row['post_message'])
puts "CREATE REPLY in topic_id #{topic.id}" puts "CREATE REPLY in current_topic_id #{current_topic_id} for reply post_id #{reply_row['post_id']}"
### def get_topic_id ### def get_topic_id
### return topic.id ### return topic.id
### end ### end
# Increment the post count for the topic # Increment the post count for the topic
post_number = fetch_db_topic_post_numbers(topic.id).to_i + 1 post_number = fetch_db_topic_post_numbers(current_topic_id).to_i + 1
sqlite_mutex.synchronize do sqlite_mutex.synchronize do
update_db_topic_post_numbers(topic.id, post_number) update_db_topic_post_numbers(current_topic_id, post_number)
### update_db_topic_post_numbers(get_topic_id, post_number) ### update_db_topic_post_numbers(get_topic_id, post_number)
end end
# Fetch the number of views the post has had # Fetch the number of views the post has had
reply_post_views = fetch_post_views(reply_row['post_id']) reply_post_views = fetch_post_views(reply_row['post_id'])
puts "TIJ JJ post_id #{post_id}" # crazy sanity check
if topic.nil?
puts "ERROR: Topic is nil for reply post_id #{reply_row['post_id']}, attempting to BYPASS anyway"
end
puts "TIJ JJ post_id #{post_id} reply post_id #{reply_row['post_id']} reply_post_views #{reply_post_views || 0} post_number #{post_number} current_topic_id #{current_topic_id} reply_post_views #{reply_post_views || 0}"
# Create the reply post in the existing topic # Create the reply post in the existing topic
post = Post.create!( post = Post.create!(
topic_id: topic.id, topic_id: current_topic_id,
user_id: reply_user_id, user_id: reply_user_id,
raw: sanitized_reply_message, raw: sanitized_reply_message,
created_at: Time.at(reply_row['post_time']), created_at: Time.at(reply_row['post_time']),
@ -1398,13 +1425,13 @@ class GossamerForumsImporter < ImportScripts::Base
puts "TIJ KK post_id #{post_id}" puts "TIJ KK post_id #{post_id}"
# Increment the post count for the topic and user # Increment the post count for the topic and user
update_db_topic_post_count(topic.id, fetch_db_topic_post_count(topic.id).to_i + 1) update_db_topic_post_count(current_topic_id, fetch_db_topic_post_count(current_topic_id).to_i + 1)
update_db_user_post_count(reply_user_id, fetch_db_user_post_count(reply_user_id).to_i + 1) update_db_user_post_count(reply_user_id, fetch_db_user_post_count(reply_user_id).to_i + 1)
# Update last post time and user for the topic # Update last post time and user for the topic
if fetch_db_topic_last_post_time(topic.id).nil? || Time.at(reply_row['post_time']).to_i > fetch_db_topic_last_post_time(topic.id).to_i if fetch_db_topic_last_post_time(current_topic_id).nil? || Time.at(reply_row['post_time']).to_i > fetch_db_topic_last_post_time(current_topic_id).to_i
update_db_topic_last_post_time(topic.id, Time.at(reply_row['post_time']).to_i) update_db_topic_last_post_time(current_topic_id, Time.at(reply_row['post_time']).to_i)
update_db_topic_last_post_user(topic.id, reply_user_id) update_db_topic_last_post_user(current_topic_id, reply_user_id)
end end
# Handle any attachments associated with the reply # Handle any attachments associated with the reply
@ -1433,7 +1460,7 @@ class GossamerForumsImporter < ImportScripts::Base
end end
puts " LAST Removing MySQL connection" puts " LAST Removing MySQL connection"
mysql_client.close # if mysql_client ##### mysql_client.close # if mysql_client
end end