saint/discourse-import_scripts
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

v0.48.1 Further attempts to address MariaDB challenges and make things as foolproof as possible.

Browse Source
This commit is contained in:
David Sainty 2024-08-18 17:09:46 +10:00
parent 702d90cd34
commit 08f62e4288
1 changed files with 32 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
gossamer_forums.rb
View File

@ -1,7 +1,7 @@
# Federated Computer, Inc. # Federated Computer, Inc.
# David Sainty <saint@federated.computer> 2024 A.D. # David Sainty <saint@federated.computer> 2024 A.D.
# Gossamer Threads to Discourse -- Migration-Import Script # Gossamer Threads to Discourse -- Migration-Import Script
# v0.48 Further attempts to address MariaDB craziness. # v0.48.1 Further attempts to address MariaDB challenges and make things as foolproof as possible.
require 'mysql2' require 'mysql2'
require 'open-uri' require 'open-uri'
@ -297,7 +297,7 @@ class GossamerForumsImporter < ImportScripts::Base
end end
# Check if post_id exists and its status # Check if post_id exists and its status
def post_status(post_id) def fetch_post_status(post_id)
result = @db.execute("SELECT status FROM topic_import_status WHERE post_id = ?", post_id).flatten.first result = @db.execute("SELECT status FROM topic_import_status WHERE post_id = ?", post_id).flatten.first
result.nil? ? nil : result.to_i result.nil? ? nil : result.to_i
end end
@ -1153,7 +1153,7 @@ class GossamerForumsImporter < ImportScripts::Base
#### post_id = parent_post_ids[current_post_batch] # Fetch the post_id for the current post #### post_id = parent_post_ids[current_post_batch] # Fetch the post_id for the current post
#### # Check if the post has already been processed or is incomplete #### # Check if the post has already been processed or is incomplete
#### post_status = post_status(post_id) #### post_status = fetch_post_status(post_id)
# Submit the import job for the current post_id to the thread pool # Submit the import job for the current post_id to the thread pool
pool.post do pool.post do
@ -1166,17 +1166,18 @@ class GossamerForumsImporter < ImportScripts::Base
#### ) #### )
#### puts "PP 22 -- #{post_id}" #### puts "PP 22 -- #{post_id}"
mariadb_pool.with do |mysql_client|
begin begin
mariadb_pool.with do |mysql_client|
# Ensure the connection is active, otherwise reconnect # Ensure the connection is active, otherwise reconnect
puts "PP 11 -- #{post_id} -- Checking MySQL connections status.." puts "PP 11 -- #{post_id} -- Checking MySQL connections status.."
mysql_client.ping || mysql_client = Mysql2::Client.new( mysql_client.ping
host: "slowtwitch.northend.network", ## || mysql_client = Mysql2::Client.new(
username: "admin", ## host: "slowtwitch.northend.network",
password: "yxnh93Ybbz2Nm8#mp28zCVv", ## username: "admin",
database: "slowtwitch" ## password: "yxnh93Ybbz2Nm8#mp28zCVv",
) ## database: "slowtwitch"
## )
#### puts " FIRST Checking MySQL connection status..." #### puts " FIRST Checking MySQL connection status..."
#### if mysql_client.query('SELECT 1').nil? #### if mysql_client.query('SELECT 1').nil?
#### puts " MySQL connection is not valid" #### puts " MySQL connection is not valid"
@ -1186,7 +1187,7 @@ class GossamerForumsImporter < ImportScripts::Base
# Use connection pooling for PostgreSQL and synchronize access to shared resources # Use connection pooling for PostgreSQL and synchronize access to shared resources
ActiveRecord::Base.connection_pool.with_connection do ActiveRecord::Base.connection_pool.with_connection do
post_status = post_status(post_id) post_status = fetch_post_status(post_id)
if post_status.nil? || post_status == 0 if post_status.nil? || post_status == 0
puts "Starting import for post_id #{post_id}" puts "Starting import for post_id #{post_id}"
topic_import_job(post_id, sqlite_mutex, mysql_client) # Import topic and its replies topic_import_job(post_id, sqlite_mutex, mysql_client) # Import topic and its replies
@ -1197,13 +1198,15 @@ class GossamerForumsImporter < ImportScripts::Base
puts "Skipping post_id #{post_id}, already processed." puts "Skipping post_id #{post_id}, already processed."
end end
end end
end
rescue => e rescue => e
puts "Error processing post ID #{post_id}: #{e.message}" puts "Error processing post ID #{post_id}: #{e.message}"
puts e.backtrace.join("\n") # Print the full stack trace puts e.backtrace.join("\n") # Print the full stack trace
sqlite_mutex.synchronize do sqlite_mutex.synchronize do
mark_post_as_failed(post_id) mark_post_as_failed(post_id)
end end
if e.message =~ /MySQL client is not connected/ if e.message =~ /MySQL client is not connected/ || e.message =~ /This connection is in use by/
sleep(1)
puts "Reconnecting to MySQL for post ID #{post_id} due to connection loss..." puts "Reconnecting to MySQL for post ID #{post_id} due to connection loss..."
retry retry
end end
@ -1221,7 +1224,6 @@ class GossamerForumsImporter < ImportScripts::Base
end end
end end
end end
end
#### current_post_batch += 1 # Increment, moving to next post in the batch #### current_post_batch += 1 # Increment, moving to next post in the batch
#### break if current_post_batch >= parent_post_count #### break if current_post_batch >= parent_post_count
@ -1262,12 +1264,13 @@ class GossamerForumsImporter < ImportScripts::Base
puts " FIRST Checking MySQL connection status..." puts " FIRST Checking MySQL connection status..."
if mysql_client.query('SELECT 1').nil? if mysql_client.query('SELECT 1').nil?
puts " MySQL connection is not valid, TRY TO RECONNECT II" puts " MySQL connection is not valid, TRY TO RECONNECT II"
mysql_client.ping || mysql_client = Mysql2::Client.new( mysql_client.ping
host: "slowtwitch.northend.network", # || mysql_client = Mysql2::Client.new(
username: "admin", # host: "slowtwitch.northend.network",
password: "yxnh93Ybbz2Nm8#mp28zCVv", # username: "admin",
database: "slowtwitch" # password: "yxnh93Ybbz2Nm8#mp28zCVv",
) # database: "slowtwitch"
# )
else else
puts " MySQL connection is valid" puts " MySQL connection is valid"
end end