saint/discourse-import_scripts
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

v0.48.1 Further attempts to address MariaDB challenges and make things as foolproof as possible.

Browse Source
This commit is contained in:
David Sainty 2024-08-18 17:09:46 +10:00
parent 702d90cd34
commit 08f62e4288
1 changed files with 32 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

61
gossamer_forums.rb
View File

@ -1,7 +1,7 @@
# Federated Computer, Inc.
# David Sainty <saint@federated.computer> 2024 A.D.
# Gossamer Threads to Discourse -- Migration-Import Script
# v0.48 Further attempts to address MariaDB craziness.
# v0.48.1 Further attempts to address MariaDB challenges and make things as foolproof as possible.
require 'mysql2'
require 'open-uri'
@ -297,7 +297,7 @@ class GossamerForumsImporter < ImportScripts::Base
end
# Check if post_id exists and its status
def post_status(post_id)
def fetch_post_status(post_id)
result = @db.execute("SELECT status FROM topic_import_status WHERE post_id = ?", post_id).flatten.first
result.nil? ? nil : result.to_i
end
@ -1153,7 +1153,7 @@ class GossamerForumsImporter < ImportScripts::Base
#### post_id = parent_post_ids[current_post_batch] # Fetch the post_id for the current post
#### # Check if the post has already been processed or is incomplete
#### post_status = post_status(post_id)
#### post_status = fetch_post_status(post_id)
# Submit the import job for the current post_id to the thread pool
pool.post do
@ -1166,17 +1166,18 @@ class GossamerForumsImporter < ImportScripts::Base
#### )
#### puts "PP 22 -- #{post_id}"
mariadb_pool.with do |mysql_client|
begin
begin
mariadb_pool.with do |mysql_client|
# Ensure the connection is active, otherwise reconnect
puts "PP 11 -- #{post_id} -- Checking MySQL connections status.."
mysql_client.ping || mysql_client = Mysql2::Client.new(
host: "slowtwitch.northend.network",
username: "admin",
password: "yxnh93Ybbz2Nm8#mp28zCVv",
database: "slowtwitch"
)
mysql_client.ping
## || mysql_client = Mysql2::Client.new(
## host: "slowtwitch.northend.network",
## username: "admin",
## password: "yxnh93Ybbz2Nm8#mp28zCVv",
## database: "slowtwitch"
## )
#### puts " FIRST Checking MySQL connection status..."
#### if mysql_client.query('SELECT 1').nil?
#### puts " MySQL connection is not valid"
@ -1186,7 +1187,7 @@ class GossamerForumsImporter < ImportScripts::Base
# Use connection pooling for PostgreSQL and synchronize access to shared resources
ActiveRecord::Base.connection_pool.with_connection do
post_status = post_status(post_id)
post_status = fetch_post_status(post_id)
if post_status.nil? || post_status == 0
puts "Starting import for post_id #{post_id}"
topic_import_job(post_id, sqlite_mutex, mysql_client) # Import topic and its replies
@ -1197,16 +1198,18 @@ class GossamerForumsImporter < ImportScripts::Base
puts "Skipping post_id #{post_id}, already processed."
end
end
rescue => e
puts "Error processing post ID #{post_id}: #{e.message}"
puts e.backtrace.join("\n") # Print the full stack trace
sqlite_mutex.synchronize do
mark_post_as_failed(post_id)
end
if e.message =~ /MySQL client is not connected/
puts "Reconnecting to MySQL for post ID #{post_id} due to connection loss..."
retry
end
end
rescue => e
puts "Error processing post ID #{post_id}: #{e.message}"
puts e.backtrace.join("\n") # Print the full stack trace
sqlite_mutex.synchronize do
mark_post_as_failed(post_id)
end
if e.message =~ /MySQL client is not connected/ || e.message =~ /This connection is in use by/
sleep(1)
puts "Reconnecting to MySQL for post ID #{post_id} due to connection loss..."
retry
end
#### ensure
#### # Ensure the MariaDB connection is closed after processing
#### mysql_client.close if mysql_client
@ -1218,7 +1221,6 @@ class GossamerForumsImporter < ImportScripts::Base
#### else
#### puts " MySQL connection is valid"
#### end
end
end
end
end
@ -1262,12 +1264,13 @@ class GossamerForumsImporter < ImportScripts::Base
puts " FIRST Checking MySQL connection status..."
if mysql_client.query('SELECT 1').nil?
puts " MySQL connection is not valid, TRY TO RECONNECT II"
mysql_client.ping || mysql_client = Mysql2::Client.new(
host: "slowtwitch.northend.network",
username: "admin",
password: "yxnh93Ybbz2Nm8#mp28zCVv",
database: "slowtwitch"
)
mysql_client.ping
# || mysql_client = Mysql2::Client.new(
# host: "slowtwitch.northend.network",
# username: "admin",
# password: "yxnh93Ybbz2Nm8#mp28zCVv",
# database: "slowtwitch"
# )
else
puts " MySQL connection is valid"
end