saint/discourse-import_scripts
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

v0.43 Move to connection pooling for MySQL problem

Browse Source
This commit is contained in:
David Sainty 2024-08-18 02:41:56 +10:00
parent 416dc9c1e0
commit 855a0a9e46
1 changed files with 51 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

88
gossamer_forums.rb
View File

@ -1,7 +1,7 @@
# Federated Computer, Inc. # Federated Computer, Inc.
# David Sainty <saint@federated.computer> 2024 A.D. # David Sainty <saint@federated.computer> 2024 A.D.
# Gossamer Threads to Discourse -- Migration-Import Script # Gossamer Threads to Discourse -- Migration-Import Script
# v0.42.1 Rewrite and simplication of concurrent-ruby support # v0.43 Move to connection pooling for MySQL problem
require 'mysql2' require 'mysql2'
require 'open-uri' require 'open-uri'
@ -1064,8 +1064,17 @@ class GossamerForumsImporter < ImportScripts::Base
pool_size pool_size
end end
# Get list of TOPICS / OP posts, i.e. post ids that have no parent / root id - SELECT post_id FROM gforum_Post WHERE post_root_id = 0;
def threaded_topic_import def threaded_topic_import
# Get list of TOPICS / OP posts, i.e. post ids that have no parent / root id - SELECT post_id FROM gforum_Post WHERE post_root_id = 0; # Define the connection pool inside the method
mariadb_pool = ConnectionPool.new(size: 10, timeout: 5) do
Mysql2::Client.new(
host: "slowtwitch.northend.network",
username: "admin",
password: "yxnh93Ybbz2Nm8#mp28zCVv",
database: "slowtwitch"
)
end
# The query selects post_ids from gforum_Post where post_root_id is 0, meaning these posts are the topic starters (OPs). # The query selects post_ids from gforum_Post where post_root_id is 0, meaning these posts are the topic starters (OPs).
# Execute the query and fetch the result # Execute the query and fetch the result
@ -1124,33 +1133,38 @@ class GossamerForumsImporter < ImportScripts::Base
# Submit the import job for the current post_id to the thread pool # Submit the import job for the current post_id to the thread pool
pool.post do pool.post do
puts "PP 11 -- #{post_id}"
# Initialise a new MariaDB / Mysql2 client inside of each thread # Initialise a new MariaDB / Mysql2 client inside of each thread
mysql_client = Mysql2::Client.new( #### mysql_client = Mysql2::Client.new(
host: "slowtwitch.northend.network", #### host: "slowtwitch.northend.network",
username: "admin", #### username: "admin",
password: "yxnh93Ybbz2Nm8#mp28zCVv", #### password: "yxnh93Ybbz2Nm8#mp28zCVv",
database: "slowtwitch" #### database: "slowtwitch"
) #### )
puts "PP 22 -- #{post_id}" #### puts "PP 22 -- #{post_id}"
puts " FIRST Checking MySQL connection status..."
if mysql_client.query('SELECT 1').nil?
puts " MySQL connection is not valid"
else
puts " MySQL connection is valid"
end
begin begin
# Use connection pooling for PostgreSQL and synchronize access to shared resources mariadb_pool.with do |mysql_client|
ActiveRecord::Base.connection_pool.with_connection do
post_status = post_status(post_id) puts "PP 11 -- #{post_id} -- Checking MySQL connections status.."
if post_status.nil? || post_status == 0 #### puts " FIRST Checking MySQL connection status..."
puts "Starting import for post_id #{post_id}" #### if mysql_client.query('SELECT 1').nil?
topic_import_job(post_id, mysql_client, sqlite_mutex) # Import topic and its replies #### puts " MySQL connection is not valid"
sqlite_mutex.synchronize do #### else
mark_post_as_complete(post_id) # Mark as complete in SQLite table #### puts " MySQL connection is valid"
#### end
# Use connection pooling for PostgreSQL and synchronize access to shared resources
ActiveRecord::Base.connection_pool.with_connection do
post_status = post_status(post_id)
if post_status.nil? || post_status == 0
puts "Starting import for post_id #{post_id}"
topic_import_job(post_id, mysql_client, sqlite_mutex) # Import topic and its replies
sqlite_mutex.synchronize do
mark_post_as_complete(post_id) # Mark as complete in SQLite table
end
else
puts "Skipping post_id #{post_id}, already processed."
end end
else
puts "Skipping post_id #{post_id}, already processed."
end end
end end
rescue => e rescue => e
@ -1158,17 +1172,17 @@ class GossamerForumsImporter < ImportScripts::Base
sqlite_mutex.synchronize do sqlite_mutex.synchronize do
mark_post_as_failed(post_id) mark_post_as_failed(post_id)
end end
ensure #### ensure
# Ensure the MariaDB connection is closed after processing #### # Ensure the MariaDB connection is closed after processing
mysql_client.close if mysql_client #### mysql_client.close if mysql_client
puts "** CLOSED MariaDB client" #### puts "** CLOSED MariaDB client"
puts "PP 22 -- #{post_id}" #### puts "PP 22 -- #{post_id}"
puts " FINAL Checking MySQL connection status..." #### puts " FINAL Checking MySQL connection status..."
if mysql_client.query('SELECT 1').nil? #### if mysql_client.query('SELECT 1').nil?
puts " MySQL connection is not valid" #### puts " MySQL connection is not valid"
else #### else
puts " MySQL connection is valid" #### puts " MySQL connection is valid"
end #### end
end end
end end
end end