diff --git a/gossamer_forums.rb b/gossamer_forums.rb index ececd4a..6f4fc3a 100644 --- a/gossamer_forums.rb +++ b/gossamer_forums.rb @@ -1,7 +1,7 @@ # Federated Computer, Inc. # David Sainty 2024 A.D. # Gossamer Threads to Discourse -- Migration-Import Script -# v0.42.1 Rewrite and simplication of concurrent-ruby support +# v0.43 Move to connection pooling for MySQL problem require 'mysql2' require 'open-uri' @@ -1064,8 +1064,17 @@ class GossamerForumsImporter < ImportScripts::Base pool_size end + # Get list of TOPICS / OP posts, i.e. post ids that have no parent / root id - SELECT post_id FROM gforum_Post WHERE post_root_id = 0; def threaded_topic_import - # Get list of TOPICS / OP posts, i.e. post ids that have no parent / root id - SELECT post_id FROM gforum_Post WHERE post_root_id = 0; + # Define the connection pool inside the method + mariadb_pool = ConnectionPool.new(size: 10, timeout: 5) do + Mysql2::Client.new( + host: "slowtwitch.northend.network", + username: "admin", + password: "yxnh93Ybbz2Nm8#mp28zCVv", + database: "slowtwitch" + ) + end # The query selects post_ids from gforum_Post where post_root_id is 0, meaning these posts are the topic starters (OPs). # Execute the query and fetch the result @@ -1124,33 +1133,38 @@ class GossamerForumsImporter < ImportScripts::Base # Submit the import job for the current post_id to the thread pool pool.post do - puts "PP 11 -- #{post_id}" # Initialise a new MariaDB / Mysql2 client inside of each thread - mysql_client = Mysql2::Client.new( - host: "slowtwitch.northend.network", - username: "admin", - password: "yxnh93Ybbz2Nm8#mp28zCVv", - database: "slowtwitch" - ) - puts "PP 22 -- #{post_id}" - puts " FIRST Checking MySQL connection status..." - if mysql_client.query('SELECT 1').nil? - puts " MySQL connection is not valid" - else - puts " MySQL connection is valid" - end +#### mysql_client = Mysql2::Client.new( +#### host: "slowtwitch.northend.network", +#### username: "admin", +#### password: "yxnh93Ybbz2Nm8#mp28zCVv", +#### database: "slowtwitch" +#### ) +#### puts "PP 22 -- #{post_id}" + begin - # Use connection pooling for PostgreSQL and synchronize access to shared resources - ActiveRecord::Base.connection_pool.with_connection do - post_status = post_status(post_id) - if post_status.nil? || post_status == 0 - puts "Starting import for post_id #{post_id}" - topic_import_job(post_id, mysql_client, sqlite_mutex) # Import topic and its replies - sqlite_mutex.synchronize do - mark_post_as_complete(post_id) # Mark as complete in SQLite table + mariadb_pool.with do |mysql_client| + + puts "PP 11 -- #{post_id} -- Checking MySQL connections status.." +#### puts " FIRST Checking MySQL connection status..." +#### if mysql_client.query('SELECT 1').nil? +#### puts " MySQL connection is not valid" +#### else +#### puts " MySQL connection is valid" +#### end + + # Use connection pooling for PostgreSQL and synchronize access to shared resources + ActiveRecord::Base.connection_pool.with_connection do + post_status = post_status(post_id) + if post_status.nil? || post_status == 0 + puts "Starting import for post_id #{post_id}" + topic_import_job(post_id, mysql_client, sqlite_mutex) # Import topic and its replies + sqlite_mutex.synchronize do + mark_post_as_complete(post_id) # Mark as complete in SQLite table + end + else + puts "Skipping post_id #{post_id}, already processed." end - else - puts "Skipping post_id #{post_id}, already processed." end end rescue => e @@ -1158,17 +1172,17 @@ class GossamerForumsImporter < ImportScripts::Base sqlite_mutex.synchronize do mark_post_as_failed(post_id) end - ensure - # Ensure the MariaDB connection is closed after processing - mysql_client.close if mysql_client - puts "** CLOSED MariaDB client" - puts "PP 22 -- #{post_id}" - puts " FINAL Checking MySQL connection status..." - if mysql_client.query('SELECT 1').nil? - puts " MySQL connection is not valid" - else - puts " MySQL connection is valid" - end +#### ensure +#### # Ensure the MariaDB connection is closed after processing +#### mysql_client.close if mysql_client +#### puts "** CLOSED MariaDB client" +#### puts "PP 22 -- #{post_id}" +#### puts " FINAL Checking MySQL connection status..." +#### if mysql_client.query('SELECT 1').nil? +#### puts " MySQL connection is not valid" +#### else +#### puts " MySQL connection is valid" +#### end end end end