From 35b97db34105f2c05ed6ecbfc5a8f3ac7980fdb3 Mon Sep 17 00:00:00 2001 From: saint Date: Sat, 17 Aug 2024 04:33:57 +1000 Subject: [PATCH] v0.40 Move to per thread MySQL/MariaDB connection --- gossamer_forums.rb | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/gossamer_forums.rb b/gossamer_forums.rb index 8f93d9f..7ed483e 100644 --- a/gossamer_forums.rb +++ b/gossamer_forums.rb @@ -1,7 +1,7 @@ # Federated Computer, Inc. # David Sainty 2024 A.D. # Gossamer Threads to Discourse -- Migration-Import Script -# v0.39.7 Fixes/tweaks in calculate_dynamic_pool_size +# v0.40 Move to per thread MySQL/MariaDB connection require 'mysql2' require 'open-uri' @@ -1061,6 +1061,7 @@ class GossamerForumsImporter < ImportScripts::Base # The query selects post_ids from gforum_Post where post_root_id is 0, meaning these posts are the topic starters (OPs). # Execute the query and fetch the result + # result = execute_query("SELECT post_id FROM gforum_Post WHERE post_root_id = 0 ORDER BY post_id ASC") result = execute_query("SELECT post_id FROM gforum_Post WHERE post_root_id = 0 ORDER BY post_id ASC") # Convert the result set to an array of post_ids @@ -1094,13 +1095,22 @@ class GossamerForumsImporter < ImportScripts::Base # Submit the import job for the current post_id to the thread pool pool.post do + # Initialise a new MariaDB / Mysql2 client inside of each thread + mysql_client = Mysql2::Client.new( + host: "slowtwitch.northend.network", + user: "admin", + password: "yxnh93Ybbz2Nm8#mp28zCVv", + database: "slowtwitch" + ) begin puts "Processing post ID: #{post_id}" - topic_import_job(post_id) # Import topic and its replies + topic_import_job(post_idi, mysql_client) # Import topic and its replies mark_post_as_complete(post_id) # Mark as complete in SQLite table rescue => e puts "Error processing post ID #{post_id}: #{e.message}" mark_post_as_failed(post_id) + ensure + mysql_client.close if mysql_client end end else @@ -1130,7 +1140,7 @@ class GossamerForumsImporter < ImportScripts::Base # end # Method to import an entire topic, including its first post and all subsequent replies - def topic_import_job(post_id) + def topic_import_job(post_id, mysql_client) #Here is where you can import the entire topic #Get post -- SELECT post_id, user_id_fk, forum_id_fk, post_root_id, post_subject, post_time, post_message, post_father_id, post_replies FROM gforum_Post WHERE post_id = post_id #check if exists, create if not @@ -1138,7 +1148,7 @@ class GossamerForumsImporter < ImportScripts::Base #this parts needs to be synchronously to avoid race conditions # Fetch the post data for the given post_id (this is the first post in the topic) - row = execute_query("SELECT post_id, user_id_fk, forum_id_fk, post_root_id, post_subject, post_time, post_message, post_father_id, post_replies FROM gforum_Post WHERE post_id = #{post_id}").first + row = execute_query("SELECT post_id, user_id_fk, forum_id_fk, post_root_id, post_subject, post_time, post_message, post_father_id, post_replies FROM gforum_Post WHERE post_id = #{post_id}", mysql.client).first # Early return if the post data is not found return unless row