saint/discourse-import_scripts
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

v0.40 Move to per thread MySQL/MariaDB connection

Browse Source
This commit is contained in:
David Sainty 2024-08-17 04:33:57 +10:00
parent c11055d0f4
commit 35b97db341
1 changed files with 14 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
gossamer_forums.rb
View File

@ -1,7 +1,7 @@
# Federated Computer, Inc. # Federated Computer, Inc.
# David Sainty <saint@federated.computer> 2024 A.D. # David Sainty <saint@federated.computer> 2024 A.D.
# Gossamer Threads to Discourse -- Migration-Import Script # Gossamer Threads to Discourse -- Migration-Import Script
# v0.39.7 Fixes/tweaks in calculate_dynamic_pool_size # v0.40 Move to per thread MySQL/MariaDB connection
require 'mysql2' require 'mysql2'
require 'open-uri' require 'open-uri'
@ -1061,6 +1061,7 @@ class GossamerForumsImporter < ImportScripts::Base
# The query selects post_ids from gforum_Post where post_root_id is 0, meaning these posts are the topic starters (OPs). # The query selects post_ids from gforum_Post where post_root_id is 0, meaning these posts are the topic starters (OPs).
# Execute the query and fetch the result # Execute the query and fetch the result
# result = execute_query("SELECT post_id FROM gforum_Post WHERE post_root_id = 0 ORDER BY post_id ASC")
result = execute_query("SELECT post_id FROM gforum_Post WHERE post_root_id = 0 ORDER BY post_id ASC") result = execute_query("SELECT post_id FROM gforum_Post WHERE post_root_id = 0 ORDER BY post_id ASC")
# Convert the result set to an array of post_ids # Convert the result set to an array of post_ids
@ -1094,13 +1095,22 @@ class GossamerForumsImporter < ImportScripts::Base
# Submit the import job for the current post_id to the thread pool # Submit the import job for the current post_id to the thread pool
pool.post do pool.post do
# Initialise a new MariaDB / Mysql2 client inside of each thread
mysql_client = Mysql2::Client.new(
host: "slowtwitch.northend.network",
user: "admin",
password: "yxnh93Ybbz2Nm8#mp28zCVv",
database: "slowtwitch"
)
begin begin
puts "Processing post ID: #{post_id}" puts "Processing post ID: #{post_id}"
topic_import_job(post_id) # Import topic and its replies topic_import_job(post_idi, mysql_client) # Import topic and its replies
mark_post_as_complete(post_id) # Mark as complete in SQLite table mark_post_as_complete(post_id) # Mark as complete in SQLite table
rescue => e rescue => e
puts "Error processing post ID #{post_id}: #{e.message}" puts "Error processing post ID #{post_id}: #{e.message}"
mark_post_as_failed(post_id) mark_post_as_failed(post_id)
ensure
mysql_client.close if mysql_client
end end
end end
else else
@ -1130,7 +1140,7 @@ class GossamerForumsImporter < ImportScripts::Base
# end # end
# Method to import an entire topic, including its first post and all subsequent replies # Method to import an entire topic, including its first post and all subsequent replies
def topic_import_job(post_id) def topic_import_job(post_id, mysql_client)
#Here is where you can import the entire topic #Here is where you can import the entire topic
#Get post -- SELECT post_id, user_id_fk, forum_id_fk, post_root_id, post_subject, post_time, post_message, post_father_id, post_replies FROM gforum_Post WHERE post_id = post_id #Get post -- SELECT post_id, user_id_fk, forum_id_fk, post_root_id, post_subject, post_time, post_message, post_father_id, post_replies FROM gforum_Post WHERE post_id = post_id
#check if exists, create if not #check if exists, create if not
@ -1138,7 +1148,7 @@ class GossamerForumsImporter < ImportScripts::Base
#this parts needs to be synchronously to avoid race conditions #this parts needs to be synchronously to avoid race conditions
# Fetch the post data for the given post_id (this is the first post in the topic) # Fetch the post data for the given post_id (this is the first post in the topic)
row = execute_query("SELECT post_id, user_id_fk, forum_id_fk, post_root_id, post_subject, post_time, post_message, post_father_id, post_replies FROM gforum_Post WHERE post_id = #{post_id}").first row = execute_query("SELECT post_id, user_id_fk, forum_id_fk, post_root_id, post_subject, post_time, post_message, post_father_id, post_replies FROM gforum_Post WHERE post_id = #{post_id}", mysql.client).first
# Early return if the post data is not found # Early return if the post data is not found
return unless row return unless row