v0.41.1 Fixes for FULL concurrency support

This commit is contained in:
David Sainty 2024-08-17 17:09:36 +10:00
parent 8dd8e2f72e
commit 4f608ce87f

View File

@ -1,7 +1,7 @@
# Federated Computer, Inc. # Federated Computer, Inc.
# David Sainty <saint@federated.computer> 2024 A.D. # David Sainty <saint@federated.computer> 2024 A.D.
# Gossamer Threads to Discourse -- Migration-Import Script # Gossamer Threads to Discourse -- Migration-Import Script
# v0.41 Further improve FULL concurrency support, for both MySQL/MariaDB _and_ importantly, the PostGreSQL Discourse DB additions and changes with ActiveRecord connection pooling and Mutex # v0.41.1 Fixes for FULL concurrency support
require 'mysql2' require 'mysql2'
require 'open-uri' require 'open-uri'
@ -299,6 +299,11 @@ class GossamerForumsImporter < ImportScripts::Base
@mysql_client.query(query, as: :hash) @mysql_client.query(query, as: :hash)
end end
# Execute an SQL query on the Gossamer Forums database
def execute_query_concurrent(query, mysql_client)
mysql_client.query(query, as: :hash)
end
# Sanitize the username to meet Discourse's requirements # Sanitize the username to meet Discourse's requirements
def sanitize_username(original_username, email, real_name) def sanitize_username(original_username, email, real_name)
# original_username = username # original_username = username
@ -1110,17 +1115,17 @@ class GossamerForumsImporter < ImportScripts::Base
# Use connection ppoling for PostgreSQL and synchronize access to shared resources # Use connection ppoling for PostgreSQL and synchronize access to shared resources
ActiveRecord::Base.connection_pool.with_connection do ActiveRecord::Base.connection_pool.with_connection do
mutex.synchronize do mutex.synchronize do
# begin begin
puts "Processing post ID: #{post_id}" puts "Processing post ID: #{post_id}"
topic_import_job(post_id, mysql_client) # Import topic and its replies topic_import_job(post_id, mysql_client) # Import topic and its replies
mark_post_as_complete(post_id) # Mark as complete in SQLite table mark_post_as_complete(post_id) # Mark as complete in SQLite table
# rescue => e rescue => e
puts "Error processing post ID #{post_id}: #{e.message}" puts "Error processing post ID #{post_id}: #{e.message}"
mark_post_as_failed(post_id) mark_post_as_failed(post_id)
# ensure ensure
# Ensure the MariaDB connection is closed after processing # Ensure the MariaDB connection is closed after processing
mysql_client.close if mysql_client mysql_client.close if mysql_client
# end end
end end
end end
end end
@ -1159,7 +1164,7 @@ class GossamerForumsImporter < ImportScripts::Base
#this parts needs to be synchronously to avoid race conditions #this parts needs to be synchronously to avoid race conditions
puts "TIJ AA post_id #{post_id}" puts "TIJ AA post_id #{post_id}"
# Fetch the post data for the given post_id (this is the first post in the topic) # Fetch the post data for the given post_id (this is the first post in the topic)
row = execute_query("SELECT post_id, user_id_fk, forum_id_fk, post_root_id, post_subject, post_time, post_message, post_father_id, post_replies FROM gforum_Post WHERE post_id = #{post_id}", mysql_client).first row = execute_query_concurrent("SELECT post_id, user_id_fk, forum_id_fk, post_root_id, post_subject, post_time, post_message, post_father_id, post_replies FROM gforum_Post WHERE post_id = #{post_id}", mysql_client).first
puts "TIJ BB post_id #{post_id}" puts "TIJ BB post_id #{post_id}"
# Early return if the post data is not found # Early return if the post data is not found
@ -1263,7 +1268,7 @@ class GossamerForumsImporter < ImportScripts::Base
insert_url_mapping(row['post_id'], new_url, unique_title) insert_url_mapping(row['post_id'], new_url, unique_title)
# Fetch and import all replies to this topic # Fetch and import all replies to this topic
replies = execute_query("SELECT post_id, user_id_fk, post_message, post_time, FROM gforum_Post WHERE post_root_id = #{post_id} ORDER BY post_time ASC") replies = execute_query_concurrent("SELECT post_id, user_id_fk, post_message, post_time, FROM gforum_Post WHERE post_root_id = #{post_id} ORDER BY post_time ASC", mysql_client)
# Import each reply sequentially # Import each reply sequentially
replies.each do |reply_row| replies.each do |reply_row|