saint/discourse-import_scripts
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

v0.41.1 Fixes for FULL concurrency support

Browse Source
This commit is contained in:
David Sainty 2024-08-17 17:09:36 +10:00
parent 8dd8e2f72e
commit 4f608ce87f
1 changed files with 12 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
gossamer_forums.rb
View File

@ -1,7 +1,7 @@
# Federated Computer, Inc.
# David Sainty <saint@federated.computer> 2024 A.D.
# Gossamer Threads to Discourse -- Migration-Import Script
# v0.41 Further improve FULL concurrency support, for both MySQL/MariaDB _and_ importantly, the PostGreSQL Discourse DB additions and changes with ActiveRecord connection pooling and Mutex
# v0.41.1 Fixes for FULL concurrency support
require 'mysql2'
require 'open-uri'
@ -299,6 +299,11 @@ class GossamerForumsImporter < ImportScripts::Base
@mysql_client.query(query, as: :hash)
end
# Execute an SQL query on the Gossamer Forums database
def execute_query_concurrent(query, mysql_client)
mysql_client.query(query, as: :hash)
end
# Sanitize the username to meet Discourse's requirements
def sanitize_username(original_username, email, real_name)
# original_username = username
@ -1110,17 +1115,17 @@ class GossamerForumsImporter < ImportScripts::Base
# Use connection ppoling for PostgreSQL and synchronize access to shared resources
ActiveRecord::Base.connection_pool.with_connection do
mutex.synchronize do
# begin
begin
puts "Processing post ID: #{post_id}"
topic_import_job(post_id, mysql_client) # Import topic and its replies
mark_post_as_complete(post_id) # Mark as complete in SQLite table
# rescue => e
rescue => e
puts "Error processing post ID #{post_id}: #{e.message}"
mark_post_as_failed(post_id)
# ensure
ensure
# Ensure the MariaDB connection is closed after processing
mysql_client.close if mysql_client
# end
end
end
end
end
@ -1159,7 +1164,7 @@ class GossamerForumsImporter < ImportScripts::Base
#this parts needs to be synchronously to avoid race conditions
puts "TIJ AA post_id #{post_id}"
# Fetch the post data for the given post_id (this is the first post in the topic)
row = execute_query("SELECT post_id, user_id_fk, forum_id_fk, post_root_id, post_subject, post_time, post_message, post_father_id, post_replies FROM gforum_Post WHERE post_id = #{post_id}", mysql_client).first
row = execute_query_concurrent("SELECT post_id, user_id_fk, forum_id_fk, post_root_id, post_subject, post_time, post_message, post_father_id, post_replies FROM gforum_Post WHERE post_id = #{post_id}", mysql_client).first
puts "TIJ BB post_id #{post_id}"
# Early return if the post data is not found
@ -1263,7 +1268,7 @@ class GossamerForumsImporter < ImportScripts::Base
insert_url_mapping(row['post_id'], new_url, unique_title)
# Fetch and import all replies to this topic
replies = execute_query("SELECT post_id, user_id_fk, post_message, post_time, FROM gforum_Post WHERE post_root_id = #{post_id} ORDER BY post_time ASC")
replies = execute_query_concurrent("SELECT post_id, user_id_fk, post_message, post_time, FROM gforum_Post WHERE post_root_id = #{post_id} ORDER BY post_time ASC", mysql_client)
# Import each reply sequentially
replies.each do |reply_row|