diff --git a/gossamer_forums.rb b/gossamer_forums.rb index de3c0e3..d9357a2 100644 --- a/gossamer_forums.rb +++ b/gossamer_forums.rb @@ -1,7 +1,7 @@ # Federated Computer, Inc. # David Sainty 2024 A.D. # Gossamer Threads to Discourse -- Migration-Import Script -# v0.41 Further improve FULL concurrency support, for both MySQL/MariaDB _and_ importantly, the PostGreSQL Discourse DB additions and changes with ActiveRecord connection pooling and Mutex +# v0.41.1 Fixes for FULL concurrency support require 'mysql2' require 'open-uri' @@ -299,6 +299,11 @@ class GossamerForumsImporter < ImportScripts::Base @mysql_client.query(query, as: :hash) end + # Execute an SQL query on the Gossamer Forums database + def execute_query_concurrent(query, mysql_client) + mysql_client.query(query, as: :hash) + end + # Sanitize the username to meet Discourse's requirements def sanitize_username(original_username, email, real_name) # original_username = username @@ -1110,17 +1115,17 @@ class GossamerForumsImporter < ImportScripts::Base # Use connection ppoling for PostgreSQL and synchronize access to shared resources ActiveRecord::Base.connection_pool.with_connection do mutex.synchronize do -# begin + begin puts "Processing post ID: #{post_id}" topic_import_job(post_id, mysql_client) # Import topic and its replies mark_post_as_complete(post_id) # Mark as complete in SQLite table -# rescue => e + rescue => e puts "Error processing post ID #{post_id}: #{e.message}" mark_post_as_failed(post_id) - # ensure + ensure # Ensure the MariaDB connection is closed after processing mysql_client.close if mysql_client -# end + end end end end @@ -1159,7 +1164,7 @@ class GossamerForumsImporter < ImportScripts::Base #this parts needs to be synchronously to avoid race conditions puts "TIJ AA post_id #{post_id}" # Fetch the post data for the given post_id (this is the first post in the topic) - row = execute_query("SELECT post_id, user_id_fk, forum_id_fk, post_root_id, post_subject, post_time, post_message, post_father_id, post_replies FROM gforum_Post WHERE post_id = #{post_id}", mysql_client).first + row = execute_query_concurrent("SELECT post_id, user_id_fk, forum_id_fk, post_root_id, post_subject, post_time, post_message, post_father_id, post_replies FROM gforum_Post WHERE post_id = #{post_id}", mysql_client).first puts "TIJ BB post_id #{post_id}" # Early return if the post data is not found @@ -1263,7 +1268,7 @@ class GossamerForumsImporter < ImportScripts::Base insert_url_mapping(row['post_id'], new_url, unique_title) # Fetch and import all replies to this topic - replies = execute_query("SELECT post_id, user_id_fk, post_message, post_time, FROM gforum_Post WHERE post_root_id = #{post_id} ORDER BY post_time ASC") + replies = execute_query_concurrent("SELECT post_id, user_id_fk, post_message, post_time, FROM gforum_Post WHERE post_root_id = #{post_id} ORDER BY post_time ASC", mysql_client) # Import each reply sequentially replies.each do |reply_row|