From d3e6dac1de1054de2c130a1dcbd09058038a3f1d Mon Sep 17 00:00:00 2001 From: saint Date: Sat, 17 Aug 2024 20:25:48 +1000 Subject: [PATCH] v0.42 Mutex addition for SQLite (which may be very important --- gossamer_forums.rb | 72 ++++++++++++++++++++++++++++------------------ 1 file changed, 44 insertions(+), 28 deletions(-) diff --git a/gossamer_forums.rb b/gossamer_forums.rb index 602200a..84f54ea 100644 --- a/gossamer_forums.rb +++ b/gossamer_forums.rb @@ -1,7 +1,7 @@ # Federated Computer, Inc. # David Sainty 2024 A.D. # Gossamer Threads to Discourse -- Migration-Import Script -# v0.41.2 Several bug fixes and improvements for fully concurrent topic-post import +# v0.42 Mutex addition for SQLite (which may be very important) require 'mysql2' require 'open-uri' @@ -1081,7 +1081,8 @@ class GossamerForumsImporter < ImportScripts::Base is_complete = false # Flag to indicate whether the import process is complete. # Mutex to control access to shared resources - mutex = Mutex.new + ### mutex = Mutex.new # Mutex for MySQL2 operations -- disabled as this may not in fact be necessary - TBD. + sqlite_mutex = Mutex.new # Mutex for SQLite opreations # Run until all posts have been processed. until is_complete @@ -1107,6 +1108,7 @@ class GossamerForumsImporter < ImportScripts::Base # Submit the import job for the current post_id to the thread pool pool.post do + puts "PP 11" # Initialise a new MariaDB / Mysql2 client inside of each thread mysql_client = Mysql2::Client.new( host: "slowtwitch.northend.network", @@ -1115,21 +1117,26 @@ class GossamerForumsImporter < ImportScripts::Base database: "slowtwitch" ) + puts "PP 22" # Use connection ppoling for PostgreSQL and synchronize access to shared resources ActiveRecord::Base.connection_pool.with_connection do - mutex.synchronize do -## begin +### mutex.synchronize do + begin puts "Processing post ID: #{post_id}" - topic_import_job(post_id, mysql_client) # Import topic and its replies - mark_post_as_complete(post_id) # Mark as complete in SQLite table -## rescue => e + topic_import_job(post_id, mysql_client, sqlite_mutex) # Import topic and its replies + sqlite_mutex.sychronize do + mark_post_as_complete(post_id) # Mark as complete in SQLite table + end + rescue => e puts "Error processing post ID #{post_id}: #{e.message}" - mark_post_as_failed(post_id) -## ensure + sqlite_mutex.sychronize do + mark_post_as_failed(post_id) + end + ensure # Ensure the MariaDB connection is closed after processing mysql_client.close if mysql_client -## end - end + end +### end end end else @@ -1159,7 +1166,7 @@ class GossamerForumsImporter < ImportScripts::Base # end # Method to import an entire topic, including its first post and all subsequent replies - def topic_import_job(post_id, mysql_client) + def topic_import_job(post_id, mysql_client, sqlite_mutex) #Here is where you can import the entire topic #Get post -- SELECT post_id, user_id_fk, forum_id_fk, post_root_id, post_subject, post_time, post_message, post_father_id, post_replies FROM gforum_Post WHERE post_id = post_id #check if exists, create if not @@ -1228,21 +1235,26 @@ class GossamerForumsImporter < ImportScripts::Base end end - # Update the database with the last post time and user for the topic - update_db_topic_last_post_time(topic.id, Time.at(row['post_time']).to_i) - update_db_topic_last_post_user(topic.id, discourse_user_id) + sqlite_mutex.sychronize do + # Update the database with the last post time and user for the topic + update_db_topic_last_post_time(topic.id, Time.at(row['post_time']).to_i) + update_db_topic_last_post_user(topic.id, discourse_user_id) - # Increment the topic count for the user - update_db_user_topic_count(discourse_user_id, fetch_db_user_topic_count(discourse_user_id).to_i + 1) + # Increment the topic count for the user + update_db_user_topic_count(discourse_user_id, fetch_db_user_topic_count(discourse_user_id).to_i + 1) + end # Sanitize and prepare the post message for Discourse sanitized_post_message = sanitize_post_message(row['post_message']) puts "CREATE TOPIC POST topic.id #{topic.id} discourse_user_id #{discourse_user_id}" - # Increment the post count for the topic - post_number = fetch_db_topic_post_numbers(topic.id).to_i + 1 - update_db_topic_post_numbers(topic.id, post_number) + sqlite_mutex.synchronize do + # Increment the post count for the topic + post_number = fetch_db_topic_post_numbers(topic.id).to_i + 1 + update_db_topic_post_numbers(topic.id, post_number) + end + puts "TIJ GG post_id #{post_id}" # Create the initial post in the new topic @@ -1258,9 +1270,11 @@ class GossamerForumsImporter < ImportScripts::Base post.custom_fields['original_gossamer_id'] = row['post_id'] post.save! - # Increment the post count for the topic and user - update_db_topic_post_count(topic.id, fetch_db_topic_post_count(topic.id).to_i + 1) - update_db_user_post_count(discourse_user_id, fetch_db_user_post_count(discourse_user_id).to_i + 1) + sqlite_mutex.sychronize do + # Increment the post count for the topic and user + update_db_topic_post_count(topic.id, fetch_db_topic_post_count(topic.id).to_i + 1) + update_db_user_post_count(discourse_user_id, fetch_db_user_post_count(discourse_user_id).to_i + 1) + end puts "TIJ HH post_id #{post_id}" # Handle any attachments associated with the post @@ -1285,12 +1299,14 @@ class GossamerForumsImporter < ImportScripts::Base puts "CREATE REPLY in topic_id #{topic.id}" - # Increment the post count for the topic - post_number = fetch_db_topic_post_numbers(topic.id).to_i + 1 - update_db_topic_post_numbers(topic.id, post_number) + sqlite_mutex.sychronize do + # Increment the post count for the topic + post_number = fetch_db_topic_post_numbers(topic.id).to_i + 1 + update_db_topic_post_numbers(topic.id, post_number) - # Fetch the number of views the post has had - reply_post_views = fetch_post_views(reply_row['post_id']) + # Fetch the number of views the post has had + reply_post_views = fetch_post_views(reply_row['post_id']) + end puts "TIJ JJ post_id #{post_id}" # Create the reply post in the existing topic