v0.42 Mutex addition for SQLite (which may be very important

gossamer_forums.rb

@@ -1,7 +1,7 @@
 # Federated Computer, Inc.
 # David Sainty <saint@federated.computer>  2024 A.D.
 # Gossamer Threads to Discourse -- Migration-Import Script
-# v0.41.2 Several bug fixes and improvements for fully concurrent topic-post import
+# v0.42 Mutex addition for SQLite (which may be very important)
 
 require 'mysql2'
 require 'open-uri'
@@ -1081,7 +1081,8 @@ class GossamerForumsImporter < ImportScripts::Base
     is_complete = false # Flag to indicate whether the import process is complete.
 
     # Mutex to control access to shared resources
-    mutex = Mutex.new
+    ###   mutex = Mutex.new      # Mutex for MySQL2 operations  --  disabled as this may not in fact be necessary - TBD.
+    sqlite_mutex = Mutex.new  # Mutex for SQLite opreations
 
     # Run until all posts have been processed.
     until is_complete
@@ -1107,6 +1108,7 @@ class GossamerForumsImporter < ImportScripts::Base
   
           # Submit the import job for the current post_id to the thread pool
           pool.post do
+    puts "PP 11"
             # Initialise a new MariaDB / Mysql2 client inside of each thread
             mysql_client = Mysql2::Client.new(
               host: "slowtwitch.northend.network",
@@ -1115,21 +1117,26 @@ class GossamerForumsImporter < ImportScripts::Base
               database: "slowtwitch"
             )
 
+    puts "PP 22"
             # Use connection ppoling for PostgreSQL and synchronize access to shared resources
             ActiveRecord::Base.connection_pool.with_connection do
-              mutex.synchronize do
+###              mutex.synchronize do
-##                begin
+                begin
                   puts "Processing post ID: #{post_id}"
-                  topic_import_job(post_id, mysql_client)  # Import topic and its replies
+                  topic_import_job(post_id, mysql_client, sqlite_mutex)  # Import topic and its replies
+                  sqlite_mutex.sychronize do
                     mark_post_as_complete(post_id)  # Mark as complete in SQLite table
-##                rescue => e
+                  end
+                rescue => e
                   puts "Error processing post ID #{post_id}: #{e.message}"
+                  sqlite_mutex.sychronize do
                     mark_post_as_failed(post_id)
-##                ensure
+                  end
+                ensure
                   # Ensure the MariaDB connection is closed after processing
                   mysql_client.close if mysql_client
-##                end
                 end
+###              end
             end
           end
         else
@@ -1159,7 +1166,7 @@ class GossamerForumsImporter < ImportScripts::Base
 #  end
 
   # Method to import an entire topic, including its first post and all subsequent replies
-  def topic_import_job(post_id, mysql_client)
+  def topic_import_job(post_id, mysql_client, sqlite_mutex)
     #Here is where you can import the entire topic
     #Get post -- SELECT post_id, user_id_fk, forum_id_fk, post_root_id, post_subject, post_time, post_message, post_father_id, post_replies FROM gforum_Post WHERE post_id = post_id
     #check if exists, create if not
@@ -1228,21 +1235,26 @@ class GossamerForumsImporter < ImportScripts::Base
             end
           end
     
+          sqlite_mutex.sychronize do
             # Update the database with the last post time and user for the topic
             update_db_topic_last_post_time(topic.id, Time.at(row['post_time']).to_i)
             update_db_topic_last_post_user(topic.id, discourse_user_id)
     
             # Increment the topic count for the user
             update_db_user_topic_count(discourse_user_id, fetch_db_user_topic_count(discourse_user_id).to_i + 1)
+          end
     
           # Sanitize and prepare the post message for Discourse
           sanitized_post_message = sanitize_post_message(row['post_message'])
     
           puts "CREATE TOPIC POST topic.id #{topic.id} discourse_user_id #{discourse_user_id}"
     
+          sqlite_mutex.synchronize do
             # Increment the post count for the topic
             post_number = fetch_db_topic_post_numbers(topic.id).to_i + 1
             update_db_topic_post_numbers(topic.id, post_number)
+          end
+
      puts "TIJ GG post_id #{post_id}"
  
           # Create the initial post in the new topic
@@ -1258,9 +1270,11 @@ class GossamerForumsImporter < ImportScripts::Base
           post.custom_fields['original_gossamer_id'] = row['post_id']
           post.save!
     
+          sqlite_mutex.sychronize do
             # Increment the post count for the topic and user
             update_db_topic_post_count(topic.id, fetch_db_topic_post_count(topic.id).to_i + 1)
             update_db_user_post_count(discourse_user_id, fetch_db_user_post_count(discourse_user_id).to_i + 1)
+          end
   
       puts "TIJ HH post_id #{post_id}"
           # Handle any attachments associated with the post
@@ -1285,12 +1299,14 @@ class GossamerForumsImporter < ImportScripts::Base
     
               puts "CREATE REPLY in topic_id #{topic.id}"
     
+              sqlite_mutex.sychronize do
                 # Increment the post count for the topic
                 post_number = fetch_db_topic_post_numbers(topic.id).to_i + 1
                 update_db_topic_post_numbers(topic.id, post_number)
     
                 # Fetch the number of views the post has had
                 reply_post_views = fetch_post_views(reply_row['post_id'])
+              end
   
       puts "TIJ JJ post_id #{post_id}"
               # Create the reply post in the existing topic