v0.28 Improvement for personal message importing - tracking and corner case workaround

gossamer_forums.rb

@@ -1,5 +1,5 @@
 # gossamer threads migration-import code
-# v0.27.2
+# v0.28
 
 require 'mysql2'
 require 'open-uri'
@@ -115,6 +115,12 @@ class GossamerForumsImporter < ImportScripts::Base
       post_id INTEGER
     );
     SQL
+    @db.execute <<-SQL
+    CREATE TABLE IF NOT EXISTS highest_processed_personal_id (
+      id INTEGER PRIMARY KEY CHECK (id = 1),
+      personal_id INTEGER
+    );
+    SQL
   end
 
   def insert_user_id_mapping(old_user_id, new_user_id)
@@ -250,6 +256,15 @@ class GossamerForumsImporter < ImportScripts::Base
     @db.execute "INSERT OR REPLACE INTO highest_processed_post_id (id, post_id) VALUES (1, ?)", post_id
   end
 
+  # Fetch the highest processed personal_id from the highest_processed_personal_id table
+  def fetch_highest_processed_personal_id
+    @db.get_first_value "SELECT personal_id FROM highest_processed_personal_id WHERE id = 1"
+  end
+  
+  # Update the highest processed personal_id in the highest_processed_personal_id table
+  def update_highest_processed_personal_id(personal_id)
+    @db.execute "INSERT OR REPLACE INTO highest_processed_personal_id (id, personal_id) VALUES (1, ?)", personal_id
+  end
 
   # Execute an SQL query on the Gossamer Forums database
   def execute_query(query)
@@ -1144,7 +1159,19 @@ class GossamerForumsImporter < ImportScripts::Base
   # Import personal messages from gforum_Message table (both inbox and sent messages)
   def import_personal_messages
     puts "Importing personal (inbox and sendmail) messages..."
+
+    # Fetch the highest_processed_personal_id
+    highest_processed_personal_id = fetch_highest_processed_personal_id.to_i
+    puts "Highest processed personal_id: #{highest_processed_personal_id}"
+
+# OVERRIDE - to get to problem msg
+    highest_processed_personal_id = 350
+
     execute_query("SELECT * FROM gforum_Message").each do |row|
+      msg_id = row['msg_id'].to_i
+
+      # Skip posts that have already been processed
+      next if msg_id <= highest_processed_personal_id
 
       from_user_id = fetch_user_id_mapping(row['from_user_id_fk'])
       to_user_id = fetch_user_id_mapping(row['to_user_id_fk'])
@@ -1158,21 +1185,20 @@ class GossamerForumsImporter < ImportScripts::Base
         sanitized_message = sanitize_post_message(row['msg_body'])
   
         # Set default message body if the sanitized message is blank
-        sanitized_message = " " if sanitized_message.strip.empty?
+        sanitized_message = "<no message body>" if sanitized_message.strip.empty?
   
-#        # If we do not change the "min personal message post length" to 1, we need this.
+        # If we do not change the "min personal message post length" to 1, we need this. ... We may need this anyway.
-#        sanitized_message = sanitized_message.ljust(10, ' ') if sanitized_message.length < 10
+        sanitized_message = sanitized_message.ljust(10, '.') if sanitized_message.length < 10
         
         # Check and set a default title if the original title is nil or empty
-        title = row['msg_subject']&.strip
+        sanitized_title = row['msg_subject']&.strip
-        title = "<no subject>" if title.nil? || title.empty?
+        sanitized_title = "<no subject>" if sanitized_title.nil? || sanitized_title.empty?
   
-        puts "IMPORTING   title  #{row['msg_subject']}  user_id  #{from_user_id}  to_user_id  #{to_user_id}"
+        puts "IMPORTING msg.  sanitized_title: #{sanitized_title}  orig title: #{row['msg_subject']}  user_id  #{from_user_id}  to_user_id  #{to_user_id}"
 
         # Create a private message topic in Discourse
         topic = Topic.create!(
-#          title: row['msg_subject'],
+          title: sanitized_title,
-          title: title,
           user_id: from_user_id,
           archetype: Archetype.private_message,
           created_at: Time.at(row['msg_time']),
@@ -1185,7 +1211,6 @@ class GossamerForumsImporter < ImportScripts::Base
         post = Post.create!(
           topic_id: topic.id,
           user_id: from_user_id,
-#          raw: row['msg_body'],
           raw: sanitized_message,
           created_at: Time.at(row['msg_time']),
           updated_at: Time.at(row['msg_time'])
@@ -1196,6 +1221,9 @@ class GossamerForumsImporter < ImportScripts::Base
         # Add recipient user to the private message topic
         topic.topic_allowed_users.create!(user_id: to_user_id)
 
+        update_highest_processed_personal_id(msg_id)
+
+# N/A. These were never a thing in Slowtwitch...  
 #        handle_post_attachments(row['msg_id'], post, from_user_id)
       end
     end
@@ -1221,8 +1249,8 @@ class GossamerForumsImporter < ImportScripts::Base
 #    import_topics_and_posts_with_attachments
 #    update_topic_stats
     update_user_stats
-    export_url_mapping_to_csv("gossamer-migration-url-mapping#{timestamp}")
+#    export_url_mapping_to_csv("gossamer-migration-url-mapping#{timestamp}")
-    create_nginx_rewrite_rules("gossamer-redirects#{timestamp}.conf")
+#    create_nginx_rewrite_rules("gossamer-redirects#{timestamp}.conf")
 
     import_personal_messages