Completed goss-restorethread.rb ready for use on Prod for topics needing (re)import / restoration.

goss-checkmd5pwhash.rb

@@ -1,12 +1,12 @@
 # Load the Discourse environment
-require File.expand_path("../../../config/environment", __FILE__)
+require File.expand_path("../../../../config/environment", __FILE__)
 
 # require_relative '/var/www/discourse/config/environment'
 
 def query_md5_password_custom_field(username)
   user = User.find_by(username: username)
   if user
-    md5_password = user.custom_fields['custom_password_md5']
+    md5_password = user.custom_fields['md5_password']
     if md5_password
       puts "MD5 password custom field for user #{username}: #{md5_password}"
     else
@@ -26,3 +26,15 @@ end
 username = 'davidpaulyoung'
 
 query_md5_password_custom_field(username)
+
+username = 'Slowman'
+
+query_md5_password_custom_field(username)
+
+username = 'GT'
+
+query_md5_password_custom_field(username)
+
+username = 'Chimpking'
+
+query_md5_password_custom_field(username)

goss-cleanup.rb

@@ -1,7 +1,7 @@
 # Federated Computer, Inc.
 # David Sainty <saint@federated.computer>  2024 A.D.
 # Gossamer Threads to Discourse -- CleanUp Script
-# v0.16 Add parallel deletion of posts.
+# v0.17 Add more cleanup options.
 
 require 'concurrent-ruby'
 require File.expand_path("../../../../config/environment", __FILE__)
@@ -75,6 +75,24 @@ class GossamerForumsCleaner
     end
   end
 
+  def cleanup_topics_with_invalid_posts_count
+    puts "Cleaning up topics with invalid posts_count..."
+
+    # Iterate through all topics
+    Topic.where("posts_count IS NULL OR posts_count = -1 OR posts_count = 0").find_each do |topic|
+      puts "Identified topic for deletion: Title: #{topic.title}  Topic ID: #{topic.id}"
+  
+      # Destroy all posts in the topic
+      topic.posts.each do |post|
+        puts "Deleting post #{post.id} in topic #{topic.id}"
+        post.destroy
+      end
+
+      # Destroy the topic itself
+      topic.destroy
+    end
+  end
+
   def cleanup_posts
     puts "Cleaning up imported posts..."
     # Find all posts that were imported and delete them
@@ -131,7 +149,8 @@ class GossamerForumsCleaner
     puts "Cleanup beginning!"
 #    cleanup_messages
 #    cleanup_topics
-    cleanup_topics_former_user
+#    cleanup_topics_former_user
+    cleanup_topics_with_invalid_posts_count
 #    cleanup_posts_parallel
 #    cleanup_categories
 #    cleanup_users

goss-correctencoding.rb

@@ -0,0 +1,163 @@
+# Federated Computer, Inc.
+# David Sainty <saint@federated.computer>  2024 A.D.
+# Gossamer Threads to Discourse -- Correct Encoding
+# v0.7 Further attempt to get this reverse dobule encoding right now
+
+require 'mysql2'
+require 'active_record'
+require 'charlock_holmes'
+
+# require 'concurrent-ruby'
+require File.expand_path("../../../../config/environment", __FILE__)
+require File.expand_path("../../../../script/import_scripts/base", __FILE__)
+
+class GossamerForumsCorrectEncoding < ImportScripts::Base
+  def initialize
+    super
+      begin
+        # Initialize MySQL client to connect to Gossamer Forums database
+        @mysql_client = Mysql2::Client.new(
+          host: "slowtwitch.northend.network",
+          username: "admin",
+          password: "yxnh93Ybbz2Nm8#mp28zCVv",
+          database: "slowtwitch"
+        )
+      rescue Mysql2::Error => e
+        puts "Error connecting to MySQL: #{e.message}"
+        puts e.backtrace.join("\n")  # Print the full stack trace
+        exit 1
+      end
+      @batch_size = 1000   # Number of posts to process in each batch
+  end
+
+  # Method to detect and fix text encoding
+  def fix_text_encoding(content)
+    begin
+#      # Treat as Windows-1252 (cp1252) and then decode into UTF-8
+#      corrected_content = content.encode('CP1252').force_encoding('UTF-8')
+      # Step 1: Treat content as CP1252 and convert it back to UTF-8
+      corrected_content = content.encode('CP1252').force_encoding('UTF-8').force_encoding('UTF-8')
+      
+#      # Step 2: Check if there's still a problem (if still corrupted, apply the second pass)
+#      if corrected_content.valid_encoding?
+#        return corrected_content
+#      else
+#        # Step 3: If it's not valid UTF-8, re-encode and try to fix remaining issues
+#        corrected_content.encode('UTF-8', invalid: :replace, undef: :replace)
+#      end
+    rescue Encoding::UndefinedConversionError => e
+      puts "Error during encoding conversion: #{e.message}"
+      puts e.backtrace.join("\n")  # Print the full stack trace
+      return content # Return the original content if conversion fails
+    end
+
+    corrected_content
+  end
+
+#    # Step 1: Try to detect encoding of the corrupted (double-encoded) content
+#    detection = CharlockHolmes::EncodingDetector.detect(broken_content)
+#    original_encoding = detection[:encoding]
+#    puts "Original encoding detected: #{original_encoding}"
+#  
+#    # Step 2: First decode the double-encoded content
+#    begin
+#      # Convert the content assuming it was double-encoded, so decode twice
+#        # First, convert from the detected encoding (ISO-8859-1 or windows-1252) to UTF-8
+#      first_pass = CharlockHolmes::Converter.convert(broken_content, original_encoding, 'UTF-8')
+#      
+#      # Step 3: Now re-interpret that output as if it's broken UTF-8 and convert it back to UTF-8
+#      fixed_content = CharlockHolmes::Converter.convert(first_pass, 'UTF-8', 'UTF-8')
+#      
+#    rescue => e
+#      puts "Error during encoding fix: #{e.message}"
+#      puts e.backtrace.join("\n")  # Print the full stack trace
+#
+#      fixed_content = broken_content # Fall back to the broken content if decoding fails
+#    end
+#  
+#    return fixed_content
+#end
+
+#    # Detect encoding
+#    detection = CharlockHolmes::EncodingDetector.detect(raw_content)
+#    original_encoding = detection[:encoding]
+#    puts "Original encoding detected: #{original_encoding}"
+  
+#      # Force the encoding to the detected one, then covnert to UTF-8
+#      if original_encoding == 'ISO-8859-1' || original_encoding == 'windows-1252'
+#        # For Windows-1252 or ISO-8859-1, force the encoding and convert to UTF-8
+#        # text.force_encoding('ISO-8859-1').encode('UTF-8')
+#        text.force_encoding(original_encoding).encode('UTF-8', invalid: :replace, undef: :replace, replace: '?')
+#      else
+#        # Try to convert from detected encoding to UTF-8
+#        text.encode('UTF-8', original_encoding, invalid: :replace, undef: :replace, replace: '?')
+#      end
+
+#    if original_encoding
+#      begin
+#        decoded_content = CharlockHolmes::Converter.convert(raw_content, original_encoding, 'UTF-8')
+#      rescue => e
+#        puts "Error during encoding conversion: #{e.message}"
+#        decoded_content = raw_content # Fall back to raw content if decoding fails
+#      end
+#    else
+#      decoded_content = raw_content # Fallback if encoding detection fails
+#    end
+#
+#    # Step 3: Ensure the content is now correctly in UTF-8 (no need to encode again)
+#    return decoded_content
+#  end
+        
+#    rescue StandardError => e
+#      puts "Error during encoding conversion: #{e.message}"
+#      puts e.backtrace.join("\n")  # Print the full stack trace
+#      text
+#    end
+#  end
+
+    # Method to fix encoding issues in post content
+  def fix_encoding
+    offset = 0
+
+    loop do
+      puts "OFFSET:  #{offset}"
+      begin
+        posts = Post.limit(@batch_size).offset(offset)
+        break if posts.empty?
+
+        posts.each do |post|
+          raw_content = post.raw
+          puts "--> NEXT POST:   post.id:  #{post.id}"
+          fixed_content = fix_text_encoding(raw_content)
+          if fixed_content != raw_content
+            puts "Updating post #{post.id}"
+            puts "------- raw_content:\n#{raw_content}"
+            puts "+++++++ fixed_content:\n#{fixed_content}"
+            puts "---------------------------------------------------------------------------------------------"
+            # post.update(raw: fixed_content)
+#            post.raw = fixed_content
+#            if post.save
+#              puts "Post ##{post.id} updated successfully."
+#            else
+#              puts "Failed to update Post ##{post.id}: #{post.errors.full_messages.join(', ')}"
+#            end
+          end
+        end
+      rescue
+        puts "Error:  #{e.message}"
+        puts e.backtrace.join("\n")  # Print the full stack trace
+      end
+
+      offset += @batch_size
+    end
+  end
+
+  def perform_encoding_correction
+    puts "Encoding Correction beginning!"
+    fix_encoding
+    puts "Encoding Correction complete!"
+  end
+end
+
+GossamerForumsCorrectEncoding.new.perform_encoding_correction
+

goss-destroydeletedposts.rb

@@ -0,0 +1,112 @@
+# Federated Computer, Inc.
+# David Sainty <saint@federated.computer>  2024 A.D.
+# Gossamer Threads to Discourse -- CleanUp Script
+# v0.5 We need to handle deletion of topic posts -- delete all posts in topic and then delete topic itself
+
+require 'mysql2'
+require 'active_record'
+
+# require 'concurrent-ruby'
+require File.expand_path("../../../../config/environment", __FILE__)
+require File.expand_path("../../../../script/import_scripts/base", __FILE__)
+
+class GossamerForumsDestroyDeletedPosts < ImportScripts::Base
+  def initialize
+    super
+      begin
+        # Initialize MySQL client to connect to Gossamer Forums database
+        @mysql_client = Mysql2::Client.new(
+          host: "slowtwitch.northend.network",
+          username: "admin",
+          password: "yxnh93Ybbz2Nm8#mp28zCVv",
+          database: "slowtwitch"
+        )
+      rescue Mysql2::Error => e
+        puts "Error connecting to MySQL: #{e.message}"
+        exit 1
+      end
+  end
+
+  # Define a method to find a post by custom field
+  def find_post_by_custom_field(post_id)
+    puts "DestroyDeletedPosts: Searching for post with original_gossamer_id: #{post_id}"
+  
+    post_custom_field = PostCustomField.find_by(name: 'original_gossamer_id', value: post_id.to_s)
+  
+    if post_custom_field
+      post = post_custom_field.post
+      puts "DestroyDeletedPosts: Found post with id: #{post.id}"
+      post
+    else
+      puts "DestroyDeletedPosts: No post found with original_gossamer_id: #{post_id}"
+      nil
+    end
+  end
+
+  # Define a method to delete all posts in a topic
+  def delete_all_posts_in_topic(topic_id)
+    posts = Post.where(topic_id: topic_id)
+    posts.each do |post|
+      puts "DELETE ALL POSTS --- DiscourseDeletedPosts: Deleting post with id: #{post.id}"
+      # post.destroy
+    end
+  end
+
+  # Define a method to delete a topic
+  def delete_topic(topic_id)
+    topic = Topic.find_by(id: topic_id)
+    if topic
+      puts "DELETE TOPIC --- DiscourseDeletedPosts: Deleting topic with id: #{topic_id}"
+      # topic.destroy
+    end
+  end
+
+
+  # Define the method to delete posts based on the Gossamer Forums flag
+  def destroy_deleted_posts_from_gossamer_with_user(username)
+ 
+    # Query the user ID from the legacy MySQL database
+    user_result = @mysql_client.query("SELECT user_id FROM gforum_User WHERE user_username = '#{username}' LIMIT 1")
+    user_id_row = user_result.first
+    if user_id_row.nil?
+      puts "DiscourseDeletedPosts: No user found with username: #{username}"
+      return
+    end
+    user_id = user_id_row['user_id']
+
+    # Find all posts marked as deleted by the given user
+    posts_result = @mysql_client.query("SELECT post_id FROM gforum_Post WHERE post_deleted = 1 AND user_id_fk = #{user_id}")
+
+    posts_result.each do |legacy_post|
+      post_id = legacy_post['post_id']
+
+      # Look for the post in Discourse by custom field
+      post = find_post_by_custom_field(post_id)
+  
+      if post
+        # Check if this post is the topic post
+        if post.post_number == 1
+          topic_id = post.topic_id
+          # Delete all posts in the topic
+          delete_all_posts_in_topic(topic_id)
+          # Delete the topic itself
+          delete_topic(topic_id)
+        else
+          # If not the topic post, just delete the individual post
+          puts "DELETE POST --- DestroyDeletedPosts: Deleting post with id: #{post.id}"
+          # post.destroy
+        end
+      end
+    end
+  end
+
+  def perform_deleted_destroy
+    puts "Destroy Deleted Posts beginning!"
+    # destroy_deleted_posts_from_gossamer
+    destroy_deleted_posts_from_gossamer_with_user('spudone')
+    puts "Destroy Deleted Posts complete!"
+  end
+end
+
+GossamerForumsDestroyDeletedPosts.new.perform_deleted_destroy
+

goss-resetemail.rb

@@ -0,0 +1,67 @@
+# Federated Computer, Inc.
+# David Sainty <saint@federated.computer>  2024 A.D.
+# Gossamer Threads to Discourse -- Reset Discourse Email
+# v0.1 First addition for resetting email address for user
+
+require File.expand_path("../../../../config/environment", __FILE__)
+
+class GossamerResetEmail
+  def reset_email(discourse_username, new_email)
+    puts "Resetting email for Discourse username: '#{discourse_username}'"
+ 
+    user = User.find_by(username: discourse_username)
+    if user
+      puts "New email: #{new_email} ..."
+
+      # Delete Gossamer MD5 Password
+      # user.custom_fields.delete('md5_password') # This removes the field completely
+      # user.save!
+      # puts "      STEP 1 COMPLETED: Gossamer MD5 Password custom field removed for username: #{discourse_username}"
+
+      # Set the Discourse password with the `password=` method to properly hash the password
+      # user.password = new_password
+      # user.save!
+      # puts "      STEP 2 COMPLETED: Discourse Password updated for username: #{discourse_username}"
+
+      user.email = new_email
+      user.save!
+      puts "      STEP 1 COMPLETED: Email address updated for username: #{discourse_username}"
+
+      # Update other attributes (to be sure to remove issues with login)
+      user.active = true
+      user.approved = true
+      user.approved_at = Time.now
+      user.approved_by_id = 1
+      user.save!
+      puts "      STEP 2 COMPLETED: Other attributes (active, approved, approved_at, approved_by_id) updated for username: #{discourse_username}"
+
+      # Generate a new token, hash it, and create a confirmed email token for the user
+      token = SecureRandom.hex(20)
+      token_hash = EmailToken.hash_token(token)
+
+      EmailToken.create!(
+        user_id: user.id,
+        email: user.email,
+        token_hash: token_hash,
+        confirmed: true
+      )
+      puts "      STEP 3 COMPLETED: New token generated, hashed and set as confirmed email token for username: #{discourse_username}"
+
+    else
+      puts "User not found: #{discourse_username}"
+    end
+  end
+end
+
+# Main execution
+if ARGV.length != 2
+  puts "Usage: #{$0} discourse_username email_address@domain.com"
+  exit 1
+end
+
+discourse_username = ARGV[0]
+new_email = ARGV[1]
+# search_topic_by_title(search_string)
+
+GossamerResetEmail.new.reset_email(discourse_username, new_email)
+

goss-resetpw.rb

@@ -0,0 +1,63 @@
+# Federated Computer, Inc.
+# David Sainty <saint@federated.computer>  2024 A.D.
+# Gossamer Threads to Discourse -- Reset Discourse Password and Clear Gossamer Password
+# v0.2 Full support
+
+require File.expand_path("../../../../config/environment", __FILE__)
+
+class GossamerResetPassword
+  def reset_password(discourse_username, new_password)
+    puts "Resetting password for Discourse username: '#{discourse_username}'"
+ 
+    user = User.find_by(username: discourse_username)
+    if user
+      puts "New password: #{new_password} ..."
+
+      # Delete Gossamer MD5 Password
+      user.custom_fields.delete('md5_password') # This removes the field completely
+      user.save!
+      puts "      STEP 1 COMPLETED: Gossamer MD5 Password custom field removed for username: #{discourse_username}"
+
+      # Set the Discourse password with the `password=` method to properly hash the password
+      user.password = new_password
+      user.save!
+      puts "      STEP 2 COMPLETED: Discourse Password updated for username: #{discourse_username}"
+
+      # Update other attributes (other than password which is already correct)
+      user.active = true
+      user.approved = true
+      user.approved_at = Time.now
+      user.approved_by_id = 1
+      user.save!
+      puts "      STEP 3 COMPLETED: Other attributes (active, approved, approved_at, approved_by_id) updated for username: #{discourse_username}"
+
+      # Generate a new token, hash it, and create a confirmed email token for the user
+      token = SecureRandom.hex(20)
+      token_hash = EmailToken.hash_token(token)
+
+      EmailToken.create!(
+        user_id: user.id,
+        email: user.email,
+        token_hash: token_hash,
+        confirmed: true
+      )
+      puts "      STEP 4 COMPLETED: New token generated, hashed and set as confirmed email token for username: #{discourse_username}"
+
+    else
+      puts "User not found: #{discourse_username}"
+    end
+  end
+end
+
+# Main execution
+if ARGV.length != 2
+  puts "Usage: #{$0} discourse_username n3w_p4ssword."
+  exit 1
+end
+
+discourse_username = ARGV[0]
+new_password = ARGV[1]
+# search_topic_by_title(search_string)
+
+GossamerResetPassword.new.reset_password(discourse_username, new_password)
+

goss-search.rb

@@ -0,0 +1,67 @@
+# Federated Computer, Inc.
+# David Sainty <saint@federated.computer>  2024 A.D.
+# Gossamer Threads to Discourse -- Search By Topic Title
+# v0.1 Initial
+
+require 'concurrent-ruby'
+require File.expand_path("../../../../config/environment", __FILE__)
+
+class GossamerForumsSearch
+  def search_topic_by_title(search_string)
+    puts "Searching for topics with title containing: '#{search_string}'"
+  
+    # Search for topics with titles containing the search string
+    matching_topics = Topic.where('title LIKE ?', "%#{search_string}%")
+  
+    # Print information about each matching topic
+    matching_topics.each do |topic|
+      puts "----------------------------------------"
+      puts "ID: #{topic.id}"
+      puts "Title: #{topic.title}"
+      puts "Created At: #{topic.created_at}"
+      puts "Updated At: #{topic.updated_at}"
+      puts "Category ID: #{topic.category_id}"
+      puts "Views: #{topic.views}"
+      puts "Posts Count: #{topic.posts_count}"
+      puts "Last Posted At: #{topic.last_posted_at}"
+      puts "Bumped At: #{topic.bumped_at}"
+      puts "Last Post User ID: #{topic.last_post_user_id}"
+      puts "----------------------------------------"
+
+      # Fetch and display all posts for the topic
+      topic.posts.each do |post|
+        puts "  Post ID: #{post.id}"
+        puts "  User ID: #{post.user_id}"
+        puts "  Created At: #{post.created_at}"
+        puts "  Updated At: #{post.updated_at}"
+        puts "  Post Number: #{post.post_number}"
+        puts "  Cooked: #{post.cooked.truncate(80)}" # Limit the display of post content to 100 characters
+        puts "  Raw: #{post.raw.truncate(80)}" # Limit the display of raw post content to 100 characters
+        puts "  ----------------------------------------"
+      end
+    end
+  
+    if matching_topics.empty?
+      puts "No topics found with title containing: '#{search_string}'"
+    end
+  end
+
+  def perform_search
+    puts "Search beginning!"
+    search_topic_by_title("No father is more proud")
+    puts "Search complete!"
+  end
+end
+
+# Main execution
+if ARGV.length != 1
+  puts "Usage: #{$0} SEARCH_STRING"
+  exit 1
+end
+
+search_string = ARGV[0]
+# search_topic_by_title(search_string)
+
+# GossamerForumsSearch.new.perform_search
+GossamerForumsSearch.new.search_topic_by_title(search_string)
+

goss-setdispw.rb

@@ -1,5 +1,5 @@
 # Load the Discourse environment
-require File.expand_path("../../../config/environment", __FILE__)
+require File.expand_path("../../../../config/environment", __FILE__)
 
 # require_relative '/var/www/discourse/config/environment'
 

goss-setmd5pwhash.rb

@@ -1,12 +1,12 @@
 # Load the Discourse environment
-require File.expand_path("../../../config/environment", __FILE__)
+require File.expand_path("../../../../config/environment", __FILE__)
 
 # require_relative '/var/www/discourse/config/environment'
 
 def set_md5_password_custom_field(username, md5_password)
   user = User.find_by(username: username)
   if user
-    user.custom_fields['custom_password_md5'] = md5_password
+    user.custom_fields['md5_password'] = md5_password
     user.save!
     puts "MD5 password custom field set for user: #{username}"
   else

gossamer_forums.rb

@@ -1,7 +1,7 @@
 # Federated Computer, Inc.
 # David Sainty <saint@federated.computer>  2024 A.D.
 # Gossamer Threads to Discourse -- Migration-Import Script
-# v0.54 Tweak trust levels
+# v0.60 Final tweak for topic and user count updating for last public posts
 
 require 'mysql2'
 require 'open-uri'
@@ -114,47 +114,47 @@ class GossamerForumsImporter < ImportScripts::Base
     SQL
     # POST IMPORT - For each topic, the time of the last / most recent post/reply
     @db.execute <<-SQL
-      CREATE TABLE IF NOT EXISTS topic_last_post_time (
+      CREATE TABLE IF NOT EXISTS topic_last_post_time_final (
         topic_id INTEGER PRIMARY KEY,
         last_post_time INTEGER
       );
     SQL
     # POST IMPORT - For each topic, increment post_count as we add posts.
     @db.execute <<-SQL
-      CREATE TABLE IF NOT EXISTS topic_post_count (
+      CREATE TABLE IF NOT EXISTS topic_post_count_final (
         topic_id INTEGER PRIMARY KEY,
         post_count INTEGER DEFAULT 0
       );
     SQL
     # POST IMPORT  - For each user (_id), increment topic_count as we add topics (to see total topics per user) 
     @db.execute <<-SQL
-      CREATE TABLE IF NOT EXISTS user_topic_count (
+      CREATE TABLE IF NOT EXISTS user_topic_count_final (
         user_id INTEGER PRIMARY KEY,
         topic_count INTEGER DEFAULT 0
       );
     SQL
     # POST IMPORT - For each user (_id), increment post_count as we add posts (to see total posts per user)
     @db.execute <<-SQL
-      CREATE TABLE IF NOT EXISTS user_post_count (
+      CREATE TABLE IF NOT EXISTS user_post_count_final (
         user_id INTEGER PRIMARY KEY,
         post_count INTEGER DEFAULT 0
       );
     SQL
     # POST IMPORT - For each topic, the user_id for the last poster / replier
     @db.execute <<-SQL
-      CREATE TABLE IF NOT EXISTS topic_last_post_user (
+      CREATE TABLE IF NOT EXISTS topic_last_post_user_final (
         topic_id INTEGER PRIMARY KEY,
         user_id INTEGER
       );
     SQL
     # POST IMPORT - The number of posts in a given topic, incremented as we add a new reply post to a topic.
     @db.execute <<-SQL
-      CREATE TABLE IF NOT EXISTS topic_post_numbers (
+      CREATE TABLE IF NOT EXISTS topic_post_numbers_final (
         topic_id INTEGER PRIMARY KEY,
         post_number INTEGER DEFAULT 0
       );
     SQL
-    # POST IMPORT - Record perssitent integer value for highest processed post id -- not used
+    # POST IMPORT - Record perssitent integer value for highest processed post id
     @db.execute <<-SQL
     CREATE TABLE IF NOT EXISTS highest_processed_post_id (
       id INTEGER PRIMARY KEY CHECK (id = 1),
@@ -229,7 +229,7 @@ class GossamerForumsImporter < ImportScripts::Base
   end
 
   # Method to create Nginx rewrite rules file
-  def create_nginx_rewrite_rules(filename)
+  def export_nginx_rewrite_rules(filename)
     File.open(filename, "w") do |file|
       @db.execute("SELECT old_post_id, new_url FROM url_map") do |row|
         old_post_id, new_url = row
@@ -253,51 +253,51 @@ class GossamerForumsImporter < ImportScripts::Base
   end
 
   def fetch_db_topic_last_post_time(topic_id)
-    @db.get_first_value "SELECT last_post_time FROM topic_last_post_time WHERE topic_id = ?", topic_id
+    @db.get_first_value "SELECT last_post_time FROM topic_last_post_time_final WHERE topic_id = ?", topic_id
   end
   
   def fetch_db_topic_last_post_user(topic_id)
-    @db.get_first_value "SELECT user_id FROM topic_last_post_user WHERE topic_id = ?", topic_id
+    @db.get_first_value "SELECT user_id FROM topic_last_post_user_final WHERE topic_id = ?", topic_id
   end
   
   def fetch_db_topic_post_count(topic_id)
-    @db.get_first_value "SELECT post_count FROM topic_post_count WHERE topic_id = ?", topic_id
+    @db.get_first_value "SELECT post_count FROM topic_post_count_final WHERE topic_id = ?", topic_id
   end
   
   def fetch_db_user_topic_count(user_id)
-    @db.get_first_value "SELECT topic_count FROM user_topic_count WHERE user_id = ?", user_id
+    @db.get_first_value "SELECT topic_count FROM user_topic_count_final WHERE user_id = ?", user_id
   end
   
   def fetch_db_user_post_count(user_id)
-    @db.get_first_value "SELECT post_count FROM user_post_count WHERE user_id = ?", user_id
+    @db.get_first_value "SELECT post_count FROM user_post_count_final WHERE user_id = ?", user_id
   end
   
   def fetch_db_topic_post_numbers(topic_id)
-    @db.get_first_value "SELECT post_number FROM topic_post_numbers WHERE topic_id = ?", topic_id
+    @db.get_first_value "SELECT post_number FROM topic_post_numbers_final WHERE topic_id = ?", topic_id
   end
   
   def update_db_topic_last_post_time(topic_id, last_post_time)
-    @db.execute "INSERT OR REPLACE INTO topic_last_post_time (topic_id, last_post_time) VALUES (?, ?)", topic_id, last_post_time
+    @db.execute "INSERT OR REPLACE INTO topic_last_post_time_final (topic_id, last_post_time) VALUES (?, ?)", topic_id, last_post_time
   end
   
   def update_db_topic_last_post_user(topic_id, user_id)
-    @db.execute "INSERT OR REPLACE INTO topic_last_post_user (topic_id, user_id) VALUES (?, ?)", topic_id, user_id
+    @db.execute "INSERT OR REPLACE INTO topic_last_post_user_final (topic_id, user_id) VALUES (?, ?)", topic_id, user_id
   end
   
   def update_db_topic_post_count(topic_id, post_count)
-    @db.execute "INSERT OR REPLACE INTO topic_post_count (topic_id, post_count) VALUES (?, ?)", topic_id, post_count
+    @db.execute "INSERT OR REPLACE INTO topic_post_count_final (topic_id, post_count) VALUES (?, ?)", topic_id, post_count
   end
   
   def update_db_user_topic_count(user_id, topic_count)
-    @db.execute "INSERT OR REPLACE INTO user_topic_count (user_id, topic_count) VALUES (?, ?)", user_id, topic_count
+    @db.execute "INSERT OR REPLACE INTO user_topic_count_final (user_id, topic_count) VALUES (?, ?)", user_id, topic_count
   end
   
   def update_db_user_post_count(user_id, post_count)
-    @db.execute "INSERT OR REPLACE INTO user_post_count (user_id, post_count) VALUES (?, ?)", user_id, post_count
+    @db.execute "INSERT OR REPLACE INTO user_post_count_final (user_id, post_count) VALUES (?, ?)", user_id, post_count
   end
   
   def update_db_topic_post_numbers(topic_id, post_number)
-    @db.execute "INSERT OR REPLACE INTO topic_post_numbers (topic_id, post_number) VALUES (?, ?)", topic_id, post_number
+    @db.execute "INSERT OR REPLACE INTO topic_post_numbers_final (topic_id, post_number) VALUES (?, ?)", topic_id, post_number
   end
 
   # Fetch the highest processed post_id from the highest_processed_post_id table
@@ -624,27 +624,38 @@ class GossamerForumsImporter < ImportScripts::Base
     users = []
 
     # Fetch all users from Gossamer Forums
-    execute_query("SELECT * FROM gforum_User").each do |row|
+#    execute_query("SELECT * FROM gforum_User").each do |row|
-      users << {
+    execute_query("SELECT user_id, user_username, user_email, user_real_name, user_registered, user_last_seen, user_title, user_about, user_homepage, user_location, user_password FROM gforum_User WHERE user_id > 123381").each do |row|
-        id: row['user_id'],
+      username = sanitize_username(row['user_username'], row['user_email'], row['user_real_name'])
-        username: sanitize_username(row['user_username'], row['user_email'], row['user_real_name']),
+      email = row['user_email']
-        email: row['user_email'],
+  
-        created_at: Time.at(row['user_registered']),
+      # Check if the user already exists in Discourse by username or email
-        updated_at: Time.at(row['user_last_seen']),
+ #     existing_user = User.find_by(username: username) || User.find_by(email: email)
-        name: row['user_real_name'],
+      existing_user = User.find_by(username: username) || User.joins(:user_emails).find_by(user_emails: { email: email })
-        title: row['user_title'],
+  
-        bio_raw: row['user_about'] || "",
+      # Only add the user if they do not already exist
-        website: row['user_homepage'],
+      unless existing_user
-        location: row['user_location'],
+        users << {
-        custom_fields: {
+          id: row['user_id'],
-          md5_password: row['user_password'],
+          username: username,
-          original_username: row['user_username'],
+          email: email,
-          original_gossamer_id: row['user_id']
+          created_at: Time.at(row['user_registered']),
+          updated_at: Time.at(row['user_last_seen']),
+          name: row['user_real_name'],
+          title: row['user_title'],
+          bio_raw: row['user_about'] || "",
+          website: row['user_homepage'],
+          location: row['user_location'],
+          custom_fields: {
+            md5_password: row['user_password'],
+            original_username: row['user_username'],
+            original_gossamer_id: row['user_id']
+          }
         }
-      }
+      end
     end
 
-    # Create or update users in Discourse
+    # Create new users in Discourse
     create_users(users) do |user|
 #      insert_user_id_mapping(user[:id], user.id)
       user
@@ -657,7 +668,8 @@ class GossamerForumsImporter < ImportScripts::Base
     users = []
 
     # Fetch all users from Gossamer Forums
-    execute_query("SELECT * FROM gforum_User").each do |row|
+#    execute_query("SELECT * FROM gforum_User").each do |row|
+    execute_query("SELECT user_id, user_username, user_email, user_real_name, user_registered, user_last_seen, user_title, user_about, user_homepage, user_location, user_password FROM gforum_User WHERE user_id > 123381").each do |row|
       users << {
         id: row['user_id'],
         username: sanitize_username(row['user_username'], row['user_email'], row['user_real_name']),
@@ -701,7 +713,8 @@ class GossamerForumsImporter < ImportScripts::Base
     users = []
 
     # Fetch all users from Gossamer Forums
-    execute_query("SELECT * FROM gforum_User").each do |row|
+#    execute_query("SELECT * FROM gforum_User").each do |row|
+    execute_query("SELECT user_id, user_username, user_email, user_real_name, user_registered, user_last_seen, user_title, user_about, user_homepage, user_location, user_password FROM gforum_User WHERE user_id > 123381").each do |row|
       users << {
         id: row['user_id'],
         username: sanitize_username(row['user_username'], row['user_email'], row['user_real_name']),
@@ -737,16 +750,16 @@ class GossamerForumsImporter < ImportScripts::Base
       discourse_user.user_profile.bio_raw ||= ""
 
       # Append bio if it exists, otherwise set it to empty string to avoid nil errors
-      if discourse_user.user_profile.bio_raw.empty?
+##      if discourse_user.user_profile.bio_raw.empty?
-        discourse_user.user_profile.bio_raw = user[:bio_raw]
+      discourse_user.user_profile.bio_raw = user[:bio_raw]
-      else
+##      else
-        discourse_user.user_profile.bio_raw += "\n\n" + user[:bio_raw]
+##        discourse_user.user_profile.bio_raw += "\n\n" + user[:bio_raw]
-      end
+##      end
 
       # Ensure the bio does not exceed 3000 characters
-      if discourse_user.user_profile.bio_raw.length > 3000
+      if discourse_user.user_profile.bio_raw.length > 2999
         puts "Warning: About Me for user #{discourse_user.username} (ID: #{discourse_user.id}) exceeds 3000 characters. Truncating."
-        discourse_user.user_profile.bio_raw = discourse_user.user_profile.bio_raw[0, 3000]
+        discourse_user.user_profile.bio_raw = discourse_user.user_profile.bio_raw[0, 2999]
       end
       discourse_user.user_profile.save!
 
@@ -1350,7 +1363,7 @@ class GossamerForumsImporter < ImportScripts::Base
     
               topic_created = true
 #            rescue ActiveRecord::RecordInvalid => e
-              rescue => e
+            rescue => e
               if e.message.include?("Title has already been used")
                 unique_title = "#{title} (#{suffix})"
                 suffix += 1
@@ -1529,7 +1542,7 @@ class GossamerForumsImporter < ImportScripts::Base
 
 
   # Import topics and posts from Gossamer Forums to Discourse
-  def import_topics_and_posts_with_attachments
+  def import_topics_and_posts
     puts "Importing topics and posts with attachments..."
    
 #    topic_last_post_time = {}
@@ -1540,16 +1553,18 @@ class GossamerForumsImporter < ImportScripts::Base
 #    topic_post_numbers = Hash.new { |hash, key| hash[key] = 0 }
 
     # Fetch the highest old_post_id from the url_map table
-    highest_old_post_id = fetch_highest_old_post_id.to_i
+#    highest_old_post_id = fetch_highest_old_post_id.to_i
-    puts "Highest (OP) old_post_id in url_map: #{highest_old_post_id}"
+#    puts "Highest (OP) old_post_id in url_map: #{highest_old_post_id}"
     highest_processed_post_id = fetch_highest_processed_post_id.to_i
     puts "Highest processed post_id: #{highest_processed_post_id}"
 
 # OVERRIDE........
 #   Attachment example:  highest_processed_post_id = 1359862 
+    highest_processed_post_id = 8179621
+    puts "OVERRIDE Highest processed post_id: #{highest_processed_post_id}"
 
     # Execute the query to get all posts ordered by post_id
-    execute_query("SELECT post_id, user_id_fk, forum_id_fk, post_root_id, post_subject, post_time, post_message, post_father_id, post_likes, post_replies FROM gforum_Post ORDER BY post_id").each do |row|
+    execute_query("SELECT post_id, user_id_fk, forum_id_fk, post_root_id, post_subject, post_time, post_message, post_father_id, post_replies FROM gforum_Post WHERE post_id > #{highest_processed_post_id} ORDER BY post_id").each do |row|
       post_id = row['post_id'].to_i
 
      # Skip posts that have already been processed
@@ -1557,11 +1572,21 @@ class GossamerForumsImporter < ImportScripts::Base
 #      next if post_id < highest_old_post_id
       next if post_id <= highest_processed_post_id
 
-      puts "Processing post_id #{row['post_id']}  post_id  #{row['post_root_id']}  post_subject/title  #{row['post_subject']}  forum_id_fk/category_id  #{row['forum_id_fk']}"
+      puts "Processing post_id #{row['post_id']}  post_root_id  #{row['post_root_id']}  post_subject/title  #{row['post_subject']}  forum_id_fk (category_id)  #{row['forum_id_fk']}"
 #       discourse_user_id = @user_id_map[row['user_id_fk']]
 
-      # Fetch the Discourse user and category IP mappings
+      # Fetch the Discourse user_id based on Gossamer data and mapping
       discourse_user_id = fetch_user_id_mapping(row['user_id_fk'])
+
+       # Check to be certain user has not been deleted, etc.
+      if discourse_user_id.nil? || discourse_user_id == 0
+        puts "discourse_user_id is NIL/ZERO for post_id #{row['post_id']}"
+        discourse_former_user = User.find_by(username: 'Former_User')
+        discourse_user_id = discourse_former_user.id
+        puts "discourse_user_id is NOW Former_User id #{discourse_user_id} for post_id #{row['post_id']}"
+      end
+
+      # Fetch the Discourse category_id based on Gossamer data and mapping
       discourse_category_id = fetch_category_id_mapping(row['forum_id_fk'])
       puts "discourse_user_id  #{discourse_user_id}  discourse_category_id  #{discourse_category_id}"
       next unless discourse_user_id && discourse_category_id
@@ -1611,11 +1636,14 @@ class GossamerForumsImporter < ImportScripts::Base
                 end
               end
             end
+
+            current_topic_id = topic.id
+
             # Track last post time and user for the topic
 #            topic_last_post_time[topic.id] = Time.at(row['post_time'])
 #            topic_last_post_user[topic.id] = discourse_user_id
-            update_db_topic_last_post_time(topic.id, Time.at(row['post_time']).to_i)
+            update_db_topic_last_post_time(current_topic_id, Time.at(row['post_time']).to_i)
-            update_db_topic_last_post_user(topic.id, discourse_user_id)
+            update_db_topic_last_post_user(current_topic_id, discourse_user_id)
 
             # Increment the count of the number of topics created by each user
 #            user_topic_count[discourse_user_id] += 1
@@ -1634,26 +1662,26 @@ class GossamerForumsImporter < ImportScripts::Base
             # Sanitize the post message
             sanitized_post_message = sanitize_post_message(row['post_message'])
 
-            puts "CREATE POST  topic.id #{topic.id}  discourse_user_id #{discourse_user_id}"
+            puts "CREATE TOPIC POST  current_topic_id #{current_topic_id}  discourse_user_id #{discourse_user_id}"
 
             # Increment the number of posts in the given topic.
 #            topic_post_numbers[topic.id] += 1
-            post_number = fetch_db_topic_post_numbers(topic.id).to_i + 1
+            post_number = fetch_db_topic_post_numbers(current_topic_id).to_i + 1
-            update_db_topic_post_numbers(topic.id, post_number)
+            update_db_topic_post_numbers(current_topic_id, post_number)
 
             # Create the initial post in the topic
             post = Post.create!(
-              topic_id: topic.id,
+              topic_id: current_topic_id,
               user_id: discourse_user_id,
-#              raw: import_attachments(row['post_message'], row['post_id']),
-#              raw: row['post_message'] || "",
               raw: sanitized_post_message,
               created_at: Time.at(row['post_time']),
               updated_at: Time.at(row['post_time']),
-              like_count: row['post_likes'] || 0,
               reads: post_views || 0,
               post_number: post_number
             )
+#              like_count: row['post_likes'] || 0,
+#              raw: import_attachments(row['post_message'], row['post_id']),
+#              raw: row['post_message'] || "",
 #              post_number: topic_post_numbers[topic.id]
             post.custom_fields['original_gossamer_id'] = row['post_id']
             post.save!
@@ -1661,11 +1689,11 @@ class GossamerForumsImporter < ImportScripts::Base
             # Track the number of posts in the topic and by the user
 #            topic_post_count[topic.id] += 1
 #            user_post_count[discourse_user_id] += 1
-            update_db_topic_post_count(topic.id, fetch_db_topic_post_count(topic.id).to_i + 1)
+            update_db_topic_post_count(current_topic_id, fetch_db_topic_post_count(current_topic_id).to_i + 1)
             update_db_user_post_count(discourse_user_id, fetch_db_user_post_count(discourse_user_id).to_i + 1)
 
             # Handle attachments for the post
-            handle_post_attachments(row['post_id'], post, discourse_user_id)
+            handle_post_attachments(row['post_id'], post, discourse_user_id, @mysql_client)
   
             # Create URL mappings
 #            old_url = "https://old/forum/#{row['forum_name']}/topics/#{row['post_id']}"
@@ -1705,24 +1733,42 @@ class GossamerForumsImporter < ImportScripts::Base
 
 #            topic_post_numbers[topic_id] += 1
             post_number = fetch_db_topic_post_numbers(topic_id).to_i + 1
-            update_db_topic_post_numbers(topic_id, post_number)
+
+            puts "CREATE REPLY in topic_id #{topic_id} for reply post_id #{row['post_id']} with post_number #{post_number}"
 
             # Create the post in the existing topic
-            post = Post.create!(
+            post_created = false
-              topic_id: topic_id,
+            while ! post_created
-              user_id: discourse_user_id,
+              begin
-#              raw: import_attachments(row['post_message'], row['post_id']),
+                post = Post.create!(
-#              raw: row['post_message'] || "",
+                  topic_id: topic_id,
-              raw: sanitized_post_message,
+                  user_id: discourse_user_id,
-              created_at: Time.at(row['post_time']),
+                  raw: sanitized_post_message,
-              updated_at: Time.at(row['post_time']),
+                  created_at: Time.at(row['post_time']),
-              reply_to_post_number: reply_to_post_number,
+                  updated_at: Time.at(row['post_time']),
-              like_count: row['post_replies'] || 0,
+                  reads: post_views || 0,
-              reads: post_views || fetch_db_topic_post_count(topic_id).to_i,
+                  post_number: post_number
-              post_number: post_number
+                )
-            )
+#                  raw: import_attachments(row['post_message'], row['post_id']),
-            post.custom_fields['original_gossamer_id'] = row['post_id']
+#                  raw: row['post_message'] || "",
-            post.save!
+#                  reply_to_post_number: reply_to_post_number,
+#                  like_count: row['post_replies'] || 0,
+#                  reads: post_views || fetch_db_topic_post_count(topic_id).to_i,
+                post.custom_fields['original_gossamer_id'] = row['post_id']
+                post.save!
+
+                post_created = true
+
+              rescue ActiveRecord::RecordNotUnique => e
+                if e.message.include?("duplicate key value violates unique constraint")
+                  post_number = post_number + 1
+                else
+                  raise e
+                end
+              end
+            end
+
+            update_db_topic_post_numbers(topic_id, post_number)
 
             # Track the number of posts in the topic and by the user
 #            topic_post_count[topic_id] += 1
@@ -1741,7 +1787,7 @@ class GossamerForumsImporter < ImportScripts::Base
             end
 
             # Handle attachments for the post
-            handle_post_attachments(row['post_id'], post, discourse_user_id)
+            handle_post_attachments(row['post_id'], post, discourse_user_id, @mysql_client)
         
             # Update the highest processed post_id
             puts "Updated highest processed post_id #{post_id}"
@@ -1772,17 +1818,55 @@ class GossamerForumsImporter < ImportScripts::Base
 #    end
     @db.execute("SELECT * FROM topic_last_post_time").each do |row|
       topic_id, last_post_time = row
-# Validation error:     Topic.find(topic_id).update!(
-      topic = Topic.find(topic_id)
 
-      # Ensure we are only updating necessary fields
+      begin
-      topic.update_columns(
+        # Validation error:     Topic.find(topic_id).update!(
-        updated_at: Time.at(last_post_time),
+        topic = Topic.find(topic_id)
-        posts_count: fetch_db_topic_post_count(topic_id).to_i,
+  
-        last_posted_at: Time.at(last_post_time),
+        # Ensure we are only updating necessary fields
-        bumped_at: Time.at(last_post_time),
+        topic.update_columns(
-        last_post_user_id: fetch_db_topic_last_post_user(topic_id).to_i
+          updated_at: Time.at(last_post_time),
-      )
+          posts_count: fetch_db_topic_post_count(topic_id).to_i,
+          last_posted_at: Time.at(last_post_time),
+          bumped_at: Time.at(last_post_time),
+          last_post_user_id: fetch_db_topic_last_post_user(topic_id).to_i
+        )
+      rescue ActiveRecord::RecordNotFound
+        puts "WARNING: Could not find Topic with id=#{topic_id}. Skipping..."
+      end
+    end
+  end
+
+  def update_topic_stats_final
+    # Update topics with the correct last post time, post count, and last post user
+    puts "Update topics with the correct last post time, post count, and last post user"
+  
+    @db.execute("SELECT * FROM topic_last_post_time_final").each do |row|
+      topic_id, last_post_time = row
+      puts "update_topic_stats_final topic_id #{topic_id}"
+  
+      begin
+        topic = Topic.find(topic_id)
+  
+        # Calculate the new values based on the given conditions
+        new_updated_at = Time.at(last_post_time)
+        new_posts_count = topic.posts_count + fetch_db_topic_post_count(topic_id).to_i
+        new_last_posted_at = Time.at(last_post_time)
+        new_bumped_at = Time.at(last_post_time)
+        new_last_post_user_id = fetch_db_topic_last_post_user(topic_id).to_i
+  
+        # Update the topic with the calculated values
+        topic.update_columns(
+          updated_at: new_updated_at,
+          posts_count: new_posts_count,
+          last_posted_at: new_last_posted_at,
+          bumped_at: new_bumped_at,
+          last_post_user_id: new_last_post_user_id
+        )
+  
+      rescue ActiveRecord::RecordNotFound
+        puts "WARNING: Could not find Topic with id=#{topic_id}. Skipping..."
+      end
     end
   end
 
@@ -1797,46 +1881,154 @@ class GossamerForumsImporter < ImportScripts::Base
 #      user = User.find(user_id)
 #      user.update!(post_count: count)
 #    end
-    @db.execute("SELECT * FROM user_topic_count").each do |row|
+    @db.execute("SELECT * FROM user_topic_count_final").each do |row|
       user_id, count = row
 #      user = User.find(user_id)
 #      user.update!(topic_count: count)
       puts "update_user_stats  user_id  #{user_id}  topic_count  #{count}"
-      user_stat = UserStat.find_or_initialize_by(user_id: user_id)
+      begin
-      user_stat.update_columns(topic_count: count)
+        user_stat = UserStat.find_or_initialize_by(user_id: user_id)
+        user_stat.update_columns(topic_count: count)
+      rescue
+        puts "WARNING: Could not find User with id=#{user_id}. Skipping..."
+      end
     end
   
-    @db.execute("SELECT * FROM user_post_count").each do |row|
+    @db.execute("SELECT * FROM user_post_count_final").each do |row|
       user_id, count = row
 #      user = User.find(user_id)
 #      user.update!(post_count: count)
       puts "update_user_stats  user_id  #{user_id}  post_count  #{count}"
-      user_stat = UserStat.find_or_initialize_by(user_id: user_id)
+      begin
-      user_stat.update_columns(post_count: count)
+        user_stat = UserStat.find_or_initialize_by(user_id: user_id)
+        user_stat.update_columns(post_count: count)
 
-      # Determine the new Trust Level based on post_count
+        # Fetch the current user and check if Trust Level needs updating
-      user = User.find(user_id)
+        new_trust_level = case count
-      new_trust_level = case count
+          when 0..29 then 1   # basic user
-                  when 0..29 then 1   # basic user
+          else 2              # member, regular reserved for now.
-                  else 2              # member, regular reserved for now.
+#          when 3..50 then 2  # member
-#                  when 3..50 then 2  # member
+#          else 3             # regular or above when 51..100
-#                  else 3             # regular or above when 51..100
+        end
-                  end
+        user = User.find(user_id)
+        current_trust_level = user.trust_level || 1 # default to 1 if not set
 
-      # Fetch the current user and check if Trust Level needs updating
+        # Only update trust level if the new level is higher than the current one
-      user = User.find(user_id)
+        if new_trust_level != current_trust_level
-      current_trust_level = user.trust_level || 1 # default to 1 if not set
+          user.update!(trust_level: new_trust_level)
-
+          puts "update_user_stats  user_id  #{user_id}  trust_level was #{current_trust_level}, now updated to #{new_trust_level}"
-      # Only update trust level if the new level is higher than the current one
+        else
-      if new_trust_level > current_trust_level
+          puts "update_user_stats  user_id  #{user_id}  trust_level remains at #{current_trust_level}"
-        user.update!(trust_level: new_trust_level)
+        end
-        puts "update_user_stats  user_id  #{user_id}  trust_level updated to #{new_trust_level}"
+      rescue
-      else
+        puts "WARNING: Could not find or modify User with id=#{user_id}. Skipping..."
-        puts "update_user_stats  user_id  #{user_id}  trust_level remains at #{current_trust_level}"
       end
     end
   end
 
+  def update_user_stats_finalfinal
+    @db.execute("SELECT * FROM user_post_count_final").each do |row|
+      user_id, count = row
+      puts "update_user_stats  user_id  #{user_id}"
+      begin
+        user_stat = UserStat.find_or_initialize_by(user_id: user_id)
+
+        # Fetch the current post count for the user
+        post_count = user_stat.post_count.to_i
+##        user_stat.update_columns(post_count: new_post_count)
+
+        # Fetch the current user and check if Trust Level needs updating
+#        new_trust_level = case post_count
+#          when 0..29 then 1   # basic user
+#          else 2              # member, regular reserved for now.
+#          when 3..50 then 2  # member
+#          else 3             # regular or above when 51..100
+#        end
+
+        user = User.find(user_id)
+        current_trust_level = user.trust_level
+
+        if user.id == 101978
+          puts "USER ID 101978, so TRUST LEVEL 4"
+          new_trust_level = 4
+        elsif user.id == 2
+          puts "USER ID 2, so TRUST LEVEL 2"
+          new_trust_level = 2
+        elsif [111533, 161794, 163535, 172916, 177877, 183157, 189831, 191083, 192666, 195599, 198625, 200072, 200823, 201878, 201968].include?(user.id)
+          new_trust_level = 1
+        else
+          new_trust_level = 2
+        end
+
+        # Only update trust level if the new level is higher than the current one
+        if new_trust_level != current_trust_level
+          user.update!(trust_level: new_trust_level)
+          puts "update_user_stats  user_id  #{user_id}  trust_level was #{current_trust_level}, now updated to #{new_trust_level}"
+        else
+          puts "update_user_stats  user_id  #{user_id}  trust_level remains at #{current_trust_level}"
+        end
+      rescue
+        puts "WARNING: Could not find or modify User with id=#{user_id}:  #{e.message} ... Skipping..."
+        puts e.backtrace.join("\n")  # Print the full stack trace
+      end
+    end
+  end
+
+
+
+  def update_user_stats_final
+    # Update user profiles with the number of topics and posts created
+    puts "Update user profiles with the number of topics and posts created"
+    @db.execute("SELECT * FROM user_topic_count_final").each do |row|
+      user_id, count = row
+      puts "update_user_stats  user_id  #{user_id}  topic_count  #{count}"
+      begin
+        user_stat = UserStat.find_or_initialize_by(user_id: user_id)
+#        user_stat.update_columns(topic_count: count)
+
+        # Accumulate the current topic count with the new count
+        new_topic_count = user_stat.topic_count.to_i + count.to_i
+        user_stat.update_columns(topic_count: new_topic_count)
+      rescue
+        puts "WARNING: Could not find User with id=#{user_id}. Skipping..."
+      end
+    end
+
+    @db.execute("SELECT * FROM user_post_count_final").each do |row|
+      user_id, count = row
+      puts "update_user_stats  user_id  #{user_id}  post_count  #{count}"
+      begin
+        user_stat = UserStat.find_or_initialize_by(user_id: user_id)
+#        user_stat.update_columns(post_count: count)
+
+        # Accumulate the current post count with the new count
+        new_post_count = user_stat.post_count.to_i + count.to_i
+        user_stat.update_columns(post_count: new_post_count)
+
+        # Fetch the current user and check if Trust Level needs updating
+        new_trust_level = case count
+          when 0..29 then 1   # basic user
+          else 2              # member, regular reserved for now.
+#          when 3..50 then 2  # member
+#          else 3             # regular or above when 51..100
+        end
+        user = User.find(user_id)
+        current_trust_level = user.trust_level || 1 # default to 1 if not set
+
+        # Only update trust level if the new level is higher than the current one
+        if new_trust_level != current_trust_level
+          user.update!(trust_level: new_trust_level)
+          puts "update_user_stats  user_id  #{user_id}  trust_level was #{current_trust_level}, now updated to #{new_trust_level}"
+        else
+          puts "update_user_stats  user_id  #{user_id}  trust_level remains at #{current_trust_level}"
+        end
+      rescue
+        puts "WARNING: Could not find or modify User with id=#{user_id}. Skipping..."
+      end
+    end
+  end
+
+
 
   # Import personal messages from gforum_Message table (both inbox and sent messages)
   def import_personal_messages
@@ -1846,17 +2038,20 @@ class GossamerForumsImporter < ImportScripts::Base
     highest_processed_personal_id = fetch_highest_processed_personal_id.to_i
     puts "Highest processed personal_id: #{highest_processed_personal_id}"
 
-# OVERRIDE - to speed getting to problem msg
+# OVERRIDE - to speed getting to lower limit msg
 # # #    highest_processed_personal_id = 1543840
     puts "Highest processed personal_id override: #{highest_processed_personal_id}"
 
-    execute_query("SELECT * FROM gforum_Message").each do |row|
+##    execute_query("SELECT * FROM gforum_Message").each do |row|
+    execute_query("SELECT * FROM gforum_Message WHERE msg_id > #{highest_processed_personal_id}").each do |row|
       begin
         msg_id = row['msg_id'].to_i
         puts "msg_id  #{msg_id}"
   
         # Skip posts that have already been processed
-        next if msg_id <= highest_processed_personal_id
+# OVERRIDE - upper limit msg
+# # #    next if msg_id <= highest_processed_personal_id || msg_id > 342443
+##        next if msg_id <= highest_processed_personal_id
   
         from_user_id = fetch_user_id_mapping(row['from_user_id_fk'])
         to_user_id = fetch_user_id_mapping(row['to_user_id_fk'])
@@ -1938,8 +2133,9 @@ class GossamerForumsImporter < ImportScripts::Base
   
           # Update the topic's last reply information
           topic.update!(
-            last_posted_at: post.updated_at,
+            last_posted_at: post.created_at,
-            last_post_user_id: post.user_id
+            last_post_user_id: post.user_id,
+            bumped_at: post.created_at
           )
   
           update_highest_processed_personal_id(msg_id)
@@ -1953,6 +2149,26 @@ class GossamerForumsImporter < ImportScripts::Base
     end
   end
 
+  def update_existing_personal_message_activity
+    puts "Updating Activity field for existing private / personal message topics..."
+  
+    private_message_topics = Topic.where(archetype: Archetype.private_message)
+  
+    private_message_topics.each do |topic|
+      last_post = topic.posts.order(created_at: :desc).first
+
+      if last_post
+        topic.update_columns(
+          last_posted_at: last_post.created_at,
+          last_post_user_id: last_post.user_id,
+          bumped_at: last_post.created_at
+        )
+        puts "Updated topic ID #{topic.id} with last activity at #{last_post.updated_at}"
+      else
+        puts "No posts found for topic ID #{topic.id}; skipping update."
+      end
+    end
+  end
 
   # Main method to perform the import
   def perform_import
@@ -1984,25 +2200,30 @@ class GossamerForumsImporter < ImportScripts::Base
 
     puts "Starting Gossamer Forums import... #{timestamp}"
 
-#    add_former_user
+###    add_former_user
-#    import_users
+###    import_users
 
-#    generate_user_id_mapping
+###    generate_user_id_mapping
-#    export_username_mapping_to_csv("/bitnami/discourse/sqlite/gossamer-migration-username-mapping#{timestamp}")
+###    export_username_mapping_to_csv("/bitnami/discourse/sqlite/gossamer-migration-username-mapping#{timestamp}")
 
-    set_user_bio_images
+###    set_user_bio_images
 
-#    import_categories
+###    import_categories
 
-#######    import_topics_and_posts_with_attachments
+###    import_topics_and_posts
 ###    threaded_topic_import
 
-    update_topic_stats
+###    update_topic_stats
-    update_user_stats
+###    update_user_stats
-    export_url_mapping_to_csv("/bitnami/discourse/sqlite/gossamer-migration-url-mapping#{timestamp}")
-    create_nginx_rewrite_rules("/bitnami/discourse/sqlite/gossamer-redirects#{timestamp}.conf")
 
-    import_personal_messages
+###    update_topic_stats_final
+    update_user_stats_finalfinal
+
+###    export_url_mapping_to_csv("/bitnami/discourse/sqlite/gossamer-migration-url-mapping#{timestamp}")
+###    export_nginx_rewrite_rules("/bitnami/discourse/sqlite/gossamer-redirects#{timestamp}.conf")
+
+####     update_existing_personal_message_activity
+###    import_personal_messages
 
     puts "Gossamer Forums import complete! #{timestamp}"
   end

softdeldeletedposts.rb

@@ -0,0 +1,207 @@
+# Federated Computer, Inc.
+# David Sainty <saint@federated.computer>  2024 A.D.
+# Gossamer Threads to Discourse -- Soft Deletion of Marked-As-Deleted Posts
+# v0.3 Ready for actual soft-delete run on live-prod
+
+require 'mysql2'
+require 'active_record'
+
+# require 'concurrent-ruby'
+require File.expand_path("../../../../config/environment", __FILE__)
+require File.expand_path("../../../../script/import_scripts/base", __FILE__)
+
+class GossamerForumsSoftDelDeletedPosts < ImportScripts::Base
+  def initialize
+    super
+      begin
+        # Initialize MySQL client to connect to Gossamer Forums database
+        @mysql_client = Mysql2::Client.new(
+          host: "slowtwitch.northend.network",
+          username: "admin",
+          password: "yxnh93Ybbz2Nm8#mp28zCVv",
+          database: "slowtwitch"
+        )
+      rescue Mysql2::Error => e
+        puts "Error connecting to MySQL: #{e.message}"
+        exit 1
+      end
+  end
+
+  # Define a method to find a post by custom field
+  def find_post_by_custom_field(post_id)
+    puts "SoftDelDeletedPosts: Searching for post with original_gossamer_id: #{post_id}"
+  
+    post_custom_field = PostCustomField.find_by(name: 'original_gossamer_id', value: post_id.to_s)
+  
+    if post_custom_field
+      post = post_custom_field.post
+      puts "SoftDelDeletedPosts: Found post with id: #{post.id}"
+      post
+    else
+      puts "SoftDelDeletedPosts: No post found with original_gossamer_id: #{post_id}"
+      nil
+    end
+  end
+
+  # Soft delete the post, whether topic post (OP) or reply post
+  def soft_delete_post(post, deleted_by_id)
+    if post.deleted_at.nil?
+#      # Get the post's owner and topic information
+#      user = User.find_by(id: post.user_id)  # Fetch user details
+#      topic = Topic.find_by(id: post.topic_id)  # Fetch topic details
+  
+#      # Display post, user, and topic information
+#      puts "DELETING POST: SoftDelDeletedPosts: Soft deleting post with id: #{post.id}"
+#      puts " - Post owner: #{user.username} (ID: #{user.id})"
+#      puts " - Topic title: '#{topic.title}' (Topic ID: #{topic.id})"
+  
+      # Perform the soft delete by updating the fields
+##      post.deleted_at = Time.now
+##      post.deleted_by_id = deleted_by_id
+##      post.save
+      puts "... actual deleted_at step"
+      post.update(deleted_at: Time.now, deleted_by_id: deleted_by_id)
+    else
+      puts "SoftDelDeletedPosts: Post with id: #{post.id} is already soft deleted."
+    end
+  end
+
+  # Define the method to delete posts based on the Gossamer Forums flag
+  def soft_del_deleted_posts_from_gossamer_with_user(username)
+ 
+    # Query the user ID from the legacy MySQL database
+    user_result = @mysql_client.query("SELECT user_id FROM gforum_User WHERE user_username = '#{username}' LIMIT 1")
+    user_id_row = user_result.first
+    if user_id_row.nil?
+      puts "DiscourseDeletedPosts: No user found with username: #{username}"
+      return
+    end
+    user_id = user_id_row['user_id']
+
+    # Find all posts marked as deleted by the given user
+    posts_result = @mysql_client.query("SELECT post_id FROM gforum_Post WHERE post_deleted = 1 AND user_id_fk = #{user_id}")
+
+    # "admin" user is ID 1, so set as the one performing the deletions
+    deleted_by_id = 1
+
+    posts_result.each do |legacy_post|
+      post_id = legacy_post['post_id']
+
+      # Look for the post in Discourse by custom field
+      post = find_post_by_custom_field(post_id)
+  
+      if post
+        # Soft delete only the individual post (whether topic post / OP or a reply post)
+        soft_delete_post(post, deleted_by_id)
+      end
+    end
+  end
+
+  # Define the method to reverse (soft undelete) the soft deletion
+  def soft_undelete_post(post)
+    if post.deleted_at.present?
+      # Get the post's owner and topic information
+      user = User.find_by(id: post.user_id)  # Fetch user details
+      topic = Topic.find_by(id: post.topic_id)  # Fetch topic details
+  
+      # Display post, user, and topic information
+      puts "RESTORING POST: SoftDelDeletedPosts: Soft undeleting post with id: #{post.id}"
+      puts " - Post owner: #{user.username} (ID: #{user.id})"
+      puts " - Topic title: '#{topic.title}' (Topic ID: #{topic.id})"
+  
+      # Reverse the soft delete by setting deleted_at and deleted_by_id to nil
+      ### post.update(deleted_at: nil, deleted_by_id: nil)
+    else
+      puts "SoftDelDeletedPosts: Post with id: #{post.id} is not soft deleted."
+    end
+  end
+
+  # Define the method to reverse the soft deletion for all legacy posts
+  def soft_undelete_all_deleted_posts
+    # Query the legacy database for posts marked as deleted
+    posts_result = @mysql_client.query("SELECT post_id FROM gforum_Post WHERE post_deleted = 1")
+  
+    posts_result.each do |legacy_post|
+      post_id = legacy_post['post_id']
+  
+      # Look for the post in Discourse by custom field
+      post = find_post_by_custom_field(post_id)
+  
+      if post
+        # Reverse the soft deletion (restore the post)
+        soft_undelete_post(post)
+      end
+    end
+  end
+
+  # Define the method to delete posts based on the Gossamer Forums flag
+  def soft_del_all_deleted_posts
+
+    # Find all posts marked as deleted by the given user
+    posts_result = @mysql_client.query("SELECT post_id FROM gforum_Post WHERE post_deleted = 1")
+
+    # "admin" user is ID 1, so set as the one performing the deletions
+    deleted_by_id = 1
+
+    posts_result.each do |legacy_post|
+      post_id = legacy_post['post_id']
+
+#      # Look for the post in Discourse by custom field
+#      post = find_post_by_custom_field(post_id)
+#
+#      if post
+#        # Soft delete only the individual post (whether topic post / OP or a reply post)
+#        soft_delete_post(post, deleted_by_id)
+#      end
+      soft_del_all_deleted_posts_by_custom_field(post_id, deleted_by_id)
+    end
+  end
+
+  # Define a method to find all posts by the original Gossamer Forums post ID and soft delete them
+  def soft_del_all_deleted_posts_by_custom_field(post_id, deleted_by_id)
+    puts "SoftDelDeletedPosts: Searching for all posts with original_gossamer_id: #{post_id}"
+  
+    # Find all PostCustomField records with the given Gossamer Forums post_id
+    post_custom_fields = PostCustomField.where(name: 'original_gossamer_id', value: post_id.to_s)
+  
+    if post_custom_fields.any?
+      post_custom_fields.each do |post_custom_field|
+        post = post_custom_field.post
+  
+        if post
+          puts "DELETING POST: SoftDelDeletedPosts: Soft deleting post with id: #{post.id}  and user_id: #{post.user_id}  and topic_id: #{post.topic_id}"
+          # Get the post's owner and topic information
+#          user = User.find_by(id: post.user_id)  # Fetch user details
+#          topic = Topic.find_by(id: post.topic_id)  # Fetch topic details
+  
+          # Display post, user, and topic information
+#          puts " - Post owner: #{user.username} (ID: #{user.id})"
+#          puts " - Topic title: '#{topic.title}' (Topic ID: #{topic.id})"
+  
+          # Soft delete the post if not already deleted
+          soft_delete_post(post, deleted_by_id)
+        else
+          puts "SoftDelDeletedPosts: Found PostCustomField with no corresponding post (ID: #{post_custom_field.id})"
+        end
+      end
+    else
+      puts "SoftDelDeletedPosts: No PostCustomField records found with original_gossamer_id: #{post_id}"
+    end
+  end
+
+
+  def perform_deleted_soft_del
+    puts "Soft Del Deleted Posts beginning!"
+
+    ## soft_del_deleted_posts_from_gossamer_with_user('spudone')
+
+    ## soft_undelete_all_deleted_posts
+
+    soft_del_all_deleted_posts
+
+    puts "Soft Del Deleted Posts complete!"
+  end
+end
+
+GossamerForumsSoftDelDeletedPosts.new.perform_deleted_soft_del
+