Completed goss-restorethread.rb ready for use on Prod for topics needing (re)import / restoration.

goss-restorethread.rb

@@ -114,42 +114,42 @@ class GossamerForumsImporter < ImportScripts::Base
     SQL
     # POST IMPORT - For each topic, the time of the last / most recent post/reply
     @db.execute <<-SQL
-      CREATE TABLE IF NOT EXISTS topic_last_post_time_final2 (
+      CREATE TABLE IF NOT EXISTS topic_last_post_time_final11 (
         topic_id INTEGER PRIMARY KEY,
         last_post_time INTEGER
       );
     SQL
     # POST IMPORT - For each topic, increment post_count as we add posts.
     @db.execute <<-SQL
-      CREATE TABLE IF NOT EXISTS topic_post_count_final2 (
+      CREATE TABLE IF NOT EXISTS topic_post_count_final11 (
         topic_id INTEGER PRIMARY KEY,
         post_count INTEGER DEFAULT 0
       );
     SQL
     # POST IMPORT  - For each user (_id), increment topic_count as we add topics (to see total topics per user) 
     @db.execute <<-SQL
-      CREATE TABLE IF NOT EXISTS user_topic_count_final2 (
+      CREATE TABLE IF NOT EXISTS user_topic_count_final11 (
         user_id INTEGER PRIMARY KEY,
         topic_count INTEGER DEFAULT 0
       );
     SQL
     # POST IMPORT - For each user (_id), increment post_count as we add posts (to see total posts per user)
     @db.execute <<-SQL
-      CREATE TABLE IF NOT EXISTS user_post_count_final2 (
+      CREATE TABLE IF NOT EXISTS user_post_count_final11 (
         user_id INTEGER PRIMARY KEY,
         post_count INTEGER DEFAULT 0
       );
     SQL
     # POST IMPORT - For each topic, the user_id for the last poster / replier
     @db.execute <<-SQL
-      CREATE TABLE IF NOT EXISTS topic_last_post_user_final2 (
+      CREATE TABLE IF NOT EXISTS topic_last_post_user_final11 (
         topic_id INTEGER PRIMARY KEY,
         user_id INTEGER
       );
     SQL
     # POST IMPORT - The number of posts in a given topic, incremented as we add a new reply post to a topic.
     @db.execute <<-SQL
-      CREATE TABLE IF NOT EXISTS topic_post_numbers_final2 (
+      CREATE TABLE IF NOT EXISTS topic_post_numbers_final11 (
         topic_id INTEGER PRIMARY KEY,
         post_number INTEGER DEFAULT 0
       );
@@ -253,55 +253,55 @@ class GossamerForumsImporter < ImportScripts::Base
   end
 
   def fetch_db_topic_last_post_time(topic_id)
-    @db.get_first_value("SELECT last_post_time FROM topic_last_post_time_final2 WHERE topic_id = ?", [topic_id])
+    @db.get_first_value("SELECT last_post_time FROM topic_last_post_time_final11 WHERE topic_id = ?", [topic_id])
   end
   
   def fetch_db_topic_last_post_user(topic_id)
-    @db.get_first_value("SELECT user_id FROM topic_last_post_user_final2 WHERE topic_id = ?", [topic_id])
+    @db.get_first_value("SELECT user_id FROM topic_last_post_user_final11 WHERE topic_id = ?", [topic_id])
   end
   
   def fetch_db_topic_post_count(topic_id)
-    @db.get_first_value("SELECT post_count FROM topic_post_count_final2 WHERE topic_id = ?", [topic_id])
+    @db.get_first_value("SELECT post_count FROM topic_post_count_final11 WHERE topic_id = ?", [topic_id])
   end
   
   def fetch_db_user_topic_count(user_id)
-    @db.get_first_value("SELECT topic_count FROM user_topic_count_final2 WHERE user_id = ?", [user_id])
+    @db.get_first_value("SELECT topic_count FROM user_topic_count_final11 WHERE user_id = ?", [user_id])
   end
   
   def fetch_db_user_post_count(user_id)
-    @db.get_first_value("SELECT post_count FROM user_post_count_final2 WHERE user_id = ?", [user_id])
+    @db.get_first_value("SELECT post_count FROM user_post_count_final11 WHERE user_id = ?", [user_id])
   end
   
   def fetch_db_topic_post_numbers(topic_id)
-    # @db.get_first_value "SELECT post_number FROM topic_post_numbers_final2 WHERE topic_id = ?", topic_id
+    # @db.get_first_value "SELECT post_number FROM topic_post_numbers_final11 WHERE topic_id = ?", topic_id
-    @db.get_first_value("SELECT post_number FROM topic_post_numbers_final2 WHERE topic_id = ?", [topic_id])
+    @db.get_first_value("SELECT post_number FROM topic_post_numbers_final11 WHERE topic_id = ?", [topic_id])
 
   end
   
   def update_db_topic_last_post_time(topic_id, last_post_time)
     # puts "zzzzzzzzz1 #{topic_id}  #{last_post_time}"
-    # @db.execute "INSERT OR REPLACE INTO topic_last_post_time_final2 (topic_id, last_post_time) VALUES (?, ?)", topic_id, last_post_time
+    # @db.execute "INSERT OR REPLACE INTO topic_last_post_time_final11 (topic_id, last_post_time) VALUES (?, ?)", topic_id, last_post_time
-    @db.execute("INSERT OR REPLACE INTO topic_last_post_time_final2 (topic_id, last_post_time) VALUES (?, ?)", [topic_id, last_post_time])
+    @db.execute("INSERT OR REPLACE INTO topic_last_post_time_final11 (topic_id, last_post_time) VALUES (?, ?)", [topic_id, last_post_time])
   end
   
   def update_db_topic_last_post_user(topic_id, user_id)
-    @db.execute("INSERT OR REPLACE INTO topic_last_post_user_final2 (topic_id, user_id) VALUES (?, ?)", [topic_id, user_id])
+    @db.execute("INSERT OR REPLACE INTO topic_last_post_user_final11 (topic_id, user_id) VALUES (?, ?)", [topic_id, user_id])
   end
   
   def update_db_topic_post_count(topic_id, post_count)
-    @db.execute("INSERT OR REPLACE INTO topic_post_count_final2 (topic_id, post_count) VALUES (?, ?)", [topic_id, post_count])
+    @db.execute("INSERT OR REPLACE INTO topic_post_count_final11 (topic_id, post_count) VALUES (?, ?)", [topic_id, post_count])
   end
   
   def update_db_user_topic_count(user_id, topic_count)
-    @db.execute("INSERT OR REPLACE INTO user_topic_count_final2 (user_id, topic_count) VALUES (?, ?)", [user_id, topic_count])
+    @db.execute("INSERT OR REPLACE INTO user_topic_count_final11 (user_id, topic_count) VALUES (?, ?)", [user_id, topic_count])
   end
   
   def update_db_user_post_count(user_id, post_count)
-    @db.execute("INSERT OR REPLACE INTO user_post_count_final2 (user_id, post_count) VALUES (?, ?)", [user_id, post_count])
+    @db.execute("INSERT OR REPLACE INTO user_post_count_final11 (user_id, post_count) VALUES (?, ?)", [user_id, post_count])
   end
   
   def update_db_topic_post_numbers(topic_id, post_number)
-    @db.execute("INSERT OR REPLACE INTO topic_post_numbers_final2 (topic_id, post_number) VALUES (?, ?)", [topic_id, post_number])
+    @db.execute("INSERT OR REPLACE INTO topic_post_numbers_final11 (topic_id, post_number) VALUES (?, ?)", [topic_id, post_number])
   end
 
   # Fetch the highest processed post_id from the highest_processed_post_id table
@@ -1049,12 +1049,19 @@ class GossamerForumsImporter < ImportScripts::Base
 #    sanitized_message.sub!(/\n?\[signature\]\n?\z/, '')
 #    sanitized_message.gsub(/\n?\[signature\]\n?/, '')
     sanitized_message.gsub!(/[\r\n]*\[signature\][\r\n]*/, '')
+
+    # Remove [inline] style inlining.
     sanitized_message.gsub!(/[\r\n]*\[inline.*\][\r\n]*/, '')
 
-    # Ensure minimum length
+    # Remove ![http://data:image/, etc. as this is not supported.
-    if sanitized_message.strip.empty? || sanitized_message.length < 5
+    # sanitized_message.gsub!(/!\[data:image\/[^\]]+\]\([^)]+\)/, '')
-      sanitized_message = "Empty post contents."
+    # sanitized_message.gsub!(/!\[[^\]]*\]\(https?:\/\/data:image\/[a-zA-Z]+;base64,[^)]+\)/, '')
-    end
+    # sanitized_message.gsub!(/!\[.*?\]\((?:http|https):\/\/data:image\/[^;]+;base64,[^)]+\)/i, '')
+    # sanitized_message.gsub!(/!\[(?:http|https):\/\/data:image\/[^]]+\]\((?:http|https):\/\/data:image\/[^)]+\)/, '')
+    # sanitized_message.gsub!(/!\[(https?:\/\/data:image\/[^]]+)\]\((https?:\/\/data:image\/[^)]+)\)/, '')
+    # sanitized_message.gsub!(/!\[(https?:\/\/data:image\/[^\]]+)\]\((https?:\/\/data:image\/[^\)]+)\)/)
+    # sanitized_message.gsub!(/!\[https?:\/\/data:image[^\]]*\]\(https?:\/\/data:image[^\)]*\)/, '')
+    sanitized_message.gsub!(/\[img\]https?:\/\/data:image[^\[]*\[\/img\]/, '')
 
     # Ensure sentence structure
     unless sanitized_message.match?(/[.!?]\s|[.!?]$/)
@@ -1079,6 +1086,11 @@ class GossamerForumsImporter < ImportScripts::Base
     # Convert inline image syntax from `!(url)` to `![url](url)`
     sanitized_message.gsub!(/!\((http[s]?:\/\/[^\)]+)\)/, '![\1](\1)')
 
+    # Ensure minimum length
+    if sanitized_message.strip.empty? || sanitized_message.length < 5
+      sanitized_message = "Empty post contents."
+    end
+
     sanitized_message
   end
 
@@ -1847,13 +1859,14 @@ class GossamerForumsImporter < ImportScripts::Base
     end
   end
 
-  def update_topic_stats_final2
+  # Update topic stats for the restored topic.
+  def update_topic_stats_final11
     # Update topics with the correct last post time, post count, and last post user
     puts "Update topics with the correct last post time, post count, and last post user"
   
-    @db.execute("SELECT * FROM topic_last_post_time_final2").each do |row|
+    @db.execute("SELECT * FROM topic_last_post_time_final11").each do |row|
       topic_id, last_post_time = row
-      puts "update_topic_stats_final2 topic_id #{topic_id}"
+      puts "update_topic_stats_final11 topic_id #{topic_id}"
   
       begin
         topic = Topic.find(topic_id)
@@ -1880,6 +1893,42 @@ class GossamerForumsImporter < ImportScripts::Base
     end
   end
 
+  # Update (increment) user stats, but don't touch trust levels, after topic restoration.
+  def update_user_stats_final11
+    # Update user profiles with the number of topics and posts created
+    puts "Update user profiles with the number of topics and posts created"
+    @db.execute("SELECT * FROM user_topic_count_final11").each do |row|
+      user_id, count = row
+      puts "update_user_stats  user_id  #{user_id}  topic_count  #{count}"
+      begin
+        user_stat = UserStat.find_or_initialize_by(user_id: user_id)
+#        user_stat.update_columns(topic_count: count)
+
+        # Accumulate the current topic count with the new count
+        new_topic_count = user_stat.topic_count.to_i + count.to_i
+        user_stat.update_columns(topic_count: new_topic_count)
+      rescue
+        puts "WARNING: Could not find User with id=#{user_id}. Skipping..."
+      end
+    end
+
+    @db.execute("SELECT * FROM user_post_count_final11").each do |row|
+      user_id, count = row
+      puts "update_user_stats  user_id  #{user_id}  post_count  #{count}"
+      begin
+        user_stat = UserStat.find_or_initialize_by(user_id: user_id)
+#        user_stat.update_columns(post_count: count)
+
+        # Accumulate the current post count with the new count
+        new_post_count = user_stat.post_count.to_i + count.to_i
+        user_stat.update_columns(post_count: new_post_count)
+      rescue
+        puts "WARNING: Could not find or modify User with id=#{user_id}. Skipping..."
+      end
+    end
+  end
+
+
   def update_user_stats
     # Update user profiles with the number of topics and posts created
     puts "Update user profiles with the number of topics and posts created"
@@ -1936,6 +1985,7 @@ class GossamerForumsImporter < ImportScripts::Base
     end
   end
 
+  # Correct user stats, notably the trust levels, after the final post-migration import run.
   def update_user_stats_finalfinal
     @db.execute("SELECT * FROM user_post_count_final").each do |row|
       user_id, count = row
@@ -1985,7 +2035,7 @@ class GossamerForumsImporter < ImportScripts::Base
   end
 
 
-
+  # Set user stats, including the trust levels (albeit incorrectly...)  after the final post-migration import run.
   def update_user_stats_final
     # Update user profiles with the number of topics and posts created
     puts "Update user profiles with the number of topics and posts created"
@@ -2230,8 +2280,8 @@ class GossamerForumsImporter < ImportScripts::Base
 ###    update_topic_stats_final
 ###    update_user_stats_finalfinal
 
-#    update_topic_stats_final2
+    update_topic_stats_final11
-#    update_user_stats_finalfinal2
+    update_user_stats_final11
 
 ###    export_url_mapping_to_csv("/bitnami/discourse/sqlite/gossamer-migration-url-mapping#{timestamp}")
 ###    export_nginx_rewrite_rules("/bitnami/discourse/sqlite/gossamer-redirects#{timestamp}.conf")