From 09bf758f42c229d2d00b3b04caf3325381ab1af9 Mon Sep 17 00:00:00 2001 From: saint Date: Sat, 7 Dec 2024 17:49:51 +1100 Subject: [PATCH] Finish takling the multiple issues which Discourse had with the very large number of posts in this particular topic --- goss-restorethread.rb | 53 ++++++++++++++++++++++--------------------- 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/goss-restorethread.rb b/goss-restorethread.rb index 4558e2d..7eb5e99 100644 --- a/goss-restorethread.rb +++ b/goss-restorethread.rb @@ -114,42 +114,42 @@ class GossamerForumsImporter < ImportScripts::Base SQL # POST IMPORT - For each topic, the time of the last / most recent post/reply @db.execute <<-SQL - CREATE TABLE IF NOT EXISTS topic_last_post_time_final4 ( + CREATE TABLE IF NOT EXISTS topic_last_post_time_final5 ( topic_id INTEGER PRIMARY KEY, last_post_time INTEGER ); SQL # POST IMPORT - For each topic, increment post_count as we add posts. @db.execute <<-SQL - CREATE TABLE IF NOT EXISTS topic_post_count_final4 ( + CREATE TABLE IF NOT EXISTS topic_post_count_final5 ( topic_id INTEGER PRIMARY KEY, post_count INTEGER DEFAULT 0 ); SQL # POST IMPORT - For each user (_id), increment topic_count as we add topics (to see total topics per user) @db.execute <<-SQL - CREATE TABLE IF NOT EXISTS user_topic_count_final4 ( + CREATE TABLE IF NOT EXISTS user_topic_count_final5 ( user_id INTEGER PRIMARY KEY, topic_count INTEGER DEFAULT 0 ); SQL # POST IMPORT - For each user (_id), increment post_count as we add posts (to see total posts per user) @db.execute <<-SQL - CREATE TABLE IF NOT EXISTS user_post_count_final4 ( + CREATE TABLE IF NOT EXISTS user_post_count_final5 ( user_id INTEGER PRIMARY KEY, post_count INTEGER DEFAULT 0 ); SQL # POST IMPORT - For each topic, the user_id for the last poster / replier @db.execute <<-SQL - CREATE TABLE IF NOT EXISTS topic_last_post_user_final4 ( + CREATE TABLE IF NOT EXISTS topic_last_post_user_final5 ( topic_id INTEGER PRIMARY KEY, user_id INTEGER ); SQL # POST IMPORT - The number of posts in a given topic, incremented as we add a new reply post to a topic. @db.execute <<-SQL - CREATE TABLE IF NOT EXISTS topic_post_numbers_final4 ( + CREATE TABLE IF NOT EXISTS topic_post_numbers_final5 ( topic_id INTEGER PRIMARY KEY, post_number INTEGER DEFAULT 0 ); @@ -253,55 +253,55 @@ class GossamerForumsImporter < ImportScripts::Base end def fetch_db_topic_last_post_time(topic_id) - @db.get_first_value("SELECT last_post_time FROM topic_last_post_time_final4 WHERE topic_id = ?", [topic_id]) + @db.get_first_value("SELECT last_post_time FROM topic_last_post_time_final5 WHERE topic_id = ?", [topic_id]) end def fetch_db_topic_last_post_user(topic_id) - @db.get_first_value("SELECT user_id FROM topic_last_post_user_final4 WHERE topic_id = ?", [topic_id]) + @db.get_first_value("SELECT user_id FROM topic_last_post_user_final5 WHERE topic_id = ?", [topic_id]) end def fetch_db_topic_post_count(topic_id) - @db.get_first_value("SELECT post_count FROM topic_post_count_final4 WHERE topic_id = ?", [topic_id]) + @db.get_first_value("SELECT post_count FROM topic_post_count_final5 WHERE topic_id = ?", [topic_id]) end def fetch_db_user_topic_count(user_id) - @db.get_first_value("SELECT topic_count FROM user_topic_count_final4 WHERE user_id = ?", [user_id]) + @db.get_first_value("SELECT topic_count FROM user_topic_count_final5 WHERE user_id = ?", [user_id]) end def fetch_db_user_post_count(user_id) - @db.get_first_value("SELECT post_count FROM user_post_count_final4 WHERE user_id = ?", [user_id]) + @db.get_first_value("SELECT post_count FROM user_post_count_final5 WHERE user_id = ?", [user_id]) end def fetch_db_topic_post_numbers(topic_id) - # @db.get_first_value "SELECT post_number FROM topic_post_numbers_final4 WHERE topic_id = ?", topic_id - @db.get_first_value("SELECT post_number FROM topic_post_numbers_final4 WHERE topic_id = ?", [topic_id]) + # @db.get_first_value "SELECT post_number FROM topic_post_numbers_final5 WHERE topic_id = ?", topic_id + @db.get_first_value("SELECT post_number FROM topic_post_numbers_final5 WHERE topic_id = ?", [topic_id]) end def update_db_topic_last_post_time(topic_id, last_post_time) # puts "zzzzzzzzz1 #{topic_id} #{last_post_time}" - # @db.execute "INSERT OR REPLACE INTO topic_last_post_time_final4 (topic_id, last_post_time) VALUES (?, ?)", topic_id, last_post_time - @db.execute("INSERT OR REPLACE INTO topic_last_post_time_final4 (topic_id, last_post_time) VALUES (?, ?)", [topic_id, last_post_time]) + # @db.execute "INSERT OR REPLACE INTO topic_last_post_time_final5 (topic_id, last_post_time) VALUES (?, ?)", topic_id, last_post_time + @db.execute("INSERT OR REPLACE INTO topic_last_post_time_final5 (topic_id, last_post_time) VALUES (?, ?)", [topic_id, last_post_time]) end def update_db_topic_last_post_user(topic_id, user_id) - @db.execute("INSERT OR REPLACE INTO topic_last_post_user_final4 (topic_id, user_id) VALUES (?, ?)", [topic_id, user_id]) + @db.execute("INSERT OR REPLACE INTO topic_last_post_user_final5 (topic_id, user_id) VALUES (?, ?)", [topic_id, user_id]) end def update_db_topic_post_count(topic_id, post_count) - @db.execute("INSERT OR REPLACE INTO topic_post_count_final4 (topic_id, post_count) VALUES (?, ?)", [topic_id, post_count]) + @db.execute("INSERT OR REPLACE INTO topic_post_count_final5 (topic_id, post_count) VALUES (?, ?)", [topic_id, post_count]) end def update_db_user_topic_count(user_id, topic_count) - @db.execute("INSERT OR REPLACE INTO user_topic_count_final4 (user_id, topic_count) VALUES (?, ?)", [user_id, topic_count]) + @db.execute("INSERT OR REPLACE INTO user_topic_count_final5 (user_id, topic_count) VALUES (?, ?)", [user_id, topic_count]) end def update_db_user_post_count(user_id, post_count) - @db.execute("INSERT OR REPLACE INTO user_post_count_final4 (user_id, post_count) VALUES (?, ?)", [user_id, post_count]) + @db.execute("INSERT OR REPLACE INTO user_post_count_final5 (user_id, post_count) VALUES (?, ?)", [user_id, post_count]) end def update_db_topic_post_numbers(topic_id, post_number) - @db.execute("INSERT OR REPLACE INTO topic_post_numbers_final4 (topic_id, post_number) VALUES (?, ?)", [topic_id, post_number]) + @db.execute("INSERT OR REPLACE INTO topic_post_numbers_final5 (topic_id, post_number) VALUES (?, ?)", [topic_id, post_number]) end # Fetch the highest processed post_id from the highest_processed_post_id table @@ -1055,7 +1055,8 @@ class GossamerForumsImporter < ImportScripts::Base # Remove ![http://data:image/, etc. as this is not supported. # sanitized_message.gsub!(/!\[data:image\/[^\]]+\]\([^)]+\)/, '') - sanitized_message.gsub!(/!\[[^\]]*\]\(https?:\/\/data:image\/[a-zA-Z]+;base64,[^)]+\)/, '') + # sanitized_message.gsub!(/!\[[^\]]*\]\(https?:\/\/data:image\/[a-zA-Z]+;base64,[^)]+\)/, '') + sanitized_message.gsub!(/!\[.*?\]\((?:http|https):\/\/data:image\/[^;]+;base64,[^)]+\)/i, '') # Ensure sentence structure unless sanitized_message.match?(/[.!?]\s|[.!?]$/) @@ -1853,13 +1854,13 @@ class GossamerForumsImporter < ImportScripts::Base end end - def update_topic_stats_final4 + def update_topic_stats_final5 # Update topics with the correct last post time, post count, and last post user puts "Update topics with the correct last post time, post count, and last post user" - @db.execute("SELECT * FROM topic_last_post_time_final4").each do |row| + @db.execute("SELECT * FROM topic_last_post_time_final5").each do |row| topic_id, last_post_time = row - puts "update_topic_stats_final4 topic_id #{topic_id}" + puts "update_topic_stats_final5 topic_id #{topic_id}" begin topic = Topic.find(topic_id) @@ -2236,8 +2237,8 @@ class GossamerForumsImporter < ImportScripts::Base ### update_topic_stats_final ### update_user_stats_finalfinal -# update_topic_stats_final4 -# update_user_stats_finalfinal4 +# update_topic_stats_final5 +# update_user_stats_finalfinal5 ### export_url_mapping_to_csv("/bitnami/discourse/sqlite/gossamer-migration-url-mapping#{timestamp}") ### export_nginx_rewrite_rules("/bitnami/discourse/sqlite/gossamer-redirects#{timestamp}.conf")