From f18016750cd368c4846f66e3e4f8115cc081fb6a Mon Sep 17 00:00:00 2001 From: saint Date: Mon, 9 Dec 2024 18:29:54 +1100 Subject: [PATCH] Finish taking the multiple issues which Discourse had with the very large number of posts in this particular topic --- goss-restorethread.rb | 56 ++++++++++++++++++++++--------------------- 1 file changed, 29 insertions(+), 27 deletions(-) diff --git a/goss-restorethread.rb b/goss-restorethread.rb index 05e6eeb..83fbdaa 100644 --- a/goss-restorethread.rb +++ b/goss-restorethread.rb @@ -114,42 +114,42 @@ class GossamerForumsImporter < ImportScripts::Base SQL # POST IMPORT - For each topic, the time of the last / most recent post/reply @db.execute <<-SQL - CREATE TABLE IF NOT EXISTS topic_last_post_time_final7 ( + CREATE TABLE IF NOT EXISTS topic_last_post_time_final8 ( topic_id INTEGER PRIMARY KEY, last_post_time INTEGER ); SQL # POST IMPORT - For each topic, increment post_count as we add posts. @db.execute <<-SQL - CREATE TABLE IF NOT EXISTS topic_post_count_final7 ( + CREATE TABLE IF NOT EXISTS topic_post_count_final8 ( topic_id INTEGER PRIMARY KEY, post_count INTEGER DEFAULT 0 ); SQL # POST IMPORT - For each user (_id), increment topic_count as we add topics (to see total topics per user) @db.execute <<-SQL - CREATE TABLE IF NOT EXISTS user_topic_count_final7 ( + CREATE TABLE IF NOT EXISTS user_topic_count_final8 ( user_id INTEGER PRIMARY KEY, topic_count INTEGER DEFAULT 0 ); SQL # POST IMPORT - For each user (_id), increment post_count as we add posts (to see total posts per user) @db.execute <<-SQL - CREATE TABLE IF NOT EXISTS user_post_count_final7 ( + CREATE TABLE IF NOT EXISTS user_post_count_final8 ( user_id INTEGER PRIMARY KEY, post_count INTEGER DEFAULT 0 ); SQL # POST IMPORT - For each topic, the user_id for the last poster / replier @db.execute <<-SQL - CREATE TABLE IF NOT EXISTS topic_last_post_user_final7 ( + CREATE TABLE IF NOT EXISTS topic_last_post_user_final8 ( topic_id INTEGER PRIMARY KEY, user_id INTEGER ); SQL # POST IMPORT - The number of posts in a given topic, incremented as we add a new reply post to a topic. @db.execute <<-SQL - CREATE TABLE IF NOT EXISTS topic_post_numbers_final7 ( + CREATE TABLE IF NOT EXISTS topic_post_numbers_final8 ( topic_id INTEGER PRIMARY KEY, post_number INTEGER DEFAULT 0 ); @@ -253,55 +253,55 @@ class GossamerForumsImporter < ImportScripts::Base end def fetch_db_topic_last_post_time(topic_id) - @db.get_first_value("SELECT last_post_time FROM topic_last_post_time_final7 WHERE topic_id = ?", [topic_id]) + @db.get_first_value("SELECT last_post_time FROM topic_last_post_time_final8 WHERE topic_id = ?", [topic_id]) end def fetch_db_topic_last_post_user(topic_id) - @db.get_first_value("SELECT user_id FROM topic_last_post_user_final7 WHERE topic_id = ?", [topic_id]) + @db.get_first_value("SELECT user_id FROM topic_last_post_user_final8 WHERE topic_id = ?", [topic_id]) end def fetch_db_topic_post_count(topic_id) - @db.get_first_value("SELECT post_count FROM topic_post_count_final7 WHERE topic_id = ?", [topic_id]) + @db.get_first_value("SELECT post_count FROM topic_post_count_final8 WHERE topic_id = ?", [topic_id]) end def fetch_db_user_topic_count(user_id) - @db.get_first_value("SELECT topic_count FROM user_topic_count_final7 WHERE user_id = ?", [user_id]) + @db.get_first_value("SELECT topic_count FROM user_topic_count_final8 WHERE user_id = ?", [user_id]) end def fetch_db_user_post_count(user_id) - @db.get_first_value("SELECT post_count FROM user_post_count_final7 WHERE user_id = ?", [user_id]) + @db.get_first_value("SELECT post_count FROM user_post_count_final8 WHERE user_id = ?", [user_id]) end def fetch_db_topic_post_numbers(topic_id) - # @db.get_first_value "SELECT post_number FROM topic_post_numbers_final7 WHERE topic_id = ?", topic_id - @db.get_first_value("SELECT post_number FROM topic_post_numbers_final7 WHERE topic_id = ?", [topic_id]) + # @db.get_first_value "SELECT post_number FROM topic_post_numbers_final8 WHERE topic_id = ?", topic_id + @db.get_first_value("SELECT post_number FROM topic_post_numbers_final8 WHERE topic_id = ?", [topic_id]) end def update_db_topic_last_post_time(topic_id, last_post_time) # puts "zzzzzzzzz1 #{topic_id} #{last_post_time}" - # @db.execute "INSERT OR REPLACE INTO topic_last_post_time_final7 (topic_id, last_post_time) VALUES (?, ?)", topic_id, last_post_time - @db.execute("INSERT OR REPLACE INTO topic_last_post_time_final7 (topic_id, last_post_time) VALUES (?, ?)", [topic_id, last_post_time]) + # @db.execute "INSERT OR REPLACE INTO topic_last_post_time_final8 (topic_id, last_post_time) VALUES (?, ?)", topic_id, last_post_time + @db.execute("INSERT OR REPLACE INTO topic_last_post_time_final8 (topic_id, last_post_time) VALUES (?, ?)", [topic_id, last_post_time]) end def update_db_topic_last_post_user(topic_id, user_id) - @db.execute("INSERT OR REPLACE INTO topic_last_post_user_final7 (topic_id, user_id) VALUES (?, ?)", [topic_id, user_id]) + @db.execute("INSERT OR REPLACE INTO topic_last_post_user_final8 (topic_id, user_id) VALUES (?, ?)", [topic_id, user_id]) end def update_db_topic_post_count(topic_id, post_count) - @db.execute("INSERT OR REPLACE INTO topic_post_count_final7 (topic_id, post_count) VALUES (?, ?)", [topic_id, post_count]) + @db.execute("INSERT OR REPLACE INTO topic_post_count_final8 (topic_id, post_count) VALUES (?, ?)", [topic_id, post_count]) end def update_db_user_topic_count(user_id, topic_count) - @db.execute("INSERT OR REPLACE INTO user_topic_count_final7 (user_id, topic_count) VALUES (?, ?)", [user_id, topic_count]) + @db.execute("INSERT OR REPLACE INTO user_topic_count_final8 (user_id, topic_count) VALUES (?, ?)", [user_id, topic_count]) end def update_db_user_post_count(user_id, post_count) - @db.execute("INSERT OR REPLACE INTO user_post_count_final7 (user_id, post_count) VALUES (?, ?)", [user_id, post_count]) + @db.execute("INSERT OR REPLACE INTO user_post_count_final8 (user_id, post_count) VALUES (?, ?)", [user_id, post_count]) end def update_db_topic_post_numbers(topic_id, post_number) - @db.execute("INSERT OR REPLACE INTO topic_post_numbers_final7 (topic_id, post_number) VALUES (?, ?)", [topic_id, post_number]) + @db.execute("INSERT OR REPLACE INTO topic_post_numbers_final8 (topic_id, post_number) VALUES (?, ?)", [topic_id, post_number]) end # Fetch the highest processed post_id from the highest_processed_post_id table @@ -1057,8 +1057,10 @@ class GossamerForumsImporter < ImportScripts::Base # sanitized_message.gsub!(/!\[data:image\/[^\]]+\]\([^)]+\)/, '') # sanitized_message.gsub!(/!\[[^\]]*\]\(https?:\/\/data:image\/[a-zA-Z]+;base64,[^)]+\)/, '') # sanitized_message.gsub!(/!\[.*?\]\((?:http|https):\/\/data:image\/[^;]+;base64,[^)]+\)/i, '') - sanitized_message.gsub!(/!\[(?:http|https):\/\/data:image\/[^]]+\]\((?:http|https):\/\/data:image\/[^)]+\)/, '') - sanitized_message.gsub!(/!\[(https?:\/\/data:image\/[^]]+)\]\((https?:\/\/data:image\/[^)]+)\)/, '') + # sanitized_message.gsub!(/!\[(?:http|https):\/\/data:image\/[^]]+\]\((?:http|https):\/\/data:image\/[^)]+\)/, '') + # sanitized_message.gsub!(/!\[(https?:\/\/data:image\/[^]]+)\]\((https?:\/\/data:image\/[^)]+)\)/, '') + sanitized_message.gsub!(/!\[(https?:\/\/data:image\/[^\]]+)\]\((https?:\/\/data:image\/[^\)]+)\)/) + # Ensure sentence structure unless sanitized_message.match?(/[.!?]\s|[.!?]$/) @@ -1856,13 +1858,13 @@ class GossamerForumsImporter < ImportScripts::Base end end - def update_topic_stats_final7 + def update_topic_stats_final8 # Update topics with the correct last post time, post count, and last post user puts "Update topics with the correct last post time, post count, and last post user" - @db.execute("SELECT * FROM topic_last_post_time_final7").each do |row| + @db.execute("SELECT * FROM topic_last_post_time_final8").each do |row| topic_id, last_post_time = row - puts "update_topic_stats_final7 topic_id #{topic_id}" + puts "update_topic_stats_final8 topic_id #{topic_id}" begin topic = Topic.find(topic_id) @@ -2239,8 +2241,8 @@ class GossamerForumsImporter < ImportScripts::Base ### update_topic_stats_final ### update_user_stats_finalfinal -# update_topic_stats_final7 -# update_user_stats_finalfinal7 +# update_topic_stats_final8 +# update_user_stats_finalfinal8 ### export_url_mapping_to_csv("/bitnami/discourse/sqlite/gossamer-migration-url-mapping#{timestamp}") ### export_nginx_rewrite_rules("/bitnami/discourse/sqlite/gossamer-redirects#{timestamp}.conf")