From 9c0f2fa5ccef3bb6ead9bd4dedc361d80fbf339d Mon Sep 17 00:00:00 2001 From: saint Date: Mon, 9 Dec 2024 20:56:51 +1100 Subject: [PATCH] Finish taking the multiple issues which Discourse had with the very large number of posts in this particular topic --- goss-restorethread.rb | 56 +++++++++++++++++++++---------------------- 1 file changed, 27 insertions(+), 29 deletions(-) diff --git a/goss-restorethread.rb b/goss-restorethread.rb index c52364f..7e156fd 100644 --- a/goss-restorethread.rb +++ b/goss-restorethread.rb @@ -114,42 +114,42 @@ class GossamerForumsImporter < ImportScripts::Base SQL # POST IMPORT - For each topic, the time of the last / most recent post/reply @db.execute <<-SQL - CREATE TABLE IF NOT EXISTS topic_last_post_time_final9 ( + CREATE TABLE IF NOT EXISTS topic_last_post_time_final11 ( topic_id INTEGER PRIMARY KEY, last_post_time INTEGER ); SQL # POST IMPORT - For each topic, increment post_count as we add posts. @db.execute <<-SQL - CREATE TABLE IF NOT EXISTS topic_post_count_final9 ( + CREATE TABLE IF NOT EXISTS topic_post_count_final11 ( topic_id INTEGER PRIMARY KEY, post_count INTEGER DEFAULT 0 ); SQL # POST IMPORT - For each user (_id), increment topic_count as we add topics (to see total topics per user) @db.execute <<-SQL - CREATE TABLE IF NOT EXISTS user_topic_count_final9 ( + CREATE TABLE IF NOT EXISTS user_topic_count_final11 ( user_id INTEGER PRIMARY KEY, topic_count INTEGER DEFAULT 0 ); SQL # POST IMPORT - For each user (_id), increment post_count as we add posts (to see total posts per user) @db.execute <<-SQL - CREATE TABLE IF NOT EXISTS user_post_count_final9 ( + CREATE TABLE IF NOT EXISTS user_post_count_final11 ( user_id INTEGER PRIMARY KEY, post_count INTEGER DEFAULT 0 ); SQL # POST IMPORT - For each topic, the user_id for the last poster / replier @db.execute <<-SQL - CREATE TABLE IF NOT EXISTS topic_last_post_user_final9 ( + CREATE TABLE IF NOT EXISTS topic_last_post_user_final11 ( topic_id INTEGER PRIMARY KEY, user_id INTEGER ); SQL # POST IMPORT - The number of posts in a given topic, incremented as we add a new reply post to a topic. @db.execute <<-SQL - CREATE TABLE IF NOT EXISTS topic_post_numbers_final9 ( + CREATE TABLE IF NOT EXISTS topic_post_numbers_final11 ( topic_id INTEGER PRIMARY KEY, post_number INTEGER DEFAULT 0 ); @@ -253,55 +253,55 @@ class GossamerForumsImporter < ImportScripts::Base end def fetch_db_topic_last_post_time(topic_id) - @db.get_first_value("SELECT last_post_time FROM topic_last_post_time_final9 WHERE topic_id = ?", [topic_id]) + @db.get_first_value("SELECT last_post_time FROM topic_last_post_time_final11 WHERE topic_id = ?", [topic_id]) end def fetch_db_topic_last_post_user(topic_id) - @db.get_first_value("SELECT user_id FROM topic_last_post_user_final9 WHERE topic_id = ?", [topic_id]) + @db.get_first_value("SELECT user_id FROM topic_last_post_user_final11 WHERE topic_id = ?", [topic_id]) end def fetch_db_topic_post_count(topic_id) - @db.get_first_value("SELECT post_count FROM topic_post_count_final9 WHERE topic_id = ?", [topic_id]) + @db.get_first_value("SELECT post_count FROM topic_post_count_final11 WHERE topic_id = ?", [topic_id]) end def fetch_db_user_topic_count(user_id) - @db.get_first_value("SELECT topic_count FROM user_topic_count_final9 WHERE user_id = ?", [user_id]) + @db.get_first_value("SELECT topic_count FROM user_topic_count_final11 WHERE user_id = ?", [user_id]) end def fetch_db_user_post_count(user_id) - @db.get_first_value("SELECT post_count FROM user_post_count_final9 WHERE user_id = ?", [user_id]) + @db.get_first_value("SELECT post_count FROM user_post_count_final11 WHERE user_id = ?", [user_id]) end def fetch_db_topic_post_numbers(topic_id) - # @db.get_first_value "SELECT post_number FROM topic_post_numbers_final9 WHERE topic_id = ?", topic_id - @db.get_first_value("SELECT post_number FROM topic_post_numbers_final9 WHERE topic_id = ?", [topic_id]) + # @db.get_first_value "SELECT post_number FROM topic_post_numbers_final11 WHERE topic_id = ?", topic_id + @db.get_first_value("SELECT post_number FROM topic_post_numbers_final11 WHERE topic_id = ?", [topic_id]) end def update_db_topic_last_post_time(topic_id, last_post_time) # puts "zzzzzzzzz1 #{topic_id} #{last_post_time}" - # @db.execute "INSERT OR REPLACE INTO topic_last_post_time_final9 (topic_id, last_post_time) VALUES (?, ?)", topic_id, last_post_time - @db.execute("INSERT OR REPLACE INTO topic_last_post_time_final9 (topic_id, last_post_time) VALUES (?, ?)", [topic_id, last_post_time]) + # @db.execute "INSERT OR REPLACE INTO topic_last_post_time_final11 (topic_id, last_post_time) VALUES (?, ?)", topic_id, last_post_time + @db.execute("INSERT OR REPLACE INTO topic_last_post_time_final11 (topic_id, last_post_time) VALUES (?, ?)", [topic_id, last_post_time]) end def update_db_topic_last_post_user(topic_id, user_id) - @db.execute("INSERT OR REPLACE INTO topic_last_post_user_final9 (topic_id, user_id) VALUES (?, ?)", [topic_id, user_id]) + @db.execute("INSERT OR REPLACE INTO topic_last_post_user_final11 (topic_id, user_id) VALUES (?, ?)", [topic_id, user_id]) end def update_db_topic_post_count(topic_id, post_count) - @db.execute("INSERT OR REPLACE INTO topic_post_count_final9 (topic_id, post_count) VALUES (?, ?)", [topic_id, post_count]) + @db.execute("INSERT OR REPLACE INTO topic_post_count_final11 (topic_id, post_count) VALUES (?, ?)", [topic_id, post_count]) end def update_db_user_topic_count(user_id, topic_count) - @db.execute("INSERT OR REPLACE INTO user_topic_count_final9 (user_id, topic_count) VALUES (?, ?)", [user_id, topic_count]) + @db.execute("INSERT OR REPLACE INTO user_topic_count_final11 (user_id, topic_count) VALUES (?, ?)", [user_id, topic_count]) end def update_db_user_post_count(user_id, post_count) - @db.execute("INSERT OR REPLACE INTO user_post_count_final9 (user_id, post_count) VALUES (?, ?)", [user_id, post_count]) + @db.execute("INSERT OR REPLACE INTO user_post_count_final11 (user_id, post_count) VALUES (?, ?)", [user_id, post_count]) end def update_db_topic_post_numbers(topic_id, post_number) - @db.execute("INSERT OR REPLACE INTO topic_post_numbers_final9 (topic_id, post_number) VALUES (?, ?)", [topic_id, post_number]) + @db.execute("INSERT OR REPLACE INTO topic_post_numbers_final11 (topic_id, post_number) VALUES (?, ?)", [topic_id, post_number]) end # Fetch the highest processed post_id from the highest_processed_post_id table @@ -1060,10 +1060,8 @@ class GossamerForumsImporter < ImportScripts::Base # sanitized_message.gsub!(/!\[(?:http|https):\/\/data:image\/[^]]+\]\((?:http|https):\/\/data:image\/[^)]+\)/, '') # sanitized_message.gsub!(/!\[(https?:\/\/data:image\/[^]]+)\]\((https?:\/\/data:image\/[^)]+)\)/, '') # sanitized_message.gsub!(/!\[(https?:\/\/data:image\/[^\]]+)\]\((https?:\/\/data:image\/[^\)]+)\)/) - puts "Here 111 #{sanitized_message}" - sanitized_message.gsub!(/!\[https?:\/\/data:image[^\]]*\]\(https?:\/\/data:image[^\)]*\)/, '') - puts "Here 222 #{sanitized_message}" - + # sanitized_message.gsub!(/!\[https?:\/\/data:image[^\]]*\]\(https?:\/\/data:image[^\)]*\)/, '') + sanitized_message.gsub!(/\[img\]https?:\/\/data:image[^\[]*\[\/img\]/, '') # Ensure sentence structure unless sanitized_message.match?(/[.!?]\s|[.!?]$/) @@ -1861,13 +1859,13 @@ class GossamerForumsImporter < ImportScripts::Base end end - def update_topic_stats_final9 + def update_topic_stats_final11 # Update topics with the correct last post time, post count, and last post user puts "Update topics with the correct last post time, post count, and last post user" - @db.execute("SELECT * FROM topic_last_post_time_final9").each do |row| + @db.execute("SELECT * FROM topic_last_post_time_final11").each do |row| topic_id, last_post_time = row - puts "update_topic_stats_final9 topic_id #{topic_id}" + puts "update_topic_stats_final11 topic_id #{topic_id}" begin topic = Topic.find(topic_id) @@ -2244,8 +2242,8 @@ class GossamerForumsImporter < ImportScripts::Base ### update_topic_stats_final ### update_user_stats_finalfinal -# update_topic_stats_final9 -# update_user_stats_finalfinal9 +# update_topic_stats_final11 +# update_user_stats_finalfinal11 ### export_url_mapping_to_csv("/bitnami/discourse/sqlite/gossamer-migration-url-mapping#{timestamp}") ### export_nginx_rewrite_rules("/bitnami/discourse/sqlite/gossamer-redirects#{timestamp}.conf")