saint/discourse-import_scripts
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Finish takling the multiple issues which Discourse had with the very large number of posts in this particular topic

Browse Source
This commit is contained in:
David Sainty 2024-12-07 17:37:43 +11:00
parent ce865db09e
commit a46d02ede8
1 changed files with 27 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
goss-restorethread.rb
View File

@ -114,42 +114,42 @@ class GossamerForumsImporter < ImportScripts::Base
SQL SQL
# POST IMPORT - For each topic, the time of the last / most recent post/reply # POST IMPORT - For each topic, the time of the last / most recent post/reply
@db.execute <<-SQL @db.execute <<-SQL
CREATE TABLE IF NOT EXISTS topic_last_post_time_final3 ( CREATE TABLE IF NOT EXISTS topic_last_post_time_final4 (
topic_id INTEGER PRIMARY KEY, topic_id INTEGER PRIMARY KEY,
last_post_time INTEGER last_post_time INTEGER
); );
SQL SQL
# POST IMPORT - For each topic, increment post_count as we add posts. # POST IMPORT - For each topic, increment post_count as we add posts.
@db.execute <<-SQL @db.execute <<-SQL
CREATE TABLE IF NOT EXISTS topic_post_count_final3 ( CREATE TABLE IF NOT EXISTS topic_post_count_final4 (
topic_id INTEGER PRIMARY KEY, topic_id INTEGER PRIMARY KEY,
post_count INTEGER DEFAULT 0 post_count INTEGER DEFAULT 0
); );
SQL SQL
# POST IMPORT - For each user (_id), increment topic_count as we add topics (to see total topics per user) # POST IMPORT - For each user (_id), increment topic_count as we add topics (to see total topics per user)
@db.execute <<-SQL @db.execute <<-SQL
CREATE TABLE IF NOT EXISTS user_topic_count_final3 ( CREATE TABLE IF NOT EXISTS user_topic_count_final4 (
user_id INTEGER PRIMARY KEY, user_id INTEGER PRIMARY KEY,
topic_count INTEGER DEFAULT 0 topic_count INTEGER DEFAULT 0
); );
SQL SQL
# POST IMPORT - For each user (_id), increment post_count as we add posts (to see total posts per user) # POST IMPORT - For each user (_id), increment post_count as we add posts (to see total posts per user)
@db.execute <<-SQL @db.execute <<-SQL
CREATE TABLE IF NOT EXISTS user_post_count_final3 ( CREATE TABLE IF NOT EXISTS user_post_count_final4 (
user_id INTEGER PRIMARY KEY, user_id INTEGER PRIMARY KEY,
post_count INTEGER DEFAULT 0 post_count INTEGER DEFAULT 0
); );
SQL SQL
# POST IMPORT - For each topic, the user_id for the last poster / replier # POST IMPORT - For each topic, the user_id for the last poster / replier
@db.execute <<-SQL @db.execute <<-SQL
CREATE TABLE IF NOT EXISTS topic_last_post_user_final3 ( CREATE TABLE IF NOT EXISTS topic_last_post_user_final4 (
topic_id INTEGER PRIMARY KEY, topic_id INTEGER PRIMARY KEY,
user_id INTEGER user_id INTEGER
); );
SQL SQL
# POST IMPORT - The number of posts in a given topic, incremented as we add a new reply post to a topic. # POST IMPORT - The number of posts in a given topic, incremented as we add a new reply post to a topic.
@db.execute <<-SQL @db.execute <<-SQL
CREATE TABLE IF NOT EXISTS topic_post_numbers_final3 ( CREATE TABLE IF NOT EXISTS topic_post_numbers_final4 (
topic_id INTEGER PRIMARY KEY, topic_id INTEGER PRIMARY KEY,
post_number INTEGER DEFAULT 0 post_number INTEGER DEFAULT 0
); );
@ -253,55 +253,55 @@ class GossamerForumsImporter < ImportScripts::Base
end end
def fetch_db_topic_last_post_time(topic_id) def fetch_db_topic_last_post_time(topic_id)
@db.get_first_value("SELECT last_post_time FROM topic_last_post_time_final3 WHERE topic_id = ?", [topic_id]) @db.get_first_value("SELECT last_post_time FROM topic_last_post_time_final4 WHERE topic_id = ?", [topic_id])
end end
def fetch_db_topic_last_post_user(topic_id) def fetch_db_topic_last_post_user(topic_id)
@db.get_first_value("SELECT user_id FROM topic_last_post_user_final3 WHERE topic_id = ?", [topic_id]) @db.get_first_value("SELECT user_id FROM topic_last_post_user_final4 WHERE topic_id = ?", [topic_id])
end end
def fetch_db_topic_post_count(topic_id) def fetch_db_topic_post_count(topic_id)
@db.get_first_value("SELECT post_count FROM topic_post_count_final3 WHERE topic_id = ?", [topic_id]) @db.get_first_value("SELECT post_count FROM topic_post_count_final4 WHERE topic_id = ?", [topic_id])
end end
def fetch_db_user_topic_count(user_id) def fetch_db_user_topic_count(user_id)
@db.get_first_value("SELECT topic_count FROM user_topic_count_final3 WHERE user_id = ?", [user_id]) @db.get_first_value("SELECT topic_count FROM user_topic_count_final4 WHERE user_id = ?", [user_id])
end end
def fetch_db_user_post_count(user_id) def fetch_db_user_post_count(user_id)
@db.get_first_value("SELECT post_count FROM user_post_count_final3 WHERE user_id = ?", [user_id]) @db.get_first_value("SELECT post_count FROM user_post_count_final4 WHERE user_id = ?", [user_id])
end end
def fetch_db_topic_post_numbers(topic_id) def fetch_db_topic_post_numbers(topic_id)
# @db.get_first_value "SELECT post_number FROM topic_post_numbers_final3 WHERE topic_id = ?", topic_id # @db.get_first_value "SELECT post_number FROM topic_post_numbers_final4 WHERE topic_id = ?", topic_id
@db.get_first_value("SELECT post_number FROM topic_post_numbers_final3 WHERE topic_id = ?", [topic_id]) @db.get_first_value("SELECT post_number FROM topic_post_numbers_final4 WHERE topic_id = ?", [topic_id])
end end
def update_db_topic_last_post_time(topic_id, last_post_time) def update_db_topic_last_post_time(topic_id, last_post_time)
# puts "zzzzzzzzz1 #{topic_id} #{last_post_time}" # puts "zzzzzzzzz1 #{topic_id} #{last_post_time}"
# @db.execute "INSERT OR REPLACE INTO topic_last_post_time_final3 (topic_id, last_post_time) VALUES (?, ?)", topic_id, last_post_time # @db.execute "INSERT OR REPLACE INTO topic_last_post_time_final4 (topic_id, last_post_time) VALUES (?, ?)", topic_id, last_post_time
@db.execute("INSERT OR REPLACE INTO topic_last_post_time_final3 (topic_id, last_post_time) VALUES (?, ?)", [topic_id, last_post_time]) @db.execute("INSERT OR REPLACE INTO topic_last_post_time_final4 (topic_id, last_post_time) VALUES (?, ?)", [topic_id, last_post_time])
end end
def update_db_topic_last_post_user(topic_id, user_id) def update_db_topic_last_post_user(topic_id, user_id)
@db.execute("INSERT OR REPLACE INTO topic_last_post_user_final3 (topic_id, user_id) VALUES (?, ?)", [topic_id, user_id]) @db.execute("INSERT OR REPLACE INTO topic_last_post_user_final4 (topic_id, user_id) VALUES (?, ?)", [topic_id, user_id])
end end
def update_db_topic_post_count(topic_id, post_count) def update_db_topic_post_count(topic_id, post_count)
@db.execute("INSERT OR REPLACE INTO topic_post_count_final3 (topic_id, post_count) VALUES (?, ?)", [topic_id, post_count]) @db.execute("INSERT OR REPLACE INTO topic_post_count_final4 (topic_id, post_count) VALUES (?, ?)", [topic_id, post_count])
end end
def update_db_user_topic_count(user_id, topic_count) def update_db_user_topic_count(user_id, topic_count)
@db.execute("INSERT OR REPLACE INTO user_topic_count_final3 (user_id, topic_count) VALUES (?, ?)", [user_id, topic_count]) @db.execute("INSERT OR REPLACE INTO user_topic_count_final4 (user_id, topic_count) VALUES (?, ?)", [user_id, topic_count])
end end
def update_db_user_post_count(user_id, post_count) def update_db_user_post_count(user_id, post_count)
@db.execute("INSERT OR REPLACE INTO user_post_count_final3 (user_id, post_count) VALUES (?, ?)", [user_id, post_count]) @db.execute("INSERT OR REPLACE INTO user_post_count_final4 (user_id, post_count) VALUES (?, ?)", [user_id, post_count])
end end
def update_db_topic_post_numbers(topic_id, post_number) def update_db_topic_post_numbers(topic_id, post_number)
@db.execute("INSERT OR REPLACE INTO topic_post_numbers_final3 (topic_id, post_number) VALUES (?, ?)", [topic_id, post_number]) @db.execute("INSERT OR REPLACE INTO topic_post_numbers_final4 (topic_id, post_number) VALUES (?, ?)", [topic_id, post_number])
end end
# Fetch the highest processed post_id from the highest_processed_post_id table # Fetch the highest processed post_id from the highest_processed_post_id table
@ -1054,7 +1054,8 @@ class GossamerForumsImporter < ImportScripts::Base
sanitized_message.gsub!(/[\r\n]*\[inline.*\][\r\n]*/, '') sanitized_message.gsub!(/[\r\n]*\[inline.*\][\r\n]*/, '')
# Remove ![http://data:image/, etc. as this is not supported. # Remove ![http://data:image/, etc. as this is not supported.
sanitized_message.gsub!(/!\[data:image\/[^\]]+\]\([^)]+\)/, '') # sanitized_message.gsub!(/!\[data:image\/[^\]]+\]\([^)]+\)/, '')
sanitized_message.gsub!(/!\[[^\]]*\]\(https?:\/\/data:image\/[a-zA-Z]+;base64,[^)]+\)/, '')
# Ensure sentence structure # Ensure sentence structure
unless sanitized_message.match?(/[.!?]\s|[.!?]$/) unless sanitized_message.match?(/[.!?]\s|[.!?]$/)
@ -1852,13 +1853,13 @@ class GossamerForumsImporter < ImportScripts::Base
end end
end end
def update_topic_stats_final3 def update_topic_stats_final4
# Update topics with the correct last post time, post count, and last post user # Update topics with the correct last post time, post count, and last post user
puts "Update topics with the correct last post time, post count, and last post user" puts "Update topics with the correct last post time, post count, and last post user"
@db.execute("SELECT * FROM topic_last_post_time_final3").each do |row| @db.execute("SELECT * FROM topic_last_post_time_final4").each do |row|
topic_id, last_post_time = row topic_id, last_post_time = row
puts "update_topic_stats_final3 topic_id #{topic_id}" puts "update_topic_stats_final4 topic_id #{topic_id}"
begin begin
topic = Topic.find(topic_id) topic = Topic.find(topic_id)
@ -2235,8 +2236,8 @@ class GossamerForumsImporter < ImportScripts::Base
### update_topic_stats_final ### update_topic_stats_final
### update_user_stats_finalfinal ### update_user_stats_finalfinal
# update_topic_stats_final3 # update_topic_stats_final4
# update_user_stats_finalfinal3 # update_user_stats_finalfinal4
### export_url_mapping_to_csv("/bitnami/discourse/sqlite/gossamer-migration-url-mapping#{timestamp}") ### export_url_mapping_to_csv("/bitnami/discourse/sqlite/gossamer-migration-url-mapping#{timestamp}")
### export_nginx_rewrite_rules("/bitnami/discourse/sqlite/gossamer-redirects#{timestamp}.conf") ### export_nginx_rewrite_rules("/bitnami/discourse/sqlite/gossamer-redirects#{timestamp}.conf")