saint/discourse-import_scripts
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Finish taking the multiple issues which Discourse had with the very large number of posts in this particular topic

Browse Source
This commit is contained in:
David Sainty
2024-12-09 20:56:51 +11:00
parent 8787035865
commit 9c0f2fa5cc
1 changed files with 27 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

56
goss-restorethread.rb
View File

@@ -114,42 +114,42 @@ class GossamerForumsImporter < ImportScripts::Base
SQL SQL
# POST IMPORT - For each topic, the time of the last / most recent post/reply # POST IMPORT - For each topic, the time of the last / most recent post/reply
@db.execute <<-SQL @db.execute <<-SQL
CREATE TABLE IF NOT EXISTS topic_last_post_time_final9 ( CREATE TABLE IF NOT EXISTS topic_last_post_time_final11 (
topic_id INTEGER PRIMARY KEY, topic_id INTEGER PRIMARY KEY,
last_post_time INTEGER last_post_time INTEGER
); );
SQL SQL
# POST IMPORT - For each topic, increment post_count as we add posts. # POST IMPORT - For each topic, increment post_count as we add posts.
@db.execute <<-SQL @db.execute <<-SQL
CREATE TABLE IF NOT EXISTS topic_post_count_final9 ( CREATE TABLE IF NOT EXISTS topic_post_count_final11 (
topic_id INTEGER PRIMARY KEY, topic_id INTEGER PRIMARY KEY,
post_count INTEGER DEFAULT 0 post_count INTEGER DEFAULT 0
); );
SQL SQL
# POST IMPORT - For each user (_id), increment topic_count as we add topics (to see total topics per user) # POST IMPORT - For each user (_id), increment topic_count as we add topics (to see total topics per user)
@db.execute <<-SQL @db.execute <<-SQL
CREATE TABLE IF NOT EXISTS user_topic_count_final9 ( CREATE TABLE IF NOT EXISTS user_topic_count_final11 (
user_id INTEGER PRIMARY KEY, user_id INTEGER PRIMARY KEY,
topic_count INTEGER DEFAULT 0 topic_count INTEGER DEFAULT 0
); );
SQL SQL
# POST IMPORT - For each user (_id), increment post_count as we add posts (to see total posts per user) # POST IMPORT - For each user (_id), increment post_count as we add posts (to see total posts per user)
@db.execute <<-SQL @db.execute <<-SQL
CREATE TABLE IF NOT EXISTS user_post_count_final9 ( CREATE TABLE IF NOT EXISTS user_post_count_final11 (
user_id INTEGER PRIMARY KEY, user_id INTEGER PRIMARY KEY,
post_count INTEGER DEFAULT 0 post_count INTEGER DEFAULT 0
); );
SQL SQL
# POST IMPORT - For each topic, the user_id for the last poster / replier # POST IMPORT - For each topic, the user_id for the last poster / replier
@db.execute <<-SQL @db.execute <<-SQL
CREATE TABLE IF NOT EXISTS topic_last_post_user_final9 ( CREATE TABLE IF NOT EXISTS topic_last_post_user_final11 (
topic_id INTEGER PRIMARY KEY, topic_id INTEGER PRIMARY KEY,
user_id INTEGER user_id INTEGER
); );
SQL SQL
# POST IMPORT - The number of posts in a given topic, incremented as we add a new reply post to a topic. # POST IMPORT - The number of posts in a given topic, incremented as we add a new reply post to a topic.
@db.execute <<-SQL @db.execute <<-SQL
CREATE TABLE IF NOT EXISTS topic_post_numbers_final9 ( CREATE TABLE IF NOT EXISTS topic_post_numbers_final11 (
topic_id INTEGER PRIMARY KEY, topic_id INTEGER PRIMARY KEY,
post_number INTEGER DEFAULT 0 post_number INTEGER DEFAULT 0
); );
@@ -253,55 +253,55 @@ class GossamerForumsImporter < ImportScripts::Base
end end
def fetch_db_topic_last_post_time(topic_id) def fetch_db_topic_last_post_time(topic_id)
@db.get_first_value("SELECT last_post_time FROM topic_last_post_time_final9 WHERE topic_id = ?", [topic_id]) @db.get_first_value("SELECT last_post_time FROM topic_last_post_time_final11 WHERE topic_id = ?", [topic_id])
end end
def fetch_db_topic_last_post_user(topic_id) def fetch_db_topic_last_post_user(topic_id)
@db.get_first_value("SELECT user_id FROM topic_last_post_user_final9 WHERE topic_id = ?", [topic_id]) @db.get_first_value("SELECT user_id FROM topic_last_post_user_final11 WHERE topic_id = ?", [topic_id])
end end
def fetch_db_topic_post_count(topic_id) def fetch_db_topic_post_count(topic_id)
@db.get_first_value("SELECT post_count FROM topic_post_count_final9 WHERE topic_id = ?", [topic_id]) @db.get_first_value("SELECT post_count FROM topic_post_count_final11 WHERE topic_id = ?", [topic_id])
end end
def fetch_db_user_topic_count(user_id) def fetch_db_user_topic_count(user_id)
@db.get_first_value("SELECT topic_count FROM user_topic_count_final9 WHERE user_id = ?", [user_id]) @db.get_first_value("SELECT topic_count FROM user_topic_count_final11 WHERE user_id = ?", [user_id])
end end
def fetch_db_user_post_count(user_id) def fetch_db_user_post_count(user_id)
@db.get_first_value("SELECT post_count FROM user_post_count_final9 WHERE user_id = ?", [user_id]) @db.get_first_value("SELECT post_count FROM user_post_count_final11 WHERE user_id = ?", [user_id])
end end
def fetch_db_topic_post_numbers(topic_id) def fetch_db_topic_post_numbers(topic_id)
# @db.get_first_value "SELECT post_number FROM topic_post_numbers_final9 WHERE topic_id = ?", topic_id # @db.get_first_value "SELECT post_number FROM topic_post_numbers_final11 WHERE topic_id = ?", topic_id
@db.get_first_value("SELECT post_number FROM topic_post_numbers_final9 WHERE topic_id = ?", [topic_id]) @db.get_first_value("SELECT post_number FROM topic_post_numbers_final11 WHERE topic_id = ?", [topic_id])
end end
def update_db_topic_last_post_time(topic_id, last_post_time) def update_db_topic_last_post_time(topic_id, last_post_time)
# puts "zzzzzzzzz1 #{topic_id} #{last_post_time}" # puts "zzzzzzzzz1 #{topic_id} #{last_post_time}"
# @db.execute "INSERT OR REPLACE INTO topic_last_post_time_final9 (topic_id, last_post_time) VALUES (?, ?)", topic_id, last_post_time # @db.execute "INSERT OR REPLACE INTO topic_last_post_time_final11 (topic_id, last_post_time) VALUES (?, ?)", topic_id, last_post_time
@db.execute("INSERT OR REPLACE INTO topic_last_post_time_final9 (topic_id, last_post_time) VALUES (?, ?)", [topic_id, last_post_time]) @db.execute("INSERT OR REPLACE INTO topic_last_post_time_final11 (topic_id, last_post_time) VALUES (?, ?)", [topic_id, last_post_time])
end end
def update_db_topic_last_post_user(topic_id, user_id) def update_db_topic_last_post_user(topic_id, user_id)
@db.execute("INSERT OR REPLACE INTO topic_last_post_user_final9 (topic_id, user_id) VALUES (?, ?)", [topic_id, user_id]) @db.execute("INSERT OR REPLACE INTO topic_last_post_user_final11 (topic_id, user_id) VALUES (?, ?)", [topic_id, user_id])
end end
def update_db_topic_post_count(topic_id, post_count) def update_db_topic_post_count(topic_id, post_count)
@db.execute("INSERT OR REPLACE INTO topic_post_count_final9 (topic_id, post_count) VALUES (?, ?)", [topic_id, post_count]) @db.execute("INSERT OR REPLACE INTO topic_post_count_final11 (topic_id, post_count) VALUES (?, ?)", [topic_id, post_count])
end end
def update_db_user_topic_count(user_id, topic_count) def update_db_user_topic_count(user_id, topic_count)
@db.execute("INSERT OR REPLACE INTO user_topic_count_final9 (user_id, topic_count) VALUES (?, ?)", [user_id, topic_count]) @db.execute("INSERT OR REPLACE INTO user_topic_count_final11 (user_id, topic_count) VALUES (?, ?)", [user_id, topic_count])
end end
def update_db_user_post_count(user_id, post_count) def update_db_user_post_count(user_id, post_count)
@db.execute("INSERT OR REPLACE INTO user_post_count_final9 (user_id, post_count) VALUES (?, ?)", [user_id, post_count]) @db.execute("INSERT OR REPLACE INTO user_post_count_final11 (user_id, post_count) VALUES (?, ?)", [user_id, post_count])
end end
def update_db_topic_post_numbers(topic_id, post_number) def update_db_topic_post_numbers(topic_id, post_number)
@db.execute("INSERT OR REPLACE INTO topic_post_numbers_final9 (topic_id, post_number) VALUES (?, ?)", [topic_id, post_number]) @db.execute("INSERT OR REPLACE INTO topic_post_numbers_final11 (topic_id, post_number) VALUES (?, ?)", [topic_id, post_number])
end end
# Fetch the highest processed post_id from the highest_processed_post_id table # Fetch the highest processed post_id from the highest_processed_post_id table
@@ -1060,10 +1060,8 @@ class GossamerForumsImporter < ImportScripts::Base
# sanitized_message.gsub!(/!\[(?:http|https):\/\/data:image\/[^]]+\]\((?:http|https):\/\/data:image\/[^)]+\)/, '') # sanitized_message.gsub!(/!\[(?:http|https):\/\/data:image\/[^]]+\]\((?:http|https):\/\/data:image\/[^)]+\)/, '')
# sanitized_message.gsub!(/!\[(https?:\/\/data:image\/[^]]+)\]\((https?:\/\/data:image\/[^)]+)\)/, '') # sanitized_message.gsub!(/!\[(https?:\/\/data:image\/[^]]+)\]\((https?:\/\/data:image\/[^)]+)\)/, '')
# sanitized_message.gsub!(/!\[(https?:\/\/data:image\/[^\]]+)\]\((https?:\/\/data:image\/[^\)]+)\)/) # sanitized_message.gsub!(/!\[(https?:\/\/data:image\/[^\]]+)\]\((https?:\/\/data:image\/[^\)]+)\)/)
puts "Here 111 #{sanitized_message}" # sanitized_message.gsub!(/!\[https?:\/\/data:image[^\]]*\]\(https?:\/\/data:image[^\)]*\)/, '')
sanitized_message.gsub!(/!\[https?:\/\/data:image[^\]]*\]\(https?:\/\/data:image[^\)]*\)/, '') sanitized_message.gsub!(/\[img\]https?:\/\/data:image[^\[]*\[\/img\]/, '')
puts "Here 222 #{sanitized_message}"
# Ensure sentence structure # Ensure sentence structure
unless sanitized_message.match?(/[.!?]\s|[.!?]$/) unless sanitized_message.match?(/[.!?]\s|[.!?]$/)
@@ -1861,13 +1859,13 @@ class GossamerForumsImporter < ImportScripts::Base
end end
end end
def update_topic_stats_final9 def update_topic_stats_final11
# Update topics with the correct last post time, post count, and last post user # Update topics with the correct last post time, post count, and last post user
puts "Update topics with the correct last post time, post count, and last post user" puts "Update topics with the correct last post time, post count, and last post user"
@db.execute("SELECT * FROM topic_last_post_time_final9").each do |row| @db.execute("SELECT * FROM topic_last_post_time_final11").each do |row|
topic_id, last_post_time = row topic_id, last_post_time = row
puts "update_topic_stats_final9 topic_id #{topic_id}" puts "update_topic_stats_final11 topic_id #{topic_id}"
begin begin
topic = Topic.find(topic_id) topic = Topic.find(topic_id)
@@ -2244,8 +2242,8 @@ class GossamerForumsImporter < ImportScripts::Base
### update_topic_stats_final ### update_topic_stats_final
### update_user_stats_finalfinal ### update_user_stats_finalfinal
# update_topic_stats_final9 # update_topic_stats_final11
# update_user_stats_finalfinal9 # update_user_stats_finalfinal11
### export_url_mapping_to_csv("/bitnami/discourse/sqlite/gossamer-migration-url-mapping#{timestamp}") ### export_url_mapping_to_csv("/bitnami/discourse/sqlite/gossamer-migration-url-mapping#{timestamp}")
### export_nginx_rewrite_rules("/bitnami/discourse/sqlite/gossamer-redirects#{timestamp}.conf") ### export_nginx_rewrite_rules("/bitnami/discourse/sqlite/gossamer-redirects#{timestamp}.conf")