saint/discourse-import_scripts
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Finish takling the multiple issues which Discourse had with the very large number of posts in this particular topic

Browse Source
This commit is contained in:
David Sainty 2024-12-07 17:12:06 +11:00
parent 1885af1a34
commit ce865db09e
1 changed files with 25 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
goss-restorethread.rb
View File

@ -114,42 +114,42 @@ class GossamerForumsImporter < ImportScripts::Base
SQL SQL
# POST IMPORT - For each topic, the time of the last / most recent post/reply # POST IMPORT - For each topic, the time of the last / most recent post/reply
@db.execute <<-SQL @db.execute <<-SQL
CREATE TABLE IF NOT EXISTS topic_last_post_time_final2 ( CREATE TABLE IF NOT EXISTS topic_last_post_time_final3 (
topic_id INTEGER PRIMARY KEY, topic_id INTEGER PRIMARY KEY,
last_post_time INTEGER last_post_time INTEGER
); );
SQL SQL
# POST IMPORT - For each topic, increment post_count as we add posts. # POST IMPORT - For each topic, increment post_count as we add posts.
@db.execute <<-SQL @db.execute <<-SQL
CREATE TABLE IF NOT EXISTS topic_post_count_final2 ( CREATE TABLE IF NOT EXISTS topic_post_count_final3 (
topic_id INTEGER PRIMARY KEY, topic_id INTEGER PRIMARY KEY,
post_count INTEGER DEFAULT 0 post_count INTEGER DEFAULT 0
); );
SQL SQL
# POST IMPORT - For each user (_id), increment topic_count as we add topics (to see total topics per user) # POST IMPORT - For each user (_id), increment topic_count as we add topics (to see total topics per user)
@db.execute <<-SQL @db.execute <<-SQL
CREATE TABLE IF NOT EXISTS user_topic_count_final2 ( CREATE TABLE IF NOT EXISTS user_topic_count_final3 (
user_id INTEGER PRIMARY KEY, user_id INTEGER PRIMARY KEY,
topic_count INTEGER DEFAULT 0 topic_count INTEGER DEFAULT 0
); );
SQL SQL
# POST IMPORT - For each user (_id), increment post_count as we add posts (to see total posts per user) # POST IMPORT - For each user (_id), increment post_count as we add posts (to see total posts per user)
@db.execute <<-SQL @db.execute <<-SQL
CREATE TABLE IF NOT EXISTS user_post_count_final2 ( CREATE TABLE IF NOT EXISTS user_post_count_final3 (
user_id INTEGER PRIMARY KEY, user_id INTEGER PRIMARY KEY,
post_count INTEGER DEFAULT 0 post_count INTEGER DEFAULT 0
); );
SQL SQL
# POST IMPORT - For each topic, the user_id for the last poster / replier # POST IMPORT - For each topic, the user_id for the last poster / replier
@db.execute <<-SQL @db.execute <<-SQL
CREATE TABLE IF NOT EXISTS topic_last_post_user_final2 ( CREATE TABLE IF NOT EXISTS topic_last_post_user_final3 (
topic_id INTEGER PRIMARY KEY, topic_id INTEGER PRIMARY KEY,
user_id INTEGER user_id INTEGER
); );
SQL SQL
# POST IMPORT - The number of posts in a given topic, incremented as we add a new reply post to a topic. # POST IMPORT - The number of posts in a given topic, incremented as we add a new reply post to a topic.
@db.execute <<-SQL @db.execute <<-SQL
CREATE TABLE IF NOT EXISTS topic_post_numbers_final2 ( CREATE TABLE IF NOT EXISTS topic_post_numbers_final3 (
topic_id INTEGER PRIMARY KEY, topic_id INTEGER PRIMARY KEY,
post_number INTEGER DEFAULT 0 post_number INTEGER DEFAULT 0
); );
@ -253,55 +253,55 @@ class GossamerForumsImporter < ImportScripts::Base
end end
def fetch_db_topic_last_post_time(topic_id) def fetch_db_topic_last_post_time(topic_id)
@db.get_first_value("SELECT last_post_time FROM topic_last_post_time_final2 WHERE topic_id = ?", [topic_id]) @db.get_first_value("SELECT last_post_time FROM topic_last_post_time_final3 WHERE topic_id = ?", [topic_id])
end end
def fetch_db_topic_last_post_user(topic_id) def fetch_db_topic_last_post_user(topic_id)
@db.get_first_value("SELECT user_id FROM topic_last_post_user_final2 WHERE topic_id = ?", [topic_id]) @db.get_first_value("SELECT user_id FROM topic_last_post_user_final3 WHERE topic_id = ?", [topic_id])
end end
def fetch_db_topic_post_count(topic_id) def fetch_db_topic_post_count(topic_id)
@db.get_first_value("SELECT post_count FROM topic_post_count_final2 WHERE topic_id = ?", [topic_id]) @db.get_first_value("SELECT post_count FROM topic_post_count_final3 WHERE topic_id = ?", [topic_id])
end end
def fetch_db_user_topic_count(user_id) def fetch_db_user_topic_count(user_id)
@db.get_first_value("SELECT topic_count FROM user_topic_count_final2 WHERE user_id = ?", [user_id]) @db.get_first_value("SELECT topic_count FROM user_topic_count_final3 WHERE user_id = ?", [user_id])
end end
def fetch_db_user_post_count(user_id) def fetch_db_user_post_count(user_id)
@db.get_first_value("SELECT post_count FROM user_post_count_final2 WHERE user_id = ?", [user_id]) @db.get_first_value("SELECT post_count FROM user_post_count_final3 WHERE user_id = ?", [user_id])
end end
def fetch_db_topic_post_numbers(topic_id) def fetch_db_topic_post_numbers(topic_id)
# @db.get_first_value "SELECT post_number FROM topic_post_numbers_final2 WHERE topic_id = ?", topic_id # @db.get_first_value "SELECT post_number FROM topic_post_numbers_final3 WHERE topic_id = ?", topic_id
@db.get_first_value("SELECT post_number FROM topic_post_numbers_final2 WHERE topic_id = ?", [topic_id]) @db.get_first_value("SELECT post_number FROM topic_post_numbers_final3 WHERE topic_id = ?", [topic_id])
end end
def update_db_topic_last_post_time(topic_id, last_post_time) def update_db_topic_last_post_time(topic_id, last_post_time)
# puts "zzzzzzzzz1 #{topic_id} #{last_post_time}" # puts "zzzzzzzzz1 #{topic_id} #{last_post_time}"
# @db.execute "INSERT OR REPLACE INTO topic_last_post_time_final2 (topic_id, last_post_time) VALUES (?, ?)", topic_id, last_post_time # @db.execute "INSERT OR REPLACE INTO topic_last_post_time_final3 (topic_id, last_post_time) VALUES (?, ?)", topic_id, last_post_time
@db.execute("INSERT OR REPLACE INTO topic_last_post_time_final2 (topic_id, last_post_time) VALUES (?, ?)", [topic_id, last_post_time]) @db.execute("INSERT OR REPLACE INTO topic_last_post_time_final3 (topic_id, last_post_time) VALUES (?, ?)", [topic_id, last_post_time])
end end
def update_db_topic_last_post_user(topic_id, user_id) def update_db_topic_last_post_user(topic_id, user_id)
@db.execute("INSERT OR REPLACE INTO topic_last_post_user_final2 (topic_id, user_id) VALUES (?, ?)", [topic_id, user_id]) @db.execute("INSERT OR REPLACE INTO topic_last_post_user_final3 (topic_id, user_id) VALUES (?, ?)", [topic_id, user_id])
end end
def update_db_topic_post_count(topic_id, post_count) def update_db_topic_post_count(topic_id, post_count)
@db.execute("INSERT OR REPLACE INTO topic_post_count_final2 (topic_id, post_count) VALUES (?, ?)", [topic_id, post_count]) @db.execute("INSERT OR REPLACE INTO topic_post_count_final3 (topic_id, post_count) VALUES (?, ?)", [topic_id, post_count])
end end
def update_db_user_topic_count(user_id, topic_count) def update_db_user_topic_count(user_id, topic_count)
@db.execute("INSERT OR REPLACE INTO user_topic_count_final2 (user_id, topic_count) VALUES (?, ?)", [user_id, topic_count]) @db.execute("INSERT OR REPLACE INTO user_topic_count_final3 (user_id, topic_count) VALUES (?, ?)", [user_id, topic_count])
end end
def update_db_user_post_count(user_id, post_count) def update_db_user_post_count(user_id, post_count)
@db.execute("INSERT OR REPLACE INTO user_post_count_final2 (user_id, post_count) VALUES (?, ?)", [user_id, post_count]) @db.execute("INSERT OR REPLACE INTO user_post_count_final3 (user_id, post_count) VALUES (?, ?)", [user_id, post_count])
end end
def update_db_topic_post_numbers(topic_id, post_number) def update_db_topic_post_numbers(topic_id, post_number)
@db.execute("INSERT OR REPLACE INTO topic_post_numbers_final2 (topic_id, post_number) VALUES (?, ?)", [topic_id, post_number]) @db.execute("INSERT OR REPLACE INTO topic_post_numbers_final3 (topic_id, post_number) VALUES (?, ?)", [topic_id, post_number])
end end
# Fetch the highest processed post_id from the highest_processed_post_id table # Fetch the highest processed post_id from the highest_processed_post_id table
@ -1852,13 +1852,13 @@ class GossamerForumsImporter < ImportScripts::Base
end end
end end
def update_topic_stats_final2 def update_topic_stats_final3
# Update topics with the correct last post time, post count, and last post user # Update topics with the correct last post time, post count, and last post user
puts "Update topics with the correct last post time, post count, and last post user" puts "Update topics with the correct last post time, post count, and last post user"
@db.execute("SELECT * FROM topic_last_post_time_final2").each do |row| @db.execute("SELECT * FROM topic_last_post_time_final3").each do |row|
topic_id, last_post_time = row topic_id, last_post_time = row
puts "update_topic_stats_final2 topic_id #{topic_id}" puts "update_topic_stats_final3 topic_id #{topic_id}"
begin begin
topic = Topic.find(topic_id) topic = Topic.find(topic_id)
@ -2235,8 +2235,8 @@ class GossamerForumsImporter < ImportScripts::Base
### update_topic_stats_final ### update_topic_stats_final
### update_user_stats_finalfinal ### update_user_stats_finalfinal
# update_topic_stats_final2 # update_topic_stats_final3
# update_user_stats_finalfinal2 # update_user_stats_finalfinal3
### export_url_mapping_to_csv("/bitnami/discourse/sqlite/gossamer-migration-url-mapping#{timestamp}") ### export_url_mapping_to_csv("/bitnami/discourse/sqlite/gossamer-migration-url-mapping#{timestamp}")
### export_nginx_rewrite_rules("/bitnami/discourse/sqlite/gossamer-redirects#{timestamp}.conf") ### export_nginx_rewrite_rules("/bitnami/discourse/sqlite/gossamer-redirects#{timestamp}.conf")