saint/discourse-import_scripts
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Finish taking the multiple issues which Discourse had with the very large number of posts in this particular topic

Browse Source
This commit is contained in:
David Sainty 2024-12-09 18:29:54 +11:00
parent 8b7ff8983e
commit f18016750c
1 changed files with 29 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
goss-restorethread.rb
View File

@ -114,42 +114,42 @@ class GossamerForumsImporter < ImportScripts::Base
SQL
# POST IMPORT - For each topic, the time of the last / most recent post/reply
@db.execute <<-SQL
CREATE TABLE IF NOT EXISTS topic_last_post_time_final7 (
CREATE TABLE IF NOT EXISTS topic_last_post_time_final8 (
topic_id INTEGER PRIMARY KEY,
last_post_time INTEGER
);
SQL
# POST IMPORT - For each topic, increment post_count as we add posts.
@db.execute <<-SQL
CREATE TABLE IF NOT EXISTS topic_post_count_final7 (
CREATE TABLE IF NOT EXISTS topic_post_count_final8 (
topic_id INTEGER PRIMARY KEY,
post_count INTEGER DEFAULT 0
);
SQL
# POST IMPORT - For each user (_id), increment topic_count as we add topics (to see total topics per user)
@db.execute <<-SQL
CREATE TABLE IF NOT EXISTS user_topic_count_final7 (
CREATE TABLE IF NOT EXISTS user_topic_count_final8 (
user_id INTEGER PRIMARY KEY,
topic_count INTEGER DEFAULT 0
);
SQL
# POST IMPORT - For each user (_id), increment post_count as we add posts (to see total posts per user)
@db.execute <<-SQL
CREATE TABLE IF NOT EXISTS user_post_count_final7 (
CREATE TABLE IF NOT EXISTS user_post_count_final8 (
user_id INTEGER PRIMARY KEY,
post_count INTEGER DEFAULT 0
);
SQL
# POST IMPORT - For each topic, the user_id for the last poster / replier
@db.execute <<-SQL
CREATE TABLE IF NOT EXISTS topic_last_post_user_final7 (
CREATE TABLE IF NOT EXISTS topic_last_post_user_final8 (
topic_id INTEGER PRIMARY KEY,
user_id INTEGER
);
SQL
# POST IMPORT - The number of posts in a given topic, incremented as we add a new reply post to a topic.
@db.execute <<-SQL
CREATE TABLE IF NOT EXISTS topic_post_numbers_final7 (
CREATE TABLE IF NOT EXISTS topic_post_numbers_final8 (
topic_id INTEGER PRIMARY KEY,
post_number INTEGER DEFAULT 0
);
@ -253,55 +253,55 @@ class GossamerForumsImporter < ImportScripts::Base
end
def fetch_db_topic_last_post_time(topic_id)
@db.get_first_value("SELECT last_post_time FROM topic_last_post_time_final7 WHERE topic_id = ?", [topic_id])
@db.get_first_value("SELECT last_post_time FROM topic_last_post_time_final8 WHERE topic_id = ?", [topic_id])
end
def fetch_db_topic_last_post_user(topic_id)
@db.get_first_value("SELECT user_id FROM topic_last_post_user_final7 WHERE topic_id = ?", [topic_id])
@db.get_first_value("SELECT user_id FROM topic_last_post_user_final8 WHERE topic_id = ?", [topic_id])
end
def fetch_db_topic_post_count(topic_id)
@db.get_first_value("SELECT post_count FROM topic_post_count_final7 WHERE topic_id = ?", [topic_id])
@db.get_first_value("SELECT post_count FROM topic_post_count_final8 WHERE topic_id = ?", [topic_id])
end
def fetch_db_user_topic_count(user_id)
@db.get_first_value("SELECT topic_count FROM user_topic_count_final7 WHERE user_id = ?", [user_id])
@db.get_first_value("SELECT topic_count FROM user_topic_count_final8 WHERE user_id = ?", [user_id])
end
def fetch_db_user_post_count(user_id)
@db.get_first_value("SELECT post_count FROM user_post_count_final7 WHERE user_id = ?", [user_id])
@db.get_first_value("SELECT post_count FROM user_post_count_final8 WHERE user_id = ?", [user_id])
end
def fetch_db_topic_post_numbers(topic_id)
# @db.get_first_value "SELECT post_number FROM topic_post_numbers_final7 WHERE topic_id = ?", topic_id
@db.get_first_value("SELECT post_number FROM topic_post_numbers_final7 WHERE topic_id = ?", [topic_id])
# @db.get_first_value "SELECT post_number FROM topic_post_numbers_final8 WHERE topic_id = ?", topic_id
@db.get_first_value("SELECT post_number FROM topic_post_numbers_final8 WHERE topic_id = ?", [topic_id])
end
def update_db_topic_last_post_time(topic_id, last_post_time)
# puts "zzzzzzzzz1 #{topic_id} #{last_post_time}"
# @db.execute "INSERT OR REPLACE INTO topic_last_post_time_final7 (topic_id, last_post_time) VALUES (?, ?)", topic_id, last_post_time
@db.execute("INSERT OR REPLACE INTO topic_last_post_time_final7 (topic_id, last_post_time) VALUES (?, ?)", [topic_id, last_post_time])
# @db.execute "INSERT OR REPLACE INTO topic_last_post_time_final8 (topic_id, last_post_time) VALUES (?, ?)", topic_id, last_post_time
@db.execute("INSERT OR REPLACE INTO topic_last_post_time_final8 (topic_id, last_post_time) VALUES (?, ?)", [topic_id, last_post_time])
end
def update_db_topic_last_post_user(topic_id, user_id)
@db.execute("INSERT OR REPLACE INTO topic_last_post_user_final7 (topic_id, user_id) VALUES (?, ?)", [topic_id, user_id])
@db.execute("INSERT OR REPLACE INTO topic_last_post_user_final8 (topic_id, user_id) VALUES (?, ?)", [topic_id, user_id])
end
def update_db_topic_post_count(topic_id, post_count)
@db.execute("INSERT OR REPLACE INTO topic_post_count_final7 (topic_id, post_count) VALUES (?, ?)", [topic_id, post_count])
@db.execute("INSERT OR REPLACE INTO topic_post_count_final8 (topic_id, post_count) VALUES (?, ?)", [topic_id, post_count])
end
def update_db_user_topic_count(user_id, topic_count)
@db.execute("INSERT OR REPLACE INTO user_topic_count_final7 (user_id, topic_count) VALUES (?, ?)", [user_id, topic_count])
@db.execute("INSERT OR REPLACE INTO user_topic_count_final8 (user_id, topic_count) VALUES (?, ?)", [user_id, topic_count])
end
def update_db_user_post_count(user_id, post_count)
@db.execute("INSERT OR REPLACE INTO user_post_count_final7 (user_id, post_count) VALUES (?, ?)", [user_id, post_count])
@db.execute("INSERT OR REPLACE INTO user_post_count_final8 (user_id, post_count) VALUES (?, ?)", [user_id, post_count])
end
def update_db_topic_post_numbers(topic_id, post_number)
@db.execute("INSERT OR REPLACE INTO topic_post_numbers_final7 (topic_id, post_number) VALUES (?, ?)", [topic_id, post_number])
@db.execute("INSERT OR REPLACE INTO topic_post_numbers_final8 (topic_id, post_number) VALUES (?, ?)", [topic_id, post_number])
end
# Fetch the highest processed post_id from the highest_processed_post_id table
@ -1057,8 +1057,10 @@ class GossamerForumsImporter < ImportScripts::Base
# sanitized_message.gsub!(/!\[data:image\/[^\]]+\]\([^)]+\)/, '')
# sanitized_message.gsub!(/!\[[^\]]*\]\(https?:\/\/data:image\/[a-zA-Z]+;base64,[^)]+\)/, '')
# sanitized_message.gsub!(/!\[.*?\]\((?:http|https):\/\/data:image\/[^;]+;base64,[^)]+\)/i, '')
sanitized_message.gsub!(/!\[(?:http|https):\/\/data:image\/[^]]+\]\((?:http|https):\/\/data:image\/[^)]+\)/, '')
sanitized_message.gsub!(/!\[(https?:\/\/data:image\/[^]]+)\]\((https?:\/\/data:image\/[^)]+)\)/, '')
# sanitized_message.gsub!(/!\[(?:http|https):\/\/data:image\/[^]]+\]\((?:http|https):\/\/data:image\/[^)]+\)/, '')
# sanitized_message.gsub!(/!\[(https?:\/\/data:image\/[^]]+)\]\((https?:\/\/data:image\/[^)]+)\)/, '')
sanitized_message.gsub!(/!\[(https?:\/\/data:image\/[^\]]+)\]\((https?:\/\/data:image\/[^\)]+)\)/)
# Ensure sentence structure
unless sanitized_message.match?(/[.!?]\s|[.!?]$/)
@ -1856,13 +1858,13 @@ class GossamerForumsImporter < ImportScripts::Base
end
end
def update_topic_stats_final7
def update_topic_stats_final8
# Update topics with the correct last post time, post count, and last post user
puts "Update topics with the correct last post time, post count, and last post user"
@db.execute("SELECT * FROM topic_last_post_time_final7").each do |row|
@db.execute("SELECT * FROM topic_last_post_time_final8").each do |row|
topic_id, last_post_time = row
puts "update_topic_stats_final7 topic_id #{topic_id}"
puts "update_topic_stats_final8 topic_id #{topic_id}"
begin
topic = Topic.find(topic_id)
@ -2239,8 +2241,8 @@ class GossamerForumsImporter < ImportScripts::Base
### update_topic_stats_final
### update_user_stats_finalfinal
# update_topic_stats_final7
# update_user_stats_finalfinal7
# update_topic_stats_final8
# update_user_stats_finalfinal8
### export_url_mapping_to_csv("/bitnami/discourse/sqlite/gossamer-migration-url-mapping#{timestamp}")
### export_nginx_rewrite_rules("/bitnami/discourse/sqlite/gossamer-redirects#{timestamp}.conf")