saint/discourse-import_scripts
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

v0.59 Final topic-post import run -- add logic for appending counts since we are adding to a live Discourse host now

Browse Source
This commit is contained in:
David Sainty 2024-09-02 13:45:44 +10:00
parent c8e7ba97bb
commit 958e37771e
1 changed files with 139 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

182
gossamer_forums.rb
View File

@ -1,7 +1,7 @@
# Federated Computer, Inc. # Federated Computer, Inc.
# David Sainty <saint@federated.computer> 2024 A.D. # David Sainty <saint@federated.computer> 2024 A.D.
# Gossamer Threads to Discourse -- Migration-Import Script # Gossamer Threads to Discourse -- Migration-Import Script
# v0.58 Final user import and personal message import run # v0.59 Final topic-post import run -- add logic for appending counts since we are adding to a live Discourse host now
require 'mysql2' require 'mysql2'
require 'open-uri' require 'open-uri'
@ -114,47 +114,47 @@ class GossamerForumsImporter < ImportScripts::Base
SQL SQL
# POST IMPORT - For each topic, the time of the last / most recent post/reply # POST IMPORT - For each topic, the time of the last / most recent post/reply
@db.execute <<-SQL @db.execute <<-SQL
CREATE TABLE IF NOT EXISTS topic_last_post_time ( CREATE TABLE IF NOT EXISTS topic_last_post_time_final (
topic_id INTEGER PRIMARY KEY, topic_id INTEGER PRIMARY KEY,
last_post_time INTEGER last_post_time INTEGER
); );
SQL SQL
# POST IMPORT - For each topic, increment post_count as we add posts. # POST IMPORT - For each topic, increment post_count as we add posts.
@db.execute <<-SQL @db.execute <<-SQL
CREATE TABLE IF NOT EXISTS topic_post_count ( CREATE TABLE IF NOT EXISTS topic_post_count_final (
topic_id INTEGER PRIMARY KEY, topic_id INTEGER PRIMARY KEY,
post_count INTEGER DEFAULT 0 post_count INTEGER DEFAULT 0
); );
SQL SQL
# POST IMPORT - For each user (_id), increment topic_count as we add topics (to see total topics per user) # POST IMPORT - For each user (_id), increment topic_count as we add topics (to see total topics per user)
@db.execute <<-SQL @db.execute <<-SQL
CREATE TABLE IF NOT EXISTS user_topic_count ( CREATE TABLE IF NOT EXISTS user_topic_count_final (
user_id INTEGER PRIMARY KEY, user_id INTEGER PRIMARY KEY,
topic_count INTEGER DEFAULT 0 topic_count INTEGER DEFAULT 0
); );
SQL SQL
# POST IMPORT - For each user (_id), increment post_count as we add posts (to see total posts per user) # POST IMPORT - For each user (_id), increment post_count as we add posts (to see total posts per user)
@db.execute <<-SQL @db.execute <<-SQL
CREATE TABLE IF NOT EXISTS user_post_count ( CREATE TABLE IF NOT EXISTS user_post_count_final (
user_id INTEGER PRIMARY KEY, user_id INTEGER PRIMARY KEY,
post_count INTEGER DEFAULT 0 post_count INTEGER DEFAULT 0
); );
SQL SQL
# POST IMPORT - For each topic, the user_id for the last poster / replier # POST IMPORT - For each topic, the user_id for the last poster / replier
@db.execute <<-SQL @db.execute <<-SQL
CREATE TABLE IF NOT EXISTS topic_last_post_user ( CREATE TABLE IF NOT EXISTS topic_last_post_user_final (
topic_id INTEGER PRIMARY KEY, topic_id INTEGER PRIMARY KEY,
user_id INTEGER user_id INTEGER
); );
SQL SQL
# POST IMPORT - The number of posts in a given topic, incremented as we add a new reply post to a topic. # POST IMPORT - The number of posts in a given topic, incremented as we add a new reply post to a topic.
@db.execute <<-SQL @db.execute <<-SQL
CREATE TABLE IF NOT EXISTS topic_post_numbers ( CREATE TABLE IF NOT EXISTS topic_post_numbers_final (
topic_id INTEGER PRIMARY KEY, topic_id INTEGER PRIMARY KEY,
post_number INTEGER DEFAULT 0 post_number INTEGER DEFAULT 0
); );
SQL SQL
# POST IMPORT - Record perssitent integer value for highest processed post id -- not used # POST IMPORT - Record perssitent integer value for highest processed post id
@db.execute <<-SQL @db.execute <<-SQL
CREATE TABLE IF NOT EXISTS highest_processed_post_id ( CREATE TABLE IF NOT EXISTS highest_processed_post_id (
id INTEGER PRIMARY KEY CHECK (id = 1), id INTEGER PRIMARY KEY CHECK (id = 1),
@ -253,51 +253,51 @@ class GossamerForumsImporter < ImportScripts::Base
end end
def fetch_db_topic_last_post_time(topic_id) def fetch_db_topic_last_post_time(topic_id)
@db.get_first_value "SELECT last_post_time FROM topic_last_post_time WHERE topic_id = ?", topic_id @db.get_first_value "SELECT last_post_time FROM topic_last_post_time_final WHERE topic_id = ?", topic_id
end end
def fetch_db_topic_last_post_user(topic_id) def fetch_db_topic_last_post_user(topic_id)
@db.get_first_value "SELECT user_id FROM topic_last_post_user WHERE topic_id = ?", topic_id @db.get_first_value "SELECT user_id FROM topic_last_post_user_final WHERE topic_id = ?", topic_id
end end
def fetch_db_topic_post_count(topic_id) def fetch_db_topic_post_count(topic_id)
@db.get_first_value "SELECT post_count FROM topic_post_count WHERE topic_id = ?", topic_id @db.get_first_value "SELECT post_count FROM topic_post_count_final WHERE topic_id = ?", topic_id
end end
def fetch_db_user_topic_count(user_id) def fetch_db_user_topic_count(user_id)
@db.get_first_value "SELECT topic_count FROM user_topic_count WHERE user_id = ?", user_id @db.get_first_value "SELECT topic_count FROM user_topic_count_final WHERE user_id = ?", user_id
end end
def fetch_db_user_post_count(user_id) def fetch_db_user_post_count(user_id)
@db.get_first_value "SELECT post_count FROM user_post_count WHERE user_id = ?", user_id @db.get_first_value "SELECT post_count FROM user_post_count_final WHERE user_id = ?", user_id
end end
def fetch_db_topic_post_numbers(topic_id) def fetch_db_topic_post_numbers(topic_id)
@db.get_first_value "SELECT post_number FROM topic_post_numbers WHERE topic_id = ?", topic_id @db.get_first_value "SELECT post_number FROM topic_post_numbers_final WHERE topic_id = ?", topic_id
end end
def update_db_topic_last_post_time(topic_id, last_post_time) def update_db_topic_last_post_time(topic_id, last_post_time)
@db.execute "INSERT OR REPLACE INTO topic_last_post_time (topic_id, last_post_time) VALUES (?, ?)", topic_id, last_post_time @db.execute "INSERT OR REPLACE INTO topic_last_post_time_final (topic_id, last_post_time) VALUES (?, ?)", topic_id, last_post_time
end end
def update_db_topic_last_post_user(topic_id, user_id) def update_db_topic_last_post_user(topic_id, user_id)
@db.execute "INSERT OR REPLACE INTO topic_last_post_user (topic_id, user_id) VALUES (?, ?)", topic_id, user_id @db.execute "INSERT OR REPLACE INTO topic_last_post_user_final (topic_id, user_id) VALUES (?, ?)", topic_id, user_id
end end
def update_db_topic_post_count(topic_id, post_count) def update_db_topic_post_count(topic_id, post_count)
@db.execute "INSERT OR REPLACE INTO topic_post_count (topic_id, post_count) VALUES (?, ?)", topic_id, post_count @db.execute "INSERT OR REPLACE INTO topic_post_count_final (topic_id, post_count) VALUES (?, ?)", topic_id, post_count
end end
def update_db_user_topic_count(user_id, topic_count) def update_db_user_topic_count(user_id, topic_count)
@db.execute "INSERT OR REPLACE INTO user_topic_count (user_id, topic_count) VALUES (?, ?)", user_id, topic_count @db.execute "INSERT OR REPLACE INTO user_topic_count_final (user_id, topic_count) VALUES (?, ?)", user_id, topic_count
end end
def update_db_user_post_count(user_id, post_count) def update_db_user_post_count(user_id, post_count)
@db.execute "INSERT OR REPLACE INTO user_post_count (user_id, post_count) VALUES (?, ?)", user_id, post_count @db.execute "INSERT OR REPLACE INTO user_post_count_final (user_id, post_count) VALUES (?, ?)", user_id, post_count
end end
def update_db_topic_post_numbers(topic_id, post_number) def update_db_topic_post_numbers(topic_id, post_number)
@db.execute "INSERT OR REPLACE INTO topic_post_numbers (topic_id, post_number) VALUES (?, ?)", topic_id, post_number @db.execute "INSERT OR REPLACE INTO topic_post_numbers_final (topic_id, post_number) VALUES (?, ?)", topic_id, post_number
end end
# Fetch the highest processed post_id from the highest_processed_post_id table # Fetch the highest processed post_id from the highest_processed_post_id table
@ -1555,12 +1555,13 @@ class GossamerForumsImporter < ImportScripts::Base
# Fetch the highest old_post_id from the url_map table # Fetch the highest old_post_id from the url_map table
# highest_old_post_id = fetch_highest_old_post_id.to_i # highest_old_post_id = fetch_highest_old_post_id.to_i
# puts "Highest (OP) old_post_id in url_map: #{highest_old_post_id}" # puts "Highest (OP) old_post_id in url_map: #{highest_old_post_id}"
### highest_processed_post_id = fetch_highest_processed_post_id.to_i highest_processed_post_id = fetch_highest_processed_post_id.to_i
highest_processed_post_id = 8179621
puts "Highest processed post_id: #{highest_processed_post_id}" puts "Highest processed post_id: #{highest_processed_post_id}"
# OVERRIDE........ # OVERRIDE........
# Attachment example: highest_processed_post_id = 1359862 # Attachment example: highest_processed_post_id = 1359862
highest_processed_post_id = 8179621
puts "OVERRIDE Highest processed post_id: #{highest_processed_post_id}"
# Execute the query to get all posts ordered by post_id # Execute the query to get all posts ordered by post_id
execute_query("SELECT post_id, user_id_fk, forum_id_fk, post_root_id, post_subject, post_time, post_message, post_father_id, post_likes, post_replies FROM gforum_Post WHERE post_id > #{highest_processed_post_id} ORDER BY post_id").each do |row| execute_query("SELECT post_id, user_id_fk, forum_id_fk, post_root_id, post_subject, post_time, post_message, post_father_id, post_likes, post_replies FROM gforum_Post WHERE post_id > #{highest_processed_post_id} ORDER BY post_id").each do |row|
@ -1635,11 +1636,14 @@ class GossamerForumsImporter < ImportScripts::Base
end end
end end
end end
current_topic_id = topic.id
# Track last post time and user for the topic # Track last post time and user for the topic
# topic_last_post_time[topic.id] = Time.at(row['post_time']) # topic_last_post_time[topic.id] = Time.at(row['post_time'])
# topic_last_post_user[topic.id] = discourse_user_id # topic_last_post_user[topic.id] = discourse_user_id
update_db_topic_last_post_time(topic.id, Time.at(row['post_time']).to_i) update_db_topic_last_post_time(current_topic_id, Time.at(row['post_time']).to_i)
update_db_topic_last_post_user(topic.id, discourse_user_id) update_db_topic_last_post_user(current_topic_id, discourse_user_id)
# Increment the count of the number of topics created by each user # Increment the count of the number of topics created by each user
# user_topic_count[discourse_user_id] += 1 # user_topic_count[discourse_user_id] += 1
@ -1658,16 +1662,16 @@ class GossamerForumsImporter < ImportScripts::Base
# Sanitize the post message # Sanitize the post message
sanitized_post_message = sanitize_post_message(row['post_message']) sanitized_post_message = sanitize_post_message(row['post_message'])
puts "CREATE TOPIC POST topic.id #{topic.id} discourse_user_id #{discourse_user_id}" puts "CREATE TOPIC POST current_topic_id #{current_topic_id} discourse_user_id #{discourse_user_id}"
# Increment the number of posts in the given topic. # Increment the number of posts in the given topic.
# topic_post_numbers[topic.id] += 1 # topic_post_numbers[topic.id] += 1
post_number = fetch_db_topic_post_numbers(topic.id).to_i + 1 post_number = fetch_db_topic_post_numbers(current_topic_id).to_i + 1
update_db_topic_post_numbers(topic.id, post_number) update_db_topic_post_numbers(current_topic_id, post_number)
# Create the initial post in the topic # Create the initial post in the topic
post = Post.create!( post = Post.create!(
topic_id: topic.id, topic_id: current_topic_id,
user_id: discourse_user_id, user_id: discourse_user_id,
# raw: import_attachments(row['post_message'], row['post_id']), # raw: import_attachments(row['post_message'], row['post_id']),
# raw: row['post_message'] || "", # raw: row['post_message'] || "",
@ -1685,11 +1689,11 @@ class GossamerForumsImporter < ImportScripts::Base
# Track the number of posts in the topic and by the user # Track the number of posts in the topic and by the user
# topic_post_count[topic.id] += 1 # topic_post_count[topic.id] += 1
# user_post_count[discourse_user_id] += 1 # user_post_count[discourse_user_id] += 1
update_db_topic_post_count(topic.id, fetch_db_topic_post_count(topic.id).to_i + 1) update_db_topic_post_count(current_topic_id, fetch_db_topic_post_count(current_topic_id).to_i + 1)
update_db_user_post_count(discourse_user_id, fetch_db_user_post_count(discourse_user_id).to_i + 1) update_db_user_post_count(discourse_user_id, fetch_db_user_post_count(discourse_user_id).to_i + 1)
# Handle attachments for the post # Handle attachments for the post
handle_post_attachments(row['post_id'], post, discourse_user_id) handle_post_attachments(row['post_id'], post, discourse_user_id, @mysql_client)
# Create URL mappings # Create URL mappings
# old_url = "https://old/forum/#{row['forum_name']}/topics/#{row['post_id']}" # old_url = "https://old/forum/#{row['forum_name']}/topics/#{row['post_id']}"
@ -1727,6 +1731,8 @@ class GossamerForumsImporter < ImportScripts::Base
# Sanitize the post message # Sanitize the post message
sanitized_post_message = sanitize_post_message(row['post_message']) sanitized_post_message = sanitize_post_message(row['post_message'])
puts "CREATE REPLY in topic_id #{topic_id} for reply post_id #{row['post_id']}"
# topic_post_numbers[topic_id] += 1 # topic_post_numbers[topic_id] += 1
post_number = fetch_db_topic_post_numbers(topic_id).to_i + 1 post_number = fetch_db_topic_post_numbers(topic_id).to_i + 1
update_db_topic_post_numbers(topic_id, post_number) update_db_topic_post_numbers(topic_id, post_number)
@ -1735,16 +1741,17 @@ class GossamerForumsImporter < ImportScripts::Base
post = Post.create!( post = Post.create!(
topic_id: topic_id, topic_id: topic_id,
user_id: discourse_user_id, user_id: discourse_user_id,
# raw: import_attachments(row['post_message'], row['post_id']),
# raw: row['post_message'] || "",
raw: sanitized_post_message, raw: sanitized_post_message,
created_at: Time.at(row['post_time']), created_at: Time.at(row['post_time']),
updated_at: Time.at(row['post_time']), updated_at: Time.at(row['post_time']),
reply_to_post_number: reply_to_post_number, reads: post_views || 0,
like_count: row['post_replies'] || 0,
reads: post_views || fetch_db_topic_post_count(topic_id).to_i,
post_number: post_number post_number: post_number
) )
# raw: import_attachments(row['post_message'], row['post_id']),
# raw: row['post_message'] || "",
# reply_to_post_number: reply_to_post_number,
# like_count: row['post_replies'] || 0,
# reads: post_views || fetch_db_topic_post_count(topic_id).to_i,
post.custom_fields['original_gossamer_id'] = row['post_id'] post.custom_fields['original_gossamer_id'] = row['post_id']
post.save! post.save!
@ -1765,7 +1772,7 @@ class GossamerForumsImporter < ImportScripts::Base
end end
# Handle attachments for the post # Handle attachments for the post
handle_post_attachments(row['post_id'], post, discourse_user_id) handle_post_attachments(row['post_id'], post, discourse_user_id, @mysql_client)
# Update the highest processed post_id # Update the highest processed post_id
puts "Updated highest processed post_id #{post_id}" puts "Updated highest processed post_id #{post_id}"
@ -1794,7 +1801,7 @@ class GossamerForumsImporter < ImportScripts::Base
# last_post_user_id: topic_last_post_user[topic_id] # last_post_user_id: topic_last_post_user[topic_id]
# ) # )
# end # end
@db.execute("SELECT * FROM topic_last_post_time").each do |row| @db.execute("SELECT * FROM topic_last_post_time_final").each do |row|
topic_id, last_post_time = row topic_id, last_post_time = row
begin begin
@ -1815,6 +1822,38 @@ class GossamerForumsImporter < ImportScripts::Base
end end
end end
def update_topic_stats_final
# Update topics with the correct last post time, post count, and last post user
puts "Update topics with the correct last post time, post count, and last post user"
@db.execute("SELECT * FROM topic_last_post_time_final").each do |row|
topic_id, last_post_time = row
begin
topic = Topic.find(topic_id)
# Calculate the new values based on the given conditions
new_updated_at = [Time.at(last_post_time), topic.updated_at].max
new_posts_count = topic.posts_count + fetch_db_topic_post_count(topic_id).to_i
new_last_posted_at = [Time.at(last_post_time), topic.last_posted_at].max
new_bumped_at = [Time.at(last_post_time), topic.bumped_at].max
new_last_post_user_id = fetch_db_topic_last_post_user(topic_id).to_i
# Update the topic with the calculated values
topic.update_columns(
updated_at: new_updated_at,
posts_count: new_posts_count,
last_posted_at: new_last_posted_at,
bumped_at: new_bumped_at,
last_post_user_id: new_last_post_user_id
)
rescue ActiveRecord::RecordNotFound
puts "WARNING: Could not find Topic with id=#{topic_id}. Skipping..."
end
end
end
def update_user_stats def update_user_stats
# Update user profiles with the number of topics and posts created # Update user profiles with the number of topics and posts created
puts "Update user profiles with the number of topics and posts created" puts "Update user profiles with the number of topics and posts created"
@ -1826,7 +1865,7 @@ class GossamerForumsImporter < ImportScripts::Base
# user = User.find(user_id) # user = User.find(user_id)
# user.update!(post_count: count) # user.update!(post_count: count)
# end # end
@db.execute("SELECT * FROM user_topic_count").each do |row| @db.execute("SELECT * FROM user_topic_count_final").each do |row|
user_id, count = row user_id, count = row
# user = User.find(user_id) # user = User.find(user_id)
# user.update!(topic_count: count) # user.update!(topic_count: count)
@ -1839,7 +1878,7 @@ class GossamerForumsImporter < ImportScripts::Base
end end
end end
@db.execute("SELECT * FROM user_post_count").each do |row| @db.execute("SELECT * FROM user_post_count_final").each do |row|
user_id, count = row user_id, count = row
# user = User.find(user_id) # user = User.find(user_id)
# user.update!(post_count: count) # user.update!(post_count: count)
@ -1871,6 +1910,59 @@ class GossamerForumsImporter < ImportScripts::Base
end end
end end
def update_user_stats_final
# Update user profiles with the number of topics and posts created
puts "Update user profiles with the number of topics and posts created"
@db.execute("SELECT * FROM user_topic_count_final").each do |row|
user_id, count = row
puts "update_user_stats user_id #{user_id} topic_count #{count}"
begin
user_stat = UserStat.find_or_initialize_by(user_id: user_id)
# user_stat.update_columns(topic_count: count)
# Accumulate the current topic count with the new count
new_topic_count = user_stat.topic_count.to_i + count.to_i
user_stat.update_columns(topic_count: new_topic_count)
rescue
puts "WARNING: Could not find User with id=#{user_id}. Skipping..."
end
end
@db.execute("SELECT * FROM user_post_count_final").each do |row|
user_id, count = row
puts "update_user_stats user_id #{user_id} post_count #{count}"
begin
user_stat = UserStat.find_or_initialize_by(user_id: user_id)
# user_stat.update_columns(post_count: count)
# Accumulate the current post count with the new count
new_post_count = user_stat.post_count.to_i + count.to_i
user_stat.update_columns(post_count: new_post_count)
# Fetch the current user and check if Trust Level needs updating
new_trust_level = case count
when 0..29 then 1 # basic user
else 2 # member, regular reserved for now.
# when 3..50 then 2 # member
# else 3 # regular or above when 51..100
end
user = User.find(user_id)
current_trust_level = user.trust_level || 1 # default to 1 if not set
# Only update trust level if the new level is higher than the current one
if new_trust_level != current_trust_level
user.update!(trust_level: new_trust_level)
puts "update_user_stats user_id #{user_id} trust_level was #{current_trust_level}, now updated to #{new_trust_level}"
else
puts "update_user_stats user_id #{user_id} trust_level remains at #{current_trust_level}"
end
rescue
puts "WARNING: Could not find or modify User with id=#{user_id}. Skipping..."
end
end
end
# Import personal messages from gforum_Message table (both inbox and sent messages) # Import personal messages from gforum_Message table (both inbox and sent messages)
def import_personal_messages def import_personal_messages
@ -2052,16 +2144,20 @@ class GossamerForumsImporter < ImportScripts::Base
### import_categories ### import_categories
## import_topics_and_posts import_topics_and_posts
# threaded_topic_import # threaded_topic_import
### update_topic_stats #### update_topic_stats
### update_user_stats #### update_user_stats
update_topic_stats_final
update_user_stats_final
### export_url_mapping_to_csv("/bitnami/discourse/sqlite/gossamer-migration-url-mapping#{timestamp}") ### export_url_mapping_to_csv("/bitnami/discourse/sqlite/gossamer-migration-url-mapping#{timestamp}")
### export_nginx_rewrite_rules("/bitnami/discourse/sqlite/gossamer-redirects#{timestamp}.conf") ### export_nginx_rewrite_rules("/bitnami/discourse/sqlite/gossamer-redirects#{timestamp}.conf")
#### update_existing_personal_message_activity #### update_existing_personal_message_activity
import_personal_messages ### import_personal_messages
puts "Gossamer Forums import complete! #{timestamp}" puts "Gossamer Forums import complete! #{timestamp}"
end end