saint/discourse-import_scripts
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

v0.59 Final topic-post import run -- add logic for appending counts since we are adding to a live Discourse host now

Browse Source
This commit is contained in:
David Sainty 2024-09-02 13:45:44 +10:00
parent c8e7ba97bb
commit 958e37771e
1 changed files with 139 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

182
gossamer_forums.rb
View File

@ -1,7 +1,7 @@
# Federated Computer, Inc.
# David Sainty <saint@federated.computer> 2024 A.D.
# Gossamer Threads to Discourse -- Migration-Import Script
# v0.58 Final user import and personal message import run
# v0.59 Final topic-post import run -- add logic for appending counts since we are adding to a live Discourse host now
require 'mysql2'
require 'open-uri'
@ -114,47 +114,47 @@ class GossamerForumsImporter < ImportScripts::Base
SQL
# POST IMPORT - For each topic, the time of the last / most recent post/reply
@db.execute <<-SQL
CREATE TABLE IF NOT EXISTS topic_last_post_time (
CREATE TABLE IF NOT EXISTS topic_last_post_time_final (
topic_id INTEGER PRIMARY KEY,
last_post_time INTEGER
);
SQL
# POST IMPORT - For each topic, increment post_count as we add posts.
@db.execute <<-SQL
CREATE TABLE IF NOT EXISTS topic_post_count (
CREATE TABLE IF NOT EXISTS topic_post_count_final (
topic_id INTEGER PRIMARY KEY,
post_count INTEGER DEFAULT 0
);
SQL
# POST IMPORT - For each user (_id), increment topic_count as we add topics (to see total topics per user)
@db.execute <<-SQL
CREATE TABLE IF NOT EXISTS user_topic_count (
CREATE TABLE IF NOT EXISTS user_topic_count_final (
user_id INTEGER PRIMARY KEY,
topic_count INTEGER DEFAULT 0
);
SQL
# POST IMPORT - For each user (_id), increment post_count as we add posts (to see total posts per user)
@db.execute <<-SQL
CREATE TABLE IF NOT EXISTS user_post_count (
CREATE TABLE IF NOT EXISTS user_post_count_final (
user_id INTEGER PRIMARY KEY,
post_count INTEGER DEFAULT 0
);
SQL
# POST IMPORT - For each topic, the user_id for the last poster / replier
@db.execute <<-SQL
CREATE TABLE IF NOT EXISTS topic_last_post_user (
CREATE TABLE IF NOT EXISTS topic_last_post_user_final (
topic_id INTEGER PRIMARY KEY,
user_id INTEGER
);
SQL
# POST IMPORT - The number of posts in a given topic, incremented as we add a new reply post to a topic.
@db.execute <<-SQL
CREATE TABLE IF NOT EXISTS topic_post_numbers (
CREATE TABLE IF NOT EXISTS topic_post_numbers_final (
topic_id INTEGER PRIMARY KEY,
post_number INTEGER DEFAULT 0
);
SQL
# POST IMPORT - Record perssitent integer value for highest processed post id -- not used
# POST IMPORT - Record perssitent integer value for highest processed post id
@db.execute <<-SQL
CREATE TABLE IF NOT EXISTS highest_processed_post_id (
id INTEGER PRIMARY KEY CHECK (id = 1),
@ -253,51 +253,51 @@ class GossamerForumsImporter < ImportScripts::Base
end
def fetch_db_topic_last_post_time(topic_id)
@db.get_first_value "SELECT last_post_time FROM topic_last_post_time WHERE topic_id = ?", topic_id
@db.get_first_value "SELECT last_post_time FROM topic_last_post_time_final WHERE topic_id = ?", topic_id
end
def fetch_db_topic_last_post_user(topic_id)
@db.get_first_value "SELECT user_id FROM topic_last_post_user WHERE topic_id = ?", topic_id
@db.get_first_value "SELECT user_id FROM topic_last_post_user_final WHERE topic_id = ?", topic_id
end
def fetch_db_topic_post_count(topic_id)
@db.get_first_value "SELECT post_count FROM topic_post_count WHERE topic_id = ?", topic_id
@db.get_first_value "SELECT post_count FROM topic_post_count_final WHERE topic_id = ?", topic_id
end
def fetch_db_user_topic_count(user_id)
@db.get_first_value "SELECT topic_count FROM user_topic_count WHERE user_id = ?", user_id
@db.get_first_value "SELECT topic_count FROM user_topic_count_final WHERE user_id = ?", user_id
end
def fetch_db_user_post_count(user_id)
@db.get_first_value "SELECT post_count FROM user_post_count WHERE user_id = ?", user_id
@db.get_first_value "SELECT post_count FROM user_post_count_final WHERE user_id = ?", user_id
end
def fetch_db_topic_post_numbers(topic_id)
@db.get_first_value "SELECT post_number FROM topic_post_numbers WHERE topic_id = ?", topic_id
@db.get_first_value "SELECT post_number FROM topic_post_numbers_final WHERE topic_id = ?", topic_id
end
def update_db_topic_last_post_time(topic_id, last_post_time)
@db.execute "INSERT OR REPLACE INTO topic_last_post_time (topic_id, last_post_time) VALUES (?, ?)", topic_id, last_post_time
@db.execute "INSERT OR REPLACE INTO topic_last_post_time_final (topic_id, last_post_time) VALUES (?, ?)", topic_id, last_post_time
end
def update_db_topic_last_post_user(topic_id, user_id)
@db.execute "INSERT OR REPLACE INTO topic_last_post_user (topic_id, user_id) VALUES (?, ?)", topic_id, user_id
@db.execute "INSERT OR REPLACE INTO topic_last_post_user_final (topic_id, user_id) VALUES (?, ?)", topic_id, user_id
end
def update_db_topic_post_count(topic_id, post_count)
@db.execute "INSERT OR REPLACE INTO topic_post_count (topic_id, post_count) VALUES (?, ?)", topic_id, post_count
@db.execute "INSERT OR REPLACE INTO topic_post_count_final (topic_id, post_count) VALUES (?, ?)", topic_id, post_count
end
def update_db_user_topic_count(user_id, topic_count)
@db.execute "INSERT OR REPLACE INTO user_topic_count (user_id, topic_count) VALUES (?, ?)", user_id, topic_count
@db.execute "INSERT OR REPLACE INTO user_topic_count_final (user_id, topic_count) VALUES (?, ?)", user_id, topic_count
end
def update_db_user_post_count(user_id, post_count)
@db.execute "INSERT OR REPLACE INTO user_post_count (user_id, post_count) VALUES (?, ?)", user_id, post_count
@db.execute "INSERT OR REPLACE INTO user_post_count_final (user_id, post_count) VALUES (?, ?)", user_id, post_count
end
def update_db_topic_post_numbers(topic_id, post_number)
@db.execute "INSERT OR REPLACE INTO topic_post_numbers (topic_id, post_number) VALUES (?, ?)", topic_id, post_number
@db.execute "INSERT OR REPLACE INTO topic_post_numbers_final (topic_id, post_number) VALUES (?, ?)", topic_id, post_number
end
# Fetch the highest processed post_id from the highest_processed_post_id table
@ -1555,12 +1555,13 @@ class GossamerForumsImporter < ImportScripts::Base
# Fetch the highest old_post_id from the url_map table
# highest_old_post_id = fetch_highest_old_post_id.to_i
# puts "Highest (OP) old_post_id in url_map: #{highest_old_post_id}"
### highest_processed_post_id = fetch_highest_processed_post_id.to_i
highest_processed_post_id = 8179621
highest_processed_post_id = fetch_highest_processed_post_id.to_i
puts "Highest processed post_id: #{highest_processed_post_id}"
# OVERRIDE........
# Attachment example: highest_processed_post_id = 1359862
highest_processed_post_id = 8179621
puts "OVERRIDE Highest processed post_id: #{highest_processed_post_id}"
# Execute the query to get all posts ordered by post_id
execute_query("SELECT post_id, user_id_fk, forum_id_fk, post_root_id, post_subject, post_time, post_message, post_father_id, post_likes, post_replies FROM gforum_Post WHERE post_id > #{highest_processed_post_id} ORDER BY post_id").each do |row|
@ -1635,11 +1636,14 @@ class GossamerForumsImporter < ImportScripts::Base
end
end
end
current_topic_id = topic.id
# Track last post time and user for the topic
# topic_last_post_time[topic.id] = Time.at(row['post_time'])
# topic_last_post_user[topic.id] = discourse_user_id
update_db_topic_last_post_time(topic.id, Time.at(row['post_time']).to_i)
update_db_topic_last_post_user(topic.id, discourse_user_id)
update_db_topic_last_post_time(current_topic_id, Time.at(row['post_time']).to_i)
update_db_topic_last_post_user(current_topic_id, discourse_user_id)
# Increment the count of the number of topics created by each user
# user_topic_count[discourse_user_id] += 1
@ -1658,16 +1662,16 @@ class GossamerForumsImporter < ImportScripts::Base
# Sanitize the post message
sanitized_post_message = sanitize_post_message(row['post_message'])
puts "CREATE TOPIC POST topic.id #{topic.id} discourse_user_id #{discourse_user_id}"
puts "CREATE TOPIC POST current_topic_id #{current_topic_id} discourse_user_id #{discourse_user_id}"
# Increment the number of posts in the given topic.
# topic_post_numbers[topic.id] += 1
post_number = fetch_db_topic_post_numbers(topic.id).to_i + 1
update_db_topic_post_numbers(topic.id, post_number)
post_number = fetch_db_topic_post_numbers(current_topic_id).to_i + 1
update_db_topic_post_numbers(current_topic_id, post_number)
# Create the initial post in the topic
post = Post.create!(
topic_id: topic.id,
topic_id: current_topic_id,
user_id: discourse_user_id,
# raw: import_attachments(row['post_message'], row['post_id']),
# raw: row['post_message'] || "",
@ -1685,11 +1689,11 @@ class GossamerForumsImporter < ImportScripts::Base
# Track the number of posts in the topic and by the user
# topic_post_count[topic.id] += 1
# user_post_count[discourse_user_id] += 1
update_db_topic_post_count(topic.id, fetch_db_topic_post_count(topic.id).to_i + 1)
update_db_topic_post_count(current_topic_id, fetch_db_topic_post_count(current_topic_id).to_i + 1)
update_db_user_post_count(discourse_user_id, fetch_db_user_post_count(discourse_user_id).to_i + 1)
# Handle attachments for the post
handle_post_attachments(row['post_id'], post, discourse_user_id)
handle_post_attachments(row['post_id'], post, discourse_user_id, @mysql_client)
# Create URL mappings
# old_url = "https://old/forum/#{row['forum_name']}/topics/#{row['post_id']}"
@ -1727,6 +1731,8 @@ class GossamerForumsImporter < ImportScripts::Base
# Sanitize the post message
sanitized_post_message = sanitize_post_message(row['post_message'])
puts "CREATE REPLY in topic_id #{topic_id} for reply post_id #{row['post_id']}"
# topic_post_numbers[topic_id] += 1
post_number = fetch_db_topic_post_numbers(topic_id).to_i + 1
update_db_topic_post_numbers(topic_id, post_number)
@ -1735,16 +1741,17 @@ class GossamerForumsImporter < ImportScripts::Base
post = Post.create!(
topic_id: topic_id,
user_id: discourse_user_id,
# raw: import_attachments(row['post_message'], row['post_id']),
# raw: row['post_message'] || "",
raw: sanitized_post_message,
created_at: Time.at(row['post_time']),
updated_at: Time.at(row['post_time']),
reply_to_post_number: reply_to_post_number,
like_count: row['post_replies'] || 0,
reads: post_views || fetch_db_topic_post_count(topic_id).to_i,
reads: post_views || 0,
post_number: post_number
)
# raw: import_attachments(row['post_message'], row['post_id']),
# raw: row['post_message'] || "",
# reply_to_post_number: reply_to_post_number,
# like_count: row['post_replies'] || 0,
# reads: post_views || fetch_db_topic_post_count(topic_id).to_i,
post.custom_fields['original_gossamer_id'] = row['post_id']
post.save!
@ -1765,7 +1772,7 @@ class GossamerForumsImporter < ImportScripts::Base
end
# Handle attachments for the post
handle_post_attachments(row['post_id'], post, discourse_user_id)
handle_post_attachments(row['post_id'], post, discourse_user_id, @mysql_client)
# Update the highest processed post_id
puts "Updated highest processed post_id #{post_id}"
@ -1794,7 +1801,7 @@ class GossamerForumsImporter < ImportScripts::Base
# last_post_user_id: topic_last_post_user[topic_id]
# )
# end
@db.execute("SELECT * FROM topic_last_post_time").each do |row|
@db.execute("SELECT * FROM topic_last_post_time_final").each do |row|
topic_id, last_post_time = row
begin
@ -1815,6 +1822,38 @@ class GossamerForumsImporter < ImportScripts::Base
end
end
def update_topic_stats_final
# Update topics with the correct last post time, post count, and last post user
puts "Update topics with the correct last post time, post count, and last post user"
@db.execute("SELECT * FROM topic_last_post_time_final").each do |row|
topic_id, last_post_time = row
begin
topic = Topic.find(topic_id)
# Calculate the new values based on the given conditions
new_updated_at = [Time.at(last_post_time), topic.updated_at].max
new_posts_count = topic.posts_count + fetch_db_topic_post_count(topic_id).to_i
new_last_posted_at = [Time.at(last_post_time), topic.last_posted_at].max
new_bumped_at = [Time.at(last_post_time), topic.bumped_at].max
new_last_post_user_id = fetch_db_topic_last_post_user(topic_id).to_i
# Update the topic with the calculated values
topic.update_columns(
updated_at: new_updated_at,
posts_count: new_posts_count,
last_posted_at: new_last_posted_at,
bumped_at: new_bumped_at,
last_post_user_id: new_last_post_user_id
)
rescue ActiveRecord::RecordNotFound
puts "WARNING: Could not find Topic with id=#{topic_id}. Skipping..."
end
end
end
def update_user_stats
# Update user profiles with the number of topics and posts created
puts "Update user profiles with the number of topics and posts created"
@ -1826,7 +1865,7 @@ class GossamerForumsImporter < ImportScripts::Base
# user = User.find(user_id)
# user.update!(post_count: count)
# end
@db.execute("SELECT * FROM user_topic_count").each do |row|
@db.execute("SELECT * FROM user_topic_count_final").each do |row|
user_id, count = row
# user = User.find(user_id)
# user.update!(topic_count: count)
@ -1839,7 +1878,7 @@ class GossamerForumsImporter < ImportScripts::Base
end
end
@db.execute("SELECT * FROM user_post_count").each do |row|
@db.execute("SELECT * FROM user_post_count_final").each do |row|
user_id, count = row
# user = User.find(user_id)
# user.update!(post_count: count)
@ -1871,6 +1910,59 @@ class GossamerForumsImporter < ImportScripts::Base
end
end
def update_user_stats_final
# Update user profiles with the number of topics and posts created
puts "Update user profiles with the number of topics and posts created"
@db.execute("SELECT * FROM user_topic_count_final").each do |row|
user_id, count = row
puts "update_user_stats user_id #{user_id} topic_count #{count}"
begin
user_stat = UserStat.find_or_initialize_by(user_id: user_id)
# user_stat.update_columns(topic_count: count)
# Accumulate the current topic count with the new count
new_topic_count = user_stat.topic_count.to_i + count.to_i
user_stat.update_columns(topic_count: new_topic_count)
rescue
puts "WARNING: Could not find User with id=#{user_id}. Skipping..."
end
end
@db.execute("SELECT * FROM user_post_count_final").each do |row|
user_id, count = row
puts "update_user_stats user_id #{user_id} post_count #{count}"
begin
user_stat = UserStat.find_or_initialize_by(user_id: user_id)
# user_stat.update_columns(post_count: count)
# Accumulate the current post count with the new count
new_post_count = user_stat.post_count.to_i + count.to_i
user_stat.update_columns(post_count: new_post_count)
# Fetch the current user and check if Trust Level needs updating
new_trust_level = case count
when 0..29 then 1 # basic user
else 2 # member, regular reserved for now.
# when 3..50 then 2 # member
# else 3 # regular or above when 51..100
end
user = User.find(user_id)
current_trust_level = user.trust_level || 1 # default to 1 if not set
# Only update trust level if the new level is higher than the current one
if new_trust_level != current_trust_level
user.update!(trust_level: new_trust_level)
puts "update_user_stats user_id #{user_id} trust_level was #{current_trust_level}, now updated to #{new_trust_level}"
else
puts "update_user_stats user_id #{user_id} trust_level remains at #{current_trust_level}"
end
rescue
puts "WARNING: Could not find or modify User with id=#{user_id}. Skipping..."
end
end
end
# Import personal messages from gforum_Message table (both inbox and sent messages)
def import_personal_messages
@ -2052,16 +2144,20 @@ class GossamerForumsImporter < ImportScripts::Base
### import_categories
## import_topics_and_posts
import_topics_and_posts
# threaded_topic_import
### update_topic_stats
### update_user_stats
#### update_topic_stats
#### update_user_stats
update_topic_stats_final
update_user_stats_final
### export_url_mapping_to_csv("/bitnami/discourse/sqlite/gossamer-migration-url-mapping#{timestamp}")
### export_nginx_rewrite_rules("/bitnami/discourse/sqlite/gossamer-redirects#{timestamp}.conf")
#### update_existing_personal_message_activity
import_personal_messages
### import_personal_messages
puts "Gossamer Forums import complete! #{timestamp}"
end