saint/discourse-import_scripts
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

v0.20 Make code persistent with SQLite instead of using hashes for stats/numbers for posts

Browse Source
This commit is contained in:
David Sainty 2024-06-28 22:25:26 +10:00
parent 69be52fcf3
commit 0bba1c4638
1 changed files with 117 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

155
gossamer_forums.rb
View File

@ -1,5 +1,5 @@
# gossamer threads migration-import code # gossamer threads migration-import code
# v0.19.3 # v0.20
require 'mysql2' require 'mysql2'
require 'open-uri' require 'open-uri'
@ -59,12 +59,9 @@ class GossamerForumsImporter < ImportScripts::Base
real_name TEXT real_name TEXT
); );
SQL SQL
@db.execute <<-SQL # @db.execute <<-SQL
DROP TABLE IF EXISTS url_map; # DROP TABLE IF EXISTS url_map;
SQL # SQL
@db.execute <<-SQL
DROP TABLE IF EXISTS url_map2;
SQL
@db.execute <<-SQL @db.execute <<-SQL
CREATE TABLE IF NOT EXISTS url_map ( CREATE TABLE IF NOT EXISTS url_map (
old_post_id INTEGER PRIMARY KEY, old_post_id INTEGER PRIMARY KEY,
@ -72,6 +69,43 @@ class GossamerForumsImporter < ImportScripts::Base
title TEXT title TEXT
); );
SQL SQL
@db.execute <<-SQL
CREATE TABLE IF NOT EXISTS topic_last_post_time (
topic_id INTEGER PRIMARY KEY,
last_post_time INTEGER
);
SQL
@db.execute <<-SQL
CREATE TABLE IF NOT EXISTS topic_post_count (
topic_id INTEGER PRIMARY KEY,
post_count INTEGER DEFAULT 0
);
SQL
@db.execute <<-SQL
CREATE TABLE IF NOT EXISTS user_topic_count (
user_id INTEGER PRIMARY KEY,
topic_count INTEGER DEFAULT 0
);
SQL
@db.execute <<-SQL
CREATE TABLE IF NOT EXISTS user_post_count (
user_id INTEGER PRIMARY KEY,
post_count INTEGER DEFAULT 0
);
SQL
@db.execute <<-SQL
CREATE TABLE IF NOT EXISTS topic_last_post_user (
topic_id INTEGER PRIMARY KEY,
user_id INTEGER
);
SQL
@db.execute <<-SQL
CREATE TABLE IF NOT EXISTS topic_post_numbers (
topic_id INTEGER PRIMARY KEY,
post_number INTEGER DEFAULT 0
);
SQL
end end
def insert_user_id_mapping(old_user_id, new_user_id) def insert_user_id_mapping(old_user_id, new_user_id)
@ -135,6 +169,15 @@ class GossamerForumsImporter < ImportScripts::Base
end end
end end
# Helper methods to interact with the SQLite database for persisting topic-post related values
def fetch_from_db(table, key)
@db.get_first_value "SELECT value FROM #{table} WHERE key = ?", key
end
def update_db(table, key, value)
@db.execute "INSERT OR REPLACE INTO #{table} (key, value) VALUES (?, ?)", key, value
end
# Execute an SQL query on the Gossamer Forums database # Execute an SQL query on the Gossamer Forums database
def execute_query(query) def execute_query(query)
@mysql_client.query(query, as: :hash) @mysql_client.query(query, as: :hash)
@ -657,12 +700,12 @@ class GossamerForumsImporter < ImportScripts::Base
def import_topics_and_posts_with_attachments def import_topics_and_posts_with_attachments
puts "Importing topics and posts with attachments..." puts "Importing topics and posts with attachments..."
topic_last_post_time = {} # topic_last_post_time = {}
topic_post_count = Hash.new(0) # topic_post_count = Hash.new(0)
user_topic_count = Hash.new(0) # user_topic_count = Hash.new(0)
user_post_count = Hash.new(0) # user_post_count = Hash.new(0)
topic_last_post_user = {} # topic_last_post_user = {}
topic_post_numbers = Hash.new { |hash, key| hash[key] = 0 } # topic_post_numbers = Hash.new { |hash, key| hash[key] = 0 }
# Execute the query to get all posts ordered by post_id # Execute the query to get all posts ordered by post_id
execute_query("SELECT * FROM gforum_Post ORDER BY post_id").each do |row| execute_query("SELECT * FROM gforum_Post ORDER BY post_id").each do |row|
@ -703,11 +746,14 @@ class GossamerForumsImporter < ImportScripts::Base
topic.save! topic.save!
# Track last post time and user for the topic # Track last post time and user for the topic
topic_last_post_time[topic.id] = Time.at(row['post_time']) # topic_last_post_time[topic.id] = Time.at(row['post_time'])
topic_last_post_user[topic.id] = discourse_user_id # topic_last_post_user[topic.id] = discourse_user_id
update_db('topic_last_post_time', topic.id, Time.at(row['post_time']).to_i)
update_db('topic_last_post_user', topic.id, discourse_user_id)
# Increment the count of the number of topics created by each user # Increment the count of the number of topics created by each user
user_topic_count[discourse_user_id] += 1 # user_topic_count[discourse_user_id] += 1
update_db('user_topic_count', discourse_user_id, fetch_from_db(db, 'user_topic_count', discourse_user_id).to_i + 1)
# # Ensure the raw post stirng contents itself is acceptable to Discourse # # Ensure the raw post stirng contents itself is acceptable to Discourse
# sanitized_post_message = row['post_message']&.tr("\0", '') || "" # sanitized_post_message = row['post_message']&.tr("\0", '') || ""
@ -725,7 +771,9 @@ class GossamerForumsImporter < ImportScripts::Base
puts "CREATE POST topic.id #{topic.id} discourse_user_id #{discourse_user_id}" puts "CREATE POST topic.id #{topic.id} discourse_user_id #{discourse_user_id}"
# Increment the number of posts in the given topic. # Increment the number of posts in the given topic.
topic_post_numbers[topic.id] += 1 # topic_post_numbers[topic.id] += 1
post_number = fetch_from_db(db, 'topic_post_numbers', topic.id).to_i + 1
update_db(db, 'topic_post_numbers', topic.id, post_number)
# Create the initial post in the topic # Create the initial post in the topic
post = Post.create!( post = Post.create!(
@ -738,14 +786,17 @@ class GossamerForumsImporter < ImportScripts::Base
updated_at: Time.at(row['post_time']), updated_at: Time.at(row['post_time']),
like_count: row['post_likes'] || 0, like_count: row['post_likes'] || 0,
reads: post_views || 0, reads: post_views || 0,
post_number: topic_post_numbers[topic.id] post_number: post_number
) )
# post_number: topic_post_numbers[topic.id]
post.custom_fields['original_gossamer_id'] = row['post_id'] post.custom_fields['original_gossamer_id'] = row['post_id']
post.save! post.save!
# Track the number of posts in the topic and by the user # Track the number of posts in the topic and by the user
topic_post_count[topic.id] += 1 # topic_post_count[topic.id] += 1
user_post_count[discourse_user_id] += 1 # user_post_count[discourse_user_id] += 1
update_db('topic_post_count', topic.id, fetch_from_db(db, 'topic_post_count', topic.id).to_i + 1)
update_db('user_post_count', discourse_user_id, fetch_from_db(db, 'user_post_count', discourse_user_id).to_i + 1)
# Handle attachments for the post # Handle attachments for the post
handle_post_attachments(row['post_id'], post, discourse_user_id) handle_post_attachments(row['post_id'], post, discourse_user_id)
@ -782,9 +833,11 @@ class GossamerForumsImporter < ImportScripts::Base
# Sanitize the post message # Sanitize the post message
sanitized_post_message = sanitize_post_message(row['post_message']) sanitized_post_message = sanitize_post_message(row['post_message'])
# Create the post in the existing topic # topic_post_numbers[topic_id] += 1
topic_post_numbers[topic_id] += 1 post_number = fetch_from_db(db, 'topic_post_numbers', topic_id).to_i + 1
update_db('topic_post_numbers', topic_id, post_number)
# Create the post in the existing topic
post = Post.create!( post = Post.create!(
topic_id: topic_id, topic_id: topic_id,
user_id: discourse_user_id, user_id: discourse_user_id,
@ -795,20 +848,26 @@ class GossamerForumsImporter < ImportScripts::Base
updated_at: Time.at(row['post_time']), updated_at: Time.at(row['post_time']),
reply_to_post_number: reply_to_post_number, reply_to_post_number: reply_to_post_number,
like_count: row['post_replies'] || 0, like_count: row['post_replies'] || 0,
reads: post_views || topic_post_count[topic_id], reads: post_views || fetch_from_db('topic_post_count', topic_id).to_i,
post_number: topic_post_numbers[topic_id] post_number: post_number
) )
post.custom_fields['original_gossamer_id'] = row['post_id'] post.custom_fields['original_gossamer_id'] = row['post_id']
post.save! post.save!
# Track the number of posts in the topic and by the user # Track the number of posts in the topic and by the user
topic_post_count[topic_id] += 1 # topic_post_count[topic_id] += 1
user_post_count[discourse_user_id] += 1 # user_post_count[discourse_user_id] += 1
update_db('topic_post_count', topic_id, fetch_from_db('topic_post_count', topic_id).to_i + 1)
update_db('user_post_count', discourse_user_id, fetch_from_db('user_post_count', discourse_user_id).to_i + 1)
# Update last post time and user for the topic # Update last post time and user for the topic
if topic_last_post_time[topic_id].nil? || Time.at(row['post_time']) > topic_last_post_time[topic_id] # if topic_last_post_time[topic_id].nil? || Time.at(row['post_time']) > topic_last_post_time[topic_id]
topic_last_post_time[topic_id] = Time.at(row['post_time']) # topic_last_post_time[topic_id] = Time.at(row['post_time'])
topic_last_post_user[topic_id] = discourse_user_id # topic_last_post_user[topic_id] = discourse_user_id
# end
if fetch_from_db('topic_last_post_time', topic_id).nil? || Time.at(row['post_time']).to_i > fetch_from_db('topic_last_post_time', topic_id).to_i
update_db('topic_last_post_time', topic_id, Time.at(row['post_time']).to_i)
update_db('topic_last_post_user', topic_id, discourse_user_id)
end end
# Handle attachments for the post # Handle attachments for the post
@ -824,23 +883,43 @@ class GossamerForumsImporter < ImportScripts::Base
end end
# Update topics with the correct last post time, post count, and last post user # Update topics with the correct last post time, post count, and last post user
topic_last_post_time.each do |topic_id, last_post_time| # topic_last_post_time.each do |topic_id, last_post_time|
# Topic.find(topic_id).update!(
# updated_at: last_post_time,
# posts_count: topic_post_count[topic_id],
# last_posted_at: last_post_time,
# bumped_at: last_post_time,
# last_post_user_id: topic_last_post_user[topic_id]
# )
# end
@db.execute("SELECT * FROM topic_last_post_time").each do |row|
topic_id, last_post_time = row
Topic.find(topic_id).update!( Topic.find(topic_id).update!(
updated_at: last_post_time, updated_at: Time.at(last_post_time),
posts_count: topic_post_count[topic_id], posts_count: fetch_from_db(db, 'topic_post_count', topic_id).to_i,
last_posted_at: last_post_time, last_posted_at: Time.at(last_post_time),
bumped_at: last_post_time, bumped_at: Time.at(last_post_time),
last_post_user_id: topic_last_post_user[topic_id] last_post_user_id: fetch_from_db(db, 'topic_last_post_user', topic_id).to_i
) )
end end
# Update user profiles with the number of topics and posts created # Update user profiles with the number of topics and posts created
user_topic_count.each do |user_id, count| # user_topic_count.each do |user_id, count|
# user = User.find(user_id)
# user.update!(topic_count: count)
# end
# user_post_count.each do |user_id, count|
# user = User.find(user_id)
# user.update!(post_count: count)
# end
@db.execute("SELECT * FROM user_topic_count").each do |row|
user_id, count = row
user = User.find(user_id) user = User.find(user_id)
user.update!(topic_count: count) user.update!(topic_count: count)
end end
user_post_count.each do |user_id, count| @db.execute("SELECT * FROM user_post_count").each do |row|
user_id, count = row
user = User.find(user_id) user = User.find(user_id)
user.update!(post_count: count) user.update!(post_count: count)
end end