saint/discourse-import_scripts
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

20240531-2

Browse Source
This commit is contained in:
dsainty 2024-06-17 19:35:13 +10:00
parent d2edc3f39f
commit d0c5884cda
1 changed files with 146 additions and 116 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

262
gossamer_forums.rb
View File

@ -14,7 +14,7 @@ class GossamerForumsImporter < ImportScripts::Base
) )
end end
# Execute a query on the MySQL database # Execute an SQL query on the Gossamer Forums MySQL database
def execute_query(query) def execute_query(query)
@mysql_client.query(query, as: :hash) @mysql_client.query(query, as: :hash)
end end
@ -24,7 +24,7 @@ class GossamerForumsImporter < ImportScripts::Base
original_username = username original_username = username
# Replace unacceptable characters with underscores # Replace unacceptable characters with underscores
sanitized = username.gsub(/[^a-zA-Z0-9._-]/, '_') sanitized = username.gsub(/[^a-zA-Z0-9._-]/, '_')
# Ensure the username is at least 3 characters long # Ensure the username is at least 2 characters long
sanitized = "#{sanitized}." if sanitized.length < 2 sanitized = "#{sanitized}." if sanitized.length < 2
# Ensure the username is no more than 20 characters long # Ensure the username is no more than 20 characters long
sanitized = sanitized[0, 20] if sanitized.length > 20 sanitized = sanitized[0, 20] if sanitized.length > 20
@ -74,18 +74,20 @@ class GossamerForumsImporter < ImportScripts::Base
email email
end end
# Import users from gforum_User table # Import users from Gossamer Forums gforum_User table to Discourse
def import_users def import_users
puts "Importing users..." puts "Importing users..."
users = [] users = []
# Fetch all users from Gossamer Forums
execute_query("SELECT * FROM gforum_User").each do |row| execute_query("SELECT * FROM gforum_User").each do |row|
sanitized_username = sanitize_username(row['user_username'], row['user_email'], row['user_real_name'])
# sanitized_username = sanitize_username(row['user_username'], row['user_email'], row['user_real_name'])
# sanitized_email = sanitize_email(row['user_email']) # sanitized_email = sanitize_email(row['user_email'])
users << { users << {
id: row['user_id'], id: row['user_id'],
username: sanitized_username, username: sanitize_username(row['user_username'], row['user_email'], row['user_real_name']),
email: row['user_email'], email: row['user_email'],
created_at: Time.at(row['user_registered']), created_at: Time.at(row['user_registered']),
updated_at: Time.at(row['user_last_seen']), updated_at: Time.at(row['user_last_seen']),
@ -95,35 +97,45 @@ class GossamerForumsImporter < ImportScripts::Base
website: row['user_homepage'], website: row['user_homepage'],
location: row['user_location'], location: row['user_location'],
custom_fields: { custom_fields: {
'md5_password' => row['user_password'], md5_password: row['user_password'],
'original_gossamer_username' => row['user_username'], original_username: row['user_username'],
'original_gossamer_id' => row['user_id'] original_gossamer_id: row['user_id']
} }
} }
end end
# Create users in Discourse with the required block # Create or update users in Discourse
create_users(users) do |user| create_users(users) do |user|
user user
end end
# Import user files after creating users # Append user bio and import user files
users.each do |user| users.each do |user|
discourse_user = User.find_by(username: user[:username]) discourse_username = sanitize_username(user[:username], user[:email], user[:name])
discourse_user = User.find_by(username: discourse_username)
if discourse_user.nil? if discourse_user.nil?
puts "User #{user[:username]} not found in Discourse. Skipping file import." puts "User #{user[:username]} --> #{discourse_username} not found in Discourse. Skipping file import."
next next
end end
# Update user profile bio_raw with user_about if discourse_user.user_profile.bio_raw.nil? || discourse_user.user_profile.bio_raw.empty?
discourse_user.user_profile.bio_raw = (discourse_user.user_profile.bio_raw || "") + "\n\n" + (user[:bio_raw] || "") discourse_user.user_profile.bio_raw = user[:bio_raw]
else
discourse_user.user_profile.bio_raw += "\n\n" + user[:bio_raw]
end
if discourse_user.user_profile.bio_raw.length > 2999
puts "Warning: About Me for user #{discourse_user.username} (ID: #{discourse_user.id}) exceeds 3000 characters. Truncating."
discourse_user.user_profile.bio_raw = discourse_user.user_profile.bio_raw[0, 2999]
end
discourse_user.user_profile.save! discourse_user.user_profile.save!
import_user_files(discourse_user) import_user_files(discourse_user)
end end
end end
# Import user files and append to user's bio # Import user files from Gossamer Forums to Discourse
def import_user_files(user) def import_user_files(user)
print "\rImporting files for user #{user.username}..." print "\rImporting files for user #{user.username}..."
@ -136,83 +148,93 @@ class GossamerForumsImporter < ImportScripts::Base
# puts "Original Gossamer ID for user #{user.username}: #{original_gossamer_id}" # puts "Original Gossamer ID for user #{user.username}: #{original_gossamer_id}"
execute_query("SELECT * FROM gforum_User_Files WHERE ForeignColKey = #{original_gossamer_id}").each do |file| execute_query("SELECT * FROM gforum_User_Files WHERE ForeignColKey = #{original_gossamer_id}").each do |file|
# Construct the file URL
# execute_query("SELECT * FROM gforum_User_Files WHERE ForeignColName = 'user_id' AND ForeignColKey = #{user.id}").each do |file|
# execute_query("SELECT * FROM gforum_User_Files WHERE ForeignColKey = #{user.custom_fields['original_gossamer_id']}").each do |file|
# Construct file URL
file_url = "https://forum.slowtwitch.com/images/users/images/#{file['ID'] % 10}/#{file['ID']}-#{file['File_Name']}" file_url = "https://forum.slowtwitch.com/images/users/images/#{file['ID'] % 10}/#{file['ID']}-#{file['File_Name']}"
puts "User #{user.username} User ID: #{user.id} original_gossamer_id: #{user.custom_fields['original_gossamer_id']} file_url: #{file_url}" puts "User #{user.username} User ID: #{user.id} original_gossamer_id: #{original_gossamer_id} file_url: #{file_url}"
# Append image link to user's bio
user.user_profile.bio_raw += "\n\n![#{file['File_Name']}](#{file_url})" new_bio = user.user_profile.bio_raw + "\n\n![#{file['File_Name']}](#{file_url})"
if new_bio.length > 3000
puts "Warning: About Me for user #{user.username} (ID: #{user.id}) exceeds 3000 characters after adding file link. Truncating."
new_bio = new_bio[0, 3000]
end
user.user_profile.bio_raw = new_bio
user.user_profile.save! user.user_profile.save!
end end
print "Importing files for user #{user.username}... Done.\n" print "Importing files for user #{user.username}... Done.\n"
end end
# Import categories from gforum_Category table # Import categories from Gossamer Forums to Discourse
def import_categories def import_categories
puts "Importing categories..." puts "Importing categories..."
execute_query("SELECT * FROM gforum_Category").each do |row| execute_query("SELECT * FROM gforum_Category").each do |row|
# Use current time if created_at or updated_at is null # Only create category if it does not exist
created_at = row['created_at'] ? Time.at(row['created_at']) : Time.now unless CategoryCustomField.exists?(name: 'original_gossamer_id', value: row['category_id'])
updated_at = row['updated_at'] ? Time.at(row['updated_at']) : Time.now category = create_category(
id: row['category_id'],
create_category( name: row['name'],
id: row['category_id'], description: row['description'],
name: row['name'], created_at: row['created_at'] ? Time.at(row['created_at']) : Time.now,
description: row['description'], updated_at: row['updated_at'] ? Time.at(row['updated_at']) : Time.now
created_at: created_at,
updated_at: updated_at
)
end
end
# Import topics and posts from gforum_Post table
def import_topics_and_posts
puts "Importing topics and posts..."
execute_query("SELECT * FROM gforum_Post ORDER BY post_root_id, post_time").each do |row|
if row['post_id'] == row['post_root_id']
# This is the root post, create a new topic
topic = create_topic(
id: row['post_id'],
title: row['post_subject'],
user_id: row['user_id_fk'],
created_at: Time.at(row['post_time']),
updated_at: Time.at(row['post_latest_reply']),
category_id: row['forum_id_fk']
)
# Create the first post in the topic
create_post(
id: row['post_id'],
topic_id: row['post_id'],
user_id: row['user_id_fk'],
raw: import_post_attachments(row['post_message'], row['post_id']),
created_at: Time.at(row['post_time']),
updated_at: Time.at(row['post_latest_reply'])
)
else
# This is a reply post, add to the existing topic
create_post(
id: row['post_id'],
topic_id: row['post_root_id'],
user_id: row['user_id_fk'],
raw: import_post_attachments(row['post_message'], row['post_id']),
created_at: Time.at(row['post_time']),
updated_at: Time.at(row['post_latest_reply']),
reply_to_post_number: row['post_father_id']
) )
category.custom_fields.create!(name: 'original_gossamer_id', value: row['category_id'])
end end
end end
end end
# Import post attachments from gforum_PostAttachment table # Import topics and posts from Gossamer Forums to Discourse
def import_topics_and_posts
puts "Importing topics and posts..."
execute_query("SELECT * FROM gforum_Post ORDER BY post_root_id, post_time").each do |row|
if row['post_id'] == row['post_root_id']
# Skip if the topic already exists
unless TopicCustomField.exists?(name: 'original_gossamer_id', value: row['post_id'])
# Create the topic
topic = create_topic(
id: row['post_id'],
title: row['post_subject'],
user_id: row['user_id_fk'],
created_at: Time.at(row['post_time']),
updated_at: Time.at(row['post_latest_reply']),
category_id: row['forum_id_fk']
)
topic.custom_fields.create!(name: 'original_gossamer_id', value: row['post_id'])
# Create the initial post in the topic
post = create_post(
id: row['post_id'],
topic_id: row['post_id'],
user_id: row['user_id_fk'],
raw: import_post_attachments(row['post_message'], row['post_id']),
created_at: Time.at(row['post_time']),
updated_at: Time.at(row['post_latest_reply'])
)
post.custom_fields.create!(name: 'original_gossamer_id', value: row['post_id'])
end
else
# Skip if the post already exists
unless PostCustomField.exists?(name: 'original_gossamer_id', value: row['post_id'])
# Create the post in the existing topic
post = create_post(
id: row['post_id'],
topic_id: row['post_root_id'],
user_id: row['user_id_fk'],
raw: import_post_attachments(row['post_message'], row['post_id']),
created_at: Time.at(row['post_time']),
updated_at: Time.at(row['post_latest_reply']),
reply_to_post_number: row['post_father_id']
)
post.custom_fields.create!(name: 'original_gossamer_id', value: row['post_id'])
end
end
end
end
# Import attachments for a post
def import_post_attachments(post_message, post_id) def import_post_attachments(post_message, post_id)
# Query for attachments related to the post
attachments = execute_query("SELECT * FROM gforum_PostAttachment WHERE post_id_fk = #{post_id}") attachments = execute_query("SELECT * FROM gforum_PostAttachment WHERE post_id_fk = #{post_id}")
attachments.each do |attachment| attachments.each do |attachment|
# Append attachment link to the post message attachment_url = "https://forum.slowtwitch.com/images/posts/#{post_id}/#{attachment['attachment_name']}"
post_message += "\n\n![#{attachment['postatt_filename']}](https://forum.slowtwitch.com/forum/?do=post_attachment;postatt_id=#{attachment['postatt_filename']})" post_message += "\n\n![#{attachment['attachment_name']}](#{attachment_url})"
end end
post_message post_message
end end
@ -228,27 +250,29 @@ class GossamerForumsImporter < ImportScripts::Base
def import_inbox_messages def import_inbox_messages
puts "Importing inbox messages..." puts "Importing inbox messages..."
execute_query("SELECT * FROM gforum_Message").each do |row| execute_query("SELECT * FROM gforum_Message").each do |row|
# Create a private message topic in Discourse unless TopicCustomField.exists?(name: 'original_gossamer_msg_id', value: row['msg_id'])
topic = create_topic( # Create a private message topic in Discourse
title: row['msg_subject'], topic = create_topic(
user_id: row['from_user_id_fk'], title: row['msg_subject'],
archetype: Archetype.private_message, user_id: row['from_user_id_fk'],
created_at: Time.at(row['msg_time']), archetype: Archetype.private_message,
updated_at: Time.at(row['msg_time']) created_at: Time.at(row['msg_time']),
) updated_at: Time.at(row['msg_time'])
)
topic.custom_fields.create!(name: 'original_gossamer_msg_id', value: row['msg_id'])
# Create the message as a post in the private topic # Create the message as a post in the private topic
create_post( create_post(
topic_id: topic.id, topic_id: topic.id,
user_id: row['from_user_id_fk'], user_id: row['from_user_id_fk'],
raw: row['msg_body'], raw: row['msg_body'],
created_at: Time.at(row['msg_time']), created_at: Time.at(row['msg_time']),
updated_at: Time.at(row['msg_time']) updated_at: Time.at(row['msg_time'])
) )
# Add recipient user to the private message topic # Add recipient user to the private message topic
topic.add_user_by_id(row['to_user_id_fk']) topic.topic_allowed_users.create!(user_id: row['to_user_id_fk'])
topic.save! end
end end
end end
@ -256,40 +280,46 @@ class GossamerForumsImporter < ImportScripts::Base
def import_sent_messages def import_sent_messages
puts "Importing sent messages..." puts "Importing sent messages..."
execute_query("SELECT * FROM gforum_SentMessage").each do |row| execute_query("SELECT * FROM gforum_SentMessage").each do |row|
# Create a private message topic in Discourse unless TopicCustomField.exists?(name: 'original_gossamer_sent_msg_id', value: row['msg_id'])
topic = create_topic( # Create a private message topic in Discourse
title: row['msg_subject'], topic = create_topic(
user_id: row['from_user_id_fk'], title: row['msg_subject'],
archetype: Archetype.private_message, user_id: row['from_user_id_fk'],
created_at: Time.at(row['msg_time']), archetype: Archetype.private_message,
updated_at: Time.at(row['msg_time']) created_at: Time.at(row['msg_time']),
) updated_at: Time.at(row['msg_time'])
)
topic.custom_fields.create!(name: 'original_gossamer_sent_msg_id', value: row['msg_id'])
# Create the message as a post in the private topic # Create the message as a post in the private topic
create_post( create_post(
topic_id: topic.id, topic_id: topic.id,
user_id: row['from_user_id_fk'], user_id: row['from_user_id_fk'],
raw: row['msg_body'], raw: row['msg_body'],
created_at: Time.at(row['msg_time']), created_at: Time.at(row['msg_time']),
updated_at: Time.at(row['msg_time']) updated_at: Time.at(row['msg_time'])
) )
# Add recipient user to the private message topic # Add recipient user to the private message topic
topic.add_user_by_id(row['to_user_id_fk']) topic.topic_allowed_users.create!(user_id: row['to_user_id_fk'])
topic.save! end
end end
end end
# Perform the full import process # Main method to perform the import
def perform_import def perform_import
puts "Starting Gossamer Forums import..."
import_users import_users
import_categories import_categories
import_topics_and_posts import_topics_and_posts
import_personal_messages import_personal_messages
puts "Gossamer Forums import complete!"
end end
end end
# Create an instance of the importer and start the import process # Create an instance of the importer and start the import process
importer = GossamerForumsImporter.new # importer = GossamerForumsImporter.new
importer.perform_import # importer.perform_import
GossamerForumsImporter.new.perform_import