20240601-3
This commit is contained in:
dsainty
parent
d0c5884cda
commit
3bb197da4f
@ -1,11 +1,14 @@
|
||||
require 'mysql2'
|
||||
require 'open-uri'
|
||||
require 'net/http'
|
||||
|
||||
require File.expand_path("../../../config/environment", __FILE__)
|
||||
require_relative 'base'
|
||||
|
||||
class GossamerForumsImporter < ImportScripts::Base
|
||||
def initialize
|
||||
super
|
||||
# Initialize MySQL client with connection details
|
||||
# Initialize MySQL client to connect to Gossamer Forums database
|
||||
@mysql_client = Mysql2::Client.new(
|
||||
host: "slowtwitch.northend.network",
|
||||
username: "admin",
|
||||
@ -14,33 +17,25 @@ class GossamerForumsImporter < ImportScripts::Base
|
||||
)
|
||||
end
|
||||
|
||||
# Execute an SQL query on the Gossamer Forums MySQL database
|
||||
# Execute an SQL query on the Gossamer Forums database
|
||||
def execute_query(query)
|
||||
@mysql_client.query(query, as: :hash)
|
||||
end
|
||||
|
||||
# Sanitize username to comply with Discourse's rules
|
||||
# Sanitize the username to meet Discourse's requirements
|
||||
def sanitize_username(username, email, name)
|
||||
original_username = username
|
||||
# Replace unacceptable characters with underscores
|
||||
sanitized = username.gsub(/[^a-zA-Z0-9._-]/, '_')
|
||||
# Ensure the username is at least 2 characters long
|
||||
sanitized = "#{sanitized}." if sanitized.length < 2
|
||||
# Ensure the username is no more than 20 characters long
|
||||
sanitized = "#{sanitized}." if sanitized.length < 2 # Allow two-character usernames
|
||||
sanitized = sanitized[0, 20] if sanitized.length > 20
|
||||
original_sanitized = sanitized
|
||||
|
||||
# Check for existing user with the same username
|
||||
existing_user = User.find_by(username: sanitized)
|
||||
|
||||
if existing_user
|
||||
# If email and name match, do not modify the username
|
||||
# puts "existing_user.email: '#{existing_user.email}' vs. email: '#{email}'"
|
||||
# puts "existing_user.name: '#{existing_user.name}' vs. email: '#{name}'"
|
||||
if existing_user.email.downcase == email.downcase && existing_user.name == name
|
||||
return sanitized
|
||||
else
|
||||
# Ensure the username is unique
|
||||
counter = 1
|
||||
while User.exists?(username: sanitized)
|
||||
sanitized = "#{original_sanitized}_#{counter}"
|
||||
@ -50,7 +45,6 @@ class GossamerForumsImporter < ImportScripts::Base
|
||||
end
|
||||
end
|
||||
|
||||
# Print the original and sanitized usernames if they differ
|
||||
if original_username != sanitized
|
||||
puts "Sanitized username: '#{original_username}' --> '#{sanitized}'"
|
||||
# else
|
||||
@ -60,6 +54,7 @@ class GossamerForumsImporter < ImportScripts::Base
|
||||
sanitized
|
||||
end
|
||||
|
||||
|
||||
# Sanitize email to replace restricted domains
|
||||
def sanitize_email(email)
|
||||
restricted_domains = ['mailinator.com', 'example.com'] # Add more restricted domains as needed
|
||||
@ -74,17 +69,38 @@ class GossamerForumsImporter < ImportScripts::Base
|
||||
email
|
||||
end
|
||||
|
||||
# Import users from Gossamer Forums gforum_User table to Discourse
|
||||
|
||||
# Helper method to download an image from a URL
|
||||
def download_image(url)
|
||||
begin
|
||||
URI.open(url).read
|
||||
rescue OpenURI::HTTPError => e
|
||||
puts "Failed to download image from #{url}: #{e.message}"
|
||||
nil
|
||||
end
|
||||
end
|
||||
|
||||
# Helper method to upload an image to Discourse
|
||||
def upload_image(user, image_data, filename)
|
||||
return if image_data.nil?
|
||||
|
||||
upload = Upload.create_for(user.id, File.open(image_data.path), filename, 'image/jpeg')
|
||||
if upload.nil? || !upload.persisted?
|
||||
puts "Failed to upload image for user #{user.username}"
|
||||
return
|
||||
end
|
||||
|
||||
upload
|
||||
end
|
||||
|
||||
|
||||
# Import users from Gossamer Forums to Discourse
|
||||
def import_users
|
||||
puts "Importing users..."
|
||||
users = []
|
||||
|
||||
# Fetch all users from Gossamer Forums
|
||||
execute_query("SELECT * FROM gforum_User").each do |row|
|
||||
|
||||
# sanitized_username = sanitize_username(row['user_username'], row['user_email'], row['user_real_name'])
|
||||
# sanitized_email = sanitize_email(row['user_email'])
|
||||
|
||||
users << {
|
||||
id: row['user_id'],
|
||||
username: sanitize_username(row['user_username'], row['user_email'], row['user_real_name']),
|
||||
@ -119,23 +135,56 @@ class GossamerForumsImporter < ImportScripts::Base
|
||||
next
|
||||
end
|
||||
|
||||
if discourse_user.user_profile.bio_raw.nil? || discourse_user.user_profile.bio_raw.empty?
|
||||
# Append bio if it exists, otherwise set it to empty string to avoid nil errors
|
||||
discourse_user.user_profile.bio_raw ||= ""
|
||||
if discourse_user.user_profile.bio_raw.empty?
|
||||
discourse_user.user_profile.bio_raw = user[:bio_raw]
|
||||
else
|
||||
discourse_user.user_profile.bio_raw += "\n\n" + user[:bio_raw]
|
||||
end
|
||||
|
||||
if discourse_user.user_profile.bio_raw.length > 2999
|
||||
# Ensure the bio does not exceed 3000 characters
|
||||
if discourse_user.user_profile.bio_raw.length > 3000
|
||||
puts "Warning: About Me for user #{discourse_user.username} (ID: #{discourse_user.id}) exceeds 3000 characters. Truncating."
|
||||
discourse_user.user_profile.bio_raw = discourse_user.user_profile.bio_raw[0, 2999]
|
||||
discourse_user.user_profile.bio_raw = discourse_user.user_profile.bio_raw[0, 3000]
|
||||
end
|
||||
discourse_user.user_profile.save!
|
||||
|
||||
# Import user files
|
||||
import_user_files(discourse_user)
|
||||
end
|
||||
end
|
||||
|
||||
# Import user files from Gossamer Forums to Discourse
|
||||
# # Import user files from Gossamer Forums to Discourse
|
||||
# def import_user_files(user)
|
||||
# print "\rImporting files for user #{user.username}..."
|
||||
#
|
||||
# original_gossamer_id = user.custom_fields['original_gossamer_id']
|
||||
# if original_gossamer_id.nil? || original_gossamer_id.empty?
|
||||
# puts "User #{user.username} does not have a valid original_gossamer_id. Skipping file import."
|
||||
# return
|
||||
# end
|
||||
#
|
||||
# # puts "Original Gossamer ID for user #{user.username}: #{original_gossamer_id}"
|
||||
#
|
||||
# # Fetch and import user files
|
||||
# execute_query("SELECT * FROM gforum_User_Files WHERE ForeignColKey = #{original_gossamer_id}").each do |file|
|
||||
# # Construct the file URL
|
||||
# file_url = "https://forum.slowtwitch.com/images/users/images/#{file['ID'] % 10}/#{file['ID']}-#{file['File_Name']}"
|
||||
# puts "User #{user.username} User ID: #{user.id} original_gossamer_id: #{original_gossamer_id} file_url: #{file_url}"
|
||||
#
|
||||
# new_bio = user.user_profile.bio_raw + "\n\n![#{file['File_Name']}](#{file_url})"
|
||||
# if new_bio.length > 3000
|
||||
# puts "Warning: About Me for user #{user.username} (ID: #{user.id}) exceeds 3000 characters after adding file link. Truncating."
|
||||
# new_bio = new_bio[0, 3000]
|
||||
# end
|
||||
# user.user_profile.bio_raw = new_bio
|
||||
# user.user_profile.save!
|
||||
# end
|
||||
# print "Importing files for user #{user.username}... Done.\n"
|
||||
# end
|
||||
|
||||
# Import user files (profile images) from Gossamer Forums to Discourse
|
||||
def import_user_files(user)
|
||||
print "\rImporting files for user #{user.username}..."
|
||||
|
||||
@ -145,77 +194,110 @@ class GossamerForumsImporter < ImportScripts::Base
|
||||
return
|
||||
end
|
||||
|
||||
# puts "Original Gossamer ID for user #{user.username}: #{original_gossamer_id}"
|
||||
puts "Original Gossamer ID for user #{user.username}: #{original_gossamer_id}"
|
||||
|
||||
images_imported = 0
|
||||
|
||||
execute_query("SELECT * FROM gforum_User_Files WHERE ForeignColKey = #{original_gossamer_id}").each do |file|
|
||||
# Construct the file URL
|
||||
file_url = "https://forum.slowtwitch.com/images/users/images/#{file['ID'] % 10}/#{file['ID']}-#{file['File_Name']}"
|
||||
puts "User #{user.username} User ID: #{user.id} original_gossamer_id: #{original_gossamer_id} file_url: #{file_url}"
|
||||
|
||||
new_bio = user.user_profile.bio_raw + "\n\n![#{file['File_Name']}](#{file_url})"
|
||||
if new_bio.length > 3000
|
||||
puts "Warning: About Me for user #{user.username} (ID: #{user.id}) exceeds 3000 characters after adding file link. Truncating."
|
||||
new_bio = new_bio[0, 3000]
|
||||
end
|
||||
user.user_profile.bio_raw = new_bio
|
||||
next unless file['ForeignColName'] =~ /^user_image\d+$/
|
||||
|
||||
image_data = download_image(file_url)
|
||||
next if image_data.nil?
|
||||
|
||||
temp_file = Tempfile.new(['user_image', '.jpg'])
|
||||
temp_file.binmode
|
||||
temp_file.write(image_data)
|
||||
temp_file.rewind
|
||||
|
||||
if images_imported == 0
|
||||
upload = upload_image(user, temp_file, file['File_Name'])
|
||||
next if upload.nil?
|
||||
|
||||
user.user_avatar = UserAvatar.create!(user_id: user.id, custom_upload_id: upload.id)
|
||||
user.save!
|
||||
images_imported += 1
|
||||
else
|
||||
user.user_profile.bio_raw ||= ""
|
||||
user.user_profile.bio_raw += "\n\n![#{file['File_Name']}](#{file_url})"
|
||||
user.user_profile.save!
|
||||
end
|
||||
|
||||
temp_file.close
|
||||
temp_file.unlink
|
||||
end
|
||||
print "Importing files for user #{user.username}... Done.\n"
|
||||
end
|
||||
|
||||
|
||||
# Import categories from Gossamer Forums to Discourse
|
||||
def import_categories
|
||||
puts "Importing categories..."
|
||||
execute_query("SELECT * FROM gforum_Category").each do |row|
|
||||
puts "Importing categories (forums)..."
|
||||
execute_query("SELECT * FROM gforum_Forum").each do |row|
|
||||
# Only create category if it does not exist
|
||||
unless CategoryCustomField.exists?(name: 'original_gossamer_id', value: row['category_id'])
|
||||
unless CategoryCustomField.exists?(name: 'original_gossamer_id', value: row['forum_id'])
|
||||
category_name = row['forum_name']
|
||||
category_description = row['forum_desc'] || "No description provided"
|
||||
|
||||
# Create category in Discourse
|
||||
category = create_category(
|
||||
id: row['category_id'],
|
||||
name: row['name'],
|
||||
description: row['description'],
|
||||
created_at: row['created_at'] ? Time.at(row['created_at']) : Time.now,
|
||||
updated_at: row['updated_at'] ? Time.at(row['updated_at']) : Time.now
|
||||
{
|
||||
id: row['forum_id'],
|
||||
name: category_name,
|
||||
description: category_description,
|
||||
created_at: row['forum_last'] ? Time.at(row['forum_last']) : Time.now,
|
||||
updated_at: row['forum_last'] ? Time.at(row['forum_last']) : Time.now
|
||||
},
|
||||
row['forum_id'] # import_id argument
|
||||
)
|
||||
category.custom_fields.create!(name: 'original_gossamer_id', value: row['category_id'])
|
||||
|
||||
# # Map Gossamer forum ID to Discourse category ID for future reference
|
||||
# @forum_id_map[row['forum_id']] = category.id
|
||||
|
||||
# category.custom_fields.create!(name: 'original_gossamer_id', value: row['forum_id'])
|
||||
category.custom_fields['original_gossamer_id'] = row['forum_id']
|
||||
category.save!
|
||||
end
|
||||
end
|
||||
puts "Importing categories... Done."
|
||||
end
|
||||
|
||||
# Import topics and posts from Gossamer Forums to Discourse
|
||||
def import_topics_and_posts
|
||||
# Import topics and posts from Gossamer Forums to Discourse
|
||||
def import_topics_and_posts
|
||||
puts "Importing topics and posts..."
|
||||
execute_query("SELECT * FROM gforum_Post ORDER BY post_root_id, post_time").each do |row|
|
||||
if row['post_id'] == row['post_root_id']
|
||||
# Skip if the topic already exists
|
||||
unless TopicCustomField.exists?(name: 'original_gossamer_id', value: row['post_id'])
|
||||
# Create the topic
|
||||
topic = create_topic(
|
||||
id: row['post_id'],
|
||||
topic = Topic.create!(
|
||||
title: row['post_subject'],
|
||||
user_id: row['user_id_fk'],
|
||||
created_at: Time.at(row['post_time']),
|
||||
updated_at: Time.at(row['post_latest_reply']),
|
||||
category_id: row['forum_id_fk']
|
||||
)
|
||||
topic.custom_fields.create!(name: 'original_gossamer_id', value: row['post_id'])
|
||||
topic.custom_fields['original_gossamer_id'] = row['post_id']
|
||||
topic.save!
|
||||
|
||||
# Create the initial post in the topic
|
||||
post = create_post(
|
||||
id: row['post_id'],
|
||||
topic_id: row['post_id'],
|
||||
post = Post.create!(
|
||||
topic_id: topic.id,
|
||||
user_id: row['user_id_fk'],
|
||||
raw: import_post_attachments(row['post_message'], row['post_id']),
|
||||
created_at: Time.at(row['post_time']),
|
||||
updated_at: Time.at(row['post_latest_reply'])
|
||||
)
|
||||
post.custom_fields.create!(name: 'original_gossamer_id', value: row['post_id'])
|
||||
post.custom_fields['original_gossamer_id'] = row['post_id']
|
||||
post.save!
|
||||
end
|
||||
else
|
||||
# Skip if the post already exists
|
||||
unless PostCustomField.exists?(name: 'original_gossamer_id', value: row['post_id'])
|
||||
# Create the post in the existing topic
|
||||
post = create_post(
|
||||
id: row['post_id'],
|
||||
post = Post.create!(
|
||||
topic_id: row['post_root_id'],
|
||||
user_id: row['user_id_fk'],
|
||||
raw: import_post_attachments(row['post_message'], row['post_id']),
|
||||
@ -223,46 +305,51 @@ class GossamerForumsImporter < ImportScripts::Base
|
||||
updated_at: Time.at(row['post_latest_reply']),
|
||||
reply_to_post_number: row['post_father_id']
|
||||
)
|
||||
post.custom_fields.create!(name: 'original_gossamer_id', value: row['post_id'])
|
||||
end
|
||||
post.custom_fields['original_gossamer_id'] = row['post_id']
|
||||
post.save!
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# Import attachments for a post
|
||||
def import_post_attachments(post_message, post_id)
|
||||
# Import attachments for a post
|
||||
def import_post_attachments(post_message, post_id)
|
||||
# Fetch attachments related to the post
|
||||
attachments = execute_query("SELECT * FROM gforum_PostAttachment WHERE post_id_fk = #{post_id}")
|
||||
attachments.each do |attachment|
|
||||
attachment_url = "https://forum.slowtwitch.com/images/posts/#{post_id}/#{attachment['attachment_name']}"
|
||||
post_message += "\n\n![#{attachment['attachment_name']}](#{attachment_url})"
|
||||
# Append attachment links to the post message
|
||||
file_url = "https://forum.slowtwitch.com/images/posts/attachments/#{attachment['ID'] % 10}/#{attachment['ID']}-#{attachment['File_Name']}"
|
||||
post_message += "\n\n![#{attachment['File_Name']}](#{file_url})"
|
||||
end
|
||||
post_message
|
||||
end
|
||||
end
|
||||
|
||||
# Import personal messages (both inbox and sent messages)
|
||||
def import_personal_messages
|
||||
# Import personal messages (both inbox and sent messages)
|
||||
def import_personal_messages
|
||||
puts "Importing personal messages..."
|
||||
import_inbox_messages
|
||||
import_sent_messages
|
||||
end
|
||||
end
|
||||
|
||||
# Import inbox messages from gforum_Message table
|
||||
def import_inbox_messages
|
||||
# Import inbox messages from gforum_Message table
|
||||
def import_inbox_messages
|
||||
puts "Importing inbox messages..."
|
||||
execute_query("SELECT * FROM gforum_Message").each do |row|
|
||||
# Skip if the message already exists
|
||||
unless TopicCustomField.exists?(name: 'original_gossamer_msg_id', value: row['msg_id'])
|
||||
# Create a private message topic in Discourse
|
||||
topic = create_topic(
|
||||
topic = Topic.create!(
|
||||
title: row['msg_subject'],
|
||||
user_id: row['from_user_id_fk'],
|
||||
archetype: Archetype.private_message,
|
||||
created_at: Time.at(row['msg_time']),
|
||||
updated_at: Time.at(row['msg_time'])
|
||||
)
|
||||
topic.custom_fields.create!(name: 'original_gossamer_msg_id', value: row['msg_id'])
|
||||
topic.custom_fields['original_gossamer_msg_id'] = row['msg_id']
|
||||
topic.save!
|
||||
|
||||
# Create the message as a post in the private topic
|
||||
create_post(
|
||||
Post.create!(
|
||||
topic_id: topic.id,
|
||||
user_id: row['from_user_id_fk'],
|
||||
raw: row['msg_body'],
|
||||
@ -274,25 +361,27 @@ class GossamerForumsImporter < ImportScripts::Base
|
||||
topic.topic_allowed_users.create!(user_id: row['to_user_id_fk'])
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# Import sent messages from gforum_SentMessage table
|
||||
def import_sent_messages
|
||||
# Import sent messages from gforum_SentMessage table
|
||||
def import_sent_messages
|
||||
puts "Importing sent messages..."
|
||||
execute_query("SELECT * FROM gforum_SentMessage").each do |row|
|
||||
# Skip if the message already exists
|
||||
unless TopicCustomField.exists?(name: 'original_gossamer_sent_msg_id', value: row['msg_id'])
|
||||
# Create a private message topic in Discourse
|
||||
topic = create_topic(
|
||||
topic = Topic.create!(
|
||||
title: row['msg_subject'],
|
||||
user_id: row['from_user_id_fk'],
|
||||
archetype: Archetype.private_message,
|
||||
created_at: Time.at(row['msg_time']),
|
||||
updated_at: Time.at(row['msg_time'])
|
||||
)
|
||||
topic.custom_fields.create!(name: 'original_gossamer_sent_msg_id', value: row['msg_id'])
|
||||
topic.custom_fields['original_gossamer_sent_msg_id'] = row['msg_id']
|
||||
topic.save!
|
||||
|
||||
# Create the message as a post in the private topic
|
||||
create_post(
|
||||
Post.create!(
|
||||
topic_id: topic.id,
|
||||
user_id: row['from_user_id_fk'],
|
||||
raw: row['msg_body'],
|
||||
@ -304,12 +393,12 @@ class GossamerForumsImporter < ImportScripts::Base
|
||||
topic.topic_allowed_users.create!(user_id: row['to_user_id_fk'])
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# Main method to perform the import
|
||||
def perform_import
|
||||
puts "Starting Gossamer Forums import..."
|
||||
import_users
|
||||
# import_users
|
||||
import_categories
|
||||
import_topics_and_posts
|
||||
import_personal_messages
|
||||
@ -317,9 +406,5 @@ class GossamerForumsImporter < ImportScripts::Base
|
||||
end
|
||||
end
|
||||
|
||||
# Create an instance of the importer and start the import process
|
||||
# importer = GossamerForumsImporter.new
|
||||
# importer.perform_import
|
||||
|
||||
GossamerForumsImporter.new.perform_import
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user