20240606
This commit is contained in:
dsainty
parent
dc4afeb39d
commit
33ea3e2d6a
@ -36,6 +36,12 @@ class GossamerForumsImporter < ImportScripts::Base
|
||||
new_user_id INTEGER
|
||||
);
|
||||
SQL
|
||||
@db.execute <<-SQL
|
||||
CREATE TABLE IF NOT EXISTS category_id_map (
|
||||
old_category_id INTEGER PRIMARY KEY,
|
||||
new_category_id INTEGER
|
||||
);
|
||||
SQL
|
||||
end
|
||||
|
||||
def insert_user_id_mapping(old_user_id, new_user_id)
|
||||
@ -46,6 +52,14 @@ class GossamerForumsImporter < ImportScripts::Base
|
||||
@db.get_first_value "SELECT new_user_id FROM user_id_map WHERE old_user_id = ?", old_user_id
|
||||
end
|
||||
|
||||
def insert_category_id_mapping(old_category_id, new_category_id)
|
||||
@db.execute "INSERT OR REPLACE INTO category_id_map (old_category_id, new_category_id) VALUES (?, ?)", old_category_id, new_category_id
|
||||
end
|
||||
|
||||
def fetch_category_id_mapping(old_category_id)
|
||||
@db.get_first_value "SELECT new_category_id FROM category_id_map WHERE old_category_id = ?", old_category_id
|
||||
end
|
||||
|
||||
# Execute an SQL query on the Gossamer Forums database
|
||||
def execute_query(query)
|
||||
@mysql_client.query(query, as: :hash)
|
||||
@ -106,21 +120,65 @@ class GossamerForumsImporter < ImportScripts::Base
|
||||
rescue OpenURI::HTTPError => e
|
||||
puts "Failed to download image from #{url}: #{e.message}"
|
||||
nil
|
||||
rescue URI::InvalidURIError => e
|
||||
puts "Failed to handle invalid URL/URI for #{url}: #{e.message}"
|
||||
nil
|
||||
end
|
||||
end
|
||||
|
||||
# Helper method to upload an image to Discourse
|
||||
def upload_image(user, image_data, filename)
|
||||
return if image_data.nil?
|
||||
def upload_image(user, file, filename, gossamer_url)
|
||||
begin
|
||||
upload = Upload.create!(
|
||||
user_id: user.id,
|
||||
original_filename: filename,
|
||||
filesize: file.size,
|
||||
# filesize: File.size(file.path),
|
||||
# content_type: `file --brief --mime-type #{file.path}`.strip,
|
||||
# sha1: Digest::SHA1.file(file.path).hexdigest,
|
||||
# origin: 'user_avatar',
|
||||
# retain_hours: nil,
|
||||
url: gossamer_url
|
||||
)
|
||||
# Error -- non-existent method upload.ensure_consistency!
|
||||
|
||||
upload = Upload.create_for(user.id, File.open(image_data.path), filename, 'image/jpeg')
|
||||
if upload.nil? || !upload.persisted?
|
||||
puts "Failed to upload image for user #{user.username}"
|
||||
return
|
||||
end
|
||||
# Move the file to the correct location
|
||||
# FileUtils.mv(file.path, upload.path)
|
||||
upload.save!
|
||||
|
||||
upload
|
||||
rescue => e
|
||||
puts "Failed to upload image #{filename} for user #{user.username}: #{e.message}"
|
||||
nil
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
# def download_file(url)
|
||||
# require 'open-uri'
|
||||
# begin
|
||||
# file = Tempfile.new
|
||||
# file.binmode
|
||||
# file.write(URI.open(url).read)
|
||||
# file.rewind
|
||||
# file
|
||||
# rescue => e
|
||||
# puts "Failed to download file from #{url}: #{e.message}"
|
||||
# nil
|
||||
# end
|
||||
# end
|
||||
|
||||
# Helper method to upload an image to Discourse
|
||||
# def upload_image(user, image_data, filename)
|
||||
# return if image_data.nil?
|
||||
#
|
||||
# upload = Upload.create_for(user.id, File.open(image_data.path), filename, 'image/jpeg')
|
||||
# if upload.nil? || !upload.persisted?
|
||||
# puts "Failed to upload image for user #{user.username}"
|
||||
# return
|
||||
# end
|
||||
#
|
||||
# upload
|
||||
# end
|
||||
|
||||
|
||||
# Import users from Gossamer Forums to Discourse
|
||||
@ -155,7 +213,7 @@ class GossamerForumsImporter < ImportScripts::Base
|
||||
user
|
||||
end
|
||||
|
||||
# For each user, add user ID mapping to @user_id_map now that we know what the Discourse user ID is, ... and append user bio and import user files
|
||||
# For each user, add user ID mapping to SQLite now that we know what the Discourse user ID is, ... and append user bio and import user files
|
||||
users.each do |user|
|
||||
discourse_username = sanitize_username(user[:username], user[:email], user[:name])
|
||||
discourse_user = User.find_by(username: discourse_username)
|
||||
@ -167,6 +225,7 @@ class GossamerForumsImporter < ImportScripts::Base
|
||||
|
||||
# # Store the user ID mapping
|
||||
# @user_id_map[user[:id]] = discourse_user.id
|
||||
puts "for insert_user_id_mapping: user[:id] #{user[:id]} discourse_user.id #{discourse_user.id}"
|
||||
insert_user_id_mapping(user[:id], discourse_user.id)
|
||||
|
||||
# Ensure user profile exists and bio_raw is a string
|
||||
@ -188,7 +247,7 @@ class GossamerForumsImporter < ImportScripts::Base
|
||||
discourse_user.user_profile.save!
|
||||
|
||||
# Import user files
|
||||
# import_user_files(discourse_user)
|
||||
import_user_files(discourse_user)
|
||||
end
|
||||
end
|
||||
|
||||
@ -240,27 +299,39 @@ class GossamerForumsImporter < ImportScripts::Base
|
||||
puts "User #{user.username} User ID: #{user.id} original_gossamer_id: #{original_gossamer_id} file_url: #{file_url}"
|
||||
|
||||
next unless file['ForeignColName'] =~ /^user_image\d+$/
|
||||
puts "#A"
|
||||
next unless ['image/jpeg', 'image/png'].include?(file['File_MimeType'])
|
||||
puts "#B"
|
||||
|
||||
image_data = download_image(file_url)
|
||||
next if image_data.nil?
|
||||
puts "#C"
|
||||
|
||||
temp_file = Tempfile.new(['user_image', '.jpg'])
|
||||
temp_file = Tempfile.new(['user_image', File.extname(file['File_Name'])])
|
||||
temp_file.binmode
|
||||
temp_file.write(image_data)
|
||||
temp_file.rewind
|
||||
|
||||
if images_imported == 0
|
||||
upload = upload_image(user, temp_file, file['File_Name'])
|
||||
puts "#D"
|
||||
upload = upload_image(user, temp_file, file['File_Name'], file_url)
|
||||
next if upload.nil?
|
||||
|
||||
user.user_avatar = UserAvatar.create!(user_id: user.id, custom_upload_id: upload.id)
|
||||
user.save!
|
||||
|
||||
# Set the Profile Header
|
||||
UserProfile.find_by(user_id: user.id).update!(profile_background_upload_id: upload.id)
|
||||
|
||||
# Set the User Card Background
|
||||
UserProfile.find_by(user_id: user.id).update!(card_background_upload_id: upload.id)
|
||||
|
||||
images_imported += 1
|
||||
else
|
||||
user.user_profile.bio_raw ||= ""
|
||||
user.user_profile.bio_raw += "\n\n![#{file['File_Name']}](#{file_url})"
|
||||
user.user_profile.save!
|
||||
end
|
||||
puts "#E"
|
||||
user.user_profile.bio_raw ||= ""
|
||||
user.user_profile.bio_raw += "\n\n![#{file['File_Name']}](#{file_url})"
|
||||
user.user_profile.save!
|
||||
|
||||
temp_file.close
|
||||
temp_file.unlink
|
||||
@ -277,11 +348,11 @@ class GossamerForumsImporter < ImportScripts::Base
|
||||
unless CategoryCustomField.exists?(name: 'original_gossamer_id', value: row['forum_id'])
|
||||
category_name = row['forum_name']
|
||||
category_description = row['forum_desc'] || "No description provided"
|
||||
|
||||
puts "id #{row['forum_id']} name #{category_name} description #{category_description}"
|
||||
# Create category in Discourse
|
||||
category = create_category(
|
||||
{
|
||||
id: row['forum_id'] + 10,
|
||||
# id: row['forum_id'] + 10,
|
||||
name: category_name,
|
||||
description: category_description,
|
||||
created_at: row['forum_last'] ? Time.at(row['forum_last']) : Time.now,
|
||||
@ -296,6 +367,10 @@ class GossamerForumsImporter < ImportScripts::Base
|
||||
# category.custom_fields.create!(name: 'original_gossamer_id', value: row['forum_id'])
|
||||
category.custom_fields['original_gossamer_id'] = row['forum_id']
|
||||
category.save!
|
||||
|
||||
# Store the user ID mapping
|
||||
puts "for insert_category_id_mapping: category[:id] #{category[:id]} row['forum_id'] #{row['forum_id']}"
|
||||
insert_category_id_mapping(row['forum_id'], category[:id])
|
||||
end
|
||||
end
|
||||
puts "Importing categories... Done."
|
||||
@ -316,12 +391,15 @@ def import_topics_and_posts
|
||||
# Execute the query to get all posts ordered by post_id
|
||||
execute_query("SELECT * FROM gforum_Post ORDER BY post_id").each do |row|
|
||||
puts "post_id #{row['post_id']} post_root_id #{row['post_root_id']} post_subject/title #{row['post_subject']} forum_id_fk/category_id #{row['forum_id_fk']}"
|
||||
discourse_user_id = fetch_user_id_mapping(row['user_id_fk'])
|
||||
# discourse_user_id = @user_id_map[row['user_id_fk']]
|
||||
discourse_user_id = fetch_user_id_mapping(row['user_id_fk'])
|
||||
discourse_category_id = fetch_category_id_mapping(row['forum_id_fk'])
|
||||
|
||||
next unless discourse_user_id
|
||||
puts "discourse_user_id #{discourse_user_id} discourse_category_id #{discourse_category_id}"
|
||||
next unless discourse_user_id && discourse_category_id
|
||||
|
||||
if row['post_root_id'] == 0
|
||||
puts "#1"
|
||||
# Ensure the title is valid
|
||||
title = ensure_valid_title(row['post_subject'])
|
||||
|
||||
@ -329,21 +407,27 @@ def import_topics_and_posts
|
||||
unless TopicCustomField.exists?(name: 'original_gossamer_id', value: row['post_id'])
|
||||
# Create the topic
|
||||
begin
|
||||
puts "#2"
|
||||
puts "CREATE TOPIC title #{title} discourse_user_id #{discourse_user_id} category_id #{discourse_category_id}"
|
||||
topic = Topic.create!(
|
||||
title: title,
|
||||
user_id: discourse_user_id,
|
||||
created_at: Time.at(row['post_time']),
|
||||
updated_at: Time.at(row['post_latest_reply']),
|
||||
category_id: row['forum_id_fk'] + 10
|
||||
category_id: discourse_category_id
|
||||
)
|
||||
topic.custom_fields['original_gossamer_id'] = row['post_id']
|
||||
topic.save!
|
||||
|
||||
# Create the initial post in the topic
|
||||
puts "CREATE POST topic.id #{topic.id} discourse_user_id #{discourse_user_id}"
|
||||
sanitized_post_message = row['post_message']&.tr("\0", '') || ""
|
||||
post = Post.create!(
|
||||
topic_id: topic.id,
|
||||
user_id: discourse_user_id,
|
||||
raw: import_post_attachments(row['post_message'], row['post_id']),
|
||||
# raw: import_post_attachments(row['post_message'], row['post_id']),
|
||||
# raw: row['post_message'] || "",
|
||||
raw: sanitized_post_message,
|
||||
created_at: Time.at(row['post_time']),
|
||||
updated_at: Time.at(row['post_latest_reply'])
|
||||
)
|
||||
@ -355,6 +439,7 @@ def import_topics_and_posts
|
||||
end
|
||||
|
||||
else
|
||||
puts "#3"
|
||||
# Find the root topic for the post
|
||||
root_topic_field = TopicCustomField.find_by(name: 'original_gossamer_id', value: row['post_root_id'])
|
||||
|
||||
@ -367,10 +452,14 @@ def import_topics_and_posts
|
||||
|
||||
# Create the post in the existing topic
|
||||
begin
|
||||
puts "#4"
|
||||
sanitized_post_message = row['post_message']&.tr("\0", '') || ""
|
||||
post = Post.create!(
|
||||
topic_id: topic_id,
|
||||
user_id: discourse_user_id,
|
||||
raw: import_post_attachments(row['post_message'], row['post_id']),
|
||||
# raw: import_post_attachments(row['post_message'], row['post_id']),
|
||||
# raw: row['post_message'] || "",
|
||||
raw: sanitized_post_message,
|
||||
created_at: Time.at(row['post_time']),
|
||||
updated_at: Time.at(row['post_latest_reply']),
|
||||
reply_to_post_number: reply_to_post_number
|
||||
@ -387,42 +476,23 @@ def import_topics_and_posts
|
||||
end
|
||||
end
|
||||
|
||||
# Import topics and posts from Gossamer Forums to Discourse
|
||||
# def import_topics_and_posts
|
||||
# puts "Importing topics and posts..."
|
||||
# execute_query("SELECT * FROM gforum_Post ORDER BY post_root_id, post_time").each do |row|
|
||||
# puts "post_id #{row['post_id']} post_root_id #{row['post_root_id']} post_subject #{row['post_subject']}"
|
||||
|
||||
# Import attachments for a post
|
||||
def import_post_attachments(post_message, post_id)
|
||||
# Fetch attachments related to the post
|
||||
attachments = execute_query("SELECT * FROM gforum_PostAttachment WHERE post_id_fk = #{post_id}")
|
||||
attachments.each do |attachment|
|
||||
# Append attachment links to the post message
|
||||
file_url = "https://forum.slowtwitch.com/images/posts/attachments/#{attachment['ID'] % 10}/#{attachment['ID']}-#{attachment['File_Name']}"
|
||||
post_message += "\n\n![#{attachment['File_Name']}](#{file_url})"
|
||||
end
|
||||
post_message
|
||||
end
|
||||
|
||||
# Import personal messages (both inbox and sent messages)
|
||||
# Import personal messages from gforum_Message table (both inbox and sent messages)
|
||||
def import_personal_messages
|
||||
puts "Importing personal messages..."
|
||||
import_inbox_messages
|
||||
import_sent_messages
|
||||
end
|
||||
|
||||
# Import inbox messages from gforum_Message table
|
||||
def import_inbox_messages
|
||||
puts "Importing inbox messages..."
|
||||
puts "Importing personal (inbox and sendmail) messages..."
|
||||
execute_query("SELECT * FROM gforum_Message").each do |row|
|
||||
|
||||
from_user_id = fetch_user_id_mapping(row['from_user_id_fk'])
|
||||
to_user_id = fetch_user_id_mapping(row['to_user_id_fk'])
|
||||
|
||||
next unless from_user_id && to_user_id
|
||||
|
||||
# Skip if the message already exists
|
||||
unless TopicCustomField.exists?(name: 'original_gossamer_msg_id', value: row['msg_id'])
|
||||
# Create a private message topic in Discourse
|
||||
discourse_user_id = @user_id_map[row['from_user_id_fk']]
|
||||
topic = Topic.create!(
|
||||
title: row['msg_subject'],
|
||||
user_id: discourse_user_id,
|
||||
user_id: from_user_id,
|
||||
archetype: Archetype.private_message,
|
||||
created_at: Time.at(row['msg_time']),
|
||||
updated_at: Time.at(row['msg_time'])
|
||||
@ -431,61 +501,50 @@ def import_inbox_messages
|
||||
topic.save!
|
||||
|
||||
# Create the message as a post in the private topic
|
||||
Post.create!(
|
||||
sanitized_message = row['msg_body']&.tr("\0", '') || ""
|
||||
post = Post.create!(
|
||||
topic_id: topic.id,
|
||||
user_id: discourse_user_id,
|
||||
raw: row['msg_body'],
|
||||
user_id: from_user_id,
|
||||
# raw: row['msg_body'],
|
||||
raw: sanitized_message,
|
||||
created_at: Time.at(row['msg_time']),
|
||||
updated_at: Time.at(row['msg_time'])
|
||||
)
|
||||
post.custom_fields['original_gossamer_msg_id'] = row['msg_id']
|
||||
post.save!
|
||||
|
||||
# Add recipient user to the private message topic
|
||||
topic.topic_allowed_users.create!(user_id: row['to_user_id_fk'])
|
||||
topic.topic_allowed_users.create!(user_id: to_user_id)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# Import sent messages from gforum_SentMessage table
|
||||
def import_sent_messages
|
||||
puts "Importing sent messages..."
|
||||
execute_query("SELECT * FROM gforum_SentMessage").each do |row|
|
||||
# Skip if the message already exists
|
||||
unless TopicCustomField.exists?(name: 'original_gossamer_sent_msg_id', value: row['msg_id'])
|
||||
# Create a private message topic in Discourse
|
||||
discourse_user_id = @user_id_map[row['from_user_id_fk']]
|
||||
topic = Topic.create!(
|
||||
title: row['msg_subject'],
|
||||
user_id: discourse_user_id,
|
||||
archetype: Archetype.private_message,
|
||||
created_at: Time.at(row['msg_time']),
|
||||
updated_at: Time.at(row['msg_time'])
|
||||
)
|
||||
topic.custom_fields['original_gossamer_sent_msg_id'] = row['msg_id']
|
||||
topic.save!
|
||||
|
||||
# Create the message as a post in the private topic
|
||||
Post.create!(
|
||||
topic_id: topic.id,
|
||||
user_id: discourse_user_id,
|
||||
raw: row['msg_body'],
|
||||
created_at: Time.at(row['msg_time']),
|
||||
updated_at: Time.at(row['msg_time'])
|
||||
)
|
||||
# Import attachments for a post
|
||||
# def import_post_attachments(post_message, post_id)
|
||||
# # Fetch attachments related to the post
|
||||
# attachments = execute_query("SELECT * FROM gforum_PostAttachment WHERE post_id_fk = #{post_id}")
|
||||
# attachments.each do |attachment|
|
||||
# # Append attachment links to the post message
|
||||
# file_url = "https://forum.slowtwitch.com/images/posts/attachments/#{attachment['ID'] % 10}/#{attachment['ID']}-#{attachment['File_Name']}"
|
||||
# post_message += "\n\n![#{attachment['File_Name']}](#{file_url})"
|
||||
# end
|
||||
# post_message
|
||||
# end
|
||||
|
||||
|
||||
|
||||
|
||||
# Add recipient user to the private message topic
|
||||
topic.topic_allowed_users.create!(user_id: row['to_user_id_fk'])
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# Main method to perform the import
|
||||
def perform_import
|
||||
RateLimiter.disable
|
||||
puts "Starting Gossamer Forums import..."
|
||||
import_users
|
||||
import_categories
|
||||
# import_users
|
||||
# import_categories
|
||||
# import_topics_and_posts
|
||||
# import_personal_messages
|
||||
import_personal_messages
|
||||
# import_attachments
|
||||
puts "Gossamer Forums import complete!"
|
||||
end
|
||||
end
|
||||
|
||||
Loading…
Reference in New Issue
Block a user