saint/discourse-import_scripts
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

20240606

Browse Source
This commit is contained in:
dsainty 2024-06-17 19:37:13 +10:00
parent dc4afeb39d
commit 33ea3e2d6a
1 changed files with 152 additions and 93 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

231
gossamer_forums.rb
View File

@ -36,6 +36,12 @@ class GossamerForumsImporter < ImportScripts::Base
new_user_id INTEGER
);
SQL
@db.execute <<-SQL
CREATE TABLE IF NOT EXISTS category_id_map (
old_category_id INTEGER PRIMARY KEY,
new_category_id INTEGER
);
SQL
end
def insert_user_id_mapping(old_user_id, new_user_id)
@ -46,6 +52,14 @@ class GossamerForumsImporter < ImportScripts::Base
@db.get_first_value "SELECT new_user_id FROM user_id_map WHERE old_user_id = ?", old_user_id
end
def insert_category_id_mapping(old_category_id, new_category_id)
@db.execute "INSERT OR REPLACE INTO category_id_map (old_category_id, new_category_id) VALUES (?, ?)", old_category_id, new_category_id
end
def fetch_category_id_mapping(old_category_id)
@db.get_first_value "SELECT new_category_id FROM category_id_map WHERE old_category_id = ?", old_category_id
end
# Execute an SQL query on the Gossamer Forums database
def execute_query(query)
@mysql_client.query(query, as: :hash)
@ -106,21 +120,65 @@ class GossamerForumsImporter < ImportScripts::Base
rescue OpenURI::HTTPError => e
puts "Failed to download image from #{url}: #{e.message}"
nil
rescue URI::InvalidURIError => e
puts "Failed to handle invalid URL/URI for #{url}: #{e.message}"
nil
end
end
# Helper method to upload an image to Discourse
def upload_image(user, image_data, filename)
return if image_data.nil?
def upload_image(user, file, filename, gossamer_url)
begin
upload = Upload.create!(
user_id: user.id,
original_filename: filename,
filesize: file.size,
# filesize: File.size(file.path),
# content_type: `file --brief --mime-type #{file.path}`.strip,
# sha1: Digest::SHA1.file(file.path).hexdigest,
# origin: 'user_avatar',
# retain_hours: nil,
url: gossamer_url
)
# Error -- non-existent method upload.ensure_consistency!
upload = Upload.create_for(user.id, File.open(image_data.path), filename, 'image/jpeg')
if upload.nil? || !upload.persisted?
puts "Failed to upload image for user #{user.username}"
return
end
# Move the file to the correct location
# FileUtils.mv(file.path, upload.path)
upload.save!
upload
rescue => e
puts "Failed to upload image #{filename} for user #{user.username}: #{e.message}"
nil
end
end
# def download_file(url)
# require 'open-uri'
# begin
# file = Tempfile.new
# file.binmode
# file.write(URI.open(url).read)
# file.rewind
# file
# rescue => e
# puts "Failed to download file from #{url}: #{e.message}"
# nil
# end
# end
# Helper method to upload an image to Discourse
# def upload_image(user, image_data, filename)
# return if image_data.nil?
#
# upload = Upload.create_for(user.id, File.open(image_data.path), filename, 'image/jpeg')
# if upload.nil? || !upload.persisted?
# puts "Failed to upload image for user #{user.username}"
# return
# end
#
# upload
# end
# Import users from Gossamer Forums to Discourse
@ -155,7 +213,7 @@ class GossamerForumsImporter < ImportScripts::Base
user
end
# For each user, add user ID mapping to @user_id_map now that we know what the Discourse user ID is, ... and append user bio and import user files
# For each user, add user ID mapping to SQLite now that we know what the Discourse user ID is, ... and append user bio and import user files
users.each do |user|
discourse_username = sanitize_username(user[:username], user[:email], user[:name])
discourse_user = User.find_by(username: discourse_username)
@ -167,6 +225,7 @@ class GossamerForumsImporter < ImportScripts::Base
# # Store the user ID mapping
# @user_id_map[user[:id]] = discourse_user.id
puts "for insert_user_id_mapping: user[:id] #{user[:id]} discourse_user.id #{discourse_user.id}"
insert_user_id_mapping(user[:id], discourse_user.id)
# Ensure user profile exists and bio_raw is a string
@ -188,7 +247,7 @@ class GossamerForumsImporter < ImportScripts::Base
discourse_user.user_profile.save!
# Import user files
# import_user_files(discourse_user)
import_user_files(discourse_user)
end
end
@ -240,27 +299,39 @@ class GossamerForumsImporter < ImportScripts::Base
puts "User #{user.username} User ID: #{user.id} original_gossamer_id: #{original_gossamer_id} file_url: #{file_url}"
next unless file['ForeignColName'] =~ /^user_image\d+$/
puts "#A"
next unless ['image/jpeg', 'image/png'].include?(file['File_MimeType'])
puts "#B"
image_data = download_image(file_url)
next if image_data.nil?
puts "#C"
temp_file = Tempfile.new(['user_image', '.jpg'])
temp_file = Tempfile.new(['user_image', File.extname(file['File_Name'])])
temp_file.binmode
temp_file.write(image_data)
temp_file.rewind
if images_imported == 0
upload = upload_image(user, temp_file, file['File_Name'])
puts "#D"
upload = upload_image(user, temp_file, file['File_Name'], file_url)
next if upload.nil?
user.user_avatar = UserAvatar.create!(user_id: user.id, custom_upload_id: upload.id)
user.save!
# Set the Profile Header
UserProfile.find_by(user_id: user.id).update!(profile_background_upload_id: upload.id)
# Set the User Card Background
UserProfile.find_by(user_id: user.id).update!(card_background_upload_id: upload.id)
images_imported += 1
else
end
puts "#E"
user.user_profile.bio_raw ||= ""
user.user_profile.bio_raw += "\n\n![#{file['File_Name']}](#{file_url})"
user.user_profile.save!
end
temp_file.close
temp_file.unlink
@ -277,11 +348,11 @@ class GossamerForumsImporter < ImportScripts::Base
unless CategoryCustomField.exists?(name: 'original_gossamer_id', value: row['forum_id'])
category_name = row['forum_name']
category_description = row['forum_desc'] || "No description provided"
puts "id #{row['forum_id']} name #{category_name} description #{category_description}"
# Create category in Discourse
category = create_category(
{
id: row['forum_id'] + 10,
# id: row['forum_id'] + 10,
name: category_name,
description: category_description,
created_at: row['forum_last'] ? Time.at(row['forum_last']) : Time.now,
@ -296,6 +367,10 @@ class GossamerForumsImporter < ImportScripts::Base
# category.custom_fields.create!(name: 'original_gossamer_id', value: row['forum_id'])
category.custom_fields['original_gossamer_id'] = row['forum_id']
category.save!
# Store the user ID mapping
puts "for insert_category_id_mapping: category[:id] #{category[:id]} row['forum_id'] #{row['forum_id']}"
insert_category_id_mapping(row['forum_id'], category[:id])
end
end
puts "Importing categories... Done."
@ -316,12 +391,15 @@ def import_topics_and_posts
# Execute the query to get all posts ordered by post_id
execute_query("SELECT * FROM gforum_Post ORDER BY post_id").each do |row|
puts "post_id #{row['post_id']} post_root_id #{row['post_root_id']} post_subject/title #{row['post_subject']} forum_id_fk/category_id #{row['forum_id_fk']}"
discourse_user_id = fetch_user_id_mapping(row['user_id_fk'])
# discourse_user_id = @user_id_map[row['user_id_fk']]
discourse_user_id = fetch_user_id_mapping(row['user_id_fk'])
discourse_category_id = fetch_category_id_mapping(row['forum_id_fk'])
next unless discourse_user_id
puts "discourse_user_id #{discourse_user_id} discourse_category_id #{discourse_category_id}"
next unless discourse_user_id && discourse_category_id
if row['post_root_id'] == 0
puts "#1"
# Ensure the title is valid
title = ensure_valid_title(row['post_subject'])
@ -329,21 +407,27 @@ def import_topics_and_posts
unless TopicCustomField.exists?(name: 'original_gossamer_id', value: row['post_id'])
# Create the topic
begin
puts "#2"
puts "CREATE TOPIC title #{title} discourse_user_id #{discourse_user_id} category_id #{discourse_category_id}"
topic = Topic.create!(
title: title,
user_id: discourse_user_id,
created_at: Time.at(row['post_time']),
updated_at: Time.at(row['post_latest_reply']),
category_id: row['forum_id_fk'] + 10
category_id: discourse_category_id
)
topic.custom_fields['original_gossamer_id'] = row['post_id']
topic.save!
# Create the initial post in the topic
puts "CREATE POST topic.id #{topic.id} discourse_user_id #{discourse_user_id}"
sanitized_post_message = row['post_message']&.tr("\0", '') || ""
post = Post.create!(
topic_id: topic.id,
user_id: discourse_user_id,
raw: import_post_attachments(row['post_message'], row['post_id']),
# raw: import_post_attachments(row['post_message'], row['post_id']),
# raw: row['post_message'] || "",
raw: sanitized_post_message,
created_at: Time.at(row['post_time']),
updated_at: Time.at(row['post_latest_reply'])
)
@ -355,6 +439,7 @@ def import_topics_and_posts
end
else
puts "#3"
# Find the root topic for the post
root_topic_field = TopicCustomField.find_by(name: 'original_gossamer_id', value: row['post_root_id'])
@ -367,10 +452,14 @@ def import_topics_and_posts
# Create the post in the existing topic
begin
puts "#4"
sanitized_post_message = row['post_message']&.tr("\0", '') || ""
post = Post.create!(
topic_id: topic_id,
user_id: discourse_user_id,
raw: import_post_attachments(row['post_message'], row['post_id']),
# raw: import_post_attachments(row['post_message'], row['post_id']),
# raw: row['post_message'] || "",
raw: sanitized_post_message,
created_at: Time.at(row['post_time']),
updated_at: Time.at(row['post_latest_reply']),
reply_to_post_number: reply_to_post_number
@ -387,42 +476,23 @@ def import_topics_and_posts
end
end
# Import topics and posts from Gossamer Forums to Discourse
# def import_topics_and_posts
# puts "Importing topics and posts..."
# execute_query("SELECT * FROM gforum_Post ORDER BY post_root_id, post_time").each do |row|
# puts "post_id #{row['post_id']} post_root_id #{row['post_root_id']} post_subject #{row['post_subject']}"
# Import attachments for a post
def import_post_attachments(post_message, post_id)
# Fetch attachments related to the post
attachments = execute_query("SELECT * FROM gforum_PostAttachment WHERE post_id_fk = #{post_id}")
attachments.each do |attachment|
# Append attachment links to the post message
file_url = "https://forum.slowtwitch.com/images/posts/attachments/#{attachment['ID'] % 10}/#{attachment['ID']}-#{attachment['File_Name']}"
post_message += "\n\n![#{attachment['File_Name']}](#{file_url})"
end
post_message
end
# Import personal messages (both inbox and sent messages)
# Import personal messages from gforum_Message table (both inbox and sent messages)
def import_personal_messages
puts "Importing personal messages..."
import_inbox_messages
import_sent_messages
end
# Import inbox messages from gforum_Message table
def import_inbox_messages
puts "Importing inbox messages..."
puts "Importing personal (inbox and sendmail) messages..."
execute_query("SELECT * FROM gforum_Message").each do |row|
from_user_id = fetch_user_id_mapping(row['from_user_id_fk'])
to_user_id = fetch_user_id_mapping(row['to_user_id_fk'])
next unless from_user_id && to_user_id
# Skip if the message already exists
unless TopicCustomField.exists?(name: 'original_gossamer_msg_id', value: row['msg_id'])
# Create a private message topic in Discourse
discourse_user_id = @user_id_map[row['from_user_id_fk']]
topic = Topic.create!(
title: row['msg_subject'],
user_id: discourse_user_id,
user_id: from_user_id,
archetype: Archetype.private_message,
created_at: Time.at(row['msg_time']),
updated_at: Time.at(row['msg_time'])
@ -431,61 +501,50 @@ def import_inbox_messages
topic.save!
# Create the message as a post in the private topic
Post.create!(
sanitized_message = row['msg_body']&.tr("\0", '') || ""
post = Post.create!(
topic_id: topic.id,
user_id: discourse_user_id,
raw: row['msg_body'],
user_id: from_user_id,
# raw: row['msg_body'],
raw: sanitized_message,
created_at: Time.at(row['msg_time']),
updated_at: Time.at(row['msg_time'])
)
post.custom_fields['original_gossamer_msg_id'] = row['msg_id']
post.save!
# Add recipient user to the private message topic
topic.topic_allowed_users.create!(user_id: row['to_user_id_fk'])
topic.topic_allowed_users.create!(user_id: to_user_id)
end
end
end
# Import sent messages from gforum_SentMessage table
def import_sent_messages
puts "Importing sent messages..."
execute_query("SELECT * FROM gforum_SentMessage").each do |row|
# Skip if the message already exists
unless TopicCustomField.exists?(name: 'original_gossamer_sent_msg_id', value: row['msg_id'])
# Create a private message topic in Discourse
discourse_user_id = @user_id_map[row['from_user_id_fk']]
topic = Topic.create!(
title: row['msg_subject'],
user_id: discourse_user_id,
archetype: Archetype.private_message,
created_at: Time.at(row['msg_time']),
updated_at: Time.at(row['msg_time'])
)
topic.custom_fields['original_gossamer_sent_msg_id'] = row['msg_id']
topic.save!
# Create the message as a post in the private topic
Post.create!(
topic_id: topic.id,
user_id: discourse_user_id,
raw: row['msg_body'],
created_at: Time.at(row['msg_time']),
updated_at: Time.at(row['msg_time'])
)
# Import attachments for a post
# def import_post_attachments(post_message, post_id)
# # Fetch attachments related to the post
# attachments = execute_query("SELECT * FROM gforum_PostAttachment WHERE post_id_fk = #{post_id}")
# attachments.each do |attachment|
# # Append attachment links to the post message
# file_url = "https://forum.slowtwitch.com/images/posts/attachments/#{attachment['ID'] % 10}/#{attachment['ID']}-#{attachment['File_Name']}"
# post_message += "\n\n![#{attachment['File_Name']}](#{file_url})"
# end
# post_message
# end
# Add recipient user to the private message topic
topic.topic_allowed_users.create!(user_id: row['to_user_id_fk'])
end
end
end
# Main method to perform the import
def perform_import
RateLimiter.disable
puts "Starting Gossamer Forums import..."
import_users
import_categories
# import_users
# import_categories
# import_topics_and_posts
# import_personal_messages
import_personal_messages
# import_attachments
puts "Gossamer Forums import complete!"
end
end