From 3bb197da4f2b25f7bafa0a5da6eda76468f91bd4 Mon Sep 17 00:00:00 2001 From: dsainty Date: Mon, 17 Jun 2024 19:35:50 +1000 Subject: [PATCH] 20240601-3 --- gossamer_forums.rb | 405 +++++++++++++++++++++++++++------------------ 1 file changed, 245 insertions(+), 160 deletions(-) diff --git a/gossamer_forums.rb b/gossamer_forums.rb index 2ccf397..c11339f 100644 --- a/gossamer_forums.rb +++ b/gossamer_forums.rb @@ -1,11 +1,14 @@ require 'mysql2' +require 'open-uri' +require 'net/http' + require File.expand_path("../../../config/environment", __FILE__) require_relative 'base' class GossamerForumsImporter < ImportScripts::Base def initialize super - # Initialize MySQL client with connection details + # Initialize MySQL client to connect to Gossamer Forums database @mysql_client = Mysql2::Client.new( host: "slowtwitch.northend.network", username: "admin", @@ -14,33 +17,25 @@ class GossamerForumsImporter < ImportScripts::Base ) end - # Execute an SQL query on the Gossamer Forums MySQL database + # Execute an SQL query on the Gossamer Forums database def execute_query(query) @mysql_client.query(query, as: :hash) end - # Sanitize username to comply with Discourse's rules + # Sanitize the username to meet Discourse's requirements def sanitize_username(username, email, name) original_username = username - # Replace unacceptable characters with underscores sanitized = username.gsub(/[^a-zA-Z0-9._-]/, '_') - # Ensure the username is at least 2 characters long - sanitized = "#{sanitized}." if sanitized.length < 2 - # Ensure the username is no more than 20 characters long + sanitized = "#{sanitized}." if sanitized.length < 2 # Allow two-character usernames sanitized = sanitized[0, 20] if sanitized.length > 20 original_sanitized = sanitized - # Check for existing user with the same username existing_user = User.find_by(username: sanitized) if existing_user - # If email and name match, do not modify the username -# puts "existing_user.email: '#{existing_user.email}' vs. email: '#{email}'" -# puts "existing_user.name: '#{existing_user.name}' vs. email: '#{name}'" if existing_user.email.downcase == email.downcase && existing_user.name == name return sanitized else - # Ensure the username is unique counter = 1 while User.exists?(username: sanitized) sanitized = "#{original_sanitized}_#{counter}" @@ -50,7 +45,6 @@ class GossamerForumsImporter < ImportScripts::Base end end - # Print the original and sanitized usernames if they differ if original_username != sanitized puts "Sanitized username: '#{original_username}' --> '#{sanitized}'" # else @@ -60,6 +54,7 @@ class GossamerForumsImporter < ImportScripts::Base sanitized end + # Sanitize email to replace restricted domains def sanitize_email(email) restricted_domains = ['mailinator.com', 'example.com'] # Add more restricted domains as needed @@ -74,17 +69,38 @@ class GossamerForumsImporter < ImportScripts::Base email end - # Import users from Gossamer Forums gforum_User table to Discourse + + # Helper method to download an image from a URL + def download_image(url) + begin + URI.open(url).read + rescue OpenURI::HTTPError => e + puts "Failed to download image from #{url}: #{e.message}" + nil + end + end + + # Helper method to upload an image to Discourse + def upload_image(user, image_data, filename) + return if image_data.nil? + + upload = Upload.create_for(user.id, File.open(image_data.path), filename, 'image/jpeg') + if upload.nil? || !upload.persisted? + puts "Failed to upload image for user #{user.username}" + return + end + + upload + end + + + # Import users from Gossamer Forums to Discourse def import_users puts "Importing users..." users = [] # Fetch all users from Gossamer Forums execute_query("SELECT * FROM gforum_User").each do |row| - -# sanitized_username = sanitize_username(row['user_username'], row['user_email'], row['user_real_name']) -# sanitized_email = sanitize_email(row['user_email']) - users << { id: row['user_id'], username: sanitize_username(row['user_username'], row['user_email'], row['user_real_name']), @@ -119,197 +135,270 @@ class GossamerForumsImporter < ImportScripts::Base next end - if discourse_user.user_profile.bio_raw.nil? || discourse_user.user_profile.bio_raw.empty? + # Append bio if it exists, otherwise set it to empty string to avoid nil errors + discourse_user.user_profile.bio_raw ||= "" + if discourse_user.user_profile.bio_raw.empty? discourse_user.user_profile.bio_raw = user[:bio_raw] else discourse_user.user_profile.bio_raw += "\n\n" + user[:bio_raw] end - if discourse_user.user_profile.bio_raw.length > 2999 + # Ensure the bio does not exceed 3000 characters + if discourse_user.user_profile.bio_raw.length > 3000 puts "Warning: About Me for user #{discourse_user.username} (ID: #{discourse_user.id}) exceeds 3000 characters. Truncating." - discourse_user.user_profile.bio_raw = discourse_user.user_profile.bio_raw[0, 2999] + discourse_user.user_profile.bio_raw = discourse_user.user_profile.bio_raw[0, 3000] end discourse_user.user_profile.save! + # Import user files import_user_files(discourse_user) end end - # Import user files from Gossamer Forums to Discourse +# # Import user files from Gossamer Forums to Discourse +# def import_user_files(user) +# print "\rImporting files for user #{user.username}..." +# +# original_gossamer_id = user.custom_fields['original_gossamer_id'] +# if original_gossamer_id.nil? || original_gossamer_id.empty? +# puts "User #{user.username} does not have a valid original_gossamer_id. Skipping file import." +# return +# end +# +# # puts "Original Gossamer ID for user #{user.username}: #{original_gossamer_id}" +# +# # Fetch and import user files +# execute_query("SELECT * FROM gforum_User_Files WHERE ForeignColKey = #{original_gossamer_id}").each do |file| +# # Construct the file URL +# file_url = "https://forum.slowtwitch.com/images/users/images/#{file['ID'] % 10}/#{file['ID']}-#{file['File_Name']}" +# puts "User #{user.username} User ID: #{user.id} original_gossamer_id: #{original_gossamer_id} file_url: #{file_url}" +# +# new_bio = user.user_profile.bio_raw + "\n\n![#{file['File_Name']}](#{file_url})" +# if new_bio.length > 3000 +# puts "Warning: About Me for user #{user.username} (ID: #{user.id}) exceeds 3000 characters after adding file link. Truncating." +# new_bio = new_bio[0, 3000] +# end +# user.user_profile.bio_raw = new_bio +# user.user_profile.save! +# end +# print "Importing files for user #{user.username}... Done.\n" +# end + + # Import user files (profile images) from Gossamer Forums to Discourse def import_user_files(user) print "\rImporting files for user #{user.username}..." - + original_gossamer_id = user.custom_fields['original_gossamer_id'] if original_gossamer_id.nil? || original_gossamer_id.empty? puts "User #{user.username} does not have a valid original_gossamer_id. Skipping file import." return end - -# puts "Original Gossamer ID for user #{user.username}: #{original_gossamer_id}" - + + puts "Original Gossamer ID for user #{user.username}: #{original_gossamer_id}" + + images_imported = 0 + execute_query("SELECT * FROM gforum_User_Files WHERE ForeignColKey = #{original_gossamer_id}").each do |file| - # Construct the file URL file_url = "https://forum.slowtwitch.com/images/users/images/#{file['ID'] % 10}/#{file['ID']}-#{file['File_Name']}" puts "User #{user.username} User ID: #{user.id} original_gossamer_id: #{original_gossamer_id} file_url: #{file_url}" - - new_bio = user.user_profile.bio_raw + "\n\n![#{file['File_Name']}](#{file_url})" - if new_bio.length > 3000 - puts "Warning: About Me for user #{user.username} (ID: #{user.id}) exceeds 3000 characters after adding file link. Truncating." - new_bio = new_bio[0, 3000] + + next unless file['ForeignColName'] =~ /^user_image\d+$/ + + image_data = download_image(file_url) + next if image_data.nil? + + temp_file = Tempfile.new(['user_image', '.jpg']) + temp_file.binmode + temp_file.write(image_data) + temp_file.rewind + + if images_imported == 0 + upload = upload_image(user, temp_file, file['File_Name']) + next if upload.nil? + + user.user_avatar = UserAvatar.create!(user_id: user.id, custom_upload_id: upload.id) + user.save! + images_imported += 1 + else + user.user_profile.bio_raw ||= "" + user.user_profile.bio_raw += "\n\n![#{file['File_Name']}](#{file_url})" + user.user_profile.save! end - user.user_profile.bio_raw = new_bio - user.user_profile.save! + + temp_file.close + temp_file.unlink end print "Importing files for user #{user.username}... Done.\n" end + # Import categories from Gossamer Forums to Discourse def import_categories - puts "Importing categories..." - execute_query("SELECT * FROM gforum_Category").each do |row| + puts "Importing categories (forums)..." + execute_query("SELECT * FROM gforum_Forum").each do |row| # Only create category if it does not exist - unless CategoryCustomField.exists?(name: 'original_gossamer_id', value: row['category_id']) + unless CategoryCustomField.exists?(name: 'original_gossamer_id', value: row['forum_id']) + category_name = row['forum_name'] + category_description = row['forum_desc'] || "No description provided" + + # Create category in Discourse category = create_category( - id: row['category_id'], - name: row['name'], - description: row['description'], - created_at: row['created_at'] ? Time.at(row['created_at']) : Time.now, - updated_at: row['updated_at'] ? Time.at(row['updated_at']) : Time.now + { + id: row['forum_id'], + name: category_name, + description: category_description, + created_at: row['forum_last'] ? Time.at(row['forum_last']) : Time.now, + updated_at: row['forum_last'] ? Time.at(row['forum_last']) : Time.now + }, + row['forum_id'] # import_id argument ) - category.custom_fields.create!(name: 'original_gossamer_id', value: row['category_id']) + +# # Map Gossamer forum ID to Discourse category ID for future reference +# @forum_id_map[row['forum_id']] = category.id + +# category.custom_fields.create!(name: 'original_gossamer_id', value: row['forum_id']) + category.custom_fields['original_gossamer_id'] = row['forum_id'] + category.save! end end + puts "Importing categories... Done." end - # Import topics and posts from Gossamer Forums to Discourse - def import_topics_and_posts - puts "Importing topics and posts..." - execute_query("SELECT * FROM gforum_Post ORDER BY post_root_id, post_time").each do |row| - if row['post_id'] == row['post_root_id'] - # Skip if the topic already exists - unless TopicCustomField.exists?(name: 'original_gossamer_id', value: row['post_id']) - # Create the topic - topic = create_topic( - id: row['post_id'], - title: row['post_subject'], - user_id: row['user_id_fk'], - created_at: Time.at(row['post_time']), - updated_at: Time.at(row['post_latest_reply']), - category_id: row['forum_id_fk'] - ) - topic.custom_fields.create!(name: 'original_gossamer_id', value: row['post_id']) - - # Create the initial post in the topic - post = create_post( - id: row['post_id'], - topic_id: row['post_id'], - user_id: row['user_id_fk'], - raw: import_post_attachments(row['post_message'], row['post_id']), - created_at: Time.at(row['post_time']), - updated_at: Time.at(row['post_latest_reply']) - ) - post.custom_fields.create!(name: 'original_gossamer_id', value: row['post_id']) - end - else - # Skip if the post already exists - unless PostCustomField.exists?(name: 'original_gossamer_id', value: row['post_id']) - # Create the post in the existing topic - post = create_post( - id: row['post_id'], - topic_id: row['post_root_id'], - user_id: row['user_id_fk'], - raw: import_post_attachments(row['post_message'], row['post_id']), - created_at: Time.at(row['post_time']), - updated_at: Time.at(row['post_latest_reply']), - reply_to_post_number: row['post_father_id'] - ) - post.custom_fields.create!(name: 'original_gossamer_id', value: row['post_id']) - end - end - end - end - - # Import attachments for a post - def import_post_attachments(post_message, post_id) - attachments = execute_query("SELECT * FROM gforum_PostAttachment WHERE post_id_fk = #{post_id}") - attachments.each do |attachment| - attachment_url = "https://forum.slowtwitch.com/images/posts/#{post_id}/#{attachment['attachment_name']}" - post_message += "\n\n![#{attachment['attachment_name']}](#{attachment_url})" - end - post_message - end - - # Import personal messages (both inbox and sent messages) - def import_personal_messages - puts "Importing personal messages..." - import_inbox_messages - import_sent_messages - end - - # Import inbox messages from gforum_Message table - def import_inbox_messages - puts "Importing inbox messages..." - execute_query("SELECT * FROM gforum_Message").each do |row| - unless TopicCustomField.exists?(name: 'original_gossamer_msg_id', value: row['msg_id']) - # Create a private message topic in Discourse - topic = create_topic( - title: row['msg_subject'], - user_id: row['from_user_id_fk'], - archetype: Archetype.private_message, - created_at: Time.at(row['msg_time']), - updated_at: Time.at(row['msg_time']) +# Import topics and posts from Gossamer Forums to Discourse +def import_topics_and_posts + puts "Importing topics and posts..." + execute_query("SELECT * FROM gforum_Post ORDER BY post_root_id, post_time").each do |row| + if row['post_id'] == row['post_root_id'] + # Skip if the topic already exists + unless TopicCustomField.exists?(name: 'original_gossamer_id', value: row['post_id']) + # Create the topic + topic = Topic.create!( + title: row['post_subject'], + user_id: row['user_id_fk'], + created_at: Time.at(row['post_time']), + updated_at: Time.at(row['post_latest_reply']), + category_id: row['forum_id_fk'] ) - topic.custom_fields.create!(name: 'original_gossamer_msg_id', value: row['msg_id']) + topic.custom_fields['original_gossamer_id'] = row['post_id'] + topic.save! - # Create the message as a post in the private topic - create_post( + # Create the initial post in the topic + post = Post.create!( topic_id: topic.id, - user_id: row['from_user_id_fk'], - raw: row['msg_body'], - created_at: Time.at(row['msg_time']), - updated_at: Time.at(row['msg_time']) + user_id: row['user_id_fk'], + raw: import_post_attachments(row['post_message'], row['post_id']), + created_at: Time.at(row['post_time']), + updated_at: Time.at(row['post_latest_reply']) ) - - # Add recipient user to the private message topic - topic.topic_allowed_users.create!(user_id: row['to_user_id_fk']) + post.custom_fields['original_gossamer_id'] = row['post_id'] + post.save! + end + else + # Skip if the post already exists + unless PostCustomField.exists?(name: 'original_gossamer_id', value: row['post_id']) + # Create the post in the existing topic + post = Post.create!( + topic_id: row['post_root_id'], + user_id: row['user_id_fk'], + raw: import_post_attachments(row['post_message'], row['post_id']), + created_at: Time.at(row['post_time']), + updated_at: Time.at(row['post_latest_reply']), + reply_to_post_number: row['post_father_id'] + ) + post.custom_fields['original_gossamer_id'] = row['post_id'] + post.save! end end end +end - # Import sent messages from gforum_SentMessage table - def import_sent_messages - puts "Importing sent messages..." - execute_query("SELECT * FROM gforum_SentMessage").each do |row| - unless TopicCustomField.exists?(name: 'original_gossamer_sent_msg_id', value: row['msg_id']) - # Create a private message topic in Discourse - topic = create_topic( - title: row['msg_subject'], - user_id: row['from_user_id_fk'], - archetype: Archetype.private_message, - created_at: Time.at(row['msg_time']), - updated_at: Time.at(row['msg_time']) - ) - topic.custom_fields.create!(name: 'original_gossamer_sent_msg_id', value: row['msg_id']) +# Import attachments for a post +def import_post_attachments(post_message, post_id) + # Fetch attachments related to the post + attachments = execute_query("SELECT * FROM gforum_PostAttachment WHERE post_id_fk = #{post_id}") + attachments.each do |attachment| + # Append attachment links to the post message + file_url = "https://forum.slowtwitch.com/images/posts/attachments/#{attachment['ID'] % 10}/#{attachment['ID']}-#{attachment['File_Name']}" + post_message += "\n\n![#{attachment['File_Name']}](#{file_url})" + end + post_message +end - # Create the message as a post in the private topic - create_post( - topic_id: topic.id, - user_id: row['from_user_id_fk'], - raw: row['msg_body'], - created_at: Time.at(row['msg_time']), - updated_at: Time.at(row['msg_time']) - ) +# Import personal messages (both inbox and sent messages) +def import_personal_messages + puts "Importing personal messages..." + import_inbox_messages + import_sent_messages +end - # Add recipient user to the private message topic - topic.topic_allowed_users.create!(user_id: row['to_user_id_fk']) - end +# Import inbox messages from gforum_Message table +def import_inbox_messages + puts "Importing inbox messages..." + execute_query("SELECT * FROM gforum_Message").each do |row| + # Skip if the message already exists + unless TopicCustomField.exists?(name: 'original_gossamer_msg_id', value: row['msg_id']) + # Create a private message topic in Discourse + topic = Topic.create!( + title: row['msg_subject'], + user_id: row['from_user_id_fk'], + archetype: Archetype.private_message, + created_at: Time.at(row['msg_time']), + updated_at: Time.at(row['msg_time']) + ) + topic.custom_fields['original_gossamer_msg_id'] = row['msg_id'] + topic.save! + + # Create the message as a post in the private topic + Post.create!( + topic_id: topic.id, + user_id: row['from_user_id_fk'], + raw: row['msg_body'], + created_at: Time.at(row['msg_time']), + updated_at: Time.at(row['msg_time']) + ) + + # Add recipient user to the private message topic + topic.topic_allowed_users.create!(user_id: row['to_user_id_fk']) end end +end + +# Import sent messages from gforum_SentMessage table +def import_sent_messages + puts "Importing sent messages..." + execute_query("SELECT * FROM gforum_SentMessage").each do |row| + # Skip if the message already exists + unless TopicCustomField.exists?(name: 'original_gossamer_sent_msg_id', value: row['msg_id']) + # Create a private message topic in Discourse + topic = Topic.create!( + title: row['msg_subject'], + user_id: row['from_user_id_fk'], + archetype: Archetype.private_message, + created_at: Time.at(row['msg_time']), + updated_at: Time.at(row['msg_time']) + ) + topic.custom_fields['original_gossamer_sent_msg_id'] = row['msg_id'] + topic.save! + + # Create the message as a post in the private topic + Post.create!( + topic_id: topic.id, + user_id: row['from_user_id_fk'], + raw: row['msg_body'], + created_at: Time.at(row['msg_time']), + updated_at: Time.at(row['msg_time']) + ) + + # Add recipient user to the private message topic + topic.topic_allowed_users.create!(user_id: row['to_user_id_fk']) + end + end +end # Main method to perform the import def perform_import puts "Starting Gossamer Forums import..." - import_users +# import_users import_categories import_topics_and_posts import_personal_messages @@ -317,9 +406,5 @@ class GossamerForumsImporter < ImportScripts::Base end end -# Create an instance of the importer and start the import process -# importer = GossamerForumsImporter.new -# importer.perform_import - GossamerForumsImporter.new.perform_import