From d0c5884cda2f45e7f416ca9471bc2ce849f010a4 Mon Sep 17 00:00:00 2001 From: dsainty Date: Mon, 17 Jun 2024 19:35:13 +1000 Subject: [PATCH] 20240531-2 --- gossamer_forums.rb | 262 +++++++++++++++++++++++++-------------------- 1 file changed, 146 insertions(+), 116 deletions(-) diff --git a/gossamer_forums.rb b/gossamer_forums.rb index 92f3168..2ccf397 100644 --- a/gossamer_forums.rb +++ b/gossamer_forums.rb @@ -14,7 +14,7 @@ class GossamerForumsImporter < ImportScripts::Base ) end - # Execute a query on the MySQL database + # Execute an SQL query on the Gossamer Forums MySQL database def execute_query(query) @mysql_client.query(query, as: :hash) end @@ -24,7 +24,7 @@ class GossamerForumsImporter < ImportScripts::Base original_username = username # Replace unacceptable characters with underscores sanitized = username.gsub(/[^a-zA-Z0-9._-]/, '_') - # Ensure the username is at least 3 characters long + # Ensure the username is at least 2 characters long sanitized = "#{sanitized}." if sanitized.length < 2 # Ensure the username is no more than 20 characters long sanitized = sanitized[0, 20] if sanitized.length > 20 @@ -74,18 +74,20 @@ class GossamerForumsImporter < ImportScripts::Base email end - # Import users from gforum_User table + # Import users from Gossamer Forums gforum_User table to Discourse def import_users puts "Importing users..." users = [] + # Fetch all users from Gossamer Forums execute_query("SELECT * FROM gforum_User").each do |row| - sanitized_username = sanitize_username(row['user_username'], row['user_email'], row['user_real_name']) + +# sanitized_username = sanitize_username(row['user_username'], row['user_email'], row['user_real_name']) # sanitized_email = sanitize_email(row['user_email']) users << { id: row['user_id'], - username: sanitized_username, + username: sanitize_username(row['user_username'], row['user_email'], row['user_real_name']), email: row['user_email'], created_at: Time.at(row['user_registered']), updated_at: Time.at(row['user_last_seen']), @@ -95,35 +97,45 @@ class GossamerForumsImporter < ImportScripts::Base website: row['user_homepage'], location: row['user_location'], custom_fields: { - 'md5_password' => row['user_password'], - 'original_gossamer_username' => row['user_username'], - 'original_gossamer_id' => row['user_id'] + md5_password: row['user_password'], + original_username: row['user_username'], + original_gossamer_id: row['user_id'] } } end - # Create users in Discourse with the required block + # Create or update users in Discourse create_users(users) do |user| user end - # Import user files after creating users + # Append user bio and import user files users.each do |user| - discourse_user = User.find_by(username: user[:username]) + discourse_username = sanitize_username(user[:username], user[:email], user[:name]) + discourse_user = User.find_by(username: discourse_username) + if discourse_user.nil? - puts "User #{user[:username]} not found in Discourse. Skipping file import." + puts "User #{user[:username]} --> #{discourse_username} not found in Discourse. Skipping file import." next end - # Update user profile bio_raw with user_about - discourse_user.user_profile.bio_raw = (discourse_user.user_profile.bio_raw || "") + "\n\n" + (user[:bio_raw] || "") + if discourse_user.user_profile.bio_raw.nil? || discourse_user.user_profile.bio_raw.empty? + discourse_user.user_profile.bio_raw = user[:bio_raw] + else + discourse_user.user_profile.bio_raw += "\n\n" + user[:bio_raw] + end + + if discourse_user.user_profile.bio_raw.length > 2999 + puts "Warning: About Me for user #{discourse_user.username} (ID: #{discourse_user.id}) exceeds 3000 characters. Truncating." + discourse_user.user_profile.bio_raw = discourse_user.user_profile.bio_raw[0, 2999] + end discourse_user.user_profile.save! import_user_files(discourse_user) end end - # Import user files and append to user's bio + # Import user files from Gossamer Forums to Discourse def import_user_files(user) print "\rImporting files for user #{user.username}..." @@ -136,83 +148,93 @@ class GossamerForumsImporter < ImportScripts::Base # puts "Original Gossamer ID for user #{user.username}: #{original_gossamer_id}" execute_query("SELECT * FROM gforum_User_Files WHERE ForeignColKey = #{original_gossamer_id}").each do |file| - -# execute_query("SELECT * FROM gforum_User_Files WHERE ForeignColName = 'user_id' AND ForeignColKey = #{user.id}").each do |file| -# execute_query("SELECT * FROM gforum_User_Files WHERE ForeignColKey = #{user.custom_fields['original_gossamer_id']}").each do |file| - # Construct file URL + # Construct the file URL file_url = "https://forum.slowtwitch.com/images/users/images/#{file['ID'] % 10}/#{file['ID']}-#{file['File_Name']}" - puts "User #{user.username} User ID: #{user.id} original_gossamer_id: #{user.custom_fields['original_gossamer_id']} file_url: #{file_url}" - # Append image link to user's bio - user.user_profile.bio_raw += "\n\n![#{file['File_Name']}](#{file_url})" + puts "User #{user.username} User ID: #{user.id} original_gossamer_id: #{original_gossamer_id} file_url: #{file_url}" + + new_bio = user.user_profile.bio_raw + "\n\n![#{file['File_Name']}](#{file_url})" + if new_bio.length > 3000 + puts "Warning: About Me for user #{user.username} (ID: #{user.id}) exceeds 3000 characters after adding file link. Truncating." + new_bio = new_bio[0, 3000] + end + user.user_profile.bio_raw = new_bio user.user_profile.save! end print "Importing files for user #{user.username}... Done.\n" end - # Import categories from gforum_Category table + # Import categories from Gossamer Forums to Discourse def import_categories puts "Importing categories..." execute_query("SELECT * FROM gforum_Category").each do |row| - # Use current time if created_at or updated_at is null - created_at = row['created_at'] ? Time.at(row['created_at']) : Time.now - updated_at = row['updated_at'] ? Time.at(row['updated_at']) : Time.now - - create_category( - id: row['category_id'], - name: row['name'], - description: row['description'], - created_at: created_at, - updated_at: updated_at - ) - end - end - - # Import topics and posts from gforum_Post table - def import_topics_and_posts - puts "Importing topics and posts..." - execute_query("SELECT * FROM gforum_Post ORDER BY post_root_id, post_time").each do |row| - if row['post_id'] == row['post_root_id'] - # This is the root post, create a new topic - topic = create_topic( - id: row['post_id'], - title: row['post_subject'], - user_id: row['user_id_fk'], - created_at: Time.at(row['post_time']), - updated_at: Time.at(row['post_latest_reply']), - category_id: row['forum_id_fk'] - ) - - # Create the first post in the topic - create_post( - id: row['post_id'], - topic_id: row['post_id'], - user_id: row['user_id_fk'], - raw: import_post_attachments(row['post_message'], row['post_id']), - created_at: Time.at(row['post_time']), - updated_at: Time.at(row['post_latest_reply']) - ) - else - # This is a reply post, add to the existing topic - create_post( - id: row['post_id'], - topic_id: row['post_root_id'], - user_id: row['user_id_fk'], - raw: import_post_attachments(row['post_message'], row['post_id']), - created_at: Time.at(row['post_time']), - updated_at: Time.at(row['post_latest_reply']), - reply_to_post_number: row['post_father_id'] + # Only create category if it does not exist + unless CategoryCustomField.exists?(name: 'original_gossamer_id', value: row['category_id']) + category = create_category( + id: row['category_id'], + name: row['name'], + description: row['description'], + created_at: row['created_at'] ? Time.at(row['created_at']) : Time.now, + updated_at: row['updated_at'] ? Time.at(row['updated_at']) : Time.now ) + category.custom_fields.create!(name: 'original_gossamer_id', value: row['category_id']) end end end - # Import post attachments from gforum_PostAttachment table + # Import topics and posts from Gossamer Forums to Discourse + def import_topics_and_posts + puts "Importing topics and posts..." + execute_query("SELECT * FROM gforum_Post ORDER BY post_root_id, post_time").each do |row| + if row['post_id'] == row['post_root_id'] + # Skip if the topic already exists + unless TopicCustomField.exists?(name: 'original_gossamer_id', value: row['post_id']) + # Create the topic + topic = create_topic( + id: row['post_id'], + title: row['post_subject'], + user_id: row['user_id_fk'], + created_at: Time.at(row['post_time']), + updated_at: Time.at(row['post_latest_reply']), + category_id: row['forum_id_fk'] + ) + topic.custom_fields.create!(name: 'original_gossamer_id', value: row['post_id']) + + # Create the initial post in the topic + post = create_post( + id: row['post_id'], + topic_id: row['post_id'], + user_id: row['user_id_fk'], + raw: import_post_attachments(row['post_message'], row['post_id']), + created_at: Time.at(row['post_time']), + updated_at: Time.at(row['post_latest_reply']) + ) + post.custom_fields.create!(name: 'original_gossamer_id', value: row['post_id']) + end + else + # Skip if the post already exists + unless PostCustomField.exists?(name: 'original_gossamer_id', value: row['post_id']) + # Create the post in the existing topic + post = create_post( + id: row['post_id'], + topic_id: row['post_root_id'], + user_id: row['user_id_fk'], + raw: import_post_attachments(row['post_message'], row['post_id']), + created_at: Time.at(row['post_time']), + updated_at: Time.at(row['post_latest_reply']), + reply_to_post_number: row['post_father_id'] + ) + post.custom_fields.create!(name: 'original_gossamer_id', value: row['post_id']) + end + end + end + end + + # Import attachments for a post def import_post_attachments(post_message, post_id) - # Query for attachments related to the post attachments = execute_query("SELECT * FROM gforum_PostAttachment WHERE post_id_fk = #{post_id}") attachments.each do |attachment| - # Append attachment link to the post message - post_message += "\n\n![#{attachment['postatt_filename']}](https://forum.slowtwitch.com/forum/?do=post_attachment;postatt_id=#{attachment['postatt_filename']})" + attachment_url = "https://forum.slowtwitch.com/images/posts/#{post_id}/#{attachment['attachment_name']}" + post_message += "\n\n![#{attachment['attachment_name']}](#{attachment_url})" end post_message end @@ -228,27 +250,29 @@ class GossamerForumsImporter < ImportScripts::Base def import_inbox_messages puts "Importing inbox messages..." execute_query("SELECT * FROM gforum_Message").each do |row| - # Create a private message topic in Discourse - topic = create_topic( - title: row['msg_subject'], - user_id: row['from_user_id_fk'], - archetype: Archetype.private_message, - created_at: Time.at(row['msg_time']), - updated_at: Time.at(row['msg_time']) - ) + unless TopicCustomField.exists?(name: 'original_gossamer_msg_id', value: row['msg_id']) + # Create a private message topic in Discourse + topic = create_topic( + title: row['msg_subject'], + user_id: row['from_user_id_fk'], + archetype: Archetype.private_message, + created_at: Time.at(row['msg_time']), + updated_at: Time.at(row['msg_time']) + ) + topic.custom_fields.create!(name: 'original_gossamer_msg_id', value: row['msg_id']) - # Create the message as a post in the private topic - create_post( - topic_id: topic.id, - user_id: row['from_user_id_fk'], - raw: row['msg_body'], - created_at: Time.at(row['msg_time']), - updated_at: Time.at(row['msg_time']) - ) + # Create the message as a post in the private topic + create_post( + topic_id: topic.id, + user_id: row['from_user_id_fk'], + raw: row['msg_body'], + created_at: Time.at(row['msg_time']), + updated_at: Time.at(row['msg_time']) + ) - # Add recipient user to the private message topic - topic.add_user_by_id(row['to_user_id_fk']) - topic.save! + # Add recipient user to the private message topic + topic.topic_allowed_users.create!(user_id: row['to_user_id_fk']) + end end end @@ -256,40 +280,46 @@ class GossamerForumsImporter < ImportScripts::Base def import_sent_messages puts "Importing sent messages..." execute_query("SELECT * FROM gforum_SentMessage").each do |row| - # Create a private message topic in Discourse - topic = create_topic( - title: row['msg_subject'], - user_id: row['from_user_id_fk'], - archetype: Archetype.private_message, - created_at: Time.at(row['msg_time']), - updated_at: Time.at(row['msg_time']) - ) + unless TopicCustomField.exists?(name: 'original_gossamer_sent_msg_id', value: row['msg_id']) + # Create a private message topic in Discourse + topic = create_topic( + title: row['msg_subject'], + user_id: row['from_user_id_fk'], + archetype: Archetype.private_message, + created_at: Time.at(row['msg_time']), + updated_at: Time.at(row['msg_time']) + ) + topic.custom_fields.create!(name: 'original_gossamer_sent_msg_id', value: row['msg_id']) - # Create the message as a post in the private topic - create_post( - topic_id: topic.id, - user_id: row['from_user_id_fk'], - raw: row['msg_body'], - created_at: Time.at(row['msg_time']), - updated_at: Time.at(row['msg_time']) - ) + # Create the message as a post in the private topic + create_post( + topic_id: topic.id, + user_id: row['from_user_id_fk'], + raw: row['msg_body'], + created_at: Time.at(row['msg_time']), + updated_at: Time.at(row['msg_time']) + ) - # Add recipient user to the private message topic - topic.add_user_by_id(row['to_user_id_fk']) - topic.save! + # Add recipient user to the private message topic + topic.topic_allowed_users.create!(user_id: row['to_user_id_fk']) + end end end - # Perform the full import process + # Main method to perform the import def perform_import + puts "Starting Gossamer Forums import..." import_users import_categories import_topics_and_posts import_personal_messages + puts "Gossamer Forums import complete!" end end # Create an instance of the importer and start the import process -importer = GossamerForumsImporter.new -importer.perform_import +# importer = GossamerForumsImporter.new +# importer.perform_import + +GossamerForumsImporter.new.perform_import