diff --git a/gossamer_forums.rb b/gossamer_forums.rb index 5e08854..4d0a395 100644 --- a/gossamer_forums.rb +++ b/gossamer_forums.rb @@ -1,5 +1,5 @@ # gossamer threads migration-import code -# v0.13 +# v0.14 require 'mysql2' require 'open-uri' @@ -12,8 +12,8 @@ require 'fileutils' require 'csv' require 'time' -require File.expand_path("../../../config/environment", __FILE__) -require_relative 'base' +require File.expand_path("../../../../config/environment", __FILE__) +require_relative '../base' class GossamerForumsImporter < ImportScripts::Base def initialize @@ -483,179 +483,178 @@ class GossamerForumsImporter < ImportScripts::Base title end -# Import topics and posts from Gossamer Forums to Discourse -def import_topics_and_posts_with_attachments - puts "Importing topics and posts with attachments..." + # Import topics and posts from Gossamer Forums to Discourse + def import_topics_and_posts_with_attachments + puts "Importing topics and posts with attachments..." + + # Execute the query to get all posts ordered by post_id + execute_query("SELECT * FROM gforum_Post ORDER BY post_id").each do |row| + puts "post_id #{row['post_id']} post_root_id #{row['post_root_id']} post_subject/title #{row['post_subject']} forum_id_fk/category_id #{row['forum_id_fk']}" + # discourse_user_id = @user_id_map[row['user_id_fk']] + discourse_user_id = fetch_user_id_mapping(row['user_id_fk']) + discourse_category_id = fetch_category_id_mapping(row['forum_id_fk']) + puts "discourse_user_id #{discourse_user_id} discourse_category_id #{discourse_category_id}" + next unless discourse_user_id && discourse_category_id - # Execute the query to get all posts ordered by post_id - execute_query("SELECT * FROM gforum_Post ORDER BY post_id").each do |row| - puts "post_id #{row['post_id']} post_root_id #{row['post_root_id']} post_subject/title #{row['post_subject']} forum_id_fk/category_id #{row['forum_id_fk']}" -# discourse_user_id = @user_id_map[row['user_id_fk']] - discourse_user_id = fetch_user_id_mapping(row['user_id_fk']) - discourse_category_id = fetch_category_id_mapping(row['forum_id_fk']) - puts "discourse_user_id #{discourse_user_id} discourse_category_id #{discourse_category_id}" - next unless discourse_user_id && discourse_category_id - - if row['post_root_id'] == 0 - puts "#1" - # Ensure the title is valid - title = ensure_valid_title(row['post_subject']) - - # Skip if the topic already exists - unless TopicCustomField.exists?(name: 'original_gossamer_id', value: row['post_id']) - # Create the topic - begin - puts "#2" - puts "CREATE TOPIC title #{title} discourse_user_id #{discourse_user_id} category_id #{discourse_category_id}" - topic = Topic.create!( - title: title, - user_id: discourse_user_id, - created_at: Time.at(row['post_time']), - updated_at: Time.at(row['post_latest_reply']), - category_id: discourse_category_id - ) - topic.custom_fields['original_gossamer_id'] = row['post_id'] - topic.save! - - # Create the initial post in the topic - puts "CREATE POST topic.id #{topic.id} discourse_user_id #{discourse_user_id}" - - # Ensure the raw post stirng contents itself is acceptable to Discourse - sanitized_post_message = row['post_message']&.tr("\0", '') || "" - - # Remove the [signature] label from appearing at the end of the messages after import - sanitized_post_message.sub(/\n?\[signature\]\n?\z/, '') - post = Post.create!( - topic_id: topic.id, - user_id: discourse_user_id, -# raw: import_attachments(row['post_message'], row['post_id']), -# raw: row['post_message'] || "", - raw: sanitized_post_message, - created_at: Time.at(row['post_time']), - updated_at: Time.at(row['post_latest_reply']) - ) - post.custom_fields['original_gossamer_id'] = row['post_id'] - post.save! - - # Handle attachments for the post - handle_post_attachments(row['post_id'], post, discourse_user_id) - - # Create URL mappings -# old_url = "https://old/forum/#{row['forum_name']}/topics/#{row['post_id']}" - new_url = "https://new/t/#{topic.slug}/#{topic.id}" - insert_url_mapping(row['post_id'], new_url, title) - - rescue ActiveRecord::RecordInvalid => e - puts "Error importing topic with post_id #{row['post_id']}: #{e.message}" - end - end - - else - puts "#3" - # Find the root topic for the post - root_topic_field = TopicCustomField.find_by(name: 'original_gossamer_id', value: row['post_root_id']) - - if root_topic_field - topic_id = root_topic_field.topic_id - - # Find the parent post for the reply - parent_post_field = PostCustomField.find_by(name: 'original_gossamer_id', value: row['post_father_id']) - reply_to_post_number = parent_post_field ? Post.find(parent_post_field.post_id).post_number : nil - - # Create the post in the existing topic - begin - puts "#4" - - # Ensure the raw post string contents itself is acceptable to Discourse - sanitized_post_message = row['post_message']&.tr("\0", '') || "" - - # Remove the [signature] label from appearing at the end of the messages after import - sanitized_post_message.sub(/\n?\[signature\]\n?\z/, '') - post = Post.create!( - topic_id: topic_id, - user_id: discourse_user_id, -# raw: import_attachments(row['post_message'], row['post_id']), -# raw: row['post_message'] || "", - raw: sanitized_post_message, - created_at: Time.at(row['post_time']), - updated_at: Time.at(row['post_latest_reply']), - reply_to_post_number: reply_to_post_number - ) - post.custom_fields['original_gossamer_id'] = row['post_id'] - post.save! - - # Handle attachments for the post - handle_post_attachments(row['post_id'], post, discourse_user_id) - rescue ActiveRecord::RecordInvalid => e - puts "Error importing post with post_id #{row['post_id']}: #{e.message}" - end + if row['post_root_id'] == 0 + puts "#1" + # Ensure the title is valid + title = ensure_valid_title(row['post_subject']) + + # Skip if the topic already exists + unless TopicCustomField.exists?(name: 'original_gossamer_id', value: row['post_id']) + # Create the topic + begin + puts "#2" + puts "CREATE TOPIC title #{title} discourse_user_id #{discourse_user_id} category_id #{discourse_category_id}" + topic = Topic.create!( + title: title, + user_id: discourse_user_id, + created_at: Time.at(row['post_time']), + updated_at: Time.at(row['post_latest_reply']), + category_id: discourse_category_id + ) + topic.custom_fields['original_gossamer_id'] = row['post_id'] + topic.save! + + # Create the initial post in the topic + puts "CREATE POST topic.id #{topic.id} discourse_user_id #{discourse_user_id}" + + # Ensure the raw post stirng contents itself is acceptable to Discourse + sanitized_post_message = row['post_message']&.tr("\0", '') || "" + + # Remove the [signature] label from appearing at the end of the messages after import + sanitized_post_message.sub(/\n?\[signature\]\n?\z/, '') + post = Post.create!( + topic_id: topic.id, + user_id: discourse_user_id, +# raw: import_attachments(row['post_message'], row['post_id']), +# raw: row['post_message'] || "", + raw: sanitized_post_message, + created_at: Time.at(row['post_time']), + updated_at: Time.at(row['post_latest_reply']) + ) + post.custom_fields['original_gossamer_id'] = row['post_id'] + post.save! + + # Handle attachments for the post + handle_post_attachments(row['post_id'], post, discourse_user_id) + + # Create URL mappings +# old_url = "https://old/forum/#{row['forum_name']}/topics/#{row['post_id']}" + new_url = "https://new/t/#{topic.slug}/#{topic.id}" + insert_url_mapping(row['post_id'], new_url, title) + + rescue ActiveRecord::RecordInvalid => e + puts "Error importing topic with post_id #{row['post_id']}: #{e.message}" + end + end else - puts "Warning: Root topic not found for post_id #{row['post_id']} with post_root_id #{row['post_root_id']}" + puts "#3" + # Find the root topic for the post + root_topic_field = TopicCustomField.find_by(name: 'original_gossamer_id', value: row['post_root_id']) + + if root_topic_field + topic_id = root_topic_field.topic_id + + # Find the parent post for the reply + parent_post_field = PostCustomField.find_by(name: 'original_gossamer_id', value: row['post_father_id']) + reply_to_post_number = parent_post_field ? Post.find(parent_post_field.post_id).post_number : nil + + # Create the post in the existing topic + begin + puts "#4" + + # Ensure the raw post string contents itself is acceptable to Discourse + sanitized_post_message = row['post_message']&.tr("\0", '') || "" + + # Remove the [signature] label from appearing at the end of the messages after import + sanitized_post_message.sub(/\n?\[signature\]\n?\z/, '') + post = Post.create!( + topic_id: topic_id, + user_id: discourse_user_id, +# raw: import_attachments(row['post_message'], row['post_id']), +# raw: row['post_message'] || "", + raw: sanitized_post_message, + created_at: Time.at(row['post_time']), + updated_at: Time.at(row['post_latest_reply']), + reply_to_post_number: reply_to_post_number + ) + post.custom_fields['original_gossamer_id'] = row['post_id'] + post.save! + + # Handle attachments for the post + handle_post_attachments(row['post_id'], post, discourse_user_id) + rescue ActiveRecord::RecordInvalid => e + puts "Error importing post with post_id #{row['post_id']}: #{e.message}" + end + else + puts "Warning: Root topic not found for post_id #{row['post_id']} with post_root_id #{row['post_root_id']}" + end end end end -end -# Import personal messages from gforum_Message table (both inbox and sent messages) -def import_personal_messages - puts "Importing personal (inbox and sendmail) messages..." - execute_query("SELECT * FROM gforum_Message").each do |row| + # Import personal messages from gforum_Message table (both inbox and sent messages) + def import_personal_messages + puts "Importing personal (inbox and sendmail) messages..." + execute_query("SELECT * FROM gforum_Message").each do |row| + + from_user_id = fetch_user_id_mapping(row['from_user_id_fk']) + to_user_id = fetch_user_id_mapping(row['to_user_id_fk']) + + next unless from_user_id && to_user_id + + # Skip if the message already exists + unless TopicCustomField.exists?(name: 'original_gossamer_msg_id', value: row['msg_id']) + + # Sanitize the message, ensuring we have an empty string or the content without any \0 + sanitized_message = row['msg_body']&.tr("\0", '') || "" + + # Set default message body if the sanitized message is blank + sanitized_message = " " if sanitized_message.strip.empty? + +# # If we do not change the "min personal message post length" to 1, we need this. +# sanitized_message = sanitized_message.ljust(10, ' ') if sanitized_message.length < 10 - from_user_id = fetch_user_id_mapping(row['from_user_id_fk']) - to_user_id = fetch_user_id_mapping(row['to_user_id_fk']) + # Check and set a default title if the original title is nil or empty + title = row['msg_subject']&.strip + title = "" if title.nil? || title.empty? + + puts "IMPORTING title #{row['msg_subject']} user_id #{from_user_id} to_user_id #{to_user_id}" - next unless from_user_id && to_user_id - - # Skip if the message already exists - unless TopicCustomField.exists?(name: 'original_gossamer_msg_id', value: row['msg_id']) - - # Sanitize the message, ensuring we have an empty string or the content without any \0 - sanitized_message = row['msg_body']&.tr("\0", '') || "" - - # Set default message body if the sanitized message is blank - sanitized_message = " " if sanitized_message.strip.empty? - -# # If we do not change the "min personal message post length" to 1, we need this. -# sanitized_message = sanitized_message.ljust(10, ' ') if sanitized_message.length < 10 - - # Check and set a default title if the original title is nil or empty - title = row['msg_subject']&.strip - title = "" if title.nil? || title.empty? - - puts "IMPORTING title #{row['msg_subject']} user_id #{from_user_id} to_user_id #{to_user_id}" - - # Create a private message topic in Discourse - topic = Topic.create!( -# title: row['msg_subject'], - title: title, - user_id: from_user_id, - archetype: Archetype.private_message, - created_at: Time.at(row['msg_time']), - updated_at: Time.at(row['msg_time']) - ) - topic.custom_fields['original_gossamer_msg_id'] = row['msg_id'] - topic.save! - - # Create the message as a post in the private topic - post = Post.create!( - topic_id: topic.id, - user_id: from_user_id, -# raw: row['msg_body'], - raw: sanitized_message, - created_at: Time.at(row['msg_time']), - updated_at: Time.at(row['msg_time']) - ) - post.custom_fields['original_gossamer_msg_id'] = row['msg_id'] - post.save! - - # Add recipient user to the private message topic - topic.topic_allowed_users.create!(user_id: to_user_id) - -# handle_post_attachments(row['msg_id'], post, from_user_id) + # Create a private message topic in Discourse + topic = Topic.create!( +# title: row['msg_subject'], + title: title, + user_id: from_user_id, + archetype: Archetype.private_message, + created_at: Time.at(row['msg_time']), + updated_at: Time.at(row['msg_time']) + ) + topic.custom_fields['original_gossamer_msg_id'] = row['msg_id'] + topic.save! + + # Create the message as a post in the private topic + post = Post.create!( + topic_id: topic.id, + user_id: from_user_id, +# raw: row['msg_body'], + raw: sanitized_message, + created_at: Time.at(row['msg_time']), + updated_at: Time.at(row['msg_time']) + ) + post.custom_fields['original_gossamer_msg_id'] = row['msg_id'] + post.save! + + # Add recipient user to the private message topic + topic.topic_allowed_users.create!(user_id: to_user_id) + +# handle_post_attachments(row['msg_id'], post, from_user_id) + end end end -end