diff --git a/gossamer_forums.rb b/gossamer_forums.rb index cf557e0..82c4c15 100644 --- a/gossamer_forums.rb +++ b/gossamer_forums.rb @@ -36,6 +36,12 @@ class GossamerForumsImporter < ImportScripts::Base new_user_id INTEGER ); SQL + @db.execute <<-SQL + CREATE TABLE IF NOT EXISTS category_id_map ( + old_category_id INTEGER PRIMARY KEY, + new_category_id INTEGER + ); + SQL end def insert_user_id_mapping(old_user_id, new_user_id) @@ -46,6 +52,14 @@ class GossamerForumsImporter < ImportScripts::Base @db.get_first_value "SELECT new_user_id FROM user_id_map WHERE old_user_id = ?", old_user_id end + def insert_category_id_mapping(old_category_id, new_category_id) + @db.execute "INSERT OR REPLACE INTO category_id_map (old_category_id, new_category_id) VALUES (?, ?)", old_category_id, new_category_id + end + + def fetch_category_id_mapping(old_category_id) + @db.get_first_value "SELECT new_category_id FROM category_id_map WHERE old_category_id = ?", old_category_id + end + # Execute an SQL query on the Gossamer Forums database def execute_query(query) @mysql_client.query(query, as: :hash) @@ -106,21 +120,65 @@ class GossamerForumsImporter < ImportScripts::Base rescue OpenURI::HTTPError => e puts "Failed to download image from #{url}: #{e.message}" nil + rescue URI::InvalidURIError => e + puts "Failed to handle invalid URL/URI for #{url}: #{e.message}" + nil end end +def upload_image(user, file, filename, gossamer_url) + begin + upload = Upload.create!( + user_id: user.id, + original_filename: filename, + filesize: file.size, +# filesize: File.size(file.path), +# content_type: `file --brief --mime-type #{file.path}`.strip, +# sha1: Digest::SHA1.file(file.path).hexdigest, +# origin: 'user_avatar', +# retain_hours: nil, + url: gossamer_url + ) +# Error -- non-existent method upload.ensure_consistency! + + # Move the file to the correct location +# FileUtils.mv(file.path, upload.path) + upload.save! + + upload + rescue => e + puts "Failed to upload image #{filename} for user #{user.username}: #{e.message}" + nil + end +end + + +# def download_file(url) +# require 'open-uri' +# begin +# file = Tempfile.new +# file.binmode +# file.write(URI.open(url).read) +# file.rewind +# file +# rescue => e +# puts "Failed to download file from #{url}: #{e.message}" +# nil +# end +# end + # Helper method to upload an image to Discourse - def upload_image(user, image_data, filename) - return if image_data.nil? - - upload = Upload.create_for(user.id, File.open(image_data.path), filename, 'image/jpeg') - if upload.nil? || !upload.persisted? - puts "Failed to upload image for user #{user.username}" - return - end - - upload - end +# def upload_image(user, image_data, filename) +# return if image_data.nil? +# +# upload = Upload.create_for(user.id, File.open(image_data.path), filename, 'image/jpeg') +# if upload.nil? || !upload.persisted? +# puts "Failed to upload image for user #{user.username}" +# return +# end +# +# upload +# end # Import users from Gossamer Forums to Discourse @@ -155,7 +213,7 @@ class GossamerForumsImporter < ImportScripts::Base user end - # For each user, add user ID mapping to @user_id_map now that we know what the Discourse user ID is, ... and append user bio and import user files + # For each user, add user ID mapping to SQLite now that we know what the Discourse user ID is, ... and append user bio and import user files users.each do |user| discourse_username = sanitize_username(user[:username], user[:email], user[:name]) discourse_user = User.find_by(username: discourse_username) @@ -167,6 +225,7 @@ class GossamerForumsImporter < ImportScripts::Base # # Store the user ID mapping # @user_id_map[user[:id]] = discourse_user.id + puts "for insert_user_id_mapping: user[:id] #{user[:id]} discourse_user.id #{discourse_user.id}" insert_user_id_mapping(user[:id], discourse_user.id) # Ensure user profile exists and bio_raw is a string @@ -188,7 +247,7 @@ class GossamerForumsImporter < ImportScripts::Base discourse_user.user_profile.save! # Import user files - # import_user_files(discourse_user) + import_user_files(discourse_user) end end @@ -240,28 +299,40 @@ class GossamerForumsImporter < ImportScripts::Base puts "User #{user.username} User ID: #{user.id} original_gossamer_id: #{original_gossamer_id} file_url: #{file_url}" next unless file['ForeignColName'] =~ /^user_image\d+$/ + puts "#A" + next unless ['image/jpeg', 'image/png'].include?(file['File_MimeType']) + puts "#B" image_data = download_image(file_url) next if image_data.nil? + puts "#C" - temp_file = Tempfile.new(['user_image', '.jpg']) + temp_file = Tempfile.new(['user_image', File.extname(file['File_Name'])]) temp_file.binmode temp_file.write(image_data) temp_file.rewind if images_imported == 0 - upload = upload_image(user, temp_file, file['File_Name']) + puts "#D" + upload = upload_image(user, temp_file, file['File_Name'], file_url) next if upload.nil? user.user_avatar = UserAvatar.create!(user_id: user.id, custom_upload_id: upload.id) user.save! + + # Set the Profile Header + UserProfile.find_by(user_id: user.id).update!(profile_background_upload_id: upload.id) + + # Set the User Card Background + UserProfile.find_by(user_id: user.id).update!(card_background_upload_id: upload.id) + images_imported += 1 - else - user.user_profile.bio_raw ||= "" - user.user_profile.bio_raw += "\n\n![#{file['File_Name']}](#{file_url})" - user.user_profile.save! - end - + end + puts "#E" + user.user_profile.bio_raw ||= "" + user.user_profile.bio_raw += "\n\n![#{file['File_Name']}](#{file_url})" + user.user_profile.save! + temp_file.close temp_file.unlink end @@ -277,11 +348,11 @@ class GossamerForumsImporter < ImportScripts::Base unless CategoryCustomField.exists?(name: 'original_gossamer_id', value: row['forum_id']) category_name = row['forum_name'] category_description = row['forum_desc'] || "No description provided" - + puts "id #{row['forum_id']} name #{category_name} description #{category_description}" # Create category in Discourse category = create_category( { - id: row['forum_id'] + 10, +# id: row['forum_id'] + 10, name: category_name, description: category_description, created_at: row['forum_last'] ? Time.at(row['forum_last']) : Time.now, @@ -296,6 +367,10 @@ class GossamerForumsImporter < ImportScripts::Base # category.custom_fields.create!(name: 'original_gossamer_id', value: row['forum_id']) category.custom_fields['original_gossamer_id'] = row['forum_id'] category.save! + + # Store the user ID mapping + puts "for insert_category_id_mapping: category[:id] #{category[:id]} row['forum_id'] #{row['forum_id']}" + insert_category_id_mapping(row['forum_id'], category[:id]) end end puts "Importing categories... Done." @@ -316,12 +391,15 @@ def import_topics_and_posts # Execute the query to get all posts ordered by post_id execute_query("SELECT * FROM gforum_Post ORDER BY post_id").each do |row| puts "post_id #{row['post_id']} post_root_id #{row['post_root_id']} post_subject/title #{row['post_subject']} forum_id_fk/category_id #{row['forum_id_fk']}" - discourse_user_id = fetch_user_id_mapping(row['user_id_fk']) # discourse_user_id = @user_id_map[row['user_id_fk']] + discourse_user_id = fetch_user_id_mapping(row['user_id_fk']) + discourse_category_id = fetch_category_id_mapping(row['forum_id_fk']) - next unless discourse_user_id + puts "discourse_user_id #{discourse_user_id} discourse_category_id #{discourse_category_id}" + next unless discourse_user_id && discourse_category_id if row['post_root_id'] == 0 + puts "#1" # Ensure the title is valid title = ensure_valid_title(row['post_subject']) @@ -329,21 +407,27 @@ def import_topics_and_posts unless TopicCustomField.exists?(name: 'original_gossamer_id', value: row['post_id']) # Create the topic begin + puts "#2" + puts "CREATE TOPIC title #{title} discourse_user_id #{discourse_user_id} category_id #{discourse_category_id}" topic = Topic.create!( title: title, user_id: discourse_user_id, created_at: Time.at(row['post_time']), updated_at: Time.at(row['post_latest_reply']), - category_id: row['forum_id_fk'] + 10 + category_id: discourse_category_id ) topic.custom_fields['original_gossamer_id'] = row['post_id'] topic.save! # Create the initial post in the topic + puts "CREATE POST topic.id #{topic.id} discourse_user_id #{discourse_user_id}" + sanitized_post_message = row['post_message']&.tr("\0", '') || "" post = Post.create!( topic_id: topic.id, user_id: discourse_user_id, - raw: import_post_attachments(row['post_message'], row['post_id']), +# raw: import_post_attachments(row['post_message'], row['post_id']), +# raw: row['post_message'] || "", + raw: sanitized_post_message, created_at: Time.at(row['post_time']), updated_at: Time.at(row['post_latest_reply']) ) @@ -355,6 +439,7 @@ def import_topics_and_posts end else + puts "#3" # Find the root topic for the post root_topic_field = TopicCustomField.find_by(name: 'original_gossamer_id', value: row['post_root_id']) @@ -367,10 +452,14 @@ def import_topics_and_posts # Create the post in the existing topic begin + puts "#4" + sanitized_post_message = row['post_message']&.tr("\0", '') || "" post = Post.create!( topic_id: topic_id, user_id: discourse_user_id, - raw: import_post_attachments(row['post_message'], row['post_id']), +# raw: import_post_attachments(row['post_message'], row['post_id']), +# raw: row['post_message'] || "", + raw: sanitized_post_message, created_at: Time.at(row['post_time']), updated_at: Time.at(row['post_latest_reply']), reply_to_post_number: reply_to_post_number @@ -387,42 +476,23 @@ def import_topics_and_posts end end -# Import topics and posts from Gossamer Forums to Discourse -# def import_topics_and_posts -# puts "Importing topics and posts..." -# execute_query("SELECT * FROM gforum_Post ORDER BY post_root_id, post_time").each do |row| -# puts "post_id #{row['post_id']} post_root_id #{row['post_root_id']} post_subject #{row['post_subject']}" -# Import attachments for a post -def import_post_attachments(post_message, post_id) - # Fetch attachments related to the post - attachments = execute_query("SELECT * FROM gforum_PostAttachment WHERE post_id_fk = #{post_id}") - attachments.each do |attachment| - # Append attachment links to the post message - file_url = "https://forum.slowtwitch.com/images/posts/attachments/#{attachment['ID'] % 10}/#{attachment['ID']}-#{attachment['File_Name']}" - post_message += "\n\n![#{attachment['File_Name']}](#{file_url})" - end - post_message -end - -# Import personal messages (both inbox and sent messages) +# Import personal messages from gforum_Message table (both inbox and sent messages) def import_personal_messages - puts "Importing personal messages..." - import_inbox_messages - import_sent_messages -end - -# Import inbox messages from gforum_Message table -def import_inbox_messages - puts "Importing inbox messages..." + puts "Importing personal (inbox and sendmail) messages..." execute_query("SELECT * FROM gforum_Message").each do |row| + + from_user_id = fetch_user_id_mapping(row['from_user_id_fk']) + to_user_id = fetch_user_id_mapping(row['to_user_id_fk']) + + next unless from_user_id && to_user_id + # Skip if the message already exists unless TopicCustomField.exists?(name: 'original_gossamer_msg_id', value: row['msg_id']) # Create a private message topic in Discourse - discourse_user_id = @user_id_map[row['from_user_id_fk']] topic = Topic.create!( title: row['msg_subject'], - user_id: discourse_user_id, + user_id: from_user_id, archetype: Archetype.private_message, created_at: Time.at(row['msg_time']), updated_at: Time.at(row['msg_time']) @@ -431,61 +501,50 @@ def import_inbox_messages topic.save! # Create the message as a post in the private topic - Post.create!( + sanitized_message = row['msg_body']&.tr("\0", '') || "" + post = Post.create!( topic_id: topic.id, - user_id: discourse_user_id, - raw: row['msg_body'], + user_id: from_user_id, +# raw: row['msg_body'], + raw: sanitized_message, created_at: Time.at(row['msg_time']), updated_at: Time.at(row['msg_time']) ) + post.custom_fields['original_gossamer_msg_id'] = row['msg_id'] + post.save! # Add recipient user to the private message topic - topic.topic_allowed_users.create!(user_id: row['to_user_id_fk']) + topic.topic_allowed_users.create!(user_id: to_user_id) end end end -# Import sent messages from gforum_SentMessage table -def import_sent_messages - puts "Importing sent messages..." - execute_query("SELECT * FROM gforum_SentMessage").each do |row| - # Skip if the message already exists - unless TopicCustomField.exists?(name: 'original_gossamer_sent_msg_id', value: row['msg_id']) - # Create a private message topic in Discourse - discourse_user_id = @user_id_map[row['from_user_id_fk']] - topic = Topic.create!( - title: row['msg_subject'], - user_id: discourse_user_id, - archetype: Archetype.private_message, - created_at: Time.at(row['msg_time']), - updated_at: Time.at(row['msg_time']) - ) - topic.custom_fields['original_gossamer_sent_msg_id'] = row['msg_id'] - topic.save! - # Create the message as a post in the private topic - Post.create!( - topic_id: topic.id, - user_id: discourse_user_id, - raw: row['msg_body'], - created_at: Time.at(row['msg_time']), - updated_at: Time.at(row['msg_time']) - ) +# Import attachments for a post +# def import_post_attachments(post_message, post_id) +# # Fetch attachments related to the post +# attachments = execute_query("SELECT * FROM gforum_PostAttachment WHERE post_id_fk = #{post_id}") +# attachments.each do |attachment| +# # Append attachment links to the post message +# file_url = "https://forum.slowtwitch.com/images/posts/attachments/#{attachment['ID'] % 10}/#{attachment['ID']}-#{attachment['File_Name']}" +# post_message += "\n\n![#{attachment['File_Name']}](#{file_url})" +# end +# post_message +# end + + + - # Add recipient user to the private message topic - topic.topic_allowed_users.create!(user_id: row['to_user_id_fk']) - end - end -end # Main method to perform the import def perform_import RateLimiter.disable puts "Starting Gossamer Forums import..." - import_users - import_categories + # import_users + # import_categories # import_topics_and_posts - # import_personal_messages + import_personal_messages + # import_attachments puts "Gossamer Forums import complete!" end end