From 4816641be3dae956e41d23f75d86920596222b7d Mon Sep 17 00:00:00 2001 From: saint Date: Sat, 22 Jun 2024 15:13:41 +1000 Subject: [PATCH] v0.13 Support for import and attachment of images/attachments to posts --- gossamer_forums.rb | 100 +++++++++++++++++++++++++++++---------------- 1 file changed, 65 insertions(+), 35 deletions(-) diff --git a/gossamer_forums.rb b/gossamer_forums.rb index 0339762..5e08854 100644 --- a/gossamer_forums.rb +++ b/gossamer_forums.rb @@ -1,5 +1,5 @@ # gossamer threads migration-import code -# v0.12 +# v0.13 require 'mysql2' require 'open-uri' @@ -10,6 +10,7 @@ require 'sqlite3' require 'digest' require 'fileutils' require 'csv' +require 'time' require File.expand_path("../../../config/environment", __FILE__) require_relative 'base' @@ -182,12 +183,12 @@ class GossamerForumsImporter < ImportScripts::Base email end - # Helper method to download an image from a URL - def download_image(url) + # Helper method to download an attachment / image from a URL + def download_attachment(url) begin URI.open(url).read rescue OpenURI::HTTPError => e - puts "Failed to download image from #{url}: #{e.message}" + puts "Failed to download attachment from #{url}: #{e.message}" nil rescue URI::InvalidURIError => e puts "Failed to handle invalid URL/URI for #{url}: #{e.message}" @@ -195,7 +196,8 @@ class GossamerForumsImporter < ImportScripts::Base end end - def upload_image(user, file, filename, gossamer_url) + # Helper method to upload an attachment / image to Discourse + def upload_attachment(user, file, filename, gossamer_url) begin upload = Upload.create!( user_id: user.id, @@ -213,14 +215,42 @@ class GossamerForumsImporter < ImportScripts::Base # Move the file to the correct location # FileUtils.mv(file.path, upload.path) upload.save! - upload rescue => e - puts "Failed to upload image #{filename} for user #{user.username}: #{e.message}" + puts "Failed to upload attachment #{filename} for user #{user.username}: #{e.message}" nil end end + # Helper method to handle post attachments + def handle_post_attachments(gossamer_post_id, post, user_id) + execute_query("SELECT * FROM gforum_PostAttachment WHERE post_id_fk = #{post_id}").each do |att_row| + attachment_url = "https://forum.slowtwitch.com/forum/?do=post_attachment;postatt_id=#{att_row['postatt_id']}" + attachment_data = download_attachment(attachment_url) + next unless attachment_data + + mime_type = att_row['postatt_content'] + temp_file = Tempfile.new(['attachment', File.extname(att_row['postatt_filename'])]) + temp_file.binmode + temp_file.write(attachment_data) + temp_file.rewind + + upload = upload_attachment(user_id, temp_file, att_row['postatt_filename'], attachment_url) + next unless upload + + upload_url = upload.url + if mime_type.start_with?('image/') + post.raw += "\n![#{att_row['postatt_filename']}](#{upload_url})" + else + post.raw += "\n[#{att_row['postatt_filename']}](#{upload_url})" + end + post.save! + + temp_file.close + temp_file.unlink + end + end + # def download_file(url) # require 'open-uri' @@ -372,7 +402,7 @@ class GossamerForumsImporter < ImportScripts::Base next unless ['image/jpeg', 'image/png'].include?(file['File_MimeType']) puts "#B" - image_data = download_image(file_url) + image_data = download_attachment(file_url) next if image_data.nil? puts "#C" @@ -383,7 +413,7 @@ class GossamerForumsImporter < ImportScripts::Base if images_imported == 0 puts "#D" - upload = upload_image(user, temp_file, file['File_Name'], file_url) + upload = upload_attachment(user, temp_file, file['File_Name'], file_url) next if upload.nil? user.user_avatar = UserAvatar.create!(user_id: user.id, custom_upload_id: upload.id) @@ -454,8 +484,8 @@ class GossamerForumsImporter < ImportScripts::Base end # Import topics and posts from Gossamer Forums to Discourse -def import_topics_and_posts - puts "Importing topics and posts..." +def import_topics_and_posts_with_attachments + puts "Importing topics and posts with attachments..." # Execute the query to get all posts ordered by post_id execute_query("SELECT * FROM gforum_Post ORDER BY post_id").each do |row| @@ -463,7 +493,6 @@ def import_topics_and_posts # discourse_user_id = @user_id_map[row['user_id_fk']] discourse_user_id = fetch_user_id_mapping(row['user_id_fk']) discourse_category_id = fetch_category_id_mapping(row['forum_id_fk']) - puts "discourse_user_id #{discourse_user_id} discourse_category_id #{discourse_category_id}" next unless discourse_user_id && discourse_category_id @@ -490,7 +519,12 @@ def import_topics_and_posts # Create the initial post in the topic puts "CREATE POST topic.id #{topic.id} discourse_user_id #{discourse_user_id}" + + # Ensure the raw post stirng contents itself is acceptable to Discourse sanitized_post_message = row['post_message']&.tr("\0", '') || "" + + # Remove the [signature] label from appearing at the end of the messages after import + sanitized_post_message.sub(/\n?\[signature\]\n?\z/, '') post = Post.create!( topic_id: topic.id, user_id: discourse_user_id, @@ -503,6 +537,9 @@ def import_topics_and_posts post.custom_fields['original_gossamer_id'] = row['post_id'] post.save! + # Handle attachments for the post + handle_post_attachments(row['post_id'], post, discourse_user_id) + # Create URL mappings # old_url = "https://old/forum/#{row['forum_name']}/topics/#{row['post_id']}" new_url = "https://new/t/#{topic.slug}/#{topic.id}" @@ -528,7 +565,12 @@ def import_topics_and_posts # Create the post in the existing topic begin puts "#4" + + # Ensure the raw post string contents itself is acceptable to Discourse sanitized_post_message = row['post_message']&.tr("\0", '') || "" + + # Remove the [signature] label from appearing at the end of the messages after import + sanitized_post_message.sub(/\n?\[signature\]\n?\z/, '') post = Post.create!( topic_id: topic_id, user_id: discourse_user_id, @@ -541,6 +583,9 @@ def import_topics_and_posts ) post.custom_fields['original_gossamer_id'] = row['post_id'] post.save! + + # Handle attachments for the post + handle_post_attachments(row['post_id'], post, discourse_user_id) rescue ActiveRecord::RecordInvalid => e puts "Error importing post with post_id #{row['post_id']}: #{e.message}" end @@ -557,10 +602,10 @@ def import_personal_messages puts "Importing personal (inbox and sendmail) messages..." execute_query("SELECT * FROM gforum_Message").each do |row| - from_user_id = fetch_user_id_mapping(row['from_user_id_fk']) - to_user_id = fetch_user_id_mapping(row['to_user_id_fk']) + from_user_id = fetch_user_id_mapping(row['from_user_id_fk']) + to_user_id = fetch_user_id_mapping(row['to_user_id_fk']) - next unless from_user_id && to_user_id + next unless from_user_id && to_user_id # Skip if the message already exists unless TopicCustomField.exists?(name: 'original_gossamer_msg_id', value: row['msg_id']) @@ -582,7 +627,8 @@ def import_personal_messages # Create a private message topic in Discourse topic = Topic.create!( - title: row['msg_subject'], +# title: row['msg_subject'], + title: title, user_id: from_user_id, archetype: Archetype.private_message, created_at: Time.at(row['msg_time']), @@ -605,27 +651,13 @@ def import_personal_messages # Add recipient user to the private message topic topic.topic_allowed_users.create!(user_id: to_user_id) + +# handle_post_attachments(row['msg_id'], post, from_user_id) end end end - # Import attachments for a post - def import_post_attachments(post_message, post_id) - # Fetch attachments related to the post - attachments = execute_query("SELECT * FROM gforum_PostAttachment WHERE post_id_fk = #{post_id}") - attachments.each do |attachment| - - # Append attachment links to the post message - file_url = "https://forum.slowtwitch.com/images/posts/attachments/#{attachment['ID'] % 10}/#{attachment['ID']}-#{attachment['File_Name']}" - post_message += "\n\n![#{attachment['File_Name']}](#{file_url})" - end -1# post_message - end - - - - # Main method to perform the import def perform_import @@ -641,14 +673,12 @@ end export_username_mapping_to_csv("gossamer-migration-username-mapping#{timestamp}") import_categories - import_topics_and_posts + import_topics_and_posts_with_attachments export_url_mapping_to_csv("gossamer-migration-url-mapping#{timestamp}") create_nginx_rewrite_rules("gossamer-redirects.conf") import_personal_messages - # import_attachments - puts "Gossamer Forums import complete! #{timestamp}" end end