v0.13 Support for import and attachment of images/attachments to posts

gossamer_forums.rb

@@ -1,5 +1,5 @@
 # gossamer threads migration-import code
-# v0.12
+# v0.13
 
 require 'mysql2'
 require 'open-uri'
@@ -10,6 +10,7 @@ require 'sqlite3'
 require 'digest'
 require 'fileutils'
 require 'csv'
+require 'time'
 
 require File.expand_path("../../../config/environment", __FILE__)
 require_relative 'base'
@@ -182,12 +183,12 @@ class GossamerForumsImporter < ImportScripts::Base
     email
   end
 
-  # Helper method to download an image from a URL
+  # Helper method to download an attachment / image from a URL
-  def download_image(url)
+  def download_attachment(url)
     begin
       URI.open(url).read
     rescue OpenURI::HTTPError => e
-      puts "Failed to download image from #{url}: #{e.message}"
+      puts "Failed to download attachment from #{url}: #{e.message}"
       nil
     rescue URI::InvalidURIError => e
       puts "Failed to handle invalid URL/URI for #{url}: #{e.message}"
@@ -195,7 +196,8 @@ class GossamerForumsImporter < ImportScripts::Base
     end
   end
 
-  def upload_image(user, file, filename, gossamer_url)
+  # Helper method to upload an attachment / image to Discourse
+  def upload_attachment(user, file, filename, gossamer_url)
     begin
       upload = Upload.create!(
         user_id: user.id,
@@ -213,14 +215,42 @@ class GossamerForumsImporter < ImportScripts::Base
       # Move the file to the correct location
 #      FileUtils.mv(file.path, upload.path)
       upload.save!
-    
       upload
     rescue => e
-      puts "Failed to upload image #{filename} for user #{user.username}: #{e.message}"
+      puts "Failed to upload attachment #{filename} for user #{user.username}: #{e.message}"
       nil
     end
   end
 
+  # Helper method to handle post attachments
+  def handle_post_attachments(gossamer_post_id, post, user_id)
+    execute_query("SELECT * FROM gforum_PostAttachment WHERE post_id_fk = #{post_id}").each do |att_row|
+      attachment_url = "https://forum.slowtwitch.com/forum/?do=post_attachment;postatt_id=#{att_row['postatt_id']}"
+      attachment_data = download_attachment(attachment_url)
+      next unless attachment_data
+  
+      mime_type = att_row['postatt_content']
+      temp_file = Tempfile.new(['attachment', File.extname(att_row['postatt_filename'])])
+      temp_file.binmode
+      temp_file.write(attachment_data)
+      temp_file.rewind
+  
+      upload = upload_attachment(user_id, temp_file, att_row['postatt_filename'], attachment_url)
+      next unless upload
+  
+      upload_url = upload.url
+      if mime_type.start_with?('image/')
+        post.raw += "\n![#{att_row['postatt_filename']}](#{upload_url})"
+      else
+        post.raw += "\n[#{att_row['postatt_filename']}](#{upload_url})"
+      end
+      post.save!
+  
+      temp_file.close
+      temp_file.unlink
+    end
+  end
+
 
 #  def download_file(url)
 #    require 'open-uri'
@@ -372,7 +402,7 @@ class GossamerForumsImporter < ImportScripts::Base
       next unless ['image/jpeg', 'image/png'].include?(file['File_MimeType'])
       puts "#B"
   
-      image_data = download_image(file_url)
+      image_data = download_attachment(file_url)
       next if image_data.nil?
       puts "#C"
   
@@ -383,7 +413,7 @@ class GossamerForumsImporter < ImportScripts::Base
   
       if images_imported == 0
         puts "#D"
-        upload = upload_image(user, temp_file, file['File_Name'], file_url)
+        upload = upload_attachment(user, temp_file, file['File_Name'], file_url)
         next if upload.nil?
   
         user.user_avatar = UserAvatar.create!(user_id: user.id, custom_upload_id: upload.id)
@@ -454,8 +484,8 @@ class GossamerForumsImporter < ImportScripts::Base
   end
 
 # Import topics and posts from Gossamer Forums to Discourse
-def import_topics_and_posts
+def import_topics_and_posts_with_attachments
-  puts "Importing topics and posts..."
+  puts "Importing topics and posts with attachments..."
   
   # Execute the query to get all posts ordered by post_id
   execute_query("SELECT * FROM gforum_Post ORDER BY post_id").each do |row|
@@ -463,7 +493,6 @@ def import_topics_and_posts
 #    discourse_user_id = @user_id_map[row['user_id_fk']]
     discourse_user_id = fetch_user_id_mapping(row['user_id_fk'])
     discourse_category_id = fetch_category_id_mapping(row['forum_id_fk'])
-
     puts "discourse_user_id  #{discourse_user_id}  discourse_category_id  #{discourse_category_id}"
     next unless discourse_user_id && discourse_category_id
 
@@ -490,7 +519,12 @@ def import_topics_and_posts
 
           # Create the initial post in the topic
           puts "CREATE POST  topic.id #{topic.id}  discourse_user_id #{discourse_user_id}"
+
+          # Ensure the raw post stirng contents itself is acceptable to Discourse
           sanitized_post_message = row['post_message']&.tr("\0", '') || ""
+
+          # Remove the [signature] label from appearing at the end of the messages after import
+          sanitized_post_message.sub(/\n?\[signature\]\n?\z/, '')
           post = Post.create!(
             topic_id: topic.id,
             user_id: discourse_user_id,
@@ -503,6 +537,9 @@ def import_topics_and_posts
           post.custom_fields['original_gossamer_id'] = row['post_id']
           post.save!
 
+          # Handle attachments for the post
+          handle_post_attachments(row['post_id'], post, discourse_user_id)
+
           # Create URL mappings
 #          old_url = "https://old/forum/#{row['forum_name']}/topics/#{row['post_id']}"
           new_url = "https://new/t/#{topic.slug}/#{topic.id}"
@@ -528,7 +565,12 @@ def import_topics_and_posts
         # Create the post in the existing topic
         begin
           puts "#4"
+
+          # Ensure the raw post string contents itself is acceptable to Discourse
           sanitized_post_message = row['post_message']&.tr("\0", '') || ""
+
+          # Remove the [signature] label from appearing at the end of the messages after import
+          sanitized_post_message.sub(/\n?\[signature\]\n?\z/, '')
           post = Post.create!(
             topic_id: topic_id,
             user_id: discourse_user_id,
@@ -541,6 +583,9 @@ def import_topics_and_posts
           )
           post.custom_fields['original_gossamer_id'] = row['post_id']
           post.save!
+
+          # Handle attachments for the post
+          handle_post_attachments(row['post_id'], post, discourse_user_id)
         rescue ActiveRecord::RecordInvalid => e
           puts "Error importing post with post_id #{row['post_id']}: #{e.message}"
         end
@@ -582,7 +627,8 @@ def import_personal_messages
 
       # Create a private message topic in Discourse
       topic = Topic.create!(
-        title: row['msg_subject'],
+#         title: row['msg_subject'],
+        title: title,
         user_id: from_user_id,
         archetype: Archetype.private_message,
         created_at: Time.at(row['msg_time']),
@@ -605,27 +651,13 @@ def import_personal_messages
 
       # Add recipient user to the private message topic
       topic.topic_allowed_users.create!(user_id: to_user_id)
+
+#       handle_post_attachments(row['msg_id'], post, from_user_id)
     end
   end
 end
 
 
-  # Import attachments for a post
- def import_post_attachments(post_message, post_id)
-    # Fetch attachments related to the post
-    attachments = execute_query("SELECT * FROM gforum_PostAttachment WHERE post_id_fk = #{post_id}")
-    attachments.each do |attachment|
-
-      # Append attachment links to the post message
-      file_url = "https://forum.slowtwitch.com/images/posts/attachments/#{attachment['ID'] % 10}/#{attachment['ID']}-#{attachment['File_Name']}"
-      post_message += "\n\n![#{attachment['File_Name']}](#{file_url})"
-    end
-1#    post_message
-  end
-
-
-
-
 
   # Main method to perform the import
   def perform_import
@@ -641,14 +673,12 @@ end
     export_username_mapping_to_csv("gossamer-migration-username-mapping#{timestamp}")
 
     import_categories
-    import_topics_and_posts
+    import_topics_and_posts_with_attachments
     export_url_mapping_to_csv("gossamer-migration-url-mapping#{timestamp}")
     create_nginx_rewrite_rules("gossamer-redirects.conf")
 
     import_personal_messages
 
-    # import_attachments
-
     puts "Gossamer Forums import complete! #{timestamp}"
   end
 end