saint/discourse-import_scripts
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

v0.13 Support for import and attachment of images/attachments to posts

Browse Source
This commit is contained in:
David Sainty 2024-06-22 15:13:41 +10:00
parent 9825f919b9
commit 4816641be3
1 changed files with 65 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

100
gossamer_forums.rb
View File

@ -1,5 +1,5 @@
# gossamer threads migration-import code
# v0.12
# v0.13
require 'mysql2'
require 'open-uri'
@ -10,6 +10,7 @@ require 'sqlite3'
require 'digest'
require 'fileutils'
require 'csv'
require 'time'
require File.expand_path("../../../config/environment", __FILE__)
require_relative 'base'
@ -182,12 +183,12 @@ class GossamerForumsImporter < ImportScripts::Base
email
end
# Helper method to download an image from a URL
def download_image(url)
# Helper method to download an attachment / image from a URL
def download_attachment(url)
begin
URI.open(url).read
rescue OpenURI::HTTPError => e
puts "Failed to download image from #{url}: #{e.message}"
puts "Failed to download attachment from #{url}: #{e.message}"
nil
rescue URI::InvalidURIError => e
puts "Failed to handle invalid URL/URI for #{url}: #{e.message}"
@ -195,7 +196,8 @@ class GossamerForumsImporter < ImportScripts::Base
end
end
def upload_image(user, file, filename, gossamer_url)
# Helper method to upload an attachment / image to Discourse
def upload_attachment(user, file, filename, gossamer_url)
begin
upload = Upload.create!(
user_id: user.id,
@ -213,14 +215,42 @@ class GossamerForumsImporter < ImportScripts::Base
# Move the file to the correct location
# FileUtils.mv(file.path, upload.path)
upload.save!
upload
rescue => e
puts "Failed to upload image #{filename} for user #{user.username}: #{e.message}"
puts "Failed to upload attachment #{filename} for user #{user.username}: #{e.message}"
nil
end
end
# Helper method to handle post attachments
def handle_post_attachments(gossamer_post_id, post, user_id)
execute_query("SELECT * FROM gforum_PostAttachment WHERE post_id_fk = #{post_id}").each do |att_row|
attachment_url = "https://forum.slowtwitch.com/forum/?do=post_attachment;postatt_id=#{att_row['postatt_id']}"
attachment_data = download_attachment(attachment_url)
next unless attachment_data
mime_type = att_row['postatt_content']
temp_file = Tempfile.new(['attachment', File.extname(att_row['postatt_filename'])])
temp_file.binmode
temp_file.write(attachment_data)
temp_file.rewind
upload = upload_attachment(user_id, temp_file, att_row['postatt_filename'], attachment_url)
next unless upload
upload_url = upload.url
if mime_type.start_with?('image/')
post.raw += "\n![#{att_row['postatt_filename']}](#{upload_url})"
else
post.raw += "\n[#{att_row['postatt_filename']}](#{upload_url})"
end
post.save!
temp_file.close
temp_file.unlink
end
end
# def download_file(url)
# require 'open-uri'
@ -372,7 +402,7 @@ class GossamerForumsImporter < ImportScripts::Base
next unless ['image/jpeg', 'image/png'].include?(file['File_MimeType'])
puts "#B"
image_data = download_image(file_url)
image_data = download_attachment(file_url)
next if image_data.nil?
puts "#C"
@ -383,7 +413,7 @@ class GossamerForumsImporter < ImportScripts::Base
if images_imported == 0
puts "#D"
upload = upload_image(user, temp_file, file['File_Name'], file_url)
upload = upload_attachment(user, temp_file, file['File_Name'], file_url)
next if upload.nil?
user.user_avatar = UserAvatar.create!(user_id: user.id, custom_upload_id: upload.id)
@ -454,8 +484,8 @@ class GossamerForumsImporter < ImportScripts::Base
end
# Import topics and posts from Gossamer Forums to Discourse
def import_topics_and_posts
puts "Importing topics and posts..."
def import_topics_and_posts_with_attachments
puts "Importing topics and posts with attachments..."
# Execute the query to get all posts ordered by post_id
execute_query("SELECT * FROM gforum_Post ORDER BY post_id").each do |row|
@ -463,7 +493,6 @@ def import_topics_and_posts
# discourse_user_id = @user_id_map[row['user_id_fk']]
discourse_user_id = fetch_user_id_mapping(row['user_id_fk'])
discourse_category_id = fetch_category_id_mapping(row['forum_id_fk'])
puts "discourse_user_id #{discourse_user_id} discourse_category_id #{discourse_category_id}"
next unless discourse_user_id && discourse_category_id
@ -490,7 +519,12 @@ def import_topics_and_posts
# Create the initial post in the topic
puts "CREATE POST topic.id #{topic.id} discourse_user_id #{discourse_user_id}"
# Ensure the raw post stirng contents itself is acceptable to Discourse
sanitized_post_message = row['post_message']&.tr("\0", '') || ""
# Remove the [signature] label from appearing at the end of the messages after import
sanitized_post_message.sub(/\n?\[signature\]\n?\z/, '')
post = Post.create!(
topic_id: topic.id,
user_id: discourse_user_id,
@ -503,6 +537,9 @@ def import_topics_and_posts
post.custom_fields['original_gossamer_id'] = row['post_id']
post.save!
# Handle attachments for the post
handle_post_attachments(row['post_id'], post, discourse_user_id)
# Create URL mappings
# old_url = "https://old/forum/#{row['forum_name']}/topics/#{row['post_id']}"
new_url = "https://new/t/#{topic.slug}/#{topic.id}"
@ -528,7 +565,12 @@ def import_topics_and_posts
# Create the post in the existing topic
begin
puts "#4"
# Ensure the raw post string contents itself is acceptable to Discourse
sanitized_post_message = row['post_message']&.tr("\0", '') || ""
# Remove the [signature] label from appearing at the end of the messages after import
sanitized_post_message.sub(/\n?\[signature\]\n?\z/, '')
post = Post.create!(
topic_id: topic_id,
user_id: discourse_user_id,
@ -541,6 +583,9 @@ def import_topics_and_posts
)
post.custom_fields['original_gossamer_id'] = row['post_id']
post.save!
# Handle attachments for the post
handle_post_attachments(row['post_id'], post, discourse_user_id)
rescue ActiveRecord::RecordInvalid => e
puts "Error importing post with post_id #{row['post_id']}: #{e.message}"
end
@ -557,10 +602,10 @@ def import_personal_messages
puts "Importing personal (inbox and sendmail) messages..."
execute_query("SELECT * FROM gforum_Message").each do |row|
from_user_id = fetch_user_id_mapping(row['from_user_id_fk'])
to_user_id = fetch_user_id_mapping(row['to_user_id_fk'])
from_user_id = fetch_user_id_mapping(row['from_user_id_fk'])
to_user_id = fetch_user_id_mapping(row['to_user_id_fk'])
next unless from_user_id && to_user_id
next unless from_user_id && to_user_id
# Skip if the message already exists
unless TopicCustomField.exists?(name: 'original_gossamer_msg_id', value: row['msg_id'])
@ -582,7 +627,8 @@ def import_personal_messages
# Create a private message topic in Discourse
topic = Topic.create!(
title: row['msg_subject'],
# title: row['msg_subject'],
title: title,
user_id: from_user_id,
archetype: Archetype.private_message,
created_at: Time.at(row['msg_time']),
@ -605,27 +651,13 @@ def import_personal_messages
# Add recipient user to the private message topic
topic.topic_allowed_users.create!(user_id: to_user_id)
# handle_post_attachments(row['msg_id'], post, from_user_id)
end
end
end
# Import attachments for a post
def import_post_attachments(post_message, post_id)
# Fetch attachments related to the post
attachments = execute_query("SELECT * FROM gforum_PostAttachment WHERE post_id_fk = #{post_id}")
attachments.each do |attachment|
# Append attachment links to the post message
file_url = "https://forum.slowtwitch.com/images/posts/attachments/#{attachment['ID'] % 10}/#{attachment['ID']}-#{attachment['File_Name']}"
post_message += "\n\n![#{attachment['File_Name']}](#{file_url})"
end
1# post_message
end
# Main method to perform the import
def perform_import
@ -641,14 +673,12 @@ end
export_username_mapping_to_csv("gossamer-migration-username-mapping#{timestamp}")
import_categories
import_topics_and_posts
import_topics_and_posts_with_attachments
export_url_mapping_to_csv("gossamer-migration-url-mapping#{timestamp}")
create_nginx_rewrite_rules("gossamer-redirects.conf")
import_personal_messages
# import_attachments
puts "Gossamer Forums import complete! #{timestamp}"
end
end