saint/discourse-import_scripts
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

v0.14 Code indentation fix and relative path fixes for require at start

Browse Source
This commit is contained in:
David Sainty 2024-06-26 15:26:26 +10:00
parent 30827f69b4
commit fdb69f2d73
1 changed files with 164 additions and 165 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

291
gossamer_forums.rb
View File

@ -1,5 +1,5 @@
# gossamer threads migration-import code # gossamer threads migration-import code
# v0.13 # v0.14
require 'mysql2' require 'mysql2'
require 'open-uri' require 'open-uri'
@ -12,8 +12,8 @@ require 'fileutils'
require 'csv' require 'csv'
require 'time' require 'time'
require File.expand_path("../../../config/environment", __FILE__) require File.expand_path("../../../../config/environment", __FILE__)
require_relative 'base' require_relative '../base'
class GossamerForumsImporter < ImportScripts::Base class GossamerForumsImporter < ImportScripts::Base
def initialize def initialize
@ -483,179 +483,178 @@ class GossamerForumsImporter < ImportScripts::Base
title title
end end
# Import topics and posts from Gossamer Forums to Discourse # Import topics and posts from Gossamer Forums to Discourse
def import_topics_and_posts_with_attachments def import_topics_and_posts_with_attachments
puts "Importing topics and posts with attachments..." puts "Importing topics and posts with attachments..."
# Execute the query to get all posts ordered by post_id # Execute the query to get all posts ordered by post_id
execute_query("SELECT * FROM gforum_Post ORDER BY post_id").each do |row| execute_query("SELECT * FROM gforum_Post ORDER BY post_id").each do |row|
puts "post_id #{row['post_id']} post_root_id #{row['post_root_id']} post_subject/title #{row['post_subject']} forum_id_fk/category_id #{row['forum_id_fk']}" puts "post_id #{row['post_id']} post_root_id #{row['post_root_id']} post_subject/title #{row['post_subject']} forum_id_fk/category_id #{row['forum_id_fk']}"
# discourse_user_id = @user_id_map[row['user_id_fk']] # discourse_user_id = @user_id_map[row['user_id_fk']]
discourse_user_id = fetch_user_id_mapping(row['user_id_fk']) discourse_user_id = fetch_user_id_mapping(row['user_id_fk'])
discourse_category_id = fetch_category_id_mapping(row['forum_id_fk']) discourse_category_id = fetch_category_id_mapping(row['forum_id_fk'])
puts "discourse_user_id #{discourse_user_id} discourse_category_id #{discourse_category_id}" puts "discourse_user_id #{discourse_user_id} discourse_category_id #{discourse_category_id}"
next unless discourse_user_id && discourse_category_id next unless discourse_user_id && discourse_category_id
if row['post_root_id'] == 0 if row['post_root_id'] == 0
puts "#1" puts "#1"
# Ensure the title is valid # Ensure the title is valid
title = ensure_valid_title(row['post_subject']) title = ensure_valid_title(row['post_subject'])
# Skip if the topic already exists # Skip if the topic already exists
unless TopicCustomField.exists?(name: 'original_gossamer_id', value: row['post_id']) unless TopicCustomField.exists?(name: 'original_gossamer_id', value: row['post_id'])
# Create the topic # Create the topic
begin begin
puts "#2" puts "#2"
puts "CREATE TOPIC title #{title} discourse_user_id #{discourse_user_id} category_id #{discourse_category_id}" puts "CREATE TOPIC title #{title} discourse_user_id #{discourse_user_id} category_id #{discourse_category_id}"
topic = Topic.create!( topic = Topic.create!(
title: title, title: title,
user_id: discourse_user_id, user_id: discourse_user_id,
created_at: Time.at(row['post_time']), created_at: Time.at(row['post_time']),
updated_at: Time.at(row['post_latest_reply']), updated_at: Time.at(row['post_latest_reply']),
category_id: discourse_category_id category_id: discourse_category_id
) )
topic.custom_fields['original_gossamer_id'] = row['post_id'] topic.custom_fields['original_gossamer_id'] = row['post_id']
topic.save! topic.save!
# Create the initial post in the topic # Create the initial post in the topic
puts "CREATE POST topic.id #{topic.id} discourse_user_id #{discourse_user_id}" puts "CREATE POST topic.id #{topic.id} discourse_user_id #{discourse_user_id}"
# Ensure the raw post stirng contents itself is acceptable to Discourse # Ensure the raw post stirng contents itself is acceptable to Discourse
sanitized_post_message = row['post_message']&.tr("\0", '') || "" sanitized_post_message = row['post_message']&.tr("\0", '') || ""
# Remove the [signature] label from appearing at the end of the messages after import # Remove the [signature] label from appearing at the end of the messages after import
sanitized_post_message.sub(/\n?\[signature\]\n?\z/, '') sanitized_post_message.sub(/\n?\[signature\]\n?\z/, '')
post = Post.create!( post = Post.create!(
topic_id: topic.id, topic_id: topic.id,
user_id: discourse_user_id, user_id: discourse_user_id,
# raw: import_attachments(row['post_message'], row['post_id']), # raw: import_attachments(row['post_message'], row['post_id']),
# raw: row['post_message'] || "", # raw: row['post_message'] || "",
raw: sanitized_post_message, raw: sanitized_post_message,
created_at: Time.at(row['post_time']), created_at: Time.at(row['post_time']),
updated_at: Time.at(row['post_latest_reply']) updated_at: Time.at(row['post_latest_reply'])
) )
post.custom_fields['original_gossamer_id'] = row['post_id'] post.custom_fields['original_gossamer_id'] = row['post_id']
post.save! post.save!
# Handle attachments for the post # Handle attachments for the post
handle_post_attachments(row['post_id'], post, discourse_user_id) handle_post_attachments(row['post_id'], post, discourse_user_id)
# Create URL mappings # Create URL mappings
# old_url = "https://old/forum/#{row['forum_name']}/topics/#{row['post_id']}" # old_url = "https://old/forum/#{row['forum_name']}/topics/#{row['post_id']}"
new_url = "https://new/t/#{topic.slug}/#{topic.id}" new_url = "https://new/t/#{topic.slug}/#{topic.id}"
insert_url_mapping(row['post_id'], new_url, title) insert_url_mapping(row['post_id'], new_url, title)
rescue ActiveRecord::RecordInvalid => e rescue ActiveRecord::RecordInvalid => e
puts "Error importing topic with post_id #{row['post_id']}: #{e.message}" puts "Error importing topic with post_id #{row['post_id']}: #{e.message}"
end end
end
else
puts "#3"
# Find the root topic for the post
root_topic_field = TopicCustomField.find_by(name: 'original_gossamer_id', value: row['post_root_id'])
if root_topic_field
topic_id = root_topic_field.topic_id
# Find the parent post for the reply
parent_post_field = PostCustomField.find_by(name: 'original_gossamer_id', value: row['post_father_id'])
reply_to_post_number = parent_post_field ? Post.find(parent_post_field.post_id).post_number : nil
# Create the post in the existing topic
begin
puts "#4"
# Ensure the raw post string contents itself is acceptable to Discourse
sanitized_post_message = row['post_message']&.tr("\0", '') || ""
# Remove the [signature] label from appearing at the end of the messages after import
sanitized_post_message.sub(/\n?\[signature\]\n?\z/, '')
post = Post.create!(
topic_id: topic_id,
user_id: discourse_user_id,
# raw: import_attachments(row['post_message'], row['post_id']),
# raw: row['post_message'] || "",
raw: sanitized_post_message,
created_at: Time.at(row['post_time']),
updated_at: Time.at(row['post_latest_reply']),
reply_to_post_number: reply_to_post_number
)
post.custom_fields['original_gossamer_id'] = row['post_id']
post.save!
# Handle attachments for the post
handle_post_attachments(row['post_id'], post, discourse_user_id)
rescue ActiveRecord::RecordInvalid => e
puts "Error importing post with post_id #{row['post_id']}: #{e.message}"
end end
else else
puts "Warning: Root topic not found for post_id #{row['post_id']} with post_root_id #{row['post_root_id']}" puts "#3"
# Find the root topic for the post
root_topic_field = TopicCustomField.find_by(name: 'original_gossamer_id', value: row['post_root_id'])
if root_topic_field
topic_id = root_topic_field.topic_id
# Find the parent post for the reply
parent_post_field = PostCustomField.find_by(name: 'original_gossamer_id', value: row['post_father_id'])
reply_to_post_number = parent_post_field ? Post.find(parent_post_field.post_id).post_number : nil
# Create the post in the existing topic
begin
puts "#4"
# Ensure the raw post string contents itself is acceptable to Discourse
sanitized_post_message = row['post_message']&.tr("\0", '') || ""
# Remove the [signature] label from appearing at the end of the messages after import
sanitized_post_message.sub(/\n?\[signature\]\n?\z/, '')
post = Post.create!(
topic_id: topic_id,
user_id: discourse_user_id,
# raw: import_attachments(row['post_message'], row['post_id']),
# raw: row['post_message'] || "",
raw: sanitized_post_message,
created_at: Time.at(row['post_time']),
updated_at: Time.at(row['post_latest_reply']),
reply_to_post_number: reply_to_post_number
)
post.custom_fields['original_gossamer_id'] = row['post_id']
post.save!
# Handle attachments for the post
handle_post_attachments(row['post_id'], post, discourse_user_id)
rescue ActiveRecord::RecordInvalid => e
puts "Error importing post with post_id #{row['post_id']}: #{e.message}"
end
else
puts "Warning: Root topic not found for post_id #{row['post_id']} with post_root_id #{row['post_root_id']}"
end
end end
end end
end end
end
# Import personal messages from gforum_Message table (both inbox and sent messages) # Import personal messages from gforum_Message table (both inbox and sent messages)
def import_personal_messages def import_personal_messages
puts "Importing personal (inbox and sendmail) messages..." puts "Importing personal (inbox and sendmail) messages..."
execute_query("SELECT * FROM gforum_Message").each do |row| execute_query("SELECT * FROM gforum_Message").each do |row|
from_user_id = fetch_user_id_mapping(row['from_user_id_fk']) from_user_id = fetch_user_id_mapping(row['from_user_id_fk'])
to_user_id = fetch_user_id_mapping(row['to_user_id_fk']) to_user_id = fetch_user_id_mapping(row['to_user_id_fk'])
next unless from_user_id && to_user_id next unless from_user_id && to_user_id
# Skip if the message already exists # Skip if the message already exists
unless TopicCustomField.exists?(name: 'original_gossamer_msg_id', value: row['msg_id']) unless TopicCustomField.exists?(name: 'original_gossamer_msg_id', value: row['msg_id'])
# Sanitize the message, ensuring we have an empty string or the content without any \0 # Sanitize the message, ensuring we have an empty string or the content without any \0
sanitized_message = row['msg_body']&.tr("\0", '') || "" sanitized_message = row['msg_body']&.tr("\0", '') || ""
# Set default message body if the sanitized message is blank # Set default message body if the sanitized message is blank
sanitized_message = " " if sanitized_message.strip.empty? sanitized_message = " " if sanitized_message.strip.empty?
# # If we do not change the "min personal message post length" to 1, we need this. # # If we do not change the "min personal message post length" to 1, we need this.
# sanitized_message = sanitized_message.ljust(10, ' ') if sanitized_message.length < 10 # sanitized_message = sanitized_message.ljust(10, ' ') if sanitized_message.length < 10
# Check and set a default title if the original title is nil or empty # Check and set a default title if the original title is nil or empty
title = row['msg_subject']&.strip title = row['msg_subject']&.strip
title = "<no subject>" if title.nil? || title.empty? title = "<no subject>" if title.nil? || title.empty?
puts "IMPORTING title #{row['msg_subject']} user_id #{from_user_id} to_user_id #{to_user_id}" puts "IMPORTING title #{row['msg_subject']} user_id #{from_user_id} to_user_id #{to_user_id}"
# Create a private message topic in Discourse # Create a private message topic in Discourse
topic = Topic.create!( topic = Topic.create!(
# title: row['msg_subject'], # title: row['msg_subject'],
title: title, title: title,
user_id: from_user_id, user_id: from_user_id,
archetype: Archetype.private_message, archetype: Archetype.private_message,
created_at: Time.at(row['msg_time']), created_at: Time.at(row['msg_time']),
updated_at: Time.at(row['msg_time']) updated_at: Time.at(row['msg_time'])
) )
topic.custom_fields['original_gossamer_msg_id'] = row['msg_id'] topic.custom_fields['original_gossamer_msg_id'] = row['msg_id']
topic.save! topic.save!
# Create the message as a post in the private topic # Create the message as a post in the private topic
post = Post.create!( post = Post.create!(
topic_id: topic.id, topic_id: topic.id,
user_id: from_user_id, user_id: from_user_id,
# raw: row['msg_body'], # raw: row['msg_body'],
raw: sanitized_message, raw: sanitized_message,
created_at: Time.at(row['msg_time']), created_at: Time.at(row['msg_time']),
updated_at: Time.at(row['msg_time']) updated_at: Time.at(row['msg_time'])
) )
post.custom_fields['original_gossamer_msg_id'] = row['msg_id'] post.custom_fields['original_gossamer_msg_id'] = row['msg_id']
post.save! post.save!
# Add recipient user to the private message topic # Add recipient user to the private message topic
topic.topic_allowed_users.create!(user_id: to_user_id) topic.topic_allowed_users.create!(user_id: to_user_id)
# handle_post_attachments(row['msg_id'], post, from_user_id) # handle_post_attachments(row['msg_id'], post, from_user_id)
end
end end
end end
end