saint/discourse-import_scripts
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

v0.14 Code indentation fix and relative path fixes for require at start

Browse Source
This commit is contained in:
David Sainty 2024-06-26 15:26:26 +10:00
parent 30827f69b4
commit fdb69f2d73
1 changed files with 164 additions and 165 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

329
gossamer_forums.rb
View File

@ -1,5 +1,5 @@
# gossamer threads migration-import code # gossamer threads migration-import code
# v0.13 # v0.14
require 'mysql2' require 'mysql2'
require 'open-uri' require 'open-uri'
@ -12,8 +12,8 @@ require 'fileutils'
require 'csv' require 'csv'
require 'time' require 'time'
require File.expand_path("../../../config/environment", __FILE__) require File.expand_path("../../../../config/environment", __FILE__)
require_relative 'base' require_relative '../base'
class GossamerForumsImporter < ImportScripts::Base class GossamerForumsImporter < ImportScripts::Base
def initialize def initialize
@ -483,179 +483,178 @@ class GossamerForumsImporter < ImportScripts::Base
title title
end end
# Import topics and posts from Gossamer Forums to Discourse # Import topics and posts from Gossamer Forums to Discourse
def import_topics_and_posts_with_attachments def import_topics_and_posts_with_attachments
puts "Importing topics and posts with attachments..." puts "Importing topics and posts with attachments..."
# Execute the query to get all posts ordered by post_id
execute_query("SELECT * FROM gforum_Post ORDER BY post_id").each do |row|
puts "post_id #{row['post_id']} post_root_id #{row['post_root_id']} post_subject/title #{row['post_subject']} forum_id_fk/category_id #{row['forum_id_fk']}"
# discourse_user_id = @user_id_map[row['user_id_fk']]
discourse_user_id = fetch_user_id_mapping(row['user_id_fk'])
discourse_category_id = fetch_category_id_mapping(row['forum_id_fk'])
puts "discourse_user_id #{discourse_user_id} discourse_category_id #{discourse_category_id}"
next unless discourse_user_id && discourse_category_id
# Execute the query to get all posts ordered by post_id if row['post_root_id'] == 0
execute_query("SELECT * FROM gforum_Post ORDER BY post_id").each do |row| puts "#1"
puts "post_id #{row['post_id']} post_root_id #{row['post_root_id']} post_subject/title #{row['post_subject']} forum_id_fk/category_id #{row['forum_id_fk']}" # Ensure the title is valid
# discourse_user_id = @user_id_map[row['user_id_fk']] title = ensure_valid_title(row['post_subject'])
discourse_user_id = fetch_user_id_mapping(row['user_id_fk'])
discourse_category_id = fetch_category_id_mapping(row['forum_id_fk']) # Skip if the topic already exists
puts "discourse_user_id #{discourse_user_id} discourse_category_id #{discourse_category_id}" unless TopicCustomField.exists?(name: 'original_gossamer_id', value: row['post_id'])
next unless discourse_user_id && discourse_category_id # Create the topic
begin
if row['post_root_id'] == 0 puts "#2"
puts "#1" puts "CREATE TOPIC title #{title} discourse_user_id #{discourse_user_id} category_id #{discourse_category_id}"
# Ensure the title is valid topic = Topic.create!(
title = ensure_valid_title(row['post_subject']) title: title,
user_id: discourse_user_id,
# Skip if the topic already exists created_at: Time.at(row['post_time']),
unless TopicCustomField.exists?(name: 'original_gossamer_id', value: row['post_id']) updated_at: Time.at(row['post_latest_reply']),
# Create the topic category_id: discourse_category_id
begin )
puts "#2" topic.custom_fields['original_gossamer_id'] = row['post_id']
puts "CREATE TOPIC title #{title} discourse_user_id #{discourse_user_id} category_id #{discourse_category_id}" topic.save!
topic = Topic.create!(
title: title, # Create the initial post in the topic
user_id: discourse_user_id, puts "CREATE POST topic.id #{topic.id} discourse_user_id #{discourse_user_id}"
created_at: Time.at(row['post_time']),
updated_at: Time.at(row['post_latest_reply']), # Ensure the raw post stirng contents itself is acceptable to Discourse
category_id: discourse_category_id sanitized_post_message = row['post_message']&.tr("\0", '') || ""
)
topic.custom_fields['original_gossamer_id'] = row['post_id'] # Remove the [signature] label from appearing at the end of the messages after import
topic.save! sanitized_post_message.sub(/\n?\[signature\]\n?\z/, '')
post = Post.create!(
# Create the initial post in the topic topic_id: topic.id,
puts "CREATE POST topic.id #{topic.id} discourse_user_id #{discourse_user_id}" user_id: discourse_user_id,
# raw: import_attachments(row['post_message'], row['post_id']),
# Ensure the raw post stirng contents itself is acceptable to Discourse # raw: row['post_message'] || "",
sanitized_post_message = row['post_message']&.tr("\0", '') || "" raw: sanitized_post_message,
created_at: Time.at(row['post_time']),
# Remove the [signature] label from appearing at the end of the messages after import updated_at: Time.at(row['post_latest_reply'])
sanitized_post_message.sub(/\n?\[signature\]\n?\z/, '') )
post = Post.create!( post.custom_fields['original_gossamer_id'] = row['post_id']
topic_id: topic.id, post.save!
user_id: discourse_user_id,
# raw: import_attachments(row['post_message'], row['post_id']), # Handle attachments for the post
# raw: row['post_message'] || "", handle_post_attachments(row['post_id'], post, discourse_user_id)
raw: sanitized_post_message,
created_at: Time.at(row['post_time']), # Create URL mappings
updated_at: Time.at(row['post_latest_reply']) # old_url = "https://old/forum/#{row['forum_name']}/topics/#{row['post_id']}"
) new_url = "https://new/t/#{topic.slug}/#{topic.id}"
post.custom_fields['original_gossamer_id'] = row['post_id'] insert_url_mapping(row['post_id'], new_url, title)
post.save!
rescue ActiveRecord::RecordInvalid => e
# Handle attachments for the post puts "Error importing topic with post_id #{row['post_id']}: #{e.message}"
handle_post_attachments(row['post_id'], post, discourse_user_id) end
end
# Create URL mappings
# old_url = "https://old/forum/#{row['forum_name']}/topics/#{row['post_id']}"
new_url = "https://new/t/#{topic.slug}/#{topic.id}"
insert_url_mapping(row['post_id'], new_url, title)
rescue ActiveRecord::RecordInvalid => e
puts "Error importing topic with post_id #{row['post_id']}: #{e.message}"
end
end
else
puts "#3"
# Find the root topic for the post
root_topic_field = TopicCustomField.find_by(name: 'original_gossamer_id', value: row['post_root_id'])
if root_topic_field
topic_id = root_topic_field.topic_id
# Find the parent post for the reply
parent_post_field = PostCustomField.find_by(name: 'original_gossamer_id', value: row['post_father_id'])
reply_to_post_number = parent_post_field ? Post.find(parent_post_field.post_id).post_number : nil
# Create the post in the existing topic
begin
puts "#4"
# Ensure the raw post string contents itself is acceptable to Discourse
sanitized_post_message = row['post_message']&.tr("\0", '') || ""
# Remove the [signature] label from appearing at the end of the messages after import
sanitized_post_message.sub(/\n?\[signature\]\n?\z/, '')
post = Post.create!(
topic_id: topic_id,
user_id: discourse_user_id,
# raw: import_attachments(row['post_message'], row['post_id']),
# raw: row['post_message'] || "",
raw: sanitized_post_message,
created_at: Time.at(row['post_time']),
updated_at: Time.at(row['post_latest_reply']),
reply_to_post_number: reply_to_post_number
)
post.custom_fields['original_gossamer_id'] = row['post_id']
post.save!
# Handle attachments for the post
handle_post_attachments(row['post_id'], post, discourse_user_id)
rescue ActiveRecord::RecordInvalid => e
puts "Error importing post with post_id #{row['post_id']}: #{e.message}"
end
else else
puts "Warning: Root topic not found for post_id #{row['post_id']} with post_root_id #{row['post_root_id']}" puts "#3"
# Find the root topic for the post
root_topic_field = TopicCustomField.find_by(name: 'original_gossamer_id', value: row['post_root_id'])
if root_topic_field
topic_id = root_topic_field.topic_id
# Find the parent post for the reply
parent_post_field = PostCustomField.find_by(name: 'original_gossamer_id', value: row['post_father_id'])
reply_to_post_number = parent_post_field ? Post.find(parent_post_field.post_id).post_number : nil
# Create the post in the existing topic
begin
puts "#4"
# Ensure the raw post string contents itself is acceptable to Discourse
sanitized_post_message = row['post_message']&.tr("\0", '') || ""
# Remove the [signature] label from appearing at the end of the messages after import
sanitized_post_message.sub(/\n?\[signature\]\n?\z/, '')
post = Post.create!(
topic_id: topic_id,
user_id: discourse_user_id,
# raw: import_attachments(row['post_message'], row['post_id']),
# raw: row['post_message'] || "",
raw: sanitized_post_message,
created_at: Time.at(row['post_time']),
updated_at: Time.at(row['post_latest_reply']),
reply_to_post_number: reply_to_post_number
)
post.custom_fields['original_gossamer_id'] = row['post_id']
post.save!
# Handle attachments for the post
handle_post_attachments(row['post_id'], post, discourse_user_id)
rescue ActiveRecord::RecordInvalid => e
puts "Error importing post with post_id #{row['post_id']}: #{e.message}"
end
else
puts "Warning: Root topic not found for post_id #{row['post_id']} with post_root_id #{row['post_root_id']}"
end
end end
end end
end end
end
# Import personal messages from gforum_Message table (both inbox and sent messages) # Import personal messages from gforum_Message table (both inbox and sent messages)
def import_personal_messages def import_personal_messages
puts "Importing personal (inbox and sendmail) messages..." puts "Importing personal (inbox and sendmail) messages..."
execute_query("SELECT * FROM gforum_Message").each do |row| execute_query("SELECT * FROM gforum_Message").each do |row|
from_user_id = fetch_user_id_mapping(row['from_user_id_fk'])
to_user_id = fetch_user_id_mapping(row['to_user_id_fk'])
next unless from_user_id && to_user_id
# Skip if the message already exists
unless TopicCustomField.exists?(name: 'original_gossamer_msg_id', value: row['msg_id'])
# Sanitize the message, ensuring we have an empty string or the content without any \0
sanitized_message = row['msg_body']&.tr("\0", '') || ""
# Set default message body if the sanitized message is blank
sanitized_message = " " if sanitized_message.strip.empty?
# # If we do not change the "min personal message post length" to 1, we need this.
# sanitized_message = sanitized_message.ljust(10, ' ') if sanitized_message.length < 10
from_user_id = fetch_user_id_mapping(row['from_user_id_fk']) # Check and set a default title if the original title is nil or empty
to_user_id = fetch_user_id_mapping(row['to_user_id_fk']) title = row['msg_subject']&.strip
title = "<no subject>" if title.nil? || title.empty?
puts "IMPORTING title #{row['msg_subject']} user_id #{from_user_id} to_user_id #{to_user_id}"
next unless from_user_id && to_user_id # Create a private message topic in Discourse
topic = Topic.create!(
# Skip if the message already exists # title: row['msg_subject'],
unless TopicCustomField.exists?(name: 'original_gossamer_msg_id', value: row['msg_id']) title: title,
user_id: from_user_id,
# Sanitize the message, ensuring we have an empty string or the content without any \0 archetype: Archetype.private_message,
sanitized_message = row['msg_body']&.tr("\0", '') || "" created_at: Time.at(row['msg_time']),
updated_at: Time.at(row['msg_time'])
# Set default message body if the sanitized message is blank )
sanitized_message = " " if sanitized_message.strip.empty? topic.custom_fields['original_gossamer_msg_id'] = row['msg_id']
topic.save!
# # If we do not change the "min personal message post length" to 1, we need this.
# sanitized_message = sanitized_message.ljust(10, ' ') if sanitized_message.length < 10 # Create the message as a post in the private topic
post = Post.create!(
# Check and set a default title if the original title is nil or empty topic_id: topic.id,
title = row['msg_subject']&.strip user_id: from_user_id,
title = "<no subject>" if title.nil? || title.empty? # raw: row['msg_body'],
raw: sanitized_message,
puts "IMPORTING title #{row['msg_subject']} user_id #{from_user_id} to_user_id #{to_user_id}" created_at: Time.at(row['msg_time']),
updated_at: Time.at(row['msg_time'])
# Create a private message topic in Discourse )
topic = Topic.create!( post.custom_fields['original_gossamer_msg_id'] = row['msg_id']
# title: row['msg_subject'], post.save!
title: title,
user_id: from_user_id, # Add recipient user to the private message topic
archetype: Archetype.private_message, topic.topic_allowed_users.create!(user_id: to_user_id)
created_at: Time.at(row['msg_time']),
updated_at: Time.at(row['msg_time']) # handle_post_attachments(row['msg_id'], post, from_user_id)
) end
topic.custom_fields['original_gossamer_msg_id'] = row['msg_id']
topic.save!
# Create the message as a post in the private topic
post = Post.create!(
topic_id: topic.id,
user_id: from_user_id,
# raw: row['msg_body'],
raw: sanitized_message,
created_at: Time.at(row['msg_time']),
updated_at: Time.at(row['msg_time'])
)
post.custom_fields['original_gossamer_msg_id'] = row['msg_id']
post.save!
# Add recipient user to the private message topic
topic.topic_allowed_users.create!(user_id: to_user_id)
# handle_post_attachments(row['msg_id'], post, from_user_id)
end end
end end
end