saint/discourse-import_scripts
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

v0.12 Significant improvement with URL topic-post mapping support

Browse Source
This commit is contained in:
David Sainty 2024-06-20 19:23:30 +10:00
parent 1e62f5ee09
commit 9825f919b9
1 changed files with 65 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

120
gossamer_forums.rb
View File

@ -1,5 +1,5 @@
# gossamer threads migration-import code # gossamer threads migration-import code
# v0.11 # v0.12
require 'mysql2' require 'mysql2'
require 'open-uri' require 'open-uri'
@ -60,9 +60,10 @@ class GossamerForumsImporter < ImportScripts::Base
SQL SQL
@db.execute <<-SQL @db.execute <<-SQL
CREATE TABLE IF NOT EXISTS url_map ( CREATE TABLE IF NOT EXISTS url_map (
old_url TEXT, old_post_id INTEGER PRIMARY KEY,
new_url TEXT, new_url TEXT,
title TEXT title TEXT
# created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
); );
SQL SQL
end end
@ -89,7 +90,7 @@ class GossamerForumsImporter < ImportScripts::Base
# Define a method to export the username mapping table to a CSV file # Define a method to export the username mapping table to a CSV file
def export_username_mapping_to_csv(filename) def export_username_mapping_to_csv(filename)
CSV.open(filename, 'wb') do |csv| CSV.open(filename, 'w') do |csv|
# Add headers # Add headers
csv << ['Old Username', 'New Username', 'Email', 'Full Name'] csv << ['Old Username', 'New Username', 'Email', 'Full Name']
@ -102,22 +103,31 @@ class GossamerForumsImporter < ImportScripts::Base
end end
# Insert a URL mapping into the SQLite database # Insert a URL mapping into the SQLite database
def insert_url_mapping(old_url, new_url, title) def insert_url_mapping(old_post_id, new_url, title)
@db.execute "INSERT INTO url_map (old_url, new_url, title) VALUES (?, ?, ?, ?, ?)", [old_url, new_url, title] @db.execute "INSERT INTO url_map (old_post_id, new_url, title) VALUES (?, ?, ?)", [old_post_id, new_url, title]
end end
# Export the URL mappings to a CSV file # Export the URL mappings to a CSV file
def export_url_mapping_to_csv(filename) def export_url_mapping_to_csv(filename)
CSV.open(filename, "wb") do |csv| CSV.open(filename, "w") do |csv|
# Add headers # Add headers
csv << ["Old URL", "New URL", "Title"] csv << ["Old Post ID", "New URL", "Title"]
@db.execute("SELECT * FROM url_map") do |row| @db.execute("SELECT old_post_id, new_url, title FROM url_map") do |row|
csv << row csv << row
end end
end end
puts "Exported URL mappings to #{filename}" puts "Exported URL mappings to #{filename}"
end end
# Method to create Nginx rewrite rules file
def create_nginx_rewrite_rules(filename)
File.open(filename, "w") do |file|
@db.execute("SELECT old_post_id, new_url FROM url_map") do |row|
old_post_id, new_url = row
file.puts "rewrite ^/forum/.*P#{old_post_id}/$ #{new_url} permanent;"
end
end
end
# Execute an SQL query on the Gossamer Forums database # Execute an SQL query on the Gossamer Forums database
def execute_query(query) def execute_query(query)
@ -158,7 +168,6 @@ class GossamerForumsImporter < ImportScripts::Base
sanitized_username sanitized_username
end end
# Sanitize email to replace restricted domains # Sanitize email to replace restricted domains
def sanitize_email(email) def sanitize_email(email)
restricted_domains = ['mailinator.com', 'example.com'] # Add more restricted domains as needed restricted_domains = ['mailinator.com', 'example.com'] # Add more restricted domains as needed
@ -173,7 +182,6 @@ class GossamerForumsImporter < ImportScripts::Base
email email
end end
# Helper method to download an image from a URL # Helper method to download an image from a URL
def download_image(url) def download_image(url)
begin begin
@ -187,31 +195,31 @@ class GossamerForumsImporter < ImportScripts::Base
end end
end end
def upload_image(user, file, filename, gossamer_url) def upload_image(user, file, filename, gossamer_url)
begin begin
upload = Upload.create!( upload = Upload.create!(
user_id: user.id, user_id: user.id,
original_filename: filename, original_filename: filename,
filesize: file.size, filesize: file.size,
# filesize: File.size(file.path), # filesize: File.size(file.path),
# content_type: `file --brief --mime-type #{file.path}`.strip, # content_type: `file --brief --mime-type #{file.path}`.strip,
# sha1: Digest::SHA1.file(file.path).hexdigest, # sha1: Digest::SHA1.file(file.path).hexdigest,
# origin: 'user_avatar', # origin: 'user_avatar',
# retain_hours: nil, # retain_hours: nil,
url: gossamer_url url: gossamer_url
) )
# Error -- non-existent method upload.ensure_consistency! # Error -- non-existent method upload.ensure_consistency!
# Move the file to the correct location # Move the file to the correct location
# FileUtils.mv(file.path, upload.path) # FileUtils.mv(file.path, upload.path)
upload.save! upload.save!
upload upload
rescue => e rescue => e
puts "Failed to upload image #{filename} for user #{user.username}: #{e.message}" puts "Failed to upload image #{filename} for user #{user.username}: #{e.message}"
nil nil
end
end end
end
# def download_file(url) # def download_file(url)
@ -437,13 +445,13 @@ end
puts "Importing categories... Done." puts "Importing categories... Done."
end end
# Helper function to ensure title meets the minimum length requirement # Helper function to ensure title meets the minimum length requirement
def ensure_valid_title(title, min_length = 5) def ensure_valid_title(title, min_length = 5)
if title.length < min_length if title.length < min_length
title += "." * (min_length - title.length) # Append dots to make it longer title += "." * (min_length - title.length) # Append dots to make it longer
end
title
end end
title
end
# Import topics and posts from Gossamer Forums to Discourse # Import topics and posts from Gossamer Forums to Discourse
def import_topics_and_posts def import_topics_and_posts
@ -486,7 +494,7 @@ def import_topics_and_posts
post = Post.create!( post = Post.create!(
topic_id: topic.id, topic_id: topic.id,
user_id: discourse_user_id, user_id: discourse_user_id,
# raw: import_post_attachments(row['post_message'], row['post_id']), # raw: import_attachments(row['post_message'], row['post_id']),
# raw: row['post_message'] || "", # raw: row['post_message'] || "",
raw: sanitized_post_message, raw: sanitized_post_message,
created_at: Time.at(row['post_time']), created_at: Time.at(row['post_time']),
@ -496,9 +504,9 @@ def import_topics_and_posts
post.save! post.save!
# Create URL mappings # Create URL mappings
old_url = "https://old/forum/#{row['forum_name']}/topics/#{row['post_id']}" # old_url = "https://old/forum/#{row['forum_name']}/topics/#{row['post_id']}"
new_url = "https://new/t/#{topic.slug}/#{topic.id}" new_url = "https://new/t/#{topic.slug}/#{topic.id}"
insert_url_mapping(old_url, new_url, title) insert_url_mapping(row['post_id'], new_url, title)
rescue ActiveRecord::RecordInvalid => e rescue ActiveRecord::RecordInvalid => e
puts "Error importing topic with post_id #{row['post_id']}: #{e.message}" puts "Error importing topic with post_id #{row['post_id']}: #{e.message}"
@ -524,7 +532,7 @@ def import_topics_and_posts
post = Post.create!( post = Post.create!(
topic_id: topic_id, topic_id: topic_id,
user_id: discourse_user_id, user_id: discourse_user_id,
# raw: import_post_attachments(row['post_message'], row['post_id']), # raw: import_attachments(row['post_message'], row['post_id']),
# raw: row['post_message'] || "", # raw: row['post_message'] || "",
raw: sanitized_post_message, raw: sanitized_post_message,
created_at: Time.at(row['post_time']), created_at: Time.at(row['post_time']),
@ -602,17 +610,18 @@ def import_personal_messages
end end
# Import attachments for a post # Import attachments for a post
# def import_post_attachments(post_message, post_id) def import_post_attachments(post_message, post_id)
# # Fetch attachments related to the post # Fetch attachments related to the post
# attachments = execute_query("SELECT * FROM gforum_PostAttachment WHERE post_id_fk = #{post_id}") attachments = execute_query("SELECT * FROM gforum_PostAttachment WHERE post_id_fk = #{post_id}")
# attachments.each do |attachment| attachments.each do |attachment|
# # Append attachment links to the post message
# file_url = "https://forum.slowtwitch.com/images/posts/attachments/#{attachment['ID'] % 10}/#{attachment['ID']}-#{attachment['File_Name']}" # Append attachment links to the post message
# post_message += "\n\n![#{attachment['File_Name']}](#{file_url})" file_url = "https://forum.slowtwitch.com/images/posts/attachments/#{attachment['ID'] % 10}/#{attachment['ID']}-#{attachment['File_Name']}"
# end post_message += "\n\n![#{attachment['File_Name']}](#{file_url})"
# post_message end
# end 1# post_message
end
@ -620,7 +629,7 @@ end
# Main method to perform the import # Main method to perform the import
def perform_import def perform_import
# Secret trick to disable RateLimiting support in Discourse # Secret trick to disable RateLimiting protection in Discourse
RateLimiter.disable RateLimiter.disable
# Set our unique timestamp for this migration run # Set our unique timestamp for this migration run
@ -628,12 +637,13 @@ end
puts "Starting Gossamer Forums import... #{timestamp}" puts "Starting Gossamer Forums import... #{timestamp}"
import_users import_users
export_username_mapping_to_csv("gossamer-migration-username-mapping#{timestamp}") export_username_mapping_to_csv("gossamer-migration-username-mapping#{timestamp}")
import_categories import_categories
import_topics_and_posts import_topics_and_posts
export_url_mapping_to_csv("gossamer-migration-url-mapping#{timestamp}") export_url_mapping_to_csv("gossamer-migration-url-mapping#{timestamp}")
create_nginx_rewrite_rules("gossamer-redirects.conf")
import_personal_messages import_personal_messages