v0.12 Significant improvement with URL topic-post mapping support
This commit is contained in:
parent
1e62f5ee09
commit
9825f919b9
@ -1,5 +1,5 @@
|
||||
# gossamer threads migration-import code
|
||||
# v0.11
|
||||
# v0.12
|
||||
|
||||
require 'mysql2'
|
||||
require 'open-uri'
|
||||
@ -60,9 +60,10 @@ class GossamerForumsImporter < ImportScripts::Base
|
||||
SQL
|
||||
@db.execute <<-SQL
|
||||
CREATE TABLE IF NOT EXISTS url_map (
|
||||
old_url TEXT,
|
||||
old_post_id INTEGER PRIMARY KEY,
|
||||
new_url TEXT,
|
||||
title TEXT
|
||||
# created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
SQL
|
||||
end
|
||||
@ -89,7 +90,7 @@ class GossamerForumsImporter < ImportScripts::Base
|
||||
|
||||
# Define a method to export the username mapping table to a CSV file
|
||||
def export_username_mapping_to_csv(filename)
|
||||
CSV.open(filename, 'wb') do |csv|
|
||||
CSV.open(filename, 'w') do |csv|
|
||||
# Add headers
|
||||
csv << ['Old Username', 'New Username', 'Email', 'Full Name']
|
||||
|
||||
@ -102,22 +103,31 @@ class GossamerForumsImporter < ImportScripts::Base
|
||||
end
|
||||
|
||||
# Insert a URL mapping into the SQLite database
|
||||
def insert_url_mapping(old_url, new_url, title)
|
||||
@db.execute "INSERT INTO url_map (old_url, new_url, title) VALUES (?, ?, ?, ?, ?)", [old_url, new_url, title]
|
||||
def insert_url_mapping(old_post_id, new_url, title)
|
||||
@db.execute "INSERT INTO url_map (old_post_id, new_url, title) VALUES (?, ?, ?)", [old_post_id, new_url, title]
|
||||
end
|
||||
|
||||
# Export the URL mappings to a CSV file
|
||||
def export_url_mapping_to_csv(filename)
|
||||
CSV.open(filename, "wb") do |csv|
|
||||
CSV.open(filename, "w") do |csv|
|
||||
# Add headers
|
||||
csv << ["Old URL", "New URL", "Title"]
|
||||
@db.execute("SELECT * FROM url_map") do |row|
|
||||
csv << ["Old Post ID", "New URL", "Title"]
|
||||
@db.execute("SELECT old_post_id, new_url, title FROM url_map") do |row|
|
||||
csv << row
|
||||
end
|
||||
end
|
||||
puts "Exported URL mappings to #{filename}"
|
||||
end
|
||||
|
||||
# Method to create Nginx rewrite rules file
|
||||
def create_nginx_rewrite_rules(filename)
|
||||
File.open(filename, "w") do |file|
|
||||
@db.execute("SELECT old_post_id, new_url FROM url_map") do |row|
|
||||
old_post_id, new_url = row
|
||||
file.puts "rewrite ^/forum/.*P#{old_post_id}/$ #{new_url} permanent;"
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# Execute an SQL query on the Gossamer Forums database
|
||||
def execute_query(query)
|
||||
@ -158,7 +168,6 @@ class GossamerForumsImporter < ImportScripts::Base
|
||||
sanitized_username
|
||||
end
|
||||
|
||||
|
||||
# Sanitize email to replace restricted domains
|
||||
def sanitize_email(email)
|
||||
restricted_domains = ['mailinator.com', 'example.com'] # Add more restricted domains as needed
|
||||
@ -173,7 +182,6 @@ class GossamerForumsImporter < ImportScripts::Base
|
||||
email
|
||||
end
|
||||
|
||||
|
||||
# Helper method to download an image from a URL
|
||||
def download_image(url)
|
||||
begin
|
||||
@ -187,31 +195,31 @@ class GossamerForumsImporter < ImportScripts::Base
|
||||
end
|
||||
end
|
||||
|
||||
def upload_image(user, file, filename, gossamer_url)
|
||||
begin
|
||||
upload = Upload.create!(
|
||||
user_id: user.id,
|
||||
original_filename: filename,
|
||||
filesize: file.size,
|
||||
# filesize: File.size(file.path),
|
||||
# content_type: `file --brief --mime-type #{file.path}`.strip,
|
||||
# sha1: Digest::SHA1.file(file.path).hexdigest,
|
||||
# origin: 'user_avatar',
|
||||
# retain_hours: nil,
|
||||
url: gossamer_url
|
||||
)
|
||||
# Error -- non-existent method upload.ensure_consistency!
|
||||
def upload_image(user, file, filename, gossamer_url)
|
||||
begin
|
||||
upload = Upload.create!(
|
||||
user_id: user.id,
|
||||
original_filename: filename,
|
||||
filesize: file.size,
|
||||
# filesize: File.size(file.path),
|
||||
# content_type: `file --brief --mime-type #{file.path}`.strip,
|
||||
# sha1: Digest::SHA1.file(file.path).hexdigest,
|
||||
# origin: 'user_avatar',
|
||||
# retain_hours: nil,
|
||||
url: gossamer_url
|
||||
)
|
||||
# Error -- non-existent method upload.ensure_consistency!
|
||||
|
||||
# Move the file to the correct location
|
||||
# FileUtils.mv(file.path, upload.path)
|
||||
upload.save!
|
||||
# Move the file to the correct location
|
||||
# FileUtils.mv(file.path, upload.path)
|
||||
upload.save!
|
||||
|
||||
upload
|
||||
rescue => e
|
||||
puts "Failed to upload image #{filename} for user #{user.username}: #{e.message}"
|
||||
nil
|
||||
upload
|
||||
rescue => e
|
||||
puts "Failed to upload image #{filename} for user #{user.username}: #{e.message}"
|
||||
nil
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
# def download_file(url)
|
||||
@ -437,13 +445,13 @@ end
|
||||
puts "Importing categories... Done."
|
||||
end
|
||||
|
||||
# Helper function to ensure title meets the minimum length requirement
|
||||
def ensure_valid_title(title, min_length = 5)
|
||||
if title.length < min_length
|
||||
title += "." * (min_length - title.length) # Append dots to make it longer
|
||||
# Helper function to ensure title meets the minimum length requirement
|
||||
def ensure_valid_title(title, min_length = 5)
|
||||
if title.length < min_length
|
||||
title += "." * (min_length - title.length) # Append dots to make it longer
|
||||
end
|
||||
title
|
||||
end
|
||||
title
|
||||
end
|
||||
|
||||
# Import topics and posts from Gossamer Forums to Discourse
|
||||
def import_topics_and_posts
|
||||
@ -486,7 +494,7 @@ def import_topics_and_posts
|
||||
post = Post.create!(
|
||||
topic_id: topic.id,
|
||||
user_id: discourse_user_id,
|
||||
# raw: import_post_attachments(row['post_message'], row['post_id']),
|
||||
# raw: import_attachments(row['post_message'], row['post_id']),
|
||||
# raw: row['post_message'] || "",
|
||||
raw: sanitized_post_message,
|
||||
created_at: Time.at(row['post_time']),
|
||||
@ -496,9 +504,9 @@ def import_topics_and_posts
|
||||
post.save!
|
||||
|
||||
# Create URL mappings
|
||||
old_url = "https://old/forum/#{row['forum_name']}/topics/#{row['post_id']}"
|
||||
# old_url = "https://old/forum/#{row['forum_name']}/topics/#{row['post_id']}"
|
||||
new_url = "https://new/t/#{topic.slug}/#{topic.id}"
|
||||
insert_url_mapping(old_url, new_url, title)
|
||||
insert_url_mapping(row['post_id'], new_url, title)
|
||||
|
||||
rescue ActiveRecord::RecordInvalid => e
|
||||
puts "Error importing topic with post_id #{row['post_id']}: #{e.message}"
|
||||
@ -524,7 +532,7 @@ def import_topics_and_posts
|
||||
post = Post.create!(
|
||||
topic_id: topic_id,
|
||||
user_id: discourse_user_id,
|
||||
# raw: import_post_attachments(row['post_message'], row['post_id']),
|
||||
# raw: import_attachments(row['post_message'], row['post_id']),
|
||||
# raw: row['post_message'] || "",
|
||||
raw: sanitized_post_message,
|
||||
created_at: Time.at(row['post_time']),
|
||||
@ -602,17 +610,18 @@ def import_personal_messages
|
||||
end
|
||||
|
||||
|
||||
# Import attachments for a post
|
||||
# def import_post_attachments(post_message, post_id)
|
||||
# # Fetch attachments related to the post
|
||||
# attachments = execute_query("SELECT * FROM gforum_PostAttachment WHERE post_id_fk = #{post_id}")
|
||||
# attachments.each do |attachment|
|
||||
# # Append attachment links to the post message
|
||||
# file_url = "https://forum.slowtwitch.com/images/posts/attachments/#{attachment['ID'] % 10}/#{attachment['ID']}-#{attachment['File_Name']}"
|
||||
# post_message += "\n\n![#{attachment['File_Name']}](#{file_url})"
|
||||
# end
|
||||
# post_message
|
||||
# end
|
||||
# Import attachments for a post
|
||||
def import_post_attachments(post_message, post_id)
|
||||
# Fetch attachments related to the post
|
||||
attachments = execute_query("SELECT * FROM gforum_PostAttachment WHERE post_id_fk = #{post_id}")
|
||||
attachments.each do |attachment|
|
||||
|
||||
# Append attachment links to the post message
|
||||
file_url = "https://forum.slowtwitch.com/images/posts/attachments/#{attachment['ID'] % 10}/#{attachment['ID']}-#{attachment['File_Name']}"
|
||||
post_message += "\n\n![#{attachment['File_Name']}](#{file_url})"
|
||||
end
|
||||
1# post_message
|
||||
end
|
||||
|
||||
|
||||
|
||||
@ -620,7 +629,7 @@ end
|
||||
|
||||
# Main method to perform the import
|
||||
def perform_import
|
||||
# Secret trick to disable RateLimiting support in Discourse
|
||||
# Secret trick to disable RateLimiting protection in Discourse
|
||||
RateLimiter.disable
|
||||
|
||||
# Set our unique timestamp for this migration run
|
||||
@ -628,12 +637,13 @@ end
|
||||
|
||||
puts "Starting Gossamer Forums import... #{timestamp}"
|
||||
|
||||
import_users
|
||||
import_users
|
||||
export_username_mapping_to_csv("gossamer-migration-username-mapping#{timestamp}")
|
||||
|
||||
import_categories
|
||||
import_topics_and_posts
|
||||
export_url_mapping_to_csv("gossamer-migration-url-mapping#{timestamp}")
|
||||
create_nginx_rewrite_rules("gossamer-redirects.conf")
|
||||
|
||||
import_personal_messages
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user