From 9825f919b99334f1be8c0d6b2b50957bfd15d0b4 Mon Sep 17 00:00:00 2001 From: saint Date: Thu, 20 Jun 2024 19:23:30 +1000 Subject: [PATCH] v0.12 Significant improvement with URL topic-post mapping support --- gossamer_forums.rb | 120 ++++++++++++++++++++++++--------------------- 1 file changed, 65 insertions(+), 55 deletions(-) diff --git a/gossamer_forums.rb b/gossamer_forums.rb index 2ef41c6..0339762 100644 --- a/gossamer_forums.rb +++ b/gossamer_forums.rb @@ -1,5 +1,5 @@ # gossamer threads migration-import code -# v0.11 +# v0.12 require 'mysql2' require 'open-uri' @@ -60,9 +60,10 @@ class GossamerForumsImporter < ImportScripts::Base SQL @db.execute <<-SQL CREATE TABLE IF NOT EXISTS url_map ( - old_url TEXT, + old_post_id INTEGER PRIMARY KEY, new_url TEXT, title TEXT +# created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ); SQL end @@ -89,7 +90,7 @@ class GossamerForumsImporter < ImportScripts::Base # Define a method to export the username mapping table to a CSV file def export_username_mapping_to_csv(filename) - CSV.open(filename, 'wb') do |csv| + CSV.open(filename, 'w') do |csv| # Add headers csv << ['Old Username', 'New Username', 'Email', 'Full Name'] @@ -102,22 +103,31 @@ class GossamerForumsImporter < ImportScripts::Base end # Insert a URL mapping into the SQLite database - def insert_url_mapping(old_url, new_url, title) - @db.execute "INSERT INTO url_map (old_url, new_url, title) VALUES (?, ?, ?, ?, ?)", [old_url, new_url, title] + def insert_url_mapping(old_post_id, new_url, title) + @db.execute "INSERT INTO url_map (old_post_id, new_url, title) VALUES (?, ?, ?)", [old_post_id, new_url, title] end # Export the URL mappings to a CSV file def export_url_mapping_to_csv(filename) - CSV.open(filename, "wb") do |csv| + CSV.open(filename, "w") do |csv| # Add headers - csv << ["Old URL", "New URL", "Title"] - @db.execute("SELECT * FROM url_map") do |row| + csv << ["Old Post ID", "New URL", "Title"] + @db.execute("SELECT old_post_id, new_url, title FROM url_map") do |row| csv << row end end puts "Exported URL mappings to #{filename}" end + # Method to create Nginx rewrite rules file + def create_nginx_rewrite_rules(filename) + File.open(filename, "w") do |file| + @db.execute("SELECT old_post_id, new_url FROM url_map") do |row| + old_post_id, new_url = row + file.puts "rewrite ^/forum/.*P#{old_post_id}/$ #{new_url} permanent;" + end + end + end # Execute an SQL query on the Gossamer Forums database def execute_query(query) @@ -158,7 +168,6 @@ class GossamerForumsImporter < ImportScripts::Base sanitized_username end - # Sanitize email to replace restricted domains def sanitize_email(email) restricted_domains = ['mailinator.com', 'example.com'] # Add more restricted domains as needed @@ -173,7 +182,6 @@ class GossamerForumsImporter < ImportScripts::Base email end - # Helper method to download an image from a URL def download_image(url) begin @@ -187,31 +195,31 @@ class GossamerForumsImporter < ImportScripts::Base end end -def upload_image(user, file, filename, gossamer_url) - begin - upload = Upload.create!( - user_id: user.id, - original_filename: filename, - filesize: file.size, -# filesize: File.size(file.path), -# content_type: `file --brief --mime-type #{file.path}`.strip, -# sha1: Digest::SHA1.file(file.path).hexdigest, -# origin: 'user_avatar', -# retain_hours: nil, - url: gossamer_url - ) -# Error -- non-existent method upload.ensure_consistency! + def upload_image(user, file, filename, gossamer_url) + begin + upload = Upload.create!( + user_id: user.id, + original_filename: filename, + filesize: file.size, +# filesize: File.size(file.path), +# content_type: `file --brief --mime-type #{file.path}`.strip, +# sha1: Digest::SHA1.file(file.path).hexdigest, +# origin: 'user_avatar', +# retain_hours: nil, + url: gossamer_url + ) +# Error -- non-existent method upload.ensure_consistency! - # Move the file to the correct location -# FileUtils.mv(file.path, upload.path) - upload.save! + # Move the file to the correct location +# FileUtils.mv(file.path, upload.path) + upload.save! - upload - rescue => e - puts "Failed to upload image #{filename} for user #{user.username}: #{e.message}" - nil + upload + rescue => e + puts "Failed to upload image #{filename} for user #{user.username}: #{e.message}" + nil + end end -end # def download_file(url) @@ -437,13 +445,13 @@ end puts "Importing categories... Done." end -# Helper function to ensure title meets the minimum length requirement -def ensure_valid_title(title, min_length = 5) - if title.length < min_length - title += "." * (min_length - title.length) # Append dots to make it longer + # Helper function to ensure title meets the minimum length requirement + def ensure_valid_title(title, min_length = 5) + if title.length < min_length + title += "." * (min_length - title.length) # Append dots to make it longer + end + title end - title -end # Import topics and posts from Gossamer Forums to Discourse def import_topics_and_posts @@ -486,7 +494,7 @@ def import_topics_and_posts post = Post.create!( topic_id: topic.id, user_id: discourse_user_id, -# raw: import_post_attachments(row['post_message'], row['post_id']), +# raw: import_attachments(row['post_message'], row['post_id']), # raw: row['post_message'] || "", raw: sanitized_post_message, created_at: Time.at(row['post_time']), @@ -496,9 +504,9 @@ def import_topics_and_posts post.save! # Create URL mappings - old_url = "https://old/forum/#{row['forum_name']}/topics/#{row['post_id']}" +# old_url = "https://old/forum/#{row['forum_name']}/topics/#{row['post_id']}" new_url = "https://new/t/#{topic.slug}/#{topic.id}" - insert_url_mapping(old_url, new_url, title) + insert_url_mapping(row['post_id'], new_url, title) rescue ActiveRecord::RecordInvalid => e puts "Error importing topic with post_id #{row['post_id']}: #{e.message}" @@ -524,7 +532,7 @@ def import_topics_and_posts post = Post.create!( topic_id: topic_id, user_id: discourse_user_id, -# raw: import_post_attachments(row['post_message'], row['post_id']), +# raw: import_attachments(row['post_message'], row['post_id']), # raw: row['post_message'] || "", raw: sanitized_post_message, created_at: Time.at(row['post_time']), @@ -602,17 +610,18 @@ def import_personal_messages end -# Import attachments for a post -# def import_post_attachments(post_message, post_id) -# # Fetch attachments related to the post -# attachments = execute_query("SELECT * FROM gforum_PostAttachment WHERE post_id_fk = #{post_id}") -# attachments.each do |attachment| -# # Append attachment links to the post message -# file_url = "https://forum.slowtwitch.com/images/posts/attachments/#{attachment['ID'] % 10}/#{attachment['ID']}-#{attachment['File_Name']}" -# post_message += "\n\n![#{attachment['File_Name']}](#{file_url})" -# end -# post_message -# end + # Import attachments for a post + def import_post_attachments(post_message, post_id) + # Fetch attachments related to the post + attachments = execute_query("SELECT * FROM gforum_PostAttachment WHERE post_id_fk = #{post_id}") + attachments.each do |attachment| + + # Append attachment links to the post message + file_url = "https://forum.slowtwitch.com/images/posts/attachments/#{attachment['ID'] % 10}/#{attachment['ID']}-#{attachment['File_Name']}" + post_message += "\n\n![#{attachment['File_Name']}](#{file_url})" + end +1# post_message + end @@ -620,7 +629,7 @@ end # Main method to perform the import def perform_import - # Secret trick to disable RateLimiting support in Discourse + # Secret trick to disable RateLimiting protection in Discourse RateLimiter.disable # Set our unique timestamp for this migration run @@ -628,12 +637,13 @@ end puts "Starting Gossamer Forums import... #{timestamp}" - import_users + import_users export_username_mapping_to_csv("gossamer-migration-username-mapping#{timestamp}") import_categories import_topics_and_posts export_url_mapping_to_csv("gossamer-migration-url-mapping#{timestamp}") + create_nginx_rewrite_rules("gossamer-redirects.conf") import_personal_messages