Compare commits
4 Commits
main
...
ross-impro
| Author | SHA1 | Date | |
|---|---|---|---|
| 008b9d74dd | |||
| 1811f7f3e0 | |||
| 6c85157c28 | |||
| 349df7239f |
Binary file not shown.
Binary file not shown.
@ -1,12 +1,12 @@
|
||||
# Load the Discourse environment
|
||||
require File.expand_path("../../../../config/environment", __FILE__)
|
||||
require File.expand_path("../../../config/environment", __FILE__)
|
||||
|
||||
# require_relative '/var/www/discourse/config/environment'
|
||||
|
||||
def query_md5_password_custom_field(username)
|
||||
user = User.find_by(username: username)
|
||||
if user
|
||||
md5_password = user.custom_fields['md5_password']
|
||||
md5_password = user.custom_fields['custom_password_md5']
|
||||
if md5_password
|
||||
puts "MD5 password custom field for user #{username}: #{md5_password}"
|
||||
else
|
||||
@ -26,15 +26,3 @@ end
|
||||
username = 'davidpaulyoung'
|
||||
|
||||
query_md5_password_custom_field(username)
|
||||
|
||||
username = 'Slowman'
|
||||
|
||||
query_md5_password_custom_field(username)
|
||||
|
||||
username = 'GT'
|
||||
|
||||
query_md5_password_custom_field(username)
|
||||
|
||||
username = 'Chimpking'
|
||||
|
||||
query_md5_password_custom_field(username)
|
||||
|
||||
@ -1,10 +1,5 @@
|
||||
# Federated Computer, Inc.
|
||||
# David Sainty <saint@federated.computer> 2024 A.D.
|
||||
# Gossamer Threads to Discourse -- CleanUp Script
|
||||
# v0.17 Add more cleanup options.
|
||||
|
||||
require 'concurrent-ruby'
|
||||
require File.expand_path("../../../../config/environment", __FILE__)
|
||||
# v0.13
|
||||
require File.expand_path("../../../config/environment", __FILE__)
|
||||
|
||||
class GossamerForumsCleaner
|
||||
def cleanup_users
|
||||
@ -38,61 +33,15 @@ class GossamerForumsCleaner
|
||||
topic = Topic.find_by(id: field.topic_id)
|
||||
if topic
|
||||
puts "Deleting topic #{topic.title} (ID: #{topic.id})"
|
||||
# topic.posts.each do |post|
|
||||
# puts "Deleting post #{post.id} in topic #{topic.id}"
|
||||
# post.destroy
|
||||
# end
|
||||
topic.destroy
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def cleanup_topics_former_user
|
||||
puts "Cleaning up imported topics..."
|
||||
|
||||
# Find all topics that were imported
|
||||
TopicCustomField.where(name: 'original_gossamer_id').each do |field|
|
||||
topic = Topic.find_by(id: field.topic_id)
|
||||
|
||||
next unless topic
|
||||
|
||||
# Fetch the first post in the topic
|
||||
first_post = topic.posts.order(:created_at).first
|
||||
|
||||
# Check if the first post has user_id 2
|
||||
if first_post && first_post.user_id == 2
|
||||
puts "Deleting topic #{topic.title} (ID: #{topic.id})"
|
||||
|
||||
# Destroy all posts in the topic
|
||||
topic.posts.each do |post|
|
||||
puts "Deleting post #{post.id} in topic #{topic.id}"
|
||||
post.destroy
|
||||
end
|
||||
|
||||
# Destroy the topic itself
|
||||
topic.destroy
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def cleanup_topics_with_invalid_posts_count
|
||||
puts "Cleaning up topics with invalid posts_count..."
|
||||
|
||||
# Iterate through all topics
|
||||
Topic.where("posts_count IS NULL OR posts_count = -1 OR posts_count = 0").find_each do |topic|
|
||||
puts "Identified topic for deletion: Title: #{topic.title} Topic ID: #{topic.id}"
|
||||
|
||||
# Destroy all posts in the topic
|
||||
topic.posts.each do |post|
|
||||
puts "Deleting post #{post.id} in topic #{topic.id}"
|
||||
post.destroy
|
||||
end
|
||||
|
||||
# Destroy the topic itself
|
||||
topic.destroy
|
||||
end
|
||||
end
|
||||
|
||||
def cleanup_posts
|
||||
puts "Cleaning up imported posts..."
|
||||
# Find all posts that were imported and delete them
|
||||
@ -105,30 +54,6 @@ class GossamerForumsCleaner
|
||||
end
|
||||
end
|
||||
|
||||
def cleanup_posts_parallel
|
||||
puts "Cleaning up imported posts..."
|
||||
|
||||
# Define the number of threads to use
|
||||
num_threads = 8
|
||||
pool = Concurrent::FixedThreadPool.new(num_threads)
|
||||
|
||||
PostCustomField.where(name: 'original_gossamer_id').in_batches(of: 1000) do |batch|
|
||||
batch.each do |field|
|
||||
pool.post do
|
||||
post = Post.find_by(id: field.post_id)
|
||||
if post
|
||||
puts "Deleting post #{post.id} (ID: #{post.id})"
|
||||
post.destroy
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# Wait for all threads to complete
|
||||
pool.shutdown
|
||||
pool.wait_for_termination
|
||||
end
|
||||
|
||||
def cleanup_messages
|
||||
puts "Cleaning up imported personal messages..."
|
||||
# Find all personal messages (inbox) that were imported and delete them
|
||||
@ -148,11 +73,9 @@ class GossamerForumsCleaner
|
||||
def perform_cleanup
|
||||
puts "Cleanup beginning!"
|
||||
# cleanup_messages
|
||||
# cleanup_topics
|
||||
# cleanup_topics_former_user
|
||||
cleanup_topics_with_invalid_posts_count
|
||||
# cleanup_posts_parallel
|
||||
# cleanup_categories
|
||||
cleanup_posts
|
||||
cleanup_topics
|
||||
cleanup_categories
|
||||
# cleanup_users
|
||||
puts "Cleanup complete!"
|
||||
end
|
||||
|
||||
@ -1,163 +0,0 @@
|
||||
# Federated Computer, Inc.
|
||||
# David Sainty <saint@federated.computer> 2024 A.D.
|
||||
# Gossamer Threads to Discourse -- Correct Encoding
|
||||
# v0.7 Further attempt to get this reverse dobule encoding right now
|
||||
|
||||
require 'mysql2'
|
||||
require 'active_record'
|
||||
require 'charlock_holmes'
|
||||
|
||||
# require 'concurrent-ruby'
|
||||
require File.expand_path("../../../../config/environment", __FILE__)
|
||||
require File.expand_path("../../../../script/import_scripts/base", __FILE__)
|
||||
|
||||
class GossamerForumsCorrectEncoding < ImportScripts::Base
|
||||
def initialize
|
||||
super
|
||||
begin
|
||||
# Initialize MySQL client to connect to Gossamer Forums database
|
||||
@mysql_client = Mysql2::Client.new(
|
||||
host: "slowtwitch.northend.network",
|
||||
username: "admin",
|
||||
password: "yxnh93Ybbz2Nm8#mp28zCVv",
|
||||
database: "slowtwitch"
|
||||
)
|
||||
rescue Mysql2::Error => e
|
||||
puts "Error connecting to MySQL: #{e.message}"
|
||||
puts e.backtrace.join("\n") # Print the full stack trace
|
||||
exit 1
|
||||
end
|
||||
@batch_size = 1000 # Number of posts to process in each batch
|
||||
end
|
||||
|
||||
# Method to detect and fix text encoding
|
||||
def fix_text_encoding(content)
|
||||
begin
|
||||
# # Treat as Windows-1252 (cp1252) and then decode into UTF-8
|
||||
# corrected_content = content.encode('CP1252').force_encoding('UTF-8')
|
||||
# Step 1: Treat content as CP1252 and convert it back to UTF-8
|
||||
corrected_content = content.encode('CP1252').force_encoding('UTF-8').force_encoding('UTF-8')
|
||||
|
||||
# # Step 2: Check if there's still a problem (if still corrupted, apply the second pass)
|
||||
# if corrected_content.valid_encoding?
|
||||
# return corrected_content
|
||||
# else
|
||||
# # Step 3: If it's not valid UTF-8, re-encode and try to fix remaining issues
|
||||
# corrected_content.encode('UTF-8', invalid: :replace, undef: :replace)
|
||||
# end
|
||||
rescue Encoding::UndefinedConversionError => e
|
||||
puts "Error during encoding conversion: #{e.message}"
|
||||
puts e.backtrace.join("\n") # Print the full stack trace
|
||||
return content # Return the original content if conversion fails
|
||||
end
|
||||
|
||||
corrected_content
|
||||
end
|
||||
|
||||
# # Step 1: Try to detect encoding of the corrupted (double-encoded) content
|
||||
# detection = CharlockHolmes::EncodingDetector.detect(broken_content)
|
||||
# original_encoding = detection[:encoding]
|
||||
# puts "Original encoding detected: #{original_encoding}"
|
||||
#
|
||||
# # Step 2: First decode the double-encoded content
|
||||
# begin
|
||||
# # Convert the content assuming it was double-encoded, so decode twice
|
||||
# # First, convert from the detected encoding (ISO-8859-1 or windows-1252) to UTF-8
|
||||
# first_pass = CharlockHolmes::Converter.convert(broken_content, original_encoding, 'UTF-8')
|
||||
#
|
||||
# # Step 3: Now re-interpret that output as if it's broken UTF-8 and convert it back to UTF-8
|
||||
# fixed_content = CharlockHolmes::Converter.convert(first_pass, 'UTF-8', 'UTF-8')
|
||||
#
|
||||
# rescue => e
|
||||
# puts "Error during encoding fix: #{e.message}"
|
||||
# puts e.backtrace.join("\n") # Print the full stack trace
|
||||
#
|
||||
# fixed_content = broken_content # Fall back to the broken content if decoding fails
|
||||
# end
|
||||
#
|
||||
# return fixed_content
|
||||
#end
|
||||
|
||||
# # Detect encoding
|
||||
# detection = CharlockHolmes::EncodingDetector.detect(raw_content)
|
||||
# original_encoding = detection[:encoding]
|
||||
# puts "Original encoding detected: #{original_encoding}"
|
||||
|
||||
# # Force the encoding to the detected one, then covnert to UTF-8
|
||||
# if original_encoding == 'ISO-8859-1' || original_encoding == 'windows-1252'
|
||||
# # For Windows-1252 or ISO-8859-1, force the encoding and convert to UTF-8
|
||||
# # text.force_encoding('ISO-8859-1').encode('UTF-8')
|
||||
# text.force_encoding(original_encoding).encode('UTF-8', invalid: :replace, undef: :replace, replace: '?')
|
||||
# else
|
||||
# # Try to convert from detected encoding to UTF-8
|
||||
# text.encode('UTF-8', original_encoding, invalid: :replace, undef: :replace, replace: '?')
|
||||
# end
|
||||
|
||||
# if original_encoding
|
||||
# begin
|
||||
# decoded_content = CharlockHolmes::Converter.convert(raw_content, original_encoding, 'UTF-8')
|
||||
# rescue => e
|
||||
# puts "Error during encoding conversion: #{e.message}"
|
||||
# decoded_content = raw_content # Fall back to raw content if decoding fails
|
||||
# end
|
||||
# else
|
||||
# decoded_content = raw_content # Fallback if encoding detection fails
|
||||
# end
|
||||
#
|
||||
# # Step 3: Ensure the content is now correctly in UTF-8 (no need to encode again)
|
||||
# return decoded_content
|
||||
# end
|
||||
|
||||
# rescue StandardError => e
|
||||
# puts "Error during encoding conversion: #{e.message}"
|
||||
# puts e.backtrace.join("\n") # Print the full stack trace
|
||||
# text
|
||||
# end
|
||||
# end
|
||||
|
||||
# Method to fix encoding issues in post content
|
||||
def fix_encoding
|
||||
offset = 0
|
||||
|
||||
loop do
|
||||
puts "OFFSET: #{offset}"
|
||||
begin
|
||||
posts = Post.limit(@batch_size).offset(offset)
|
||||
break if posts.empty?
|
||||
|
||||
posts.each do |post|
|
||||
raw_content = post.raw
|
||||
puts "--> NEXT POST: post.id: #{post.id}"
|
||||
fixed_content = fix_text_encoding(raw_content)
|
||||
if fixed_content != raw_content
|
||||
puts "Updating post #{post.id}"
|
||||
puts "------- raw_content:\n#{raw_content}"
|
||||
puts "+++++++ fixed_content:\n#{fixed_content}"
|
||||
puts "---------------------------------------------------------------------------------------------"
|
||||
# post.update(raw: fixed_content)
|
||||
# post.raw = fixed_content
|
||||
# if post.save
|
||||
# puts "Post ##{post.id} updated successfully."
|
||||
# else
|
||||
# puts "Failed to update Post ##{post.id}: #{post.errors.full_messages.join(', ')}"
|
||||
# end
|
||||
end
|
||||
end
|
||||
rescue
|
||||
puts "Error: #{e.message}"
|
||||
puts e.backtrace.join("\n") # Print the full stack trace
|
||||
end
|
||||
|
||||
offset += @batch_size
|
||||
end
|
||||
end
|
||||
|
||||
def perform_encoding_correction
|
||||
puts "Encoding Correction beginning!"
|
||||
fix_encoding
|
||||
puts "Encoding Correction complete!"
|
||||
end
|
||||
end
|
||||
|
||||
GossamerForumsCorrectEncoding.new.perform_encoding_correction
|
||||
|
||||
@ -1,8 +1,8 @@
|
||||
# Load the Discourse environment
|
||||
require File.expand_path("../../../../config/environment", __FILE__)
|
||||
require File.expand_path("../../../config/environment", __FILE__)
|
||||
|
||||
# Define usernames to exclude from deletion
|
||||
excluded_usernames = ["saint", "discobot", "system","admin"]
|
||||
excluded_usernames = ["saint", "discobot", "system"]
|
||||
|
||||
# Find all users except the excluded ones
|
||||
users_to_delete = User.where.not(username: excluded_usernames)
|
||||
|
||||
@ -1,112 +0,0 @@
|
||||
# Federated Computer, Inc.
|
||||
# David Sainty <saint@federated.computer> 2024 A.D.
|
||||
# Gossamer Threads to Discourse -- CleanUp Script
|
||||
# v0.5 We need to handle deletion of topic posts -- delete all posts in topic and then delete topic itself
|
||||
|
||||
require 'mysql2'
|
||||
require 'active_record'
|
||||
|
||||
# require 'concurrent-ruby'
|
||||
require File.expand_path("../../../../config/environment", __FILE__)
|
||||
require File.expand_path("../../../../script/import_scripts/base", __FILE__)
|
||||
|
||||
class GossamerForumsDestroyDeletedPosts < ImportScripts::Base
|
||||
def initialize
|
||||
super
|
||||
begin
|
||||
# Initialize MySQL client to connect to Gossamer Forums database
|
||||
@mysql_client = Mysql2::Client.new(
|
||||
host: "slowtwitch.northend.network",
|
||||
username: "admin",
|
||||
password: "yxnh93Ybbz2Nm8#mp28zCVv",
|
||||
database: "slowtwitch"
|
||||
)
|
||||
rescue Mysql2::Error => e
|
||||
puts "Error connecting to MySQL: #{e.message}"
|
||||
exit 1
|
||||
end
|
||||
end
|
||||
|
||||
# Define a method to find a post by custom field
|
||||
def find_post_by_custom_field(post_id)
|
||||
puts "DestroyDeletedPosts: Searching for post with original_gossamer_id: #{post_id}"
|
||||
|
||||
post_custom_field = PostCustomField.find_by(name: 'original_gossamer_id', value: post_id.to_s)
|
||||
|
||||
if post_custom_field
|
||||
post = post_custom_field.post
|
||||
puts "DestroyDeletedPosts: Found post with id: #{post.id}"
|
||||
post
|
||||
else
|
||||
puts "DestroyDeletedPosts: No post found with original_gossamer_id: #{post_id}"
|
||||
nil
|
||||
end
|
||||
end
|
||||
|
||||
# Define a method to delete all posts in a topic
|
||||
def delete_all_posts_in_topic(topic_id)
|
||||
posts = Post.where(topic_id: topic_id)
|
||||
posts.each do |post|
|
||||
puts "DELETE ALL POSTS --- DiscourseDeletedPosts: Deleting post with id: #{post.id}"
|
||||
# post.destroy
|
||||
end
|
||||
end
|
||||
|
||||
# Define a method to delete a topic
|
||||
def delete_topic(topic_id)
|
||||
topic = Topic.find_by(id: topic_id)
|
||||
if topic
|
||||
puts "DELETE TOPIC --- DiscourseDeletedPosts: Deleting topic with id: #{topic_id}"
|
||||
# topic.destroy
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
# Define the method to delete posts based on the Gossamer Forums flag
|
||||
def destroy_deleted_posts_from_gossamer_with_user(username)
|
||||
|
||||
# Query the user ID from the legacy MySQL database
|
||||
user_result = @mysql_client.query("SELECT user_id FROM gforum_User WHERE user_username = '#{username}' LIMIT 1")
|
||||
user_id_row = user_result.first
|
||||
if user_id_row.nil?
|
||||
puts "DiscourseDeletedPosts: No user found with username: #{username}"
|
||||
return
|
||||
end
|
||||
user_id = user_id_row['user_id']
|
||||
|
||||
# Find all posts marked as deleted by the given user
|
||||
posts_result = @mysql_client.query("SELECT post_id FROM gforum_Post WHERE post_deleted = 1 AND user_id_fk = #{user_id}")
|
||||
|
||||
posts_result.each do |legacy_post|
|
||||
post_id = legacy_post['post_id']
|
||||
|
||||
# Look for the post in Discourse by custom field
|
||||
post = find_post_by_custom_field(post_id)
|
||||
|
||||
if post
|
||||
# Check if this post is the topic post
|
||||
if post.post_number == 1
|
||||
topic_id = post.topic_id
|
||||
# Delete all posts in the topic
|
||||
delete_all_posts_in_topic(topic_id)
|
||||
# Delete the topic itself
|
||||
delete_topic(topic_id)
|
||||
else
|
||||
# If not the topic post, just delete the individual post
|
||||
puts "DELETE POST --- DestroyDeletedPosts: Deleting post with id: #{post.id}"
|
||||
# post.destroy
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def perform_deleted_destroy
|
||||
puts "Destroy Deleted Posts beginning!"
|
||||
# destroy_deleted_posts_from_gossamer
|
||||
destroy_deleted_posts_from_gossamer_with_user('spudone')
|
||||
puts "Destroy Deleted Posts complete!"
|
||||
end
|
||||
end
|
||||
|
||||
GossamerForumsDestroyDeletedPosts.new.perform_deleted_destroy
|
||||
|
||||
@ -1,67 +0,0 @@
|
||||
# Federated Computer, Inc.
|
||||
# David Sainty <saint@federated.computer> 2024 A.D.
|
||||
# Gossamer Threads to Discourse -- Reset Discourse Email
|
||||
# v0.1 First addition for resetting email address for user
|
||||
|
||||
require File.expand_path("../../../../config/environment", __FILE__)
|
||||
|
||||
class GossamerResetEmail
|
||||
def reset_email(discourse_username, new_email)
|
||||
puts "Resetting email for Discourse username: '#{discourse_username}'"
|
||||
|
||||
user = User.find_by(username: discourse_username)
|
||||
if user
|
||||
puts "New email: #{new_email} ..."
|
||||
|
||||
# Delete Gossamer MD5 Password
|
||||
# user.custom_fields.delete('md5_password') # This removes the field completely
|
||||
# user.save!
|
||||
# puts " STEP 1 COMPLETED: Gossamer MD5 Password custom field removed for username: #{discourse_username}"
|
||||
|
||||
# Set the Discourse password with the `password=` method to properly hash the password
|
||||
# user.password = new_password
|
||||
# user.save!
|
||||
# puts " STEP 2 COMPLETED: Discourse Password updated for username: #{discourse_username}"
|
||||
|
||||
user.email = new_email
|
||||
user.save!
|
||||
puts " STEP 1 COMPLETED: Email address updated for username: #{discourse_username}"
|
||||
|
||||
# Update other attributes (to be sure to remove issues with login)
|
||||
user.active = true
|
||||
user.approved = true
|
||||
user.approved_at = Time.now
|
||||
user.approved_by_id = 1
|
||||
user.save!
|
||||
puts " STEP 2 COMPLETED: Other attributes (active, approved, approved_at, approved_by_id) updated for username: #{discourse_username}"
|
||||
|
||||
# Generate a new token, hash it, and create a confirmed email token for the user
|
||||
token = SecureRandom.hex(20)
|
||||
token_hash = EmailToken.hash_token(token)
|
||||
|
||||
EmailToken.create!(
|
||||
user_id: user.id,
|
||||
email: user.email,
|
||||
token_hash: token_hash,
|
||||
confirmed: true
|
||||
)
|
||||
puts " STEP 3 COMPLETED: New token generated, hashed and set as confirmed email token for username: #{discourse_username}"
|
||||
|
||||
else
|
||||
puts "User not found: #{discourse_username}"
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# Main execution
|
||||
if ARGV.length != 2
|
||||
puts "Usage: #{$0} discourse_username email_address@domain.com"
|
||||
exit 1
|
||||
end
|
||||
|
||||
discourse_username = ARGV[0]
|
||||
new_email = ARGV[1]
|
||||
# search_topic_by_title(search_string)
|
||||
|
||||
GossamerResetEmail.new.reset_email(discourse_username, new_email)
|
||||
|
||||
@ -1,63 +0,0 @@
|
||||
# Federated Computer, Inc.
|
||||
# David Sainty <saint@federated.computer> 2024 A.D.
|
||||
# Gossamer Threads to Discourse -- Reset Discourse Password and Clear Gossamer Password
|
||||
# v0.2 Full support
|
||||
|
||||
require File.expand_path("../../../../config/environment", __FILE__)
|
||||
|
||||
class GossamerResetPassword
|
||||
def reset_password(discourse_username, new_password)
|
||||
puts "Resetting password for Discourse username: '#{discourse_username}'"
|
||||
|
||||
user = User.find_by(username: discourse_username)
|
||||
if user
|
||||
puts "New password: #{new_password} ..."
|
||||
|
||||
# Delete Gossamer MD5 Password
|
||||
user.custom_fields.delete('md5_password') # This removes the field completely
|
||||
user.save!
|
||||
puts " STEP 1 COMPLETED: Gossamer MD5 Password custom field removed for username: #{discourse_username}"
|
||||
|
||||
# Set the Discourse password with the `password=` method to properly hash the password
|
||||
user.password = new_password
|
||||
user.save!
|
||||
puts " STEP 2 COMPLETED: Discourse Password updated for username: #{discourse_username}"
|
||||
|
||||
# Update other attributes (other than password which is already correct)
|
||||
user.active = true
|
||||
user.approved = true
|
||||
user.approved_at = Time.now
|
||||
user.approved_by_id = 1
|
||||
user.save!
|
||||
puts " STEP 3 COMPLETED: Other attributes (active, approved, approved_at, approved_by_id) updated for username: #{discourse_username}"
|
||||
|
||||
# Generate a new token, hash it, and create a confirmed email token for the user
|
||||
token = SecureRandom.hex(20)
|
||||
token_hash = EmailToken.hash_token(token)
|
||||
|
||||
EmailToken.create!(
|
||||
user_id: user.id,
|
||||
email: user.email,
|
||||
token_hash: token_hash,
|
||||
confirmed: true
|
||||
)
|
||||
puts " STEP 4 COMPLETED: New token generated, hashed and set as confirmed email token for username: #{discourse_username}"
|
||||
|
||||
else
|
||||
puts "User not found: #{discourse_username}"
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# Main execution
|
||||
if ARGV.length != 2
|
||||
puts "Usage: #{$0} discourse_username n3w_p4ssword."
|
||||
exit 1
|
||||
end
|
||||
|
||||
discourse_username = ARGV[0]
|
||||
new_password = ARGV[1]
|
||||
# search_topic_by_title(search_string)
|
||||
|
||||
GossamerResetPassword.new.reset_password(discourse_username, new_password)
|
||||
|
||||
@ -1,67 +0,0 @@
|
||||
# Federated Computer, Inc.
|
||||
# David Sainty <saint@federated.computer> 2024 A.D.
|
||||
# Gossamer Threads to Discourse -- Search By Topic Title
|
||||
# v0.1 Initial
|
||||
|
||||
require 'concurrent-ruby'
|
||||
require File.expand_path("../../../../config/environment", __FILE__)
|
||||
|
||||
class GossamerForumsSearch
|
||||
def search_topic_by_title(search_string)
|
||||
puts "Searching for topics with title containing: '#{search_string}'"
|
||||
|
||||
# Search for topics with titles containing the search string
|
||||
matching_topics = Topic.where('title LIKE ?', "%#{search_string}%")
|
||||
|
||||
# Print information about each matching topic
|
||||
matching_topics.each do |topic|
|
||||
puts "----------------------------------------"
|
||||
puts "ID: #{topic.id}"
|
||||
puts "Title: #{topic.title}"
|
||||
puts "Created At: #{topic.created_at}"
|
||||
puts "Updated At: #{topic.updated_at}"
|
||||
puts "Category ID: #{topic.category_id}"
|
||||
puts "Views: #{topic.views}"
|
||||
puts "Posts Count: #{topic.posts_count}"
|
||||
puts "Last Posted At: #{topic.last_posted_at}"
|
||||
puts "Bumped At: #{topic.bumped_at}"
|
||||
puts "Last Post User ID: #{topic.last_post_user_id}"
|
||||
puts "----------------------------------------"
|
||||
|
||||
# Fetch and display all posts for the topic
|
||||
topic.posts.each do |post|
|
||||
puts " Post ID: #{post.id}"
|
||||
puts " User ID: #{post.user_id}"
|
||||
puts " Created At: #{post.created_at}"
|
||||
puts " Updated At: #{post.updated_at}"
|
||||
puts " Post Number: #{post.post_number}"
|
||||
puts " Cooked: #{post.cooked.truncate(80)}" # Limit the display of post content to 100 characters
|
||||
puts " Raw: #{post.raw.truncate(80)}" # Limit the display of raw post content to 100 characters
|
||||
puts " ----------------------------------------"
|
||||
end
|
||||
end
|
||||
|
||||
if matching_topics.empty?
|
||||
puts "No topics found with title containing: '#{search_string}'"
|
||||
end
|
||||
end
|
||||
|
||||
def perform_search
|
||||
puts "Search beginning!"
|
||||
search_topic_by_title("No father is more proud")
|
||||
puts "Search complete!"
|
||||
end
|
||||
end
|
||||
|
||||
# Main execution
|
||||
if ARGV.length != 1
|
||||
puts "Usage: #{$0} SEARCH_STRING"
|
||||
exit 1
|
||||
end
|
||||
|
||||
search_string = ARGV[0]
|
||||
# search_topic_by_title(search_string)
|
||||
|
||||
# GossamerForumsSearch.new.perform_search
|
||||
GossamerForumsSearch.new.search_topic_by_title(search_string)
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
# Load the Discourse environment
|
||||
require File.expand_path("../../../../config/environment", __FILE__)
|
||||
require File.expand_path("../../../config/environment", __FILE__)
|
||||
|
||||
# require_relative '/var/www/discourse/config/environment'
|
||||
|
||||
|
||||
@ -1,12 +1,12 @@
|
||||
# Load the Discourse environment
|
||||
require File.expand_path("../../../../config/environment", __FILE__)
|
||||
require File.expand_path("../../../config/environment", __FILE__)
|
||||
|
||||
# require_relative '/var/www/discourse/config/environment'
|
||||
|
||||
def set_md5_password_custom_field(username, md5_password)
|
||||
user = User.find_by(username: username)
|
||||
if user
|
||||
user.custom_fields['md5_password'] = md5_password
|
||||
user.custom_fields['custom_password_md5'] = md5_password
|
||||
user.save!
|
||||
puts "MD5 password custom field set for user: #{username}"
|
||||
else
|
||||
|
||||
1140
gossamer_forums.rb
1140
gossamer_forums.rb
File diff suppressed because it is too large
Load Diff
658
gosss.rb
658
gosss.rb
@ -1,658 +0,0 @@
|
||||
# gossamer threads migration-import code
|
||||
# v0.12
|
||||
|
||||
require 'mysql2'
|
||||
require 'open-uri'
|
||||
require 'net/http'
|
||||
require 'tempfile'
|
||||
require 'sqlite3'
|
||||
|
||||
require 'digest'
|
||||
require 'fileutils'
|
||||
require 'csv'
|
||||
require 'time'
|
||||
|
||||
require File.expand_path("../../../config/environment", __FILE__)
|
||||
require_relative 'base'
|
||||
|
||||
class GossamerForumsImporter < ImportScripts::Base
|
||||
def initialize
|
||||
super
|
||||
begin
|
||||
# Initialize MySQL client to connect to Gossamer Forums database
|
||||
@mysql_client = Mysql2::Client.new(
|
||||
host: "slot.northend.network",
|
||||
username: "admin",
|
||||
password: "yxnh93Ybbz2Nm8#mp28zCVv",
|
||||
database: "slot"
|
||||
)
|
||||
rescue Mysql2::Error => e
|
||||
puts "Error connecting to MySQL: #{e.message}"
|
||||
exit 1
|
||||
end
|
||||
|
||||
# # Create a mapping of old Gossamer user IDs to new Discourse user IDs
|
||||
# @user_id_map = {}
|
||||
initialize_sqlite_id_name_url_db
|
||||
end
|
||||
|
||||
def initialize_sqlite_id_name_url_db
|
||||
@db = SQLite3::Database.new 'id_name_url_map.db'
|
||||
@db.execute <<-SQL
|
||||
CREATE TABLE IF NOT EXISTS user_id_map (
|
||||
old_user_id INTEGER PRIMARY KEY,
|
||||
new_user_id INTEGER
|
||||
);
|
||||
SQL
|
||||
@db.execute <<-SQL
|
||||
CREATE TABLE IF NOT EXISTS category_id_map (
|
||||
old_category_id INTEGER PRIMARY KEY,
|
||||
new_category_id INTEGER
|
||||
);
|
||||
SQL
|
||||
@db.execute <<-SQL
|
||||
CREATE TABLE IF NOT EXISTS username_map (
|
||||
id INTEGER PRIMARY KEY,
|
||||
old_username TEXT,
|
||||
new_username TEXT,
|
||||
email TEXT,
|
||||
real_name TEXT
|
||||
);
|
||||
SQL
|
||||
@db.execute <<-SQL
|
||||
CREATE TABLE IF NOT EXISTS url_map (
|
||||
old_post_id INTEGER PRIMARY KEY,
|
||||
new_url TEXT,
|
||||
title TEXT
|
||||
# created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
SQL
|
||||
end
|
||||
|
||||
def insert_user_id_mapping(old_user_id, new_user_id)
|
||||
@db.execute "INSERT OR REPLACE INTO user_id_map (old_user_id, new_user_id) VALUES (?, ?)", old_user_id, new_user_id
|
||||
end
|
||||
|
||||
def fetch_user_id_mapping(old_user_id)
|
||||
@db.get_first_value "SELECT new_user_id FROM user_id_map WHERE old_user_id = ?", old_user_id
|
||||
end
|
||||
|
||||
def insert_category_id_mapping(old_category_id, new_category_id)
|
||||
@db.execute "INSERT OR REPLACE INTO category_id_map (old_category_id, new_category_id) VALUES (?, ?)", old_category_id, new_category_id
|
||||
end
|
||||
|
||||
def fetch_category_id_mapping(old_category_id)
|
||||
@db.get_first_value "SELECT new_category_id FROM category_id_map WHERE old_category_id = ?", old_category_id
|
||||
end
|
||||
|
||||
def insert_username_mapping(old_username, new_username, email, real_name)
|
||||
@db.execute "INSERT INTO username_map (old_username, new_username, email, real_name) VALUES (?, ?, ?, ?)", old_username, new_username, email, real_name
|
||||
end
|
||||
|
||||
# Define a method to export the username mapping table to a CSV file
|
||||
def export_username_mapping_to_csv(filename)
|
||||
CSV.open(filename, 'w') do |csv|
|
||||
# Add headers
|
||||
csv << ['Old Username', 'New Username', 'Email', 'Full Name']
|
||||
|
||||
# Fetch data from the database
|
||||
@db.execute("SELECT old_username, new_username, email, real_name FROM username_map") do |row|
|
||||
csv << row
|
||||
end
|
||||
end
|
||||
puts "Exported changed username mappings to #{filename}"
|
||||
end
|
||||
|
||||
# Insert a URL mapping into the SQLite database
|
||||
def insert_url_mapping(old_post_id, new_url, title)
|
||||
@db.execute "INSERT INTO url_map (old_post_id, new_url, title) VALUES (?, ?, ?)", [old_post_id, new_url, title]
|
||||
end
|
||||
|
||||
# Export the URL mappings to a CSV file
|
||||
def export_url_mapping_to_csv(filename)
|
||||
CSV.open(filename, "w") do |csv|
|
||||
# Add headers
|
||||
csv << ["Old Post ID", "New URL", "Title"]
|
||||
@db.execute("SELECT old_post_id, new_url, title FROM url_map") do |row|
|
||||
csv << row
|
||||
end
|
||||
end
|
||||
puts "Exported URL mappings to #{filename}"
|
||||
end
|
||||
|
||||
# Method to create Nginx rewrite rules file
|
||||
def create_nginx_rewrite_rules(filename)
|
||||
File.open(filename, "w") do |file|
|
||||
@db.execute("SELECT old_post_id, new_url FROM url_map") do |row|
|
||||
old_post_id, new_url = row
|
||||
file.puts "rewrite ^/forum/.*P#{old_post_id}/$ #{new_url} permanent;"
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# Execute an SQL query on the Gossamer Forums database
|
||||
def execute_query(query)
|
||||
@mysql_client.query(query, as: :hash)
|
||||
end
|
||||
|
||||
# Sanitize the username to meet Discourse's requirements
|
||||
def sanitize_username(original_username, email, real_name)
|
||||
# original_username = username
|
||||
sanitized_username = username.gsub(/[^a-zA-Z0-9._-]/, '_')
|
||||
sanitized_username = "#{sanitized_username}." if sanitized_username.length < 2 # Allow two-character usernames
|
||||
sanitized_username = sanitized_username[0, 20] if sanitized_username.length > 20
|
||||
firststep_sanitized = sanitized_username
|
||||
|
||||
existing_user = User.find_by(username: sanitized_username)
|
||||
|
||||
if existing_user
|
||||
if existing_user.email.downcase == email.downcase && existing_user.name == name
|
||||
return sanitized_username
|
||||
else
|
||||
counter = 1
|
||||
while User.exists?(username: sanitized_username)
|
||||
sanitized_username = "#{firststep_sanitized}_#{counter}"
|
||||
sanitized_username = sanitized_username[0, 20] if sanitized_username.length > 20
|
||||
counter += 1
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
if original_username != sanitized_username
|
||||
# The Discourse username is not the same as the Gossamer Forums username
|
||||
puts "Sanitized username: '#{original_username}' --> '#{sanitized_username}'"
|
||||
insert_username_mapping(original_username, sanitized_username, email, real_name)
|
||||
# else
|
||||
# puts "UNsanitized username: '#{original_username}' --> '#{sanitized_username}'"
|
||||
end
|
||||
|
||||
sanitized_username
|
||||
end
|
||||
|
||||
# Sanitize email to replace restricted domains
|
||||
def sanitize_email(email)
|
||||
restricted_domains = ['mailinator.com', 'example.com'] # Add more restricted domains as needed
|
||||
domain = email.split('@').last
|
||||
|
||||
if restricted_domains.include?(domain)
|
||||
sanitized_email = email.gsub(domain, 'example.org') # Change to a permissible domain
|
||||
puts "Sanitized email: '#{email}' --> '#{sanitized_email}'"
|
||||
return sanitized_email
|
||||
end
|
||||
|
||||
email
|
||||
end
|
||||
|
||||
# Helper method to download an image from a URL
|
||||
def download_image(url)
|
||||
begin
|
||||
URI.open(url).read
|
||||
rescue OpenURI::HTTPError => e
|
||||
puts "Failed to download image from #{url}: #{e.message}"
|
||||
nil
|
||||
rescue URI::InvalidURIError => e
|
||||
puts "Failed to handle invalid URL/URI for #{url}: #{e.message}"
|
||||
nil
|
||||
end
|
||||
end
|
||||
|
||||
def upload_image(user, file, filename, gossamer_url)
|
||||
begin
|
||||
upload = Upload.create!(
|
||||
user_id: user.id,
|
||||
original_filename: filename,
|
||||
filesize: file.size,
|
||||
# filesize: File.size(file.path),
|
||||
# content_type: `file --brief --mime-type #{file.path}`.strip,
|
||||
# sha1: Digest::SHA1.file(file.path).hexdigest,
|
||||
# origin: 'user_avatar',
|
||||
# retain_hours: nil,
|
||||
url: gossamer_url
|
||||
)
|
||||
# Error -- non-existent method upload.ensure_consistency!
|
||||
|
||||
# Move the file to the correct location
|
||||
# FileUtils.mv(file.path, upload.path)
|
||||
upload.save!
|
||||
|
||||
upload
|
||||
rescue => e
|
||||
puts "Failed to upload image #{filename} for user #{user.username}: #{e.message}"
|
||||
nil
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
# def download_file(url)
|
||||
# require 'open-uri'
|
||||
# begin
|
||||
# file = Tempfile.new
|
||||
# file.binmode
|
||||
# file.write(URI.open(url).read)
|
||||
# file.rewind
|
||||
# file
|
||||
# rescue => e
|
||||
# puts "Failed to download file from #{url}: #{e.message}"
|
||||
# nil
|
||||
# end
|
||||
# end
|
||||
|
||||
# Helper method to upload an image to Discourse
|
||||
# def upload_image(user, image_data, filename)
|
||||
# return if image_data.nil?
|
||||
#
|
||||
# upload = Upload.create_for(user.id, File.open(image_data.path), filename, 'image/jpeg')
|
||||
# if upload.nil? || !upload.persisted?
|
||||
# puts "Failed to upload image for user #{user.username}"
|
||||
# return
|
||||
# end
|
||||
#
|
||||
# upload
|
||||
# end
|
||||
|
||||
|
||||
# Import users from Gossamer Forums to Discourse
|
||||
def import_users
|
||||
puts "Importing users..."
|
||||
users = []
|
||||
|
||||
# Fetch all users from Gossamer Forums
|
||||
execute_query("SELECT * FROM gforum_User").each do |row|
|
||||
users << {
|
||||
id: row['user_id'],
|
||||
username: sanitize_username(row['user_username'], row['user_email'], row['user_real_name']),
|
||||
email: row['user_email'],
|
||||
created_at: Time.at(row['user_registered']),
|
||||
updated_at: Time.at(row['user_last_seen']),
|
||||
name: row['user_real_name'],
|
||||
title: row['user_title'],
|
||||
bio_raw: row['user_about'] || "",
|
||||
website: row['user_homepage'],
|
||||
location: row['user_location'],
|
||||
custom_fields: {
|
||||
md5_password: row['user_password'],
|
||||
original_username: row['user_username'],
|
||||
original_gossamer_id: row['user_id']
|
||||
}
|
||||
}
|
||||
end
|
||||
|
||||
# Create or update users in Discourse
|
||||
create_users(users) do |user|
|
||||
# insert_user_id_mapping(user[:id], user.id)
|
||||
user
|
||||
end
|
||||
|
||||
# For each user, add user ID mapping to SQLite now that we know what the Discourse user ID is, ... and append user bio and import user files
|
||||
users.each do |discourse_user|
|
||||
# discourse_username = sanitize_username(user[:username], user[:email], user[:name])
|
||||
# discourse_user = User.find_by(username: discourse_username)
|
||||
|
||||
# if discourse_user.nil?
|
||||
# puts "User #{user[:username]} --> #{discourse_username} not found in Discourse. Skipping file import."
|
||||
# next
|
||||
# end
|
||||
|
||||
# # Store the user ID mapping
|
||||
# @user_id_map[user[:id]] = discourse_user.id
|
||||
puts "for insert_user_id_mapping: user[:id] #{user[:id]} discourse_user.id #{discourse_user.id}"
|
||||
insert_user_id_mapping(user[:id], discourse_user.id)
|
||||
|
||||
# Ensure user profile exists and bio_raw is a string
|
||||
discourse_user.user_profile ||= UserProfile.new(user_id: discourse_user.id)
|
||||
discourse_user.user_profile.bio_raw ||= ""
|
||||
|
||||
# Append bio if it exists, otherwise set it to empty string to avoid nil errors
|
||||
if discourse_user.user_profile.bio_raw.empty?
|
||||
discourse_user.user_profile.bio_raw = user[:bio_raw]
|
||||
else
|
||||
discourse_user.user_profile.bio_raw += "\n\n" + user[:bio_raw]
|
||||
end
|
||||
|
||||
# Ensure the bio does not exceed 3000 characters
|
||||
if discourse_user.user_profile.bio_raw.length > 3000
|
||||
puts "Warning: About Me for user #{discourse_user.username} (ID: #{discourse_user.id}) exceeds 3000 characters. Truncating."
|
||||
discourse_user.user_profile.bio_raw = discourse_user.user_profile.bio_raw[0, 3000]
|
||||
end
|
||||
discourse_user.user_profile.save!
|
||||
|
||||
# Import user files
|
||||
import_user_files(discourse_user)
|
||||
end
|
||||
end
|
||||
|
||||
# # Import user files from Gossamer Forums to Discourse
|
||||
# def import_user_files(user)
|
||||
# print "\rImporting files for user #{user.username}..."
|
||||
#
|
||||
# original_gossamer_id = user.custom_fields['original_gossamer_id']
|
||||
# if original_gossamer_id.nil? || original_gossamer_id.empty?
|
||||
# puts "User #{user.username} does not have a valid original_gossamer_id. Skipping file import."
|
||||
# return
|
||||
# end
|
||||
#
|
||||
# # puts "Original Gossamer ID for user #{user.username}: #{original_gossamer_id}"
|
||||
#
|
||||
# # Fetch and import user files
|
||||
# execute_query("SELECT * FROM gforum_User_Files WHERE ForeignColKey = #{original_gossamer_id}").each do |file|
|
||||
# # Construct the file URL
|
||||
# file_url = "https://forum.slot.com/images/users/images/#{file['ID'] % 10}/#{file['ID']}-#{file['File_Name']}"
|
||||
# puts "User #{user.username} User ID: #{user.id} original_gossamer_id: #{original_gossamer_id} file_url: #{file_url}"
|
||||
#
|
||||
# new_bio = user.user_profile.bio_raw + "\n\n![#{file['File_Name']}](#{file_url})"
|
||||
# if new_bio.length > 3000
|
||||
# puts "Warning: About Me for user #{user.username} (ID: #{user.id}) exceeds 3000 characters after adding file link. Truncating."
|
||||
# new_bio = new_bio[0, 3000]
|
||||
# end
|
||||
# user.user_profile.bio_raw = new_bio
|
||||
# user.user_profile.save!
|
||||
# end
|
||||
# print "Importing files for user #{user.username}... Done.\n"
|
||||
# end
|
||||
|
||||
# Import user files (profile images) from Gossamer Forums to Discourse
|
||||
def import_user_files(user)
|
||||
print "\rImporting files for user #{user.username}..."
|
||||
|
||||
original_gossamer_id = user.custom_fields['original_gossamer_id']
|
||||
if original_gossamer_id.nil? || original_gossamer_id.empty?
|
||||
puts "User #{user.username} does not have a valid original_gossamer_id. Skipping file import."
|
||||
return
|
||||
end
|
||||
|
||||
puts "Original Gossamer ID for user #{user.username}: #{original_gossamer_id}"
|
||||
|
||||
images_imported = 0
|
||||
|
||||
execute_query("SELECT * FROM gforum_User_Files WHERE ForeignColKey = #{original_gossamer_id}").each do |file|
|
||||
file_url = "https://forum.slot.com/images/users/images/#{file['ID'] % 10}/#{file['ID']}-#{file['File_Name']}"
|
||||
puts "User #{user.username} User ID: #{user.id} original_gossamer_id: #{original_gossamer_id} file_url: #{file_url}"
|
||||
|
||||
next unless file['ForeignColName'] =~ /^user_image\d+$/
|
||||
puts "#A"
|
||||
next unless ['image/jpeg', 'image/png'].include?(file['File_MimeType'])
|
||||
puts "#B"
|
||||
|
||||
image_data = download_image(file_url)
|
||||
next if image_data.nil?
|
||||
puts "#C"
|
||||
|
||||
temp_file = Tempfile.new(['user_image', File.extname(file['File_Name'])])
|
||||
temp_file.binmode
|
||||
temp_file.write(image_data)
|
||||
temp_file.rewind
|
||||
|
||||
if images_imported == 0
|
||||
puts "#D"
|
||||
upload = upload_image(user, temp_file, file['File_Name'], file_url)
|
||||
next if upload.nil?
|
||||
|
||||
user.user_avatar = UserAvatar.create!(user_id: user.id, custom_upload_id: upload.id)
|
||||
user.save!
|
||||
|
||||
# Set the Profile Header
|
||||
UserProfile.find_by(user_id: user.id).update!(profile_background_upload_id: upload.id)
|
||||
|
||||
# Set the User Card Background
|
||||
UserProfile.find_by(user_id: user.id).update!(card_background_upload_id: upload.id)
|
||||
|
||||
images_imported += 1
|
||||
end
|
||||
puts "#E"
|
||||
user.user_profile.bio_raw ||= ""
|
||||
user.user_profile.bio_raw += "\n\n![#{file['File_Name']}](#{file_url})"
|
||||
user.user_profile.save!
|
||||
|
||||
temp_file.close
|
||||
temp_file.unlink
|
||||
end
|
||||
print "Importing files for user #{user.username}... Done.\n"
|
||||
end
|
||||
|
||||
|
||||
# Import categories from Gossamer Forums to Discourse
|
||||
def import_categories
|
||||
puts "Importing categories (forums)..."
|
||||
execute_query("SELECT * FROM gforum_Forum").each do |row|
|
||||
# Only create category if it does not exist
|
||||
unless CategoryCustomField.exists?(name: 'original_gossamer_id', value: row['forum_id'])
|
||||
category_name = row['forum_name']
|
||||
category_description = row['forum_desc'] || "No description provided"
|
||||
puts "id #{row['forum_id']} name #{category_name} description #{category_description}"
|
||||
# Create category in Discourse
|
||||
category = create_category(
|
||||
{
|
||||
# id: row['forum_id'] + 10,
|
||||
name: category_name,
|
||||
description: category_description,
|
||||
created_at: row['forum_last'] ? Time.at(row['forum_last']) : Time.now,
|
||||
updated_at: row['forum_last'] ? Time.at(row['forum_last']) : Time.now
|
||||
},
|
||||
row['forum_id'] # import_id argument
|
||||
)
|
||||
|
||||
# # Map Gossamer forum ID to Discourse category ID for future reference
|
||||
# @forum_id_map[row['forum_id']] = category.id
|
||||
|
||||
# category.custom_fields.create!(name: 'original_gossamer_id', value: row['forum_id'])
|
||||
category.custom_fields['original_gossamer_id'] = row['forum_id']
|
||||
category.save!
|
||||
|
||||
# Store the user ID mapping
|
||||
puts "for insert_category_id_mapping: category[:id] #{category[:id]} row['forum_id'] #{row['forum_id']}"
|
||||
insert_category_id_mapping(row['forum_id'], category[:id])
|
||||
end
|
||||
end
|
||||
puts "Importing categories... Done."
|
||||
end
|
||||
|
||||
# Helper function to ensure title meets the minimum length requirement
|
||||
def ensure_valid_title(title, min_length = 5)
|
||||
if title.length < min_length
|
||||
title += "." * (min_length - title.length) # Append dots to make it longer
|
||||
end
|
||||
title
|
||||
end
|
||||
|
||||
# Import topics and posts from Gossamer Forums to Discourse
|
||||
def import_topics_and_posts
|
||||
puts "Importing topics and posts..."
|
||||
|
||||
# Execute the query to get all posts ordered by post_id
|
||||
execute_query("SELECT * FROM gforum_Post ORDER BY post_id").each do |row|
|
||||
puts "post_id #{row['post_id']} post_root_id #{row['post_root_id']} post_subject/title #{row['post_subject']} forum_id_fk/category_id #{row['forum_id_fk']}"
|
||||
# discourse_user_id = @user_id_map[row['user_id_fk']]
|
||||
discourse_user_id = fetch_user_id_mapping(row['user_id_fk'])
|
||||
discourse_category_id = fetch_category_id_mapping(row['forum_id_fk'])
|
||||
|
||||
puts "discourse_user_id #{discourse_user_id} discourse_category_id #{discourse_category_id}"
|
||||
next unless discourse_user_id && discourse_category_id
|
||||
|
||||
if row['post_root_id'] == 0
|
||||
puts "#1"
|
||||
# Ensure the title is valid
|
||||
title = ensure_valid_title(row['post_subject'])
|
||||
|
||||
# Skip if the topic already exists
|
||||
unless TopicCustomField.exists?(name: 'original_gossamer_id', value: row['post_id'])
|
||||
# Create the topic
|
||||
begin
|
||||
puts "#2"
|
||||
puts "CREATE TOPIC title #{title} discourse_user_id #{discourse_user_id} category_id #{discourse_category_id}"
|
||||
topic = Topic.create!(
|
||||
title: title,
|
||||
user_id: discourse_user_id,
|
||||
created_at: Time.at(row['post_time']),
|
||||
updated_at: Time.at(row['post_latest_reply']),
|
||||
category_id: discourse_category_id
|
||||
)
|
||||
topic.custom_fields['original_gossamer_id'] = row['post_id']
|
||||
topic.save!
|
||||
|
||||
# Create the initial post in the topic
|
||||
puts "CREATE POST topic.id #{topic.id} discourse_user_id #{discourse_user_id}"
|
||||
sanitized_post_message = row['post_message']&.tr("\0", '') || ""
|
||||
post = Post.create!(
|
||||
topic_id: topic.id,
|
||||
user_id: discourse_user_id,
|
||||
# raw: import_attachments(row['post_message'], row['post_id']),
|
||||
# raw: row['post_message'] || "",
|
||||
raw: sanitized_post_message,
|
||||
created_at: Time.at(row['post_time']),
|
||||
updated_at: Time.at(row['post_latest_reply'])
|
||||
)
|
||||
post.custom_fields['original_gossamer_id'] = row['post_id']
|
||||
post.save!
|
||||
|
||||
# Create URL mappings
|
||||
# old_url = "https://old/forum/#{row['forum_name']}/topics/#{row['post_id']}"
|
||||
new_url = "https://new/t/#{topic.slug}/#{topic.id}"
|
||||
insert_url_mapping(row['post_id'], new_url, title)
|
||||
|
||||
rescue ActiveRecord::RecordInvalid => e
|
||||
puts "Error importing topic with post_id #{row['post_id']}: #{e.message}"
|
||||
end
|
||||
end
|
||||
|
||||
else
|
||||
puts "#3"
|
||||
# Find the root topic for the post
|
||||
root_topic_field = TopicCustomField.find_by(name: 'original_gossamer_id', value: row['post_root_id'])
|
||||
|
||||
if root_topic_field
|
||||
topic_id = root_topic_field.topic_id
|
||||
|
||||
# Find the parent post for the reply
|
||||
parent_post_field = PostCustomField.find_by(name: 'original_gossamer_id', value: row['post_father_id'])
|
||||
reply_to_post_number = parent_post_field ? Post.find(parent_post_field.post_id).post_number : nil
|
||||
|
||||
# Create the post in the existing topic
|
||||
begin
|
||||
puts "#4"
|
||||
sanitized_post_message = row['post_message']&.tr("\0", '') || ""
|
||||
post = Post.create!(
|
||||
topic_id: topic_id,
|
||||
user_id: discourse_user_id,
|
||||
# raw: import_attachments(row['post_message'], row['post_id']),
|
||||
# raw: row['post_message'] || "",
|
||||
raw: sanitized_post_message,
|
||||
created_at: Time.at(row['post_time']),
|
||||
updated_at: Time.at(row['post_latest_reply']),
|
||||
reply_to_post_number: reply_to_post_number
|
||||
)
|
||||
post.custom_fields['original_gossamer_id'] = row['post_id']
|
||||
post.save!
|
||||
rescue ActiveRecord::RecordInvalid => e
|
||||
puts "Error importing post with post_id #{row['post_id']}: #{e.message}"
|
||||
end
|
||||
else
|
||||
puts "Warning: Root topic not found for post_id #{row['post_id']} with post_root_id #{row['post_root_id']}"
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
# Import personal messages from gforum_Message table (both inbox and sent messages)
|
||||
def import_personal_messages
|
||||
puts "Importing personal (inbox and sendmail) messages..."
|
||||
execute_query("SELECT * FROM gforum_Message").each do |row|
|
||||
|
||||
from_user_id = fetch_user_id_mapping(row['from_user_id_fk'])
|
||||
to_user_id = fetch_user_id_mapping(row['to_user_id_fk'])
|
||||
|
||||
next unless from_user_id && to_user_id
|
||||
|
||||
# Skip if the message already exists
|
||||
unless TopicCustomField.exists?(name: 'original_gossamer_msg_id', value: row['msg_id'])
|
||||
|
||||
# Sanitize the message, ensuring we have an empty string or the content without any \0
|
||||
sanitized_message = row['msg_body']&.tr("\0", '') || ""
|
||||
|
||||
# Set default message body if the sanitized message is blank
|
||||
sanitized_message = " " if sanitized_message.strip.empty?
|
||||
|
||||
# # If we do not change the "min personal message post length" to 1, we need this.
|
||||
# sanitized_message = sanitized_message.ljust(10, ' ') if sanitized_message.length < 10
|
||||
|
||||
# Check and set a default title if the original title is nil or empty
|
||||
title = row['msg_subject']&.strip
|
||||
title = "<no subject>" if title.nil? || title.empty?
|
||||
|
||||
puts "IMPORTING title #{row['msg_subject']} user_id #{from_user_id} to_user_id #{to_user_id}"
|
||||
|
||||
# Create a private message topic in Discourse
|
||||
topic = Topic.create!(
|
||||
title: row['msg_subject'],
|
||||
user_id: from_user_id,
|
||||
archetype: Archetype.private_message,
|
||||
created_at: Time.at(row['msg_time']),
|
||||
updated_at: Time.at(row['msg_time'])
|
||||
)
|
||||
topic.custom_fields['original_gossamer_msg_id'] = row['msg_id']
|
||||
topic.save!
|
||||
|
||||
# Create the message as a post in the private topic
|
||||
post = Post.create!(
|
||||
topic_id: topic.id,
|
||||
user_id: from_user_id,
|
||||
# raw: row['msg_body'],
|
||||
raw: sanitized_message,
|
||||
created_at: Time.at(row['msg_time']),
|
||||
updated_at: Time.at(row['msg_time'])
|
||||
)
|
||||
post.custom_fields['original_gossamer_msg_id'] = row['msg_id']
|
||||
post.save!
|
||||
|
||||
# Add recipient user to the private message topic
|
||||
topic.topic_allowed_users.create!(user_id: to_user_id)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
# Import attachments for a post
|
||||
def import_post_attachments(post_message, post_id)
|
||||
# Fetch attachments related to the post
|
||||
attachments = execute_query("SELECT * FROM gforum_PostAttachment WHERE post_id_fk = #{post_id}")
|
||||
attachments.each do |attachment|
|
||||
|
||||
# Append attachment links to the post message
|
||||
file_url = "https://forum.slot.com/images/posts/attachments/#{attachment['ID'] % 10}/#{attachment['ID']}-#{attachment['File_Name']}"
|
||||
post_message += "\n\n![#{attachment['File_Name']}](#{file_url})"
|
||||
end
|
||||
1# post_message
|
||||
end
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
# Main method to perform the import
|
||||
def perform_import
|
||||
# Secret trick to disable RateLimiting protection in Discourse
|
||||
RateLimiter.disable
|
||||
|
||||
# Set our unique timestamp for this migration run
|
||||
timestamp = Time.now.strftime("-%y%m%d%H%M%S")
|
||||
|
||||
puts "Starting Gossamer Forums import... #{timestamp}"
|
||||
|
||||
import_users
|
||||
export_username_mapping_to_csv("gossamer-migration-username-mapping#{timestamp}")
|
||||
|
||||
import_categories
|
||||
import_topics_and_posts
|
||||
export_url_mapping_to_csv("gossamer-migration-url-mapping#{timestamp}")
|
||||
create_nginx_rewrite_rules("gossamer-redirects.conf")
|
||||
|
||||
import_personal_messages
|
||||
|
||||
# import_attachments
|
||||
|
||||
puts "Gossamer Forums import complete! #{timestamp}"
|
||||
end
|
||||
end
|
||||
|
||||
GossamerForumsImporter.new.perform_import
|
||||
|
||||
@ -1,207 +0,0 @@
|
||||
# Federated Computer, Inc.
|
||||
# David Sainty <saint@federated.computer> 2024 A.D.
|
||||
# Gossamer Threads to Discourse -- Soft Deletion of Marked-As-Deleted Posts
|
||||
# v0.3 Ready for actual soft-delete run on live-prod
|
||||
|
||||
require 'mysql2'
|
||||
require 'active_record'
|
||||
|
||||
# require 'concurrent-ruby'
|
||||
require File.expand_path("../../../../config/environment", __FILE__)
|
||||
require File.expand_path("../../../../script/import_scripts/base", __FILE__)
|
||||
|
||||
class GossamerForumsSoftDelDeletedPosts < ImportScripts::Base
|
||||
def initialize
|
||||
super
|
||||
begin
|
||||
# Initialize MySQL client to connect to Gossamer Forums database
|
||||
@mysql_client = Mysql2::Client.new(
|
||||
host: "slowtwitch.northend.network",
|
||||
username: "admin",
|
||||
password: "yxnh93Ybbz2Nm8#mp28zCVv",
|
||||
database: "slowtwitch"
|
||||
)
|
||||
rescue Mysql2::Error => e
|
||||
puts "Error connecting to MySQL: #{e.message}"
|
||||
exit 1
|
||||
end
|
||||
end
|
||||
|
||||
# Define a method to find a post by custom field
|
||||
def find_post_by_custom_field(post_id)
|
||||
puts "SoftDelDeletedPosts: Searching for post with original_gossamer_id: #{post_id}"
|
||||
|
||||
post_custom_field = PostCustomField.find_by(name: 'original_gossamer_id', value: post_id.to_s)
|
||||
|
||||
if post_custom_field
|
||||
post = post_custom_field.post
|
||||
puts "SoftDelDeletedPosts: Found post with id: #{post.id}"
|
||||
post
|
||||
else
|
||||
puts "SoftDelDeletedPosts: No post found with original_gossamer_id: #{post_id}"
|
||||
nil
|
||||
end
|
||||
end
|
||||
|
||||
# Soft delete the post, whether topic post (OP) or reply post
|
||||
def soft_delete_post(post, deleted_by_id)
|
||||
if post.deleted_at.nil?
|
||||
# # Get the post's owner and topic information
|
||||
# user = User.find_by(id: post.user_id) # Fetch user details
|
||||
# topic = Topic.find_by(id: post.topic_id) # Fetch topic details
|
||||
|
||||
# # Display post, user, and topic information
|
||||
# puts "DELETING POST: SoftDelDeletedPosts: Soft deleting post with id: #{post.id}"
|
||||
# puts " - Post owner: #{user.username} (ID: #{user.id})"
|
||||
# puts " - Topic title: '#{topic.title}' (Topic ID: #{topic.id})"
|
||||
|
||||
# Perform the soft delete by updating the fields
|
||||
## post.deleted_at = Time.now
|
||||
## post.deleted_by_id = deleted_by_id
|
||||
## post.save
|
||||
puts "... actual deleted_at step"
|
||||
post.update(deleted_at: Time.now, deleted_by_id: deleted_by_id)
|
||||
else
|
||||
puts "SoftDelDeletedPosts: Post with id: #{post.id} is already soft deleted."
|
||||
end
|
||||
end
|
||||
|
||||
# Define the method to delete posts based on the Gossamer Forums flag
|
||||
def soft_del_deleted_posts_from_gossamer_with_user(username)
|
||||
|
||||
# Query the user ID from the legacy MySQL database
|
||||
user_result = @mysql_client.query("SELECT user_id FROM gforum_User WHERE user_username = '#{username}' LIMIT 1")
|
||||
user_id_row = user_result.first
|
||||
if user_id_row.nil?
|
||||
puts "DiscourseDeletedPosts: No user found with username: #{username}"
|
||||
return
|
||||
end
|
||||
user_id = user_id_row['user_id']
|
||||
|
||||
# Find all posts marked as deleted by the given user
|
||||
posts_result = @mysql_client.query("SELECT post_id FROM gforum_Post WHERE post_deleted = 1 AND user_id_fk = #{user_id}")
|
||||
|
||||
# "admin" user is ID 1, so set as the one performing the deletions
|
||||
deleted_by_id = 1
|
||||
|
||||
posts_result.each do |legacy_post|
|
||||
post_id = legacy_post['post_id']
|
||||
|
||||
# Look for the post in Discourse by custom field
|
||||
post = find_post_by_custom_field(post_id)
|
||||
|
||||
if post
|
||||
# Soft delete only the individual post (whether topic post / OP or a reply post)
|
||||
soft_delete_post(post, deleted_by_id)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# Define the method to reverse (soft undelete) the soft deletion
|
||||
def soft_undelete_post(post)
|
||||
if post.deleted_at.present?
|
||||
# Get the post's owner and topic information
|
||||
user = User.find_by(id: post.user_id) # Fetch user details
|
||||
topic = Topic.find_by(id: post.topic_id) # Fetch topic details
|
||||
|
||||
# Display post, user, and topic information
|
||||
puts "RESTORING POST: SoftDelDeletedPosts: Soft undeleting post with id: #{post.id}"
|
||||
puts " - Post owner: #{user.username} (ID: #{user.id})"
|
||||
puts " - Topic title: '#{topic.title}' (Topic ID: #{topic.id})"
|
||||
|
||||
# Reverse the soft delete by setting deleted_at and deleted_by_id to nil
|
||||
### post.update(deleted_at: nil, deleted_by_id: nil)
|
||||
else
|
||||
puts "SoftDelDeletedPosts: Post with id: #{post.id} is not soft deleted."
|
||||
end
|
||||
end
|
||||
|
||||
# Define the method to reverse the soft deletion for all legacy posts
|
||||
def soft_undelete_all_deleted_posts
|
||||
# Query the legacy database for posts marked as deleted
|
||||
posts_result = @mysql_client.query("SELECT post_id FROM gforum_Post WHERE post_deleted = 1")
|
||||
|
||||
posts_result.each do |legacy_post|
|
||||
post_id = legacy_post['post_id']
|
||||
|
||||
# Look for the post in Discourse by custom field
|
||||
post = find_post_by_custom_field(post_id)
|
||||
|
||||
if post
|
||||
# Reverse the soft deletion (restore the post)
|
||||
soft_undelete_post(post)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# Define the method to delete posts based on the Gossamer Forums flag
|
||||
def soft_del_all_deleted_posts
|
||||
|
||||
# Find all posts marked as deleted by the given user
|
||||
posts_result = @mysql_client.query("SELECT post_id FROM gforum_Post WHERE post_deleted = 1")
|
||||
|
||||
# "admin" user is ID 1, so set as the one performing the deletions
|
||||
deleted_by_id = 1
|
||||
|
||||
posts_result.each do |legacy_post|
|
||||
post_id = legacy_post['post_id']
|
||||
|
||||
# # Look for the post in Discourse by custom field
|
||||
# post = find_post_by_custom_field(post_id)
|
||||
#
|
||||
# if post
|
||||
# # Soft delete only the individual post (whether topic post / OP or a reply post)
|
||||
# soft_delete_post(post, deleted_by_id)
|
||||
# end
|
||||
soft_del_all_deleted_posts_by_custom_field(post_id, deleted_by_id)
|
||||
end
|
||||
end
|
||||
|
||||
# Define a method to find all posts by the original Gossamer Forums post ID and soft delete them
|
||||
def soft_del_all_deleted_posts_by_custom_field(post_id, deleted_by_id)
|
||||
puts "SoftDelDeletedPosts: Searching for all posts with original_gossamer_id: #{post_id}"
|
||||
|
||||
# Find all PostCustomField records with the given Gossamer Forums post_id
|
||||
post_custom_fields = PostCustomField.where(name: 'original_gossamer_id', value: post_id.to_s)
|
||||
|
||||
if post_custom_fields.any?
|
||||
post_custom_fields.each do |post_custom_field|
|
||||
post = post_custom_field.post
|
||||
|
||||
if post
|
||||
puts "DELETING POST: SoftDelDeletedPosts: Soft deleting post with id: #{post.id} and user_id: #{post.user_id} and topic_id: #{post.topic_id}"
|
||||
# Get the post's owner and topic information
|
||||
# user = User.find_by(id: post.user_id) # Fetch user details
|
||||
# topic = Topic.find_by(id: post.topic_id) # Fetch topic details
|
||||
|
||||
# Display post, user, and topic information
|
||||
# puts " - Post owner: #{user.username} (ID: #{user.id})"
|
||||
# puts " - Topic title: '#{topic.title}' (Topic ID: #{topic.id})"
|
||||
|
||||
# Soft delete the post if not already deleted
|
||||
soft_delete_post(post, deleted_by_id)
|
||||
else
|
||||
puts "SoftDelDeletedPosts: Found PostCustomField with no corresponding post (ID: #{post_custom_field.id})"
|
||||
end
|
||||
end
|
||||
else
|
||||
puts "SoftDelDeletedPosts: No PostCustomField records found with original_gossamer_id: #{post_id}"
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
def perform_deleted_soft_del
|
||||
puts "Soft Del Deleted Posts beginning!"
|
||||
|
||||
## soft_del_deleted_posts_from_gossamer_with_user('spudone')
|
||||
|
||||
## soft_undelete_all_deleted_posts
|
||||
|
||||
soft_del_all_deleted_posts
|
||||
|
||||
puts "Soft Del Deleted Posts complete!"
|
||||
end
|
||||
end
|
||||
|
||||
GossamerForumsSoftDelDeletedPosts.new.perform_deleted_soft_del
|
||||
|
||||
Loading…
Reference in New Issue
Block a user