saint/discourse-import_scripts
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

v0.58 Final user import and personal message import run

Browse Source
This commit is contained in:
David Sainty 2024-08-31 22:28:48 +10:00
parent a61fa8a725
commit f27447bbc1
1 changed files with 56 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

92
gossamer_forums.rb
View File

@ -1,7 +1,7 @@
# Federated Computer, Inc. # Federated Computer, Inc.
# David Sainty <saint@federated.computer> 2024 A.D. # David Sainty <saint@federated.computer> 2024 A.D.
# Gossamer Threads to Discourse -- Migration-Import Script # Gossamer Threads to Discourse -- Migration-Import Script
# v0.57 Improve personal message handling and add commented line for doing specified blocks of PMs. # v0.58 Final user import and personal message import run
require 'mysql2' require 'mysql2'
require 'open-uri' require 'open-uri'
@ -625,26 +625,35 @@ class GossamerForumsImporter < ImportScripts::Base
# Fetch all users from Gossamer Forums # Fetch all users from Gossamer Forums
execute_query("SELECT * FROM gforum_User").each do |row| execute_query("SELECT * FROM gforum_User").each do |row|
users << { username = sanitize_username(row['user_username'], row['user_email'], row['user_real_name'])
id: row['user_id'], email = row['user_email']
username: sanitize_username(row['user_username'], row['user_email'], row['user_real_name']),
email: row['user_email'], # Check if the user already exists in Discourse by username or email
created_at: Time.at(row['user_registered']), existing_user = User.find_by(username: username) || User.find_by(email: email)
updated_at: Time.at(row['user_last_seen']),
name: row['user_real_name'], # Only add the user if they do not already exist
title: row['user_title'], unless existing_user
bio_raw: row['user_about'] || "", users << {
website: row['user_homepage'], id: row['user_id'],
location: row['user_location'], username: username,
custom_fields: { email: email,
md5_password: row['user_password'], created_at: Time.at(row['user_registered']),
original_username: row['user_username'], updated_at: Time.at(row['user_last_seen']),
original_gossamer_id: row['user_id'] name: row['user_real_name'],
title: row['user_title'],
bio_raw: row['user_about'] || "",
website: row['user_homepage'],
location: row['user_location'],
custom_fields: {
md5_password: row['user_password'],
original_username: row['user_username'],
original_gossamer_id: row['user_id']
}
} }
} end
end end
# Create or update users in Discourse # Create new users in Discourse
create_users(users) do |user| create_users(users) do |user|
# insert_user_id_mapping(user[:id], user.id) # insert_user_id_mapping(user[:id], user.id)
user user
@ -1350,7 +1359,7 @@ class GossamerForumsImporter < ImportScripts::Base
topic_created = true topic_created = true
# rescue ActiveRecord::RecordInvalid => e # rescue ActiveRecord::RecordInvalid => e
rescue => e rescue => e
if e.message.include?("Title has already been used") if e.message.include?("Title has already been used")
unique_title = "#{title} (#{suffix})" unique_title = "#{title} (#{suffix})"
suffix += 1 suffix += 1
@ -1529,7 +1538,7 @@ class GossamerForumsImporter < ImportScripts::Base
# Import topics and posts from Gossamer Forums to Discourse # Import topics and posts from Gossamer Forums to Discourse
def import_topics_and_posts_with_attachments def import_topics_and_posts
puts "Importing topics and posts with attachments..." puts "Importing topics and posts with attachments..."
# topic_last_post_time = {} # topic_last_post_time = {}
@ -1540,16 +1549,17 @@ class GossamerForumsImporter < ImportScripts::Base
# topic_post_numbers = Hash.new { |hash, key| hash[key] = 0 } # topic_post_numbers = Hash.new { |hash, key| hash[key] = 0 }
# Fetch the highest old_post_id from the url_map table # Fetch the highest old_post_id from the url_map table
highest_old_post_id = fetch_highest_old_post_id.to_i # highest_old_post_id = fetch_highest_old_post_id.to_i
puts "Highest (OP) old_post_id in url_map: #{highest_old_post_id}" # puts "Highest (OP) old_post_id in url_map: #{highest_old_post_id}"
highest_processed_post_id = fetch_highest_processed_post_id.to_i ### highest_processed_post_id = fetch_highest_processed_post_id.to_i
highest_processed_post_id = 8179621
puts "Highest processed post_id: #{highest_processed_post_id}" puts "Highest processed post_id: #{highest_processed_post_id}"
# OVERRIDE........ # OVERRIDE........
# Attachment example: highest_processed_post_id = 1359862 # Attachment example: highest_processed_post_id = 1359862
# Execute the query to get all posts ordered by post_id # Execute the query to get all posts ordered by post_id
execute_query("SELECT post_id, user_id_fk, forum_id_fk, post_root_id, post_subject, post_time, post_message, post_father_id, post_likes, post_replies FROM gforum_Post ORDER BY post_id").each do |row| execute_query("SELECT post_id, user_id_fk, forum_id_fk, post_root_id, post_subject, post_time, post_message, post_father_id, post_likes, post_replies FROM gforum_Post WHERE post_id > #{highest_processed_post_id} ORDER BY post_id").each do |row|
post_id = row['post_id'].to_i post_id = row['post_id'].to_i
# Skip posts that have already been processed # Skip posts that have already been processed
@ -1560,8 +1570,18 @@ class GossamerForumsImporter < ImportScripts::Base
puts "Processing post_id #{row['post_id']} post_id #{row['post_root_id']} post_subject/title #{row['post_subject']} forum_id_fk/category_id #{row['forum_id_fk']}" puts "Processing post_id #{row['post_id']} post_id #{row['post_root_id']} post_subject/title #{row['post_subject']} forum_id_fk/category_id #{row['forum_id_fk']}"
# discourse_user_id = @user_id_map[row['user_id_fk']] # discourse_user_id = @user_id_map[row['user_id_fk']]
# Fetch the Discourse user and category IP mappings # Fetch the Discourse user_id based on Gossamer data and mapping
discourse_user_id = fetch_user_id_mapping(row['user_id_fk']) discourse_user_id = fetch_user_id_mapping(row['user_id_fk'])
# Check to be certain user has not been deleted, etc.
if discourse_user_id.nil? || discourse_user_id == 0
puts "discourse_user_id is NIL/ZERO for post_id #{row['post_id']}"
discourse_former_user = User.find_by(username: 'Former_User')
discourse_user_id = discourse_former_user.id
puts "discourse_user_id is NOW Former_User id #{discourse_user_id} for post_id #{row['post_id']}"
end
# Fetch the Discourse category_id based on Gossamer data and mapping
discourse_category_id = fetch_category_id_mapping(row['forum_id_fk']) discourse_category_id = fetch_category_id_mapping(row['forum_id_fk'])
puts "discourse_user_id #{discourse_user_id} discourse_category_id #{discourse_category_id}" puts "discourse_user_id #{discourse_user_id} discourse_category_id #{discourse_category_id}"
next unless discourse_user_id && discourse_category_id next unless discourse_user_id && discourse_category_id
@ -1634,7 +1654,7 @@ class GossamerForumsImporter < ImportScripts::Base
# Sanitize the post message # Sanitize the post message
sanitized_post_message = sanitize_post_message(row['post_message']) sanitized_post_message = sanitize_post_message(row['post_message'])
puts "CREATE POST topic.id #{topic.id} discourse_user_id #{discourse_user_id}" puts "CREATE TOPIC POST topic.id #{topic.id} discourse_user_id #{discourse_user_id}"
# Increment the number of posts in the given topic. # Increment the number of posts in the given topic.
# topic_post_numbers[topic.id] += 1 # topic_post_numbers[topic.id] += 1
@ -2019,25 +2039,25 @@ class GossamerForumsImporter < ImportScripts::Base
puts "Starting Gossamer Forums import... #{timestamp}" puts "Starting Gossamer Forums import... #{timestamp}"
# add_former_user # add_former_user
# import_users import_users
# generate_user_id_mapping generate_user_id_mapping
# export_username_mapping_to_csv("/bitnami/discourse/sqlite/gossamer-migration-username-mapping#{timestamp}") export_username_mapping_to_csv("/bitnami/discourse/sqlite/gossamer-migration-username-mapping#{timestamp}")
# set_user_bio_images # set_user_bio_images
# import_categories # import_categories
####### import_topics_and_posts_with_attachments ## import_topics_and_posts
# threaded_topic_import # threaded_topic_import
# update_topic_stats ### update_topic_stats
# update_user_stats ### update_user_stats
# export_url_mapping_to_csv("/bitnami/discourse/sqlite/gossamer-migration-url-mapping#{timestamp}") #### export_url_mapping_to_csv("/bitnami/discourse/sqlite/gossamer-migration-url-mapping#{timestamp}")
# export_nginx_rewrite_rules("/bitnami/discourse/sqlite/gossamer-redirects#{timestamp}.conf") #### export_nginx_rewrite_rules("/bitnami/discourse/sqlite/gossamer-redirects#{timestamp}.conf")
# update_existing_personal_message_activity ##### update_existing_personal_message_activity
import_personal_messages import_personal_messages
puts "Gossamer Forums import complete! #{timestamp}" puts "Gossamer Forums import complete! #{timestamp}"
end end