saint/discourse-import_scripts
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

v0.28 Improvement for personal message importing - tracking and corner case workaround

Browse Source
This commit is contained in:
David Sainty 2024-07-05 01:28:39 +10:00
parent e052d0c939
commit 0fdfba684a
1 changed files with 42 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
gossamer_forums.rb
View File

@ -1,5 +1,5 @@
# gossamer threads migration-import code
# v0.27.2
# v0.28
require 'mysql2'
require 'open-uri'
@ -115,6 +115,12 @@ class GossamerForumsImporter < ImportScripts::Base
post_id INTEGER
);
SQL
@db.execute <<-SQL
CREATE TABLE IF NOT EXISTS highest_processed_personal_id (
id INTEGER PRIMARY KEY CHECK (id = 1),
personal_id INTEGER
);
SQL
end
def insert_user_id_mapping(old_user_id, new_user_id)
@ -250,6 +256,15 @@ class GossamerForumsImporter < ImportScripts::Base
@db.execute "INSERT OR REPLACE INTO highest_processed_post_id (id, post_id) VALUES (1, ?)", post_id
end
# Fetch the highest processed personal_id from the highest_processed_personal_id table
def fetch_highest_processed_personal_id
@db.get_first_value "SELECT personal_id FROM highest_processed_personal_id WHERE id = 1"
end
# Update the highest processed personal_id in the highest_processed_personal_id table
def update_highest_processed_personal_id(personal_id)
@db.execute "INSERT OR REPLACE INTO highest_processed_personal_id (id, personal_id) VALUES (1, ?)", personal_id
end
# Execute an SQL query on the Gossamer Forums database
def execute_query(query)
@ -1144,7 +1159,19 @@ class GossamerForumsImporter < ImportScripts::Base
# Import personal messages from gforum_Message table (both inbox and sent messages)
def import_personal_messages
puts "Importing personal (inbox and sendmail) messages..."
# Fetch the highest_processed_personal_id
highest_processed_personal_id = fetch_highest_processed_personal_id.to_i
puts "Highest processed personal_id: #{highest_processed_personal_id}"
# OVERRIDE - to get to problem msg
highest_processed_personal_id = 350
execute_query("SELECT * FROM gforum_Message").each do |row|
msg_id = row['msg_id'].to_i
# Skip posts that have already been processed
next if msg_id <= highest_processed_personal_id
from_user_id = fetch_user_id_mapping(row['from_user_id_fk'])
to_user_id = fetch_user_id_mapping(row['to_user_id_fk'])
@ -1158,21 +1185,20 @@ class GossamerForumsImporter < ImportScripts::Base
sanitized_message = sanitize_post_message(row['msg_body'])
# Set default message body if the sanitized message is blank
sanitized_message = " " if sanitized_message.strip.empty?
sanitized_message = "<no message body>" if sanitized_message.strip.empty?
# # If we do not change the "min personal message post length" to 1, we need this.
# sanitized_message = sanitized_message.ljust(10, ' ') if sanitized_message.length < 10
# If we do not change the "min personal message post length" to 1, we need this. ... We may need this anyway.
sanitized_message = sanitized_message.ljust(10, '.') if sanitized_message.length < 10
# Check and set a default title if the original title is nil or empty
title = row['msg_subject']&.strip
title = "<no subject>" if title.nil? || title.empty?
sanitized_title = row['msg_subject']&.strip
sanitized_title = "<no subject>" if sanitized_title.nil? || sanitized_title.empty?
puts "IMPORTING title #{row['msg_subject']} user_id #{from_user_id} to_user_id #{to_user_id}"
puts "IMPORTING msg. sanitized_title: #{sanitized_title} orig title: #{row['msg_subject']} user_id #{from_user_id} to_user_id #{to_user_id}"
# Create a private message topic in Discourse
topic = Topic.create!(
# title: row['msg_subject'],
title: title,
title: sanitized_title,
user_id: from_user_id,
archetype: Archetype.private_message,
created_at: Time.at(row['msg_time']),
@ -1185,7 +1211,6 @@ class GossamerForumsImporter < ImportScripts::Base
post = Post.create!(
topic_id: topic.id,
user_id: from_user_id,
# raw: row['msg_body'],
raw: sanitized_message,
created_at: Time.at(row['msg_time']),
updated_at: Time.at(row['msg_time'])
@ -1196,6 +1221,9 @@ class GossamerForumsImporter < ImportScripts::Base
# Add recipient user to the private message topic
topic.topic_allowed_users.create!(user_id: to_user_id)
update_highest_processed_personal_id(msg_id)
# N/A. These were never a thing in Slowtwitch...
# handle_post_attachments(row['msg_id'], post, from_user_id)
end
end
@ -1221,8 +1249,8 @@ class GossamerForumsImporter < ImportScripts::Base
# import_topics_and_posts_with_attachments
# update_topic_stats
update_user_stats
export_url_mapping_to_csv("gossamer-migration-url-mapping#{timestamp}")
create_nginx_rewrite_rules("gossamer-redirects#{timestamp}.conf")
# export_url_mapping_to_csv("gossamer-migration-url-mapping#{timestamp}")
# create_nginx_rewrite_rules("gossamer-redirects#{timestamp}.conf")
import_personal_messages