saint/discourse-import_scripts
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

v0.42 Mutex addition for SQLite (which may be very important

Browse Source
This commit is contained in:
David Sainty 2024-08-17 20:25:48 +10:00
parent 3a1476951e
commit d3e6dac1de
1 changed files with 44 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

72
gossamer_forums.rb
View File

@ -1,7 +1,7 @@
# Federated Computer, Inc. # Federated Computer, Inc.
# David Sainty <saint@federated.computer> 2024 A.D. # David Sainty <saint@federated.computer> 2024 A.D.
# Gossamer Threads to Discourse -- Migration-Import Script # Gossamer Threads to Discourse -- Migration-Import Script
# v0.41.2 Several bug fixes and improvements for fully concurrent topic-post import # v0.42 Mutex addition for SQLite (which may be very important)
require 'mysql2' require 'mysql2'
require 'open-uri' require 'open-uri'
@ -1081,7 +1081,8 @@ class GossamerForumsImporter < ImportScripts::Base
is_complete = false # Flag to indicate whether the import process is complete. is_complete = false # Flag to indicate whether the import process is complete.
# Mutex to control access to shared resources # Mutex to control access to shared resources
mutex = Mutex.new ### mutex = Mutex.new # Mutex for MySQL2 operations -- disabled as this may not in fact be necessary - TBD.
sqlite_mutex = Mutex.new # Mutex for SQLite opreations
# Run until all posts have been processed. # Run until all posts have been processed.
until is_complete until is_complete
@ -1107,6 +1108,7 @@ class GossamerForumsImporter < ImportScripts::Base
# Submit the import job for the current post_id to the thread pool # Submit the import job for the current post_id to the thread pool
pool.post do pool.post do
puts "PP 11"
# Initialise a new MariaDB / Mysql2 client inside of each thread # Initialise a new MariaDB / Mysql2 client inside of each thread
mysql_client = Mysql2::Client.new( mysql_client = Mysql2::Client.new(
host: "slowtwitch.northend.network", host: "slowtwitch.northend.network",
@ -1115,21 +1117,26 @@ class GossamerForumsImporter < ImportScripts::Base
database: "slowtwitch" database: "slowtwitch"
) )
puts "PP 22"
# Use connection ppoling for PostgreSQL and synchronize access to shared resources # Use connection ppoling for PostgreSQL and synchronize access to shared resources
ActiveRecord::Base.connection_pool.with_connection do ActiveRecord::Base.connection_pool.with_connection do
mutex.synchronize do ### mutex.synchronize do
## begin begin
puts "Processing post ID: #{post_id}" puts "Processing post ID: #{post_id}"
topic_import_job(post_id, mysql_client) # Import topic and its replies topic_import_job(post_id, mysql_client, sqlite_mutex) # Import topic and its replies
mark_post_as_complete(post_id) # Mark as complete in SQLite table sqlite_mutex.sychronize do
## rescue => e mark_post_as_complete(post_id) # Mark as complete in SQLite table
end
rescue => e
puts "Error processing post ID #{post_id}: #{e.message}" puts "Error processing post ID #{post_id}: #{e.message}"
mark_post_as_failed(post_id) sqlite_mutex.sychronize do
## ensure mark_post_as_failed(post_id)
end
ensure
# Ensure the MariaDB connection is closed after processing # Ensure the MariaDB connection is closed after processing
mysql_client.close if mysql_client mysql_client.close if mysql_client
## end end
end ### end
end end
end end
else else
@ -1159,7 +1166,7 @@ class GossamerForumsImporter < ImportScripts::Base
# end # end
# Method to import an entire topic, including its first post and all subsequent replies # Method to import an entire topic, including its first post and all subsequent replies
def topic_import_job(post_id, mysql_client) def topic_import_job(post_id, mysql_client, sqlite_mutex)
#Here is where you can import the entire topic #Here is where you can import the entire topic
#Get post -- SELECT post_id, user_id_fk, forum_id_fk, post_root_id, post_subject, post_time, post_message, post_father_id, post_replies FROM gforum_Post WHERE post_id = post_id #Get post -- SELECT post_id, user_id_fk, forum_id_fk, post_root_id, post_subject, post_time, post_message, post_father_id, post_replies FROM gforum_Post WHERE post_id = post_id
#check if exists, create if not #check if exists, create if not
@ -1228,21 +1235,26 @@ class GossamerForumsImporter < ImportScripts::Base
end end
end end
# Update the database with the last post time and user for the topic sqlite_mutex.sychronize do
update_db_topic_last_post_time(topic.id, Time.at(row['post_time']).to_i) # Update the database with the last post time and user for the topic
update_db_topic_last_post_user(topic.id, discourse_user_id) update_db_topic_last_post_time(topic.id, Time.at(row['post_time']).to_i)
update_db_topic_last_post_user(topic.id, discourse_user_id)
# Increment the topic count for the user # Increment the topic count for the user
update_db_user_topic_count(discourse_user_id, fetch_db_user_topic_count(discourse_user_id).to_i + 1) update_db_user_topic_count(discourse_user_id, fetch_db_user_topic_count(discourse_user_id).to_i + 1)
end
# Sanitize and prepare the post message for Discourse # Sanitize and prepare the post message for Discourse
sanitized_post_message = sanitize_post_message(row['post_message']) sanitized_post_message = sanitize_post_message(row['post_message'])
puts "CREATE TOPIC POST topic.id #{topic.id} discourse_user_id #{discourse_user_id}" puts "CREATE TOPIC POST topic.id #{topic.id} discourse_user_id #{discourse_user_id}"
# Increment the post count for the topic sqlite_mutex.synchronize do
post_number = fetch_db_topic_post_numbers(topic.id).to_i + 1 # Increment the post count for the topic
update_db_topic_post_numbers(topic.id, post_number) post_number = fetch_db_topic_post_numbers(topic.id).to_i + 1
update_db_topic_post_numbers(topic.id, post_number)
end
puts "TIJ GG post_id #{post_id}" puts "TIJ GG post_id #{post_id}"
# Create the initial post in the new topic # Create the initial post in the new topic
@ -1258,9 +1270,11 @@ class GossamerForumsImporter < ImportScripts::Base
post.custom_fields['original_gossamer_id'] = row['post_id'] post.custom_fields['original_gossamer_id'] = row['post_id']
post.save! post.save!
# Increment the post count for the topic and user sqlite_mutex.sychronize do
update_db_topic_post_count(topic.id, fetch_db_topic_post_count(topic.id).to_i + 1) # Increment the post count for the topic and user
update_db_user_post_count(discourse_user_id, fetch_db_user_post_count(discourse_user_id).to_i + 1) update_db_topic_post_count(topic.id, fetch_db_topic_post_count(topic.id).to_i + 1)
update_db_user_post_count(discourse_user_id, fetch_db_user_post_count(discourse_user_id).to_i + 1)
end
puts "TIJ HH post_id #{post_id}" puts "TIJ HH post_id #{post_id}"
# Handle any attachments associated with the post # Handle any attachments associated with the post
@ -1285,12 +1299,14 @@ class GossamerForumsImporter < ImportScripts::Base
puts "CREATE REPLY in topic_id #{topic.id}" puts "CREATE REPLY in topic_id #{topic.id}"
# Increment the post count for the topic sqlite_mutex.sychronize do
post_number = fetch_db_topic_post_numbers(topic.id).to_i + 1 # Increment the post count for the topic
update_db_topic_post_numbers(topic.id, post_number) post_number = fetch_db_topic_post_numbers(topic.id).to_i + 1
update_db_topic_post_numbers(topic.id, post_number)
# Fetch the number of views the post has had # Fetch the number of views the post has had
reply_post_views = fetch_post_views(reply_row['post_id']) reply_post_views = fetch_post_views(reply_row['post_id'])
end
puts "TIJ JJ post_id #{post_id}" puts "TIJ JJ post_id #{post_id}"
# Create the reply post in the existing topic # Create the reply post in the existing topic