saint/discourse-import_scripts
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

v0.42 Mutex addition for SQLite (which may be very important

Browse Source
This commit is contained in:
David Sainty 2024-08-17 20:25:48 +10:00
parent 3a1476951e
commit d3e6dac1de
1 changed files with 44 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

72
gossamer_forums.rb
View File

@ -1,7 +1,7 @@
# Federated Computer, Inc.
# David Sainty <saint@federated.computer> 2024 A.D.
# Gossamer Threads to Discourse -- Migration-Import Script
# v0.41.2 Several bug fixes and improvements for fully concurrent topic-post import
# v0.42 Mutex addition for SQLite (which may be very important)
require 'mysql2'
require 'open-uri'
@ -1081,7 +1081,8 @@ class GossamerForumsImporter < ImportScripts::Base
is_complete = false # Flag to indicate whether the import process is complete.
# Mutex to control access to shared resources
mutex = Mutex.new
### mutex = Mutex.new # Mutex for MySQL2 operations -- disabled as this may not in fact be necessary - TBD.
sqlite_mutex = Mutex.new # Mutex for SQLite opreations
# Run until all posts have been processed.
until is_complete
@ -1107,6 +1108,7 @@ class GossamerForumsImporter < ImportScripts::Base
# Submit the import job for the current post_id to the thread pool
pool.post do
puts "PP 11"
# Initialise a new MariaDB / Mysql2 client inside of each thread
mysql_client = Mysql2::Client.new(
host: "slowtwitch.northend.network",
@ -1115,21 +1117,26 @@ class GossamerForumsImporter < ImportScripts::Base
database: "slowtwitch"
)
puts "PP 22"
# Use connection ppoling for PostgreSQL and synchronize access to shared resources
ActiveRecord::Base.connection_pool.with_connection do
mutex.synchronize do
## begin
### mutex.synchronize do
begin
puts "Processing post ID: #{post_id}"
topic_import_job(post_id, mysql_client) # Import topic and its replies
mark_post_as_complete(post_id) # Mark as complete in SQLite table
## rescue => e
topic_import_job(post_id, mysql_client, sqlite_mutex) # Import topic and its replies
sqlite_mutex.sychronize do
mark_post_as_complete(post_id) # Mark as complete in SQLite table
end
rescue => e
puts "Error processing post ID #{post_id}: #{e.message}"
mark_post_as_failed(post_id)
## ensure
sqlite_mutex.sychronize do
mark_post_as_failed(post_id)
end
ensure
# Ensure the MariaDB connection is closed after processing
mysql_client.close if mysql_client
## end
end
end
### end
end
end
else
@ -1159,7 +1166,7 @@ class GossamerForumsImporter < ImportScripts::Base
# end
# Method to import an entire topic, including its first post and all subsequent replies
def topic_import_job(post_id, mysql_client)
def topic_import_job(post_id, mysql_client, sqlite_mutex)
#Here is where you can import the entire topic
#Get post -- SELECT post_id, user_id_fk, forum_id_fk, post_root_id, post_subject, post_time, post_message, post_father_id, post_replies FROM gforum_Post WHERE post_id = post_id
#check if exists, create if not
@ -1228,21 +1235,26 @@ class GossamerForumsImporter < ImportScripts::Base
end
end
# Update the database with the last post time and user for the topic
update_db_topic_last_post_time(topic.id, Time.at(row['post_time']).to_i)
update_db_topic_last_post_user(topic.id, discourse_user_id)
sqlite_mutex.sychronize do
# Update the database with the last post time and user for the topic
update_db_topic_last_post_time(topic.id, Time.at(row['post_time']).to_i)
update_db_topic_last_post_user(topic.id, discourse_user_id)
# Increment the topic count for the user
update_db_user_topic_count(discourse_user_id, fetch_db_user_topic_count(discourse_user_id).to_i + 1)
# Increment the topic count for the user
update_db_user_topic_count(discourse_user_id, fetch_db_user_topic_count(discourse_user_id).to_i + 1)
end
# Sanitize and prepare the post message for Discourse
sanitized_post_message = sanitize_post_message(row['post_message'])
puts "CREATE TOPIC POST topic.id #{topic.id} discourse_user_id #{discourse_user_id}"
# Increment the post count for the topic
post_number = fetch_db_topic_post_numbers(topic.id).to_i + 1
update_db_topic_post_numbers(topic.id, post_number)
sqlite_mutex.synchronize do
# Increment the post count for the topic
post_number = fetch_db_topic_post_numbers(topic.id).to_i + 1
update_db_topic_post_numbers(topic.id, post_number)
end
puts "TIJ GG post_id #{post_id}"
# Create the initial post in the new topic
@ -1258,9 +1270,11 @@ class GossamerForumsImporter < ImportScripts::Base
post.custom_fields['original_gossamer_id'] = row['post_id']
post.save!
# Increment the post count for the topic and user
update_db_topic_post_count(topic.id, fetch_db_topic_post_count(topic.id).to_i + 1)
update_db_user_post_count(discourse_user_id, fetch_db_user_post_count(discourse_user_id).to_i + 1)
sqlite_mutex.sychronize do
# Increment the post count for the topic and user
update_db_topic_post_count(topic.id, fetch_db_topic_post_count(topic.id).to_i + 1)
update_db_user_post_count(discourse_user_id, fetch_db_user_post_count(discourse_user_id).to_i + 1)
end
puts "TIJ HH post_id #{post_id}"
# Handle any attachments associated with the post
@ -1285,12 +1299,14 @@ class GossamerForumsImporter < ImportScripts::Base
puts "CREATE REPLY in topic_id #{topic.id}"
# Increment the post count for the topic
post_number = fetch_db_topic_post_numbers(topic.id).to_i + 1
update_db_topic_post_numbers(topic.id, post_number)
sqlite_mutex.sychronize do
# Increment the post count for the topic
post_number = fetch_db_topic_post_numbers(topic.id).to_i + 1
update_db_topic_post_numbers(topic.id, post_number)
# Fetch the number of views the post has had
reply_post_views = fetch_post_views(reply_row['post_id'])
# Fetch the number of views the post has had
reply_post_views = fetch_post_views(reply_row['post_id'])
end
puts "TIJ JJ post_id #{post_id}"
# Create the reply post in the existing topic