v0.41.1 Fixes for FULL concurrency support
This commit is contained in:
parent
8dd8e2f72e
commit
4f608ce87f
@ -1,7 +1,7 @@
|
|||||||
# Federated Computer, Inc.
|
# Federated Computer, Inc.
|
||||||
# David Sainty <saint@federated.computer> 2024 A.D.
|
# David Sainty <saint@federated.computer> 2024 A.D.
|
||||||
# Gossamer Threads to Discourse -- Migration-Import Script
|
# Gossamer Threads to Discourse -- Migration-Import Script
|
||||||
# v0.41 Further improve FULL concurrency support, for both MySQL/MariaDB _and_ importantly, the PostGreSQL Discourse DB additions and changes with ActiveRecord connection pooling and Mutex
|
# v0.41.1 Fixes for FULL concurrency support
|
||||||
|
|
||||||
require 'mysql2'
|
require 'mysql2'
|
||||||
require 'open-uri'
|
require 'open-uri'
|
||||||
@ -299,6 +299,11 @@ class GossamerForumsImporter < ImportScripts::Base
|
|||||||
@mysql_client.query(query, as: :hash)
|
@mysql_client.query(query, as: :hash)
|
||||||
end
|
end
|
||||||
|
|
||||||
|
# Execute an SQL query on the Gossamer Forums database
|
||||||
|
def execute_query_concurrent(query, mysql_client)
|
||||||
|
mysql_client.query(query, as: :hash)
|
||||||
|
end
|
||||||
|
|
||||||
# Sanitize the username to meet Discourse's requirements
|
# Sanitize the username to meet Discourse's requirements
|
||||||
def sanitize_username(original_username, email, real_name)
|
def sanitize_username(original_username, email, real_name)
|
||||||
# original_username = username
|
# original_username = username
|
||||||
@ -1110,17 +1115,17 @@ class GossamerForumsImporter < ImportScripts::Base
|
|||||||
# Use connection ppoling for PostgreSQL and synchronize access to shared resources
|
# Use connection ppoling for PostgreSQL and synchronize access to shared resources
|
||||||
ActiveRecord::Base.connection_pool.with_connection do
|
ActiveRecord::Base.connection_pool.with_connection do
|
||||||
mutex.synchronize do
|
mutex.synchronize do
|
||||||
# begin
|
begin
|
||||||
puts "Processing post ID: #{post_id}"
|
puts "Processing post ID: #{post_id}"
|
||||||
topic_import_job(post_id, mysql_client) # Import topic and its replies
|
topic_import_job(post_id, mysql_client) # Import topic and its replies
|
||||||
mark_post_as_complete(post_id) # Mark as complete in SQLite table
|
mark_post_as_complete(post_id) # Mark as complete in SQLite table
|
||||||
# rescue => e
|
rescue => e
|
||||||
puts "Error processing post ID #{post_id}: #{e.message}"
|
puts "Error processing post ID #{post_id}: #{e.message}"
|
||||||
mark_post_as_failed(post_id)
|
mark_post_as_failed(post_id)
|
||||||
# ensure
|
ensure
|
||||||
# Ensure the MariaDB connection is closed after processing
|
# Ensure the MariaDB connection is closed after processing
|
||||||
mysql_client.close if mysql_client
|
mysql_client.close if mysql_client
|
||||||
# end
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
@ -1159,7 +1164,7 @@ class GossamerForumsImporter < ImportScripts::Base
|
|||||||
#this parts needs to be synchronously to avoid race conditions
|
#this parts needs to be synchronously to avoid race conditions
|
||||||
puts "TIJ AA post_id #{post_id}"
|
puts "TIJ AA post_id #{post_id}"
|
||||||
# Fetch the post data for the given post_id (this is the first post in the topic)
|
# Fetch the post data for the given post_id (this is the first post in the topic)
|
||||||
row = execute_query("SELECT post_id, user_id_fk, forum_id_fk, post_root_id, post_subject, post_time, post_message, post_father_id, post_replies FROM gforum_Post WHERE post_id = #{post_id}", mysql_client).first
|
row = execute_query_concurrent("SELECT post_id, user_id_fk, forum_id_fk, post_root_id, post_subject, post_time, post_message, post_father_id, post_replies FROM gforum_Post WHERE post_id = #{post_id}", mysql_client).first
|
||||||
|
|
||||||
puts "TIJ BB post_id #{post_id}"
|
puts "TIJ BB post_id #{post_id}"
|
||||||
# Early return if the post data is not found
|
# Early return if the post data is not found
|
||||||
@ -1263,7 +1268,7 @@ class GossamerForumsImporter < ImportScripts::Base
|
|||||||
insert_url_mapping(row['post_id'], new_url, unique_title)
|
insert_url_mapping(row['post_id'], new_url, unique_title)
|
||||||
|
|
||||||
# Fetch and import all replies to this topic
|
# Fetch and import all replies to this topic
|
||||||
replies = execute_query("SELECT post_id, user_id_fk, post_message, post_time, FROM gforum_Post WHERE post_root_id = #{post_id} ORDER BY post_time ASC")
|
replies = execute_query_concurrent("SELECT post_id, user_id_fk, post_message, post_time, FROM gforum_Post WHERE post_root_id = #{post_id} ORDER BY post_time ASC", mysql_client)
|
||||||
|
|
||||||
# Import each reply sequentially
|
# Import each reply sequentially
|
||||||
replies.each do |reply_row|
|
replies.each do |reply_row|
|
||||||
|
Loading…
Reference in New Issue
Block a user