v0.52 Improve handling and look at how we can improve ActiveRecord config for PostgreSQL and avoid insufficient pool size
This commit is contained in:
parent
e37ba1c8a9
commit
af3c46e1cf
@ -1,7 +1,7 @@
|
||||
# Federated Computer, Inc.
|
||||
# David Sainty <saint@federated.computer> 2024 A.D.
|
||||
# Gossamer Threads to Discourse -- Migration-Import Script
|
||||
# v0.51 Fix mysql bug
|
||||
# v0.52 Improve handling and look at how we can improve ActiveRecord config for PostgreSQL and avoid insufficient pool size.
|
||||
|
||||
require 'mysql2'
|
||||
require 'open-uri'
|
||||
@ -30,18 +30,6 @@ class GossamerForumsImporter < ImportScripts::Base
|
||||
super
|
||||
begin
|
||||
|
||||
# # Database configuration for ActiveRecord
|
||||
# # This is not used, except for pool size... issue with our Bitnami Discourse?
|
||||
# ActiveRecord::Base.establish_connection(
|
||||
# adapter: 'postgresql',
|
||||
# database: 'slowtwitch',
|
||||
# username: 'admin',
|
||||
# password: "yxnh93Ybbz2Nm8#mp28zCVv",
|
||||
# host: 'slowtwitch.northend.network',
|
||||
# pool: 20, # Adjust based on concurrency needs
|
||||
# timeout: 5000
|
||||
# )
|
||||
|
||||
# Initialize MySQL client to connect to Gossamer Forums database
|
||||
@mysql_client = Mysql2::Client.new(
|
||||
host: "slowtwitch.northend.network",
|
||||
@ -1087,21 +1075,29 @@ class GossamerForumsImporter < ImportScripts::Base
|
||||
# Get list of TOPICS / OP posts, i.e. post ids that have no parent / root id - SELECT post_id FROM gforum_Post WHERE post_root_id = 0;
|
||||
def threaded_topic_import
|
||||
|
||||
# Define the custom connection pool settings
|
||||
custom_pool = ActiveRecord::ConnectionAdapters::ConnectionPool.new(
|
||||
ActiveRecord::Base.connection_pool.spec.to_h.merge(pool: 40, timeout: 5000)
|
||||
)
|
||||
|
||||
# Register the custom connection pool under a unique identifier
|
||||
ActiveRecord::Base.connection_handler.connection_pools['CustomPool'] = custom_pool
|
||||
|
||||
# Use CachedThreadPool for dynamic thread management
|
||||
#### pool = Concurrent::CachedThreadPool.new
|
||||
###### pool = Concurrent::FixedThreadPool.new(7)
|
||||
pool = Concurrent::FixedThreadPool.new(7)
|
||||
pool = Concurrent::FixedThreadPool.new(20)
|
||||
|
||||
# Define the connection pool inside the method
|
||||
###### mariadb_pool = ConnectionPool.new(size: 14, timeout: 100) do
|
||||
mariadb_pool = ConnectionPool.new(size: 40, timeout: 100) do
|
||||
Mysql2::Client.new(
|
||||
host: "slowtwitch.northend.network",
|
||||
username: "admin",
|
||||
password: "yxnh93Ybbz2Nm8#mp28zCVv",
|
||||
database: "slowtwitch"
|
||||
)
|
||||
end
|
||||
#### mariadb_pool = ConnectionPool.new(size: 40, timeout: 100) do
|
||||
#### Mysql2::Client.new(
|
||||
#### host: "slowtwitch.northend.network",
|
||||
#### username: "admin",
|
||||
#### password: "yxnh93Ybbz2Nm8#mp28zCVv",
|
||||
#### database: "slowtwitch"
|
||||
#### )
|
||||
#### end
|
||||
|
||||
# The query selects post_ids from gforum_Post where post_root_id is 0, meaning these posts are the topic starters (OPs).
|
||||
# Execute the query and fetch the result
|
||||
@ -1184,7 +1180,8 @@ class GossamerForumsImporter < ImportScripts::Base
|
||||
end
|
||||
|
||||
# Use connection pooling for PostgreSQL and synchronize access to shared resources
|
||||
ActiveRecord::Base.connection_pool.with_connection do
|
||||
# ActiveRecord::Base.connection_pool.with_connection do
|
||||
ActiveRecord::Base.connected_to(pool: 'CustomPool') do
|
||||
post_status = fetch_post_status(post_id)
|
||||
if post_status.nil? || post_status == 0
|
||||
puts "Starting import for post_id #{post_id}"
|
||||
@ -1203,10 +1200,28 @@ class GossamerForumsImporter < ImportScripts::Base
|
||||
sqlite_mutex.synchronize do
|
||||
mark_post_as_failed(post_id)
|
||||
end
|
||||
if e.message =~ /MySQL client is not connected/ || e.message =~ /This connection is in use by/
|
||||
sleep(1)
|
||||
puts "Reconnecting to MySQL for post ID #{post_id} due to connection loss..."
|
||||
retry
|
||||
case e.message
|
||||
when /MySQL client is not connected/, /This connection is in use by/
|
||||
puts "Lost MySQL, retrying for post ID #{post_id}..."
|
||||
# Add reconnection attempt again here... if it proves necessary?
|
||||
retries ||= 0
|
||||
retries += 1
|
||||
if retries < 5
|
||||
sleep(1)
|
||||
retry
|
||||
else
|
||||
puts "Max retries reached for post ID #{post_id}"
|
||||
end
|
||||
when /could not obtain a connection from the pool/
|
||||
puts "Connection pool exhausted, retrying for post ID #{post_id}..."
|
||||
retries ||= 0
|
||||
retries += 1
|
||||
if retries < 5
|
||||
sleep(1)
|
||||
retry
|
||||
else
|
||||
puts "Max retries reached for post ID #{post_id}"
|
||||
end
|
||||
end
|
||||
ensure
|
||||
# Ensure the MariaDB connection is closed after processing
|
||||
@ -1306,15 +1321,19 @@ class GossamerForumsImporter < ImportScripts::Base
|
||||
# Check if the topic has already been imported using the custom field 'original_gossamer_id'
|
||||
unless TopicCustomField.exists?(name: 'original_gossamer_id', value: row['post_id'])
|
||||
puts "TIJ EE post_id #{post_id}"
|
||||
ActiveRecord::Base.transaction do
|
||||
# ActiveRecord::Base.transaction do
|
||||
## ActiveRecord::Base.connected_to(pool: 'CustomPool') do
|
||||
## ActiveRecord::Base.transaction do
|
||||
# Create the new topic in Discourse
|
||||
begin
|
||||
suffix = 1
|
||||
topic_created = false
|
||||
|
||||
ActiveRecord::Base.connected_to(pool: 'CustomPool') do
|
||||
ActiveRecord::Base.transaction do
|
||||
while !topic_created
|
||||
begin
|
||||
puts "TIJ FF post_id #{post_id}"
|
||||
puts "TIJ FF post_id #{post_id}"
|
||||
puts "CREATE TOPIC unique_title #{unique_title} title #{title} discourse_user_id #{discourse_user_id} category_id #{discourse_category_id}"
|
||||
topic = Topic.create!(
|
||||
title: unique_title,
|
||||
@ -1340,6 +1359,8 @@ class GossamerForumsImporter < ImportScripts::Base
|
||||
# puts e.backtrace.join("\n") # Print the full stack trace
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# Workaround... take a copy of topic.id
|
||||
current_topic_id = topic.id
|
||||
@ -1365,8 +1386,10 @@ class GossamerForumsImporter < ImportScripts::Base
|
||||
update_db_topic_post_numbers(current_topic_id, post_number)
|
||||
end
|
||||
|
||||
puts "TIJ GG post_id #{post_id}"
|
||||
puts "TIJ GG post_id #{post_id}"
|
||||
|
||||
ActiveRecord::Base.connected_to(pool: 'CustomPool') do
|
||||
ActiveRecord::Base.transaction do
|
||||
# Create the initial post in the new topic
|
||||
post = Post.create!(
|
||||
topic_id: current_topic_id,
|
||||
@ -1379,6 +1402,8 @@ class GossamerForumsImporter < ImportScripts::Base
|
||||
)
|
||||
post.custom_fields['original_gossamer_id'] = row['post_id']
|
||||
post.save!
|
||||
end
|
||||
end
|
||||
|
||||
sqlite_mutex.synchronize do
|
||||
# Increment the post count for the topic and user
|
||||
@ -1386,9 +1411,14 @@ class GossamerForumsImporter < ImportScripts::Base
|
||||
update_db_user_post_count(discourse_user_id, fetch_db_user_post_count(discourse_user_id).to_i + 1)
|
||||
end
|
||||
|
||||
puts "TIJ HH post_id #{post_id}"
|
||||
puts "TIJ HH post_id #{post_id}"
|
||||
|
||||
ActiveRecord::Base.connected_to(pool: 'CustomPool') do
|
||||
ActiveRecord::Base.transaction do
|
||||
# Handle any attachments associated with the post
|
||||
handle_post_attachments(row['post_id'], post, discourse_user_id, mysql_client)
|
||||
end
|
||||
end
|
||||
|
||||
# Create URL mappings for the new topic
|
||||
new_url = "https://new/t/#{topic.slug}/#{current_topic_id}"
|
||||
@ -1412,7 +1442,7 @@ class GossamerForumsImporter < ImportScripts::Base
|
||||
puts "reply_user_id is NOW Former_User id #{reply_user_id} for reply post_id #{reply_row['post_id']}"
|
||||
end
|
||||
|
||||
puts "TIJ II post_id #{post_id}"
|
||||
puts "TIJ II post_id #{post_id}"
|
||||
# Sanitize and prepare the reply message for Discourse
|
||||
sanitized_reply_message = sanitize_post_message(reply_row['post_message'])
|
||||
|
||||
@ -1437,6 +1467,8 @@ class GossamerForumsImporter < ImportScripts::Base
|
||||
end
|
||||
puts "TIJ JJ post_id #{post_id} reply post_id #{reply_row['post_id']} reply_post_views #{reply_post_views || 0} post_number #{post_number} current_topic_id #{current_topic_id} reply_post_views #{reply_post_views || 0}"
|
||||
|
||||
ActiveRecord::Base.connected_to(pool: 'CustomPool') do
|
||||
ActiveRecord::Base.transaction do
|
||||
# Create the reply post in the existing topic
|
||||
post = Post.create!(
|
||||
topic_id: current_topic_id,
|
||||
@ -1449,8 +1481,10 @@ class GossamerForumsImporter < ImportScripts::Base
|
||||
)
|
||||
post.custom_fields['original_gossamer_id'] = reply_row['post_id']
|
||||
post.save!
|
||||
|
||||
puts "TIJ KK post_id #{post_id}"
|
||||
end
|
||||
end
|
||||
|
||||
puts "TIJ KK post_id #{post_id}"
|
||||
# Increment the post count for the topic and user
|
||||
sqlite_mutex.synchronize do
|
||||
update_db_topic_post_count(current_topic_id, fetch_db_topic_post_count(current_topic_id).to_i + 1)
|
||||
@ -1465,8 +1499,12 @@ class GossamerForumsImporter < ImportScripts::Base
|
||||
end
|
||||
end
|
||||
|
||||
ActiveRecord::Base.connected_to(pool: 'CustomPool') do
|
||||
ActiveRecord::Base.transaction do
|
||||
# Handle any attachments associated with the reply
|
||||
handle_post_attachments(reply_row['post_id'], post, reply_user_id, mysql_client)
|
||||
end
|
||||
end
|
||||
|
||||
# # Update the highest processed post_id in the database (thread-safe)
|
||||
# update_highest_processed_post_id_thread_safe(reply_row['post_id'])
|
||||
@ -1485,7 +1523,8 @@ class GossamerForumsImporter < ImportScripts::Base
|
||||
puts "Error importing topic with post_id #{row['post_id']}: #{e.message}"
|
||||
raise ActiveRecord::Rollback
|
||||
end
|
||||
end
|
||||
# end
|
||||
# end
|
||||
else
|
||||
puts "Topic for post_id #{row['post_id']} already exists, skipping creation."
|
||||
end
|
||||
|
||||
Loading…
Reference in New Issue
Block a user