v0.46 Restore use of ConnectionPool, tweak threads, mysql pool size, timeout, add full stacktraces
This commit is contained in:
parent
18d81d3880
commit
ddf59a09a5
@ -1,7 +1,7 @@
|
||||
# Federated Computer, Inc.
|
||||
# David Sainty <saint@federated.computer> 2024 A.D.
|
||||
# Gossamer Threads to Discourse -- Migration-Import Script
|
||||
# v0.45 SQLIte mutexes only wrap around updates now
|
||||
# v0.46 Restore use of ConnectionPool, tweak threads, mysql pool size, timeout, add full stacktraces
|
||||
|
||||
require 'mysql2'
|
||||
require 'open-uri'
|
||||
@ -469,6 +469,7 @@ class GossamerForumsImporter < ImportScripts::Base
|
||||
upload
|
||||
rescue => e
|
||||
puts "FAILURE: Failed to upload attachment #{filename} for user_id #{user_id}: #{e.message}"
|
||||
puts e.backtrace.join("\n") # Print the full stack trace
|
||||
nil
|
||||
end
|
||||
end
|
||||
@ -484,6 +485,7 @@ class GossamerForumsImporter < ImportScripts::Base
|
||||
upload
|
||||
rescue => e
|
||||
puts "Failed to upload attachment #{filename} for user_id #{user_id}: #{e.message}"
|
||||
puts e.backtrace.join("\n") # Print the full stack trace
|
||||
nil
|
||||
end
|
||||
end
|
||||
@ -537,6 +539,7 @@ class GossamerForumsImporter < ImportScripts::Base
|
||||
# file
|
||||
# rescue => e
|
||||
# puts "Failed to download file from #{url}: #{e.message}"
|
||||
# puts e.backtrace.join("\n") # Print the full stack trace
|
||||
# nil
|
||||
# end
|
||||
# end
|
||||
@ -755,6 +758,7 @@ class GossamerForumsImporter < ImportScripts::Base
|
||||
png_path if File.exist?(png_path)
|
||||
rescue => e
|
||||
puts "Failed to convert image #{file_path}: #{e.message}"
|
||||
puts e.backtrace.join("\n") # Print the full stack trace
|
||||
nil
|
||||
end
|
||||
end
|
||||
@ -767,6 +771,7 @@ class GossamerForumsImporter < ImportScripts::Base
|
||||
resized_path if File.exist?(resized_path)
|
||||
rescue => e
|
||||
puts "Failed to resize image #{file_path}: #{e.message}"
|
||||
puts e.backtrace.join("\n") # Print the full stack trace
|
||||
nil
|
||||
end
|
||||
end
|
||||
@ -858,6 +863,7 @@ class GossamerForumsImporter < ImportScripts::Base
|
||||
FileUtils.copy_file(resized_image_path, resized_temp_file.path)
|
||||
rescue => e
|
||||
puts "Skipping image due to resize failure: #{temp_file.path}"
|
||||
puts e.backtrace.join("\n") # Print the full stack trace
|
||||
temp_file.close
|
||||
temp_file.unlink
|
||||
next
|
||||
@ -1069,15 +1075,19 @@ class GossamerForumsImporter < ImportScripts::Base
|
||||
# Get list of TOPICS / OP posts, i.e. post ids that have no parent / root id - SELECT post_id FROM gforum_Post WHERE post_root_id = 0;
|
||||
def threaded_topic_import
|
||||
|
||||
### # Define the connection pool inside the method
|
||||
### mariadb_pool = ConnectionPool.new(size: 20, timeout: 500) do
|
||||
### Mysql2::Client.new(
|
||||
### host: "slowtwitch.northend.network",
|
||||
### username: "admin",
|
||||
### password: "yxnh93Ybbz2Nm8#mp28zCVv",
|
||||
### database: "slowtwitch"
|
||||
### )
|
||||
### end
|
||||
# Use CachedThreadPool for dynamic thread management
|
||||
#### pool = Concurrent::CachedThreadPool.new
|
||||
pool = Concurrent::FixedThreadPool.new(7)
|
||||
|
||||
# Define the connection pool inside the method
|
||||
mariadb_pool = ConnectionPool.new(size: 14, timeout: 100) do
|
||||
Mysql2::Client.new(
|
||||
host: "slowtwitch.northend.network",
|
||||
username: "admin",
|
||||
password: "yxnh93Ybbz2Nm8#mp28zCVv",
|
||||
database: "slowtwitch"
|
||||
)
|
||||
end
|
||||
|
||||
# The query selects post_ids from gforum_Post where post_root_id is 0, meaning these posts are the topic starters (OPs).
|
||||
# Execute the query and fetch the result
|
||||
@ -1087,7 +1097,7 @@ class GossamerForumsImporter < ImportScripts::Base
|
||||
# Convert the result set to an array of post_ids
|
||||
parent_post_ids = result.map { |row| row['post_id'] }
|
||||
|
||||
parent_post_count = parent_post_ids.count
|
||||
# parent_post_count = parent_post_ids.count
|
||||
batch_size = 10 # Set our batch size for number of posts to import in a single batch
|
||||
|
||||
#### current_post_batch = 0 # Set our current batch number. This tracks the current batch of posts being processed.
|
||||
@ -1098,10 +1108,6 @@ class GossamerForumsImporter < ImportScripts::Base
|
||||
### mutex = Mutex.new # Mutex for MySQL2 operations -- disabled as this may not in fact be necessary - TBD.
|
||||
sqlite_mutex = Mutex.new # Mutex for SQLite opreations
|
||||
|
||||
# Use CachedThreadPool for dynamic thread management
|
||||
#### pool = Concurrent::CachedThreadPool.new
|
||||
pool = Concurrent::FixedThreadPool.new(11)
|
||||
|
||||
# Run until all posts have been processed.
|
||||
until is_complete
|
||||
|
||||
@ -1145,8 +1151,8 @@ class GossamerForumsImporter < ImportScripts::Base
|
||||
#### )
|
||||
#### puts "PP 22 -- #{post_id}"
|
||||
|
||||
begin
|
||||
##### mariadb_pool.with do |mysql_client|
|
||||
mariadb_pool.with do |mysql_client|
|
||||
begin
|
||||
|
||||
puts "PP 11 -- #{post_id} -- Checking MySQL connections status.."
|
||||
#### puts " FIRST Checking MySQL connection status..."
|
||||
@ -1161,7 +1167,7 @@ class GossamerForumsImporter < ImportScripts::Base
|
||||
post_status = post_status(post_id)
|
||||
if post_status.nil? || post_status == 0
|
||||
puts "Starting import for post_id #{post_id}"
|
||||
topic_import_job(post_id, sqlite_mutex) # Import topic and its replies
|
||||
topic_import_job(post_id, sqlite_mutex, mysql_client) # Import topic and its replies
|
||||
sqlite_mutex.synchronize do
|
||||
mark_post_as_complete(post_id) # Mark as complete in SQLite table
|
||||
end
|
||||
@ -1169,12 +1175,12 @@ class GossamerForumsImporter < ImportScripts::Base
|
||||
puts "Skipping post_id #{post_id}, already processed."
|
||||
end
|
||||
end
|
||||
##### end
|
||||
rescue => e
|
||||
puts "Error processing post ID #{post_id}: #{e.message}"
|
||||
sqlite_mutex.synchronize do
|
||||
mark_post_as_failed(post_id)
|
||||
end
|
||||
rescue => e
|
||||
puts "Error processing post ID #{post_id}: #{e.message}"
|
||||
puts e.backtrace.join("\n") # Print the full stack trace
|
||||
sqlite_mutex.synchronize do
|
||||
mark_post_as_failed(post_id)
|
||||
end
|
||||
#### ensure
|
||||
#### # Ensure the MariaDB connection is closed after processing
|
||||
#### mysql_client.close if mysql_client
|
||||
@ -1186,6 +1192,7 @@ class GossamerForumsImporter < ImportScripts::Base
|
||||
#### else
|
||||
#### puts " MySQL connection is valid"
|
||||
#### end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@ -1217,15 +1224,15 @@ class GossamerForumsImporter < ImportScripts::Base
|
||||
# end
|
||||
|
||||
# Method to import an entire topic, including its first post and all subsequent replies
|
||||
##### def topic_import_job(post_id, mysql_client, sqlite_mutex)
|
||||
def topic_import_job(post_id, sqlite_mutex)
|
||||
def topic_import_job(post_id, sqlite_mutex, mysql_client)
|
||||
##### def topic_import_job(post_id, sqlite_mutex)
|
||||
puts "TIJ ZZ post_id #{post_id}"
|
||||
mysql_client = Mysql2::Client.new(
|
||||
host: "slowtwitch.northend.network",
|
||||
username: "admin",
|
||||
password: "yxnh93Ybbz2Nm8#mp28zCVv",
|
||||
database: "slowtwitch"
|
||||
)
|
||||
##### mysql_client = Mysql2::Client.new(
|
||||
##### host: "slowtwitch.northend.network",
|
||||
##### username: "admin",
|
||||
##### password: "yxnh93Ybbz2Nm8#mp28zCVv",
|
||||
##### database: "slowtwitch"
|
||||
##### )
|
||||
puts " FIRST Checking MySQL connection status..."
|
||||
if mysql_client.query('SELECT 1').nil?
|
||||
puts " MySQL connection is not valid"
|
||||
@ -1293,6 +1300,7 @@ class GossamerForumsImporter < ImportScripts::Base
|
||||
else
|
||||
raise e
|
||||
end
|
||||
puts e.backtrace.join("\n") # Print the full stack trace
|
||||
end
|
||||
end
|
||||
|
||||
@ -1408,6 +1416,7 @@ class GossamerForumsImporter < ImportScripts::Base
|
||||
# rescue ActiveRecord::RecordInvalid => e
|
||||
## rescue => e
|
||||
## puts "Error importing reply with post_id #{reply_row['post_id']}: #{e.message}"
|
||||
## puts e.backtrace.join("\n") # Print the full stack trace
|
||||
## end
|
||||
end
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user