saint/discourse-import_scripts
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

v0.46 Restore use of ConnectionPool, tweak threads, mysql pool size, timeout, add full stacktraces

Browse Source
This commit is contained in:
David Sainty 2024-08-18 15:06:30 +10:00
parent 18d81d3880
commit ddf59a09a5
1 changed files with 41 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

73
gossamer_forums.rb
View File

@ -1,7 +1,7 @@
# Federated Computer, Inc. # Federated Computer, Inc.
# David Sainty <saint@federated.computer> 2024 A.D. # David Sainty <saint@federated.computer> 2024 A.D.
# Gossamer Threads to Discourse -- Migration-Import Script # Gossamer Threads to Discourse -- Migration-Import Script
# v0.45 SQLIte mutexes only wrap around updates now # v0.46 Restore use of ConnectionPool, tweak threads, mysql pool size, timeout, add full stacktraces
require 'mysql2' require 'mysql2'
require 'open-uri' require 'open-uri'
@ -469,6 +469,7 @@ class GossamerForumsImporter < ImportScripts::Base
upload upload
rescue => e rescue => e
puts "FAILURE: Failed to upload attachment #{filename} for user_id #{user_id}: #{e.message}" puts "FAILURE: Failed to upload attachment #{filename} for user_id #{user_id}: #{e.message}"
puts e.backtrace.join("\n") # Print the full stack trace
nil nil
end end
end end
@ -484,6 +485,7 @@ class GossamerForumsImporter < ImportScripts::Base
upload upload
rescue => e rescue => e
puts "Failed to upload attachment #{filename} for user_id #{user_id}: #{e.message}" puts "Failed to upload attachment #{filename} for user_id #{user_id}: #{e.message}"
puts e.backtrace.join("\n") # Print the full stack trace
nil nil
end end
end end
@ -537,6 +539,7 @@ class GossamerForumsImporter < ImportScripts::Base
# file # file
# rescue => e # rescue => e
# puts "Failed to download file from #{url}: #{e.message}" # puts "Failed to download file from #{url}: #{e.message}"
# puts e.backtrace.join("\n") # Print the full stack trace
# nil # nil
# end # end
# end # end
@ -755,6 +758,7 @@ class GossamerForumsImporter < ImportScripts::Base
png_path if File.exist?(png_path) png_path if File.exist?(png_path)
rescue => e rescue => e
puts "Failed to convert image #{file_path}: #{e.message}" puts "Failed to convert image #{file_path}: #{e.message}"
puts e.backtrace.join("\n") # Print the full stack trace
nil nil
end end
end end
@ -767,6 +771,7 @@ class GossamerForumsImporter < ImportScripts::Base
resized_path if File.exist?(resized_path) resized_path if File.exist?(resized_path)
rescue => e rescue => e
puts "Failed to resize image #{file_path}: #{e.message}" puts "Failed to resize image #{file_path}: #{e.message}"
puts e.backtrace.join("\n") # Print the full stack trace
nil nil
end end
end end
@ -858,6 +863,7 @@ class GossamerForumsImporter < ImportScripts::Base
FileUtils.copy_file(resized_image_path, resized_temp_file.path) FileUtils.copy_file(resized_image_path, resized_temp_file.path)
rescue => e rescue => e
puts "Skipping image due to resize failure: #{temp_file.path}" puts "Skipping image due to resize failure: #{temp_file.path}"
puts e.backtrace.join("\n") # Print the full stack trace
temp_file.close temp_file.close
temp_file.unlink temp_file.unlink
next next
@ -1069,15 +1075,19 @@ class GossamerForumsImporter < ImportScripts::Base
# Get list of TOPICS / OP posts, i.e. post ids that have no parent / root id - SELECT post_id FROM gforum_Post WHERE post_root_id = 0; # Get list of TOPICS / OP posts, i.e. post ids that have no parent / root id - SELECT post_id FROM gforum_Post WHERE post_root_id = 0;
def threaded_topic_import def threaded_topic_import
### # Define the connection pool inside the method # Use CachedThreadPool for dynamic thread management
### mariadb_pool = ConnectionPool.new(size: 20, timeout: 500) do #### pool = Concurrent::CachedThreadPool.new
### Mysql2::Client.new( pool = Concurrent::FixedThreadPool.new(7)
### host: "slowtwitch.northend.network",
### username: "admin", # Define the connection pool inside the method
### password: "yxnh93Ybbz2Nm8#mp28zCVv", mariadb_pool = ConnectionPool.new(size: 14, timeout: 100) do
### database: "slowtwitch" Mysql2::Client.new(
### ) host: "slowtwitch.northend.network",
### end username: "admin",
password: "yxnh93Ybbz2Nm8#mp28zCVv",
database: "slowtwitch"
)
end
# The query selects post_ids from gforum_Post where post_root_id is 0, meaning these posts are the topic starters (OPs). # The query selects post_ids from gforum_Post where post_root_id is 0, meaning these posts are the topic starters (OPs).
# Execute the query and fetch the result # Execute the query and fetch the result
@ -1087,7 +1097,7 @@ class GossamerForumsImporter < ImportScripts::Base
# Convert the result set to an array of post_ids # Convert the result set to an array of post_ids
parent_post_ids = result.map { |row| row['post_id'] } parent_post_ids = result.map { |row| row['post_id'] }
parent_post_count = parent_post_ids.count # parent_post_count = parent_post_ids.count
batch_size = 10 # Set our batch size for number of posts to import in a single batch batch_size = 10 # Set our batch size for number of posts to import in a single batch
#### current_post_batch = 0 # Set our current batch number. This tracks the current batch of posts being processed. #### current_post_batch = 0 # Set our current batch number. This tracks the current batch of posts being processed.
@ -1098,10 +1108,6 @@ class GossamerForumsImporter < ImportScripts::Base
### mutex = Mutex.new # Mutex for MySQL2 operations -- disabled as this may not in fact be necessary - TBD. ### mutex = Mutex.new # Mutex for MySQL2 operations -- disabled as this may not in fact be necessary - TBD.
sqlite_mutex = Mutex.new # Mutex for SQLite opreations sqlite_mutex = Mutex.new # Mutex for SQLite opreations
# Use CachedThreadPool for dynamic thread management
#### pool = Concurrent::CachedThreadPool.new
pool = Concurrent::FixedThreadPool.new(11)
# Run until all posts have been processed. # Run until all posts have been processed.
until is_complete until is_complete
@ -1145,8 +1151,8 @@ class GossamerForumsImporter < ImportScripts::Base
#### ) #### )
#### puts "PP 22 -- #{post_id}" #### puts "PP 22 -- #{post_id}"
begin mariadb_pool.with do |mysql_client|
##### mariadb_pool.with do |mysql_client| begin
puts "PP 11 -- #{post_id} -- Checking MySQL connections status.." puts "PP 11 -- #{post_id} -- Checking MySQL connections status.."
#### puts " FIRST Checking MySQL connection status..." #### puts " FIRST Checking MySQL connection status..."
@ -1161,7 +1167,7 @@ class GossamerForumsImporter < ImportScripts::Base
post_status = post_status(post_id) post_status = post_status(post_id)
if post_status.nil? || post_status == 0 if post_status.nil? || post_status == 0
puts "Starting import for post_id #{post_id}" puts "Starting import for post_id #{post_id}"
topic_import_job(post_id, sqlite_mutex) # Import topic and its replies topic_import_job(post_id, sqlite_mutex, mysql_client) # Import topic and its replies
sqlite_mutex.synchronize do sqlite_mutex.synchronize do
mark_post_as_complete(post_id) # Mark as complete in SQLite table mark_post_as_complete(post_id) # Mark as complete in SQLite table
end end
@ -1169,12 +1175,12 @@ class GossamerForumsImporter < ImportScripts::Base
puts "Skipping post_id #{post_id}, already processed." puts "Skipping post_id #{post_id}, already processed."
end end
end end
##### end rescue => e
rescue => e puts "Error processing post ID #{post_id}: #{e.message}"
puts "Error processing post ID #{post_id}: #{e.message}" puts e.backtrace.join("\n") # Print the full stack trace
sqlite_mutex.synchronize do sqlite_mutex.synchronize do
mark_post_as_failed(post_id) mark_post_as_failed(post_id)
end end
#### ensure #### ensure
#### # Ensure the MariaDB connection is closed after processing #### # Ensure the MariaDB connection is closed after processing
#### mysql_client.close if mysql_client #### mysql_client.close if mysql_client
@ -1186,6 +1192,7 @@ class GossamerForumsImporter < ImportScripts::Base
#### else #### else
#### puts " MySQL connection is valid" #### puts " MySQL connection is valid"
#### end #### end
end
end end
end end
end end
@ -1217,15 +1224,15 @@ class GossamerForumsImporter < ImportScripts::Base
# end # end
# Method to import an entire topic, including its first post and all subsequent replies # Method to import an entire topic, including its first post and all subsequent replies
##### def topic_import_job(post_id, mysql_client, sqlite_mutex) def topic_import_job(post_id, sqlite_mutex, mysql_client)
def topic_import_job(post_id, sqlite_mutex) ##### def topic_import_job(post_id, sqlite_mutex)
puts "TIJ ZZ post_id #{post_id}" puts "TIJ ZZ post_id #{post_id}"
mysql_client = Mysql2::Client.new( ##### mysql_client = Mysql2::Client.new(
host: "slowtwitch.northend.network", ##### host: "slowtwitch.northend.network",
username: "admin", ##### username: "admin",
password: "yxnh93Ybbz2Nm8#mp28zCVv", ##### password: "yxnh93Ybbz2Nm8#mp28zCVv",
database: "slowtwitch" ##### database: "slowtwitch"
) ##### )
puts " FIRST Checking MySQL connection status..." puts " FIRST Checking MySQL connection status..."
if mysql_client.query('SELECT 1').nil? if mysql_client.query('SELECT 1').nil?
puts " MySQL connection is not valid" puts " MySQL connection is not valid"
@ -1293,6 +1300,7 @@ class GossamerForumsImporter < ImportScripts::Base
else else
raise e raise e
end end
puts e.backtrace.join("\n") # Print the full stack trace
end end
end end
@ -1408,6 +1416,7 @@ class GossamerForumsImporter < ImportScripts::Base
# rescue ActiveRecord::RecordInvalid => e # rescue ActiveRecord::RecordInvalid => e
## rescue => e ## rescue => e
## puts "Error importing reply with post_id #{reply_row['post_id']}: #{e.message}" ## puts "Error importing reply with post_id #{reply_row['post_id']}: #{e.message}"
## puts e.backtrace.join("\n") # Print the full stack trace
## end ## end
end end