From ddf59a09a51dde4426a8082c6a84238b5c368a02 Mon Sep 17 00:00:00 2001 From: saint Date: Sun, 18 Aug 2024 15:06:30 +1000 Subject: [PATCH] v0.46 Restore use of ConnectionPool, tweak threads, mysql pool size, timeout, add full stacktraces --- gossamer_forums.rb | 73 ++++++++++++++++++++++++++-------------------- 1 file changed, 41 insertions(+), 32 deletions(-) diff --git a/gossamer_forums.rb b/gossamer_forums.rb index 63f68e1..a405a5e 100644 --- a/gossamer_forums.rb +++ b/gossamer_forums.rb @@ -1,7 +1,7 @@ # Federated Computer, Inc. # David Sainty 2024 A.D. # Gossamer Threads to Discourse -- Migration-Import Script -# v0.45 SQLIte mutexes only wrap around updates now +# v0.46 Restore use of ConnectionPool, tweak threads, mysql pool size, timeout, add full stacktraces require 'mysql2' require 'open-uri' @@ -469,6 +469,7 @@ class GossamerForumsImporter < ImportScripts::Base upload rescue => e puts "FAILURE: Failed to upload attachment #{filename} for user_id #{user_id}: #{e.message}" + puts e.backtrace.join("\n") # Print the full stack trace nil end end @@ -484,6 +485,7 @@ class GossamerForumsImporter < ImportScripts::Base upload rescue => e puts "Failed to upload attachment #{filename} for user_id #{user_id}: #{e.message}" + puts e.backtrace.join("\n") # Print the full stack trace nil end end @@ -537,6 +539,7 @@ class GossamerForumsImporter < ImportScripts::Base # file # rescue => e # puts "Failed to download file from #{url}: #{e.message}" +# puts e.backtrace.join("\n") # Print the full stack trace # nil # end # end @@ -755,6 +758,7 @@ class GossamerForumsImporter < ImportScripts::Base png_path if File.exist?(png_path) rescue => e puts "Failed to convert image #{file_path}: #{e.message}" + puts e.backtrace.join("\n") # Print the full stack trace nil end end @@ -767,6 +771,7 @@ class GossamerForumsImporter < ImportScripts::Base resized_path if File.exist?(resized_path) rescue => e puts "Failed to resize image #{file_path}: #{e.message}" + puts e.backtrace.join("\n") # Print the full stack trace nil end end @@ -858,6 +863,7 @@ class GossamerForumsImporter < ImportScripts::Base FileUtils.copy_file(resized_image_path, resized_temp_file.path) rescue => e puts "Skipping image due to resize failure: #{temp_file.path}" + puts e.backtrace.join("\n") # Print the full stack trace temp_file.close temp_file.unlink next @@ -1069,15 +1075,19 @@ class GossamerForumsImporter < ImportScripts::Base # Get list of TOPICS / OP posts, i.e. post ids that have no parent / root id - SELECT post_id FROM gforum_Post WHERE post_root_id = 0; def threaded_topic_import -### # Define the connection pool inside the method -### mariadb_pool = ConnectionPool.new(size: 20, timeout: 500) do -### Mysql2::Client.new( -### host: "slowtwitch.northend.network", -### username: "admin", -### password: "yxnh93Ybbz2Nm8#mp28zCVv", -### database: "slowtwitch" -### ) -### end + # Use CachedThreadPool for dynamic thread management +#### pool = Concurrent::CachedThreadPool.new + pool = Concurrent::FixedThreadPool.new(7) + + # Define the connection pool inside the method + mariadb_pool = ConnectionPool.new(size: 14, timeout: 100) do + Mysql2::Client.new( + host: "slowtwitch.northend.network", + username: "admin", + password: "yxnh93Ybbz2Nm8#mp28zCVv", + database: "slowtwitch" + ) + end # The query selects post_ids from gforum_Post where post_root_id is 0, meaning these posts are the topic starters (OPs). # Execute the query and fetch the result @@ -1087,7 +1097,7 @@ class GossamerForumsImporter < ImportScripts::Base # Convert the result set to an array of post_ids parent_post_ids = result.map { |row| row['post_id'] } - parent_post_count = parent_post_ids.count +# parent_post_count = parent_post_ids.count batch_size = 10 # Set our batch size for number of posts to import in a single batch #### current_post_batch = 0 # Set our current batch number. This tracks the current batch of posts being processed. @@ -1098,10 +1108,6 @@ class GossamerForumsImporter < ImportScripts::Base ### mutex = Mutex.new # Mutex for MySQL2 operations -- disabled as this may not in fact be necessary - TBD. sqlite_mutex = Mutex.new # Mutex for SQLite opreations - # Use CachedThreadPool for dynamic thread management -#### pool = Concurrent::CachedThreadPool.new - pool = Concurrent::FixedThreadPool.new(11) - # Run until all posts have been processed. until is_complete @@ -1145,8 +1151,8 @@ class GossamerForumsImporter < ImportScripts::Base #### ) #### puts "PP 22 -- #{post_id}" - begin -##### mariadb_pool.with do |mysql_client| + mariadb_pool.with do |mysql_client| + begin puts "PP 11 -- #{post_id} -- Checking MySQL connections status.." #### puts " FIRST Checking MySQL connection status..." @@ -1161,7 +1167,7 @@ class GossamerForumsImporter < ImportScripts::Base post_status = post_status(post_id) if post_status.nil? || post_status == 0 puts "Starting import for post_id #{post_id}" - topic_import_job(post_id, sqlite_mutex) # Import topic and its replies + topic_import_job(post_id, sqlite_mutex, mysql_client) # Import topic and its replies sqlite_mutex.synchronize do mark_post_as_complete(post_id) # Mark as complete in SQLite table end @@ -1169,12 +1175,12 @@ class GossamerForumsImporter < ImportScripts::Base puts "Skipping post_id #{post_id}, already processed." end end -##### end - rescue => e - puts "Error processing post ID #{post_id}: #{e.message}" - sqlite_mutex.synchronize do - mark_post_as_failed(post_id) - end + rescue => e + puts "Error processing post ID #{post_id}: #{e.message}" + puts e.backtrace.join("\n") # Print the full stack trace + sqlite_mutex.synchronize do + mark_post_as_failed(post_id) + end #### ensure #### # Ensure the MariaDB connection is closed after processing #### mysql_client.close if mysql_client @@ -1186,6 +1192,7 @@ class GossamerForumsImporter < ImportScripts::Base #### else #### puts " MySQL connection is valid" #### end + end end end end @@ -1217,15 +1224,15 @@ class GossamerForumsImporter < ImportScripts::Base # end # Method to import an entire topic, including its first post and all subsequent replies -##### def topic_import_job(post_id, mysql_client, sqlite_mutex) - def topic_import_job(post_id, sqlite_mutex) + def topic_import_job(post_id, sqlite_mutex, mysql_client) +##### def topic_import_job(post_id, sqlite_mutex) puts "TIJ ZZ post_id #{post_id}" - mysql_client = Mysql2::Client.new( - host: "slowtwitch.northend.network", - username: "admin", - password: "yxnh93Ybbz2Nm8#mp28zCVv", - database: "slowtwitch" - ) +##### mysql_client = Mysql2::Client.new( +##### host: "slowtwitch.northend.network", +##### username: "admin", +##### password: "yxnh93Ybbz2Nm8#mp28zCVv", +##### database: "slowtwitch" +##### ) puts " FIRST Checking MySQL connection status..." if mysql_client.query('SELECT 1').nil? puts " MySQL connection is not valid" @@ -1293,6 +1300,7 @@ class GossamerForumsImporter < ImportScripts::Base else raise e end + puts e.backtrace.join("\n") # Print the full stack trace end end @@ -1408,6 +1416,7 @@ class GossamerForumsImporter < ImportScripts::Base # rescue ActiveRecord::RecordInvalid => e ## rescue => e ## puts "Error importing reply with post_id #{reply_row['post_id']}: #{e.message}" +## puts e.backtrace.join("\n") # Print the full stack trace ## end end