diff --git a/.nfs00000000000b6da800000003 b/.nfs00000000000b6da800000003 new file mode 100644 index 0000000..cf1113c Binary files /dev/null and b/.nfs00000000000b6da800000003 differ diff --git a/gossamer_forums.rb b/gossamer_forums.rb index 7506250..c763455 100644 --- a/gossamer_forums.rb +++ b/gossamer_forums.rb @@ -1,7 +1,7 @@ # Federated Computer, Inc. # David Sainty 2024 A.D. # Gossamer Threads to Discourse -- Migration-Import Script -# v0.48.6 48-48-48, add handling for deleted-former user as OP poster +# v0.49 Work on Mysql brokenness require 'mysql2' require 'open-uri' @@ -30,23 +30,23 @@ class GossamerForumsImporter < ImportScripts::Base super begin - # Database configuration for ActiveRecord - # This is not used, except for pool size... issue with our Bitnami Discourse? - ActiveRecord::Base.establish_connection( - adapter: 'postgresql', - database: 'slowtwitch', - username: 'admin', - password: "yxnh93Ybbz2Nm8#mp28zCVv", - host: 'slowtwitch.northend.network', - pool: 40, # Adjust based on concurrency needs - timeout: 5000 - ) +# # Database configuration for ActiveRecord +# # This is not used, except for pool size... issue with our Bitnami Discourse? +# ActiveRecord::Base.establish_connection( +# adapter: 'postgresql', +# database: 'slowtwitch', +# username: 'admin', +# password: "yxnh93Ybbz2Nm8#mp28zCVv", +# host: 'slowtwitch.northend.network', +# pool: 20, # Adjust based on concurrency needs +# timeout: 5000 +# ) # Initialize MySQL client to connect to Gossamer Forums database @mysql_client = Mysql2::Client.new( - host: "172.99.0.10", + host: "slowtwitch.northend.network", username: "admin", - password: "x0YGLA9252iiTFQuqaM0ROX8FmQzZuUu", + password: "yxnh93Ybbz2Nm8#mp28zCVv", database: "slowtwitch" ) rescue Mysql2::Error => e @@ -1081,45 +1081,8 @@ class GossamerForumsImporter < ImportScripts::Base result ? result['post_views'] : 0 end -########## THREADING START -------------------------------------------- +########## THREADING START ############################################################################# - # Method to dynamically calculate the optimal thread pool size based on system load - def calculate_dynamic_pool_size - # Fetch current CPU load average using Sys::ProcTable.loadavg - # load_avg = Sys::ProcTable.loadavg.last # Get the 15-minute load average - # load_avg = Sys::ProcTable.loadavg - load_avg = File.read('/proc/loadavg').split - - # Calculate the pool size based on the load average - # Adjust the multiplier and threshold as needed - # pool_size = [(Concurrent.processor_count / (load_avg + 0.1)).to_i, 1].max - - # Extract the 1-minute load average from the fetched data - one_minute_load_avg = load_avg[0].to_f - - # Determine how many logical CPU cores are available on the system - cpu_count = Concurrent.processor_count - - # Log the current load and CPU information for debugging and monitoring purposes - puts "1-minute Load Average: #{one_minute_load_avg}, CPU Count: #{cpu_count}" - - # Calculate the initial pool size based on the ratio of the 1-minute load average to the number of CPUs - # This ratio gives an idea of how many threads should be running to efficiently utilize the CPU resources - initial_pool_size = (cpu_count / one_minute_load_avg).ceil - - # Ensure the pool size is at least 1 to avoid creating a pool with zero threads - initial_pool_size = 1 if initial_pool_size < 1 - - # Cap the maximum pool size to twice the number of CPUs - # This prevents overloading the system with too many threads, which could lead to diminishing returns - max_pool_size = cpu_count * 2 - - # Adjust the final pool size to be within the valid range (1 to max_pool_size) - pool_size = [[initial_pool_size, max_pool_size].min, 1].max - - puts "Calculated and adjusted dynamic pool size: #{pool_size}" # Log the dynamically adjusted pool size - pool_size - end # Get list of TOPICS / OP posts, i.e. post ids that have no parent / root id - SELECT post_id FROM gforum_Post WHERE post_root_id = 0; def threaded_topic_import @@ -1127,19 +1090,15 @@ class GossamerForumsImporter < ImportScripts::Base # Use CachedThreadPool for dynamic thread management #### pool = Concurrent::CachedThreadPool.new ###### pool = Concurrent::FixedThreadPool.new(7) - pool = Concurrent::FixedThreadPool.new(40) + pool = Concurrent::FixedThreadPool.new(7) # Define the connection pool inside the method ###### mariadb_pool = ConnectionPool.new(size: 14, timeout: 100) do - mariadb_pool = ConnectionPool.new(size: 40, timeout: 100) do -# host: "slowtwitch.northend.network", -# username: "admin", -# password: "yxnh93Ybbz2Nm8#mp28zCVv", -# database: "slowtwitch" + mariadb_pool = ConnectionPool.new(size: 14, timeout: 100) do Mysql2::Client.new( - host: "172.99.0.10", + host: "slowtwitch.northend.network", username: "admin", - password: "x0YGLA9252iiTFQuqaM0ROX8FmQzZuUu", + password: "yxnh93Ybbz2Nm8#mp28zCVv", database: "slowtwitch" ) end @@ -1153,7 +1112,7 @@ class GossamerForumsImporter < ImportScripts::Base parent_post_ids = result.map { |row| row['post_id'] } # parent_post_count = parent_post_ids.count - batch_size = 1000 # Set our batch size for number of posts to import in a single batch + batch_size = 10 # Set our batch size for number of posts to import in a single batch #### current_post_batch = 0 # Set our current batch number. This tracks the current batch of posts being processed. @@ -1197,33 +1156,32 @@ class GossamerForumsImporter < ImportScripts::Base # Submit the import job for the current post_id to the thread pool pool.post do - # Initialise a new MariaDB / Mysql2 client inside of each thread -#### mysql_client = Mysql2::Client.new( -#### host: "slowtwitch.northend.network", -#### username: "admin", -#### password: "yxnh93Ybbz2Nm8#mp28zCVv", -#### database: "slowtwitch" -#### ) #### puts "PP 22 -- #{post_id}" begin - mariadb_pool.with do |mysql_client| +####### mariadb_pool.with do |mysql_client| + # Initialise a new MariaDB / Mysql2 client inside of each thread + mysql_client = Mysql2::Client.new( + host: "slowtwitch.northend.network", + username: "admin", + password: "yxnh93Ybbz2Nm8#mp28zCVv", + database: "slowtwitch" + ) # Ensure the connection is active, otherwise reconnect puts "PP 11 -- #{post_id} -- Checking MySQL connections status.." - mysql_client.ping -## || mysql_client = Mysql2::Client.new( -## host: "slowtwitch.northend.network", -## username: "admin", -## password: "yxnh93Ybbz2Nm8#mp28zCVv", -## database: "slowtwitch" -## ) -#### puts " FIRST Checking MySQL connection status..." -#### if mysql_client.query('SELECT 1').nil? -#### puts " MySQL connection is not valid" -#### else -#### puts " MySQL connection is valid" -#### end + mysql_client.ping || mysql_client = Mysql2::Client.new( + host: "slowtwitch.northend.network", + username: "admin", + password: "yxnh93Ybbz2Nm8#mp28zCVv", + database: "slowtwitch" + ) + puts " FIRST Checking MySQL connection status..." + if mysql_client.query('SELECT 1').nil? + puts " !! MySQL connection is NOT VALID !!" + else + puts " MySQL connection is valid" + end # Use connection pooling for PostgreSQL and synchronize access to shared resources ActiveRecord::Base.connection_pool.with_connection do @@ -1238,7 +1196,7 @@ class GossamerForumsImporter < ImportScripts::Base puts "Skipping post_id #{post_id}, already processed." end end - end +####### end rescue => e puts "Error processing post ID #{post_id}: #{e.message}" puts e.backtrace.join("\n") # Print the full stack trace @@ -1250,17 +1208,10 @@ class GossamerForumsImporter < ImportScripts::Base puts "Reconnecting to MySQL for post ID #{post_id} due to connection loss..." retry end -#### ensure -#### # Ensure the MariaDB connection is closed after processing -#### mysql_client.close if mysql_client -#### puts "** CLOSED MariaDB client" -#### puts "PP 22 -- #{post_id}" -#### puts " FINAL Checking MySQL connection status..." -#### if mysql_client.query('SELECT 1').nil? -#### puts " MySQL connection is not valid" -#### else -#### puts " MySQL connection is valid" -#### end + ensure + # Ensure the MariaDB connection is closed after processing + mysql_client.close if mysql_client + puts "** CLOSED MariaDB client" end end end