saint/discourse-import_scripts
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

v0.50 Work on MySQL brokenness

Browse Source
This commit is contained in:
David Sainty 2024-08-19 18:37:05 +10:00
parent 6b4b728751
commit 5ef554b4e1
2 changed files with 45 additions and 94 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
.nfs00000000000b6da800000003 Normal file
View File

Binary file not shown.

139
gossamer_forums.rb
View File

@ -1,7 +1,7 @@
# Federated Computer, Inc. # Federated Computer, Inc.
# David Sainty <saint@federated.computer> 2024 A.D. # David Sainty <saint@federated.computer> 2024 A.D.
# Gossamer Threads to Discourse -- Migration-Import Script # Gossamer Threads to Discourse -- Migration-Import Script
# v0.48.6 48-48-48, add handling for deleted-former user as OP poster # v0.49 Work on Mysql brokenness
require 'mysql2' require 'mysql2'
require 'open-uri' require 'open-uri'
@ -30,23 +30,23 @@ class GossamerForumsImporter < ImportScripts::Base
super super
begin begin
# Database configuration for ActiveRecord # # Database configuration for ActiveRecord
# This is not used, except for pool size... issue with our Bitnami Discourse? # # This is not used, except for pool size... issue with our Bitnami Discourse?
ActiveRecord::Base.establish_connection( # ActiveRecord::Base.establish_connection(
adapter: 'postgresql', # adapter: 'postgresql',
database: 'slowtwitch', # database: 'slowtwitch',
username: 'admin', # username: 'admin',
password: "yxnh93Ybbz2Nm8#mp28zCVv", # password: "yxnh93Ybbz2Nm8#mp28zCVv",
host: 'slowtwitch.northend.network', # host: 'slowtwitch.northend.network',
pool: 40, # Adjust based on concurrency needs # pool: 20, # Adjust based on concurrency needs
timeout: 5000 # timeout: 5000
) # )
# Initialize MySQL client to connect to Gossamer Forums database # Initialize MySQL client to connect to Gossamer Forums database
@mysql_client = Mysql2::Client.new( @mysql_client = Mysql2::Client.new(
host: "172.99.0.10", host: "slowtwitch.northend.network",
username: "admin", username: "admin",
password: "x0YGLA9252iiTFQuqaM0ROX8FmQzZuUu", password: "yxnh93Ybbz2Nm8#mp28zCVv",
database: "slowtwitch" database: "slowtwitch"
) )
rescue Mysql2::Error => e rescue Mysql2::Error => e
@ -1081,45 +1081,8 @@ class GossamerForumsImporter < ImportScripts::Base
result ? result['post_views'] : 0 result ? result['post_views'] : 0
end end
########## THREADING START -------------------------------------------- ########## THREADING START #############################################################################
# Method to dynamically calculate the optimal thread pool size based on system load
def calculate_dynamic_pool_size
# Fetch current CPU load average using Sys::ProcTable.loadavg
# load_avg = Sys::ProcTable.loadavg.last # Get the 15-minute load average
# load_avg = Sys::ProcTable.loadavg
load_avg = File.read('/proc/loadavg').split
# Calculate the pool size based on the load average
# Adjust the multiplier and threshold as needed
# pool_size = [(Concurrent.processor_count / (load_avg + 0.1)).to_i, 1].max
# Extract the 1-minute load average from the fetched data
one_minute_load_avg = load_avg[0].to_f
# Determine how many logical CPU cores are available on the system
cpu_count = Concurrent.processor_count
# Log the current load and CPU information for debugging and monitoring purposes
puts "1-minute Load Average: #{one_minute_load_avg}, CPU Count: #{cpu_count}"
# Calculate the initial pool size based on the ratio of the 1-minute load average to the number of CPUs
# This ratio gives an idea of how many threads should be running to efficiently utilize the CPU resources
initial_pool_size = (cpu_count / one_minute_load_avg).ceil
# Ensure the pool size is at least 1 to avoid creating a pool with zero threads
initial_pool_size = 1 if initial_pool_size < 1
# Cap the maximum pool size to twice the number of CPUs
# This prevents overloading the system with too many threads, which could lead to diminishing returns
max_pool_size = cpu_count * 2
# Adjust the final pool size to be within the valid range (1 to max_pool_size)
pool_size = [[initial_pool_size, max_pool_size].min, 1].max
puts "Calculated and adjusted dynamic pool size: #{pool_size}" # Log the dynamically adjusted pool size
pool_size
end
# Get list of TOPICS / OP posts, i.e. post ids that have no parent / root id - SELECT post_id FROM gforum_Post WHERE post_root_id = 0; # Get list of TOPICS / OP posts, i.e. post ids that have no parent / root id - SELECT post_id FROM gforum_Post WHERE post_root_id = 0;
def threaded_topic_import def threaded_topic_import
@ -1127,19 +1090,15 @@ class GossamerForumsImporter < ImportScripts::Base
# Use CachedThreadPool for dynamic thread management # Use CachedThreadPool for dynamic thread management
#### pool = Concurrent::CachedThreadPool.new #### pool = Concurrent::CachedThreadPool.new
###### pool = Concurrent::FixedThreadPool.new(7) ###### pool = Concurrent::FixedThreadPool.new(7)
pool = Concurrent::FixedThreadPool.new(40) pool = Concurrent::FixedThreadPool.new(7)
# Define the connection pool inside the method # Define the connection pool inside the method
###### mariadb_pool = ConnectionPool.new(size: 14, timeout: 100) do ###### mariadb_pool = ConnectionPool.new(size: 14, timeout: 100) do
mariadb_pool = ConnectionPool.new(size: 40, timeout: 100) do mariadb_pool = ConnectionPool.new(size: 14, timeout: 100) do
# host: "slowtwitch.northend.network",
# username: "admin",
# password: "yxnh93Ybbz2Nm8#mp28zCVv",
# database: "slowtwitch"
Mysql2::Client.new( Mysql2::Client.new(
host: "172.99.0.10", host: "slowtwitch.northend.network",
username: "admin", username: "admin",
password: "x0YGLA9252iiTFQuqaM0ROX8FmQzZuUu", password: "yxnh93Ybbz2Nm8#mp28zCVv",
database: "slowtwitch" database: "slowtwitch"
) )
end end
@ -1153,7 +1112,7 @@ class GossamerForumsImporter < ImportScripts::Base
parent_post_ids = result.map { |row| row['post_id'] } parent_post_ids = result.map { |row| row['post_id'] }
# parent_post_count = parent_post_ids.count # parent_post_count = parent_post_ids.count
batch_size = 1000 # Set our batch size for number of posts to import in a single batch batch_size = 10 # Set our batch size for number of posts to import in a single batch
#### current_post_batch = 0 # Set our current batch number. This tracks the current batch of posts being processed. #### current_post_batch = 0 # Set our current batch number. This tracks the current batch of posts being processed.
@ -1197,33 +1156,32 @@ class GossamerForumsImporter < ImportScripts::Base
# Submit the import job for the current post_id to the thread pool # Submit the import job for the current post_id to the thread pool
pool.post do pool.post do
# Initialise a new MariaDB / Mysql2 client inside of each thread
#### mysql_client = Mysql2::Client.new(
#### host: "slowtwitch.northend.network",
#### username: "admin",
#### password: "yxnh93Ybbz2Nm8#mp28zCVv",
#### database: "slowtwitch"
#### )
#### puts "PP 22 -- #{post_id}" #### puts "PP 22 -- #{post_id}"
begin begin
mariadb_pool.with do |mysql_client| ####### mariadb_pool.with do |mysql_client|
# Initialise a new MariaDB / Mysql2 client inside of each thread
mysql_client = Mysql2::Client.new(
host: "slowtwitch.northend.network",
username: "admin",
password: "yxnh93Ybbz2Nm8#mp28zCVv",
database: "slowtwitch"
)
# Ensure the connection is active, otherwise reconnect # Ensure the connection is active, otherwise reconnect
puts "PP 11 -- #{post_id} -- Checking MySQL connections status.." puts "PP 11 -- #{post_id} -- Checking MySQL connections status.."
mysql_client.ping mysql_client.ping || mysql_client = Mysql2::Client.new(
## || mysql_client = Mysql2::Client.new( host: "slowtwitch.northend.network",
## host: "slowtwitch.northend.network", username: "admin",
## username: "admin", password: "yxnh93Ybbz2Nm8#mp28zCVv",
## password: "yxnh93Ybbz2Nm8#mp28zCVv", database: "slowtwitch"
## database: "slowtwitch" )
## ) puts " FIRST Checking MySQL connection status..."
#### puts " FIRST Checking MySQL connection status..." if mysql_client.query('SELECT 1').nil?
#### if mysql_client.query('SELECT 1').nil? puts " !! MySQL connection is NOT VALID !!"
#### puts " MySQL connection is not valid" else
#### else puts " MySQL connection is valid"
#### puts " MySQL connection is valid" end
#### end
# Use connection pooling for PostgreSQL and synchronize access to shared resources # Use connection pooling for PostgreSQL and synchronize access to shared resources
ActiveRecord::Base.connection_pool.with_connection do ActiveRecord::Base.connection_pool.with_connection do
@ -1238,7 +1196,7 @@ class GossamerForumsImporter < ImportScripts::Base
puts "Skipping post_id #{post_id}, already processed." puts "Skipping post_id #{post_id}, already processed."
end end
end end
end ####### end
rescue => e rescue => e
puts "Error processing post ID #{post_id}: #{e.message}" puts "Error processing post ID #{post_id}: #{e.message}"
puts e.backtrace.join("\n") # Print the full stack trace puts e.backtrace.join("\n") # Print the full stack trace
@ -1250,17 +1208,10 @@ class GossamerForumsImporter < ImportScripts::Base
puts "Reconnecting to MySQL for post ID #{post_id} due to connection loss..." puts "Reconnecting to MySQL for post ID #{post_id} due to connection loss..."
retry retry
end end
#### ensure ensure
#### # Ensure the MariaDB connection is closed after processing # Ensure the MariaDB connection is closed after processing
#### mysql_client.close if mysql_client mysql_client.close if mysql_client
#### puts "** CLOSED MariaDB client" puts "** CLOSED MariaDB client"
#### puts "PP 22 -- #{post_id}"
#### puts " FINAL Checking MySQL connection status..."
#### if mysql_client.query('SELECT 1').nil?
#### puts " MySQL connection is not valid"
#### else
#### puts " MySQL connection is valid"
#### end
end end
end end
end end