saint/discourse-import_scripts
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

v0.49 Work on MySQL brokenness

Browse Source
This commit is contained in:
David Sainty 2024-08-19 18:16:35 +10:00
parent e9f5da7d02
commit cdd512bf06
1 changed files with 3 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
gossamer_forums.rb
View File

@ -1081,45 +1081,8 @@ class GossamerForumsImporter < ImportScripts::Base
result ? result['post_views'] : 0
end
########## THREADING START --------------------------------------------
########## THREADING START #############################################################################
# Method to dynamically calculate the optimal thread pool size based on system load
def calculate_dynamic_pool_size
# Fetch current CPU load average using Sys::ProcTable.loadavg
# load_avg = Sys::ProcTable.loadavg.last # Get the 15-minute load average
# load_avg = Sys::ProcTable.loadavg
load_avg = File.read('/proc/loadavg').split
# Calculate the pool size based on the load average
# Adjust the multiplier and threshold as needed
# pool_size = [(Concurrent.processor_count / (load_avg + 0.1)).to_i, 1].max
# Extract the 1-minute load average from the fetched data
one_minute_load_avg = load_avg[0].to_f
# Determine how many logical CPU cores are available on the system
cpu_count = Concurrent.processor_count
# Log the current load and CPU information for debugging and monitoring purposes
puts "1-minute Load Average: #{one_minute_load_avg}, CPU Count: #{cpu_count}"
# Calculate the initial pool size based on the ratio of the 1-minute load average to the number of CPUs
# This ratio gives an idea of how many threads should be running to efficiently utilize the CPU resources
initial_pool_size = (cpu_count / one_minute_load_avg).ceil
# Ensure the pool size is at least 1 to avoid creating a pool with zero threads
initial_pool_size = 1 if initial_pool_size < 1
# Cap the maximum pool size to twice the number of CPUs
# This prevents overloading the system with too many threads, which could lead to diminishing returns
max_pool_size = cpu_count * 2
# Adjust the final pool size to be within the valid range (1 to max_pool_size)
pool_size = [[initial_pool_size, max_pool_size].min, 1].max
puts "Calculated and adjusted dynamic pool size: #{pool_size}" # Log the dynamically adjusted pool size
pool_size
end
# Get list of TOPICS / OP posts, i.e. post ids that have no parent / root id - SELECT post_id FROM gforum_Post WHERE post_root_id = 0;
def threaded_topic_import
@ -1127,11 +1090,11 @@ class GossamerForumsImporter < ImportScripts::Base
# Use CachedThreadPool for dynamic thread management
#### pool = Concurrent::CachedThreadPool.new
###### pool = Concurrent::FixedThreadPool.new(7)
pool = Concurrent::FixedThreadPool.new(10)
pool = Concurrent::FixedThreadPool.new(7)
# Define the connection pool inside the method
###### mariadb_pool = ConnectionPool.new(size: 14, timeout: 100) do
mariadb_pool = ConnectionPool.new(size: 10, timeout: 100) do
mariadb_pool = ConnectionPool.new(size: 14, timeout: 100) do
Mysql2::Client.new(
host: "slowtwitch.northend.network",
username: "admin",