saint/discourse-import_scripts
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

v0.48.8 20-20-20, efforts to deal with MySQL connectionf failure issue

Browse Source
This commit is contained in:
David Sainty 2024-08-19 16:04:15 +10:00
parent 5305063ead
commit a34885141b
2 changed files with 5 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
.gossamer_forums.rb.swp Normal file
View File

Binary file not shown.

14
gossamer_forums.rb
View File

@ -1,7 +1,7 @@
# Federated Computer, Inc. # Federated Computer, Inc.
# David Sainty <saint@federated.computer> 2024 A.D. # David Sainty <saint@federated.computer> 2024 A.D.
# Gossamer Threads to Discourse -- Migration-Import Script # Gossamer Threads to Discourse -- Migration-Import Script
# v0.48.6 48-48-48, add handling for deleted-former user as OP poster # v0.48.8 20-20-20, efforts to deal with MySQL connectionf failure issue
require 'mysql2' require 'mysql2'
require 'open-uri' require 'open-uri'
@ -38,7 +38,7 @@ class GossamerForumsImporter < ImportScripts::Base
username: 'admin', username: 'admin',
password: "yxnh93Ybbz2Nm8#mp28zCVv", password: "yxnh93Ybbz2Nm8#mp28zCVv",
host: 'slowtwitch.northend.network', host: 'slowtwitch.northend.network',
pool: 24, # Adjust based on concurrency needs pool: 20, # Adjust based on concurrency needs
timeout: 5000 timeout: 5000
) )
@ -1132,15 +1132,11 @@ class GossamerForumsImporter < ImportScripts::Base
# Use CachedThreadPool for dynamic thread management # Use CachedThreadPool for dynamic thread management
#### pool = Concurrent::CachedThreadPool.new #### pool = Concurrent::CachedThreadPool.new
###### pool = Concurrent::FixedThreadPool.new(7) ###### pool = Concurrent::FixedThreadPool.new(7)
pool = Concurrent::FixedThreadPool.new(24) pool = Concurrent::FixedThreadPool.new(20)
# Define the connection pool inside the method # Define the connection pool inside the method
###### mariadb_pool = ConnectionPool.new(size: 14, timeout: 100) do ###### mariadb_pool = ConnectionPool.new(size: 14, timeout: 100) do
mariadb_pool = ConnectionPool.new(size: 24, timeout: 100) do mariadb_pool = ConnectionPool.new(size: 20, timeout: 100) do
# host: "172.99.0.10",
# username: "admin",
# password: "x0YGLA9252iiTFQuqaM0ROX8FmQzZuUu",
# database: "slowtwitch"
Mysql2::Client.new( Mysql2::Client.new(
host: "slowtwitch.northend.network", host: "slowtwitch.northend.network",
username: "admin", username: "admin",
@ -1158,7 +1154,7 @@ class GossamerForumsImporter < ImportScripts::Base
parent_post_ids = result.map { |row| row['post_id'] } parent_post_ids = result.map { |row| row['post_id'] }
# parent_post_count = parent_post_ids.count # parent_post_count = parent_post_ids.count
batch_size = 1000 # Set our batch size for number of posts to import in a single batch batch_size = 30 # Set our batch size for number of posts to import in a single batch
#### current_post_batch = 0 # Set our current batch number. This tracks the current batch of posts being processed. #### current_post_batch = 0 # Set our current batch number. This tracks the current batch of posts being processed.