saint/discourse-import_scripts
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

v0.48.2 batch size 1000, moving to local instance of MariaDB for slowtwitch db

Browse Source
This commit is contained in:
David Sainty 2024-08-19 11:37:12 +10:00
parent 93c3c72862
commit 5763ce8764
1 changed files with 24 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

60
gossamer_forums.rb
View File

@ -30,43 +30,31 @@ class GossamerForumsImporter < ImportScripts::Base
super super
begin begin
# # Database configuration for ActiveRecord
# ActiveRecord::Base.establish_connection(
# adapter: 'mysql2',
# database: 'slowtwitch',
# username: 'admin',
# password: "yxnh93Ybbz2Nm8#mp28zCVv",
# host: 'slowtwitch.northend.network',
# pool: 20, # Adjust based on concurrency needs
# timeout: 5000
# )
# # Initialize MySQL client to connect to Gossamer Forums database
# @mysql_client = Mysql2::Client.new(
# host: "slowtwitch.northend.network",
# username: "admin",
# password: "yxnh93Ybbz2Nm8#mp28zCVv",
# database: "slowtwitch"
# )
# Database configuration for ActiveRecord # Database configuration for ActiveRecord
ActiveRecord::Base.establish_connection( ActiveRecord::Base.establish_connection(
adapter: 'postgresql', adapter: 'postgresql',
database: 'slowtwitch', database: 'discourse',
username: 'admin', username: 'discourse',
password: 'yxnh93Ybbz2Nm8#mp28zCVv', password: "nhB5FWhQkjdvaD2ViRNO63dQagDnzaTn",
host: 'slowtwitch.northend.network', host: '10.0.0.2',
pool: 20, # Adjust based on concurrency needs pool: 20, # Adjust based on concurrency needs
timeout: 5000 timeout: 5000
) )
# Initialize MySQL client to connect to Gossamer Forums database # Initialize MySQL client to connect to Gossamer Forums database
@mysql_client = Mysql2::Client.new( @mysql_client = Mysql2::Client.new(
host: "172.99.0.10", host: "slowtwitch.northend.network",
username: "admin", username: "admin",
password: "x0YGLA9252iiTFQuqaM0ROX8FmQzZuUu", password: "yxnh93Ybbz2Nm8#mp28zCVv",
database: "slowtwitch" database: "slowtwitch"
) )
# # Initialize MySQL client to connect to Gossamer Forums database
# @mysql_client = Mysql2::Client.new(
# host: "172.99.0.10",
# username: "admin",
# password: "x0YGLA9252iiTFQuqaM0ROX8FmQzZuUu",
# database: "slowtwitch"
# )
rescue Mysql2::Error => e rescue Mysql2::Error => e
puts "Error connecting to MySQL: #{e.message}" puts "Error connecting to MySQL: #{e.message}"
@ -1150,22 +1138,22 @@ class GossamerForumsImporter < ImportScripts::Base
# Define the connection pool inside the method # Define the connection pool inside the method
###### mariadb_pool = ConnectionPool.new(size: 14, timeout: 100) do ###### mariadb_pool = ConnectionPool.new(size: 14, timeout: 100) do
# mariadb_pool = ConnectionPool.new(size: 24, timeout: 100) do
# Mysql2::Client.new(
# host: "slowtwitch.northend.network",
# username: "admin",
# password: "yxnh93Ybbz2Nm8#mp28zCVv",
# database: "slowtwitch"
# )
# end
mariadb_pool = ConnectionPool.new(size: 24, timeout: 100) do mariadb_pool = ConnectionPool.new(size: 24, timeout: 100) do
Mysql2::Client.new( Mysql2::Client.new(
host: "172.99.0.10", host: "slowtwitch.northend.network",
username: "admin", username: "admin",
password: "x0YGLA9252iiTFQuqaM0ROX8FmQzZuUu", password: "yxnh93Ybbz2Nm8#mp28zCVv",
database: "slowtwitch" database: "slowtwitch"
) )
end end
# mariadb_pool = ConnectionPool.new(size: 24, timeout: 100) do
# Mysql2::Client.new(
# host: "172.99.0.10",
# username: "admin",
# password: "x0YGLA9252iiTFQuqaM0ROX8FmQzZuUu",
# database: "slowtwitch"
# )
# end
# The query selects post_ids from gforum_Post where post_root_id is 0, meaning these posts are the topic starters (OPs). # The query selects post_ids from gforum_Post where post_root_id is 0, meaning these posts are the topic starters (OPs).
# Execute the query and fetch the result # Execute the query and fetch the result
@ -1177,7 +1165,7 @@ class GossamerForumsImporter < ImportScripts::Base
# parent_post_count = parent_post_ids.count # parent_post_count = parent_post_ids.count
# #
batch_size = 10 # Set our batch size for number of posts to import in a single batch batch_size = 100 # Set our batch size for number of posts to import in a single batch
#### current_post_batch = 0 # Set our current batch number. This tracks the current batch of posts being processed. #### current_post_batch = 0 # Set our current batch number. This tracks the current batch of posts being processed.