From 855a0a9e462b7807c642d0f1c61d050f3b4abcd7 Mon Sep 17 00:00:00 2001
From: saint <saint@saint.li>
Date: Sun, 18 Aug 2024 02:41:56 +1000
Subject: [PATCH] v0.43 Move to connection pooling for MySQL problem

---
 gossamer_forums.rb | 88 +++++++++++++++++++++++++++-------------------
 1 file changed, 51 insertions(+), 37 deletions(-)

diff --git a/gossamer_forums.rb b/gossamer_forums.rb
index ececd4a..6f4fc3a 100644
--- a/gossamer_forums.rb
+++ b/gossamer_forums.rb
@@ -1,7 +1,7 @@
 # Federated Computer, Inc.
 # David Sainty <saint@federated.computer>  2024 A.D.
 # Gossamer Threads to Discourse -- Migration-Import Script
-# v0.42.1 Rewrite and simplication of concurrent-ruby support
+# v0.43 Move to connection pooling for MySQL problem
 
 require 'mysql2'
 require 'open-uri'
@@ -1064,8 +1064,17 @@ class GossamerForumsImporter < ImportScripts::Base
     pool_size
   end
 
+  # Get list of TOPICS / OP posts, i.e. post ids that have no parent / root id - SELECT post_id FROM gforum_Post WHERE post_root_id = 0;
   def threaded_topic_import
-    # Get list of TOPICS / OP posts, i.e. post ids that have no parent / root id - SELECT post_id FROM gforum_Post WHERE post_root_id = 0;
+    # Define the connection pool inside the method
+    mariadb_pool = ConnectionPool.new(size: 10, timeout: 5) do
+      Mysql2::Client.new(
+        host: "slowtwitch.northend.network",
+        username: "admin",
+        password: "yxnh93Ybbz2Nm8#mp28zCVv",
+        database: "slowtwitch"
+      )
+    end
 
     # The query selects post_ids from gforum_Post where post_root_id is 0, meaning these posts are the topic starters (OPs).
     # Execute the query and fetch the result
@@ -1124,33 +1133,38 @@ class GossamerForumsImporter < ImportScripts::Base
   
       # Submit the import job for the current post_id to the thread pool
         pool.post do
-    puts "PP 11 --  #{post_id}"
           # Initialise a new MariaDB / Mysql2 client inside of each thread
-          mysql_client = Mysql2::Client.new(
-            host: "slowtwitch.northend.network",
-            username: "admin",
-            password: "yxnh93Ybbz2Nm8#mp28zCVv",
-            database: "slowtwitch"
-          )
-    puts "PP 22 --  #{post_id}"
-    puts "    FIRST Checking MySQL connection status..."
-    if mysql_client.query('SELECT 1').nil?
-      puts "    MySQL connection is not valid"
-    else
-      puts "    MySQL connection is valid"
-    end
+####          mysql_client = Mysql2::Client.new(
+####            host: "slowtwitch.northend.network",
+####            username: "admin",
+####            password: "yxnh93Ybbz2Nm8#mp28zCVv",
+####            database: "slowtwitch"
+####          )
+####    puts "PP 22 --  #{post_id}"
+
           begin
-            # Use connection pooling for PostgreSQL and synchronize access to shared resources
-            ActiveRecord::Base.connection_pool.with_connection do
-              post_status = post_status(post_id)
-              if post_status.nil? || post_status == 0
-                puts "Starting import for post_id #{post_id}"
-                topic_import_job(post_id, mysql_client, sqlite_mutex)  # Import topic and its replies
-                sqlite_mutex.synchronize do
-                  mark_post_as_complete(post_id)  # Mark as complete in SQLite table
+            mariadb_pool.with do |mysql_client|
+
+              puts "PP 11 --  #{post_id} -- Checking MySQL connections status.."
+####    puts "    FIRST Checking MySQL connection status..."
+####    if mysql_client.query('SELECT 1').nil?
+####      puts "    MySQL connection is not valid"
+####    else
+####      puts "    MySQL connection is valid"
+####    end
+
+              # Use connection pooling for PostgreSQL and synchronize access to shared resources
+              ActiveRecord::Base.connection_pool.with_connection do
+                post_status = post_status(post_id)
+                if post_status.nil? || post_status == 0
+                  puts "Starting import for post_id #{post_id}"
+                  topic_import_job(post_id, mysql_client, sqlite_mutex)  # Import topic and its replies
+                  sqlite_mutex.synchronize do
+                    mark_post_as_complete(post_id)  # Mark as complete in SQLite table
+                  end
+                else
+                  puts "Skipping post_id #{post_id}, already processed."
                 end
-              else
-                puts "Skipping post_id #{post_id}, already processed."
               end
             end
           rescue => e
@@ -1158,17 +1172,17 @@ class GossamerForumsImporter < ImportScripts::Base
             sqlite_mutex.synchronize do
               mark_post_as_failed(post_id)
             end
-          ensure
-            # Ensure the MariaDB connection is closed after processing
-            mysql_client.close if mysql_client
-            puts "** CLOSED MariaDB client"
-    puts "PP 22 --  #{post_id}"
-    puts "    FINAL Checking MySQL connection status..."
-    if mysql_client.query('SELECT 1').nil?
-      puts "    MySQL connection is not valid"
-    else
-      puts "    MySQL connection is valid"
-    end
+####            ensure
+####            # Ensure the MariaDB connection is closed after processing
+####            mysql_client.close if mysql_client
+####            puts "** CLOSED MariaDB client"
+####    puts "PP 22 --  #{post_id}"
+####    puts "    FINAL Checking MySQL connection status..."
+####    if mysql_client.query('SELECT 1').nil?
+####      puts "    MySQL connection is not valid"
+####    else
+####      puts "    MySQL connection is valid"
+####    end
           end
         end
       end