From 90a2d2fa37384a45ccd3783b7c5f60398cf83b6c Mon Sep 17 00:00:00 2001 From: saint Date: Sun, 18 Aug 2024 04:39:51 +1000 Subject: [PATCH] v0.44 Try removing connection_pool and starting MySQL within thread method and fix handle_post_attachment --- gossamer_forums.rb | 61 ++++++++++++++++++++++++++++------------------ 1 file changed, 37 insertions(+), 24 deletions(-) diff --git a/gossamer_forums.rb b/gossamer_forums.rb index fdd7102..154f9c8 100644 --- a/gossamer_forums.rb +++ b/gossamer_forums.rb @@ -1,7 +1,7 @@ # Federated Computer, Inc. # David Sainty 2024 A.D. # Gossamer Threads to Discourse -- Migration-Import Script -# v0.43 Move to connection pooling for MySQL problem +# v0.44 Try removing connection_pool and starting MySQL within thread method and fix handle_post_attachment require 'mysql2' require 'open-uri' @@ -490,8 +490,8 @@ class GossamerForumsImporter < ImportScripts::Base # Helper method to handle post attachments - def handle_post_attachments(gossamer_post_id, post, user_id) - execute_query("SELECT * FROM gforum_PostAttachment WHERE post_id_fk = #{gossamer_post_id}").each do |att_row| + def handle_post_attachments(gossamer_post_id, post, user_id, mysql_client) + execute_query_concurrent("SELECT * FROM gforum_PostAttachment WHERE post_id_fk = #{gossamer_post_id}", mysql_client).each do |att_row| attachment_url = "https://forum.slowtwitch.com/forum/?do=post_attachment;postatt_id=#{att_row['postatt_id']}" puts "Handling attachment: #{attachment_url}" attachment_data = download_attachment(attachment_url) @@ -1068,15 +1068,16 @@ class GossamerForumsImporter < ImportScripts::Base # Get list of TOPICS / OP posts, i.e. post ids that have no parent / root id - SELECT post_id FROM gforum_Post WHERE post_root_id = 0; def threaded_topic_import - # Define the connection pool inside the method - mariadb_pool = ConnectionPool.new(size: 20, timeout: 500) do - Mysql2::Client.new( - host: "slowtwitch.northend.network", - username: "admin", - password: "yxnh93Ybbz2Nm8#mp28zCVv", - database: "slowtwitch" - ) - end + +### # Define the connection pool inside the method +### mariadb_pool = ConnectionPool.new(size: 20, timeout: 500) do +### Mysql2::Client.new( +### host: "slowtwitch.northend.network", +### username: "admin", +### password: "yxnh93Ybbz2Nm8#mp28zCVv", +### database: "slowtwitch" +### ) +### end # The query selects post_ids from gforum_Post where post_root_id is 0, meaning these posts are the topic starters (OPs). # Execute the query and fetch the result @@ -1145,7 +1146,7 @@ class GossamerForumsImporter < ImportScripts::Base #### puts "PP 22 -- #{post_id}" begin - mariadb_pool.with do |mysql_client| +##### mariadb_pool.with do |mysql_client| puts "PP 11 -- #{post_id} -- Checking MySQL connections status.." #### puts " FIRST Checking MySQL connection status..." @@ -1160,7 +1161,7 @@ class GossamerForumsImporter < ImportScripts::Base post_status = post_status(post_id) if post_status.nil? || post_status == 0 puts "Starting import for post_id #{post_id}" - topic_import_job(post_id, mysql_client, sqlite_mutex) # Import topic and its replies + topic_import_job(post_id, sqlite_mutex) # Import topic and its replies sqlite_mutex.synchronize do mark_post_as_complete(post_id) # Mark as complete in SQLite table end @@ -1168,7 +1169,7 @@ class GossamerForumsImporter < ImportScripts::Base puts "Skipping post_id #{post_id}, already processed." end end - end +##### end rescue => e puts "Error processing post ID #{post_id}: #{e.message}" sqlite_mutex.synchronize do @@ -1216,12 +1217,21 @@ class GossamerForumsImporter < ImportScripts::Base # end # Method to import an entire topic, including its first post and all subsequent replies - def topic_import_job(post_id, mysql_client, sqlite_mutex) - #Here is where you can import the entire topic - #Get post -- SELECT post_id, user_id_fk, forum_id_fk, post_root_id, post_subject, post_time, post_message, post_father_id, post_replies FROM gforum_Post WHERE post_id = post_id - #check if exists, create if not - #get children, create -- SELECT post_id, user_id_fk, forum_id_fk, post_root_id, post_subject, post_time, post_message, post_father_id, post_replies FROM gforum_Post WHERE post_root_id = post_id - #this parts needs to be synchronously to avoid race conditions +##### def topic_import_job(post_id, mysql_client, sqlite_mutex) + def topic_import_job(post_id, sqlite_mutex) + puts "TIJ ZZ post_id #{post_id}" + mysql_client = Mysql2::Client.new( + host: "slowtwitch.northend.network", + username: "admin", + password: "yxnh93Ybbz2Nm8#mp28zCVv", + database: "slowtwitch" + ) + puts " FIRST Checking MySQL connection status..." + if mysql_client.query('SELECT 1').nil? + puts " MySQL connection is not valid" + else + puts " MySQL connection is valid" + end puts "TIJ AA post_id #{post_id}" # Fetch the post data for the given post_id (this is the first post in the topic) row = execute_query_concurrent("SELECT post_id, user_id_fk, forum_id_fk, post_root_id, post_subject, post_time, post_message, post_father_id, post_replies FROM gforum_Post WHERE post_id = #{post_id}", mysql_client).first @@ -1251,7 +1261,7 @@ class GossamerForumsImporter < ImportScripts::Base # Check if the topic has already been imported using the custom field 'original_gossamer_id' unless TopicCustomField.exists?(name: 'original_gossamer_id', value: row['post_id']) - puts "TIJ EE post_id #{post_id}" + puts "TIJ EE post_id #{post_id}" ActiveRecord::Base.transaction do # Create the new topic in Discourse begin @@ -1330,7 +1340,7 @@ class GossamerForumsImporter < ImportScripts::Base puts "TIJ HH post_id #{post_id}" # Handle any attachments associated with the post - handle_post_attachments(row['post_id'], post, discourse_user_id) + handle_post_attachments(row['post_id'], post, discourse_user_id, mysql_client) # Create URL mappings for the new topic new_url = "https://new/t/#{topic.slug}/#{topic.id}" @@ -1386,7 +1396,7 @@ class GossamerForumsImporter < ImportScripts::Base end # Handle any attachments associated with the reply - handle_post_attachments(reply_row['post_id'], post, reply_user_id) + handle_post_attachments(reply_row['post_id'], post, reply_user_id, mysql_client) # # Update the highest processed post_id in the database (thread-safe) # update_highest_processed_post_id_thread_safe(reply_row['post_id']) @@ -1408,6 +1418,9 @@ class GossamerForumsImporter < ImportScripts::Base else puts "Topic for post_id #{row['post_id']} already exists, skipping creation." end + + puts " LAST Removing MySQL connection" + mysql_client.close # if mysql_client end