v0.2 - Add debugging

This commit is contained in:
David Sainty 2024-09-09 16:29:14 +10:00
parent 85c93af281
commit 538185095d

View File

@ -1,7 +1,7 @@
# Federated Computer, Inc. # Federated Computer, Inc.
# David Sainty <saint@federated.computer> 2024 A.D. # David Sainty <saint@federated.computer> 2024 A.D.
# Gossamer Threads to Discourse -- Correct Encoding # Gossamer Threads to Discourse -- Correct Encoding
# v0.1 New script # v0.2 Debugging
require 'mysql2' require 'mysql2'
require 'active_record' require 'active_record'
@ -31,6 +31,7 @@ class GossamerForumsCorrectEncoding < ImportScripts::Base
# Method to detect and fix text encoding # Method to detect and fix text encoding
def fix_text_encoding(text) def fix_text_encoding(text)
begin
# Detect encoding # Detect encoding
detection = CharlockHolmes::Detect.detect(text) detection = CharlockHolmes::Detect.detect(text)
original_encoding = detection[:encoding] original_encoding = detection[:encoding]
@ -44,6 +45,7 @@ class GossamerForumsCorrectEncoding < ImportScripts::Base
end end
rescue StandardError => e rescue StandardError => e
puts "Error during encoding conversion: #{e.message}" puts "Error during encoding conversion: #{e.message}"
puts e.backtrace.join("\n") # Print the full stack trace
text text
end end
@ -52,14 +54,17 @@ class GossamerForumsCorrectEncoding < ImportScripts::Base
offset = 0 offset = 0
loop do loop do
puts "OFFSET: #{offset}"
begin
posts = Post.limit(@batch_size).offset(offset) posts = Post.limit(@batch_size).offset(offset)
break if posts.empty? break if posts.empty?
posts.each do |post| posts.each do |post|
raw_content = post.raw raw_content = post.raw
puts "--> NEXT POST: post.id: #{post.id}"
fixed_content = fix_text_encoding(raw_content) fixed_content = fix_text_encoding(raw_content)
if fixed_content != raw_content if fixed_content != raw_content
puts "Updating post ##{post.id}" puts "Updating post #{post.id}"
puts "------- raw_content:\n#{raw_content}" puts "------- raw_content:\n#{raw_content}"
puts "+++++++ fixed_content:\n#{fixed_content}" puts "+++++++ fixed_content:\n#{fixed_content}"
puts "---------------------------------------------------------------------------------------------" puts "---------------------------------------------------------------------------------------------"
@ -72,6 +77,10 @@ class GossamerForumsCorrectEncoding < ImportScripts::Base
# end # end
end end
end end
rescue
puts "Error: #{e.message}"
puts e.backtrace.join("\n") # Print the full stack trace
end
offset += @batch_size offset += @batch_size
end end