# Federated Computer, Inc. # David Sainty 2024 A.D. # Gossamer Threads to Discourse -- Migration-Import Script # v0.60 Final tweak for topic and user count updating for last public posts require 'mysql2' require 'open-uri' require 'net/http' require 'tempfile' require 'sqlite3' require 'digest' require 'fileutils' require 'csv' require 'time' require 'concurrent' require 'sys/proctable' require 'active_record' require 'connection_pool' require File.expand_path("../../../../config/environment", __FILE__) # require_relative '../base' require File.expand_path("../../../../script/import_scripts/base", __FILE__) class GossamerForumsImporter < ImportScripts::Base def initialize super begin # Initialize MySQL client to connect to Gossamer Forums database @mysql_client = Mysql2::Client.new( host: "slowtwitch.northend.network", username: "admin", password: "yxnh93Ybbz2Nm8#mp28zCVv", database: "slowtwitch" ) rescue Mysql2::Error => e puts "Error connecting to MySQL: #{e.message}" exit 1 end # # Create a mapping of old Gossamer user IDs to new Discourse user IDs # @user_id_map = {} initialize_sqlite_id_name_url_db end def initialize_sqlite_id_name_url_db @db = SQLite3::Database.new '/bitnami/discourse/sqlite/id_name_url_map.db' ###### ONLY when we need to clear the url_map and topic_import_status .... e.g. if reimporting topics-posts from scratch # @db.execute <<-SQL # DROP TABLE IF EXISTS url_map; # SQL # @db.execute <<-SQL # DROP TABLE IF EXISTS topic_last_post_time; # SQL # @db.execute <<-SQL # DROP TABLE IF EXISTS topic_post_count; # SQL # @db.execute <<-SQL # DROP TABLE IF EXISTS user_topic_count; # SQL # @db.execute <<-SQL # DROP TABLE IF EXISTS user_post_count; # SQL # @db.execute <<-SQL # DROP TABLE IF EXISTS topic_last_post_user; # SQL # @db.execute <<-SQL # DROP TABLE IF EXISTS topic_post_numbers; # SQL # @db.execute <<-SQL # DROP TABLE IF EXISTS highest_processed_post_id; # SQL # @db.execute <<-SQL # DROP TABLE IF EXISTS topic_import_status; # SQL ####### # USER IMPORT - map of old to new user ids, used for topic-post import. @db.execute <<-SQL CREATE TABLE IF NOT EXISTS user_id_map ( old_user_id INTEGER PRIMARY KEY, new_user_id INTEGER ); SQL # CATEGORY IMPORT - map of old to new category ids, used for topic-post import. @db.execute <<-SQL CREATE TABLE IF NOT EXISTS category_id_map ( old_category_id INTEGER PRIMARY KEY, new_category_id INTEGER ); SQL # USER IMPORT - map of old to new usernames for SENDING MIGRATION EMAIL @db.execute <<-SQL CREATE TABLE IF NOT EXISTS username_map ( id INTEGER PRIMARY KEY, old_username TEXT, new_username TEXT, email TEXT, real_name TEXT ); SQL # POST IMPORT - Generate a map of old_post_id (Gossamer) and the new URL for a WEB SERVER REDIRECT FILE @db.execute <<-SQL CREATE TABLE IF NOT EXISTS url_map ( old_post_id INTEGER PRIMARY KEY, new_url TEXT, title TEXT ); SQL # POST IMPORT - For each topic, the time of the last / most recent post/reply @db.execute <<-SQL CREATE TABLE IF NOT EXISTS topic_last_post_time_final11 ( topic_id INTEGER PRIMARY KEY, last_post_time INTEGER ); SQL # POST IMPORT - For each topic, increment post_count as we add posts. @db.execute <<-SQL CREATE TABLE IF NOT EXISTS topic_post_count_final11 ( topic_id INTEGER PRIMARY KEY, post_count INTEGER DEFAULT 0 ); SQL # POST IMPORT - For each user (_id), increment topic_count as we add topics (to see total topics per user) @db.execute <<-SQL CREATE TABLE IF NOT EXISTS user_topic_count_final11 ( user_id INTEGER PRIMARY KEY, topic_count INTEGER DEFAULT 0 ); SQL # POST IMPORT - For each user (_id), increment post_count as we add posts (to see total posts per user) @db.execute <<-SQL CREATE TABLE IF NOT EXISTS user_post_count_final11 ( user_id INTEGER PRIMARY KEY, post_count INTEGER DEFAULT 0 ); SQL # POST IMPORT - For each topic, the user_id for the last poster / replier @db.execute <<-SQL CREATE TABLE IF NOT EXISTS topic_last_post_user_final11 ( topic_id INTEGER PRIMARY KEY, user_id INTEGER ); SQL # POST IMPORT - The number of posts in a given topic, incremented as we add a new reply post to a topic. @db.execute <<-SQL CREATE TABLE IF NOT EXISTS topic_post_numbers_final11 ( topic_id INTEGER PRIMARY KEY, post_number INTEGER DEFAULT 0 ); SQL # POST IMPORT - Record perssitent integer value for highest processed post id @db.execute <<-SQL CREATE TABLE IF NOT EXISTS highest_processed_post_id ( id INTEGER PRIMARY KEY CHECK (id = 1), post_id INTEGER ); SQL # PERSONAL MESSAGE IMPORT - Record perssitent integer value for highest processed personal id @db.execute <<-SQL CREATE TABLE IF NOT EXISTS highest_processed_personal_id ( id INTEGER PRIMARY KEY CHECK (id = 1), personal_id INTEGER ); SQL # POST IMPORT - For each topic (topic post ID) record status 0 fail or 1 success @db.execute <<-SQL CREATE TABLE IF NOT EXISTS topic_import_status ( post_id INTEGER PRIMARY KEY, status INTEGER DEFAULT 0 ); SQL end def insert_user_id_mapping(old_user_id, new_user_id) @db.execute("INSERT OR REPLACE INTO user_id_map (old_user_id, new_user_id) VALUES (?, ?)", [old_user_id, new_user_id]) end def fetch_user_id_mapping(old_user_id) @db.get_first_value("SELECT new_user_id FROM user_id_map WHERE old_user_id = ?", [old_user_id]) end def insert_category_id_mapping(old_category_id, new_category_id) @db.execute("INSERT OR REPLACE INTO category_id_map (old_category_id, new_category_id) VALUES (?, ?)", [old_category_id, new_category_id]) end def fetch_category_id_mapping(old_category_id) @db.get_first_value("SELECT new_category_id FROM category_id_map WHERE old_category_id = ?", [old_category_id]) end def insert_username_mapping(old_username, new_username, email, real_name) @db.execute("INSERT OR REPLACE INTO username_map (old_username, new_username, email, real_name) VALUES (?, ?, ?, ?)", [old_username, new_username, email, real_name]) end # Define a method to export the username mapping table to a CSV file def export_username_mapping_to_csv(filename) CSV.open(filename, 'w') do |csv| # Add headers csv << ['Old Username', 'New Username', 'Email', 'Full Name'] # Fetch data from the database @db.execute("SELECT old_username, new_username, email, real_name FROM username_map") do |row| csv << row end end puts "Exported changed username mappings to #{filename}" end # Insert a URL mapping into the SQLite database def insert_url_mapping(old_post_id, new_url, title) @db.execute("INSERT OR REPLACE INTO url_map (old_post_id, new_url, title) VALUES (?, ?, ?)", [old_post_id, new_url, title]) end # Export the URL mappings to a CSV file def export_url_mapping_to_csv(filename) CSV.open(filename, "w") do |csv| # Add headers csv << ["Old Post ID", "New URL", "Title"] @db.execute("SELECT old_post_id, new_url, title FROM url_map") do |row| csv << row end end puts "Exported URL mappings to #{filename}" end # Method to create Nginx rewrite rules file def export_nginx_rewrite_rules(filename) File.open(filename, "w") do |file| @db.execute("SELECT old_post_id, new_url FROM url_map") do |row| old_post_id, new_url = row file.puts "rewrite ^/forum/.*P#{old_post_id}/$ #{new_url} permanent;" end end end # Fetch the highest old_post_id from the url_map table def fetch_highest_old_post_id @db.get_first_value("SELECT MAX(old_post_id) FROM url_map") end # Helper methods to interact with the SQLite database for persisting topic-post related values def fetch_db(table, key) @db.get_first_value("SELECT value FROM #{table} WHERE key = ?", [key]) end def update_db(table, key, value) @db.execute("INSERT OR REPLACE INTO #{table} (key, value) VALUES (?, ?)", [key, value]) end def fetch_db_topic_last_post_time(topic_id) @db.get_first_value("SELECT last_post_time FROM topic_last_post_time_final11 WHERE topic_id = ?", [topic_id]) end def fetch_db_topic_last_post_user(topic_id) @db.get_first_value("SELECT user_id FROM topic_last_post_user_final11 WHERE topic_id = ?", [topic_id]) end def fetch_db_topic_post_count(topic_id) @db.get_first_value("SELECT post_count FROM topic_post_count_final11 WHERE topic_id = ?", [topic_id]) end def fetch_db_user_topic_count(user_id) @db.get_first_value("SELECT topic_count FROM user_topic_count_final11 WHERE user_id = ?", [user_id]) end def fetch_db_user_post_count(user_id) @db.get_first_value("SELECT post_count FROM user_post_count_final11 WHERE user_id = ?", [user_id]) end def fetch_db_topic_post_numbers(topic_id) # @db.get_first_value "SELECT post_number FROM topic_post_numbers_final11 WHERE topic_id = ?", topic_id @db.get_first_value("SELECT post_number FROM topic_post_numbers_final11 WHERE topic_id = ?", [topic_id]) end def update_db_topic_last_post_time(topic_id, last_post_time) # puts "zzzzzzzzz1 #{topic_id} #{last_post_time}" # @db.execute "INSERT OR REPLACE INTO topic_last_post_time_final11 (topic_id, last_post_time) VALUES (?, ?)", topic_id, last_post_time @db.execute("INSERT OR REPLACE INTO topic_last_post_time_final11 (topic_id, last_post_time) VALUES (?, ?)", [topic_id, last_post_time]) end def update_db_topic_last_post_user(topic_id, user_id) @db.execute("INSERT OR REPLACE INTO topic_last_post_user_final11 (topic_id, user_id) VALUES (?, ?)", [topic_id, user_id]) end def update_db_topic_post_count(topic_id, post_count) @db.execute("INSERT OR REPLACE INTO topic_post_count_final11 (topic_id, post_count) VALUES (?, ?)", [topic_id, post_count]) end def update_db_user_topic_count(user_id, topic_count) @db.execute("INSERT OR REPLACE INTO user_topic_count_final11 (user_id, topic_count) VALUES (?, ?)", [user_id, topic_count]) end def update_db_user_post_count(user_id, post_count) @db.execute("INSERT OR REPLACE INTO user_post_count_final11 (user_id, post_count) VALUES (?, ?)", [user_id, post_count]) end def update_db_topic_post_numbers(topic_id, post_number) @db.execute("INSERT OR REPLACE INTO topic_post_numbers_final11 (topic_id, post_number) VALUES (?, ?)", [topic_id, post_number]) end # Fetch the highest processed post_id from the highest_processed_post_id table def fetch_highest_processed_post_id @db.get_first_value("SELECT post_id FROM highest_processed_post_id WHERE id = 1") end # Update the highest processed post_id in the highest_processed_post_id table def update_highest_processed_post_id(post_id) @db.execute("INSERT OR REPLACE INTO highest_processed_post_id (id, post_id) VALUES (1, ?)", [post_id]) end # Fetch the highest processed personal_id from the highest_processed_personal_id table def fetch_highest_processed_personal_id @db.get_first_value("SELECT personal_id FROM highest_processed_personal_id WHERE id = 1") end # Update the highest processed personal_id in the highest_processed_personal_id table def update_highest_processed_personal_id(personal_id) @db.execute("INSERT OR REPLACE INTO highest_processed_personal_id (id, personal_id) VALUES (1, ?)", [personal_id]) end # Check if post_id exists and its status def fetch_post_status(post_id) result = @db.execute("SELECT status FROM topic_import_status WHERE post_id = ?", [post_id]).flatten.first result.nil? ? nil : result.to_i end # Mark post_id as complete def mark_post_as_complete(post_id) @db.execute("INSERT OR REPLACE INTO topic_import_status (post_id, status) VALUES (?, 1)", [post_id]) end # Mark post_id as failed def mark_post_as_failed(post_id) @db.execute("INSERT OR REPLACE INTO topic_import_status (post_id, status) VALUES (?, 0)", [post_id]) end # Execute an SQL query on the Gossamer Forums database def execute_query(query) @mysql_client.query(query, as: :hash) end # Execute an SQL query on the Gossamer Forums database def execute_query_concurrent(query, mysql_client) mysql_client.query(query, as: :hash) end # Sanitize the username to meet Discourse's requirements def sanitize_username(original_username, email, real_name) # original_username = username sanitized_username = original_username.gsub(/[^a-zA-Z0-9._-]/, '_') sanitized_username = "#{sanitized_username}." if sanitized_username.length < 2 # Allow two-character usernames sanitized_username = sanitized_username[0, 20] if sanitized_username.length > 20 firststep_sanitized = sanitized_username existing_user = User.find_by(username: sanitized_username) if existing_user if existing_user.email.downcase == email.downcase && existing_user.name == real_name # The existing user with the username the same as the current proposed sanitised name _is_ the same person... return sanitized_username else # We cannot clobber another person with the same proposed username, so we resolve the conflict counter = 1 while User.exists?(username: sanitized_username) sanitized_username = "#{firststep_sanitized}_#{counter}" sanitized_username = sanitized_username[0, 20] if sanitized_username.length > 20 counter += 1 end end end if original_username != sanitized_username # The Discourse username is not the same as the Gossamer Forums username puts "Sanitized username: '#{original_username}' --> '#{sanitized_username}'" insert_username_mapping(original_username, sanitized_username, email, real_name) # else # puts "UNsanitized username: '#{original_username}' --> '#{sanitized_username}'" end sanitized_username end # Sanitize email to replace restricted domains def sanitize_email(email) restricted_domains = ['mailinator.com', 'example.com'] # Add more restricted domains as needed domain = email.split('@').last if restricted_domains.include?(domain) sanitized_email = email.gsub(domain, 'example.org') # Change to a permissible domain puts "Sanitized email: '#{email}' --> '#{sanitized_email}'" return sanitized_email end email end def generate_unique_title(base_title, user_id, timestamp) unique_title = base_title suffix = 1 puts "Generating unique title for base title: #{base_title}" while Topic.find_by(title: unique_title) # while Topic.where(title: unique_title).exists? # while Topic.exists?(title: unique_title) puts "Title '#{unique_title}' already exists, generating a new one." unique_title = "#{base_title} (#{suffix})" puts "NEW unique_title #{unique_title}" suffix += 1 end puts "Final unique title: #{unique_title}" unique_title end # Helper method to download an attachment / image from a URL def download_attachment(url) begin puts "URL: '#{url}'" URI.open(url, ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE).read rescue OpenURI::HTTPError => e puts "Failed to download attachment from #{url}: #{e.message}" nil rescue URI::InvalidURIError => e puts "Failed to handle invalid URL/URI for #{url}: #{e.message}" nil end end # Helper method to upload an attachment / image to Discourse def upload_attachment(user_id, file, filename, gossamer_url) begin # BAD # upload = Upload.create!( # user_id: user_id, # original_filename: filename, # filesize: file.size, # # filesize: File.size(file.path), # # content_type: `file --brief --mime-type #{file.path}`.strip, # # sha1: Digest::SHA1.file(file.path).hexdigest, # # origin: 'user_avatar', # # retain_hours: nil, # url: gossamer_url # ) # # Error -- non-existent method upload.ensure_consistency! # Use the correct content type and SHA1 hash for the upload # content_type = `file --brief --mime-type #{file.path}`.strip # Generate SHA1 hash for the file sha1 = Digest::SHA1.file(file.path).hexdigest # Ensure the file size is correctly handled file_size = File.size(file.path) puts "Attempting to upload attachment. filename #{filename} user_id #{user_id} file.size #{file.size} file_size #{file_size} sha1 #{sha1} gossamer_url #{gossamer_url}" # Create the upload record, adjusting fields as needed for 3.2.5 compatibility # upload = Upload.create!( upload = Upload.new( user_id: user_id, original_filename: filename, filesize: file.size, sha1: sha1, url: gossamer_url # content_type: content_type, # origin: 'composer', # retain_hours: nil ) # Save the upload object before generating the file path upload.save! # Use FileStore::LocalStore to store the file store = FileStore::LocalStore.new upload.url = store.store_file(file, store.get_path_for('original', upload.id, upload.sha1, File.extname(file.path))) # # Use Discourse's internal method to upload the file # file.rewind # Discourse.store.upload(file, upload.sha1) # # Move the file to the correct location # upload_path = Upload.get_path_for_file(upload.sha1) # FileUtils.mkdir_p(File.dirname(upload_path)) # FileUtils.mv(file.path, upload.path) # Save the upload object upload.save! # # Check if further processing or compatibility handling is needed # if Discourse.respond_to?(:store) # store = Discourse.store # upload.url = store.store_file(file, store.get_path_for('original', upload.id, upload.sha1, File.extname(file.path))) # else # # Fallback for earlier file store methods # upload_path = Upload.get_path_for_file(upload.sha1) # FileUtils.mkdir_p(File.dirname(upload_path)) # FileUtils.mv(file.path, upload_path) # end # Return the upload object upload rescue => e puts "FAILURE: Failed to upload attachment #{filename} for user_id #{user_id}: #{e.message}" puts e.backtrace.join("\n") # Print the full stack trace nil end end # Helper method to upload an attachment / image to Discourse def upload_attachment_two(user_id, file, filename) begin upload = UploadCreator.new(file, filename, origin: 'import').create_for(user_id) # Raise an erorr if the upload fails to catch issues early raise "Upload failed" unless upload upload rescue => e puts "Failed to upload attachment #{filename} for user_id #{user_id}: #{e.message}" puts e.backtrace.join("\n") # Print the full stack trace nil end end # Helper method to handle post attachments def handle_post_attachments(gossamer_post_id, post, user_id, mysql_client) execute_query_concurrent("SELECT * FROM gforum_PostAttachment WHERE post_id_fk = #{gossamer_post_id}", mysql_client).each do |att_row| attachment_url = "https://forum.slowtwitch.com/forum/?do=post_attachment;postatt_id=#{att_row['postatt_id']}" puts "Handling attachment: #{attachment_url}" attachment_data = download_attachment(attachment_url) next unless attachment_data mime_type = att_row['postatt_content'] # Create a Tempfile to pass to the UploadCreator temp_file = Tempfile.new(['attachment', File.extname(att_row['postatt_filename'])]) temp_file.binmode temp_file.write(attachment_data) temp_file.rewind puts "Attempting upload..." upload = upload_attachment_two(user_id, temp_file, att_row['postatt_filename']) next unless upload # Get the URL of the uploaded file from Discourse # upload_url = Upload.get_from_url(upload.url).url upload_url = upload.url puts "Appending to post.raw... #{upload_url} MIME type: #{mime_type}" if mime_type.start_with?('image/') post.raw += "\n![#{att_row['postatt_filename']}](#{upload_url})" else post.raw += "\n[#{att_row['postatt_filename']}](#{upload_url})" end post.save! temp_file.close temp_file.unlink end end # def download_file(url) # require 'open-uri' # begin # file = Tempfile.new # file.binmode # file.write(URI.open(url).read) # file.rewind # file # rescue => e # puts "Failed to download file from #{url}: #{e.message}" # puts e.backtrace.join("\n") # Print the full stack trace # nil # end # end # Helper method to upload an image to Discourse # def upload_image(user, image_data, filename) # return if image_data.nil? # # upload = Upload.create_for(user.id, File.open(image_data.path), filename, 'image/jpeg') # if upload.nil? || !upload.persisted? # puts "Failed to upload image for user #{user.username}" # return # end # # upload # end # Add 'Former_User' account if not already present def add_former_user puts "Adding 'Former User' account if it does not exist..." former_user = User.find_by(username: 'Former_User') if former_user.nil? former_user = User.create!( username: 'Former_User', name: 'Former User', email: 'former_user@example.com', password: SecureRandom.hex(16), active: true ) puts "'Former User' account created with user ID: #{former_user.id}" # Store the user ID mapping puts "for insert_user_id_mapping: 0 (former_user) discourse_user.id #{former_user.id}" insert_user_id_mapping(0, former_user.id) else puts "'Former User' account already exists with user ID: #{former_user.id}" end end # Import users from Gossamer Forums to Discourse def import_users puts "Importing Users..." users = [] # Fetch all users from Gossamer Forums # execute_query("SELECT * FROM gforum_User").each do |row| execute_query("SELECT user_id, user_username, user_email, user_real_name, user_registered, user_last_seen, user_title, user_about, user_homepage, user_location, user_password FROM gforum_User WHERE user_id > 123381").each do |row| username = sanitize_username(row['user_username'], row['user_email'], row['user_real_name']) email = row['user_email'] # Check if the user already exists in Discourse by username or email # existing_user = User.find_by(username: username) || User.find_by(email: email) existing_user = User.find_by(username: username) || User.joins(:user_emails).find_by(user_emails: { email: email }) # Only add the user if they do not already exist unless existing_user users << { id: row['user_id'], username: username, email: email, created_at: Time.at(row['user_registered']), updated_at: Time.at(row['user_last_seen']), name: row['user_real_name'], title: row['user_title'], bio_raw: row['user_about'] || "", website: row['user_homepage'], location: row['user_location'], custom_fields: { md5_password: row['user_password'], original_username: row['user_username'], original_gossamer_id: row['user_id'] } } end end # Create new users in Discourse create_users(users) do |user| # insert_user_id_mapping(user[:id], user.id) user end end # Generate SQLite user ID mapping between Discourse and Gossamer def generate_user_id_mapping puts "Generating User ID Mapping..." users = [] # Fetch all users from Gossamer Forums # execute_query("SELECT * FROM gforum_User").each do |row| execute_query("SELECT user_id, user_username, user_email, user_real_name, user_registered, user_last_seen, user_title, user_about, user_homepage, user_location, user_password FROM gforum_User WHERE user_id > 123381").each do |row| users << { id: row['user_id'], username: sanitize_username(row['user_username'], row['user_email'], row['user_real_name']), email: row['user_email'], created_at: Time.at(row['user_registered']), updated_at: Time.at(row['user_last_seen']), name: row['user_real_name'], title: row['user_title'], bio_raw: row['user_about'] || "", website: row['user_homepage'], location: row['user_location'], custom_fields: { md5_password: row['user_password'], original_username: row['user_username'], original_gossamer_id: row['user_id'] } } end # For each user, add user ID mapping to SQLite now that we know what the Discourse user ID is users.each do |user| # discourse_username = sanitize_username(user[:username], user[:email], user[:name]) discourse_username = user[:username] discourse_user = User.find_by(username: discourse_username) if discourse_user.nil? puts "User #{user[:username]} --> #{discourse_username} not found in Discourse. Skipping user mapping addition." next end # Store the user ID mapping # @user_id_map[user[:id]] = discourse_user.id puts "for insert_user_id_mapping: user[:id] #{user[:id]} discourse_user.id #{discourse_user.id}" insert_user_id_mapping(user[:id], discourse_user.id) end end # Import and set user Bio and Images def set_user_bio_images puts "Setting User Bio and Images..." users = [] # Fetch all users from Gossamer Forums # execute_query("SELECT * FROM gforum_User").each do |row| execute_query("SELECT user_id, user_username, user_email, user_real_name, user_registered, user_last_seen, user_title, user_about, user_homepage, user_location, user_password FROM gforum_User WHERE user_id > 123381").each do |row| users << { id: row['user_id'], username: sanitize_username(row['user_username'], row['user_email'], row['user_real_name']), email: row['user_email'], created_at: Time.at(row['user_registered']), updated_at: Time.at(row['user_last_seen']), name: row['user_real_name'], title: row['user_title'], bio_raw: row['user_about'] || "", website: row['user_homepage'], location: row['user_location'], custom_fields: { md5_password: row['user_password'], original_username: row['user_username'], original_gossamer_id: row['user_id'] } } end # For each user, append user bio and import user files users.each do |user| # discourse_username = sanitize_username(user[:username], user[:email], user[:name]) discourse_username = user[:username] discourse_user = User.find_by(username: discourse_username) if discourse_user.nil? puts "User #{user[:username]} --> #{discourse_username} not found in Discourse. Skipping bio-image setting." next end # Ensure user profile exists and bio_raw is a string discourse_user.user_profile ||= UserProfile.new(user_id: discourse_user.id) discourse_user.user_profile.bio_raw ||= "" # Append bio if it exists, otherwise set it to empty string to avoid nil errors ## if discourse_user.user_profile.bio_raw.empty? discourse_user.user_profile.bio_raw = user[:bio_raw] ## else ## discourse_user.user_profile.bio_raw += "\n\n" + user[:bio_raw] ## end # Ensure the bio does not exceed 3000 characters if discourse_user.user_profile.bio_raw.length > 2999 puts "Warning: About Me for user #{discourse_user.username} (ID: #{discourse_user.id}) exceeds 3000 characters. Truncating." discourse_user.user_profile.bio_raw = discourse_user.user_profile.bio_raw[0, 2999] end discourse_user.user_profile.save! # Import user files # TEMPORARY!! -- Use if script is interrupted mid-user-importing. # if user[:id] > 353 import_user_files(discourse_user) # end end end # # Import user files from Gossamer Forums to Discourse # def import_user_files(user) # print "\rImporting files for user #{user.username}..." # # original_gossamer_id = user.custom_fields['original_gossamer_id'] # if original_gossamer_id.nil? || original_gossamer_id.empty? # puts "User #{user.username} does not have a valid original_gossamer_id. Skipping file import." # return # end # # # puts "Original Gossamer ID for user #{user.username}: #{original_gossamer_id}" # # # Fetch and import user files # execute_query("SELECT * FROM gforum_User_Files WHERE ForeignColKey = #{original_gossamer_id}").each do |file| # # Construct the file URL # file_url = "https://forum.slowtwitch.com/images/users/images/#{file['ID'] % 10}/#{file['ID']}-#{file['File_Name']}" # puts "User #{user.username} User ID: #{user.id} original_gossamer_id: #{original_gossamer_id} file_url: #{file_url}" # # new_bio = user.user_profile.bio_raw + "\n\n![#{file['File_Name']}](#{file_url})" # if new_bio.length > 3000 # puts "Warning: About Me for user #{user.username} (ID: #{user.id}) exceeds 3000 characters after adding file link. Truncating." # new_bio = new_bio[0, 3000] # end # user.user_profile.bio_raw = new_bio # user.user_profile.save! # end # print "Importing files for user #{user.username}... Done.\n" # end # Helper method to convert TIFF to PNG def convert_tiff_to_png(file_path) png_path = file_path.sub('.tiff', '.png').sub('.tif', '.png') begin system("convert #{file_path} #{png_path}") png_path if File.exist?(png_path) rescue => e puts "Failed to convert image #{file_path}: #{e.message}" puts e.backtrace.join("\n") # Print the full stack trace nil end end # Helper method to resize an image to specified dimensions def resize_image(file_path, max_width, max_height) resized_path = file_path.sub(File.extname(file_path), "_resized#{File.extname(file_path)}") begin system("convert #{file_path} -resize #{max_width}x#{max_height} #{resized_path}") resized_path if File.exist?(resized_path) rescue => e puts "Failed to resize image #{file_path}: #{e.message}" puts e.backtrace.join("\n") # Print the full stack trace nil end end # Create avatar for the user using the provided image path def create_avatar(user, avatar_path) tempfile = Tempfile.new(['avatar', File.extname(avatar_path)]) FileUtils.copy_file(avatar_path, tempfile.path) filename = "avatar#{File.extname(avatar_path)}" upload = UploadCreator.new(tempfile, filename, type: "avatar").create_for(user.id) if upload.present? && upload.persisted? user.create_user_avatar user.user_avatar.update(custom_upload_id: upload.id) user.update(uploaded_avatar_id: upload.id) else puts "Failed to upload avatar for user #{user.username}: #{avatar_path}" puts upload.errors.inspect if upload end rescue StandardError => e puts "Failed to create avatar for user #{user.username}: #{avatar_path}. Error: #{e.message}" ensure tempfile.close! if tempfile end # Import user files (profile images) from Gossamer Forums to Discourse def import_user_files(user) print "\rImporting files for user #{user.username}..." original_gossamer_id = user.custom_fields['original_gossamer_id'] if original_gossamer_id.nil? || original_gossamer_id.empty? puts "User #{user.username} does not have a valid original_gossamer_id. Skipping file import." return end puts "Original Gossamer ID for user #{user.username}: #{original_gossamer_id}" images_imported = 0 execute_query("SELECT * FROM gforum_User_Files WHERE ForeignColKey = #{original_gossamer_id}").each do |file| # Encode the filename twice to match how it is stored on the server double_encoded_filename = file['File_Name'].gsub('%', '%25') encoded_filename = URI.encode_www_form_component(double_encoded_filename) # Construct the file URL file_url = "https://forum.slowtwitch.com/images/users/images/#{file['ID'] % 10}/#{file['ID']}-#{encoded_filename}" puts "User #{user.username} User ID: #{user.id} original_gossamer_id: #{original_gossamer_id} file_url: #{file_url}" # Ensure the file is a user image and has a supported MIME type next unless file['ForeignColName'] =~ /^user_image\d+$/ puts "#A" next unless ['image/jpeg', 'image/png', 'image/gif', 'image/webp', 'image/avif', 'image/heif', 'image/heic'].include?(file['File_MimeType']) puts "#B" # Download the attachment image_data = download_attachment(file_url) next if image_data.nil? puts "#C" # Write the image data to a temporary file temp_file = Tempfile.new(['user_image', File.extname(file['File_Name'])]) temp_file.binmode temp_file.write(image_data) temp_file.rewind # Convert TIFF to PNG if necessary # if file['File_MimeType'] == 'image/tiff' # begin # converted_tiff_to_png_path = convert_tiff_to_png(temp_file.path) # raise "Conversion of TIFF failed" if converted_tiff_to_png_path.nil? # temp_file.close # temp_file.unlink # temp_file = File.open(converted_tiff_to_png_path) # rescue => e # puts "Skipping image due to convert failure: #{temp_file.path}" # temp_file.close # temp_file.unlink # next # end # end # Resize the image for the avatar / profile picture (200x200) begin resized_image_path = resize_image(temp_file.path, 200, 200) raise "Image resize failed" if resized_image_path.nil? resized_temp_file = Tempfile.new(['user_image_resized', '.png']) FileUtils.copy_file(resized_image_path, resized_temp_file.path) rescue => e puts "Skipping image due to resize failure: #{temp_file.path}" puts e.backtrace.join("\n") # Print the full stack trace temp_file.close temp_file.unlink next end # Only handle the first image for avator and profile settings if images_imported == 0 puts "#D" # Upload the original image for profile header and user card background upload = upload_attachment(user.id, temp_file, file['File_Name'], file_url) next if upload.nil? # Upload the resized image for the avatar resized_upload = upload_attachment(user.id, resized_temp_file, file['File_Name'], file_url) next if resized_upload.nil? # Ensure the upload ID is valid and exists if resized_upload # && UserAvatar.exists?(custom_upload_id: resized_upload.id) # Set the avatar using the resized image # user.user_avatar = UserAvatar.create!(user_id: user.id, custom_upload_id: resized_upload.id) # user.save! # user.create_user_avatar # user.user_avatar.update(custom_upload_id: resized_upload.id) # user.update(uploaded_avatar_id: resized_upload.id) create_avatar(user, resized_temp_file.path) puts "Avatar set for user #{user.username} with upload ID #{resized_upload.id}" else puts "Failed to set avatar for user #{user.username}" end # Set the Profile Header UserProfile.find_by(user_id: user.id).update!(profile_background_upload_id: upload.id) # Set the User Card Background UserProfile.find_by(user_id: user.id).update!(card_background_upload_id: upload.id) images_imported += 1 end puts "#E" # Append the image to the user's bio_raw only if it does not already exist image_markdown = "\n\n![#{file['File_Name']}](#{file_url})" user.user_profile.bio_raw ||= "" unless user.user_profile.bio_raw.include?(image_markdown) user.user_profile.bio_raw += image_markdown user.user_profile.save! end # Clean up temporary files temp_file.close temp_file.unlink if temp_file.is_a?(Tempfile) resized_temp_file.close resized_temp_file.unlink end # Check for and remove exact duplicate image markdown strings user.user_profile.bio_raw = user.user_profile.bio_raw.split("\n\n").uniq.join("\n\n") user.user_profile.save! print "Importing files for user #{user.username}... Done.\n" end # Import categories from Gossamer Forums to Discourse def import_categories puts "Importing categories (forums)..." execute_query("SELECT * FROM gforum_Forum").each do |row| # Only create category if it does not exist unless CategoryCustomField.exists?(name: 'original_gossamer_id', value: row['forum_id']) category_name = row['forum_name'] category_description = row['forum_desc'] || "No description provided" puts "id #{row['forum_id']} name #{category_name} description #{category_description}" # Create category in Discourse category = create_category( { # id: row['forum_id'] + 10, name: category_name, description: category_description, created_at: row['forum_last'] ? Time.at(row['forum_last']) : Time.now, updated_at: row['forum_last'] ? Time.at(row['forum_last']) : Time.now }, row['forum_id'] # import_id argument ) # # Map Gossamer forum ID to Discourse category ID for future reference # @forum_id_map[row['forum_id']] = category.id # category.custom_fields.create!(name: 'original_gossamer_id', value: row['forum_id']) category.custom_fields['original_gossamer_id'] = row['forum_id'] category.save! # Store the user ID mapping puts "for insert_category_id_mapping: category[:id] #{category[:id]} row['forum_id'] #{row['forum_id']}" insert_category_id_mapping(row['forum_id'], category[:id]) end end puts "Importing categories... Done." end # Helper function to ensure title meets the minimum length requirement def ensure_valid_title(title, min_length = 5) if title.length < min_length title += "." * (min_length - title.length) # Append dots to make it longer end title end # # Convert Gossamer tags to Discourse markdown # def convert_gossamer_tags_to_markdown(text) # text.gsub!(/\[b\](.*?)\[\/b\]/, '**\1**') # text.gsub!(/\[i\](.*?)\[\/i\]/, '*\1*') # text.gsub!(/\[img\](.*?)\[\/img\]/, '![image](\1)') # text.gsub!(/\[quote\](.*?)\[\/quote\]/m, '[quote]\1[/quote]') # text.gsub!(/\[quote (.*?)\](.*?)\[\/quote\]/m, '[quote=\1]\2[/quote]') # text.gsub!(/\[font "(.*?)"\](.*?)\[\/font\]/m, '\2') # Ignoring font changes # text.gsub!(/\[size (\d+)\](.*?)\[\/size\]/m, '\2') # Ignoring size changes # text # end # Sanitize post message to remove Gossamer-specific tags and convert to Discourse-friendly format def sanitize_post_message(message) # Ensure the raw post string contents itself is acceptable to Discourse sanitized_message = message&.tr("\0", '') || "" # Remove the [signature] as we don't support this in Discourse # sanitized_message.sub!(/\n?\[signature\]\n?\z/, '') # sanitized_message.gsub(/\n?\[signature\]\n?/, '') sanitized_message.gsub!(/[\r\n]*\[signature\][\r\n]*/, '') # Remove [inline] style inlining. sanitized_message.gsub!(/[\r\n]*\[inline.*\][\r\n]*/, '') # Remove ![http://data:image/, etc. as this is not supported. # sanitized_message.gsub!(/!\[data:image\/[^\]]+\]\([^)]+\)/, '') # sanitized_message.gsub!(/!\[[^\]]*\]\(https?:\/\/data:image\/[a-zA-Z]+;base64,[^)]+\)/, '') # sanitized_message.gsub!(/!\[.*?\]\((?:http|https):\/\/data:image\/[^;]+;base64,[^)]+\)/i, '') # sanitized_message.gsub!(/!\[(?:http|https):\/\/data:image\/[^]]+\]\((?:http|https):\/\/data:image\/[^)]+\)/, '') # sanitized_message.gsub!(/!\[(https?:\/\/data:image\/[^]]+)\]\((https?:\/\/data:image\/[^)]+)\)/, '') # sanitized_message.gsub!(/!\[(https?:\/\/data:image\/[^\]]+)\]\((https?:\/\/data:image\/[^\)]+)\)/) # sanitized_message.gsub!(/!\[https?:\/\/data:image[^\]]*\]\(https?:\/\/data:image[^\)]*\)/, '') sanitized_message.gsub!(/\[img\]https?:\/\/data:image[^\[]*\[\/img\]/, '') # Ensure sentence structure unless sanitized_message.match?(/[.!?]\s|[.!?]$/) sanitized_message += "." end # Convert Gossamer tags to Discourse markdown sanitized_message.gsub!(/\[b\](.*?)\[\/b\]/, '**\1**') sanitized_message.gsub!(/\[i\](.*?)\[\/i\]/, '*\1*') sanitized_message.gsub!(/\[u\](.*?)\[\/u\]/, '\1') sanitized_message.gsub!(/\[quote\](.*?)\[\/quote\]/m, '[quote]\1[/quote]') sanitized_message.gsub!(/\[quote\s+user=(.*?)\](.*?)\[\/quote\]/m, '[quote \1]\2[/quote]') sanitized_message.gsub!(/\[img\](.*?)\[\/img\]/, '![image](\1)') sanitized_message.gsub!(/\[url=(.*?)\](.*?)\[\/url\]/, '[\2](\1)') sanitized_message.gsub!(/\[size=(.*?)\](.*?)\[\/size\]/, '\2') sanitized_message.gsub!(/\[color=(.*?)\](.*?)\[\/color\]/, '\2') sanitized_message.gsub!(/\[font=(.*?)\](.*?)\[\/font\]/, '\2') # Remove unsupported tags sanitized_message.gsub!(/\[.*?\]/, '') # Convert inline image syntax from `!(url)` to `![url](url)` sanitized_message.gsub!(/!\((http[s]?:\/\/[^\)]+)\)/, '![\1](\1)') # Ensure minimum length if sanitized_message.strip.empty? || sanitized_message.length < 5 sanitized_message = "Empty post contents." end sanitized_message end # Fetch post views from the gforum_PostView table def fetch_post_views(post_id, mysql_client) result = execute_query_concurrent("SELECT post_views FROM gforum_PostView WHERE post_id_fk = #{post_id} LIMIT 1", mysql_client).first result ? result['post_views'] : 0 end ########## THREADING START ############################################################################# # Restore specified TOPIC / OP post and all subsequent posts, via threading - SELECT post_id FROM gforum_Post WHERE post_id = restore_forumpost_id; def threaded_topic_restore(restore_forumpost_id) ## # Update connection pool settings ## ActiveRecord::Base.establish_connection( ## ActiveRecord::Base.connection_db_config.configuration_hash.merge(pool: 40, timeout: 5000) ## ) ### # Define the custom connection pool settings ### custom_pool = ActiveRecord::ConnectionAdapters::ConnectionPool.new( ### ActiveRecord::Base.connection_pool.spec.to_h.merge(pool: 40, timeout: 5000) ### ) ### # Register the custom connection pool under a unique identifier ### ActiveRecord::Base.connection_handler.connection_pools['CustomPool'] = custom_pool # Use CachedThreadPool for dynamic thread management #### pool = Concurrent::CachedThreadPool.new ###### pool = Concurrent::FixedThreadPool.new(7) pool = Concurrent::FixedThreadPool.new(2) # Define the connection pool inside the method ###### mariadb_pool = ConnectionPool.new(size: 14, timeout: 100) do #### mariadb_pool = ConnectionPool.new(size: 40, timeout: 100) do #### Mysql2::Client.new( #### host: "slowtwitch.northend.network", #### username: "admin", #### password: "yxnh93Ybbz2Nm8#mp28zCVv", #### database: "slowtwitch" #### ) #### end # The query selects post_ids from gforum_Post where post_root_id is 0, meaning these posts are the topic starters (OPs). # Execute the query and fetch the result # result = execute_query("SELECT post_id FROM gforum_Post WHERE post_root_id = 0 ORDER BY post_id ASC") result = execute_query("SELECT post_id FROM gforum_Post WHERE post_id = #{restore_forumpost_id} ORDER BY post_id ASC") # Convert the result set to an array of post_ids parent_post_ids = result.map { |row| row['post_id'] } # parent_post_count = parent_post_ids.count batch_size = 10 # Set our batch size for number of posts to import in a single batch #### current_post_batch = 0 # Set our current batch number. This tracks the current batch of posts being processed. is_complete = false # Flag to indicate whether the import process is complete. # Mutex to control access to shared resources ### mutex = Mutex.new # Mutex for MySQL2 operations -- disabled as this may not in fact be necessary - TBD. sqlite_mutex = Mutex.new # Mutex for SQLite opreations # Run until all posts have been processed. until is_complete # puts "QQ 11 -- GETTING NEXT BATCH ****************************************" #### # Query in batches, create pool, wait for termination, do it again #### current_post_batch_max = current_post_batch + batch_size # Get the next batch of posts current_post_batch = parent_post_ids.shift(batch_size) break if current_post_batch.empty? # Process each post in the current batch current_post_batch.each do |post_id| # puts "QQ 22 -- #{post_id}" ####### # Static pool size based on number of CPUs # # pool = Concurrent::FixedThreadPool.new(Concurrent.processor_count) # Create a thread pool that is bounded by processors avaialable # # pool = Concurrent::FixedThreadPool.new(8) # Create a thread pool of 8 pool members #### # Dynamically calculate the pool size based on system load to optimise performance #### pool_size = calculate_dynamic_pool_size # Dynamically calculate what the pool size "ought" to be. #### pool = Concurrent::FixedThreadPool.new(pool_size) # Create a thread pool with the calculated size #### # Process each post in the current batch #### while current_post_batch < current_post_batch_max #### post_id = parent_post_ids[current_post_batch] # Fetch the post_id for the current post #### # Check if the post has already been processed or is incomplete #### post_status = fetch_post_status(post_id) # Submit the import job for the current post_id to the thread pool pool.post do #### puts "PP 22 -- #{post_id}" retries = 0 begin ####### mariadb_pool.with do |mysql_client| # Initialise a new MariaDB / Mysql2 client inside of each thread mysql_client = Mysql2::Client.new( host: "slowtwitch.northend.network", username: "admin", password: "yxnh93Ybbz2Nm8#mp28zCVv", database: "slowtwitch" ) # Ensure the connection is active, otherwise reconnect puts "PP 11 -- #{post_id} -- Checking MySQL connections status.." mysql_client.ping || mysql_client = Mysql2::Client.new( host: "slowtwitch.northend.network", username: "admin", password: "yxnh93Ybbz2Nm8#mp28zCVv", database: "slowtwitch" ) puts " FIRST Checking MySQL connection status..." if mysql_client.query('SELECT 1').nil? puts " !! MySQL connection is NOT VALID !!" else puts " MySQL connection is valid" end # Use connection pooling for PostgreSQL and synchronize access to shared resources ActiveRecord::Base.connection_pool.with_connection do # ActiveRecord::Base.connected_to(pool: 'CustomPool') do ## ActiveRecord::Base.connected_to(role: :writing) do # Change from standard import # post_status = fetch_post_status(post_id) post_status = 0 if post_status.nil? || post_status == 0 puts "Starting import for post_id #{post_id}" topic_import_job(post_id, sqlite_mutex, mysql_client) # Import topic and its replies sqlite_mutex.synchronize do mark_post_as_complete(post_id) # Mark as complete in SQLite table end else puts "Skipping post_id #{post_id}, already processed." end end ####### end rescue => e puts "Error processing post ID #{post_id}: #{e.message}" puts e.backtrace.join("\n") # Print the full stack trace sqlite_mutex.synchronize do mark_post_as_failed(post_id) end case e.message when /MySQL client is not connected/, /This connection is in use by/ puts "Lost MySQL, retrying for post ID #{post_id}..." # Add reconnection attempt again here... if it proves necessary? retries += 1 if retries < 5 sleep(1) retry else puts "Max retries reached for post ID #{post_id}" end when /could not obtain a connection from the pool/ puts "Connection pool exhausted, retrying for post ID #{post_id}..." retries += 1 if retries < 5 sleep(1) retry else puts "Max retries reached for post ID #{post_id}" end end ensure # Ensure the MariaDB connection is closed after processing mysql_client.close if mysql_client puts "** CLOSED MariaDB client" end end end #### current_post_batch += 1 # Increment, moving to next post in the batch #### break if current_post_batch >= parent_post_count # Check if all posts have been processed #### is_complete = true if current_post_batch >= parent_post_count is_complete = parent_post_ids.empty? end # Wait for all jobs in the current batch to finish before proceeding puts "PP 33 -- Ready for shutdown" pool.shutdown # Initiate thread pool shutdown after all jobs submitted puts "PP 44 -- Now wait for termination" pool.wait_for_termination # Wait for all threads to finish exec end # # Method to ensure thread-safe updates to highest_processed_post_id # def update_highest_processed_post_id_thread_safe(post_id) # @highest_processed_mutex ||= Mutex.new # @highest_processed_mutex.synchronize do # if post_id > fetch_highest_processed_post_id # update_highest_processed_post_id(post_id) # end # end # end # Method to import an entire topic, including its first post and all subsequent replies def topic_import_job(post_id, sqlite_mutex, mysql_client) ##### def topic_import_job(post_id, sqlite_mutex) puts "TIJ ZZ post_id #{post_id}" ##### mysql_client = Mysql2::Client.new( ##### host: "slowtwitch.northend.network", ##### username: "admin", ##### password: "yxnh93Ybbz2Nm8#mp28zCVv", ##### database: "slowtwitch" ##### ) puts " FIRST Checking MySQL connection status..." if mysql_client.query('SELECT 1').nil? puts " MySQL connection is not valid, TRY TO RECONNECT II" mysql_client.ping # || mysql_client = Mysql2::Client.new( # host: "slowtwitch.northend.network", # username: "admin", # password: "yxnh93Ybbz2Nm8#mp28zCVv", # database: "slowtwitch" # ) else puts " MySQL connection is valid" end puts "TIJ AA post_id #{post_id}" # Fetch the post data for the given post_id (this is the first post in the topic) row = execute_query_concurrent("SELECT post_id, user_id_fk, forum_id_fk, post_root_id, post_subject, post_time, post_message, post_father_id, post_replies FROM gforum_Post WHERE post_id = #{post_id}", mysql_client).first puts "TIJ BB post_id #{post_id}" # Early return if the post data is not found return unless row puts "TIJ CC post_id #{post_id}" # Extract key values from the fetched row post_id = row['post_id'].to_i puts "Processing post_id #{row['post_id']} post_root_id #{row['post_root_id']} post_subject/title #{row['post_subject']} forum_id_fk/category_id #{row['forum_id_fk']}" # Fetch the mapped Discourse user and category ID based on Gossamer data discourse_user_id = fetch_user_id_mapping(row['user_id_fk']) # Check to be certain user has not been deleted, etc. if discourse_user_id.nil? || discourse_user_id == 0 puts "discourse_user_id is NIL/ZERO for post_id #{row['post_id']}" discourse_former_user = User.find_by(username: 'Former_User') discourse_user_id = discourse_former_user.id puts "discourse_user_id is NOW Former_User id #{discourse_user_id} for post_id #{row['post_id']}" end # Fetch the mapped Discourse user and category ID based on Gossamer data discourse_category_id = fetch_category_id_mapping(row['forum_id_fk']) puts "discourse_user_id #{discourse_user_id} discourse_category_id #{discourse_category_id}" return unless discourse_user_id && discourse_category_id puts "TIJ DD post_id #{post_id}" # Ensure the topic title is valid and generate a unique title if needed title = ensure_valid_title(row['post_subject']) unique_title = title # Fetch the number of views the post has had post_views = fetch_post_views(row['post_id'], mysql_client) # Check if the topic has already been imported using the custom field 'original_gossamer_id' unless TopicCustomField.exists?(name: 'original_gossamer_id', value: row['post_id']) puts "TIJ EE post_id #{post_id}" ActiveRecord::Base.transaction do # Create the new topic in Discourse begin suffix = 1 topic_created = false ## ActiveRecord::Base.transaction do while !topic_created begin puts "TIJ FF post_id #{post_id}" puts "CREATE TOPIC unique_title #{unique_title} title #{title} discourse_user_id #{discourse_user_id} category_id #{discourse_category_id}" topic = Topic.create!( title: unique_title, user_id: discourse_user_id, created_at: Time.at(row['post_time']), updated_at: Time.at(row['post_time']), category_id: discourse_category_id, views: post_views || 0, posts_count: 0 ) topic.custom_fields['original_gossamer_id'] = row['post_id'] topic.save! topic_created = true # rescue ActiveRecord::RecordInvalid => e rescue => e if e.message.include?("Title has already been used") unique_title = "#{title} (#{suffix})" suffix += 1 else raise e end # puts e.backtrace.join("\n") # Print the full stack trace end end ## end # Workaround... take a copy of topic.id current_topic_id = topic.id sqlite_mutex.synchronize do # Update the database with the last post time and user for the topic puts "zzzz1 #{current_topic_id} #{Time.at(row['post_time']).to_i}" update_db_topic_last_post_time(current_topic_id, Time.at(row['post_time']).to_i) update_db_topic_last_post_user(current_topic_id, discourse_user_id) puts "zzzz2" # Increment the topic count for the user update_db_user_topic_count(discourse_user_id, fetch_db_user_topic_count(discourse_user_id).to_i + 1) end # Sanitize and prepare the post message for Discourse sanitized_post_message = sanitize_post_message(row['post_message']) puts "CREATE TOPIC POST for current_topic_id #{current_topic_id} discourse_user_id #{discourse_user_id} sanitized_post_message: #{sanitized_post_message}" post_number = 1 # Increment the post count for the topic # This is a first post... post_number = fetch_db_topic_post_numbers(current_topic_id).to_i + 1 sqlite_mutex.synchronize do update_db_topic_post_numbers(current_topic_id, post_number) end puts "TIJ GG post_id #{post_id}" ## ActiveRecord::Base.transaction do # Create the initial post in the new topic post = Post.create!( topic_id: current_topic_id, user_id: discourse_user_id, raw: sanitized_post_message, created_at: Time.at(row['post_time']), updated_at: Time.at(row['post_time']), reads: post_views || 0, post_number: post_number ) post.custom_fields['original_gossamer_id'] = row['post_id'] post.save! ## end sqlite_mutex.synchronize do # Increment the post count for the topic and user update_db_topic_post_count(current_topic_id, fetch_db_topic_post_count(current_topic_id).to_i + 1) update_db_user_post_count(discourse_user_id, fetch_db_user_post_count(discourse_user_id).to_i + 1) end puts "TIJ HH post_id #{post_id}" ## ActiveRecord::Base.transaction do # Handle any attachments associated with the post handle_post_attachments(row['post_id'], post, discourse_user_id, mysql_client) ## end # Create URL mappings for the new topic new_url = "https://new/t/#{topic.slug}/#{current_topic_id}" sqlite_mutex.synchronize do insert_url_mapping(row['post_id'], new_url, unique_title) end # Fetch and import all replies to this topic replies = execute_query_concurrent("SELECT post_id, user_id_fk, post_message, post_time FROM gforum_Post WHERE post_root_id = #{post_id} ORDER BY post_time ASC", mysql_client) # Import each reply sequentially replies.each do |reply_row| ## begin # Fetch the discourse user ID for the reply reply_user_id = fetch_user_id_mapping(reply_row['user_id_fk']) if reply_user_id.nil? || reply_user_id == 0 puts "reply_user_id is NIL/ZERO for reply post_id #{reply_row['post_id']}" former_user = User.find_by(username: 'Former_User') reply_user_id = former_user.id puts "reply_user_id is NOW Former_User id #{reply_user_id} for reply post_id #{reply_row['post_id']}" end puts "TIJ II post_id #{post_id}" # Sanitize and prepare the reply message for Discourse sanitized_reply_message = sanitize_post_message(reply_row['post_message']) puts "CREATE REPLY in current_topic_id #{current_topic_id} for reply post_id #{reply_row['post_id']} -- sanitized_reply_message: #{sanitized_reply_message}" ### def get_topic_id ### return topic.id ### end # Increment the post count for the topic post_number = fetch_db_topic_post_numbers(current_topic_id).to_i + 1 sqlite_mutex.synchronize do update_db_topic_post_numbers(current_topic_id, post_number) ### update_db_topic_post_numbers(get_topic_id, post_number) end # Fetch the number of views the post has had reply_post_views = fetch_post_views(reply_row['post_id'], mysql_client) # crazy sanity check if topic.nil? puts "ERROR: Topic is nil for reply post_id #{reply_row['post_id']}, attempting to BYPASS anyway" end puts "TIJ JJ post_id #{post_id} reply post_id #{reply_row['post_id']} reply_post_views #{reply_post_views || 0} post_number #{post_number} current_topic_id #{current_topic_id} reply_post_views #{reply_post_views || 0}" ## ActiveRecord::Base.transaction do # Create the reply post in the existing topic post = Post.create!( topic_id: current_topic_id, user_id: reply_user_id, raw: sanitized_reply_message, created_at: Time.at(reply_row['post_time']), updated_at: Time.at(reply_row['post_time']), reads: reply_post_views || 0, post_number: post_number ) post.custom_fields['original_gossamer_id'] = reply_row['post_id'] post.save! ## end puts "TIJ KK post_id #{post_id}" # Increment the post count for the topic and user sqlite_mutex.synchronize do update_db_topic_post_count(current_topic_id, fetch_db_topic_post_count(current_topic_id).to_i + 1) update_db_user_post_count(reply_user_id, fetch_db_user_post_count(reply_user_id).to_i + 1) end # Update last post time and user for the topic if fetch_db_topic_last_post_time(current_topic_id).nil? || Time.at(reply_row['post_time']).to_i > fetch_db_topic_last_post_time(current_topic_id).to_i sqlite_mutex.synchronize do update_db_topic_last_post_time(current_topic_id, Time.at(reply_row['post_time']).to_i) update_db_topic_last_post_user(current_topic_id, reply_user_id) end end ## ActiveRecord::Base.transaction do # Handle any attachments associated with the reply handle_post_attachments(reply_row['post_id'], post, reply_user_id, mysql_client) ## end # # Update the highest processed post_id in the database (thread-safe) # update_highest_processed_post_id_thread_safe(reply_row['post_id']) # rescue ActiveRecord::RecordInvalid => e ## rescue => e ## puts "Error importing reply with post_id #{reply_row['post_id']}: #{e.message}" ## puts e.backtrace.join("\n") # Print the full stack trace ## end end # # After processing the entire topic, update the highest_processed_post_id to the current topic's post_id (thread-safe) # update_highest_processed_post_id_thread_safe(post_id) rescue ActiveRecord::RecordInvalid => e puts "---> ERROR importing topic with post_id #{row['post_id']}: #{e.message}" raise ActiveRecord::Rollback end end else puts "Topic for post_id #{row['post_id']} already exists, skipping creation." end puts " LAST Removing MySQL connection" ##### mysql_client.close # if mysql_client end ########## THREADING END -------------------------------------------- # Import topics and posts from Gossamer Forums to Discourse def import_topics_and_posts puts "Importing topics and posts with attachments..." # topic_last_post_time = {} # topic_post_count = Hash.new(0) # user_topic_count = Hash.new(0) # user_post_count = Hash.new(0) # topic_last_post_user = {} # topic_post_numbers = Hash.new { |hash, key| hash[key] = 0 } # Fetch the highest old_post_id from the url_map table # highest_old_post_id = fetch_highest_old_post_id.to_i # puts "Highest (OP) old_post_id in url_map: #{highest_old_post_id}" highest_processed_post_id = fetch_highest_processed_post_id.to_i puts "Highest processed post_id: #{highest_processed_post_id}" # OVERRIDE........ # Attachment example: highest_processed_post_id = 1359862 highest_processed_post_id = 8179621 puts "OVERRIDE Highest processed post_id: #{highest_processed_post_id}" # Execute the query to get all posts ordered by post_id execute_query("SELECT post_id, user_id_fk, forum_id_fk, post_root_id, post_subject, post_time, post_message, post_father_id, post_replies FROM gforum_Post WHERE post_id > #{highest_processed_post_id} ORDER BY post_id").each do |row| post_id = row['post_id'].to_i # Skip posts that have already been processed # FIXED PROPERLY FOR FUTURE RUNS # next if post_id < highest_old_post_id next if post_id <= highest_processed_post_id puts "Processing post_id #{row['post_id']} post_root_id #{row['post_root_id']} post_subject/title #{row['post_subject']} forum_id_fk (category_id) #{row['forum_id_fk']}" # discourse_user_id = @user_id_map[row['user_id_fk']] # Fetch the Discourse user_id based on Gossamer data and mapping discourse_user_id = fetch_user_id_mapping(row['user_id_fk']) # Check to be certain user has not been deleted, etc. if discourse_user_id.nil? || discourse_user_id == 0 puts "discourse_user_id is NIL/ZERO for post_id #{row['post_id']}" discourse_former_user = User.find_by(username: 'Former_User') discourse_user_id = discourse_former_user.id puts "discourse_user_id is NOW Former_User id #{discourse_user_id} for post_id #{row['post_id']}" end # Fetch the Discourse category_id based on Gossamer data and mapping discourse_category_id = fetch_category_id_mapping(row['forum_id_fk']) puts "discourse_user_id #{discourse_user_id} discourse_category_id #{discourse_category_id}" next unless discourse_user_id && discourse_category_id # Check if the post is a topic (post_root_id == 0) if row['post_root_id'] == 0 puts "#1" # Ensure the title is valid title = ensure_valid_title(row['post_subject']) unique_title = title # Confirm the number of views the post has had post_views = fetch_post_views(row['post_id'], @mysql_client) # Skip if the topic already exists unless TopicCustomField.exists?(name: 'original_gossamer_id', value: row['post_id']) # Create the topic begin suffix = 1 topic_created = false while !topic_created begin puts "#2" # unique_title = generate_unique_title(title, discourse_user_id, Time.at(row['post_time'])) puts "CREATE TOPIC unique_title #{unique_title} title #{title} discourse_user_id #{discourse_user_id} category_id #{discourse_category_id}" topic = Topic.create!( title: unique_title, user_id: discourse_user_id, created_at: Time.at(row['post_time']), updated_at: Time.at(row['post_time']), category_id: discourse_category_id, views: post_views || 0, posts_count: 0 ) topic.custom_fields['original_gossamer_id'] = row['post_id'] topic.save! topic_created = true rescue ActiveRecord::RecordInvalid => e if e.message.include?("Title has already been used") unique_title = "#{title} (#{suffix})" suffix += 1 else raise e end end end current_topic_id = topic.id # Track last post time and user for the topic # topic_last_post_time[topic.id] = Time.at(row['post_time']) # topic_last_post_user[topic.id] = discourse_user_id update_db_topic_last_post_time(current_topic_id, Time.at(row['post_time']).to_i) update_db_topic_last_post_user(current_topic_id, discourse_user_id) # Increment the count of the number of topics created by each user # user_topic_count[discourse_user_id] += 1 update_db_user_topic_count(discourse_user_id, fetch_db_user_topic_count(discourse_user_id).to_i + 1) # # Ensure the raw post stirng contents itself is acceptable to Discourse # sanitized_post_message = row['post_message']&.tr("\0", '') || "" # # # Convert Gossamer tags to Discourse markdown # sanitized_post_message = convert_gossamer_tags_to_markdown(sanitized_post_message) # # # Remove the [signature] label from appearing at the end of the messages after import # sanitized_post_message.sub(/\n?\[signature\]\n?\z/, '') # sanitized_post_message.gsub(/\n?\[signature\]\n?/, '') # Sanitize the post message sanitized_post_message = sanitize_post_message(row['post_message']) puts "CREATE TOPIC POST current_topic_id #{current_topic_id} discourse_user_id #{discourse_user_id}" # Increment the number of posts in the given topic. # topic_post_numbers[topic.id] += 1 post_number = fetch_db_topic_post_numbers(current_topic_id).to_i + 1 update_db_topic_post_numbers(current_topic_id, post_number) # Create the initial post in the topic post = Post.create!( topic_id: current_topic_id, user_id: discourse_user_id, raw: sanitized_post_message, created_at: Time.at(row['post_time']), updated_at: Time.at(row['post_time']), reads: post_views || 0, post_number: post_number ) # like_count: row['post_likes'] || 0, # raw: import_attachments(row['post_message'], row['post_id']), # raw: row['post_message'] || "", # post_number: topic_post_numbers[topic.id] post.custom_fields['original_gossamer_id'] = row['post_id'] post.save! # Track the number of posts in the topic and by the user # topic_post_count[topic.id] += 1 # user_post_count[discourse_user_id] += 1 update_db_topic_post_count(current_topic_id, fetch_db_topic_post_count(current_topic_id).to_i + 1) update_db_user_post_count(discourse_user_id, fetch_db_user_post_count(discourse_user_id).to_i + 1) # Handle attachments for the post handle_post_attachments(row['post_id'], post, discourse_user_id, @mysql_client) # Create URL mappings # old_url = "https://old/forum/#{row['forum_name']}/topics/#{row['post_id']}" new_url = "https://new/t/#{topic.slug}/#{topic.id}" insert_url_mapping(row['post_id'], new_url, unique_title) # Update the highest processed post_id puts "Updated highest processed post_id #{post_id}" update_highest_processed_post_id(post_id) rescue ActiveRecord::RecordInvalid => e puts "Error importing topic with post_id #{row['post_id']}: #{e.message}" end end else puts "#3" # Confirm the number of views the post has had post_views = fetch_post_views(row['post_id'], @mysql_client) # Find the root topic for the post root_topic_field = TopicCustomField.find_by(name: 'original_gossamer_id', value: row['post_root_id']) if root_topic_field topic_id = root_topic_field.topic_id # Find the parent post for the reply parent_post_field = PostCustomField.find_by(name: 'original_gossamer_id', value: row['post_father_id']) reply_to_post_number = parent_post_field ? Post.find(parent_post_field.post_id).post_number : nil # Create the post in the existing topic begin puts "#4" # Sanitize the post message sanitized_post_message = sanitize_post_message(row['post_message']) # topic_post_numbers[topic_id] += 1 post_number = fetch_db_topic_post_numbers(topic_id).to_i + 1 puts "CREATE REPLY in topic_id #{topic_id} for reply post_id #{row['post_id']} with post_number #{post_number}" # Create the post in the existing topic post_created = false while ! post_created begin post = Post.create!( topic_id: topic_id, user_id: discourse_user_id, raw: sanitized_post_message, created_at: Time.at(row['post_time']), updated_at: Time.at(row['post_time']), reads: post_views || 0, post_number: post_number ) # raw: import_attachments(row['post_message'], row['post_id']), # raw: row['post_message'] || "", # reply_to_post_number: reply_to_post_number, # like_count: row['post_replies'] || 0, # reads: post_views || fetch_db_topic_post_count(topic_id).to_i, post.custom_fields['original_gossamer_id'] = row['post_id'] post.save! post_created = true rescue ActiveRecord::RecordNotUnique => e if e.message.include?("duplicate key value violates unique constraint") post_number = post_number + 1 else raise e end end end update_db_topic_post_numbers(topic_id, post_number) # Track the number of posts in the topic and by the user # topic_post_count[topic_id] += 1 # user_post_count[discourse_user_id] += 1 update_db_topic_post_count(topic_id, fetch_db_topic_post_count(topic_id).to_i + 1) update_db_user_post_count(discourse_user_id, fetch_db_user_post_count(discourse_user_id).to_i + 1) # Update last post time and user for the topic # if topic_last_post_time[topic_id].nil? || Time.at(row['post_time']) > topic_last_post_time[topic_id] # topic_last_post_time[topic_id] = Time.at(row['post_time']) # topic_last_post_user[topic_id] = discourse_user_id # end if fetch_db_topic_last_post_time(topic_id).nil? || Time.at(row['post_time']).to_i > fetch_db_topic_last_post_time(topic_id).to_i update_db_topic_last_post_time(topic_id, Time.at(row['post_time']).to_i) update_db_topic_last_post_user(topic_id, discourse_user_id) end # Handle attachments for the post handle_post_attachments(row['post_id'], post, discourse_user_id, @mysql_client) # Update the highest processed post_id puts "Updated highest processed post_id #{post_id}" update_highest_processed_post_id(post_id) rescue ActiveRecord::RecordInvalid => e puts "Error importing post with post_id #{row['post_id']}: #{e.message}" end else puts "Warning: Root topic not found for post_id #{row['post_id']} with post_root_id #{row['post_root_id']}" end end end puts "Importing topics and posts with attachments... Done." end def update_topic_stats # Update topics with the correct last post time, post count, and last post user puts "Update topics with the correct last post time, post count, and last post user" # topic_last_post_time.each do |topic_id, last_post_time| # Topic.find(topic_id).update!( # updated_at: last_post_time, # posts_count: topic_post_count[topic_id], # last_posted_at: last_post_time, # bumped_at: last_post_time, # last_post_user_id: topic_last_post_user[topic_id] # ) # end @db.execute("SELECT * FROM topic_last_post_time").each do |row| topic_id, last_post_time = row begin # Validation error: Topic.find(topic_id).update!( topic = Topic.find(topic_id) # Ensure we are only updating necessary fields topic.update_columns( updated_at: Time.at(last_post_time), posts_count: fetch_db_topic_post_count(topic_id).to_i, last_posted_at: Time.at(last_post_time), bumped_at: Time.at(last_post_time), last_post_user_id: fetch_db_topic_last_post_user(topic_id).to_i ) rescue ActiveRecord::RecordNotFound puts "WARNING: Could not find Topic with id=#{topic_id}. Skipping..." end end end # Update topic stats for the restored topic. def update_topic_stats_final11 # Update topics with the correct last post time, post count, and last post user puts "Update topics with the correct last post time, post count, and last post user" @db.execute("SELECT * FROM topic_last_post_time_final11").each do |row| topic_id, last_post_time = row puts "update_topic_stats_final11 topic_id #{topic_id}" begin topic = Topic.find(topic_id) # Calculate the new values based on the given conditions new_updated_at = Time.at(last_post_time) new_posts_count = topic.posts_count + fetch_db_topic_post_count(topic_id).to_i new_last_posted_at = Time.at(last_post_time) new_bumped_at = Time.at(last_post_time) new_last_post_user_id = fetch_db_topic_last_post_user(topic_id).to_i # Update the topic with the calculated values topic.update_columns( updated_at: new_updated_at, posts_count: new_posts_count, last_posted_at: new_last_posted_at, bumped_at: new_bumped_at, last_post_user_id: new_last_post_user_id ) rescue ActiveRecord::RecordNotFound puts "WARNING: Could not find Topic with id=#{topic_id}. Skipping..." end end end # Update (increment) user stats, but don't touch trust levels, after topic restoration. def update_user_stats_final11 # Update user profiles with the number of topics and posts created puts "Update user profiles with the number of topics and posts created" @db.execute("SELECT * FROM user_topic_count_final11").each do |row| user_id, count = row puts "update_user_stats user_id #{user_id} topic_count #{count}" begin user_stat = UserStat.find_or_initialize_by(user_id: user_id) # user_stat.update_columns(topic_count: count) # Accumulate the current topic count with the new count new_topic_count = user_stat.topic_count.to_i + count.to_i user_stat.update_columns(topic_count: new_topic_count) rescue puts "WARNING: Could not find User with id=#{user_id}. Skipping..." end end @db.execute("SELECT * FROM user_post_count_final11").each do |row| user_id, count = row puts "update_user_stats user_id #{user_id} post_count #{count}" begin user_stat = UserStat.find_or_initialize_by(user_id: user_id) # user_stat.update_columns(post_count: count) # Accumulate the current post count with the new count new_post_count = user_stat.post_count.to_i + count.to_i user_stat.update_columns(post_count: new_post_count) rescue puts "WARNING: Could not find or modify User with id=#{user_id}. Skipping..." end end end def update_user_stats # Update user profiles with the number of topics and posts created puts "Update user profiles with the number of topics and posts created" # user_topic_count.each do |user_id, count| # user = User.find(user_id) # user.update!(topic_count: count) # end # user_post_count.each do |user_id, count| # user = User.find(user_id) # user.update!(post_count: count) # end @db.execute("SELECT * FROM user_topic_count_final").each do |row| user_id, count = row # user = User.find(user_id) # user.update!(topic_count: count) puts "update_user_stats user_id #{user_id} topic_count #{count}" begin user_stat = UserStat.find_or_initialize_by(user_id: user_id) user_stat.update_columns(topic_count: count) rescue puts "WARNING: Could not find User with id=#{user_id}. Skipping..." end end @db.execute("SELECT * FROM user_post_count_final").each do |row| user_id, count = row # user = User.find(user_id) # user.update!(post_count: count) puts "update_user_stats user_id #{user_id} post_count #{count}" begin user_stat = UserStat.find_or_initialize_by(user_id: user_id) user_stat.update_columns(post_count: count) # Fetch the current user and check if Trust Level needs updating new_trust_level = case count when 0..29 then 1 # basic user else 2 # member, regular reserved for now. # when 3..50 then 2 # member # else 3 # regular or above when 51..100 end user = User.find(user_id) current_trust_level = user.trust_level || 1 # default to 1 if not set # Only update trust level if the new level is higher than the current one if new_trust_level != current_trust_level user.update!(trust_level: new_trust_level) puts "update_user_stats user_id #{user_id} trust_level was #{current_trust_level}, now updated to #{new_trust_level}" else puts "update_user_stats user_id #{user_id} trust_level remains at #{current_trust_level}" end rescue puts "WARNING: Could not find or modify User with id=#{user_id}. Skipping..." end end end # Correct user stats, notably the trust levels, after the final post-migration import run. def update_user_stats_finalfinal @db.execute("SELECT * FROM user_post_count_final").each do |row| user_id, count = row puts "update_user_stats user_id #{user_id}" begin user_stat = UserStat.find_or_initialize_by(user_id: user_id) # Fetch the current post count for the user post_count = user_stat.post_count.to_i ## user_stat.update_columns(post_count: new_post_count) # Fetch the current user and check if Trust Level needs updating # new_trust_level = case post_count # when 0..29 then 1 # basic user # else 2 # member, regular reserved for now. # when 3..50 then 2 # member # else 3 # regular or above when 51..100 # end user = User.find(user_id) current_trust_level = user.trust_level if user.id == 101978 puts "USER ID 101978, so TRUST LEVEL 4" new_trust_level = 4 elsif user.id == 2 puts "USER ID 2, so TRUST LEVEL 2" new_trust_level = 2 elsif [111533, 161794, 163535, 172916, 177877, 183157, 189831, 191083, 192666, 195599, 198625, 200072, 200823, 201878, 201968].include?(user.id) new_trust_level = 1 else new_trust_level = 2 end # Only update trust level if the new level is higher than the current one if new_trust_level != current_trust_level user.update!(trust_level: new_trust_level) puts "update_user_stats user_id #{user_id} trust_level was #{current_trust_level}, now updated to #{new_trust_level}" else puts "update_user_stats user_id #{user_id} trust_level remains at #{current_trust_level}" end rescue puts "WARNING: Could not find or modify User with id=#{user_id}: #{e.message} ... Skipping..." puts e.backtrace.join("\n") # Print the full stack trace end end end # Set user stats, including the trust levels (albeit incorrectly...) after the final post-migration import run. def update_user_stats_final # Update user profiles with the number of topics and posts created puts "Update user profiles with the number of topics and posts created" @db.execute("SELECT * FROM user_topic_count_final").each do |row| user_id, count = row puts "update_user_stats user_id #{user_id} topic_count #{count}" begin user_stat = UserStat.find_or_initialize_by(user_id: user_id) # user_stat.update_columns(topic_count: count) # Accumulate the current topic count with the new count new_topic_count = user_stat.topic_count.to_i + count.to_i user_stat.update_columns(topic_count: new_topic_count) rescue puts "WARNING: Could not find User with id=#{user_id}. Skipping..." end end @db.execute("SELECT * FROM user_post_count_final").each do |row| user_id, count = row puts "update_user_stats user_id #{user_id} post_count #{count}" begin user_stat = UserStat.find_or_initialize_by(user_id: user_id) # user_stat.update_columns(post_count: count) # Accumulate the current post count with the new count new_post_count = user_stat.post_count.to_i + count.to_i user_stat.update_columns(post_count: new_post_count) # Fetch the current user and check if Trust Level needs updating new_trust_level = case count when 0..29 then 1 # basic user else 2 # member, regular reserved for now. # when 3..50 then 2 # member # else 3 # regular or above when 51..100 end user = User.find(user_id) current_trust_level = user.trust_level || 1 # default to 1 if not set # Only update trust level if the new level is higher than the current one if new_trust_level != current_trust_level user.update!(trust_level: new_trust_level) puts "update_user_stats user_id #{user_id} trust_level was #{current_trust_level}, now updated to #{new_trust_level}" else puts "update_user_stats user_id #{user_id} trust_level remains at #{current_trust_level}" end rescue puts "WARNING: Could not find or modify User with id=#{user_id}. Skipping..." end end end # Import personal messages from gforum_Message table (both inbox and sent messages) def import_personal_messages puts "Importing personal (inbox and sentmail) messages..." # Fetch the highest_processed_personal_id highest_processed_personal_id = fetch_highest_processed_personal_id.to_i puts "Highest processed personal_id: #{highest_processed_personal_id}" # OVERRIDE - to speed getting to lower limit msg # # # highest_processed_personal_id = 1543840 puts "Highest processed personal_id override: #{highest_processed_personal_id}" ## execute_query("SELECT * FROM gforum_Message").each do |row| execute_query("SELECT * FROM gforum_Message WHERE msg_id > #{highest_processed_personal_id}").each do |row| begin msg_id = row['msg_id'].to_i puts "msg_id #{msg_id}" # Skip posts that have already been processed # OVERRIDE - upper limit msg # # # next if msg_id <= highest_processed_personal_id || msg_id > 342443 ## next if msg_id <= highest_processed_personal_id from_user_id = fetch_user_id_mapping(row['from_user_id_fk']) to_user_id = fetch_user_id_mapping(row['to_user_id_fk']) next unless from_user_id && to_user_id # Skip if the message already exists unless TopicCustomField.exists?(name: 'original_gossamer_msg_id', value: row['msg_id']) # Sanitize the message, ensuring we have an empty string or the content without any \0 sanitized_message = sanitize_post_message(row['msg_body']) # Set default message body if the sanitized message is blank sanitized_message = "" if sanitized_message.strip.empty? || sanitized_message.split.size < 3 # If we do not change the "min personal message post length" to 1, we need this. ... We may need this anyway. sanitized_message = sanitized_message.ljust(10, '.') if sanitized_message.length < 10 # Check and set a default title if the original title is nil or empty sanitized_title = row['msg_subject']&.strip sanitized_title = "" if sanitized_title.nil? || sanitized_title.empty? # Check for an existing private message topic between the same two users with a similar title # topic = Topic.joins(:topic_allowed_users, :custom_fields) # .where(archetype: Archetype.private_message) # .where("topic_allowed_users.user_id = ? AND topic_allowed_users.user_id = ?", from_user_id, to_user_id) # .where("title = ? OR title = ?", sanitized_title, "Re: #{sanitized_title}") # .where(topic_custom_fields: { name: 'original_gossamer_msg_id' }) # .first topic = Topic.joins(:topic_allowed_users) .where(archetype: Archetype.private_message) .where("topics.title = ? OR topics.title = ?", sanitized_title, sanitized_title.gsub(/^Re: /, "")) .where("topic_allowed_users.user_id IN (?)", [from_user_id, to_user_id]) .group("topics.id") .having("COUNT(topic_allowed_users.user_id) = 2") .first # .where("topics.title = ? OR topics.title = ?", sanitized_title, sanitized_title.gsub(/^Re: /, "")) # .where("topics.topic_allowed_users.user_id = ? AND topics.topic_allowed_users.user_id = ?", from_user_id, to_user_id) # .where("((topics.user_id = ? AND topics.topic_allowed_users.user_id = ?) OR (topics.user_id = ? AND topics.topic_allowed_users.user_id = ?))", from_user_id, to_user_id, to_user_id, from_user_id) if topic.nil? puts "IMPORTING new message topic sanitized: #{sanitized_title} user_id #{from_user_id} to_user_id #{to_user_id}" # Create a private message topic in Discourse topic = Topic.create!( title: sanitized_title, user_id: from_user_id, archetype: Archetype.private_message, created_at: Time.at(row['msg_time']), updated_at: Time.at(row['msg_time']), last_posted_at: Time.at(row['msg_time']), last_post_user_id: from_user_id ) topic.custom_fields['original_gossamer_msg_id'] = row['msg_id'] topic.save! # Add recipient user to the private message topic topic.topic_allowed_users.create!(user_id: to_user_id) # Add sender user to the private message topic topic.topic_allowed_users.create!(user_id: from_user_id) else puts "APPENDING to existing message topic sanitized: #{sanitized_title} user_id #{from_user_id} to_user_id #{to_user_id}" # Increment the number of replies for the topic topic.increment!(:posts_count) end # Create the message as a post in the private topic post = Post.create!( topic_id: topic.id, user_id: from_user_id, raw: sanitized_message, created_at: Time.at(row['msg_time']), updated_at: Time.at(row['msg_time']) ) post.custom_fields['original_gossamer_msg_id'] = row['msg_id'] post.save! # Update the topic's last reply information topic.update!( last_posted_at: post.created_at, last_post_user_id: post.user_id, bumped_at: post.created_at ) update_highest_processed_personal_id(msg_id) # N/A. These were never a thing in Slowtwitch... # handle_post_attachments(row['msg_id'], post, from_user_id) end rescue StandardError => e puts "Error importing message #{row['msg_id']}: #{e.message}" end end end def update_existing_personal_message_activity puts "Updating Activity field for existing private / personal message topics..." private_message_topics = Topic.where(archetype: Archetype.private_message) private_message_topics.each do |topic| last_post = topic.posts.order(created_at: :desc).first if last_post topic.update_columns( last_posted_at: last_post.created_at, last_post_user_id: last_post.user_id, bumped_at: last_post.created_at ) puts "Updated topic ID #{topic.id} with last activity at #{last_post.updated_at}" else puts "No posts found for topic ID #{topic.id}; skipping update." end end end # Main method to perform the import def perform_import(restore_forumpost_id) # Secret trick to disable RateLimiting protection in Discourse puts "CCC" RateLimiter.disable # ActiveRecord::Base.connection.disconnect! rescue nil # ActiveRecord::Base.clear_active_connections! rescue nil # ActiveRecord::Base.establish_connection( # adapter: 'postgresql', # host: '10.0.0.2', # Use the external DB host # port: 5432, # Default PostgreSQL port # database: 'discourse', # username: 'discourse', # password: 'nhB5FWhQkjdvaD2ViRNO63dQagDnzaTn', # connect_timeout: 10 # ) # ENV['PGHOST'] = '10.0.0.2' ENV['PGPORT'] = '5432' ENV['PGDATABASE'] = 'discourse' ENV['PGUSER'] = 'discourse' ENV['PGPASSWORD'] = 'nhB5FWhQkjdvaD2ViRNO63dQagDnzaTn' # ActiveRecord::Base.establish # Set our unique timestamp for this migration run timestamp = Time.now.strftime("-%Y%m%d%H%M%S") puts "Starting Gossamer Forums forum post / OP restoration for id: #{restore_forumpost_id}... #{timestamp}" ### add_former_user ### import_users ### generate_user_id_mapping ### export_username_mapping_to_csv("/bitnami/discourse/sqlite/gossamer-migration-username-mapping#{timestamp}") ### set_user_bio_images ### import_categories ### import_topics_and_posts threaded_topic_restore(restore_forumpost_id) ### update_topic_stats ### update_user_stats ### update_topic_stats_final ### update_user_stats_finalfinal update_topic_stats_final11 update_user_stats_final11 ### export_url_mapping_to_csv("/bitnami/discourse/sqlite/gossamer-migration-url-mapping#{timestamp}") ### export_nginx_rewrite_rules("/bitnami/discourse/sqlite/gossamer-redirects#{timestamp}.conf") #### update_existing_personal_message_activity ### import_personal_messages puts "Gossamer Forums forum post / OP restoration complete for id: #{restore_forumpost_id}... #{timestamp}" end end # Main execution if ARGV.length != 1 puts "Usage: #{$0} " exit 1 end restore_forumpost_id = ARGV[0] puts "BBB" GossamerForumsImporter.new.perform_import(restore_forumpost_id)