From 958e37771efc15e491b3e4b0f0ad93ce18f82124 Mon Sep 17 00:00:00 2001 From: saint Date: Mon, 2 Sep 2024 13:45:44 +1000 Subject: [PATCH] v0.59 Final topic-post import run -- add logic for appending counts since we are adding to a live Discourse host now --- gossamer_forums.rb | 182 ++++++++++++++++++++++++++++++++++----------- 1 file changed, 139 insertions(+), 43 deletions(-) diff --git a/gossamer_forums.rb b/gossamer_forums.rb index 1d2ac45..3a93e69 100644 --- a/gossamer_forums.rb +++ b/gossamer_forums.rb @@ -1,7 +1,7 @@ # Federated Computer, Inc. # David Sainty 2024 A.D. # Gossamer Threads to Discourse -- Migration-Import Script -# v0.58 Final user import and personal message import run +# v0.59 Final topic-post import run -- add logic for appending counts since we are adding to a live Discourse host now require 'mysql2' require 'open-uri' @@ -114,47 +114,47 @@ class GossamerForumsImporter < ImportScripts::Base SQL # POST IMPORT - For each topic, the time of the last / most recent post/reply @db.execute <<-SQL - CREATE TABLE IF NOT EXISTS topic_last_post_time ( + CREATE TABLE IF NOT EXISTS topic_last_post_time_final ( topic_id INTEGER PRIMARY KEY, last_post_time INTEGER ); SQL # POST IMPORT - For each topic, increment post_count as we add posts. @db.execute <<-SQL - CREATE TABLE IF NOT EXISTS topic_post_count ( + CREATE TABLE IF NOT EXISTS topic_post_count_final ( topic_id INTEGER PRIMARY KEY, post_count INTEGER DEFAULT 0 ); SQL # POST IMPORT - For each user (_id), increment topic_count as we add topics (to see total topics per user) @db.execute <<-SQL - CREATE TABLE IF NOT EXISTS user_topic_count ( + CREATE TABLE IF NOT EXISTS user_topic_count_final ( user_id INTEGER PRIMARY KEY, topic_count INTEGER DEFAULT 0 ); SQL # POST IMPORT - For each user (_id), increment post_count as we add posts (to see total posts per user) @db.execute <<-SQL - CREATE TABLE IF NOT EXISTS user_post_count ( + CREATE TABLE IF NOT EXISTS user_post_count_final ( user_id INTEGER PRIMARY KEY, post_count INTEGER DEFAULT 0 ); SQL # POST IMPORT - For each topic, the user_id for the last poster / replier @db.execute <<-SQL - CREATE TABLE IF NOT EXISTS topic_last_post_user ( + CREATE TABLE IF NOT EXISTS topic_last_post_user_final ( topic_id INTEGER PRIMARY KEY, user_id INTEGER ); SQL # POST IMPORT - The number of posts in a given topic, incremented as we add a new reply post to a topic. @db.execute <<-SQL - CREATE TABLE IF NOT EXISTS topic_post_numbers ( + CREATE TABLE IF NOT EXISTS topic_post_numbers_final ( topic_id INTEGER PRIMARY KEY, post_number INTEGER DEFAULT 0 ); SQL - # POST IMPORT - Record perssitent integer value for highest processed post id -- not used + # POST IMPORT - Record perssitent integer value for highest processed post id @db.execute <<-SQL CREATE TABLE IF NOT EXISTS highest_processed_post_id ( id INTEGER PRIMARY KEY CHECK (id = 1), @@ -253,51 +253,51 @@ class GossamerForumsImporter < ImportScripts::Base end def fetch_db_topic_last_post_time(topic_id) - @db.get_first_value "SELECT last_post_time FROM topic_last_post_time WHERE topic_id = ?", topic_id + @db.get_first_value "SELECT last_post_time FROM topic_last_post_time_final WHERE topic_id = ?", topic_id end def fetch_db_topic_last_post_user(topic_id) - @db.get_first_value "SELECT user_id FROM topic_last_post_user WHERE topic_id = ?", topic_id + @db.get_first_value "SELECT user_id FROM topic_last_post_user_final WHERE topic_id = ?", topic_id end def fetch_db_topic_post_count(topic_id) - @db.get_first_value "SELECT post_count FROM topic_post_count WHERE topic_id = ?", topic_id + @db.get_first_value "SELECT post_count FROM topic_post_count_final WHERE topic_id = ?", topic_id end def fetch_db_user_topic_count(user_id) - @db.get_first_value "SELECT topic_count FROM user_topic_count WHERE user_id = ?", user_id + @db.get_first_value "SELECT topic_count FROM user_topic_count_final WHERE user_id = ?", user_id end def fetch_db_user_post_count(user_id) - @db.get_first_value "SELECT post_count FROM user_post_count WHERE user_id = ?", user_id + @db.get_first_value "SELECT post_count FROM user_post_count_final WHERE user_id = ?", user_id end def fetch_db_topic_post_numbers(topic_id) - @db.get_first_value "SELECT post_number FROM topic_post_numbers WHERE topic_id = ?", topic_id + @db.get_first_value "SELECT post_number FROM topic_post_numbers_final WHERE topic_id = ?", topic_id end def update_db_topic_last_post_time(topic_id, last_post_time) - @db.execute "INSERT OR REPLACE INTO topic_last_post_time (topic_id, last_post_time) VALUES (?, ?)", topic_id, last_post_time + @db.execute "INSERT OR REPLACE INTO topic_last_post_time_final (topic_id, last_post_time) VALUES (?, ?)", topic_id, last_post_time end def update_db_topic_last_post_user(topic_id, user_id) - @db.execute "INSERT OR REPLACE INTO topic_last_post_user (topic_id, user_id) VALUES (?, ?)", topic_id, user_id + @db.execute "INSERT OR REPLACE INTO topic_last_post_user_final (topic_id, user_id) VALUES (?, ?)", topic_id, user_id end def update_db_topic_post_count(topic_id, post_count) - @db.execute "INSERT OR REPLACE INTO topic_post_count (topic_id, post_count) VALUES (?, ?)", topic_id, post_count + @db.execute "INSERT OR REPLACE INTO topic_post_count_final (topic_id, post_count) VALUES (?, ?)", topic_id, post_count end def update_db_user_topic_count(user_id, topic_count) - @db.execute "INSERT OR REPLACE INTO user_topic_count (user_id, topic_count) VALUES (?, ?)", user_id, topic_count + @db.execute "INSERT OR REPLACE INTO user_topic_count_final (user_id, topic_count) VALUES (?, ?)", user_id, topic_count end def update_db_user_post_count(user_id, post_count) - @db.execute "INSERT OR REPLACE INTO user_post_count (user_id, post_count) VALUES (?, ?)", user_id, post_count + @db.execute "INSERT OR REPLACE INTO user_post_count_final (user_id, post_count) VALUES (?, ?)", user_id, post_count end def update_db_topic_post_numbers(topic_id, post_number) - @db.execute "INSERT OR REPLACE INTO topic_post_numbers (topic_id, post_number) VALUES (?, ?)", topic_id, post_number + @db.execute "INSERT OR REPLACE INTO topic_post_numbers_final (topic_id, post_number) VALUES (?, ?)", topic_id, post_number end # Fetch the highest processed post_id from the highest_processed_post_id table @@ -1555,12 +1555,13 @@ class GossamerForumsImporter < ImportScripts::Base # Fetch the highest old_post_id from the url_map table # highest_old_post_id = fetch_highest_old_post_id.to_i # puts "Highest (OP) old_post_id in url_map: #{highest_old_post_id}" -### highest_processed_post_id = fetch_highest_processed_post_id.to_i - highest_processed_post_id = 8179621 + highest_processed_post_id = fetch_highest_processed_post_id.to_i puts "Highest processed post_id: #{highest_processed_post_id}" # OVERRIDE........ # Attachment example: highest_processed_post_id = 1359862 + highest_processed_post_id = 8179621 + puts "OVERRIDE Highest processed post_id: #{highest_processed_post_id}" # Execute the query to get all posts ordered by post_id execute_query("SELECT post_id, user_id_fk, forum_id_fk, post_root_id, post_subject, post_time, post_message, post_father_id, post_likes, post_replies FROM gforum_Post WHERE post_id > #{highest_processed_post_id} ORDER BY post_id").each do |row| @@ -1635,11 +1636,14 @@ class GossamerForumsImporter < ImportScripts::Base end end end + + current_topic_id = topic.id + # Track last post time and user for the topic # topic_last_post_time[topic.id] = Time.at(row['post_time']) # topic_last_post_user[topic.id] = discourse_user_id - update_db_topic_last_post_time(topic.id, Time.at(row['post_time']).to_i) - update_db_topic_last_post_user(topic.id, discourse_user_id) + update_db_topic_last_post_time(current_topic_id, Time.at(row['post_time']).to_i) + update_db_topic_last_post_user(current_topic_id, discourse_user_id) # Increment the count of the number of topics created by each user # user_topic_count[discourse_user_id] += 1 @@ -1658,16 +1662,16 @@ class GossamerForumsImporter < ImportScripts::Base # Sanitize the post message sanitized_post_message = sanitize_post_message(row['post_message']) - puts "CREATE TOPIC POST topic.id #{topic.id} discourse_user_id #{discourse_user_id}" + puts "CREATE TOPIC POST current_topic_id #{current_topic_id} discourse_user_id #{discourse_user_id}" # Increment the number of posts in the given topic. # topic_post_numbers[topic.id] += 1 - post_number = fetch_db_topic_post_numbers(topic.id).to_i + 1 - update_db_topic_post_numbers(topic.id, post_number) + post_number = fetch_db_topic_post_numbers(current_topic_id).to_i + 1 + update_db_topic_post_numbers(current_topic_id, post_number) # Create the initial post in the topic post = Post.create!( - topic_id: topic.id, + topic_id: current_topic_id, user_id: discourse_user_id, # raw: import_attachments(row['post_message'], row['post_id']), # raw: row['post_message'] || "", @@ -1685,11 +1689,11 @@ class GossamerForumsImporter < ImportScripts::Base # Track the number of posts in the topic and by the user # topic_post_count[topic.id] += 1 # user_post_count[discourse_user_id] += 1 - update_db_topic_post_count(topic.id, fetch_db_topic_post_count(topic.id).to_i + 1) + update_db_topic_post_count(current_topic_id, fetch_db_topic_post_count(current_topic_id).to_i + 1) update_db_user_post_count(discourse_user_id, fetch_db_user_post_count(discourse_user_id).to_i + 1) # Handle attachments for the post - handle_post_attachments(row['post_id'], post, discourse_user_id) + handle_post_attachments(row['post_id'], post, discourse_user_id, @mysql_client) # Create URL mappings # old_url = "https://old/forum/#{row['forum_name']}/topics/#{row['post_id']}" @@ -1727,6 +1731,8 @@ class GossamerForumsImporter < ImportScripts::Base # Sanitize the post message sanitized_post_message = sanitize_post_message(row['post_message']) + puts "CREATE REPLY in topic_id #{topic_id} for reply post_id #{row['post_id']}" + # topic_post_numbers[topic_id] += 1 post_number = fetch_db_topic_post_numbers(topic_id).to_i + 1 update_db_topic_post_numbers(topic_id, post_number) @@ -1735,16 +1741,17 @@ class GossamerForumsImporter < ImportScripts::Base post = Post.create!( topic_id: topic_id, user_id: discourse_user_id, -# raw: import_attachments(row['post_message'], row['post_id']), -# raw: row['post_message'] || "", raw: sanitized_post_message, created_at: Time.at(row['post_time']), updated_at: Time.at(row['post_time']), - reply_to_post_number: reply_to_post_number, - like_count: row['post_replies'] || 0, - reads: post_views || fetch_db_topic_post_count(topic_id).to_i, + reads: post_views || 0, post_number: post_number ) +# raw: import_attachments(row['post_message'], row['post_id']), +# raw: row['post_message'] || "", +# reply_to_post_number: reply_to_post_number, +# like_count: row['post_replies'] || 0, +# reads: post_views || fetch_db_topic_post_count(topic_id).to_i, post.custom_fields['original_gossamer_id'] = row['post_id'] post.save! @@ -1765,7 +1772,7 @@ class GossamerForumsImporter < ImportScripts::Base end # Handle attachments for the post - handle_post_attachments(row['post_id'], post, discourse_user_id) + handle_post_attachments(row['post_id'], post, discourse_user_id, @mysql_client) # Update the highest processed post_id puts "Updated highest processed post_id #{post_id}" @@ -1794,7 +1801,7 @@ class GossamerForumsImporter < ImportScripts::Base # last_post_user_id: topic_last_post_user[topic_id] # ) # end - @db.execute("SELECT * FROM topic_last_post_time").each do |row| + @db.execute("SELECT * FROM topic_last_post_time_final").each do |row| topic_id, last_post_time = row begin @@ -1815,6 +1822,38 @@ class GossamerForumsImporter < ImportScripts::Base end end + def update_topic_stats_final + # Update topics with the correct last post time, post count, and last post user + puts "Update topics with the correct last post time, post count, and last post user" + + @db.execute("SELECT * FROM topic_last_post_time_final").each do |row| + topic_id, last_post_time = row + + begin + topic = Topic.find(topic_id) + + # Calculate the new values based on the given conditions + new_updated_at = [Time.at(last_post_time), topic.updated_at].max + new_posts_count = topic.posts_count + fetch_db_topic_post_count(topic_id).to_i + new_last_posted_at = [Time.at(last_post_time), topic.last_posted_at].max + new_bumped_at = [Time.at(last_post_time), topic.bumped_at].max + new_last_post_user_id = fetch_db_topic_last_post_user(topic_id).to_i + + # Update the topic with the calculated values + topic.update_columns( + updated_at: new_updated_at, + posts_count: new_posts_count, + last_posted_at: new_last_posted_at, + bumped_at: new_bumped_at, + last_post_user_id: new_last_post_user_id + ) + + rescue ActiveRecord::RecordNotFound + puts "WARNING: Could not find Topic with id=#{topic_id}. Skipping..." + end + end + end + def update_user_stats # Update user profiles with the number of topics and posts created puts "Update user profiles with the number of topics and posts created" @@ -1826,7 +1865,7 @@ class GossamerForumsImporter < ImportScripts::Base # user = User.find(user_id) # user.update!(post_count: count) # end - @db.execute("SELECT * FROM user_topic_count").each do |row| + @db.execute("SELECT * FROM user_topic_count_final").each do |row| user_id, count = row # user = User.find(user_id) # user.update!(topic_count: count) @@ -1839,7 +1878,7 @@ class GossamerForumsImporter < ImportScripts::Base end end - @db.execute("SELECT * FROM user_post_count").each do |row| + @db.execute("SELECT * FROM user_post_count_final").each do |row| user_id, count = row # user = User.find(user_id) # user.update!(post_count: count) @@ -1871,6 +1910,59 @@ class GossamerForumsImporter < ImportScripts::Base end end + def update_user_stats_final + # Update user profiles with the number of topics and posts created + puts "Update user profiles with the number of topics and posts created" + @db.execute("SELECT * FROM user_topic_count_final").each do |row| + user_id, count = row + puts "update_user_stats user_id #{user_id} topic_count #{count}" + begin + user_stat = UserStat.find_or_initialize_by(user_id: user_id) +# user_stat.update_columns(topic_count: count) + + # Accumulate the current topic count with the new count + new_topic_count = user_stat.topic_count.to_i + count.to_i + user_stat.update_columns(topic_count: new_topic_count) + rescue + puts "WARNING: Could not find User with id=#{user_id}. Skipping..." + end + end + + @db.execute("SELECT * FROM user_post_count_final").each do |row| + user_id, count = row + puts "update_user_stats user_id #{user_id} post_count #{count}" + begin + user_stat = UserStat.find_or_initialize_by(user_id: user_id) +# user_stat.update_columns(post_count: count) + + # Accumulate the current post count with the new count + new_post_count = user_stat.post_count.to_i + count.to_i + user_stat.update_columns(post_count: new_post_count) + + # Fetch the current user and check if Trust Level needs updating + new_trust_level = case count + when 0..29 then 1 # basic user + else 2 # member, regular reserved for now. +# when 3..50 then 2 # member +# else 3 # regular or above when 51..100 + end + user = User.find(user_id) + current_trust_level = user.trust_level || 1 # default to 1 if not set + + # Only update trust level if the new level is higher than the current one + if new_trust_level != current_trust_level + user.update!(trust_level: new_trust_level) + puts "update_user_stats user_id #{user_id} trust_level was #{current_trust_level}, now updated to #{new_trust_level}" + else + puts "update_user_stats user_id #{user_id} trust_level remains at #{current_trust_level}" + end + rescue + puts "WARNING: Could not find or modify User with id=#{user_id}. Skipping..." + end + end + end + + # Import personal messages from gforum_Message table (both inbox and sent messages) def import_personal_messages @@ -2052,16 +2144,20 @@ class GossamerForumsImporter < ImportScripts::Base ### import_categories -## import_topics_and_posts + import_topics_and_posts # threaded_topic_import -### update_topic_stats -### update_user_stats +#### update_topic_stats +#### update_user_stats + + update_topic_stats_final + update_user_stats_final + ### export_url_mapping_to_csv("/bitnami/discourse/sqlite/gossamer-migration-url-mapping#{timestamp}") ### export_nginx_rewrite_rules("/bitnami/discourse/sqlite/gossamer-redirects#{timestamp}.conf") #### update_existing_personal_message_activity - import_personal_messages +### import_personal_messages puts "Gossamer Forums import complete! #{timestamp}" end