From 0bba1c4638e00276fdd35b33ecd7cfc49d3d2c9d Mon Sep 17 00:00:00 2001 From: saint Date: Fri, 28 Jun 2024 22:25:26 +1000 Subject: [PATCH] v0.20 Make code persistent with SQLite instead of using hashes for stats/numbers for posts --- gossamer_forums.rb | 155 ++++++++++++++++++++++++++++++++++----------- 1 file changed, 117 insertions(+), 38 deletions(-) diff --git a/gossamer_forums.rb b/gossamer_forums.rb index 13a46de..be84983 100644 --- a/gossamer_forums.rb +++ b/gossamer_forums.rb @@ -1,5 +1,5 @@ # gossamer threads migration-import code -# v0.19.3 +# v0.20 require 'mysql2' require 'open-uri' @@ -59,12 +59,9 @@ class GossamerForumsImporter < ImportScripts::Base real_name TEXT ); SQL - @db.execute <<-SQL - DROP TABLE IF EXISTS url_map; - SQL - @db.execute <<-SQL - DROP TABLE IF EXISTS url_map2; - SQL +# @db.execute <<-SQL +# DROP TABLE IF EXISTS url_map; +# SQL @db.execute <<-SQL CREATE TABLE IF NOT EXISTS url_map ( old_post_id INTEGER PRIMARY KEY, @@ -72,6 +69,43 @@ class GossamerForumsImporter < ImportScripts::Base title TEXT ); SQL + @db.execute <<-SQL + CREATE TABLE IF NOT EXISTS topic_last_post_time ( + topic_id INTEGER PRIMARY KEY, + last_post_time INTEGER + ); + SQL + @db.execute <<-SQL + CREATE TABLE IF NOT EXISTS topic_post_count ( + topic_id INTEGER PRIMARY KEY, + post_count INTEGER DEFAULT 0 + ); + + SQL + @db.execute <<-SQL + CREATE TABLE IF NOT EXISTS user_topic_count ( + user_id INTEGER PRIMARY KEY, + topic_count INTEGER DEFAULT 0 + ); + SQL + @db.execute <<-SQL + CREATE TABLE IF NOT EXISTS user_post_count ( + user_id INTEGER PRIMARY KEY, + post_count INTEGER DEFAULT 0 + ); + SQL + @db.execute <<-SQL + CREATE TABLE IF NOT EXISTS topic_last_post_user ( + topic_id INTEGER PRIMARY KEY, + user_id INTEGER + ); + SQL + @db.execute <<-SQL + CREATE TABLE IF NOT EXISTS topic_post_numbers ( + topic_id INTEGER PRIMARY KEY, + post_number INTEGER DEFAULT 0 + ); + SQL end def insert_user_id_mapping(old_user_id, new_user_id) @@ -135,6 +169,15 @@ class GossamerForumsImporter < ImportScripts::Base end end + # Helper methods to interact with the SQLite database for persisting topic-post related values + def fetch_from_db(table, key) + @db.get_first_value "SELECT value FROM #{table} WHERE key = ?", key + end + + def update_db(table, key, value) + @db.execute "INSERT OR REPLACE INTO #{table} (key, value) VALUES (?, ?)", key, value + end + # Execute an SQL query on the Gossamer Forums database def execute_query(query) @mysql_client.query(query, as: :hash) @@ -657,12 +700,12 @@ class GossamerForumsImporter < ImportScripts::Base def import_topics_and_posts_with_attachments puts "Importing topics and posts with attachments..." - topic_last_post_time = {} - topic_post_count = Hash.new(0) - user_topic_count = Hash.new(0) - user_post_count = Hash.new(0) - topic_last_post_user = {} - topic_post_numbers = Hash.new { |hash, key| hash[key] = 0 } +# topic_last_post_time = {} +# topic_post_count = Hash.new(0) +# user_topic_count = Hash.new(0) +# user_post_count = Hash.new(0) +# topic_last_post_user = {} +# topic_post_numbers = Hash.new { |hash, key| hash[key] = 0 } # Execute the query to get all posts ordered by post_id execute_query("SELECT * FROM gforum_Post ORDER BY post_id").each do |row| @@ -703,11 +746,14 @@ class GossamerForumsImporter < ImportScripts::Base topic.save! # Track last post time and user for the topic - topic_last_post_time[topic.id] = Time.at(row['post_time']) - topic_last_post_user[topic.id] = discourse_user_id +# topic_last_post_time[topic.id] = Time.at(row['post_time']) +# topic_last_post_user[topic.id] = discourse_user_id + update_db('topic_last_post_time', topic.id, Time.at(row['post_time']).to_i) + update_db('topic_last_post_user', topic.id, discourse_user_id) # Increment the count of the number of topics created by each user - user_topic_count[discourse_user_id] += 1 +# user_topic_count[discourse_user_id] += 1 + update_db('user_topic_count', discourse_user_id, fetch_from_db(db, 'user_topic_count', discourse_user_id).to_i + 1) # # Ensure the raw post stirng contents itself is acceptable to Discourse # sanitized_post_message = row['post_message']&.tr("\0", '') || "" @@ -725,7 +771,9 @@ class GossamerForumsImporter < ImportScripts::Base puts "CREATE POST topic.id #{topic.id} discourse_user_id #{discourse_user_id}" # Increment the number of posts in the given topic. - topic_post_numbers[topic.id] += 1 +# topic_post_numbers[topic.id] += 1 + post_number = fetch_from_db(db, 'topic_post_numbers', topic.id).to_i + 1 + update_db(db, 'topic_post_numbers', topic.id, post_number) # Create the initial post in the topic post = Post.create!( @@ -738,14 +786,17 @@ class GossamerForumsImporter < ImportScripts::Base updated_at: Time.at(row['post_time']), like_count: row['post_likes'] || 0, reads: post_views || 0, - post_number: topic_post_numbers[topic.id] + post_number: post_number ) +# post_number: topic_post_numbers[topic.id] post.custom_fields['original_gossamer_id'] = row['post_id'] post.save! # Track the number of posts in the topic and by the user - topic_post_count[topic.id] += 1 - user_post_count[discourse_user_id] += 1 +# topic_post_count[topic.id] += 1 +# user_post_count[discourse_user_id] += 1 + update_db('topic_post_count', topic.id, fetch_from_db(db, 'topic_post_count', topic.id).to_i + 1) + update_db('user_post_count', discourse_user_id, fetch_from_db(db, 'user_post_count', discourse_user_id).to_i + 1) # Handle attachments for the post handle_post_attachments(row['post_id'], post, discourse_user_id) @@ -782,9 +833,11 @@ class GossamerForumsImporter < ImportScripts::Base # Sanitize the post message sanitized_post_message = sanitize_post_message(row['post_message']) - # Create the post in the existing topic - topic_post_numbers[topic_id] += 1 +# topic_post_numbers[topic_id] += 1 + post_number = fetch_from_db(db, 'topic_post_numbers', topic_id).to_i + 1 + update_db('topic_post_numbers', topic_id, post_number) + # Create the post in the existing topic post = Post.create!( topic_id: topic_id, user_id: discourse_user_id, @@ -795,20 +848,26 @@ class GossamerForumsImporter < ImportScripts::Base updated_at: Time.at(row['post_time']), reply_to_post_number: reply_to_post_number, like_count: row['post_replies'] || 0, - reads: post_views || topic_post_count[topic_id], - post_number: topic_post_numbers[topic_id] + reads: post_views || fetch_from_db('topic_post_count', topic_id).to_i, + post_number: post_number ) post.custom_fields['original_gossamer_id'] = row['post_id'] post.save! # Track the number of posts in the topic and by the user - topic_post_count[topic_id] += 1 - user_post_count[discourse_user_id] += 1 +# topic_post_count[topic_id] += 1 +# user_post_count[discourse_user_id] += 1 + update_db('topic_post_count', topic_id, fetch_from_db('topic_post_count', topic_id).to_i + 1) + update_db('user_post_count', discourse_user_id, fetch_from_db('user_post_count', discourse_user_id).to_i + 1) # Update last post time and user for the topic - if topic_last_post_time[topic_id].nil? || Time.at(row['post_time']) > topic_last_post_time[topic_id] - topic_last_post_time[topic_id] = Time.at(row['post_time']) - topic_last_post_user[topic_id] = discourse_user_id +# if topic_last_post_time[topic_id].nil? || Time.at(row['post_time']) > topic_last_post_time[topic_id] +# topic_last_post_time[topic_id] = Time.at(row['post_time']) +# topic_last_post_user[topic_id] = discourse_user_id +# end + if fetch_from_db('topic_last_post_time', topic_id).nil? || Time.at(row['post_time']).to_i > fetch_from_db('topic_last_post_time', topic_id).to_i + update_db('topic_last_post_time', topic_id, Time.at(row['post_time']).to_i) + update_db('topic_last_post_user', topic_id, discourse_user_id) end # Handle attachments for the post @@ -824,23 +883,43 @@ class GossamerForumsImporter < ImportScripts::Base end # Update topics with the correct last post time, post count, and last post user - topic_last_post_time.each do |topic_id, last_post_time| +# topic_last_post_time.each do |topic_id, last_post_time| +# Topic.find(topic_id).update!( +# updated_at: last_post_time, +# posts_count: topic_post_count[topic_id], +# last_posted_at: last_post_time, +# bumped_at: last_post_time, +# last_post_user_id: topic_last_post_user[topic_id] +# ) +# end + @db.execute("SELECT * FROM topic_last_post_time").each do |row| + topic_id, last_post_time = row Topic.find(topic_id).update!( - updated_at: last_post_time, - posts_count: topic_post_count[topic_id], - last_posted_at: last_post_time, - bumped_at: last_post_time, - last_post_user_id: topic_last_post_user[topic_id] + updated_at: Time.at(last_post_time), + posts_count: fetch_from_db(db, 'topic_post_count', topic_id).to_i, + last_posted_at: Time.at(last_post_time), + bumped_at: Time.at(last_post_time), + last_post_user_id: fetch_from_db(db, 'topic_last_post_user', topic_id).to_i ) end # Update user profiles with the number of topics and posts created - user_topic_count.each do |user_id, count| +# user_topic_count.each do |user_id, count| +# user = User.find(user_id) +# user.update!(topic_count: count) +# end +# user_post_count.each do |user_id, count| +# user = User.find(user_id) +# user.update!(post_count: count) +# end + @db.execute("SELECT * FROM user_topic_count").each do |row| + user_id, count = row user = User.find(user_id) user.update!(topic_count: count) end - - user_post_count.each do |user_id, count| + + @db.execute("SELECT * FROM user_post_count").each do |row| + user_id, count = row user = User.find(user_id) user.update!(post_count: count) end