v0.20 Make code persistent with SQLite instead of using hashes for stats/numbers for posts
This commit is contained in:
parent
69be52fcf3
commit
0bba1c4638
@ -1,5 +1,5 @@
|
||||
# gossamer threads migration-import code
|
||||
# v0.19.3
|
||||
# v0.20
|
||||
|
||||
require 'mysql2'
|
||||
require 'open-uri'
|
||||
@ -59,12 +59,9 @@ class GossamerForumsImporter < ImportScripts::Base
|
||||
real_name TEXT
|
||||
);
|
||||
SQL
|
||||
@db.execute <<-SQL
|
||||
DROP TABLE IF EXISTS url_map;
|
||||
SQL
|
||||
@db.execute <<-SQL
|
||||
DROP TABLE IF EXISTS url_map2;
|
||||
SQL
|
||||
# @db.execute <<-SQL
|
||||
# DROP TABLE IF EXISTS url_map;
|
||||
# SQL
|
||||
@db.execute <<-SQL
|
||||
CREATE TABLE IF NOT EXISTS url_map (
|
||||
old_post_id INTEGER PRIMARY KEY,
|
||||
@ -72,6 +69,43 @@ class GossamerForumsImporter < ImportScripts::Base
|
||||
title TEXT
|
||||
);
|
||||
SQL
|
||||
@db.execute <<-SQL
|
||||
CREATE TABLE IF NOT EXISTS topic_last_post_time (
|
||||
topic_id INTEGER PRIMARY KEY,
|
||||
last_post_time INTEGER
|
||||
);
|
||||
SQL
|
||||
@db.execute <<-SQL
|
||||
CREATE TABLE IF NOT EXISTS topic_post_count (
|
||||
topic_id INTEGER PRIMARY KEY,
|
||||
post_count INTEGER DEFAULT 0
|
||||
);
|
||||
|
||||
SQL
|
||||
@db.execute <<-SQL
|
||||
CREATE TABLE IF NOT EXISTS user_topic_count (
|
||||
user_id INTEGER PRIMARY KEY,
|
||||
topic_count INTEGER DEFAULT 0
|
||||
);
|
||||
SQL
|
||||
@db.execute <<-SQL
|
||||
CREATE TABLE IF NOT EXISTS user_post_count (
|
||||
user_id INTEGER PRIMARY KEY,
|
||||
post_count INTEGER DEFAULT 0
|
||||
);
|
||||
SQL
|
||||
@db.execute <<-SQL
|
||||
CREATE TABLE IF NOT EXISTS topic_last_post_user (
|
||||
topic_id INTEGER PRIMARY KEY,
|
||||
user_id INTEGER
|
||||
);
|
||||
SQL
|
||||
@db.execute <<-SQL
|
||||
CREATE TABLE IF NOT EXISTS topic_post_numbers (
|
||||
topic_id INTEGER PRIMARY KEY,
|
||||
post_number INTEGER DEFAULT 0
|
||||
);
|
||||
SQL
|
||||
end
|
||||
|
||||
def insert_user_id_mapping(old_user_id, new_user_id)
|
||||
@ -135,6 +169,15 @@ class GossamerForumsImporter < ImportScripts::Base
|
||||
end
|
||||
end
|
||||
|
||||
# Helper methods to interact with the SQLite database for persisting topic-post related values
|
||||
def fetch_from_db(table, key)
|
||||
@db.get_first_value "SELECT value FROM #{table} WHERE key = ?", key
|
||||
end
|
||||
|
||||
def update_db(table, key, value)
|
||||
@db.execute "INSERT OR REPLACE INTO #{table} (key, value) VALUES (?, ?)", key, value
|
||||
end
|
||||
|
||||
# Execute an SQL query on the Gossamer Forums database
|
||||
def execute_query(query)
|
||||
@mysql_client.query(query, as: :hash)
|
||||
@ -657,12 +700,12 @@ class GossamerForumsImporter < ImportScripts::Base
|
||||
def import_topics_and_posts_with_attachments
|
||||
puts "Importing topics and posts with attachments..."
|
||||
|
||||
topic_last_post_time = {}
|
||||
topic_post_count = Hash.new(0)
|
||||
user_topic_count = Hash.new(0)
|
||||
user_post_count = Hash.new(0)
|
||||
topic_last_post_user = {}
|
||||
topic_post_numbers = Hash.new { |hash, key| hash[key] = 0 }
|
||||
# topic_last_post_time = {}
|
||||
# topic_post_count = Hash.new(0)
|
||||
# user_topic_count = Hash.new(0)
|
||||
# user_post_count = Hash.new(0)
|
||||
# topic_last_post_user = {}
|
||||
# topic_post_numbers = Hash.new { |hash, key| hash[key] = 0 }
|
||||
|
||||
# Execute the query to get all posts ordered by post_id
|
||||
execute_query("SELECT * FROM gforum_Post ORDER BY post_id").each do |row|
|
||||
@ -703,11 +746,14 @@ class GossamerForumsImporter < ImportScripts::Base
|
||||
topic.save!
|
||||
|
||||
# Track last post time and user for the topic
|
||||
topic_last_post_time[topic.id] = Time.at(row['post_time'])
|
||||
topic_last_post_user[topic.id] = discourse_user_id
|
||||
# topic_last_post_time[topic.id] = Time.at(row['post_time'])
|
||||
# topic_last_post_user[topic.id] = discourse_user_id
|
||||
update_db('topic_last_post_time', topic.id, Time.at(row['post_time']).to_i)
|
||||
update_db('topic_last_post_user', topic.id, discourse_user_id)
|
||||
|
||||
# Increment the count of the number of topics created by each user
|
||||
user_topic_count[discourse_user_id] += 1
|
||||
# user_topic_count[discourse_user_id] += 1
|
||||
update_db('user_topic_count', discourse_user_id, fetch_from_db(db, 'user_topic_count', discourse_user_id).to_i + 1)
|
||||
|
||||
# # Ensure the raw post stirng contents itself is acceptable to Discourse
|
||||
# sanitized_post_message = row['post_message']&.tr("\0", '') || ""
|
||||
@ -725,7 +771,9 @@ class GossamerForumsImporter < ImportScripts::Base
|
||||
puts "CREATE POST topic.id #{topic.id} discourse_user_id #{discourse_user_id}"
|
||||
|
||||
# Increment the number of posts in the given topic.
|
||||
topic_post_numbers[topic.id] += 1
|
||||
# topic_post_numbers[topic.id] += 1
|
||||
post_number = fetch_from_db(db, 'topic_post_numbers', topic.id).to_i + 1
|
||||
update_db(db, 'topic_post_numbers', topic.id, post_number)
|
||||
|
||||
# Create the initial post in the topic
|
||||
post = Post.create!(
|
||||
@ -738,14 +786,17 @@ class GossamerForumsImporter < ImportScripts::Base
|
||||
updated_at: Time.at(row['post_time']),
|
||||
like_count: row['post_likes'] || 0,
|
||||
reads: post_views || 0,
|
||||
post_number: topic_post_numbers[topic.id]
|
||||
post_number: post_number
|
||||
)
|
||||
# post_number: topic_post_numbers[topic.id]
|
||||
post.custom_fields['original_gossamer_id'] = row['post_id']
|
||||
post.save!
|
||||
|
||||
# Track the number of posts in the topic and by the user
|
||||
topic_post_count[topic.id] += 1
|
||||
user_post_count[discourse_user_id] += 1
|
||||
# topic_post_count[topic.id] += 1
|
||||
# user_post_count[discourse_user_id] += 1
|
||||
update_db('topic_post_count', topic.id, fetch_from_db(db, 'topic_post_count', topic.id).to_i + 1)
|
||||
update_db('user_post_count', discourse_user_id, fetch_from_db(db, 'user_post_count', discourse_user_id).to_i + 1)
|
||||
|
||||
# Handle attachments for the post
|
||||
handle_post_attachments(row['post_id'], post, discourse_user_id)
|
||||
@ -782,9 +833,11 @@ class GossamerForumsImporter < ImportScripts::Base
|
||||
# Sanitize the post message
|
||||
sanitized_post_message = sanitize_post_message(row['post_message'])
|
||||
|
||||
# Create the post in the existing topic
|
||||
topic_post_numbers[topic_id] += 1
|
||||
# topic_post_numbers[topic_id] += 1
|
||||
post_number = fetch_from_db(db, 'topic_post_numbers', topic_id).to_i + 1
|
||||
update_db('topic_post_numbers', topic_id, post_number)
|
||||
|
||||
# Create the post in the existing topic
|
||||
post = Post.create!(
|
||||
topic_id: topic_id,
|
||||
user_id: discourse_user_id,
|
||||
@ -795,20 +848,26 @@ class GossamerForumsImporter < ImportScripts::Base
|
||||
updated_at: Time.at(row['post_time']),
|
||||
reply_to_post_number: reply_to_post_number,
|
||||
like_count: row['post_replies'] || 0,
|
||||
reads: post_views || topic_post_count[topic_id],
|
||||
post_number: topic_post_numbers[topic_id]
|
||||
reads: post_views || fetch_from_db('topic_post_count', topic_id).to_i,
|
||||
post_number: post_number
|
||||
)
|
||||
post.custom_fields['original_gossamer_id'] = row['post_id']
|
||||
post.save!
|
||||
|
||||
# Track the number of posts in the topic and by the user
|
||||
topic_post_count[topic_id] += 1
|
||||
user_post_count[discourse_user_id] += 1
|
||||
# topic_post_count[topic_id] += 1
|
||||
# user_post_count[discourse_user_id] += 1
|
||||
update_db('topic_post_count', topic_id, fetch_from_db('topic_post_count', topic_id).to_i + 1)
|
||||
update_db('user_post_count', discourse_user_id, fetch_from_db('user_post_count', discourse_user_id).to_i + 1)
|
||||
|
||||
# Update last post time and user for the topic
|
||||
if topic_last_post_time[topic_id].nil? || Time.at(row['post_time']) > topic_last_post_time[topic_id]
|
||||
topic_last_post_time[topic_id] = Time.at(row['post_time'])
|
||||
topic_last_post_user[topic_id] = discourse_user_id
|
||||
# if topic_last_post_time[topic_id].nil? || Time.at(row['post_time']) > topic_last_post_time[topic_id]
|
||||
# topic_last_post_time[topic_id] = Time.at(row['post_time'])
|
||||
# topic_last_post_user[topic_id] = discourse_user_id
|
||||
# end
|
||||
if fetch_from_db('topic_last_post_time', topic_id).nil? || Time.at(row['post_time']).to_i > fetch_from_db('topic_last_post_time', topic_id).to_i
|
||||
update_db('topic_last_post_time', topic_id, Time.at(row['post_time']).to_i)
|
||||
update_db('topic_last_post_user', topic_id, discourse_user_id)
|
||||
end
|
||||
|
||||
# Handle attachments for the post
|
||||
@ -824,23 +883,43 @@ class GossamerForumsImporter < ImportScripts::Base
|
||||
end
|
||||
|
||||
# Update topics with the correct last post time, post count, and last post user
|
||||
topic_last_post_time.each do |topic_id, last_post_time|
|
||||
# topic_last_post_time.each do |topic_id, last_post_time|
|
||||
# Topic.find(topic_id).update!(
|
||||
# updated_at: last_post_time,
|
||||
# posts_count: topic_post_count[topic_id],
|
||||
# last_posted_at: last_post_time,
|
||||
# bumped_at: last_post_time,
|
||||
# last_post_user_id: topic_last_post_user[topic_id]
|
||||
# )
|
||||
# end
|
||||
@db.execute("SELECT * FROM topic_last_post_time").each do |row|
|
||||
topic_id, last_post_time = row
|
||||
Topic.find(topic_id).update!(
|
||||
updated_at: last_post_time,
|
||||
posts_count: topic_post_count[topic_id],
|
||||
last_posted_at: last_post_time,
|
||||
bumped_at: last_post_time,
|
||||
last_post_user_id: topic_last_post_user[topic_id]
|
||||
updated_at: Time.at(last_post_time),
|
||||
posts_count: fetch_from_db(db, 'topic_post_count', topic_id).to_i,
|
||||
last_posted_at: Time.at(last_post_time),
|
||||
bumped_at: Time.at(last_post_time),
|
||||
last_post_user_id: fetch_from_db(db, 'topic_last_post_user', topic_id).to_i
|
||||
)
|
||||
end
|
||||
|
||||
# Update user profiles with the number of topics and posts created
|
||||
user_topic_count.each do |user_id, count|
|
||||
# user_topic_count.each do |user_id, count|
|
||||
# user = User.find(user_id)
|
||||
# user.update!(topic_count: count)
|
||||
# end
|
||||
# user_post_count.each do |user_id, count|
|
||||
# user = User.find(user_id)
|
||||
# user.update!(post_count: count)
|
||||
# end
|
||||
@db.execute("SELECT * FROM user_topic_count").each do |row|
|
||||
user_id, count = row
|
||||
user = User.find(user_id)
|
||||
user.update!(topic_count: count)
|
||||
end
|
||||
|
||||
user_post_count.each do |user_id, count|
|
||||
|
||||
@db.execute("SELECT * FROM user_post_count").each do |row|
|
||||
user_id, count = row
|
||||
user = User.find(user_id)
|
||||
user.update!(post_count: count)
|
||||
end
|
||||
|
||||
Loading…
Reference in New Issue
Block a user