saint/discourse-import_scripts
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

v0.48.1 Further attempts to address MariaDB challenges and make things as foolproof as possible.

Browse Source
This commit is contained in:
David Sainty 2024-08-18 18:57:22 +10:00
parent 96c2b27250
commit b0f300bf8e
1 changed files with 51 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
gossamer_forums.rb
View File

@ -62,25 +62,50 @@ class GossamerForumsImporter < ImportScripts::Base
@db = SQLite3::Database.new '/bitnami/discourse/sqlite/id_name_url_map.db'
###### ONLY when we need to clear the url_map and topic_import_status .... e.g. if reimporting topics-posts from scratch
# @db.execute <<-SQL
# DROP TABLE IF EXISTS url_map;
# SQL
# @db.execute <<-SQL
# DROP TABLE IF EXISTS topic_import_status;
# SQL
@db.execute <<-SQL
DROP TABLE IF EXISTS url_map;
SQL
@db.execute <<-SQL
DROP TABLE IF EXISTS topic_last_post_time;
SQL
@db.execute <<-SQL
DROP TABLE IF EXISTS topic_post_count;
SQL
@db.execute <<-SQL
DROP TABLE IF EXISTS user_topic_count;
SQL
@db.execute <<-SQL
DROP TABLE IF EXISTS user_post_count;
SQL
@db.execute <<-SQL
DROP TABLE IF EXISTS topic_last_post_user;
SQL
@db.execute <<-SQL
DROP TABLE IF EXISTS topic_post_numbers;
SQL
@db.execute <<-SQL
DROP TABLE IF EXISTS highest_processed_post_id;
SQL
@db.execute <<-SQL
DROP TABLE IF EXISTS topic_import_status;
SQL
#######
# USER IMPORT - map of old to new user ids, used for topic-post import.
@db.execute <<-SQL
CREATE TABLE IF NOT EXISTS user_id_map (
old_user_id INTEGER PRIMARY KEY,
new_user_id INTEGER
);
SQL
# CATEGORY IMPORT - map of old to new category ids, used for topic-post import.
@db.execute <<-SQL
CREATE TABLE IF NOT EXISTS category_id_map (
old_category_id INTEGER PRIMARY KEY,
new_category_id INTEGER
);
SQL
# USER IMPORT - map of old to new usernames for SENDING MIGRATION EMAIL
@db.execute <<-SQL
CREATE TABLE IF NOT EXISTS username_map (
id INTEGER PRIMARY KEY,
@ -90,6 +115,7 @@ class GossamerForumsImporter < ImportScripts::Base
real_name TEXT
);
SQL
# POST IMPORT - Generate a map of old_post_id (Gossamer) and the new URL for a WEB SERVER REDIRECT FILE
@db.execute <<-SQL
CREATE TABLE IF NOT EXISTS url_map (
old_post_id INTEGER PRIMARY KEY,
@ -97,54 +123,63 @@ class GossamerForumsImporter < ImportScripts::Base
title TEXT
);
SQL
# POST IMPORT - For each topic, the time of the last / most recent post/reply
@db.execute <<-SQL
CREATE TABLE IF NOT EXISTS topic_last_post_time (
topic_id INTEGER PRIMARY KEY,
last_post_time INTEGER
);
SQL
# POST IMPORT - For each topic, increment post_count as we add posts.
@db.execute <<-SQL
CREATE TABLE IF NOT EXISTS topic_post_count (
topic_id INTEGER PRIMARY KEY,
post_count INTEGER DEFAULT 0
);
SQL
# POST IMPORT - For each user (_id), increment topic_count as we add topics (to see total topics per user)
@db.execute <<-SQL
CREATE TABLE IF NOT EXISTS user_topic_count (
user_id INTEGER PRIMARY KEY,
topic_count INTEGER DEFAULT 0
);
SQL
# POST IMPORT - For each user (_id), increment post_count as we add posts (to see total posts per user)
@db.execute <<-SQL
CREATE TABLE IF NOT EXISTS user_post_count (
user_id INTEGER PRIMARY KEY,
post_count INTEGER DEFAULT 0
);
SQL
# POST IMPORT - For each topic, the user_id for the last poster / replier
@db.execute <<-SQL
CREATE TABLE IF NOT EXISTS topic_last_post_user (
topic_id INTEGER PRIMARY KEY,
user_id INTEGER
);
SQL
# POST IMPORT - The number of posts in a given topic, incremented as we add a new reply post to a topic.
@db.execute <<-SQL
CREATE TABLE IF NOT EXISTS topic_post_numbers (
topic_id INTEGER PRIMARY KEY,
post_number INTEGER DEFAULT 0
);
SQL
# POST IMPORT - Record perssitent integer value for highest processed post id -- not used
@db.execute <<-SQL
CREATE TABLE IF NOT EXISTS highest_processed_post_id (
id INTEGER PRIMARY KEY CHECK (id = 1),
post_id INTEGER
);
SQL
# PERSONAL MESSAGE IMPORT - Record perssitent integer value for highest processed personal id
@db.execute <<-SQL
CREATE TABLE IF NOT EXISTS highest_processed_personal_id (
id INTEGER PRIMARY KEY CHECK (id = 1),
personal_id INTEGER
);
SQL
# POST IMPORT - For each topic (topic post ID) record status 0 fail or 1 success
@db.execute <<-SQL
CREATE TABLE IF NOT EXISTS topic_import_status (
post_id INTEGER PRIMARY KEY,
@ -1356,9 +1391,9 @@ class GossamerForumsImporter < ImportScripts::Base
puts "CREATE TOPIC POST for current_topic_id #{current_topic_id} discourse_user_id #{discourse_user_id}"
post_number = 0
post_number = 1
# Increment the post count for the topic
post_number = fetch_db_topic_post_numbers(current_topic_id).to_i + 1
# This is a first post... post_number = fetch_db_topic_post_numbers(current_topic_id).to_i + 1
sqlite_mutex.synchronize do
update_db_topic_post_numbers(current_topic_id, post_number)
end
@ -1447,14 +1482,18 @@ class GossamerForumsImporter < ImportScripts::Base
puts "TIJ KK post_id #{post_id}"
# Increment the post count for the topic and user
sqlite_mutex.synchronize do
update_db_topic_post_count(current_topic_id, fetch_db_topic_post_count(current_topic_id).to_i + 1)
update_db_user_post_count(reply_user_id, fetch_db_user_post_count(reply_user_id).to_i + 1)
end
# Update last post time and user for the topic
if fetch_db_topic_last_post_time(current_topic_id).nil? || Time.at(reply_row['post_time']).to_i > fetch_db_topic_last_post_time(current_topic_id).to_i
sqlite_mutex.synchronize do
update_db_topic_last_post_time(current_topic_id, Time.at(reply_row['post_time']).to_i)
update_db_topic_last_post_user(current_topic_id, reply_user_id)
end
end
# Handle any attachments associated with the reply
handle_post_attachments(reply_row['post_id'], post, reply_user_id, mysql_client)