From b0f300bf8ebf1a7945cdccab8241869a7fe66c45 Mon Sep 17 00:00:00 2001 From: saint Date: Sun, 18 Aug 2024 18:57:22 +1000 Subject: [PATCH] v0.48.1 Further attempts to address MariaDB challenges and make things as foolproof as possible. --- gossamer_forums.rb | 63 +++++++++++++++++++++++++++++++++++++--------- 1 file changed, 51 insertions(+), 12 deletions(-) diff --git a/gossamer_forums.rb b/gossamer_forums.rb index 2db2f9e..32378b9 100644 --- a/gossamer_forums.rb +++ b/gossamer_forums.rb @@ -62,25 +62,50 @@ class GossamerForumsImporter < ImportScripts::Base @db = SQLite3::Database.new '/bitnami/discourse/sqlite/id_name_url_map.db' ###### ONLY when we need to clear the url_map and topic_import_status .... e.g. if reimporting topics-posts from scratch -# @db.execute <<-SQL -# DROP TABLE IF EXISTS url_map; -# SQL -# @db.execute <<-SQL -# DROP TABLE IF EXISTS topic_import_status; -# SQL + @db.execute <<-SQL + DROP TABLE IF EXISTS url_map; + SQL + @db.execute <<-SQL + DROP TABLE IF EXISTS topic_last_post_time; + SQL + @db.execute <<-SQL + DROP TABLE IF EXISTS topic_post_count; + SQL + @db.execute <<-SQL + DROP TABLE IF EXISTS user_topic_count; + SQL + @db.execute <<-SQL + DROP TABLE IF EXISTS user_post_count; + SQL + @db.execute <<-SQL + DROP TABLE IF EXISTS topic_last_post_user; + SQL + @db.execute <<-SQL + DROP TABLE IF EXISTS topic_post_numbers; + SQL + @db.execute <<-SQL + DROP TABLE IF EXISTS highest_processed_post_id; + SQL + @db.execute <<-SQL + DROP TABLE IF EXISTS topic_import_status; + SQL +####### + # USER IMPORT - map of old to new user ids, used for topic-post import. @db.execute <<-SQL CREATE TABLE IF NOT EXISTS user_id_map ( old_user_id INTEGER PRIMARY KEY, new_user_id INTEGER ); SQL + # CATEGORY IMPORT - map of old to new category ids, used for topic-post import. @db.execute <<-SQL CREATE TABLE IF NOT EXISTS category_id_map ( old_category_id INTEGER PRIMARY KEY, new_category_id INTEGER ); SQL + # USER IMPORT - map of old to new usernames for SENDING MIGRATION EMAIL @db.execute <<-SQL CREATE TABLE IF NOT EXISTS username_map ( id INTEGER PRIMARY KEY, @@ -90,6 +115,7 @@ class GossamerForumsImporter < ImportScripts::Base real_name TEXT ); SQL + # POST IMPORT - Generate a map of old_post_id (Gossamer) and the new URL for a WEB SERVER REDIRECT FILE @db.execute <<-SQL CREATE TABLE IF NOT EXISTS url_map ( old_post_id INTEGER PRIMARY KEY, @@ -97,54 +123,63 @@ class GossamerForumsImporter < ImportScripts::Base title TEXT ); SQL + # POST IMPORT - For each topic, the time of the last / most recent post/reply @db.execute <<-SQL CREATE TABLE IF NOT EXISTS topic_last_post_time ( topic_id INTEGER PRIMARY KEY, last_post_time INTEGER ); SQL + # POST IMPORT - For each topic, increment post_count as we add posts. @db.execute <<-SQL CREATE TABLE IF NOT EXISTS topic_post_count ( topic_id INTEGER PRIMARY KEY, post_count INTEGER DEFAULT 0 ); SQL + # POST IMPORT - For each user (_id), increment topic_count as we add topics (to see total topics per user) @db.execute <<-SQL CREATE TABLE IF NOT EXISTS user_topic_count ( user_id INTEGER PRIMARY KEY, topic_count INTEGER DEFAULT 0 ); SQL + # POST IMPORT - For each user (_id), increment post_count as we add posts (to see total posts per user) @db.execute <<-SQL CREATE TABLE IF NOT EXISTS user_post_count ( user_id INTEGER PRIMARY KEY, post_count INTEGER DEFAULT 0 ); SQL + # POST IMPORT - For each topic, the user_id for the last poster / replier @db.execute <<-SQL CREATE TABLE IF NOT EXISTS topic_last_post_user ( topic_id INTEGER PRIMARY KEY, user_id INTEGER ); SQL + # POST IMPORT - The number of posts in a given topic, incremented as we add a new reply post to a topic. @db.execute <<-SQL CREATE TABLE IF NOT EXISTS topic_post_numbers ( topic_id INTEGER PRIMARY KEY, post_number INTEGER DEFAULT 0 ); SQL + # POST IMPORT - Record perssitent integer value for highest processed post id -- not used @db.execute <<-SQL CREATE TABLE IF NOT EXISTS highest_processed_post_id ( id INTEGER PRIMARY KEY CHECK (id = 1), post_id INTEGER ); SQL + # PERSONAL MESSAGE IMPORT - Record perssitent integer value for highest processed personal id @db.execute <<-SQL CREATE TABLE IF NOT EXISTS highest_processed_personal_id ( id INTEGER PRIMARY KEY CHECK (id = 1), personal_id INTEGER ); SQL + # POST IMPORT - For each topic (topic post ID) record status 0 fail or 1 success @db.execute <<-SQL CREATE TABLE IF NOT EXISTS topic_import_status ( post_id INTEGER PRIMARY KEY, @@ -1356,9 +1391,9 @@ class GossamerForumsImporter < ImportScripts::Base puts "CREATE TOPIC POST for current_topic_id #{current_topic_id} discourse_user_id #{discourse_user_id}" - post_number = 0 + post_number = 1 # Increment the post count for the topic - post_number = fetch_db_topic_post_numbers(current_topic_id).to_i + 1 + # This is a first post... post_number = fetch_db_topic_post_numbers(current_topic_id).to_i + 1 sqlite_mutex.synchronize do update_db_topic_post_numbers(current_topic_id, post_number) end @@ -1447,13 +1482,17 @@ class GossamerForumsImporter < ImportScripts::Base puts "TIJ KK post_id #{post_id}" # Increment the post count for the topic and user - update_db_topic_post_count(current_topic_id, fetch_db_topic_post_count(current_topic_id).to_i + 1) - update_db_user_post_count(reply_user_id, fetch_db_user_post_count(reply_user_id).to_i + 1) + sqlite_mutex.synchronize do + update_db_topic_post_count(current_topic_id, fetch_db_topic_post_count(current_topic_id).to_i + 1) + update_db_user_post_count(reply_user_id, fetch_db_user_post_count(reply_user_id).to_i + 1) + end # Update last post time and user for the topic if fetch_db_topic_last_post_time(current_topic_id).nil? || Time.at(reply_row['post_time']).to_i > fetch_db_topic_last_post_time(current_topic_id).to_i - update_db_topic_last_post_time(current_topic_id, Time.at(reply_row['post_time']).to_i) - update_db_topic_last_post_user(current_topic_id, reply_user_id) + sqlite_mutex.synchronize do + update_db_topic_last_post_time(current_topic_id, Time.at(reply_row['post_time']).to_i) + update_db_topic_last_post_user(current_topic_id, reply_user_id) + end end # Handle any attachments associated with the reply