20240606

gossamer_forums.rb

@@ -36,6 +36,12 @@ class GossamerForumsImporter < ImportScripts::Base
         new_user_id INTEGER
       );
     SQL
+    @db.execute <<-SQL
+      CREATE TABLE IF NOT EXISTS category_id_map (
+        old_category_id INTEGER PRIMARY KEY,
+        new_category_id INTEGER
+      );
+    SQL
   end
 
   def insert_user_id_mapping(old_user_id, new_user_id)
@@ -46,6 +52,14 @@ class GossamerForumsImporter < ImportScripts::Base
     @db.get_first_value "SELECT new_user_id FROM user_id_map WHERE old_user_id = ?", old_user_id
   end
 
+  def insert_category_id_mapping(old_category_id, new_category_id)
+    @db.execute "INSERT OR REPLACE INTO category_id_map (old_category_id, new_category_id) VALUES (?, ?)", old_category_id, new_category_id
+  end
+
+  def fetch_category_id_mapping(old_category_id)
+    @db.get_first_value "SELECT new_category_id FROM category_id_map WHERE old_category_id = ?", old_category_id
+  end
+
   # Execute an SQL query on the Gossamer Forums database
   def execute_query(query)
     @mysql_client.query(query, as: :hash)
@@ -106,21 +120,65 @@ class GossamerForumsImporter < ImportScripts::Base
     rescue OpenURI::HTTPError => e
       puts "Failed to download image from #{url}: #{e.message}"
       nil
+    rescue URI::InvalidURIError => e
+      puts "Failed to handle invalid URL/URI for #{url}: #{e.message}"
+      nil
     end
   end
 
-  # Helper method to upload an image to Discourse
+def upload_image(user, file, filename, gossamer_url)
-  def upload_image(user, image_data, filename)
+  begin
-    return if image_data.nil?
+    upload = Upload.create!(
+      user_id: user.id,
+      original_filename: filename,
+      filesize: file.size,
+#      filesize: File.size(file.path),
+#      content_type: `file --brief --mime-type #{file.path}`.strip,
+#      sha1: Digest::SHA1.file(file.path).hexdigest,
+#      origin: 'user_avatar',
+#      retain_hours: nil,
+      url: gossamer_url
+    )
+# Error -- non-existent method    upload.ensure_consistency!
     
-    upload = Upload.create_for(user.id, File.open(image_data.path), filename, 'image/jpeg')
+    # Move the file to the correct location
-    if upload.nil? || !upload.persisted?
+#    FileUtils.mv(file.path, upload.path)
-      puts "Failed to upload image for user #{user.username}"
+    upload.save!
-      return
-    end
     
     upload
+  rescue => e
+    puts "Failed to upload image #{filename} for user #{user.username}: #{e.message}"
+    nil
   end
+end
+
+
+#  def download_file(url)
+#    require 'open-uri'
+#    begin
+#      file = Tempfile.new
+#      file.binmode
+#      file.write(URI.open(url).read)
+#      file.rewind
+#      file
+#    rescue => e
+#      puts "Failed to download file from #{url}: #{e.message}"
+#      nil
+#    end
+#  end
+
+  # Helper method to upload an image to Discourse
+#  def upload_image(user, image_data, filename)
+#    return if image_data.nil?
+#
+#    upload = Upload.create_for(user.id, File.open(image_data.path), filename, 'image/jpeg')
+#    if upload.nil? || !upload.persisted?
+#      puts "Failed to upload image for user #{user.username}"
+#      return
+#    end
+#
+#    upload
+#  end
 
 
   # Import users from Gossamer Forums to Discourse
@@ -155,7 +213,7 @@ class GossamerForumsImporter < ImportScripts::Base
       user
     end
 
-    # For each user, add user ID mapping to @user_id_map now that we know what the Discourse user ID is, ... and append user bio and import user files
+    # For each user, add user ID mapping to SQLite now that we know what the Discourse user ID is, ... and append user bio and import user files
     users.each do |user|
       discourse_username = sanitize_username(user[:username], user[:email], user[:name])
       discourse_user = User.find_by(username: discourse_username)
@@ -167,6 +225,7 @@ class GossamerForumsImporter < ImportScripts::Base
 
 #      # Store the user ID mapping
 #      @user_id_map[user[:id]] = discourse_user.id
+      puts "for insert_user_id_mapping:   user[:id]  #{user[:id]}  discourse_user.id  #{discourse_user.id}"
       insert_user_id_mapping(user[:id], discourse_user.id)
 
       # Ensure user profile exists and bio_raw is a string
@@ -188,7 +247,7 @@ class GossamerForumsImporter < ImportScripts::Base
       discourse_user.user_profile.save!
 
       # Import user files
-      # import_user_files(discourse_user)
+      import_user_files(discourse_user)
     end
   end
 
@@ -240,27 +299,39 @@ class GossamerForumsImporter < ImportScripts::Base
       puts "User #{user.username}  User ID: #{user.id}  original_gossamer_id: #{original_gossamer_id}  file_url: #{file_url}"
   
       next unless file['ForeignColName'] =~ /^user_image\d+$/
+      puts "#A"
+      next unless ['image/jpeg', 'image/png'].include?(file['File_MimeType'])
+      puts "#B"
   
       image_data = download_image(file_url)
       next if image_data.nil?
+      puts "#C"
   
-      temp_file = Tempfile.new(['user_image', '.jpg'])
+      temp_file = Tempfile.new(['user_image', File.extname(file['File_Name'])])
       temp_file.binmode
       temp_file.write(image_data)
       temp_file.rewind
   
       if images_imported == 0
-        upload = upload_image(user, temp_file, file['File_Name'])
+        puts "#D"
+        upload = upload_image(user, temp_file, file['File_Name'], file_url)
         next if upload.nil?
   
         user.user_avatar = UserAvatar.create!(user_id: user.id, custom_upload_id: upload.id)
         user.save!
+
+        # Set the Profile Header
+        UserProfile.find_by(user_id: user.id).update!(profile_background_upload_id: upload.id)
+
+        # Set the User Card Background
+        UserProfile.find_by(user_id: user.id).update!(card_background_upload_id: upload.id)
+
         images_imported += 1
-      else
+      end 
+      puts "#E"
       user.user_profile.bio_raw ||= ""
       user.user_profile.bio_raw += "\n\n![#{file['File_Name']}](#{file_url})"
       user.user_profile.save!
-      end
         
       temp_file.close
       temp_file.unlink
@@ -277,11 +348,11 @@ class GossamerForumsImporter < ImportScripts::Base
       unless CategoryCustomField.exists?(name: 'original_gossamer_id', value: row['forum_id'])
         category_name = row['forum_name']
         category_description = row['forum_desc'] || "No description provided"
-
+        puts "id  #{row['forum_id']}  name  #{category_name}  description  #{category_description}"
         # Create category in Discourse
         category = create_category(
           {
-            id: row['forum_id'] + 10,
+#            id: row['forum_id'] + 10,
             name: category_name,
             description: category_description,
             created_at: row['forum_last'] ? Time.at(row['forum_last']) : Time.now,
@@ -296,6 +367,10 @@ class GossamerForumsImporter < ImportScripts::Base
 #        category.custom_fields.create!(name: 'original_gossamer_id', value: row['forum_id'])
         category.custom_fields['original_gossamer_id'] = row['forum_id']
         category.save!
+
+        # Store the user ID mapping
+        puts "for insert_category_id_mapping:   category[:id]  #{category[:id]}  row['forum_id']  #{row['forum_id']}"
+        insert_category_id_mapping(row['forum_id'], category[:id])
       end
     end
     puts "Importing categories... Done."
@@ -316,12 +391,15 @@ def import_topics_and_posts
   # Execute the query to get all posts ordered by post_id
   execute_query("SELECT * FROM gforum_Post ORDER BY post_id").each do |row|
     puts "post_id  #{row['post_id']}  post_root_id  #{row['post_root_id']}  post_subject/title  #{row['post_subject']}  forum_id_fk/category_id  #{row['forum_id_fk']}"
-    discourse_user_id = fetch_user_id_mapping(row['user_id_fk'])
 #    discourse_user_id = @user_id_map[row['user_id_fk']]
+    discourse_user_id = fetch_user_id_mapping(row['user_id_fk'])
+    discourse_category_id = fetch_category_id_mapping(row['forum_id_fk'])
 
-    next unless discourse_user_id
+    puts "discourse_user_id  #{discourse_user_id}  discourse_category_id  #{discourse_category_id}"
+    next unless discourse_user_id && discourse_category_id
 
     if row['post_root_id'] == 0
+      puts "#1"
       # Ensure the title is valid
       title = ensure_valid_title(row['post_subject'])
 
@@ -329,21 +407,27 @@ def import_topics_and_posts
       unless TopicCustomField.exists?(name: 'original_gossamer_id', value: row['post_id'])
         # Create the topic
         begin
+          puts "#2"
+          puts "CREATE TOPIC  title #{title}  discourse_user_id #{discourse_user_id}  category_id #{discourse_category_id}"
           topic = Topic.create!(
             title: title,
             user_id: discourse_user_id,
             created_at: Time.at(row['post_time']),
             updated_at: Time.at(row['post_latest_reply']),
-            category_id: row['forum_id_fk'] + 10
+            category_id: discourse_category_id
           )
           topic.custom_fields['original_gossamer_id'] = row['post_id']
           topic.save!
 
           # Create the initial post in the topic
+          puts "CREATE POST  topic.id #{topic.id}  discourse_user_id #{discourse_user_id}"
+          sanitized_post_message = row['post_message']&.tr("\0", '') || ""
           post = Post.create!(
             topic_id: topic.id,
             user_id: discourse_user_id,
-            raw: import_post_attachments(row['post_message'], row['post_id']),
+#            raw: import_post_attachments(row['post_message'], row['post_id']),
+#            raw: row['post_message'] || "",
+            raw: sanitized_post_message,
             created_at: Time.at(row['post_time']),
             updated_at: Time.at(row['post_latest_reply'])
           )
@@ -355,6 +439,7 @@ def import_topics_and_posts
       end
 
     else
+      puts "#3"
       # Find the root topic for the post
       root_topic_field = TopicCustomField.find_by(name: 'original_gossamer_id', value: row['post_root_id'])
       
@@ -367,10 +452,14 @@ def import_topics_and_posts
         
         # Create the post in the existing topic
         begin
+          puts "#4"
+          sanitized_post_message = row['post_message']&.tr("\0", '') || ""
           post = Post.create!(
             topic_id: topic_id,
             user_id: discourse_user_id,
-            raw: import_post_attachments(row['post_message'], row['post_id']),
+#             raw: import_post_attachments(row['post_message'], row['post_id']),
+#            raw: row['post_message'] || "",
+            raw: sanitized_post_message,
             created_at: Time.at(row['post_time']),
             updated_at: Time.at(row['post_latest_reply']),
             reply_to_post_number: reply_to_post_number
@@ -387,42 +476,23 @@ def import_topics_and_posts
   end
 end
 
-# Import topics and posts from Gossamer Forums to Discourse
-# def import_topics_and_posts
-#   puts "Importing topics and posts..."
-#   execute_query("SELECT * FROM gforum_Post ORDER BY post_root_id, post_time").each do |row|
-#     puts "post_id  #{row['post_id']}  post_root_id  #{row['post_root_id']}  post_subject  #{row['post_subject']}" 
 
-# Import attachments for a post
+# Import personal messages from gforum_Message table (both inbox and sent messages)
-def import_post_attachments(post_message, post_id)
-  # Fetch attachments related to the post
-  attachments = execute_query("SELECT * FROM gforum_PostAttachment WHERE post_id_fk = #{post_id}")
-  attachments.each do |attachment|
-    # Append attachment links to the post message
-    file_url = "https://forum.slowtwitch.com/images/posts/attachments/#{attachment['ID'] % 10}/#{attachment['ID']}-#{attachment['File_Name']}"
-    post_message += "\n\n![#{attachment['File_Name']}](#{file_url})"
-  end
-  post_message
-end
-
-# Import personal messages (both inbox and sent messages)
 def import_personal_messages
-  puts "Importing personal messages..."
+  puts "Importing personal (inbox and sendmail) messages..."
-  import_inbox_messages
-  import_sent_messages
-end
-
-# Import inbox messages from gforum_Message table
-def import_inbox_messages
-  puts "Importing inbox messages..."
   execute_query("SELECT * FROM gforum_Message").each do |row|
+
+  from_user_id = fetch_user_id_mapping(row['from_user_id_fk'])
+  to_user_id = fetch_user_id_mapping(row['to_user_id_fk'])
+
+  next unless from_user_id && to_user_id
+
     # Skip if the message already exists
     unless TopicCustomField.exists?(name: 'original_gossamer_msg_id', value: row['msg_id'])
       # Create a private message topic in Discourse
-      discourse_user_id = @user_id_map[row['from_user_id_fk']]
       topic = Topic.create!(
         title: row['msg_subject'],
-        user_id: discourse_user_id,
+        user_id: from_user_id,
         archetype: Archetype.private_message,
         created_at: Time.at(row['msg_time']),
         updated_at: Time.at(row['msg_time'])
@@ -431,61 +501,50 @@ def import_inbox_messages
       topic.save!
 
       # Create the message as a post in the private topic
-      Post.create!(
+      sanitized_message = row['msg_body']&.tr("\0", '') || ""
+      post = Post.create!(
         topic_id: topic.id,
-        user_id: discourse_user_id,
+        user_id: from_user_id,
-        raw: row['msg_body'],
+#        raw: row['msg_body'],
+        raw: sanitized_message,
         created_at: Time.at(row['msg_time']),
         updated_at: Time.at(row['msg_time'])
       )
+      post.custom_fields['original_gossamer_msg_id'] = row['msg_id']
+      post.save!
 
       # Add recipient user to the private message topic
-      topic.topic_allowed_users.create!(user_id: row['to_user_id_fk'])
+      topic.topic_allowed_users.create!(user_id: to_user_id)
     end
   end
 end
 
-# Import sent messages from gforum_SentMessage table
-def import_sent_messages
-  puts "Importing sent messages..."
-  execute_query("SELECT * FROM gforum_SentMessage").each do |row|
-    # Skip if the message already exists
-    unless TopicCustomField.exists?(name: 'original_gossamer_sent_msg_id', value: row['msg_id'])
-      # Create a private message topic in Discourse
-      discourse_user_id = @user_id_map[row['from_user_id_fk']]
-      topic = Topic.create!(
-        title: row['msg_subject'],
-        user_id: discourse_user_id,
-        archetype: Archetype.private_message,
-        created_at: Time.at(row['msg_time']),
-        updated_at: Time.at(row['msg_time'])
-      )
-      topic.custom_fields['original_gossamer_sent_msg_id'] = row['msg_id']
-      topic.save!
 
-      # Create the message as a post in the private topic
+# Import attachments for a post
-      Post.create!(
+# def import_post_attachments(post_message, post_id)
-        topic_id: topic.id,
+#  # Fetch attachments related to the post
-        user_id: discourse_user_id,
+#  attachments = execute_query("SELECT * FROM gforum_PostAttachment WHERE post_id_fk = #{post_id}")
-        raw: row['msg_body'],
+#  attachments.each do |attachment|
-        created_at: Time.at(row['msg_time']),
+#    # Append attachment links to the post message
-        updated_at: Time.at(row['msg_time'])
+#    file_url = "https://forum.slowtwitch.com/images/posts/attachments/#{attachment['ID'] % 10}/#{attachment['ID']}-#{attachment['File_Name']}"
-      )
+#    post_message += "\n\n![#{attachment['File_Name']}](#{file_url})"
+#  end
+#  post_message
+# end
+
+
+
 
-      # Add recipient user to the private message topic
-      topic.topic_allowed_users.create!(user_id: row['to_user_id_fk'])
-    end
-  end
-end
 
   # Main method to perform the import
   def perform_import
     RateLimiter.disable
     puts "Starting Gossamer Forums import..."
-    import_users
+    # import_users
-    import_categories
+    # import_categories
     # import_topics_and_posts
-    # import_personal_messages
+    import_personal_messages
+    # import_attachments
     puts "Gossamer Forums import complete!"
   end
 end