20240601-3

gossamer_forums.rb

@@ -1,11 +1,14 @@
 require 'mysql2'
+require 'open-uri'
+require 'net/http'
+
 require File.expand_path("../../../config/environment", __FILE__)
 require_relative 'base'
 
 class GossamerForumsImporter < ImportScripts::Base
   def initialize
     super
-    # Initialize MySQL client with connection details
+    # Initialize MySQL client to connect to Gossamer Forums database
     @mysql_client = Mysql2::Client.new(
       host: "slowtwitch.northend.network",
       username: "admin",
@@ -14,33 +17,25 @@ class GossamerForumsImporter < ImportScripts::Base
     )
   end
 
-  # Execute an SQL query on the Gossamer Forums MySQL database
+  # Execute an SQL query on the Gossamer Forums database
   def execute_query(query)
     @mysql_client.query(query, as: :hash)
   end
 
-  # Sanitize username to comply with Discourse's rules
+  # Sanitize the username to meet Discourse's requirements
   def sanitize_username(username, email, name)
     original_username = username
-    # Replace unacceptable characters with underscores
     sanitized = username.gsub(/[^a-zA-Z0-9._-]/, '_')
-    # Ensure the username is at least 2 characters long
+    sanitized = "#{sanitized}." if sanitized.length < 2 # Allow two-character usernames
-    sanitized = "#{sanitized}." if sanitized.length < 2
-    # Ensure the username is no more than 20 characters long
     sanitized = sanitized[0, 20] if sanitized.length > 20
     original_sanitized = sanitized
 
-    # Check for existing user with the same username
     existing_user = User.find_by(username: sanitized)
     
     if existing_user
-      # If email and name match, do not modify the username
-#      puts "existing_user.email: '#{existing_user.email}'  vs.  email: '#{email}'"
-#      puts "existing_user.name: '#{existing_user.name}'  vs.  email: '#{name}'"
       if existing_user.email.downcase == email.downcase && existing_user.name == name
         return sanitized
       else
-        # Ensure the username is unique
         counter = 1
         while User.exists?(username: sanitized)
           sanitized = "#{original_sanitized}_#{counter}"
@@ -50,7 +45,6 @@ class GossamerForumsImporter < ImportScripts::Base
       end
     end
 
-    # Print the original and sanitized usernames if they differ
     if original_username != sanitized
       puts "Sanitized username: '#{original_username}' --> '#{sanitized}'"
 #    else
@@ -60,6 +54,7 @@ class GossamerForumsImporter < ImportScripts::Base
     sanitized
   end
 
+
   # Sanitize email to replace restricted domains
   def sanitize_email(email)
     restricted_domains = ['mailinator.com', 'example.com'] # Add more restricted domains as needed
@@ -74,17 +69,38 @@ class GossamerForumsImporter < ImportScripts::Base
     email
   end
 
-  # Import users from Gossamer Forums gforum_User table to Discourse
+
+  # Helper method to download an image from a URL
+  def download_image(url)
+    begin
+      URI.open(url).read
+    rescue OpenURI::HTTPError => e
+      puts "Failed to download image from #{url}: #{e.message}"
+      nil
+    end
+  end
+
+  # Helper method to upload an image to Discourse
+  def upload_image(user, image_data, filename)
+    return if image_data.nil?
+
+    upload = Upload.create_for(user.id, File.open(image_data.path), filename, 'image/jpeg')
+    if upload.nil? || !upload.persisted?
+      puts "Failed to upload image for user #{user.username}"
+      return
+    end
+
+    upload
+  end
+
+
+  # Import users from Gossamer Forums to Discourse
   def import_users
     puts "Importing users..."
     users = []
 
     # Fetch all users from Gossamer Forums
     execute_query("SELECT * FROM gforum_User").each do |row|
-
-#      sanitized_username = sanitize_username(row['user_username'], row['user_email'], row['user_real_name'])
-#      sanitized_email = sanitize_email(row['user_email'])
-
       users << {
         id: row['user_id'],
         username: sanitize_username(row['user_username'], row['user_email'], row['user_real_name']),
@@ -119,23 +135,56 @@ class GossamerForumsImporter < ImportScripts::Base
         next
       end
 
-      if discourse_user.user_profile.bio_raw.nil? || discourse_user.user_profile.bio_raw.empty?
+      # Append bio if it exists, otherwise set it to empty string to avoid nil errors
+      discourse_user.user_profile.bio_raw ||= ""
+      if discourse_user.user_profile.bio_raw.empty?
         discourse_user.user_profile.bio_raw = user[:bio_raw]
       else
         discourse_user.user_profile.bio_raw += "\n\n" + user[:bio_raw]
       end
 
-      if discourse_user.user_profile.bio_raw.length > 2999
+      # Ensure the bio does not exceed 3000 characters
+      if discourse_user.user_profile.bio_raw.length > 3000
         puts "Warning: About Me for user #{discourse_user.username} (ID: #{discourse_user.id}) exceeds 3000 characters. Truncating."
-        discourse_user.user_profile.bio_raw = discourse_user.user_profile.bio_raw[0, 2999]
+        discourse_user.user_profile.bio_raw = discourse_user.user_profile.bio_raw[0, 3000]
       end
       discourse_user.user_profile.save!
 
+      # Import user files
       import_user_files(discourse_user)
     end
   end
 
-  # Import user files from Gossamer Forums to Discourse
+#  # Import user files from Gossamer Forums to Discourse
+#  def import_user_files(user)
+#    print "\rImporting files for user #{user.username}..."
+#
+#    original_gossamer_id = user.custom_fields['original_gossamer_id']
+#    if original_gossamer_id.nil? || original_gossamer_id.empty?
+#      puts "User #{user.username} does not have a valid original_gossamer_id. Skipping file import."
+#      return
+#    end
+#
+# #   puts "Original Gossamer ID for user #{user.username}: #{original_gossamer_id}"
+#
+#    # Fetch and import user files
+#    execute_query("SELECT * FROM gforum_User_Files WHERE ForeignColKey = #{original_gossamer_id}").each do |file|
+#      # Construct the file URL
+#      file_url = "https://forum.slowtwitch.com/images/users/images/#{file['ID'] % 10}/#{file['ID']}-#{file['File_Name']}"
+#      puts "User #{user.username}  User ID: #{user.id}  original_gossamer_id: #{original_gossamer_id}  file_url: #{file_url}"
+#      
+#      new_bio = user.user_profile.bio_raw + "\n\n![#{file['File_Name']}](#{file_url})"
+#      if new_bio.length > 3000
+#        puts "Warning: About Me for user #{user.username} (ID: #{user.id}) exceeds 3000 characters after adding file link. Truncating."
+#        new_bio = new_bio[0, 3000]
+#      end
+#      user.user_profile.bio_raw = new_bio
+#      user.user_profile.save!
+#    end
+#    print "Importing files for user #{user.username}... Done.\n"
+#  end
+
+  # Import user files (profile images) from Gossamer Forums to Discourse
   def import_user_files(user)
     print "\rImporting files for user #{user.username}..."
   
@@ -145,171 +194,211 @@ class GossamerForumsImporter < ImportScripts::Base
       return
     end
   
-#    puts "Original Gossamer ID for user #{user.username}: #{original_gossamer_id}"
+    puts "Original Gossamer ID for user #{user.username}: #{original_gossamer_id}"
+  
+    images_imported = 0
   
     execute_query("SELECT * FROM gforum_User_Files WHERE ForeignColKey = #{original_gossamer_id}").each do |file|
-      # Construct the file URL
       file_url = "https://forum.slowtwitch.com/images/users/images/#{file['ID'] % 10}/#{file['ID']}-#{file['File_Name']}"
       puts "User #{user.username}  User ID: #{user.id}  original_gossamer_id: #{original_gossamer_id}  file_url: #{file_url}"
   
-      new_bio = user.user_profile.bio_raw + "\n\n![#{file['File_Name']}](#{file_url})"
+      next unless file['ForeignColName'] =~ /^user_image\d+$/
-      if new_bio.length > 3000
+  
-        puts "Warning: About Me for user #{user.username} (ID: #{user.id}) exceeds 3000 characters after adding file link. Truncating."
+      image_data = download_image(file_url)
-        new_bio = new_bio[0, 3000]
+      next if image_data.nil?
+  
+      temp_file = Tempfile.new(['user_image', '.jpg'])
+      temp_file.binmode
+      temp_file.write(image_data)
+      temp_file.rewind
+  
+      if images_imported == 0
+        upload = upload_image(user, temp_file, file['File_Name'])
+        next if upload.nil?
+  
+        user.user_avatar = UserAvatar.create!(user_id: user.id, custom_upload_id: upload.id)
+        user.save!
+        images_imported += 1
+      else
+        user.user_profile.bio_raw ||= ""
+        user.user_profile.bio_raw += "\n\n![#{file['File_Name']}](#{file_url})"
+        user.user_profile.save!
       end
-      user.user_profile.bio_raw = new_bio
+  
-      user.user_profile.save!
+      temp_file.close
+      temp_file.unlink
     end
     print "Importing files for user #{user.username}... Done.\n"
   end
 
+
   # Import categories from Gossamer Forums to Discourse
   def import_categories
-    puts "Importing categories..."
+    puts "Importing categories (forums)..."
-    execute_query("SELECT * FROM gforum_Category").each do |row|
+    execute_query("SELECT * FROM gforum_Forum").each do |row|
       # Only create category if it does not exist
-      unless CategoryCustomField.exists?(name: 'original_gossamer_id', value: row['category_id'])
+      unless CategoryCustomField.exists?(name: 'original_gossamer_id', value: row['forum_id'])
+        category_name = row['forum_name']
+        category_description = row['forum_desc'] || "No description provided"
+
+        # Create category in Discourse
         category = create_category(
-          id: row['category_id'],
+          {
-          name: row['name'],
+            id: row['forum_id'],
-          description: row['description'],
+            name: category_name,
-          created_at: row['created_at'] ? Time.at(row['created_at']) : Time.now,
+            description: category_description,
-          updated_at: row['updated_at'] ? Time.at(row['updated_at']) : Time.now
+            created_at: row['forum_last'] ? Time.at(row['forum_last']) : Time.now,
+            updated_at: row['forum_last'] ? Time.at(row['forum_last']) : Time.now
+          },
+          row['forum_id'] # import_id argument
         )
-        category.custom_fields.create!(name: 'original_gossamer_id', value: row['category_id'])
+
+#        # Map Gossamer forum ID to Discourse category ID for future reference
+#        @forum_id_map[row['forum_id']] = category.id
+
+#        category.custom_fields.create!(name: 'original_gossamer_id', value: row['forum_id'])
+        category.custom_fields['original_gossamer_id'] = row['forum_id']
+        category.save!
       end
     end
+    puts "Importing categories... Done."
   end
 
-  # Import topics and posts from Gossamer Forums to Discourse
+# Import topics and posts from Gossamer Forums to Discourse
-  def import_topics_and_posts
+def import_topics_and_posts
-    puts "Importing topics and posts..."
+  puts "Importing topics and posts..."
-    execute_query("SELECT * FROM gforum_Post ORDER BY post_root_id, post_time").each do |row|
+  execute_query("SELECT * FROM gforum_Post ORDER BY post_root_id, post_time").each do |row|
-      if row['post_id'] == row['post_root_id']
+    if row['post_id'] == row['post_root_id']
-        # Skip if the topic already exists
+      # Skip if the topic already exists
-        unless TopicCustomField.exists?(name: 'original_gossamer_id', value: row['post_id'])
+      unless TopicCustomField.exists?(name: 'original_gossamer_id', value: row['post_id'])
-          # Create the topic
+        # Create the topic
-          topic = create_topic(
+        topic = Topic.create!(
-            id: row['post_id'],
+          title: row['post_subject'],
-            title: row['post_subject'],
+          user_id: row['user_id_fk'],
-            user_id: row['user_id_fk'],
+          created_at: Time.at(row['post_time']),
-            created_at: Time.at(row['post_time']),
+          updated_at: Time.at(row['post_latest_reply']),
-            updated_at: Time.at(row['post_latest_reply']),
+          category_id: row['forum_id_fk']
-            category_id: row['forum_id_fk']
-          )
-          topic.custom_fields.create!(name: 'original_gossamer_id', value: row['post_id'])
-
-          # Create the initial post in the topic
-          post = create_post(
-            id: row['post_id'],
-            topic_id: row['post_id'],
-            user_id: row['user_id_fk'],
-            raw: import_post_attachments(row['post_message'], row['post_id']),
-            created_at: Time.at(row['post_time']),
-            updated_at: Time.at(row['post_latest_reply'])
-          )
-          post.custom_fields.create!(name: 'original_gossamer_id', value: row['post_id'])
-        end
-      else
-        # Skip if the post already exists
-        unless PostCustomField.exists?(name: 'original_gossamer_id', value: row['post_id'])
-          # Create the post in the existing topic
-          post = create_post(
-            id: row['post_id'],
-            topic_id: row['post_root_id'],
-            user_id: row['user_id_fk'],
-            raw: import_post_attachments(row['post_message'], row['post_id']),
-            created_at: Time.at(row['post_time']),
-            updated_at: Time.at(row['post_latest_reply']),
-            reply_to_post_number: row['post_father_id']
-          )
-          post.custom_fields.create!(name: 'original_gossamer_id', value: row['post_id'])
-        end
-      end
-    end
-  end
-
-  # Import attachments for a post
-  def import_post_attachments(post_message, post_id)
-    attachments = execute_query("SELECT * FROM gforum_PostAttachment WHERE post_id_fk = #{post_id}")
-    attachments.each do |attachment|
-      attachment_url = "https://forum.slowtwitch.com/images/posts/#{post_id}/#{attachment['attachment_name']}"
-      post_message += "\n\n![#{attachment['attachment_name']}](#{attachment_url})"
-    end
-    post_message
-  end
-
-  # Import personal messages (both inbox and sent messages)
-  def import_personal_messages
-    puts "Importing personal messages..."
-    import_inbox_messages
-    import_sent_messages
-  end
-
-  # Import inbox messages from gforum_Message table
-  def import_inbox_messages
-    puts "Importing inbox messages..."
-    execute_query("SELECT * FROM gforum_Message").each do |row|
-      unless TopicCustomField.exists?(name: 'original_gossamer_msg_id', value: row['msg_id'])
-        # Create a private message topic in Discourse
-        topic = create_topic(
-          title: row['msg_subject'],
-          user_id: row['from_user_id_fk'],
-          archetype: Archetype.private_message,
-          created_at: Time.at(row['msg_time']),
-          updated_at: Time.at(row['msg_time'])
         )
-        topic.custom_fields.create!(name: 'original_gossamer_msg_id', value: row['msg_id'])
+        topic.custom_fields['original_gossamer_id'] = row['post_id']
+        topic.save!
 
-        # Create the message as a post in the private topic
+        # Create the initial post in the topic
-        create_post(
+        post = Post.create!(
           topic_id: topic.id,
-          user_id: row['from_user_id_fk'],
+          user_id: row['user_id_fk'],
-          raw: row['msg_body'],
+          raw: import_post_attachments(row['post_message'], row['post_id']),
-          created_at: Time.at(row['msg_time']),
+          created_at: Time.at(row['post_time']),
-          updated_at: Time.at(row['msg_time'])
+          updated_at: Time.at(row['post_latest_reply'])
         )
-
+        post.custom_fields['original_gossamer_id'] = row['post_id']
-        # Add recipient user to the private message topic
+        post.save!
-        topic.topic_allowed_users.create!(user_id: row['to_user_id_fk'])
+      end
+    else
+      # Skip if the post already exists
+      unless PostCustomField.exists?(name: 'original_gossamer_id', value: row['post_id'])
+        # Create the post in the existing topic
+        post = Post.create!(
+          topic_id: row['post_root_id'],
+          user_id: row['user_id_fk'],
+          raw: import_post_attachments(row['post_message'], row['post_id']),
+          created_at: Time.at(row['post_time']),
+          updated_at: Time.at(row['post_latest_reply']),
+          reply_to_post_number: row['post_father_id']
+        )
+        post.custom_fields['original_gossamer_id'] = row['post_id']
+        post.save!
       end
     end
   end
+end
 
-  # Import sent messages from gforum_SentMessage table
+# Import attachments for a post
-  def import_sent_messages
+def import_post_attachments(post_message, post_id)
-    puts "Importing sent messages..."
+  # Fetch attachments related to the post
-    execute_query("SELECT * FROM gforum_SentMessage").each do |row|
+  attachments = execute_query("SELECT * FROM gforum_PostAttachment WHERE post_id_fk = #{post_id}")
-      unless TopicCustomField.exists?(name: 'original_gossamer_sent_msg_id', value: row['msg_id'])
+  attachments.each do |attachment|
-        # Create a private message topic in Discourse
+    # Append attachment links to the post message
-        topic = create_topic(
+    file_url = "https://forum.slowtwitch.com/images/posts/attachments/#{attachment['ID'] % 10}/#{attachment['ID']}-#{attachment['File_Name']}"
-          title: row['msg_subject'],
+    post_message += "\n\n![#{attachment['File_Name']}](#{file_url})"
-          user_id: row['from_user_id_fk'],
+  end
-          archetype: Archetype.private_message,
+  post_message
-          created_at: Time.at(row['msg_time']),
+end
-          updated_at: Time.at(row['msg_time'])
-        )
-        topic.custom_fields.create!(name: 'original_gossamer_sent_msg_id', value: row['msg_id'])
 
-        # Create the message as a post in the private topic
+# Import personal messages (both inbox and sent messages)
-        create_post(
+def import_personal_messages
-          topic_id: topic.id,
+  puts "Importing personal messages..."
-          user_id: row['from_user_id_fk'],
+  import_inbox_messages
-          raw: row['msg_body'],
+  import_sent_messages
-          created_at: Time.at(row['msg_time']),
+end
-          updated_at: Time.at(row['msg_time'])
-        )
 
-        # Add recipient user to the private message topic
+# Import inbox messages from gforum_Message table
-        topic.topic_allowed_users.create!(user_id: row['to_user_id_fk'])
+def import_inbox_messages
-      end
+  puts "Importing inbox messages..."
+  execute_query("SELECT * FROM gforum_Message").each do |row|
+    # Skip if the message already exists
+    unless TopicCustomField.exists?(name: 'original_gossamer_msg_id', value: row['msg_id'])
+      # Create a private message topic in Discourse
+      topic = Topic.create!(
+        title: row['msg_subject'],
+        user_id: row['from_user_id_fk'],
+        archetype: Archetype.private_message,
+        created_at: Time.at(row['msg_time']),
+        updated_at: Time.at(row['msg_time'])
+      )
+      topic.custom_fields['original_gossamer_msg_id'] = row['msg_id']
+      topic.save!
+
+      # Create the message as a post in the private topic
+      Post.create!(
+        topic_id: topic.id,
+        user_id: row['from_user_id_fk'],
+        raw: row['msg_body'],
+        created_at: Time.at(row['msg_time']),
+        updated_at: Time.at(row['msg_time'])
+      )
+
+      # Add recipient user to the private message topic
+      topic.topic_allowed_users.create!(user_id: row['to_user_id_fk'])
     end
   end
+end
+
+# Import sent messages from gforum_SentMessage table
+def import_sent_messages
+  puts "Importing sent messages..."
+  execute_query("SELECT * FROM gforum_SentMessage").each do |row|
+    # Skip if the message already exists
+    unless TopicCustomField.exists?(name: 'original_gossamer_sent_msg_id', value: row['msg_id'])
+      # Create a private message topic in Discourse
+      topic = Topic.create!(
+        title: row['msg_subject'],
+        user_id: row['from_user_id_fk'],
+        archetype: Archetype.private_message,
+        created_at: Time.at(row['msg_time']),
+        updated_at: Time.at(row['msg_time'])
+      )
+      topic.custom_fields['original_gossamer_sent_msg_id'] = row['msg_id']
+      topic.save!
+
+      # Create the message as a post in the private topic
+      Post.create!(
+        topic_id: topic.id,
+        user_id: row['from_user_id_fk'],
+        raw: row['msg_body'],
+        created_at: Time.at(row['msg_time']),
+        updated_at: Time.at(row['msg_time'])
+      )
+
+      # Add recipient user to the private message topic
+      topic.topic_allowed_users.create!(user_id: row['to_user_id_fk'])
+    end
+  end
+end
 
   # Main method to perform the import
   def perform_import
     puts "Starting Gossamer Forums import..."
-    import_users
+#    import_users
     import_categories
     import_topics_and_posts
     import_personal_messages
@@ -317,9 +406,5 @@ class GossamerForumsImporter < ImportScripts::Base
   end
 end
 
-# Create an instance of the importer and start the import process
-# importer = GossamerForumsImporter.new
-# importer.perform_import
-
 GossamerForumsImporter.new.perform_import