pre remove inline go routine for post threads

main.go

@@ -7,15 +7,15 @@ import (
 )
 
 // WP Config
-const baseUrl = "http://go-api-playground.local/"
-const wordpressKey = "admin"
-const wordpressSecret = "7ZeY ZCwp 3DwL EBAK vGtf NW9J"
+const baseUrl = "https://slowtwitch.cloud/"
+const wordpressKey = "admin@slowtwitch.cloud"
+const wordpressSecret = "3SrI MysB NREZ 40OD PLAP uk1W"
 
 // DB Config
 const slowtwitchAdminUser = "admin"
 const slowtwitchAdminPass = "yxnh93Ybbz2Nm8#mp28zCVv"
 const slowtwitchDbName = "slowtwitch"
-const migrationDbName = "slowtwitch_transfer_threaded_test"
+const migrationDbName = "slowtwitch_transfer"
 const federatedDbUrl = "slowtwitch.northend.network"
 const federatedDbPort = "3306"
 

services/migration/editor-result.go

@@ -18,8 +18,8 @@ func CreateEditorResult(result EditorResult, db *sql.DB) error {
 	return err
 }
 
-func GetEditor(username string, db *sql.DB) (EditorResult, error) {
-	rows, err := db.Query("select WordpressId, Username, Email, (IsSuccess = b'1') from EditorsResults where Username = ?", username)
+func GetEditor(username, email string, db *sql.DB) (EditorResult, error) {
+	rows, err := db.Query("select WordpressId, Username, Email, (IsSuccess = b'1') from EditorsResults where Username = ? OR Email = ?", username, email)
 	if err != nil {
 		fmt.Println(err)
 	}
@@ -35,7 +35,7 @@ func GetEditor(username string, db *sql.DB) (EditorResult, error) {
 			fmt.Println(err)
 		}
 
-		if editor.Username == username {
+		if editor.Username == username || editor.Email == email {
 			return editor, nil
 		}
 	}
@@ -43,8 +43,8 @@ func GetEditor(username string, db *sql.DB) (EditorResult, error) {
 	return EditorResult{}, errors.New("Could not find editor.")
 }
 
-func EditorHasBeenMigrated(username string, db *sql.DB) bool {
-	_, err := GetEditor(username, db)
+func EditorHasBeenMigrated(username, email string, db *sql.DB) bool {
+	_, err := GetEditor(username, email, db)
 
 	if err == nil {
 		return true

services/migration/migrate-authors.go

@@ -22,7 +22,7 @@ func (migration *MigrateAuthors) Execute() []EditorResult {
 	var output []EditorResult
 
 	for _, editor := range editors {
-		hasBeenMigrated := EditorHasBeenMigrated(editor.Username, migration.ResultsDatabase)
+		hasBeenMigrated := EditorHasBeenMigrated(editor.Username, editor.Email, migration.ResultsDatabase)
 
 		if hasBeenMigrated == false {
 			firstName, lastName := getFirstAndLastName(editor.Name)

services/migration/migrate-posts.go

@@ -8,6 +8,7 @@ import (
 	"slices"
 	"strconv"
 	"strings"
+	"sync"
 )
 
 type MigratePosts struct {
@@ -36,6 +37,7 @@ func (migration MigratePosts) Execute() []PostResult {
 		panic("Could not migrate posts due to tags not found:" + wpTagErr.Error())
 	}
 
+	//TODO Re-thread using a work input channel, work output channel, and waitgroup
 	var postResults []PostResult
 	batchSize := 5
 	var postBatch []int
@@ -114,13 +116,18 @@ func (migration MigratePosts) Execute() []PostResult {
 
 					createWordpressPost.Categories = wordPressCategoryIds
 					//Get Author ID
-					editor, findEditorErr := GetEditor(postBase.Author, migration.ResultsDatabase)
+					editor, findEditorErr := GetEditor(postBase.Author, postBase.AuthorEmail, migration.ResultsDatabase)
 
 					if findEditorErr != nil {
-						errorMessage = errorMessage + findEditorErr.Error()
-						failedPostResult := createPostFailureResult(batchId, errorMessage, migration.ResultsDatabase)
-						postResultsChannel <- failedPostResult
-						return
+						editor, catchAllEditorErr := GetEditor("admin", "slowman2@slowtwitch.com", migration.ResultsDatabase)
+
+						if catchAllEditorErr != nil {
+							errorMessage = errorMessage + findEditorErr.Error()
+							failedPostResult := createPostFailureResult(batchId, errorMessage, migration.ResultsDatabase)
+							postResultsChannel <- failedPostResult
+							return
+						}
+						createWordpressPost.Author = editor.WordpressId
 					}
 					createWordpressPost.Author = editor.WordpressId
 					//Get old link
@@ -272,45 +279,289 @@ func (migration MigratePosts) Execute() []PostResult {
 			postResults = append(postResults, batchResults...)
 		}
 	}
-	// Update related posts once work is done -- Why are there duplicates?
-	updatePostRelationships(postResults, migration)
+	// Update related posts once work is done -- Do for all posts?
+	allPostResults := GetAllPostResults(migration.ResultsDatabase)
+	updatePostRelationships(allPostResults, migration)
 
 	return postResults
 }
 
+func createPost(postId int, migration MigratePosts, wpTagData []wordpress.TagData) PostResult {
+	errorMessage := ""
+	//migrate
+	postBase, postBaseErr := slowtwitch.GetPostBase(postId, migration.SlowtwitchDatabase)
+
+	if postBaseErr != nil {
+		errorMessage = errorMessage + postBaseErr.Error()
+		failedPostResult := createPostFailureResult(postId, errorMessage, migration.ResultsDatabase)
+		return failedPostResult
+	}
+
+	createWordpressPost := wordpress.CreatePost{
+		Title:   postBase.Title,
+		Excerpt: postBase.Description,
+		Status:  "publish",
+	}
+
+	if postBase.DatePublished.Valid {
+		t := postBase.DatePublished.Time
+		timeString := fmt.Sprintf("%d-%02d-%02dT%02d:%02d:%02d",
+			t.Year(), t.Month(), t.Day(),
+			t.Hour(), t.Minute(), t.Second())
+		createWordpressPost.Date = timeString
+	} else {
+		errorMessage = errorMessage + "Invalid Date Published"
+		failedPostResult := createPostFailureResult(postId, errorMessage, migration.ResultsDatabase)
+		return failedPostResult
+	}
+
+	for _, tag := range wpTagData {
+		if postBase.Bike == true && tag.Name == "bike" {
+			createWordpressPost.Tags = append(createWordpressPost.Tags, tag.Id)
+		}
+		if postBase.Swim == true && tag.Name == "swim" {
+			createWordpressPost.Tags = append(createWordpressPost.Tags, tag.Id)
+		}
+		if postBase.Run == true && tag.Name == "run" {
+			createWordpressPost.Tags = append(createWordpressPost.Tags, tag.Id)
+		}
+	}
+
+	var wordPressCategoryIds []int
+	var firstCategoryResult CategoryResult
+	for _, slowtwitchCategoryId := range postBase.CategoryIds {
+		categoryResult, err := GetSlowtwitchCategoryResult(slowtwitchCategoryId, migration.ResultsDatabase)
+		if err != nil {
+			errorMessage = errorMessage + err.Error()
+			failedPostResult := createPostFailureResult(postId, errorMessage, migration.ResultsDatabase)
+			return failedPostResult
+		}
+		wordPressCategoryIds = append(wordPressCategoryIds, categoryResult.WordpressId)
+		firstCategoryResult = categoryResult
+	}
+
+	if len(wordPressCategoryIds) == 0 {
+		errorMessage = "This post has no categories and is broken on the production site."
+		failedPostResult := createPostFailureResult(postId, errorMessage, migration.ResultsDatabase)
+		return failedPostResult
+	}
+
+	createWordpressPost.Categories = wordPressCategoryIds
+	//Get Author ID
+	editor, findEditorErr := GetEditor(postBase.Author, postBase.AuthorEmail, migration.ResultsDatabase)
+
+	if findEditorErr != nil {
+		editor, catchAllEditorErr := GetEditor("admin", "slowman2@slowtwitch.com", migration.ResultsDatabase)
+
+		if catchAllEditorErr != nil {
+			errorMessage = errorMessage + findEditorErr.Error()
+			failedPostResult := createPostFailureResult(postId, errorMessage, migration.ResultsDatabase)
+			return failedPostResult
+		}
+		createWordpressPost.Author = editor.WordpressId
+	}
+	createWordpressPost.Author = editor.WordpressId
+	//Get old link
+	oldLink := strings.ReplaceAll(firstCategoryResult.OldUrl, "index.html", "") + slowtwitch.ConvertPostTitleToPath(postBase.Title, postBase.Id)
+	linkStatus := slowtwitch.GetPageStatus(oldLink)
+
+	if linkStatus == 404 {
+		errorMessage = errorMessage + "Page not found on Slowtwitch"
+		failedPostResult := createPostFailureResult(postId, errorMessage, migration.ResultsDatabase)
+		return failedPostResult
+	}
+
+	//Get page, parse out post data and images
+	//Upload images to wordpress, swap out with new image urls
+	//Submit
+	imagePaths, html, retreiveHtmlErr := slowtwitch.GetImagesAndPostHtml(oldLink)
+	if retreiveHtmlErr != nil {
+		errorMessage = errorMessage + retreiveHtmlErr.Error()
+		failedPostResult := createPostFailureResult(postId, errorMessage, migration.ResultsDatabase)
+		return failedPostResult
+	}
+	//Create images from the image paths
+	var imageResults []ImageResult
+
+	for i, imagePath := range imagePaths {
+		//construct URL
+		imageUrl := "https://www.slowtwitch.com" + imagePath
+		createWordpressImage := wordpress.CreateImage{
+			Url: imageUrl,
+		}
+		//submit image
+		wordpressImage, wordpressImageErr := createWordpressImage.Execute(migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword)
+
+		if wordpressImageErr != nil {
+			errorMessage = errorMessage + wordpressImageErr.Error()
+			imageFailureResult := ImageResult{
+				OldUrl:      imageUrl,
+				NewUrl:      "",
+				WordpressId: 0,
+				IsSuccess:   false,
+			}
+			imageResults = append(imageResults, imageFailureResult)
+			continue
+		}
+		//first photo is the featured photo
+		if i == 0 {
+			createWordpressPost.FeaturedMedia = wordpressImage.Id
+		}
+		//begin process of recording result
+		imageResult := ImageResult{
+			OldUrl:      imageUrl,
+			NewUrl:      wordpressImage.Link,
+			WordpressId: wordpressImage.Id,
+			IsSuccess:   true,
+		}
+		imageResults = append(imageResults, imageResult)
+		//replace old links with new in post html
+		newImagePath := "/wp-content/uploads/" + wordpressImage.MediaDetails.File
+		html = strings.ReplaceAll(html, imagePath, newImagePath)
+		//create redirect
+		imageRedirect := wordpress.CreateRedirect{
+			Title:      postBase.Title + "image-" + string((i + 1)),
+			Url:        imagePath,
+			MatchType:  "page",
+			ActionType: "url",
+			ActionCode: 301,
+			GroupId:    1,
+			ActionData: wordpress.ActionData{
+				Url: "/" + wordpressImage.Slug,
+			},
+		}
+
+		_, imageRedirectErr := imageRedirect.Execute(migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword)
+
+		if imageRedirectErr != nil {
+			fmt.Println("Failed to create image redirect:", imageUrl, ":", imageRedirectErr.Error())
+		}
+	}
+
+	createWordpressPost.Content = html
+	post, createWordpressPostErr := createWordpressPost.Execute(migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword)
+	if createWordpressPostErr != nil {
+		errorMessage = errorMessage + createWordpressPostErr.Error()
+		postFailureResult := createPostFailureResult(postId, errorMessage, migration.ResultsDatabase)
+		return postFailureResult
+	}
+	//set up post result here to create
+	//truncate error message for db
+	if len(errorMessage) > 1450 {
+		errorMessage = errorMessage[:1450]
+	}
+
+	postResult := PostResult{
+		SlowtwitchId: postId,
+		WordpressId:  post.Id,
+		OldUrl:       oldLink,
+		OldUrlStatus: linkStatus,
+		NewUrl:       post.Link,
+		IsSuccess:    true,
+		ErrorMessage: errorMessage,
+	}
+
+	postResultId, createPostResultErr := CreatePostResult(postResult, migration.ResultsDatabase)
+	if createPostResultErr != nil {
+		fmt.Println("Could not record post result for Slowtwitch post:" + strconv.Itoa(postId) + createPostResultErr.Error())
+	}
+	for _, imageResult := range imageResults {
+		imageResult.PostId = postResultId
+		createImageResultErr := CreateImageResult(imageResult, migration.ResultsDatabase)
+		if createImageResultErr != nil {
+			fmt.Println("Error recording image result")
+		}
+	}
+	oldPath := strings.ReplaceAll(oldLink, "https://www.slowtwitch.com", "")
+	postRedirect := wordpress.CreateRedirect{
+		Title:      "Article Redirect" + postBase.Title,
+		Url:        oldPath,
+		MatchType:  "page",
+		ActionType: "url",
+		ActionCode: 301,
+		GroupId:    1,
+		ActionData: wordpress.ActionData{
+			Url: "/" + strings.ReplaceAll(postResult.NewUrl, migration.WordpressBaseUrl, ""),
+		},
+	}
+	_, postRedirectErr := postRedirect.Execute(migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword)
+	if postRedirectErr != nil {
+		fmt.Println("Error creating redirect for", postId, ":"+postRedirectErr.Error())
+	}
+	fmt.Println("Successfully created post and result for", postId)
+	updateAcfImages(imageResults, post.Id, migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword)
+	return postResult
+}
+
 func updatePostRelationships(postResults []PostResult, migration MigratePosts) {
 	fmt.Println("Updating post relationships")
-	for i, postResult := range postResults {
-		fmt.Println("Updating post", i+1, "/", len(postResults))
-		if postResult.IsSuccess {
-			var relatedWordpressIds []int
+	batchSize := 5
+	postResultWorkInput := make(chan PostResult)
+	//Set up a wait group to wait for the input channel to close after execution
+	var wg sync.WaitGroup
+	wg.Add(1)
+	wg.Add(len(postResults))
 
-			relatedSlowtwitchIds, slowtwitchIdsErr := slowtwitch.GetRelatedArticleIds(postResult.SlowtwitchId, migration.SlowtwitchDatabase)
-
-			if slowtwitchIdsErr != nil || len(relatedSlowtwitchIds) == 0 {
-				continue
-			}
-			for _, slowtwitchRelatedId := range relatedSlowtwitchIds {
-				wordpressRelatedId, wordpressIdErr := GetWordpressPostIdBySlowtwitchPostId(slowtwitchRelatedId, migration.ResultsDatabase)
-				if wordpressIdErr != nil {
-					continue
-				}
-				relatedWordpressIds = append(relatedWordpressIds, wordpressRelatedId)
+	//Launch go routines that will read from the input channel,
+	//maxed out at the batch size
+	for i := 0; i < batchSize; i++ {
+		go func() {
+			for result := range postResultWorkInput {
+				updatePostRelationship(result, migration)
+				wg.Done()
 			}
+		}()
+	}
 
-			if len(relatedWordpressIds) > 0 {
-				updateWordpressRelatedPosts := wordpress.UpdateAcfRelatedPosts{
-					Acf: wordpress.AcfRelatedPosts{
-						PostIds: relatedWordpressIds,
-					},
-				}
-				updateRelatedPostErr := updateWordpressRelatedPosts.Execute(migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword, postResult.WordpressId)
-				if updateRelatedPostErr != nil {
-					fmt.Println("Error updating wordpressRelatedPosts", updateRelatedPostErr.Error())
-				}
+	//Launch an unbuffered channel that will fill the channel with work
+	//as the workers take work from the channel. This will be blocked while
+	//any task is waiting to be picked up from the channel
+	go func() {
+		for _, result := range postResults {
+			postResultWorkInput <- result
+		}
+
+		close(postResultWorkInput)
+		wg.Done()
+	}()
+
+	wg.Wait()
+	fmt.Println("Done with post relationships")
+}
+
+func updatePostRelationship(postResult PostResult, migration MigratePosts) bool {
+	if postResult.IsSuccess {
+		var relatedWordpressIds []int
+
+		relatedSlowtwitchIds, slowtwitchIdsErr := slowtwitch.GetRelatedArticleIds(postResult.SlowtwitchId, migration.SlowtwitchDatabase)
+
+		if slowtwitchIdsErr != nil || len(relatedSlowtwitchIds) == 0 {
+			return false
+		}
+		for _, slowtwitchRelatedId := range relatedSlowtwitchIds {
+			wordpressRelatedId, wordpressIdErr := GetWordpressPostIdBySlowtwitchPostId(slowtwitchRelatedId, migration.ResultsDatabase)
+			if wordpressIdErr != nil {
+				return false
+			}
+			relatedWordpressIds = append(relatedWordpressIds, wordpressRelatedId)
+		}
+
+		if len(relatedWordpressIds) > 0 {
+			fmt.Println("Updating post in WP", postResult.WordpressId)
+			updateWordpressRelatedPosts := wordpress.UpdateAcfRelatedPosts{
+				Acf: wordpress.AcfRelatedPosts{
+					PostIds: relatedWordpressIds,
+				},
+			}
+			updateRelatedPostErr := updateWordpressRelatedPosts.Execute(migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword, postResult.WordpressId)
+			if updateRelatedPostErr != nil {
+				fmt.Println("Error updating wordpressRelatedPosts", updateRelatedPostErr.Error())
+			} else {
+				return true
 			}
 		}
 	}
+	return false
 }
 
 func getPostIdsThatNeedMigration(slowtwitchPostIds, migratedPostIds []int) []int {

services/migration/post-result.go

@@ -1,6 +1,9 @@
 package migration
 
-import "database/sql"
+import (
+	"database/sql"
+	"log"
+)
 
 type PostResult struct {
 	WordpressId  int
@@ -12,6 +15,28 @@ type PostResult struct {
 	ErrorMessage string
 }
 
+func GetAllPostResults(db *sql.DB) []PostResult {
+	var results []PostResult
+	rows, err := db.Query("select WordpressId, SlowtwitchId, OldUrl, OldUrlStatus, NewUrl, (IsSuccess = b'1'), ErrorMessage from PostResults")
+
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	defer rows.Close()
+
+	for rows.Next() {
+		result := PostResult{}
+		err = rows.Scan(&result.WordpressId, &result.SlowtwitchId, &result.OldUrl, &result.OldUrlStatus, &result.NewUrl, &result.IsSuccess, &result.ErrorMessage)
+		if err != nil {
+			log.Fatal(err)
+		}
+		results = append(results, result)
+	}
+
+	return results
+}
+
 func CreatePostResult(parameters PostResult, db *sql.DB) (int, error) {
 	result, err := db.Exec("insert into PostResults (WordpressId, SlowtwitchId, OldUrl, OldUrlStatus, NewUrl, IsSuccess, ErrorMessage) values (?, ?, ?, ?, ?, ?, ?)", parameters.WordpressId, parameters.SlowtwitchId, parameters.OldUrl, parameters.OldUrlStatus, parameters.NewUrl, parameters.IsSuccess, parameters.ErrorMessage)
 	if err != nil {

services/slowtwitch/url-converter.go

@@ -28,5 +28,9 @@ func convert(path string) string {
 	output = strings.ReplaceAll(output, "?", "")
 	output = strings.ReplaceAll(output, ",", "")
 	output = strings.ReplaceAll(output, "%", "%25")
+	output = strings.ReplaceAll(output, "_&", "")
+	output = strings.ReplaceAll(output, "&", "")
+	output = strings.ReplaceAll(output, "#", "")
+	output = strings.ReplaceAll(output, ";", "")
 	return output
 }