threaded post migrator

.idea/sqldialects.xml

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="SqlDialectMappings">
+    <file url="file://$PROJECT_DIR$/main.go" dialect="GenericSQL" />
+    <file url="PROJECT" dialect="MySQL" />
+  </component>
+</project>

main.go

@@ -7,9 +7,9 @@ import (
 )
 
 // WP Config
-const baseUrl = "https://slowtwitch.cloud/"
+const baseUrl = "http://go-api-playground.local/"
-const wordpressKey = "admin@slowtwitch.cloud"
+const wordpressKey = "admin"
-const wordpressSecret = "6zY7 xsKZ dGIt l1Lp ypIK 6TWh"
+const wordpressSecret = "KStj VH7E 9UZf FV8C ptGQ F4Tl"
 
 // DB Config
 const slowtwitchAdminUser = "admin"

services/migration/migrate-posts.go

@@ -37,198 +37,232 @@ func (migration MigratePosts) Execute() []PostResult {
 	}
 
 	var postResults []PostResult
+	batchSize := 5
+	var postBatch []int
+
 	for _, postId := range slowtwitchPostIdsForMigration {
-		errorMessage := ""
+		//anonymous func will take in post IDs: the rest it can access from scope
-		//migrate
-		postBase, postBaseErr := slowtwitch.GetPostBase(postId, migration.SlowtwitchDatabase)
-		if postBaseErr != nil {
-			errorMessage = errorMessage + postBaseErr.Error()
-			createPostFailureResult(postId, errorMessage, migration.ResultsDatabase)
-			continue
-		}
 
-		createWordpressPost := wordpress.CreatePost{
+		postBatch = append(postBatch, postId)
-			Title:   postBase.Title,
+		if len(postBatch) == batchSize {
-			Excerpt: postBase.Description,
+			postResultsChannel := make(chan PostResult)
-			Status:  "publish",
+			for _, batchId := range postBatch {
-		}
+				go func(id int) {
+					errorMessage := ""
+					//migrate
+					postBase, postBaseErr := slowtwitch.GetPostBase(batchId, migration.SlowtwitchDatabase)
 
-		if postBase.DatePublished.Valid {
+					if postBaseErr != nil {
-			t := postBase.DatePublished.Time
+						errorMessage = errorMessage + postBaseErr.Error()
-			timeString := fmt.Sprintf("%d-%02d-%02dT%02d:%02d:%02d",
+						failedPostResult := createPostFailureResult(batchId, errorMessage, migration.ResultsDatabase)
-				t.Year(), t.Month(), t.Day(),
+						postResultsChannel <- failedPostResult
-				t.Hour(), t.Minute(), t.Second())
+						return
-			createWordpressPost.Date = timeString
+					}
-		} else {
-			errorMessage = errorMessage + "Invalid Date Published"
-			createPostFailureResult(postId, errorMessage, migration.ResultsDatabase)
-			continue
-		}
 
-		for _, tag := range wpTagData {
+					createWordpressPost := wordpress.CreatePost{
-			if postBase.Bike == true && tag.Name == "bike" {
+						Title:   postBase.Title,
-				createWordpressPost.Tags = append(createWordpressPost.Tags, tag.Id)
+						Excerpt: postBase.Description,
+						Status:  "publish",
+					}
+
+					if postBase.DatePublished.Valid {
+						t := postBase.DatePublished.Time
+						timeString := fmt.Sprintf("%d-%02d-%02dT%02d:%02d:%02d",
+							t.Year(), t.Month(), t.Day(),
+							t.Hour(), t.Minute(), t.Second())
+						createWordpressPost.Date = timeString
+					} else {
+						errorMessage = errorMessage + "Invalid Date Published"
+						failedPostResult := createPostFailureResult(batchId, errorMessage, migration.ResultsDatabase)
+						postResultsChannel <- failedPostResult
+						return
+					}
+
+					for _, tag := range wpTagData {
+						if postBase.Bike == true && tag.Name == "bike" {
+							createWordpressPost.Tags = append(createWordpressPost.Tags, tag.Id)
+						}
+						if postBase.Swim == true && tag.Name == "swim" {
+							createWordpressPost.Tags = append(createWordpressPost.Tags, tag.Id)
+						}
+						if postBase.Run == true && tag.Name == "run" {
+							createWordpressPost.Tags = append(createWordpressPost.Tags, tag.Id)
+						}
+					}
+
+					var wordPressCategoryIds []int
+					var firstCategoryResult CategoryResult
+					for _, slowtwitchCategoryId := range postBase.CategoryIds {
+						categoryResult, err := GetSlowtwitchCategoryResult(slowtwitchCategoryId, migration.ResultsDatabase)
+						if err != nil {
+							errorMessage = errorMessage + err.Error()
+							failedPostResult := createPostFailureResult(batchId, errorMessage, migration.ResultsDatabase)
+							postResultsChannel <- failedPostResult
+							return
+						}
+						wordPressCategoryIds = append(wordPressCategoryIds, categoryResult.WordpressId)
+						firstCategoryResult = categoryResult
+					}
+
+					if len(wordPressCategoryIds) == 0 {
+						errorMessage = "This post has no categories and is broken on the production site."
+						failedPostResult := createPostFailureResult(batchId, errorMessage, migration.ResultsDatabase)
+						postResultsChannel <- failedPostResult
+						return
+					}
+
+					createWordpressPost.Categories = wordPressCategoryIds
+					//Get Author ID
+					editor, findEditorErr := GetEditor(postBase.Author, migration.ResultsDatabase)
+
+					if findEditorErr != nil {
+						errorMessage = errorMessage + findEditorErr.Error()
+						failedPostResult := createPostFailureResult(batchId, errorMessage, migration.ResultsDatabase)
+						postResultsChannel <- failedPostResult
+						return
+					}
+					createWordpressPost.Author = editor.WordpressId
+					//Get old link
+					oldLink := strings.ReplaceAll(firstCategoryResult.OldUrl, "index.html", "") + slowtwitch.ConvertPostTitleToPath(postBase.Title, postBase.Id)
+					linkStatus := slowtwitch.GetPageStatus(oldLink)
+
+					if linkStatus == 404 {
+						errorMessage = errorMessage + "Page not found on Slowtwitch"
+						failedPostResult := createPostFailureResult(batchId, errorMessage, migration.ResultsDatabase)
+						postResultsChannel <- failedPostResult
+						return
+					}
+
+					//Get page, parse out post data and images
+					//Upload images to wordpress, swap out with new image urls
+					//Submit
+					imagePaths, html, retreiveHtmlErr := slowtwitch.GetImagesAndPostHtml(oldLink)
+					if retreiveHtmlErr != nil {
+						errorMessage = errorMessage + retreiveHtmlErr.Error()
+						failedPostResult := createPostFailureResult(batchId, errorMessage, migration.ResultsDatabase)
+						postResultsChannel <- failedPostResult
+						return
+					}
+					//Create images from the image paths
+					var imageResults []ImageResult
+
+					for i, imagePath := range imagePaths {
+						//construct URL
+						imageUrl := "https://www.slowtwitch.com" + imagePath
+						createWordpressImage := wordpress.CreateImage{
+							Url: imageUrl,
+						}
+						//submit image
+						wordpressImage, wordpressImageErr := createWordpressImage.Execute(migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword)
+
+						if wordpressImageErr != nil {
+							errorMessage = errorMessage + wordpressImageErr.Error()
+							createImageFailureResult(imageUrl, migration.ResultsDatabase)
+							continue
+						}
+						//first photo is the featured photo
+						if i == 0 {
+							createWordpressPost.FeaturedMedia = wordpressImage.Id
+						}
+						//begin process of recording result
+						imageResult := ImageResult{
+							OldUrl:      imageUrl,
+							NewUrl:      wordpressImage.Link,
+							WordpressId: wordpressImage.Id,
+							IsSuccess:   true,
+						}
+						imageResults = append(imageResults, imageResult)
+						//replace old links with new in post html
+						strings.ReplaceAll(html, imageUrl, wordpressImage.Link)
+						//create redirect
+						imageRedirect := wordpress.CreateRedirect{
+							Title:      postBase.Title + "image-" + string((i + 1)),
+							Url:        imagePath,
+							MatchType:  "page",
+							ActionType: "url",
+							ActionCode: 301,
+							GroupId:    1,
+							ActionData: wordpress.ActionData{
+								Url: "/" + wordpressImage.Slug,
+							},
+						}
+
+						_, imageRedirectErr := imageRedirect.Execute(migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword)
+
+						if imageRedirectErr != nil {
+							fmt.Println("Failed to create image redirect:", imageUrl, ":", imageRedirectErr.Error())
+						}
+					}
+
+					createWordpressPost.Content = html
+					post, createWordpressPostErr := createWordpressPost.Execute(migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword)
+					if createWordpressPostErr != nil {
+						errorMessage = errorMessage + createWordpressPostErr.Error()
+						postFailureResult := createPostFailureResult(batchId, errorMessage, migration.ResultsDatabase)
+						postResultsChannel <- postFailureResult
+						return
+					}
+					//set up post result here to create
+					//truncate error message for db
+					if len(errorMessage) > 1450 {
+						errorMessage = errorMessage[:1450]
+					}
+
+					postResult := PostResult{
+						SlowtwitchId: batchId,
+						WordpressId:  post.Id,
+						OldUrl:       oldLink,
+						OldUrlStatus: linkStatus,
+						NewUrl:       post.Link,
+						IsSuccess:    true,
+						ErrorMessage: errorMessage,
+					}
+
+					postResultId, createPostResultErr := CreatePostResult(postResult, migration.ResultsDatabase)
+					if createPostResultErr != nil {
+						fmt.Println("Could not record post result for Slowtwitch post:" + strconv.Itoa(batchId) + createPostResultErr.Error())
+					}
+					for _, imageResult := range imageResults {
+						imageResult.PostId = postResultId
+						createImageResultErr := CreateImageResult(imageResult, migration.ResultsDatabase)
+						if createImageResultErr != nil {
+							fmt.Println("Error recording image result")
+						}
+					}
+					oldPath := strings.ReplaceAll(oldLink, "https://www.slowtwitch.com", "")
+					postRedirect := wordpress.CreateRedirect{
+						Title:      "Article Redirect" + postBase.Title,
+						Url:        oldPath,
+						MatchType:  "page",
+						ActionType: "url",
+						ActionCode: 301,
+						GroupId:    1,
+						ActionData: wordpress.ActionData{
+							Url: "/" + strings.ReplaceAll(postResult.NewUrl, migration.WordpressBaseUrl, ""),
+						},
+					}
+					_, postRedirectErr := postRedirect.Execute(migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword)
+					if postRedirectErr != nil {
+						fmt.Println("Error creating redirect for", batchId, ":"+postRedirectErr.Error())
+					}
+					fmt.Println("Successfully created post and result for", batchId)
+					updateAcfImages(imageResults, post.Id, migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword)
+					postResults = append(postResults, postResult)
+					postResultsChannel <- postResult
+				}(batchId)
 			}
-			if postBase.Swim == true && tag.Name == "swim" {
+			//Listen to channel and record results
-				createWordpressPost.Tags = append(createWordpressPost.Tags, tag.Id)
+			var batchResults []PostResult
-			}
+			for result := range postResultsChannel {
-			if postBase.Run == true && tag.Name == "run" {
+				batchResults = append(batchResults, result)
-				createWordpressPost.Tags = append(createWordpressPost.Tags, tag.Id)
+				if len(batchResults) == batchSize {
-			}
+					close(postResultsChannel)
-		}
+				}
-
-		var wordPressCategoryIds []int
-		var firstCategoryResult CategoryResult
-		for _, slowtwitchCategoryId := range postBase.CategoryIds {
-			categoryResult, err := GetSlowtwitchCategoryResult(slowtwitchCategoryId, migration.ResultsDatabase)
-			if err != nil {
-				errorMessage = errorMessage + err.Error()
-				createPostFailureResult(postId, errorMessage, migration.ResultsDatabase)
-				continue
-			}
-			wordPressCategoryIds = append(wordPressCategoryIds, categoryResult.WordpressId)
-			firstCategoryResult = categoryResult
-		}
-
-		if len(wordPressCategoryIds) == 0 {
-			errorMessage = "This post has no categories and is broken on the production site."
-			createPostFailureResult(postId, errorMessage, migration.ResultsDatabase)
-			continue
-		}
-
-		createWordpressPost.Categories = wordPressCategoryIds
-		//Get Author ID
-		editor, findEditorErr := GetEditor(postBase.Author, migration.ResultsDatabase)
-
-		if findEditorErr != nil {
-			errorMessage = errorMessage + findEditorErr.Error()
-			createPostFailureResult(postId, errorMessage, migration.ResultsDatabase)
-			continue
-		}
-		createWordpressPost.Author = editor.WordpressId
-		//Get old link
-		oldLink := strings.ReplaceAll(firstCategoryResult.OldUrl, "index.html", "") + slowtwitch.ConvertPostTitleToPath(postBase.Title, postBase.Id)
-		linkStatus := slowtwitch.GetPageStatus(oldLink)
-
-		if linkStatus == 404 {
-			errorMessage = errorMessage + "Page not found on Slowtwitch"
-			createPostFailureResult(postId, errorMessage, migration.ResultsDatabase)
-			continue
-		}
-
-		//Get page, parse out post data and images
-		//Upload images to wordpress, swap out with new image urls
-		//Submit
-		imagePaths, html, retreiveHtmlErr := slowtwitch.GetImagesAndPostHtml(oldLink)
-		if retreiveHtmlErr != nil {
-			errorMessage = errorMessage + retreiveHtmlErr.Error()
-			createPostFailureResult(postId, errorMessage, migration.ResultsDatabase)
-			continue
-		}
-		//Create images from the image paths
-		var imageResults []ImageResult
-
-		for i, imagePath := range imagePaths {
-			//construct URL
-			imageUrl := "https://www.slowtwitch.com" + imagePath
-			createWordpressImage := wordpress.CreateImage{
-				Url: imageUrl,
-			}
-			//submit image
-			wordpressImage, wordpressImageErr := createWordpressImage.Execute(migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword)
-
-			if wordpressImageErr != nil {
-				errorMessage = errorMessage + wordpressImageErr.Error()
-				createImageFailureResult(imageUrl, migration.ResultsDatabase)
-				continue
-			}
-			//first photo is the featured photo
-			if i == 0 {
-				createWordpressPost.FeaturedMedia = wordpressImage.Id
-			}
-			//begin process of recording result
-			imageResult := ImageResult{
-				OldUrl:      imageUrl,
-				NewUrl:      wordpressImage.Link,
-				WordpressId: wordpressImage.Id,
-				IsSuccess:   true,
-			}
-			imageResults = append(imageResults, imageResult)
-			//replace old links with new in post html
-			strings.ReplaceAll(html, imageUrl, wordpressImage.Link)
-			//create redirect
-			imageRedirect := wordpress.CreateRedirect{
-				Title:      postBase.Title + "image-" + string((i + 1)),
-				Url:        imagePath,
-				MatchType:  "page",
-				ActionType: "url",
-				ActionCode: 301,
-				GroupId:    1,
-				ActionData: wordpress.ActionData{
-					Url: "/" + wordpressImage.Slug,
-				},
 			}
 
-			_, imageRedirectErr := imageRedirect.Execute(migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword)
+			postBatch = nil
-
+			postResults = append(postResults, batchResults...)
-			if imageRedirectErr != nil {
-				fmt.Println("Failed to create image redirect:", imageUrl, ":", imageRedirectErr.Error())
-			}
 		}
-
-		createWordpressPost.Content = html
-		post, createWordpressPostErr := createWordpressPost.Execute(migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword)
-		if createWordpressPostErr != nil {
-			errorMessage = errorMessage + createWordpressPostErr.Error()
-			createPostFailureResult(postId, errorMessage, migration.ResultsDatabase)
-			continue
-		}
-		//set up post result here to create
-		//truncate error message for db
-		if len(errorMessage) > 1450 {
-			errorMessage = errorMessage[:1450]
-		}
-
-		postResult := PostResult{
-			SlowtwitchId: postId,
-			WordpressId:  post.Id,
-			OldUrl:       oldLink,
-			OldUrlStatus: linkStatus,
-			NewUrl:       post.Link,
-			IsSuccess:    true,
-			ErrorMessage: errorMessage,
-		}
-
-		postResultId, createPostResultErr := CreatePostResult(postResult, migration.ResultsDatabase)
-		if createPostResultErr != nil {
-			fmt.Println("Could not record post result for Slowtwitch post:" + strconv.Itoa(postId) + createPostResultErr.Error())
-		}
-		for _, imageResult := range imageResults {
-			imageResult.PostId = postResultId
-			createImageResultErr := CreateImageResult(imageResult, migration.ResultsDatabase)
-			if createImageResultErr != nil {
-				fmt.Println("Error recording image result")
-			}
-		}
-		oldPath := strings.ReplaceAll(oldLink, "https://www.slowtwitch.com", "")
-		postRedirect := wordpress.CreateRedirect{
-			Title:      "Article Redirect" + postBase.Title,
-			Url:        oldPath,
-			MatchType:  "page",
-			ActionType: "url",
-			ActionCode: 301,
-			GroupId:    1,
-			ActionData: wordpress.ActionData{
-				Url: "/" + strings.ReplaceAll(postResult.NewUrl, migration.WordpressBaseUrl, ""),
-			},
-		}
-		_, postRedirectErr := postRedirect.Execute(migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword)
-		if postRedirectErr != nil {
-			fmt.Println("Error creating redirect for", postId, ":"+postRedirectErr.Error())
-		}
-		fmt.Println("Successfully created post and result for", postId)
-		updateAcfImages(imageResults, post.Id, migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword)
-		postResults = append(postResults, postResult)
 	}
 	// Update related posts once work is done
 	updatePostRelationships(postResults, migration)
@@ -282,7 +316,7 @@ func getPostIdsThatNeedMigration(slowtwitchPostIds, migratedPostIds []int) []int
 	return output
 }
 
-func createPostFailureResult(slowtwitchId int, errorMessage string, migrationDb *sql.DB) {
+func createPostFailureResult(slowtwitchId int, errorMessage string, migrationDb *sql.DB) PostResult {
 	fmt.Println("Error creating post: ", slowtwitchId, ":", errorMessage)
 
 	postResult := PostResult{
@@ -300,6 +334,7 @@ func createPostFailureResult(slowtwitchId int, errorMessage string, migrationDb
 	if err != nil {
 		fmt.Println("Failed to create failure result: ", err)
 	}
+	return postResult
 }
 
 func createImageFailureResult(url string, resultsDatabase *sql.DB) {