From 90b725bb8a4f0d49e578d1ed1983fcb4edc7bd9a Mon Sep 17 00:00:00 2001 From: Ross Trottier Date: Sat, 25 May 2024 09:31:59 -0600 Subject: [PATCH] threaded post migrator --- .idea/sqldialects.xml | 7 + main.go | 6 +- services/migration/migrate-posts.go | 405 +++++++++++++++------------- 3 files changed, 230 insertions(+), 188 deletions(-) create mode 100644 .idea/sqldialects.xml diff --git a/.idea/sqldialects.xml b/.idea/sqldialects.xml new file mode 100644 index 0000000..e15a914 --- /dev/null +++ b/.idea/sqldialects.xml @@ -0,0 +1,7 @@ + + + + + + + \ No newline at end of file diff --git a/main.go b/main.go index d4402b1..a18304e 100644 --- a/main.go +++ b/main.go @@ -7,9 +7,9 @@ import ( ) // WP Config -const baseUrl = "https://slowtwitch.cloud/" -const wordpressKey = "admin@slowtwitch.cloud" -const wordpressSecret = "6zY7 xsKZ dGIt l1Lp ypIK 6TWh" +const baseUrl = "http://go-api-playground.local/" +const wordpressKey = "admin" +const wordpressSecret = "KStj VH7E 9UZf FV8C ptGQ F4Tl" // DB Config const slowtwitchAdminUser = "admin" diff --git a/services/migration/migrate-posts.go b/services/migration/migrate-posts.go index 57a0a91..c042c98 100644 --- a/services/migration/migrate-posts.go +++ b/services/migration/migrate-posts.go @@ -37,198 +37,232 @@ func (migration MigratePosts) Execute() []PostResult { } var postResults []PostResult + batchSize := 5 + var postBatch []int + for _, postId := range slowtwitchPostIdsForMigration { - errorMessage := "" - //migrate - postBase, postBaseErr := slowtwitch.GetPostBase(postId, migration.SlowtwitchDatabase) - if postBaseErr != nil { - errorMessage = errorMessage + postBaseErr.Error() - createPostFailureResult(postId, errorMessage, migration.ResultsDatabase) - continue - } + //anonymous func will take in post IDs: the rest it can access from scope - createWordpressPost := wordpress.CreatePost{ - Title: postBase.Title, - Excerpt: postBase.Description, - Status: "publish", - } + postBatch = append(postBatch, postId) + if len(postBatch) == batchSize { + postResultsChannel := make(chan PostResult) + for _, batchId := range postBatch { + go func(id int) { + errorMessage := "" + //migrate + postBase, postBaseErr := slowtwitch.GetPostBase(batchId, migration.SlowtwitchDatabase) - if postBase.DatePublished.Valid { - t := postBase.DatePublished.Time - timeString := fmt.Sprintf("%d-%02d-%02dT%02d:%02d:%02d", - t.Year(), t.Month(), t.Day(), - t.Hour(), t.Minute(), t.Second()) - createWordpressPost.Date = timeString - } else { - errorMessage = errorMessage + "Invalid Date Published" - createPostFailureResult(postId, errorMessage, migration.ResultsDatabase) - continue - } + if postBaseErr != nil { + errorMessage = errorMessage + postBaseErr.Error() + failedPostResult := createPostFailureResult(batchId, errorMessage, migration.ResultsDatabase) + postResultsChannel <- failedPostResult + return + } - for _, tag := range wpTagData { - if postBase.Bike == true && tag.Name == "bike" { - createWordpressPost.Tags = append(createWordpressPost.Tags, tag.Id) + createWordpressPost := wordpress.CreatePost{ + Title: postBase.Title, + Excerpt: postBase.Description, + Status: "publish", + } + + if postBase.DatePublished.Valid { + t := postBase.DatePublished.Time + timeString := fmt.Sprintf("%d-%02d-%02dT%02d:%02d:%02d", + t.Year(), t.Month(), t.Day(), + t.Hour(), t.Minute(), t.Second()) + createWordpressPost.Date = timeString + } else { + errorMessage = errorMessage + "Invalid Date Published" + failedPostResult := createPostFailureResult(batchId, errorMessage, migration.ResultsDatabase) + postResultsChannel <- failedPostResult + return + } + + for _, tag := range wpTagData { + if postBase.Bike == true && tag.Name == "bike" { + createWordpressPost.Tags = append(createWordpressPost.Tags, tag.Id) + } + if postBase.Swim == true && tag.Name == "swim" { + createWordpressPost.Tags = append(createWordpressPost.Tags, tag.Id) + } + if postBase.Run == true && tag.Name == "run" { + createWordpressPost.Tags = append(createWordpressPost.Tags, tag.Id) + } + } + + var wordPressCategoryIds []int + var firstCategoryResult CategoryResult + for _, slowtwitchCategoryId := range postBase.CategoryIds { + categoryResult, err := GetSlowtwitchCategoryResult(slowtwitchCategoryId, migration.ResultsDatabase) + if err != nil { + errorMessage = errorMessage + err.Error() + failedPostResult := createPostFailureResult(batchId, errorMessage, migration.ResultsDatabase) + postResultsChannel <- failedPostResult + return + } + wordPressCategoryIds = append(wordPressCategoryIds, categoryResult.WordpressId) + firstCategoryResult = categoryResult + } + + if len(wordPressCategoryIds) == 0 { + errorMessage = "This post has no categories and is broken on the production site." + failedPostResult := createPostFailureResult(batchId, errorMessage, migration.ResultsDatabase) + postResultsChannel <- failedPostResult + return + } + + createWordpressPost.Categories = wordPressCategoryIds + //Get Author ID + editor, findEditorErr := GetEditor(postBase.Author, migration.ResultsDatabase) + + if findEditorErr != nil { + errorMessage = errorMessage + findEditorErr.Error() + failedPostResult := createPostFailureResult(batchId, errorMessage, migration.ResultsDatabase) + postResultsChannel <- failedPostResult + return + } + createWordpressPost.Author = editor.WordpressId + //Get old link + oldLink := strings.ReplaceAll(firstCategoryResult.OldUrl, "index.html", "") + slowtwitch.ConvertPostTitleToPath(postBase.Title, postBase.Id) + linkStatus := slowtwitch.GetPageStatus(oldLink) + + if linkStatus == 404 { + errorMessage = errorMessage + "Page not found on Slowtwitch" + failedPostResult := createPostFailureResult(batchId, errorMessage, migration.ResultsDatabase) + postResultsChannel <- failedPostResult + return + } + + //Get page, parse out post data and images + //Upload images to wordpress, swap out with new image urls + //Submit + imagePaths, html, retreiveHtmlErr := slowtwitch.GetImagesAndPostHtml(oldLink) + if retreiveHtmlErr != nil { + errorMessage = errorMessage + retreiveHtmlErr.Error() + failedPostResult := createPostFailureResult(batchId, errorMessage, migration.ResultsDatabase) + postResultsChannel <- failedPostResult + return + } + //Create images from the image paths + var imageResults []ImageResult + + for i, imagePath := range imagePaths { + //construct URL + imageUrl := "https://www.slowtwitch.com" + imagePath + createWordpressImage := wordpress.CreateImage{ + Url: imageUrl, + } + //submit image + wordpressImage, wordpressImageErr := createWordpressImage.Execute(migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword) + + if wordpressImageErr != nil { + errorMessage = errorMessage + wordpressImageErr.Error() + createImageFailureResult(imageUrl, migration.ResultsDatabase) + continue + } + //first photo is the featured photo + if i == 0 { + createWordpressPost.FeaturedMedia = wordpressImage.Id + } + //begin process of recording result + imageResult := ImageResult{ + OldUrl: imageUrl, + NewUrl: wordpressImage.Link, + WordpressId: wordpressImage.Id, + IsSuccess: true, + } + imageResults = append(imageResults, imageResult) + //replace old links with new in post html + strings.ReplaceAll(html, imageUrl, wordpressImage.Link) + //create redirect + imageRedirect := wordpress.CreateRedirect{ + Title: postBase.Title + "image-" + string((i + 1)), + Url: imagePath, + MatchType: "page", + ActionType: "url", + ActionCode: 301, + GroupId: 1, + ActionData: wordpress.ActionData{ + Url: "/" + wordpressImage.Slug, + }, + } + + _, imageRedirectErr := imageRedirect.Execute(migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword) + + if imageRedirectErr != nil { + fmt.Println("Failed to create image redirect:", imageUrl, ":", imageRedirectErr.Error()) + } + } + + createWordpressPost.Content = html + post, createWordpressPostErr := createWordpressPost.Execute(migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword) + if createWordpressPostErr != nil { + errorMessage = errorMessage + createWordpressPostErr.Error() + postFailureResult := createPostFailureResult(batchId, errorMessage, migration.ResultsDatabase) + postResultsChannel <- postFailureResult + return + } + //set up post result here to create + //truncate error message for db + if len(errorMessage) > 1450 { + errorMessage = errorMessage[:1450] + } + + postResult := PostResult{ + SlowtwitchId: batchId, + WordpressId: post.Id, + OldUrl: oldLink, + OldUrlStatus: linkStatus, + NewUrl: post.Link, + IsSuccess: true, + ErrorMessage: errorMessage, + } + + postResultId, createPostResultErr := CreatePostResult(postResult, migration.ResultsDatabase) + if createPostResultErr != nil { + fmt.Println("Could not record post result for Slowtwitch post:" + strconv.Itoa(batchId) + createPostResultErr.Error()) + } + for _, imageResult := range imageResults { + imageResult.PostId = postResultId + createImageResultErr := CreateImageResult(imageResult, migration.ResultsDatabase) + if createImageResultErr != nil { + fmt.Println("Error recording image result") + } + } + oldPath := strings.ReplaceAll(oldLink, "https://www.slowtwitch.com", "") + postRedirect := wordpress.CreateRedirect{ + Title: "Article Redirect" + postBase.Title, + Url: oldPath, + MatchType: "page", + ActionType: "url", + ActionCode: 301, + GroupId: 1, + ActionData: wordpress.ActionData{ + Url: "/" + strings.ReplaceAll(postResult.NewUrl, migration.WordpressBaseUrl, ""), + }, + } + _, postRedirectErr := postRedirect.Execute(migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword) + if postRedirectErr != nil { + fmt.Println("Error creating redirect for", batchId, ":"+postRedirectErr.Error()) + } + fmt.Println("Successfully created post and result for", batchId) + updateAcfImages(imageResults, post.Id, migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword) + postResults = append(postResults, postResult) + postResultsChannel <- postResult + }(batchId) } - if postBase.Swim == true && tag.Name == "swim" { - createWordpressPost.Tags = append(createWordpressPost.Tags, tag.Id) - } - if postBase.Run == true && tag.Name == "run" { - createWordpressPost.Tags = append(createWordpressPost.Tags, tag.Id) - } - } - - var wordPressCategoryIds []int - var firstCategoryResult CategoryResult - for _, slowtwitchCategoryId := range postBase.CategoryIds { - categoryResult, err := GetSlowtwitchCategoryResult(slowtwitchCategoryId, migration.ResultsDatabase) - if err != nil { - errorMessage = errorMessage + err.Error() - createPostFailureResult(postId, errorMessage, migration.ResultsDatabase) - continue - } - wordPressCategoryIds = append(wordPressCategoryIds, categoryResult.WordpressId) - firstCategoryResult = categoryResult - } - - if len(wordPressCategoryIds) == 0 { - errorMessage = "This post has no categories and is broken on the production site." - createPostFailureResult(postId, errorMessage, migration.ResultsDatabase) - continue - } - - createWordpressPost.Categories = wordPressCategoryIds - //Get Author ID - editor, findEditorErr := GetEditor(postBase.Author, migration.ResultsDatabase) - - if findEditorErr != nil { - errorMessage = errorMessage + findEditorErr.Error() - createPostFailureResult(postId, errorMessage, migration.ResultsDatabase) - continue - } - createWordpressPost.Author = editor.WordpressId - //Get old link - oldLink := strings.ReplaceAll(firstCategoryResult.OldUrl, "index.html", "") + slowtwitch.ConvertPostTitleToPath(postBase.Title, postBase.Id) - linkStatus := slowtwitch.GetPageStatus(oldLink) - - if linkStatus == 404 { - errorMessage = errorMessage + "Page not found on Slowtwitch" - createPostFailureResult(postId, errorMessage, migration.ResultsDatabase) - continue - } - - //Get page, parse out post data and images - //Upload images to wordpress, swap out with new image urls - //Submit - imagePaths, html, retreiveHtmlErr := slowtwitch.GetImagesAndPostHtml(oldLink) - if retreiveHtmlErr != nil { - errorMessage = errorMessage + retreiveHtmlErr.Error() - createPostFailureResult(postId, errorMessage, migration.ResultsDatabase) - continue - } - //Create images from the image paths - var imageResults []ImageResult - - for i, imagePath := range imagePaths { - //construct URL - imageUrl := "https://www.slowtwitch.com" + imagePath - createWordpressImage := wordpress.CreateImage{ - Url: imageUrl, - } - //submit image - wordpressImage, wordpressImageErr := createWordpressImage.Execute(migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword) - - if wordpressImageErr != nil { - errorMessage = errorMessage + wordpressImageErr.Error() - createImageFailureResult(imageUrl, migration.ResultsDatabase) - continue - } - //first photo is the featured photo - if i == 0 { - createWordpressPost.FeaturedMedia = wordpressImage.Id - } - //begin process of recording result - imageResult := ImageResult{ - OldUrl: imageUrl, - NewUrl: wordpressImage.Link, - WordpressId: wordpressImage.Id, - IsSuccess: true, - } - imageResults = append(imageResults, imageResult) - //replace old links with new in post html - strings.ReplaceAll(html, imageUrl, wordpressImage.Link) - //create redirect - imageRedirect := wordpress.CreateRedirect{ - Title: postBase.Title + "image-" + string((i + 1)), - Url: imagePath, - MatchType: "page", - ActionType: "url", - ActionCode: 301, - GroupId: 1, - ActionData: wordpress.ActionData{ - Url: "/" + wordpressImage.Slug, - }, + //Listen to channel and record results + var batchResults []PostResult + for result := range postResultsChannel { + batchResults = append(batchResults, result) + if len(batchResults) == batchSize { + close(postResultsChannel) + } } - _, imageRedirectErr := imageRedirect.Execute(migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword) - - if imageRedirectErr != nil { - fmt.Println("Failed to create image redirect:", imageUrl, ":", imageRedirectErr.Error()) - } + postBatch = nil + postResults = append(postResults, batchResults...) } - - createWordpressPost.Content = html - post, createWordpressPostErr := createWordpressPost.Execute(migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword) - if createWordpressPostErr != nil { - errorMessage = errorMessage + createWordpressPostErr.Error() - createPostFailureResult(postId, errorMessage, migration.ResultsDatabase) - continue - } - //set up post result here to create - //truncate error message for db - if len(errorMessage) > 1450 { - errorMessage = errorMessage[:1450] - } - - postResult := PostResult{ - SlowtwitchId: postId, - WordpressId: post.Id, - OldUrl: oldLink, - OldUrlStatus: linkStatus, - NewUrl: post.Link, - IsSuccess: true, - ErrorMessage: errorMessage, - } - - postResultId, createPostResultErr := CreatePostResult(postResult, migration.ResultsDatabase) - if createPostResultErr != nil { - fmt.Println("Could not record post result for Slowtwitch post:" + strconv.Itoa(postId) + createPostResultErr.Error()) - } - for _, imageResult := range imageResults { - imageResult.PostId = postResultId - createImageResultErr := CreateImageResult(imageResult, migration.ResultsDatabase) - if createImageResultErr != nil { - fmt.Println("Error recording image result") - } - } - oldPath := strings.ReplaceAll(oldLink, "https://www.slowtwitch.com", "") - postRedirect := wordpress.CreateRedirect{ - Title: "Article Redirect" + postBase.Title, - Url: oldPath, - MatchType: "page", - ActionType: "url", - ActionCode: 301, - GroupId: 1, - ActionData: wordpress.ActionData{ - Url: "/" + strings.ReplaceAll(postResult.NewUrl, migration.WordpressBaseUrl, ""), - }, - } - _, postRedirectErr := postRedirect.Execute(migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword) - if postRedirectErr != nil { - fmt.Println("Error creating redirect for", postId, ":"+postRedirectErr.Error()) - } - fmt.Println("Successfully created post and result for", postId) - updateAcfImages(imageResults, post.Id, migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword) - postResults = append(postResults, postResult) } // Update related posts once work is done updatePostRelationships(postResults, migration) @@ -282,7 +316,7 @@ func getPostIdsThatNeedMigration(slowtwitchPostIds, migratedPostIds []int) []int return output } -func createPostFailureResult(slowtwitchId int, errorMessage string, migrationDb *sql.DB) { +func createPostFailureResult(slowtwitchId int, errorMessage string, migrationDb *sql.DB) PostResult { fmt.Println("Error creating post: ", slowtwitchId, ":", errorMessage) postResult := PostResult{ @@ -300,6 +334,7 @@ func createPostFailureResult(slowtwitchId int, errorMessage string, migrationDb if err != nil { fmt.Println("Failed to create failure result: ", err) } + return postResult } func createImageFailureResult(url string, resultsDatabase *sql.DB) {