package migration import ( "database/sql" "federated.computer/wp-sync-slowtwitch/services/slowtwitch" "federated.computer/wp-sync-slowtwitch/services/wordpress" "fmt" "slices" "strconv" "strings" ) type MigratePosts struct { SlowtwitchDatabase *sql.DB ResultsDatabase *sql.DB WordpressBaseUrl string WordpressUser string WordpressPassword string } func (migration MigratePosts) Execute() []PostResult { slowtwitchPostIds, err := slowtwitch.GetAllPostIds(migration.SlowtwitchDatabase) if err != nil { panic("Could not migrate posts: " + err.Error()) } migratedPostIds, err := GetAllMigratedPostIds(migration.ResultsDatabase) if err != nil { panic("Could not migrate posts:" + err.Error()) } slowtwitchPostIdsForMigration := getPostIdsThatNeedMigration(slowtwitchPostIds, migratedPostIds) //get wordpress tag data, there are only 3 tags := []string{"swim", "bike", "run"} wpTagData, wpTagErr := wordpress.GetTags(tags, migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword) if wpTagErr != nil { panic("Could not migrate posts due to tags not found:" + wpTagErr.Error()) } var postResults []PostResult batchSize := 5 var postBatch []int for i, postId := range slowtwitchPostIdsForMigration { //anonymous func will take in post IDs: the rest it can access from scope postBatch = append(postBatch, postId) if len(postBatch) == batchSize || i == len(slowtwitchPostIdsForMigration)-1 { postResultsChannel := make(chan PostResult) for _, batchId := range postBatch { go func(id int) { errorMessage := "" //migrate postBase, postBaseErr := slowtwitch.GetPostBase(batchId, migration.SlowtwitchDatabase) if postBaseErr != nil { errorMessage = errorMessage + postBaseErr.Error() failedPostResult := createPostFailureResult(batchId, errorMessage, migration.ResultsDatabase) postResultsChannel <- failedPostResult return } createWordpressPost := wordpress.CreatePost{ Title: postBase.Title, Excerpt: postBase.Description, Status: "publish", } if postBase.DatePublished.Valid { t := postBase.DatePublished.Time timeString := fmt.Sprintf("%d-%02d-%02dT%02d:%02d:%02d", t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second()) createWordpressPost.Date = timeString } else { errorMessage = errorMessage + "Invalid Date Published" failedPostResult := createPostFailureResult(batchId, errorMessage, migration.ResultsDatabase) postResultsChannel <- failedPostResult return } for _, tag := range wpTagData { if postBase.Bike == true && tag.Name == "bike" { createWordpressPost.Tags = append(createWordpressPost.Tags, tag.Id) } if postBase.Swim == true && tag.Name == "swim" { createWordpressPost.Tags = append(createWordpressPost.Tags, tag.Id) } if postBase.Run == true && tag.Name == "run" { createWordpressPost.Tags = append(createWordpressPost.Tags, tag.Id) } } var wordPressCategoryIds []int var firstCategoryResult CategoryResult for _, slowtwitchCategoryId := range postBase.CategoryIds { categoryResult, err := GetSlowtwitchCategoryResult(slowtwitchCategoryId, migration.ResultsDatabase) if err != nil { errorMessage = errorMessage + err.Error() failedPostResult := createPostFailureResult(batchId, errorMessage, migration.ResultsDatabase) postResultsChannel <- failedPostResult return } wordPressCategoryIds = append(wordPressCategoryIds, categoryResult.WordpressId) firstCategoryResult = categoryResult } if len(wordPressCategoryIds) == 0 { errorMessage = "This post has no categories and is broken on the production site." failedPostResult := createPostFailureResult(batchId, errorMessage, migration.ResultsDatabase) postResultsChannel <- failedPostResult return } createWordpressPost.Categories = wordPressCategoryIds //Get Author ID editor, findEditorErr := GetEditor(postBase.Author, migration.ResultsDatabase) if findEditorErr != nil { errorMessage = errorMessage + findEditorErr.Error() failedPostResult := createPostFailureResult(batchId, errorMessage, migration.ResultsDatabase) postResultsChannel <- failedPostResult return } createWordpressPost.Author = editor.WordpressId //Get old link oldLink := strings.ReplaceAll(firstCategoryResult.OldUrl, "index.html", "") + slowtwitch.ConvertPostTitleToPath(postBase.Title, postBase.Id) linkStatus := slowtwitch.GetPageStatus(oldLink) if linkStatus == 404 { errorMessage = errorMessage + "Page not found on Slowtwitch" failedPostResult := createPostFailureResult(batchId, errorMessage, migration.ResultsDatabase) postResultsChannel <- failedPostResult return } //Get page, parse out post data and images //Upload images to wordpress, swap out with new image urls //Submit imagePaths, html, retreiveHtmlErr := slowtwitch.GetImagesAndPostHtml(oldLink) if retreiveHtmlErr != nil { errorMessage = errorMessage + retreiveHtmlErr.Error() failedPostResult := createPostFailureResult(batchId, errorMessage, migration.ResultsDatabase) postResultsChannel <- failedPostResult return } //Create images from the image paths var imageResults []ImageResult for i, imagePath := range imagePaths { //construct URL imageUrl := "https://www.slowtwitch.com" + imagePath createWordpressImage := wordpress.CreateImage{ Url: imageUrl, } //submit image wordpressImage, wordpressImageErr := createWordpressImage.Execute(migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword) if wordpressImageErr != nil { errorMessage = errorMessage + wordpressImageErr.Error() imageFailureResult := ImageResult{ OldUrl: imageUrl, NewUrl: "", WordpressId: 0, IsSuccess: false, } imageResults = append(imageResults, imageFailureResult) continue } //first photo is the featured photo if i == 0 { createWordpressPost.FeaturedMedia = wordpressImage.Id } //begin process of recording result imageResult := ImageResult{ OldUrl: imageUrl, NewUrl: wordpressImage.Link, WordpressId: wordpressImage.Id, IsSuccess: true, } imageResults = append(imageResults, imageResult) //replace old links with new in post html strings.ReplaceAll(html, imageUrl, wordpressImage.Link) //create redirect imageRedirect := wordpress.CreateRedirect{ Title: postBase.Title + "image-" + string((i + 1)), Url: imagePath, MatchType: "page", ActionType: "url", ActionCode: 301, GroupId: 1, ActionData: wordpress.ActionData{ Url: "/" + wordpressImage.Slug, }, } _, imageRedirectErr := imageRedirect.Execute(migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword) if imageRedirectErr != nil { fmt.Println("Failed to create image redirect:", imageUrl, ":", imageRedirectErr.Error()) } } createWordpressPost.Content = html post, createWordpressPostErr := createWordpressPost.Execute(migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword) if createWordpressPostErr != nil { errorMessage = errorMessage + createWordpressPostErr.Error() postFailureResult := createPostFailureResult(batchId, errorMessage, migration.ResultsDatabase) postResultsChannel <- postFailureResult return } //set up post result here to create //truncate error message for db if len(errorMessage) > 1450 { errorMessage = errorMessage[:1450] } postResult := PostResult{ SlowtwitchId: batchId, WordpressId: post.Id, OldUrl: oldLink, OldUrlStatus: linkStatus, NewUrl: post.Link, IsSuccess: true, ErrorMessage: errorMessage, } postResultId, createPostResultErr := CreatePostResult(postResult, migration.ResultsDatabase) if createPostResultErr != nil { fmt.Println("Could not record post result for Slowtwitch post:" + strconv.Itoa(batchId) + createPostResultErr.Error()) } for _, imageResult := range imageResults { imageResult.PostId = postResultId createImageResultErr := CreateImageResult(imageResult, migration.ResultsDatabase) if createImageResultErr != nil { fmt.Println("Error recording image result") } } oldPath := strings.ReplaceAll(oldLink, "https://www.slowtwitch.com", "") postRedirect := wordpress.CreateRedirect{ Title: "Article Redirect" + postBase.Title, Url: oldPath, MatchType: "page", ActionType: "url", ActionCode: 301, GroupId: 1, ActionData: wordpress.ActionData{ Url: "/" + strings.ReplaceAll(postResult.NewUrl, migration.WordpressBaseUrl, ""), }, } _, postRedirectErr := postRedirect.Execute(migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword) if postRedirectErr != nil { fmt.Println("Error creating redirect for", batchId, ":"+postRedirectErr.Error()) } fmt.Println("Successfully created post and result for", batchId) updateAcfImages(imageResults, post.Id, migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword) postResults = append(postResults, postResult) postResultsChannel <- postResult }(batchId) } //Listen to channel and record results var batchResults []PostResult for result := range postResultsChannel { batchResults = append(batchResults, result) if len(batchResults) == batchSize { close(postResultsChannel) } } postBatch = nil postResults = append(postResults, batchResults...) } } // Update related posts once work is done updatePostRelationships(postResults, migration) return postResults } func updatePostRelationships(postResults []PostResult, migration MigratePosts) { fmt.Println("Updating post relationships") for i, postResult := range postResults { fmt.Println("Updating post", i+1, "/", len(postResults)) if postResult.IsSuccess { var relatedWordpressIds []int relatedSlowtwitchIds, slowtwitchIdsErr := slowtwitch.GetRelatedArticleIds(postResult.SlowtwitchId, migration.SlowtwitchDatabase) if slowtwitchIdsErr != nil || len(relatedSlowtwitchIds) == 0 { continue } for _, slowtwitchRelatedId := range relatedSlowtwitchIds { wordpressRelatedId, wordpressIdErr := GetWordpressPostIdBySlowtwitchPostId(slowtwitchRelatedId, migration.ResultsDatabase) if wordpressIdErr != nil { continue } relatedWordpressIds = append(relatedWordpressIds, wordpressRelatedId) } if len(relatedWordpressIds) > 0 { updateWordpressRelatedPosts := wordpress.UpdateAcfRelatedPosts{ Acf: wordpress.AcfRelatedPosts{ PostIds: relatedWordpressIds, }, } updateRelatedPostErr := updateWordpressRelatedPosts.Execute(migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword, postResult.WordpressId) if updateRelatedPostErr != nil { fmt.Println("Error updating wordpressRelatedPosts", updateRelatedPostErr.Error()) } } } } } func getPostIdsThatNeedMigration(slowtwitchPostIds, migratedPostIds []int) []int { var output []int for _, slowtwitchPostId := range slowtwitchPostIds { hasId := slices.Contains(migratedPostIds, slowtwitchPostId) if hasId == false { output = append(output, slowtwitchPostId) } } return output } func createPostFailureResult(slowtwitchId int, errorMessage string, migrationDb *sql.DB) PostResult { fmt.Println("Error creating post: ", slowtwitchId, ":", errorMessage) postResult := PostResult{ SlowtwitchId: slowtwitchId, WordpressId: 0, OldUrl: "", OldUrlStatus: 0, NewUrl: "", IsSuccess: false, ErrorMessage: errorMessage, } _, err := CreatePostResult(postResult, migrationDb) if err != nil { fmt.Println("Failed to create failure result: ", err) } return postResult } func createImageFailureResult(url string, resultsDatabase *sql.DB) { fmt.Println("Error creating image failure result: ", url) imageResult := ImageResult{ OldUrl: url, NewUrl: "", WordpressId: 0, IsSuccess: false, } err := CreateImageResult(imageResult, resultsDatabase) if err != nil { fmt.Println("Failed to create failure result: ", err) } } func getSuccessfulWordpressImageIds(imageResults []ImageResult) []int { var output []int for _, imageResult := range imageResults { if imageResult.IsSuccess { output = append(output, imageResult.WordpressId) } } return output } func updateAcfImages(imageResults []ImageResult, postId int, baseUrl, user, pass string) { if len(imageResults) > 0 { wordpressImageIds := getSuccessfulWordpressImageIds(imageResults) if len(wordpressImageIds) > 0 { updateAcfImages := wordpress.UpdateAcfImages{ Acf: wordpress.AcfImages{ PostImages: wordpressImageIds, }, } acfImagesErr := updateAcfImages.Execute(baseUrl, user, pass, postId) if acfImagesErr != nil { fmt.Println("Error updating acf images for post", postId, ":", acfImagesErr.Error()) } } } }