package migration import ( "database/sql" "federated.computer/wp-sync-slowtwitch/services/slowtwitch" "federated.computer/wp-sync-slowtwitch/services/wordpress" "fmt" "slices" "strconv" "strings" "sync" ) type MigratePosts struct { SlowtwitchDatabase *sql.DB ResultsDatabase *sql.DB WordpressBaseUrl string WordpressUser string WordpressPassword string } func (migration MigratePosts) Execute() []PostResult { slowtwitchPostIds, err := slowtwitch.GetAllPostIds(migration.SlowtwitchDatabase) if err != nil { panic("Could not migrate posts: " + err.Error()) } migratedPostIds, err := GetAllMigratedPostIds(migration.ResultsDatabase) if err != nil { panic("Could not migrate posts:" + err.Error()) } slowtwitchPostIdsForMigration := getPostIdsThatNeedMigration(slowtwitchPostIds, migratedPostIds) //get wordpress tag data, there are only 3 tags := []string{"swim", "bike", "run"} wpTagData, wpTagErr := wordpress.GetTags(tags, migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword) if wpTagErr != nil { panic("Could not migrate posts due to tags not found:" + wpTagErr.Error()) } var postResults []PostResult batchSize := 5 workInputChannel := make(chan int) workOutputChannel := make(chan PostResult) doneChannel := make(chan bool) defer close(doneChannel) var wg sync.WaitGroup wg.Add(len(slowtwitchPostIdsForMigration)) for i := 0; i < batchSize; i++ { go func() { for { select { case <-doneChannel: return case postId, ok := <-workInputChannel: if ok { result := createPost(postId, migration, wpTagData) workOutputChannel <- result } else { return } } } }() } go func() { defer close(workInputChannel) for _, postId := range slowtwitchPostIdsForMigration { select { case <-doneChannel: return case workInputChannel <- postId: } } }() go func() { defer close(workOutputChannel) for { select { case <-doneChannel: return case postResult, ok := <-workOutputChannel: if ok { postResults = append(postResults, postResult) wg.Done() } else { return } } } }() wg.Wait() doneChannel <- true updatePostRelationships(postResults, migration) return postResults } func createPost(postId int, migration MigratePosts, wpTagData []wordpress.TagData) PostResult { fmt.Println("Creating post " + strconv.Itoa(postId)) errorMessage := "" //migrate postBase, postBaseErr := slowtwitch.GetPostBase(postId, migration.SlowtwitchDatabase) if postBaseErr != nil { errorMessage = errorMessage + postBaseErr.Error() failedPostResult := createPostFailureResult(postId, errorMessage, migration.ResultsDatabase) return failedPostResult } createWordpressPost := wordpress.CreatePost{ Title: postBase.Title, Excerpt: postBase.Description, Status: "publish", } fmt.Println("Checking date: " + strconv.Itoa(postId)) if postBase.DatePublished.Valid { t := postBase.DatePublished.Time timeString := fmt.Sprintf("%d-%02d-%02dT%02d:%02d:%02d", t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second()) createWordpressPost.Date = timeString } else { errorMessage = errorMessage + "Invalid Date Published" failedPostResult := createPostFailureResult(postId, errorMessage, migration.ResultsDatabase) return failedPostResult } fmt.Println("Checking tags: " + strconv.Itoa(postId)) for _, tag := range wpTagData { if postBase.Bike == true && tag.Name == "bike" { createWordpressPost.Tags = append(createWordpressPost.Tags, tag.Id) } if postBase.Swim == true && tag.Name == "swim" { createWordpressPost.Tags = append(createWordpressPost.Tags, tag.Id) } if postBase.Run == true && tag.Name == "run" { createWordpressPost.Tags = append(createWordpressPost.Tags, tag.Id) } } fmt.Println("Checking categories: " + strconv.Itoa(postId)) var wordPressCategoryIds []int var firstCategoryResult CategoryResult for _, slowtwitchCategoryId := range postBase.CategoryIds { categoryResult, err := GetSlowtwitchCategoryResult(slowtwitchCategoryId, migration.ResultsDatabase) if err != nil { errorMessage = errorMessage + err.Error() failedPostResult := createPostFailureResult(postId, errorMessage, migration.ResultsDatabase) return failedPostResult } wordPressCategoryIds = append(wordPressCategoryIds, categoryResult.WordpressId) firstCategoryResult = categoryResult } if len(wordPressCategoryIds) == 0 { errorMessage = "This post has no categories and is broken on the production site." failedPostResult := createPostFailureResult(postId, errorMessage, migration.ResultsDatabase) return failedPostResult } createWordpressPost.Categories = wordPressCategoryIds //Get Author ID editor, findEditorErr := GetEditor(postBase.Author, postBase.AuthorEmail, migration.ResultsDatabase) fmt.Println("Finding editor: " + strconv.Itoa(postId)) if findEditorErr != nil { editor, catchAllEditorErr := GetEditor("admin", "slowman2@slowtwitch.com", migration.ResultsDatabase) if catchAllEditorErr != nil { errorMessage = errorMessage + findEditorErr.Error() failedPostResult := createPostFailureResult(postId, errorMessage, migration.ResultsDatabase) return failedPostResult } createWordpressPost.Author = editor.WordpressId } createWordpressPost.Author = editor.WordpressId //Get old link oldLink := strings.ReplaceAll(firstCategoryResult.OldUrl, "index.html", "") + slowtwitch.ConvertPostTitleToPath(postBase.Title, postBase.Id) linkStatus := slowtwitch.GetPageStatus(oldLink) fmt.Println("Checking link: " + strconv.Itoa(postId)) if linkStatus == 404 { errorMessage = errorMessage + "Page not found on Slowtwitch" failedPostResult := createPostFailureResult(postId, errorMessage, migration.ResultsDatabase) return failedPostResult } //Get page, parse out post data and images //Upload images to wordpress, swap out with new image urls //Submit fmt.Println("Checking images: " + strconv.Itoa(postId)) imagePaths, html, retreiveHtmlErr := slowtwitch.GetImagesAndPostHtml(oldLink) if retreiveHtmlErr != nil { errorMessage = errorMessage + retreiveHtmlErr.Error() failedPostResult := createPostFailureResult(postId, errorMessage, migration.ResultsDatabase) return failedPostResult } //Create images from the image paths var imageResults []ImageResult for i, imagePath := range imagePaths { //construct URL imageUrl := "https://www.slowtwitch.com" + imagePath createWordpressImage := wordpress.CreateImage{ Url: imageUrl, } //submit image wordpressImage, wordpressImageErr := createWordpressImage.Execute(migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword) if wordpressImageErr != nil { errorMessage = errorMessage + wordpressImageErr.Error() imageFailureResult := ImageResult{ OldUrl: imageUrl, NewUrl: "", WordpressId: 0, IsSuccess: false, } imageResults = append(imageResults, imageFailureResult) continue } //first photo is the featured photo if i == 0 { createWordpressPost.FeaturedMedia = wordpressImage.Id } //begin process of recording result imageResult := ImageResult{ OldUrl: imageUrl, NewUrl: wordpressImage.Link, WordpressId: wordpressImage.Id, IsSuccess: true, } imageResults = append(imageResults, imageResult) //replace old links with new in post html newImagePath := "/wp-content/uploads/" + wordpressImage.MediaDetails.File html = strings.ReplaceAll(html, imagePath, newImagePath) //create redirect imageRedirect := wordpress.CreateRedirect{ Title: postBase.Title + "image-" + string((i + 1)), Url: imagePath, MatchType: "page", ActionType: "url", ActionCode: 301, GroupId: 1, ActionData: wordpress.ActionData{ Url: "/" + wordpressImage.Slug, }, } _, imageRedirectErr := imageRedirect.Execute(migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword) if imageRedirectErr != nil { fmt.Println("Failed to create image redirect:", imageUrl, ":", imageRedirectErr.Error()) } } fmt.Println("Checking html: " + strconv.Itoa(postId)) createWordpressPost.Content = html post, createWordpressPostErr := createWordpressPost.Execute(migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword) if createWordpressPostErr != nil { errorMessage = errorMessage + createWordpressPostErr.Error() postFailureResult := createPostFailureResult(postId, errorMessage, migration.ResultsDatabase) return postFailureResult } //set up post result here to create //truncate error message for db if len(errorMessage) > 1450 { errorMessage = errorMessage[:1450] } postResult := PostResult{ SlowtwitchId: postId, WordpressId: post.Id, OldUrl: oldLink, OldUrlStatus: linkStatus, NewUrl: post.Link, IsSuccess: true, ErrorMessage: errorMessage, } fmt.Println("Recording result: " + strconv.Itoa(postId)) postResultId, createPostResultErr := CreatePostResult(postResult, migration.ResultsDatabase) if createPostResultErr != nil { fmt.Println("Could not record post result for Slowtwitch post:" + strconv.Itoa(postId) + createPostResultErr.Error()) } for _, imageResult := range imageResults { imageResult.PostId = postResultId createImageResultErr := CreateImageResult(imageResult, migration.ResultsDatabase) if createImageResultErr != nil { fmt.Println("Error recording image result") } } oldPath := strings.ReplaceAll(oldLink, "https://www.slowtwitch.com", "") postRedirect := wordpress.CreateRedirect{ Title: "Article Redirect" + postBase.Title, Url: oldPath, MatchType: "page", ActionType: "url", ActionCode: 301, GroupId: 1, ActionData: wordpress.ActionData{ Url: "/" + strings.ReplaceAll(postResult.NewUrl, migration.WordpressBaseUrl, ""), }, } _, postRedirectErr := postRedirect.Execute(migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword) if postRedirectErr != nil { fmt.Println("Error creating redirect for", postId, ":"+postRedirectErr.Error()) } fmt.Println("Successfully created post and result for", postId) updateAcfImages(imageResults, post.Id, migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword) return postResult } func updatePostRelationships(postResults []PostResult, migration MigratePosts) { fmt.Println("Updating post relationships") totalPostResults := len(postResults) batchSize := 5 postResultWorkInput := make(chan PostResult) doneChannel := make(chan bool) defer close(doneChannel) //Set up a wait group to wait for the input channel to close after execution var wg sync.WaitGroup wg.Add(totalPostResults + 1) //Launch go routines that will read from the input channel, //maxed out at the batch size for i := 0; i < batchSize; i++ { go func() { for { select { case <-doneChannel: return case result, ok := <-postResultWorkInput: if ok { updatePostRelationship(result, migration) wg.Done() } else { return } } } }() } //Launch an unbuffered channel that will fill the channel with work //as the workers take work from the channel. This will be blocked while //any task is waiting to be picked up from the channel go func() { //defer close(postResultWorkInput) for _, result := range postResults { select { case <-doneChannel: return case postResultWorkInput <- result: } } wg.Done() }() wg.Wait() doneChannel <- true fmt.Println("Done with post relationships") } func updatePostRelationship(postResult PostResult, migration MigratePosts) bool { if postResult.IsSuccess { var relatedWordpressIds []int relatedSlowtwitchIds, slowtwitchIdsErr := slowtwitch.GetRelatedArticleIds(postResult.SlowtwitchId, migration.SlowtwitchDatabase) if slowtwitchIdsErr != nil || len(relatedSlowtwitchIds) == 0 { return false } for _, slowtwitchRelatedId := range relatedSlowtwitchIds { wordpressRelatedId, wordpressIdErr := GetWordpressPostIdBySlowtwitchPostId(slowtwitchRelatedId, migration.ResultsDatabase) if wordpressIdErr != nil { return false } relatedWordpressIds = append(relatedWordpressIds, wordpressRelatedId) } if len(relatedWordpressIds) > 0 { fmt.Println("Updating post in WP", postResult.WordpressId) updateWordpressRelatedPosts := wordpress.UpdateAcfRelatedPosts{ Acf: wordpress.AcfRelatedPosts{ PostIds: relatedWordpressIds, }, } updateRelatedPostErr := updateWordpressRelatedPosts.Execute(migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword, postResult.WordpressId) if updateRelatedPostErr != nil { fmt.Println("Error updating wordpressRelatedPosts", updateRelatedPostErr.Error()) } else { return true } } } return false } func getPostIdsThatNeedMigration(slowtwitchPostIds, migratedPostIds []int) []int { var output []int for _, slowtwitchPostId := range slowtwitchPostIds { hasId := slices.Contains(migratedPostIds, slowtwitchPostId) if hasId == false { output = append(output, slowtwitchPostId) } } return output } func createPostFailureResult(slowtwitchId int, errorMessage string, migrationDb *sql.DB) PostResult { fmt.Println("Error creating post: ", slowtwitchId, ":", errorMessage) postResult := PostResult{ SlowtwitchId: slowtwitchId, WordpressId: 0, OldUrl: "", OldUrlStatus: 0, NewUrl: "", IsSuccess: false, ErrorMessage: errorMessage, } _, err := CreatePostResult(postResult, migrationDb) if err != nil { fmt.Println("Failed to create failure result: ", err) } return postResult } func createImageFailureResult(url string, resultsDatabase *sql.DB) { fmt.Println("Error creating image failure result: ", url) imageResult := ImageResult{ OldUrl: url, NewUrl: "", WordpressId: 0, IsSuccess: false, } err := CreateImageResult(imageResult, resultsDatabase) if err != nil { fmt.Println("Failed to create failure result: ", err) } } func getSuccessfulWordpressImageIds(imageResults []ImageResult) []int { var output []int for _, imageResult := range imageResults { if imageResult.WordpressId > 0 && imageResult.IsSuccess { output = append(output, imageResult.WordpressId) } } return output } func updateAcfImages(imageResults []ImageResult, postId int, baseUrl, user, pass string) { if len(imageResults) > 0 { wordpressImageIds := getSuccessfulWordpressImageIds(imageResults) if len(wordpressImageIds) > 0 { updateAcfImages := wordpress.UpdateAcfImages{ Acf: wordpress.AcfImages{ PostImages: wordpressImageIds, }, } acfImagesErr := updateAcfImages.Execute(baseUrl, user, pass, postId) if acfImagesErr != nil { fmt.Println("Error updating acf images for post", postId, ":", acfImagesErr.Error()) } } } }