439 lines
15 KiB
Go
439 lines
15 KiB
Go
package migration
|
|
|
|
import (
|
|
"database/sql"
|
|
"federated.computer/wp-sync-slowtwitch/services/slowtwitch"
|
|
"federated.computer/wp-sync-slowtwitch/services/wordpress"
|
|
"fmt"
|
|
"slices"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
)
|
|
|
|
type MigratePosts struct {
|
|
SlowtwitchDatabase *sql.DB
|
|
ResultsDatabase *sql.DB
|
|
WordpressBaseUrl string
|
|
WordpressUser string
|
|
WordpressPassword string
|
|
}
|
|
|
|
func (migration MigratePosts) Execute() []PostResult {
|
|
slowtwitchPostIds, err := slowtwitch.GetAllPostIds(migration.SlowtwitchDatabase)
|
|
if err != nil {
|
|
panic("Could not migrate posts: " + err.Error())
|
|
}
|
|
migratedPostIds, err := GetAllMigratedPostIds(migration.ResultsDatabase)
|
|
if err != nil {
|
|
panic("Could not migrate posts:" + err.Error())
|
|
}
|
|
slowtwitchPostIdsForMigration := getPostIdsThatNeedMigration(slowtwitchPostIds, migratedPostIds)
|
|
|
|
//get wordpress tag data, there are only 3
|
|
tags := []string{"swim", "bike", "run"}
|
|
wpTagData, wpTagErr := wordpress.GetTags(tags, migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword)
|
|
if wpTagErr != nil {
|
|
panic("Could not migrate posts due to tags not found:" + wpTagErr.Error())
|
|
}
|
|
|
|
var postResults []PostResult
|
|
batchSize := 5
|
|
var postBatch []int
|
|
|
|
for i, postId := range slowtwitchPostIdsForMigration {
|
|
//anonymous func will take in post IDs: the rest it can access from scope
|
|
postBatch = append(postBatch, postId)
|
|
|
|
if len(postBatch) == batchSize || i == len(slowtwitchPostIdsForMigration)-1 {
|
|
postResultsChannel := make(chan PostResult)
|
|
|
|
for _, batchId := range postBatch {
|
|
go func(id int) {
|
|
errorMessage := ""
|
|
//migrate
|
|
postBase, postBaseErr := slowtwitch.GetPostBase(batchId, migration.SlowtwitchDatabase)
|
|
|
|
if postBaseErr != nil {
|
|
errorMessage = errorMessage + postBaseErr.Error()
|
|
failedPostResult := createPostFailureResult(batchId, errorMessage, migration.ResultsDatabase)
|
|
postResultsChannel <- failedPostResult
|
|
return
|
|
}
|
|
|
|
createWordpressPost := wordpress.CreatePost{
|
|
Title: postBase.Title,
|
|
Excerpt: postBase.Description,
|
|
Status: "publish",
|
|
}
|
|
|
|
if postBase.DatePublished.Valid {
|
|
t := postBase.DatePublished.Time
|
|
timeString := fmt.Sprintf("%d-%02d-%02dT%02d:%02d:%02d",
|
|
t.Year(), t.Month(), t.Day(),
|
|
t.Hour(), t.Minute(), t.Second())
|
|
createWordpressPost.Date = timeString
|
|
} else {
|
|
errorMessage = errorMessage + "Invalid Date Published"
|
|
failedPostResult := createPostFailureResult(batchId, errorMessage, migration.ResultsDatabase)
|
|
postResultsChannel <- failedPostResult
|
|
return
|
|
}
|
|
|
|
for _, tag := range wpTagData {
|
|
if postBase.Bike == true && tag.Name == "bike" {
|
|
createWordpressPost.Tags = append(createWordpressPost.Tags, tag.Id)
|
|
}
|
|
if postBase.Swim == true && tag.Name == "swim" {
|
|
createWordpressPost.Tags = append(createWordpressPost.Tags, tag.Id)
|
|
}
|
|
if postBase.Run == true && tag.Name == "run" {
|
|
createWordpressPost.Tags = append(createWordpressPost.Tags, tag.Id)
|
|
}
|
|
}
|
|
|
|
var wordPressCategoryIds []int
|
|
var firstCategoryResult CategoryResult
|
|
for _, slowtwitchCategoryId := range postBase.CategoryIds {
|
|
categoryResult, err := GetSlowtwitchCategoryResult(slowtwitchCategoryId, migration.ResultsDatabase)
|
|
if err != nil {
|
|
errorMessage = errorMessage + err.Error()
|
|
failedPostResult := createPostFailureResult(batchId, errorMessage, migration.ResultsDatabase)
|
|
postResultsChannel <- failedPostResult
|
|
return
|
|
}
|
|
wordPressCategoryIds = append(wordPressCategoryIds, categoryResult.WordpressId)
|
|
firstCategoryResult = categoryResult
|
|
}
|
|
|
|
if len(wordPressCategoryIds) == 0 {
|
|
errorMessage = "This post has no categories and is broken on the production site."
|
|
failedPostResult := createPostFailureResult(batchId, errorMessage, migration.ResultsDatabase)
|
|
postResultsChannel <- failedPostResult
|
|
return
|
|
}
|
|
|
|
createWordpressPost.Categories = wordPressCategoryIds
|
|
//Get Author ID
|
|
editor, findEditorErr := GetEditor(postBase.Author, postBase.AuthorEmail, migration.ResultsDatabase)
|
|
|
|
if findEditorErr != nil {
|
|
editor, catchAllEditorErr := GetEditor("admin", "slowman2@slowtwitch.com", migration.ResultsDatabase)
|
|
|
|
if catchAllEditorErr != nil {
|
|
errorMessage = errorMessage + findEditorErr.Error()
|
|
failedPostResult := createPostFailureResult(batchId, errorMessage, migration.ResultsDatabase)
|
|
postResultsChannel <- failedPostResult
|
|
return
|
|
}
|
|
createWordpressPost.Author = editor.WordpressId
|
|
}
|
|
createWordpressPost.Author = editor.WordpressId
|
|
//Get old link
|
|
oldLink := strings.ReplaceAll(firstCategoryResult.OldUrl, "index.html", "") + slowtwitch.ConvertPostTitleToPath(postBase.Title, postBase.Id)
|
|
linkStatus := slowtwitch.GetPageStatus(oldLink)
|
|
|
|
if linkStatus == 404 {
|
|
errorMessage = errorMessage + "Page not found on Slowtwitch"
|
|
failedPostResult := createPostFailureResult(batchId, errorMessage, migration.ResultsDatabase)
|
|
postResultsChannel <- failedPostResult
|
|
return
|
|
}
|
|
|
|
//Get page, parse out post data and images
|
|
//Upload images to wordpress, swap out with new image urls
|
|
//Submit
|
|
imagePaths, html, retreiveHtmlErr := slowtwitch.GetImagesAndPostHtml(oldLink)
|
|
if retreiveHtmlErr != nil {
|
|
errorMessage = errorMessage + retreiveHtmlErr.Error()
|
|
failedPostResult := createPostFailureResult(batchId, errorMessage, migration.ResultsDatabase)
|
|
postResultsChannel <- failedPostResult
|
|
return
|
|
}
|
|
//Create images from the image paths
|
|
var imageResults []ImageResult
|
|
|
|
for i, imagePath := range imagePaths {
|
|
//construct URL
|
|
imageUrl := "https://www.slowtwitch.com" + imagePath
|
|
createWordpressImage := wordpress.CreateImage{
|
|
Url: imageUrl,
|
|
}
|
|
//submit image
|
|
wordpressImage, wordpressImageErr := createWordpressImage.Execute(migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword)
|
|
|
|
if wordpressImageErr != nil {
|
|
errorMessage = errorMessage + wordpressImageErr.Error()
|
|
imageFailureResult := ImageResult{
|
|
OldUrl: imageUrl,
|
|
NewUrl: "",
|
|
WordpressId: 0,
|
|
IsSuccess: false,
|
|
}
|
|
imageResults = append(imageResults, imageFailureResult)
|
|
continue
|
|
}
|
|
//first photo is the featured photo
|
|
if i == 0 {
|
|
createWordpressPost.FeaturedMedia = wordpressImage.Id
|
|
}
|
|
//begin process of recording result
|
|
imageResult := ImageResult{
|
|
OldUrl: imageUrl,
|
|
NewUrl: wordpressImage.Link,
|
|
WordpressId: wordpressImage.Id,
|
|
IsSuccess: true,
|
|
}
|
|
imageResults = append(imageResults, imageResult)
|
|
//replace old links with new in post html
|
|
newImagePath := "/wp-content/uploads/" + wordpressImage.MediaDetails.File
|
|
html = strings.ReplaceAll(html, imagePath, newImagePath)
|
|
//create redirect
|
|
imageRedirect := wordpress.CreateRedirect{
|
|
Title: postBase.Title + "image-" + string((i + 1)),
|
|
Url: imagePath,
|
|
MatchType: "page",
|
|
ActionType: "url",
|
|
ActionCode: 301,
|
|
GroupId: 1,
|
|
ActionData: wordpress.ActionData{
|
|
Url: "/" + wordpressImage.Slug,
|
|
},
|
|
}
|
|
|
|
_, imageRedirectErr := imageRedirect.Execute(migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword)
|
|
|
|
if imageRedirectErr != nil {
|
|
fmt.Println("Failed to create image redirect:", imageUrl, ":", imageRedirectErr.Error())
|
|
}
|
|
}
|
|
|
|
createWordpressPost.Content = html
|
|
post, createWordpressPostErr := createWordpressPost.Execute(migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword)
|
|
if createWordpressPostErr != nil {
|
|
errorMessage = errorMessage + createWordpressPostErr.Error()
|
|
postFailureResult := createPostFailureResult(batchId, errorMessage, migration.ResultsDatabase)
|
|
postResultsChannel <- postFailureResult
|
|
return
|
|
}
|
|
//set up post result here to create
|
|
//truncate error message for db
|
|
if len(errorMessage) > 1450 {
|
|
errorMessage = errorMessage[:1450]
|
|
}
|
|
|
|
postResult := PostResult{
|
|
SlowtwitchId: batchId,
|
|
WordpressId: post.Id,
|
|
OldUrl: oldLink,
|
|
OldUrlStatus: linkStatus,
|
|
NewUrl: post.Link,
|
|
IsSuccess: true,
|
|
ErrorMessage: errorMessage,
|
|
}
|
|
|
|
postResultId, createPostResultErr := CreatePostResult(postResult, migration.ResultsDatabase)
|
|
if createPostResultErr != nil {
|
|
fmt.Println("Could not record post result for Slowtwitch post:" + strconv.Itoa(batchId) + createPostResultErr.Error())
|
|
}
|
|
for _, imageResult := range imageResults {
|
|
imageResult.PostId = postResultId
|
|
createImageResultErr := CreateImageResult(imageResult, migration.ResultsDatabase)
|
|
if createImageResultErr != nil {
|
|
fmt.Println("Error recording image result")
|
|
}
|
|
}
|
|
oldPath := strings.ReplaceAll(oldLink, "https://www.slowtwitch.com", "")
|
|
postRedirect := wordpress.CreateRedirect{
|
|
Title: "Article Redirect" + postBase.Title,
|
|
Url: oldPath,
|
|
MatchType: "page",
|
|
ActionType: "url",
|
|
ActionCode: 301,
|
|
GroupId: 1,
|
|
ActionData: wordpress.ActionData{
|
|
Url: "/" + strings.ReplaceAll(postResult.NewUrl, migration.WordpressBaseUrl, ""),
|
|
},
|
|
}
|
|
_, postRedirectErr := postRedirect.Execute(migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword)
|
|
if postRedirectErr != nil {
|
|
fmt.Println("Error creating redirect for", batchId, ":"+postRedirectErr.Error())
|
|
}
|
|
fmt.Println("Successfully created post and result for", batchId)
|
|
updateAcfImages(imageResults, post.Id, migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword)
|
|
postResults = append(postResults, postResult)
|
|
postResultsChannel <- postResult
|
|
}(batchId)
|
|
}
|
|
//Listen to channel and record results
|
|
var batchResults []PostResult
|
|
for result := range postResultsChannel {
|
|
batchResults = append(batchResults, result)
|
|
if len(batchResults) == batchSize {
|
|
close(postResultsChannel)
|
|
}
|
|
}
|
|
|
|
postBatch = nil
|
|
postResults = append(postResults, batchResults...)
|
|
}
|
|
}
|
|
// Update related posts once work is done -- Do for all posts?
|
|
allPostResults := GetAllPostResults(migration.ResultsDatabase)
|
|
updatePostRelationships(allPostResults, migration)
|
|
|
|
return postResults
|
|
}
|
|
|
|
func updatePostRelationships(postResults []PostResult, migration MigratePosts) {
|
|
fmt.Println("Updating post relationships")
|
|
batchSize := 5
|
|
postResultWorkInput := make(chan PostResult)
|
|
//Set up a wait group to wait for the input channel to close after execution
|
|
var wg sync.WaitGroup
|
|
wg.Add(1)
|
|
wg.Add(len(postResults))
|
|
|
|
//Launch go routines that will read from the input channel,
|
|
//maxed out at the batch size
|
|
for i := 0; i < batchSize; i++ {
|
|
go func() {
|
|
for result := range postResultWorkInput {
|
|
updatePostRelationship(result, migration)
|
|
wg.Done()
|
|
}
|
|
}()
|
|
}
|
|
|
|
//Launch an unbuffered channel that will fill the channel with work
|
|
//as the workers take work from the channel. This will be blocked while
|
|
//any task is waiting to be picked up from the channel
|
|
go func() {
|
|
for _, result := range postResults {
|
|
postResultWorkInput <- result
|
|
}
|
|
|
|
close(postResultWorkInput)
|
|
wg.Done()
|
|
}()
|
|
|
|
wg.Wait()
|
|
fmt.Println("Done with post relationships")
|
|
}
|
|
|
|
func updatePostRelationship(postResult PostResult, migration MigratePosts) bool {
|
|
if postResult.IsSuccess {
|
|
var relatedWordpressIds []int
|
|
|
|
relatedSlowtwitchIds, slowtwitchIdsErr := slowtwitch.GetRelatedArticleIds(postResult.SlowtwitchId, migration.SlowtwitchDatabase)
|
|
|
|
if slowtwitchIdsErr != nil || len(relatedSlowtwitchIds) == 0 {
|
|
return false
|
|
}
|
|
for _, slowtwitchRelatedId := range relatedSlowtwitchIds {
|
|
wordpressRelatedId, wordpressIdErr := GetWordpressPostIdBySlowtwitchPostId(slowtwitchRelatedId, migration.ResultsDatabase)
|
|
if wordpressIdErr != nil {
|
|
return false
|
|
}
|
|
relatedWordpressIds = append(relatedWordpressIds, wordpressRelatedId)
|
|
}
|
|
|
|
if len(relatedWordpressIds) > 0 {
|
|
fmt.Println("Updating post in WP", postResult.WordpressId)
|
|
updateWordpressRelatedPosts := wordpress.UpdateAcfRelatedPosts{
|
|
Acf: wordpress.AcfRelatedPosts{
|
|
PostIds: relatedWordpressIds,
|
|
},
|
|
}
|
|
updateRelatedPostErr := updateWordpressRelatedPosts.Execute(migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword, postResult.WordpressId)
|
|
if updateRelatedPostErr != nil {
|
|
fmt.Println("Error updating wordpressRelatedPosts", updateRelatedPostErr.Error())
|
|
} else {
|
|
return true
|
|
}
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func getPostIdsThatNeedMigration(slowtwitchPostIds, migratedPostIds []int) []int {
|
|
var output []int
|
|
|
|
for _, slowtwitchPostId := range slowtwitchPostIds {
|
|
hasId := slices.Contains(migratedPostIds, slowtwitchPostId)
|
|
if hasId == false {
|
|
output = append(output, slowtwitchPostId)
|
|
}
|
|
}
|
|
|
|
return output
|
|
}
|
|
|
|
func createPostFailureResult(slowtwitchId int, errorMessage string, migrationDb *sql.DB) PostResult {
|
|
fmt.Println("Error creating post: ", slowtwitchId, ":", errorMessage)
|
|
|
|
postResult := PostResult{
|
|
SlowtwitchId: slowtwitchId,
|
|
WordpressId: 0,
|
|
OldUrl: "",
|
|
OldUrlStatus: 0,
|
|
NewUrl: "",
|
|
IsSuccess: false,
|
|
ErrorMessage: errorMessage,
|
|
}
|
|
|
|
_, err := CreatePostResult(postResult, migrationDb)
|
|
|
|
if err != nil {
|
|
fmt.Println("Failed to create failure result: ", err)
|
|
}
|
|
return postResult
|
|
}
|
|
|
|
func createImageFailureResult(url string, resultsDatabase *sql.DB) {
|
|
fmt.Println("Error creating image failure result: ", url)
|
|
|
|
imageResult := ImageResult{
|
|
OldUrl: url,
|
|
NewUrl: "",
|
|
WordpressId: 0,
|
|
IsSuccess: false,
|
|
}
|
|
|
|
err := CreateImageResult(imageResult, resultsDatabase)
|
|
if err != nil {
|
|
fmt.Println("Failed to create failure result: ", err)
|
|
}
|
|
}
|
|
|
|
func getSuccessfulWordpressImageIds(imageResults []ImageResult) []int {
|
|
var output []int
|
|
|
|
for _, imageResult := range imageResults {
|
|
if imageResult.WordpressId > 0 && imageResult.IsSuccess {
|
|
output = append(output, imageResult.WordpressId)
|
|
}
|
|
}
|
|
|
|
return output
|
|
}
|
|
|
|
func updateAcfImages(imageResults []ImageResult, postId int, baseUrl, user, pass string) {
|
|
if len(imageResults) > 0 {
|
|
wordpressImageIds := getSuccessfulWordpressImageIds(imageResults)
|
|
|
|
if len(wordpressImageIds) > 0 {
|
|
updateAcfImages := wordpress.UpdateAcfImages{
|
|
Acf: wordpress.AcfImages{
|
|
PostImages: wordpressImageIds,
|
|
},
|
|
}
|
|
acfImagesErr := updateAcfImages.Execute(baseUrl, user, pass, postId)
|
|
|
|
if acfImagesErr != nil {
|
|
fmt.Println("Error updating acf images for post", postId, ":", acfImagesErr.Error())
|
|
}
|
|
}
|
|
}
|
|
}
|