threaded post migrator
This commit is contained in:
parent
cd8ac17775
commit
90b725bb8a
7
.idea/sqldialects.xml
Normal file
7
.idea/sqldialects.xml
Normal file
@ -0,0 +1,7 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="SqlDialectMappings">
|
||||
<file url="file://$PROJECT_DIR$/main.go" dialect="GenericSQL" />
|
||||
<file url="PROJECT" dialect="MySQL" />
|
||||
</component>
|
||||
</project>
|
||||
6
main.go
6
main.go
@ -7,9 +7,9 @@ import (
|
||||
)
|
||||
|
||||
// WP Config
|
||||
const baseUrl = "https://slowtwitch.cloud/"
|
||||
const wordpressKey = "admin@slowtwitch.cloud"
|
||||
const wordpressSecret = "6zY7 xsKZ dGIt l1Lp ypIK 6TWh"
|
||||
const baseUrl = "http://go-api-playground.local/"
|
||||
const wordpressKey = "admin"
|
||||
const wordpressSecret = "KStj VH7E 9UZf FV8C ptGQ F4Tl"
|
||||
|
||||
// DB Config
|
||||
const slowtwitchAdminUser = "admin"
|
||||
|
||||
@ -37,198 +37,232 @@ func (migration MigratePosts) Execute() []PostResult {
|
||||
}
|
||||
|
||||
var postResults []PostResult
|
||||
batchSize := 5
|
||||
var postBatch []int
|
||||
|
||||
for _, postId := range slowtwitchPostIdsForMigration {
|
||||
errorMessage := ""
|
||||
//migrate
|
||||
postBase, postBaseErr := slowtwitch.GetPostBase(postId, migration.SlowtwitchDatabase)
|
||||
if postBaseErr != nil {
|
||||
errorMessage = errorMessage + postBaseErr.Error()
|
||||
createPostFailureResult(postId, errorMessage, migration.ResultsDatabase)
|
||||
continue
|
||||
}
|
||||
//anonymous func will take in post IDs: the rest it can access from scope
|
||||
|
||||
createWordpressPost := wordpress.CreatePost{
|
||||
Title: postBase.Title,
|
||||
Excerpt: postBase.Description,
|
||||
Status: "publish",
|
||||
}
|
||||
postBatch = append(postBatch, postId)
|
||||
if len(postBatch) == batchSize {
|
||||
postResultsChannel := make(chan PostResult)
|
||||
for _, batchId := range postBatch {
|
||||
go func(id int) {
|
||||
errorMessage := ""
|
||||
//migrate
|
||||
postBase, postBaseErr := slowtwitch.GetPostBase(batchId, migration.SlowtwitchDatabase)
|
||||
|
||||
if postBase.DatePublished.Valid {
|
||||
t := postBase.DatePublished.Time
|
||||
timeString := fmt.Sprintf("%d-%02d-%02dT%02d:%02d:%02d",
|
||||
t.Year(), t.Month(), t.Day(),
|
||||
t.Hour(), t.Minute(), t.Second())
|
||||
createWordpressPost.Date = timeString
|
||||
} else {
|
||||
errorMessage = errorMessage + "Invalid Date Published"
|
||||
createPostFailureResult(postId, errorMessage, migration.ResultsDatabase)
|
||||
continue
|
||||
}
|
||||
if postBaseErr != nil {
|
||||
errorMessage = errorMessage + postBaseErr.Error()
|
||||
failedPostResult := createPostFailureResult(batchId, errorMessage, migration.ResultsDatabase)
|
||||
postResultsChannel <- failedPostResult
|
||||
return
|
||||
}
|
||||
|
||||
for _, tag := range wpTagData {
|
||||
if postBase.Bike == true && tag.Name == "bike" {
|
||||
createWordpressPost.Tags = append(createWordpressPost.Tags, tag.Id)
|
||||
createWordpressPost := wordpress.CreatePost{
|
||||
Title: postBase.Title,
|
||||
Excerpt: postBase.Description,
|
||||
Status: "publish",
|
||||
}
|
||||
|
||||
if postBase.DatePublished.Valid {
|
||||
t := postBase.DatePublished.Time
|
||||
timeString := fmt.Sprintf("%d-%02d-%02dT%02d:%02d:%02d",
|
||||
t.Year(), t.Month(), t.Day(),
|
||||
t.Hour(), t.Minute(), t.Second())
|
||||
createWordpressPost.Date = timeString
|
||||
} else {
|
||||
errorMessage = errorMessage + "Invalid Date Published"
|
||||
failedPostResult := createPostFailureResult(batchId, errorMessage, migration.ResultsDatabase)
|
||||
postResultsChannel <- failedPostResult
|
||||
return
|
||||
}
|
||||
|
||||
for _, tag := range wpTagData {
|
||||
if postBase.Bike == true && tag.Name == "bike" {
|
||||
createWordpressPost.Tags = append(createWordpressPost.Tags, tag.Id)
|
||||
}
|
||||
if postBase.Swim == true && tag.Name == "swim" {
|
||||
createWordpressPost.Tags = append(createWordpressPost.Tags, tag.Id)
|
||||
}
|
||||
if postBase.Run == true && tag.Name == "run" {
|
||||
createWordpressPost.Tags = append(createWordpressPost.Tags, tag.Id)
|
||||
}
|
||||
}
|
||||
|
||||
var wordPressCategoryIds []int
|
||||
var firstCategoryResult CategoryResult
|
||||
for _, slowtwitchCategoryId := range postBase.CategoryIds {
|
||||
categoryResult, err := GetSlowtwitchCategoryResult(slowtwitchCategoryId, migration.ResultsDatabase)
|
||||
if err != nil {
|
||||
errorMessage = errorMessage + err.Error()
|
||||
failedPostResult := createPostFailureResult(batchId, errorMessage, migration.ResultsDatabase)
|
||||
postResultsChannel <- failedPostResult
|
||||
return
|
||||
}
|
||||
wordPressCategoryIds = append(wordPressCategoryIds, categoryResult.WordpressId)
|
||||
firstCategoryResult = categoryResult
|
||||
}
|
||||
|
||||
if len(wordPressCategoryIds) == 0 {
|
||||
errorMessage = "This post has no categories and is broken on the production site."
|
||||
failedPostResult := createPostFailureResult(batchId, errorMessage, migration.ResultsDatabase)
|
||||
postResultsChannel <- failedPostResult
|
||||
return
|
||||
}
|
||||
|
||||
createWordpressPost.Categories = wordPressCategoryIds
|
||||
//Get Author ID
|
||||
editor, findEditorErr := GetEditor(postBase.Author, migration.ResultsDatabase)
|
||||
|
||||
if findEditorErr != nil {
|
||||
errorMessage = errorMessage + findEditorErr.Error()
|
||||
failedPostResult := createPostFailureResult(batchId, errorMessage, migration.ResultsDatabase)
|
||||
postResultsChannel <- failedPostResult
|
||||
return
|
||||
}
|
||||
createWordpressPost.Author = editor.WordpressId
|
||||
//Get old link
|
||||
oldLink := strings.ReplaceAll(firstCategoryResult.OldUrl, "index.html", "") + slowtwitch.ConvertPostTitleToPath(postBase.Title, postBase.Id)
|
||||
linkStatus := slowtwitch.GetPageStatus(oldLink)
|
||||
|
||||
if linkStatus == 404 {
|
||||
errorMessage = errorMessage + "Page not found on Slowtwitch"
|
||||
failedPostResult := createPostFailureResult(batchId, errorMessage, migration.ResultsDatabase)
|
||||
postResultsChannel <- failedPostResult
|
||||
return
|
||||
}
|
||||
|
||||
//Get page, parse out post data and images
|
||||
//Upload images to wordpress, swap out with new image urls
|
||||
//Submit
|
||||
imagePaths, html, retreiveHtmlErr := slowtwitch.GetImagesAndPostHtml(oldLink)
|
||||
if retreiveHtmlErr != nil {
|
||||
errorMessage = errorMessage + retreiveHtmlErr.Error()
|
||||
failedPostResult := createPostFailureResult(batchId, errorMessage, migration.ResultsDatabase)
|
||||
postResultsChannel <- failedPostResult
|
||||
return
|
||||
}
|
||||
//Create images from the image paths
|
||||
var imageResults []ImageResult
|
||||
|
||||
for i, imagePath := range imagePaths {
|
||||
//construct URL
|
||||
imageUrl := "https://www.slowtwitch.com" + imagePath
|
||||
createWordpressImage := wordpress.CreateImage{
|
||||
Url: imageUrl,
|
||||
}
|
||||
//submit image
|
||||
wordpressImage, wordpressImageErr := createWordpressImage.Execute(migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword)
|
||||
|
||||
if wordpressImageErr != nil {
|
||||
errorMessage = errorMessage + wordpressImageErr.Error()
|
||||
createImageFailureResult(imageUrl, migration.ResultsDatabase)
|
||||
continue
|
||||
}
|
||||
//first photo is the featured photo
|
||||
if i == 0 {
|
||||
createWordpressPost.FeaturedMedia = wordpressImage.Id
|
||||
}
|
||||
//begin process of recording result
|
||||
imageResult := ImageResult{
|
||||
OldUrl: imageUrl,
|
||||
NewUrl: wordpressImage.Link,
|
||||
WordpressId: wordpressImage.Id,
|
||||
IsSuccess: true,
|
||||
}
|
||||
imageResults = append(imageResults, imageResult)
|
||||
//replace old links with new in post html
|
||||
strings.ReplaceAll(html, imageUrl, wordpressImage.Link)
|
||||
//create redirect
|
||||
imageRedirect := wordpress.CreateRedirect{
|
||||
Title: postBase.Title + "image-" + string((i + 1)),
|
||||
Url: imagePath,
|
||||
MatchType: "page",
|
||||
ActionType: "url",
|
||||
ActionCode: 301,
|
||||
GroupId: 1,
|
||||
ActionData: wordpress.ActionData{
|
||||
Url: "/" + wordpressImage.Slug,
|
||||
},
|
||||
}
|
||||
|
||||
_, imageRedirectErr := imageRedirect.Execute(migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword)
|
||||
|
||||
if imageRedirectErr != nil {
|
||||
fmt.Println("Failed to create image redirect:", imageUrl, ":", imageRedirectErr.Error())
|
||||
}
|
||||
}
|
||||
|
||||
createWordpressPost.Content = html
|
||||
post, createWordpressPostErr := createWordpressPost.Execute(migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword)
|
||||
if createWordpressPostErr != nil {
|
||||
errorMessage = errorMessage + createWordpressPostErr.Error()
|
||||
postFailureResult := createPostFailureResult(batchId, errorMessage, migration.ResultsDatabase)
|
||||
postResultsChannel <- postFailureResult
|
||||
return
|
||||
}
|
||||
//set up post result here to create
|
||||
//truncate error message for db
|
||||
if len(errorMessage) > 1450 {
|
||||
errorMessage = errorMessage[:1450]
|
||||
}
|
||||
|
||||
postResult := PostResult{
|
||||
SlowtwitchId: batchId,
|
||||
WordpressId: post.Id,
|
||||
OldUrl: oldLink,
|
||||
OldUrlStatus: linkStatus,
|
||||
NewUrl: post.Link,
|
||||
IsSuccess: true,
|
||||
ErrorMessage: errorMessage,
|
||||
}
|
||||
|
||||
postResultId, createPostResultErr := CreatePostResult(postResult, migration.ResultsDatabase)
|
||||
if createPostResultErr != nil {
|
||||
fmt.Println("Could not record post result for Slowtwitch post:" + strconv.Itoa(batchId) + createPostResultErr.Error())
|
||||
}
|
||||
for _, imageResult := range imageResults {
|
||||
imageResult.PostId = postResultId
|
||||
createImageResultErr := CreateImageResult(imageResult, migration.ResultsDatabase)
|
||||
if createImageResultErr != nil {
|
||||
fmt.Println("Error recording image result")
|
||||
}
|
||||
}
|
||||
oldPath := strings.ReplaceAll(oldLink, "https://www.slowtwitch.com", "")
|
||||
postRedirect := wordpress.CreateRedirect{
|
||||
Title: "Article Redirect" + postBase.Title,
|
||||
Url: oldPath,
|
||||
MatchType: "page",
|
||||
ActionType: "url",
|
||||
ActionCode: 301,
|
||||
GroupId: 1,
|
||||
ActionData: wordpress.ActionData{
|
||||
Url: "/" + strings.ReplaceAll(postResult.NewUrl, migration.WordpressBaseUrl, ""),
|
||||
},
|
||||
}
|
||||
_, postRedirectErr := postRedirect.Execute(migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword)
|
||||
if postRedirectErr != nil {
|
||||
fmt.Println("Error creating redirect for", batchId, ":"+postRedirectErr.Error())
|
||||
}
|
||||
fmt.Println("Successfully created post and result for", batchId)
|
||||
updateAcfImages(imageResults, post.Id, migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword)
|
||||
postResults = append(postResults, postResult)
|
||||
postResultsChannel <- postResult
|
||||
}(batchId)
|
||||
}
|
||||
if postBase.Swim == true && tag.Name == "swim" {
|
||||
createWordpressPost.Tags = append(createWordpressPost.Tags, tag.Id)
|
||||
}
|
||||
if postBase.Run == true && tag.Name == "run" {
|
||||
createWordpressPost.Tags = append(createWordpressPost.Tags, tag.Id)
|
||||
}
|
||||
}
|
||||
|
||||
var wordPressCategoryIds []int
|
||||
var firstCategoryResult CategoryResult
|
||||
for _, slowtwitchCategoryId := range postBase.CategoryIds {
|
||||
categoryResult, err := GetSlowtwitchCategoryResult(slowtwitchCategoryId, migration.ResultsDatabase)
|
||||
if err != nil {
|
||||
errorMessage = errorMessage + err.Error()
|
||||
createPostFailureResult(postId, errorMessage, migration.ResultsDatabase)
|
||||
continue
|
||||
}
|
||||
wordPressCategoryIds = append(wordPressCategoryIds, categoryResult.WordpressId)
|
||||
firstCategoryResult = categoryResult
|
||||
}
|
||||
|
||||
if len(wordPressCategoryIds) == 0 {
|
||||
errorMessage = "This post has no categories and is broken on the production site."
|
||||
createPostFailureResult(postId, errorMessage, migration.ResultsDatabase)
|
||||
continue
|
||||
}
|
||||
|
||||
createWordpressPost.Categories = wordPressCategoryIds
|
||||
//Get Author ID
|
||||
editor, findEditorErr := GetEditor(postBase.Author, migration.ResultsDatabase)
|
||||
|
||||
if findEditorErr != nil {
|
||||
errorMessage = errorMessage + findEditorErr.Error()
|
||||
createPostFailureResult(postId, errorMessage, migration.ResultsDatabase)
|
||||
continue
|
||||
}
|
||||
createWordpressPost.Author = editor.WordpressId
|
||||
//Get old link
|
||||
oldLink := strings.ReplaceAll(firstCategoryResult.OldUrl, "index.html", "") + slowtwitch.ConvertPostTitleToPath(postBase.Title, postBase.Id)
|
||||
linkStatus := slowtwitch.GetPageStatus(oldLink)
|
||||
|
||||
if linkStatus == 404 {
|
||||
errorMessage = errorMessage + "Page not found on Slowtwitch"
|
||||
createPostFailureResult(postId, errorMessage, migration.ResultsDatabase)
|
||||
continue
|
||||
}
|
||||
|
||||
//Get page, parse out post data and images
|
||||
//Upload images to wordpress, swap out with new image urls
|
||||
//Submit
|
||||
imagePaths, html, retreiveHtmlErr := slowtwitch.GetImagesAndPostHtml(oldLink)
|
||||
if retreiveHtmlErr != nil {
|
||||
errorMessage = errorMessage + retreiveHtmlErr.Error()
|
||||
createPostFailureResult(postId, errorMessage, migration.ResultsDatabase)
|
||||
continue
|
||||
}
|
||||
//Create images from the image paths
|
||||
var imageResults []ImageResult
|
||||
|
||||
for i, imagePath := range imagePaths {
|
||||
//construct URL
|
||||
imageUrl := "https://www.slowtwitch.com" + imagePath
|
||||
createWordpressImage := wordpress.CreateImage{
|
||||
Url: imageUrl,
|
||||
}
|
||||
//submit image
|
||||
wordpressImage, wordpressImageErr := createWordpressImage.Execute(migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword)
|
||||
|
||||
if wordpressImageErr != nil {
|
||||
errorMessage = errorMessage + wordpressImageErr.Error()
|
||||
createImageFailureResult(imageUrl, migration.ResultsDatabase)
|
||||
continue
|
||||
}
|
||||
//first photo is the featured photo
|
||||
if i == 0 {
|
||||
createWordpressPost.FeaturedMedia = wordpressImage.Id
|
||||
}
|
||||
//begin process of recording result
|
||||
imageResult := ImageResult{
|
||||
OldUrl: imageUrl,
|
||||
NewUrl: wordpressImage.Link,
|
||||
WordpressId: wordpressImage.Id,
|
||||
IsSuccess: true,
|
||||
}
|
||||
imageResults = append(imageResults, imageResult)
|
||||
//replace old links with new in post html
|
||||
strings.ReplaceAll(html, imageUrl, wordpressImage.Link)
|
||||
//create redirect
|
||||
imageRedirect := wordpress.CreateRedirect{
|
||||
Title: postBase.Title + "image-" + string((i + 1)),
|
||||
Url: imagePath,
|
||||
MatchType: "page",
|
||||
ActionType: "url",
|
||||
ActionCode: 301,
|
||||
GroupId: 1,
|
||||
ActionData: wordpress.ActionData{
|
||||
Url: "/" + wordpressImage.Slug,
|
||||
},
|
||||
//Listen to channel and record results
|
||||
var batchResults []PostResult
|
||||
for result := range postResultsChannel {
|
||||
batchResults = append(batchResults, result)
|
||||
if len(batchResults) == batchSize {
|
||||
close(postResultsChannel)
|
||||
}
|
||||
}
|
||||
|
||||
_, imageRedirectErr := imageRedirect.Execute(migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword)
|
||||
|
||||
if imageRedirectErr != nil {
|
||||
fmt.Println("Failed to create image redirect:", imageUrl, ":", imageRedirectErr.Error())
|
||||
}
|
||||
postBatch = nil
|
||||
postResults = append(postResults, batchResults...)
|
||||
}
|
||||
|
||||
createWordpressPost.Content = html
|
||||
post, createWordpressPostErr := createWordpressPost.Execute(migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword)
|
||||
if createWordpressPostErr != nil {
|
||||
errorMessage = errorMessage + createWordpressPostErr.Error()
|
||||
createPostFailureResult(postId, errorMessage, migration.ResultsDatabase)
|
||||
continue
|
||||
}
|
||||
//set up post result here to create
|
||||
//truncate error message for db
|
||||
if len(errorMessage) > 1450 {
|
||||
errorMessage = errorMessage[:1450]
|
||||
}
|
||||
|
||||
postResult := PostResult{
|
||||
SlowtwitchId: postId,
|
||||
WordpressId: post.Id,
|
||||
OldUrl: oldLink,
|
||||
OldUrlStatus: linkStatus,
|
||||
NewUrl: post.Link,
|
||||
IsSuccess: true,
|
||||
ErrorMessage: errorMessage,
|
||||
}
|
||||
|
||||
postResultId, createPostResultErr := CreatePostResult(postResult, migration.ResultsDatabase)
|
||||
if createPostResultErr != nil {
|
||||
fmt.Println("Could not record post result for Slowtwitch post:" + strconv.Itoa(postId) + createPostResultErr.Error())
|
||||
}
|
||||
for _, imageResult := range imageResults {
|
||||
imageResult.PostId = postResultId
|
||||
createImageResultErr := CreateImageResult(imageResult, migration.ResultsDatabase)
|
||||
if createImageResultErr != nil {
|
||||
fmt.Println("Error recording image result")
|
||||
}
|
||||
}
|
||||
oldPath := strings.ReplaceAll(oldLink, "https://www.slowtwitch.com", "")
|
||||
postRedirect := wordpress.CreateRedirect{
|
||||
Title: "Article Redirect" + postBase.Title,
|
||||
Url: oldPath,
|
||||
MatchType: "page",
|
||||
ActionType: "url",
|
||||
ActionCode: 301,
|
||||
GroupId: 1,
|
||||
ActionData: wordpress.ActionData{
|
||||
Url: "/" + strings.ReplaceAll(postResult.NewUrl, migration.WordpressBaseUrl, ""),
|
||||
},
|
||||
}
|
||||
_, postRedirectErr := postRedirect.Execute(migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword)
|
||||
if postRedirectErr != nil {
|
||||
fmt.Println("Error creating redirect for", postId, ":"+postRedirectErr.Error())
|
||||
}
|
||||
fmt.Println("Successfully created post and result for", postId)
|
||||
updateAcfImages(imageResults, post.Id, migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword)
|
||||
postResults = append(postResults, postResult)
|
||||
}
|
||||
// Update related posts once work is done
|
||||
updatePostRelationships(postResults, migration)
|
||||
@ -282,7 +316,7 @@ func getPostIdsThatNeedMigration(slowtwitchPostIds, migratedPostIds []int) []int
|
||||
return output
|
||||
}
|
||||
|
||||
func createPostFailureResult(slowtwitchId int, errorMessage string, migrationDb *sql.DB) {
|
||||
func createPostFailureResult(slowtwitchId int, errorMessage string, migrationDb *sql.DB) PostResult {
|
||||
fmt.Println("Error creating post: ", slowtwitchId, ":", errorMessage)
|
||||
|
||||
postResult := PostResult{
|
||||
@ -300,6 +334,7 @@ func createPostFailureResult(slowtwitchId int, errorMessage string, migrationDb
|
||||
if err != nil {
|
||||
fmt.Println("Failed to create failure result: ", err)
|
||||
}
|
||||
return postResult
|
||||
}
|
||||
|
||||
func createImageFailureResult(url string, resultsDatabase *sql.DB) {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user