ross/Post-Migrator
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

threaded post migrator

Browse Source
This commit is contained in:
Ross Trottier 2024-05-25 09:31:59 -06:00
parent cd8ac17775
commit 90b725bb8a
3 changed files with 230 additions and 188 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
.idea/sqldialects.xml Normal file
View File

@ -0,0 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="SqlDialectMappings">
<file url="file://$PROJECT_DIR$/main.go" dialect="GenericSQL" />
<file url="PROJECT" dialect="MySQL" />
</component>
</project>

6
main.go
View File

@ -7,9 +7,9 @@ import (
) )
// WP Config // WP Config
const baseUrl = "https://slowtwitch.cloud/" const baseUrl = "http://go-api-playground.local/"
const wordpressKey = "admin@slowtwitch.cloud" const wordpressKey = "admin"
const wordpressSecret = "6zY7 xsKZ dGIt l1Lp ypIK 6TWh" const wordpressSecret = "KStj VH7E 9UZf FV8C ptGQ F4Tl"
// DB Config // DB Config
const slowtwitchAdminUser = "admin" const slowtwitchAdminUser = "admin"

79
services/migration/migrate-posts.go
View File

@ -37,14 +37,26 @@ func (migration MigratePosts) Execute() []PostResult {
} }
var postResults []PostResult var postResults []PostResult
batchSize := 5
var postBatch []int
for _, postId := range slowtwitchPostIdsForMigration { for _, postId := range slowtwitchPostIdsForMigration {
//anonymous func will take in post IDs: the rest it can access from scope
postBatch = append(postBatch, postId)
if len(postBatch) == batchSize {
postResultsChannel := make(chan PostResult)
for _, batchId := range postBatch {
go func(id int) {
errorMessage := "" errorMessage := ""
//migrate //migrate
postBase, postBaseErr := slowtwitch.GetPostBase(postId, migration.SlowtwitchDatabase) postBase, postBaseErr := slowtwitch.GetPostBase(batchId, migration.SlowtwitchDatabase)
if postBaseErr != nil { if postBaseErr != nil {
errorMessage = errorMessage + postBaseErr.Error() errorMessage = errorMessage + postBaseErr.Error()
createPostFailureResult(postId, errorMessage, migration.ResultsDatabase) failedPostResult := createPostFailureResult(batchId, errorMessage, migration.ResultsDatabase)
continue postResultsChannel <- failedPostResult
return
} }
createWordpressPost := wordpress.CreatePost{ createWordpressPost := wordpress.CreatePost{
@ -61,8 +73,9 @@ func (migration MigratePosts) Execute() []PostResult {
createWordpressPost.Date = timeString createWordpressPost.Date = timeString
} else { } else {
errorMessage = errorMessage + "Invalid Date Published" errorMessage = errorMessage + "Invalid Date Published"
createPostFailureResult(postId, errorMessage, migration.ResultsDatabase) failedPostResult := createPostFailureResult(batchId, errorMessage, migration.ResultsDatabase)
continue postResultsChannel <- failedPostResult
return
} }
for _, tag := range wpTagData { for _, tag := range wpTagData {
@ -83,8 +96,9 @@ func (migration MigratePosts) Execute() []PostResult {
categoryResult, err := GetSlowtwitchCategoryResult(slowtwitchCategoryId, migration.ResultsDatabase) categoryResult, err := GetSlowtwitchCategoryResult(slowtwitchCategoryId, migration.ResultsDatabase)
if err != nil { if err != nil {
errorMessage = errorMessage + err.Error() errorMessage = errorMessage + err.Error()
createPostFailureResult(postId, errorMessage, migration.ResultsDatabase) failedPostResult := createPostFailureResult(batchId, errorMessage, migration.ResultsDatabase)
continue postResultsChannel <- failedPostResult
return
} }
wordPressCategoryIds = append(wordPressCategoryIds, categoryResult.WordpressId) wordPressCategoryIds = append(wordPressCategoryIds, categoryResult.WordpressId)
firstCategoryResult = categoryResult firstCategoryResult = categoryResult
@ -92,8 +106,9 @@ func (migration MigratePosts) Execute() []PostResult {
if len(wordPressCategoryIds) == 0 { if len(wordPressCategoryIds) == 0 {
errorMessage = "This post has no categories and is broken on the production site." errorMessage = "This post has no categories and is broken on the production site."
createPostFailureResult(postId, errorMessage, migration.ResultsDatabase) failedPostResult := createPostFailureResult(batchId, errorMessage, migration.ResultsDatabase)
continue postResultsChannel <- failedPostResult
return
} }
createWordpressPost.Categories = wordPressCategoryIds createWordpressPost.Categories = wordPressCategoryIds
@ -102,8 +117,9 @@ func (migration MigratePosts) Execute() []PostResult {
if findEditorErr != nil { if findEditorErr != nil {
errorMessage = errorMessage + findEditorErr.Error() errorMessage = errorMessage + findEditorErr.Error()
createPostFailureResult(postId, errorMessage, migration.ResultsDatabase) failedPostResult := createPostFailureResult(batchId, errorMessage, migration.ResultsDatabase)
continue postResultsChannel <- failedPostResult
return
} }
createWordpressPost.Author = editor.WordpressId createWordpressPost.Author = editor.WordpressId
//Get old link //Get old link
@ -112,8 +128,9 @@ func (migration MigratePosts) Execute() []PostResult {
if linkStatus == 404 { if linkStatus == 404 {
errorMessage = errorMessage + "Page not found on Slowtwitch" errorMessage = errorMessage + "Page not found on Slowtwitch"
createPostFailureResult(postId, errorMessage, migration.ResultsDatabase) failedPostResult := createPostFailureResult(batchId, errorMessage, migration.ResultsDatabase)
continue postResultsChannel <- failedPostResult
return
} }
//Get page, parse out post data and images //Get page, parse out post data and images
@ -122,8 +139,9 @@ func (migration MigratePosts) Execute() []PostResult {
imagePaths, html, retreiveHtmlErr := slowtwitch.GetImagesAndPostHtml(oldLink) imagePaths, html, retreiveHtmlErr := slowtwitch.GetImagesAndPostHtml(oldLink)
if retreiveHtmlErr != nil { if retreiveHtmlErr != nil {
errorMessage = errorMessage + retreiveHtmlErr.Error() errorMessage = errorMessage + retreiveHtmlErr.Error()
createPostFailureResult(postId, errorMessage, migration.ResultsDatabase) failedPostResult := createPostFailureResult(batchId, errorMessage, migration.ResultsDatabase)
continue postResultsChannel <- failedPostResult
return
} }
//Create images from the image paths //Create images from the image paths
var imageResults []ImageResult var imageResults []ImageResult
@ -180,8 +198,9 @@ func (migration MigratePosts) Execute() []PostResult {
post, createWordpressPostErr := createWordpressPost.Execute(migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword) post, createWordpressPostErr := createWordpressPost.Execute(migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword)
if createWordpressPostErr != nil { if createWordpressPostErr != nil {
errorMessage = errorMessage + createWordpressPostErr.Error() errorMessage = errorMessage + createWordpressPostErr.Error()
createPostFailureResult(postId, errorMessage, migration.ResultsDatabase) postFailureResult := createPostFailureResult(batchId, errorMessage, migration.ResultsDatabase)
continue postResultsChannel <- postFailureResult
return
} }
//set up post result here to create //set up post result here to create
//truncate error message for db //truncate error message for db
@ -190,7 +209,7 @@ func (migration MigratePosts) Execute() []PostResult {
} }
postResult := PostResult{ postResult := PostResult{
SlowtwitchId: postId, SlowtwitchId: batchId,
WordpressId: post.Id, WordpressId: post.Id,
OldUrl: oldLink, OldUrl: oldLink,
OldUrlStatus: linkStatus, OldUrlStatus: linkStatus,
@ -201,7 +220,7 @@ func (migration MigratePosts) Execute() []PostResult {
postResultId, createPostResultErr := CreatePostResult(postResult, migration.ResultsDatabase) postResultId, createPostResultErr := CreatePostResult(postResult, migration.ResultsDatabase)
if createPostResultErr != nil { if createPostResultErr != nil {
fmt.Println("Could not record post result for Slowtwitch post:" + strconv.Itoa(postId) + createPostResultErr.Error()) fmt.Println("Could not record post result for Slowtwitch post:" + strconv.Itoa(batchId) + createPostResultErr.Error())
} }
for _, imageResult := range imageResults { for _, imageResult := range imageResults {
imageResult.PostId = postResultId imageResult.PostId = postResultId
@ -224,11 +243,26 @@ func (migration MigratePosts) Execute() []PostResult {
} }
_, postRedirectErr := postRedirect.Execute(migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword) _, postRedirectErr := postRedirect.Execute(migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword)
if postRedirectErr != nil { if postRedirectErr != nil {
fmt.Println("Error creating redirect for", postId, ":"+postRedirectErr.Error()) fmt.Println("Error creating redirect for", batchId, ":"+postRedirectErr.Error())
} }
fmt.Println("Successfully created post and result for", postId) fmt.Println("Successfully created post and result for", batchId)
updateAcfImages(imageResults, post.Id, migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword) updateAcfImages(imageResults, post.Id, migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword)
postResults = append(postResults, postResult) postResults = append(postResults, postResult)
postResultsChannel <- postResult
}(batchId)
}
//Listen to channel and record results
var batchResults []PostResult
for result := range postResultsChannel {
batchResults = append(batchResults, result)
if len(batchResults) == batchSize {
close(postResultsChannel)
}
}
postBatch = nil
postResults = append(postResults, batchResults...)
}
} }
// Update related posts once work is done // Update related posts once work is done
updatePostRelationships(postResults, migration) updatePostRelationships(postResults, migration)
@ -282,7 +316,7 @@ func getPostIdsThatNeedMigration(slowtwitchPostIds, migratedPostIds []int) []int
return output return output
} }
func createPostFailureResult(slowtwitchId int, errorMessage string, migrationDb *sql.DB) { func createPostFailureResult(slowtwitchId int, errorMessage string, migrationDb *sql.DB) PostResult {
fmt.Println("Error creating post: ", slowtwitchId, ":", errorMessage) fmt.Println("Error creating post: ", slowtwitchId, ":", errorMessage)
postResult := PostResult{ postResult := PostResult{
@ -300,6 +334,7 @@ func createPostFailureResult(slowtwitchId int, errorMessage string, migrationDb
if err != nil { if err != nil {
fmt.Println("Failed to create failure result: ", err) fmt.Println("Failed to create failure result: ", err)
} }
return postResult
} }
func createImageFailureResult(url string, resultsDatabase *sql.DB) { func createImageFailureResult(url string, resultsDatabase *sql.DB) {