ross/Post-Migrator
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

threaded post migrator

Browse Source
This commit is contained in:
Ross Trottier 2024-05-25 09:31:59 -06:00
parent cd8ac17775
commit 90b725bb8a
3 changed files with 230 additions and 188 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
.idea/sqldialects.xml Normal file
View File

@ -0,0 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="SqlDialectMappings">
<file url="file://$PROJECT_DIR$/main.go" dialect="GenericSQL" />
<file url="PROJECT" dialect="MySQL" />
</component>
</project>

6
main.go
View File

@ -7,9 +7,9 @@ import (
)
// WP Config
const baseUrl = "https://slowtwitch.cloud/"
const wordpressKey = "admin@slowtwitch.cloud"
const wordpressSecret = "6zY7 xsKZ dGIt l1Lp ypIK 6TWh"
const baseUrl = "http://go-api-playground.local/"
const wordpressKey = "admin"
const wordpressSecret = "KStj VH7E 9UZf FV8C ptGQ F4Tl"
// DB Config
const slowtwitchAdminUser = "admin"

79
services/migration/migrate-posts.go
View File

@ -37,14 +37,26 @@ func (migration MigratePosts) Execute() []PostResult {
}
var postResults []PostResult
batchSize := 5
var postBatch []int
for _, postId := range slowtwitchPostIdsForMigration {
//anonymous func will take in post IDs: the rest it can access from scope
postBatch = append(postBatch, postId)
if len(postBatch) == batchSize {
postResultsChannel := make(chan PostResult)
for _, batchId := range postBatch {
go func(id int) {
errorMessage := ""
//migrate
postBase, postBaseErr := slowtwitch.GetPostBase(postId, migration.SlowtwitchDatabase)
postBase, postBaseErr := slowtwitch.GetPostBase(batchId, migration.SlowtwitchDatabase)
if postBaseErr != nil {
errorMessage = errorMessage + postBaseErr.Error()
createPostFailureResult(postId, errorMessage, migration.ResultsDatabase)
continue
failedPostResult := createPostFailureResult(batchId, errorMessage, migration.ResultsDatabase)
postResultsChannel <- failedPostResult
return
}
createWordpressPost := wordpress.CreatePost{
@ -61,8 +73,9 @@ func (migration MigratePosts) Execute() []PostResult {
createWordpressPost.Date = timeString
} else {
errorMessage = errorMessage + "Invalid Date Published"
createPostFailureResult(postId, errorMessage, migration.ResultsDatabase)
continue
failedPostResult := createPostFailureResult(batchId, errorMessage, migration.ResultsDatabase)
postResultsChannel <- failedPostResult
return
}
for _, tag := range wpTagData {
@ -83,8 +96,9 @@ func (migration MigratePosts) Execute() []PostResult {
categoryResult, err := GetSlowtwitchCategoryResult(slowtwitchCategoryId, migration.ResultsDatabase)
if err != nil {
errorMessage = errorMessage + err.Error()
createPostFailureResult(postId, errorMessage, migration.ResultsDatabase)
continue
failedPostResult := createPostFailureResult(batchId, errorMessage, migration.ResultsDatabase)
postResultsChannel <- failedPostResult
return
}
wordPressCategoryIds = append(wordPressCategoryIds, categoryResult.WordpressId)
firstCategoryResult = categoryResult
@ -92,8 +106,9 @@ func (migration MigratePosts) Execute() []PostResult {
if len(wordPressCategoryIds) == 0 {
errorMessage = "This post has no categories and is broken on the production site."
createPostFailureResult(postId, errorMessage, migration.ResultsDatabase)
continue
failedPostResult := createPostFailureResult(batchId, errorMessage, migration.ResultsDatabase)
postResultsChannel <- failedPostResult
return
}
createWordpressPost.Categories = wordPressCategoryIds
@ -102,8 +117,9 @@ func (migration MigratePosts) Execute() []PostResult {
if findEditorErr != nil {
errorMessage = errorMessage + findEditorErr.Error()
createPostFailureResult(postId, errorMessage, migration.ResultsDatabase)
continue
failedPostResult := createPostFailureResult(batchId, errorMessage, migration.ResultsDatabase)
postResultsChannel <- failedPostResult
return
}
createWordpressPost.Author = editor.WordpressId
//Get old link
@ -112,8 +128,9 @@ func (migration MigratePosts) Execute() []PostResult {
if linkStatus == 404 {
errorMessage = errorMessage + "Page not found on Slowtwitch"
createPostFailureResult(postId, errorMessage, migration.ResultsDatabase)
continue
failedPostResult := createPostFailureResult(batchId, errorMessage, migration.ResultsDatabase)
postResultsChannel <- failedPostResult
return
}
//Get page, parse out post data and images
@ -122,8 +139,9 @@ func (migration MigratePosts) Execute() []PostResult {
imagePaths, html, retreiveHtmlErr := slowtwitch.GetImagesAndPostHtml(oldLink)
if retreiveHtmlErr != nil {
errorMessage = errorMessage + retreiveHtmlErr.Error()
createPostFailureResult(postId, errorMessage, migration.ResultsDatabase)
continue
failedPostResult := createPostFailureResult(batchId, errorMessage, migration.ResultsDatabase)
postResultsChannel <- failedPostResult
return
}
//Create images from the image paths
var imageResults []ImageResult
@ -180,8 +198,9 @@ func (migration MigratePosts) Execute() []PostResult {
post, createWordpressPostErr := createWordpressPost.Execute(migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword)
if createWordpressPostErr != nil {
errorMessage = errorMessage + createWordpressPostErr.Error()
createPostFailureResult(postId, errorMessage, migration.ResultsDatabase)
continue
postFailureResult := createPostFailureResult(batchId, errorMessage, migration.ResultsDatabase)
postResultsChannel <- postFailureResult
return
}
//set up post result here to create
//truncate error message for db
@ -190,7 +209,7 @@ func (migration MigratePosts) Execute() []PostResult {
}
postResult := PostResult{
SlowtwitchId: postId,
SlowtwitchId: batchId,
WordpressId: post.Id,
OldUrl: oldLink,
OldUrlStatus: linkStatus,
@ -201,7 +220,7 @@ func (migration MigratePosts) Execute() []PostResult {
postResultId, createPostResultErr := CreatePostResult(postResult, migration.ResultsDatabase)
if createPostResultErr != nil {
fmt.Println("Could not record post result for Slowtwitch post:" + strconv.Itoa(postId) + createPostResultErr.Error())
fmt.Println("Could not record post result for Slowtwitch post:" + strconv.Itoa(batchId) + createPostResultErr.Error())
}
for _, imageResult := range imageResults {
imageResult.PostId = postResultId
@ -224,11 +243,26 @@ func (migration MigratePosts) Execute() []PostResult {
}
_, postRedirectErr := postRedirect.Execute(migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword)
if postRedirectErr != nil {
fmt.Println("Error creating redirect for", postId, ":"+postRedirectErr.Error())
fmt.Println("Error creating redirect for", batchId, ":"+postRedirectErr.Error())
}
fmt.Println("Successfully created post and result for", postId)
fmt.Println("Successfully created post and result for", batchId)
updateAcfImages(imageResults, post.Id, migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword)
postResults = append(postResults, postResult)
postResultsChannel <- postResult
}(batchId)
}
//Listen to channel and record results
var batchResults []PostResult
for result := range postResultsChannel {
batchResults = append(batchResults, result)
if len(batchResults) == batchSize {
close(postResultsChannel)
}
}
postBatch = nil
postResults = append(postResults, batchResults...)
}
}
// Update related posts once work is done
updatePostRelationships(postResults, migration)
@ -282,7 +316,7 @@ func getPostIdsThatNeedMigration(slowtwitchPostIds, migratedPostIds []int) []int
return output
}
func createPostFailureResult(slowtwitchId int, errorMessage string, migrationDb *sql.DB) {
func createPostFailureResult(slowtwitchId int, errorMessage string, migrationDb *sql.DB) PostResult {
fmt.Println("Error creating post: ", slowtwitchId, ":", errorMessage)
postResult := PostResult{
@ -300,6 +334,7 @@ func createPostFailureResult(slowtwitchId int, errorMessage string, migrationDb
if err != nil {
fmt.Println("Failed to create failure result: ", err)
}
return postResult
}
func createImageFailureResult(url string, resultsDatabase *sql.DB) {