ross/Post-Migrator
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: ross:e02a35069a1f9490304a31a5c4661f0994c2ebb2
Branches Tags
ross:master
..
compare: ross:cb6829f175faa669ce64712b432483e3f65aca04
Branches Tags
ross:master

No commits in common. "e02a35069a1f9490304a31a5c4661f0994c2ebb2" and "cb6829f175faa669ce64712b432483e3f65aca04" have entirely different histories.
e02a35069a ... cb6829f175

6 changed files with 42 additions and 322 deletions
Show Stats Expand all files Collapse all files

8
main.go
View File

@ -7,15 +7,15 @@ import (
)
// WP Config
const baseUrl = "https://slowtwitch.cloud/"
const wordpressKey = "admin@slowtwitch.cloud"
const wordpressSecret = "3SrI MysB NREZ 40OD PLAP uk1W"
const baseUrl = "http://go-api-playground.local/"
const wordpressKey = "admin"
const wordpressSecret = "7ZeY ZCwp 3DwL EBAK vGtf NW9J"
// DB Config
const slowtwitchAdminUser = "admin"
const slowtwitchAdminPass = "yxnh93Ybbz2Nm8#mp28zCVv"
const slowtwitchDbName = "slowtwitch"
const migrationDbName = "slowtwitch_transfer"
const migrationDbName = "slowtwitch_transfer_threaded_test"
const federatedDbUrl = "slowtwitch.northend.network"
const federatedDbPort = "3306"

10
services/migration/editor-result.go
View File

@ -18,8 +18,8 @@ func CreateEditorResult(result EditorResult, db *sql.DB) error {
return err
}
func GetEditor(username, email string, db *sql.DB) (EditorResult, error) {
rows, err := db.Query("select WordpressId, Username, Email, (IsSuccess = b'1') from EditorsResults where Username = ? OR Email = ?", username, email)
func GetEditor(username string, db *sql.DB) (EditorResult, error) {
rows, err := db.Query("select WordpressId, Username, Email, (IsSuccess = b'1') from EditorsResults where Username = ?", username)
if err != nil {
fmt.Println(err)
}
@ -35,7 +35,7 @@ func GetEditor(username, email string, db *sql.DB) (EditorResult, error) {
fmt.Println(err)
}
if editor.Username == username || editor.Email == email {
if editor.Username == username {
return editor, nil
}
}
@ -43,8 +43,8 @@ func GetEditor(username, email string, db *sql.DB) (EditorResult, error) {
return EditorResult{}, errors.New("Could not find editor.")
}
func EditorHasBeenMigrated(username, email string, db *sql.DB) bool {
_, err := GetEditor(username, email, db)
func EditorHasBeenMigrated(username string, db *sql.DB) bool {
_, err := GetEditor(username, db)
if err == nil {
return true

2
services/migration/migrate-authors.go
View File

@ -22,7 +22,7 @@ func (migration *MigrateAuthors) Execute() []EditorResult {
var output []EditorResult
for _, editor := range editors {
hasBeenMigrated := EditorHasBeenMigrated(editor.Username, editor.Email, migration.ResultsDatabase)
hasBeenMigrated := EditorHasBeenMigrated(editor.Username, migration.ResultsDatabase)
if hasBeenMigrated == false {
firstName, lastName := getFirstAndLastName(editor.Name)

267
services/migration/migrate-posts.go
View File

@ -8,7 +8,6 @@ import (
"slices"
"strconv"
"strings"
"sync"
)
type MigratePosts struct {
@ -37,7 +36,6 @@ func (migration MigratePosts) Execute() []PostResult {
panic("Could not migrate posts due to tags not found:" + wpTagErr.Error())
}
//TODO Re-thread using a work input channel, work output channel, and waitgroup
var postResults []PostResult
batchSize := 5
var postBatch []int
@ -116,20 +114,15 @@ func (migration MigratePosts) Execute() []PostResult {
createWordpressPost.Categories = wordPressCategoryIds
//Get Author ID
editor, findEditorErr := GetEditor(postBase.Author, postBase.AuthorEmail, migration.ResultsDatabase)
editor, findEditorErr := GetEditor(postBase.Author, migration.ResultsDatabase)
if findEditorErr != nil {
editor, catchAllEditorErr := GetEditor("admin", "slowman2@slowtwitch.com", migration.ResultsDatabase)
if catchAllEditorErr != nil {
errorMessage = errorMessage + findEditorErr.Error()
failedPostResult := createPostFailureResult(batchId, errorMessage, migration.ResultsDatabase)
postResultsChannel <- failedPostResult
return
}
createWordpressPost.Author = editor.WordpressId
}
createWordpressPost.Author = editor.WordpressId
//Get old link
oldLink := strings.ReplaceAll(firstCategoryResult.OldUrl, "index.html", "") + slowtwitch.ConvertPostTitleToPath(postBase.Title, postBase.Id)
linkStatus := slowtwitch.GetPageStatus(oldLink)
@ -279,275 +272,33 @@ func (migration MigratePosts) Execute() []PostResult {
postResults = append(postResults, batchResults...)
}
}
// Update related posts once work is done -- Do for all posts?
allPostResults := GetAllPostResults(migration.ResultsDatabase)
updatePostRelationships(allPostResults, migration)
// Update related posts once work is done -- Why are there duplicates?
updatePostRelationships(postResults, migration)
return postResults
}
func createPost(postId int, migration MigratePosts, wpTagData []wordpress.TagData) PostResult {
errorMessage := ""
//migrate
postBase, postBaseErr := slowtwitch.GetPostBase(postId, migration.SlowtwitchDatabase)
if postBaseErr != nil {
errorMessage = errorMessage + postBaseErr.Error()
failedPostResult := createPostFailureResult(postId, errorMessage, migration.ResultsDatabase)
return failedPostResult
}
createWordpressPost := wordpress.CreatePost{
Title: postBase.Title,
Excerpt: postBase.Description,
Status: "publish",
}
if postBase.DatePublished.Valid {
t := postBase.DatePublished.Time
timeString := fmt.Sprintf("%d-%02d-%02dT%02d:%02d:%02d",
t.Year(), t.Month(), t.Day(),
t.Hour(), t.Minute(), t.Second())
createWordpressPost.Date = timeString
} else {
errorMessage = errorMessage + "Invalid Date Published"
failedPostResult := createPostFailureResult(postId, errorMessage, migration.ResultsDatabase)
return failedPostResult
}
for _, tag := range wpTagData {
if postBase.Bike == true && tag.Name == "bike" {
createWordpressPost.Tags = append(createWordpressPost.Tags, tag.Id)
}
if postBase.Swim == true && tag.Name == "swim" {
createWordpressPost.Tags = append(createWordpressPost.Tags, tag.Id)
}
if postBase.Run == true && tag.Name == "run" {
createWordpressPost.Tags = append(createWordpressPost.Tags, tag.Id)
}
}
var wordPressCategoryIds []int
var firstCategoryResult CategoryResult
for _, slowtwitchCategoryId := range postBase.CategoryIds {
categoryResult, err := GetSlowtwitchCategoryResult(slowtwitchCategoryId, migration.ResultsDatabase)
if err != nil {
errorMessage = errorMessage + err.Error()
failedPostResult := createPostFailureResult(postId, errorMessage, migration.ResultsDatabase)
return failedPostResult
}
wordPressCategoryIds = append(wordPressCategoryIds, categoryResult.WordpressId)
firstCategoryResult = categoryResult
}
if len(wordPressCategoryIds) == 0 {
errorMessage = "This post has no categories and is broken on the production site."
failedPostResult := createPostFailureResult(postId, errorMessage, migration.ResultsDatabase)
return failedPostResult
}
createWordpressPost.Categories = wordPressCategoryIds
//Get Author ID
editor, findEditorErr := GetEditor(postBase.Author, postBase.AuthorEmail, migration.ResultsDatabase)
if findEditorErr != nil {
editor, catchAllEditorErr := GetEditor("admin", "slowman2@slowtwitch.com", migration.ResultsDatabase)
if catchAllEditorErr != nil {
errorMessage = errorMessage + findEditorErr.Error()
failedPostResult := createPostFailureResult(postId, errorMessage, migration.ResultsDatabase)
return failedPostResult
}
createWordpressPost.Author = editor.WordpressId
}
createWordpressPost.Author = editor.WordpressId
//Get old link
oldLink := strings.ReplaceAll(firstCategoryResult.OldUrl, "index.html", "") + slowtwitch.ConvertPostTitleToPath(postBase.Title, postBase.Id)
linkStatus := slowtwitch.GetPageStatus(oldLink)
if linkStatus == 404 {
errorMessage = errorMessage + "Page not found on Slowtwitch"
failedPostResult := createPostFailureResult(postId, errorMessage, migration.ResultsDatabase)
return failedPostResult
}
//Get page, parse out post data and images
//Upload images to wordpress, swap out with new image urls
//Submit
imagePaths, html, retreiveHtmlErr := slowtwitch.GetImagesAndPostHtml(oldLink)
if retreiveHtmlErr != nil {
errorMessage = errorMessage + retreiveHtmlErr.Error()
failedPostResult := createPostFailureResult(postId, errorMessage, migration.ResultsDatabase)
return failedPostResult
}
//Create images from the image paths
var imageResults []ImageResult
for i, imagePath := range imagePaths {
//construct URL
imageUrl := "https://www.slowtwitch.com" + imagePath
createWordpressImage := wordpress.CreateImage{
Url: imageUrl,
}
//submit image
wordpressImage, wordpressImageErr := createWordpressImage.Execute(migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword)
if wordpressImageErr != nil {
errorMessage = errorMessage + wordpressImageErr.Error()
imageFailureResult := ImageResult{
OldUrl: imageUrl,
NewUrl: "",
WordpressId: 0,
IsSuccess: false,
}
imageResults = append(imageResults, imageFailureResult)
continue
}
//first photo is the featured photo
if i == 0 {
createWordpressPost.FeaturedMedia = wordpressImage.Id
}
//begin process of recording result
imageResult := ImageResult{
OldUrl: imageUrl,
NewUrl: wordpressImage.Link,
WordpressId: wordpressImage.Id,
IsSuccess: true,
}
imageResults = append(imageResults, imageResult)
//replace old links with new in post html
newImagePath := "/wp-content/uploads/" + wordpressImage.MediaDetails.File
html = strings.ReplaceAll(html, imagePath, newImagePath)
//create redirect
imageRedirect := wordpress.CreateRedirect{
Title: postBase.Title + "image-" + string((i + 1)),
Url: imagePath,
MatchType: "page",
ActionType: "url",
ActionCode: 301,
GroupId: 1,
ActionData: wordpress.ActionData{
Url: "/" + wordpressImage.Slug,
},
}
_, imageRedirectErr := imageRedirect.Execute(migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword)
if imageRedirectErr != nil {
fmt.Println("Failed to create image redirect:", imageUrl, ":", imageRedirectErr.Error())
}
}
createWordpressPost.Content = html
post, createWordpressPostErr := createWordpressPost.Execute(migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword)
if createWordpressPostErr != nil {
errorMessage = errorMessage + createWordpressPostErr.Error()
postFailureResult := createPostFailureResult(postId, errorMessage, migration.ResultsDatabase)
return postFailureResult
}
//set up post result here to create
//truncate error message for db
if len(errorMessage) > 1450 {
errorMessage = errorMessage[:1450]
}
postResult := PostResult{
SlowtwitchId: postId,
WordpressId: post.Id,
OldUrl: oldLink,
OldUrlStatus: linkStatus,
NewUrl: post.Link,
IsSuccess: true,
ErrorMessage: errorMessage,
}
postResultId, createPostResultErr := CreatePostResult(postResult, migration.ResultsDatabase)
if createPostResultErr != nil {
fmt.Println("Could not record post result for Slowtwitch post:" + strconv.Itoa(postId) + createPostResultErr.Error())
}
for _, imageResult := range imageResults {
imageResult.PostId = postResultId
createImageResultErr := CreateImageResult(imageResult, migration.ResultsDatabase)
if createImageResultErr != nil {
fmt.Println("Error recording image result")
}
}
oldPath := strings.ReplaceAll(oldLink, "https://www.slowtwitch.com", "")
postRedirect := wordpress.CreateRedirect{
Title: "Article Redirect" + postBase.Title,
Url: oldPath,
MatchType: "page",
ActionType: "url",
ActionCode: 301,
GroupId: 1,
ActionData: wordpress.ActionData{
Url: "/" + strings.ReplaceAll(postResult.NewUrl, migration.WordpressBaseUrl, ""),
},
}
_, postRedirectErr := postRedirect.Execute(migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword)
if postRedirectErr != nil {
fmt.Println("Error creating redirect for", postId, ":"+postRedirectErr.Error())
}
fmt.Println("Successfully created post and result for", postId)
updateAcfImages(imageResults, post.Id, migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword)
return postResult
}
func updatePostRelationships(postResults []PostResult, migration MigratePosts) {
fmt.Println("Updating post relationships")
batchSize := 5
postResultWorkInput := make(chan PostResult)
//Set up a wait group to wait for the input channel to close after execution
var wg sync.WaitGroup
wg.Add(1)
wg.Add(len(postResults))
//Launch go routines that will read from the input channel,
//maxed out at the batch size
for i := 0; i < batchSize; i++ {
go func() {
for result := range postResultWorkInput {
updatePostRelationship(result, migration)
wg.Done()
}
}()
}
//Launch an unbuffered channel that will fill the channel with work
//as the workers take work from the channel. This will be blocked while
//any task is waiting to be picked up from the channel
go func() {
for _, result := range postResults {
postResultWorkInput <- result
}
close(postResultWorkInput)
wg.Done()
}()
wg.Wait()
fmt.Println("Done with post relationships")
}
func updatePostRelationship(postResult PostResult, migration MigratePosts) bool {
for i, postResult := range postResults {
fmt.Println("Updating post", i+1, "/", len(postResults))
if postResult.IsSuccess {
var relatedWordpressIds []int
relatedSlowtwitchIds, slowtwitchIdsErr := slowtwitch.GetRelatedArticleIds(postResult.SlowtwitchId, migration.SlowtwitchDatabase)
if slowtwitchIdsErr != nil || len(relatedSlowtwitchIds) == 0 {
return false
continue
}
for _, slowtwitchRelatedId := range relatedSlowtwitchIds {
wordpressRelatedId, wordpressIdErr := GetWordpressPostIdBySlowtwitchPostId(slowtwitchRelatedId, migration.ResultsDatabase)
if wordpressIdErr != nil {
return false
continue
}
relatedWordpressIds = append(relatedWordpressIds, wordpressRelatedId)
}
if len(relatedWordpressIds) > 0 {
fmt.Println("Updating post in WP", postResult.WordpressId)
updateWordpressRelatedPosts := wordpress.UpdateAcfRelatedPosts{
Acf: wordpress.AcfRelatedPosts{
PostIds: relatedWordpressIds,
@ -556,12 +307,10 @@ func updatePostRelationship(postResult PostResult, migration MigratePosts) bool
updateRelatedPostErr := updateWordpressRelatedPosts.Execute(migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword, postResult.WordpressId)
if updateRelatedPostErr != nil {
fmt.Println("Error updating wordpressRelatedPosts", updateRelatedPostErr.Error())
} else {
return true
}
}
}
return false
}
}
func getPostIdsThatNeedMigration(slowtwitchPostIds, migratedPostIds []int) []int {

27
services/migration/post-result.go
View File

@ -1,9 +1,6 @@
package migration
import (
"database/sql"
"log"
)
import "database/sql"
type PostResult struct {
WordpressId int
@ -15,28 +12,6 @@ type PostResult struct {
ErrorMessage string
}
func GetAllPostResults(db *sql.DB) []PostResult {
var results []PostResult
rows, err := db.Query("select WordpressId, SlowtwitchId, OldUrl, OldUrlStatus, NewUrl, (IsSuccess = b'1'), ErrorMessage from PostResults")
if err != nil {
log.Fatal(err)
}
defer rows.Close()
for rows.Next() {
result := PostResult{}
err = rows.Scan(&result.WordpressId, &result.SlowtwitchId, &result.OldUrl, &result.OldUrlStatus, &result.NewUrl, &result.IsSuccess, &result.ErrorMessage)
if err != nil {
log.Fatal(err)
}
results = append(results, result)
}
return results
}
func CreatePostResult(parameters PostResult, db *sql.DB) (int, error) {
result, err := db.Exec("insert into PostResults (WordpressId, SlowtwitchId, OldUrl, OldUrlStatus, NewUrl, IsSuccess, ErrorMessage) values (?, ?, ?, ?, ?, ?, ?)", parameters.WordpressId, parameters.SlowtwitchId, parameters.OldUrl, parameters.OldUrlStatus, parameters.NewUrl, parameters.IsSuccess, parameters.ErrorMessage)
if err != nil {

4
services/slowtwitch/url-converter.go
View File

@ -28,9 +28,5 @@ func convert(path string) string {
output = strings.ReplaceAll(output, "?", "")
output = strings.ReplaceAll(output, ",", "")
output = strings.ReplaceAll(output, "%", "%25")
output = strings.ReplaceAll(output, "_&", "")
output = strings.ReplaceAll(output, "&", "")
output = strings.ReplaceAll(output, "#", "")
output = strings.ReplaceAll(output, ";", "")
return output
}