ross/Post-Migrator
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

checkpoint

Browse Source
This commit is contained in:
Ross Trottier 2024-05-19 15:00:33 -06:00
parent 74dfcc4266
commit 683e3a6a87
8 changed files with 258 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

82
main.go
View File

@ -1,15 +1,10 @@
package main
import (
"database/sql"
"federated.computer/wp-sync-slowtwitch/services/migration"
"federated.computer/wp-sync-slowtwitch/services/slowtwitch"
"fmt"
"github.com/PuerkitoBio/goquery"
"golang.org/x/net/html"
"io"
"log"
"net/http"
"strings"
)
const baseUrl = "https://slowtwitch.cloud/"
@ -24,73 +19,34 @@ const federatedDbUrl = "slowtwitch.northend.network"
const federatedDbPort = "3306"
var appCache AppCache
var slowtwitchDB *sql.DB
var resultsDB *sql.DB
func main() {
// TODO Article migration
slowtwitchDB, slowtwitchDbErr := migration.Connect(slowtwitchAdminUser, slowtwitchAdminPass, federatedDbUrl, federatedDbPort, slowtwitchDbName+"?parseTime=true")
//Connect to databases
slowtwitchDatabase, slowtwitchDbErr := migration.Connect(slowtwitchAdminUser, slowtwitchAdminPass, federatedDbUrl, federatedDbPort, slowtwitchDbName+"?parseTime=true")
if slowtwitchDbErr != nil {
fmt.Println(slowtwitchDbErr)
panic("Could not connect to slowtwitch database.")
} else {
slowtwitchDB = slowtwitchDatabase
}
resultsDB, resultsDBerr := migration.Connect(slowtwitchAdminUser, slowtwitchAdminPass, federatedDbUrl, federatedDbPort, migrationDbName)
resultsDatabase, resultsDBerr := migration.Connect(slowtwitchAdminUser, slowtwitchAdminPass, federatedDbUrl, federatedDbPort, migrationDbName)
if resultsDBerr != nil {
fmt.Println(resultsDBerr)
panic("Could not connect to results database.")
} else {
resultsDB = resultsDatabase
}
//EXPERIMENT START
res, err := http.Get("https://www.slowtwitch.com/Products/Components/SRAM_Drops_New_RED_AXS_Groupset_8950.html")
if err != nil {
log.Fatalf("http.Get -> %v", err)
}
defer res.Body.Close()
// Read the HTML content
htmlContent, err := io.ReadAll(res.Body)
if err != nil {
log.Fatalf("ioutil.ReadAll -> %v", err)
}
doc, err := goquery.NewDocumentFromReader(strings.NewReader(string(htmlContent)))
if err != nil {
log.Fatalf("goquery.NewDocumentFromReader -> %v", err)
}
// Find all image tags and extract their 'src' attributes
var imagePaths []string
doc.Find(".detail_text img").Each(func(i int, img *goquery.Selection) {
imgUrl, exists := img.Attr("src")
if exists {
log.Printf("Image URL %d: %s", i+1, slowtwitch.GetURL(imgUrl))
imagePaths = append(imagePaths, imgUrl)
}
})
blog := doc.Find(".detail_text")
blog.Find(":first-child").Remove()
blog.Find("img").Each(func(i int, img *goquery.Selection) {
imgUrl, exists := img.Attr("src")
if exists {
newEle := goquery.NewDocumentFromNode(&html.Node{
Type: html.ElementNode,
Data: "img",
Attr: []html.Attribute{
html.Attribute{
Key: "src",
Val: "www.slowtwitch.cloud" + imgUrl,
},
html.Attribute{
Key: "class",
Val: "class1 class2 class3",
},
},
})
img.AfterSelection(newEle.Selection)
}
img.Remove()
})
blogContent, err := blog.Html()
// TODO Article migration
//EXPERIMENT AREA START
imageUrls, html, err := slowtwitch.GetImagesAndPostHtml("https://www.slowtwitch.com/Products/Components/SRAM_Drops_New_RED_AXS_Groupset_8950.html")
if err != nil {
fmt.Println(err)
}
fmt.Println(blogContent)
//First image in list will be the featured image, remove that img tag from the returned html
//images will need their src updated after upload
for i, imageUrl := range imageUrls {
fmt.Println(i, imageUrl)
}
fmt.Println(html)
//EXPERIMENT END
editorMigration := migration.MigrateAuthors{
SlowtwitchDatabase: slowtwitchDB,

17
services/migration/image-result.go Normal file
View File

@ -0,0 +1,17 @@
package migration
import "database/sql"
type ImageResult struct {
WordpressId int
PostId int
OldUrl string
NewUrl string
IsSuccess bool
ErrorMessage string
}
func CreateImageResult(parameters ImageResult, db *sql.DB) error {
_, err := db.Exec("insert into ImageResults (PostId, WordpressId, OldUrl, NewUrl, IsSuccess, ErrorMessage) values (?, ?, ?, ?, ?, ?)", parameters.PostId, parameters.WordpressId, parameters.OldUrl, parameters.NewUrl, parameters.IsSuccess, parameters.ErrorMessage)
return err
}

110
services/migration/migrate-posts.go
View File

@ -32,6 +32,17 @@ func (migration MigratePosts) Execute() {
fmt.Println("Could not migrate posts:", err)
return
}
//get wordpress tag data, there are only 3
tags := []string{"swim", "bike", "run"}
var wpTagData []wordpress.TagData
for _, tag := range tags {
tagData, ok := wordpress.GetTag(tag, migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword)
if ok == false {
panic("could not get tag data from wp")
}
wpTagData = append(wpTagData, tagData)
}
slowtwitchPostIdsForMigration := getPostIdsThatNeedMigration(slowtwitchPostIds, migratedPostIds)
@ -42,6 +53,7 @@ func (migration MigratePosts) Execute() {
createWordpressPost := wordpress.CreatePost{
Title: postBase.Title,
Excerpt: postBase.Description,
Status: "published",
}
if postBase.DatePublished.Valid {
@ -52,6 +64,18 @@ func (migration MigratePosts) Execute() {
continue
}
for _, tag := range wpTagData {
if postBase.Bike == true && tag.Name == "bike" {
createWordpressPost.Tags = append(createWordpressPost.Tags, tag.Id)
}
if postBase.Swim == true && tag.Name == "swim" {
createWordpressPost.Tags = append(createWordpressPost.Tags, tag.Id)
}
if postBase.Run == true && tag.Name == "run" {
createWordpressPost.Tags = append(createWordpressPost.Tags, tag.Id)
}
}
if err != nil {
errorMessage = errorMessage + err.Error()
// TODO SEND TO RESULTS DB WITH CALL
@ -89,16 +113,94 @@ func (migration MigratePosts) Execute() {
// TODO SEND TO RESULTS DB WITH CALL
continue
}
//Get page, parse out post data and images
//Upload images to wordpress, swap out with new image urls
//Submit
imagePaths, html, err := slowtwitch.GetImagesAndPostHtml(oldLink)
if err != nil {
errorMessage = errorMessage + err.Error()
// TODO SEND TO RESULTS DB WITH CALL
continue
}
//Get new link
//Store results
//Update advanced Custom Fields with images
var imageResults []ImageResult
for i, imagePath := range imagePaths {
imageUrl := "https://www.slowtwitch.com" + imagePath
createWordpressImage := wordpress.CreateImage{
Url: imageUrl,
}
wordpressImage, err := createWordpressImage.Execute(migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword)
if err != nil {
errorMessage = errorMessage + err.Error()
// TODO SEND TO RESULTS DB WITH CALL
continue
}
//first photo is the featured photo
if i == 0 {
createWordpressPost.FeaturedMedia = wordpressImage.Id
}
//begin process of recording result
imageResult := ImageResult{
OldUrl: imageUrl,
NewUrl: wordpressImage.Link,
WordpressId: wordpressImage.Id,
IsSuccess: true,
}
imageResults = append(imageResults, imageResult)
//replace old links with new in post html
strings.ReplaceAll(html, imageUrl, wordpressImage.Link)
//create redirect
createRedirect := wordpress.CreateRedirect{
Title: postBase.Title + "image-" + string((i + 1)),
Url: imagePath,
MatchType: "page",
ActionType: "url",
ActionCode: 301,
GroupId: 1,
ActionData: wordpress.ActionData{
Url: "/" + wordpressImage.Slug,
},
}
createRedirect.Execute(migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword)
}
createWordpressPost.Content = html
post, err := createWordpressPost.Execute(migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword)
if err != nil {
errorMessage = errorMessage + err.Error()
// TODO SEND TO RESULTS DB WITH CALL
continue
}
//set up post result here to create
postResult := PostResult{
SlowtwitchId: postId,
WordpressId: post.Id,
OldUrl: oldLink,
OldUrlStatus: linkStatus,
NewUrl: post.Link,
IsSuccess: true,
ErrorMessage: errorMessage,
}
postResultId, err := CreatePostResult(postResult, migration.ResultsDatabase)
if err != nil {
panic("Could not record post result for Slowtwitch post:" + string(postId))
}
for _, imageResult := range imageResults {
imageResult.PostId = postResultId
err := CreateImageResult(imageResult, migration.ResultsDatabase)
if err != nil {
fmt.Println("Error recording image result")
}
}
// TODO record redirect
}
//Update related posts (get from post results db) as second loop
//Update advanced Custom Fields with images
}
func getPostIdsThatNeedMigration(slowtwitchPostIds, migratedPostIds []int) []int {

26
services/migration/post-result.go Normal file
View File

@ -0,0 +1,26 @@
package migration
import "database/sql"
type PostResult struct {
WordpressId int
SlowtwitchId int
OldUrl string
OldUrlStatus int
NewUrl string
IsSuccess bool
ErrorMessage string
}
func CreatePostResult(parameters PostResult, db *sql.DB) (int, error) {
result, err := db.Exec("insert into ImageResults (WordpressId, SlowtwitchId, OldUrl, OldUrlStatus, NewUrl, IsSuccess, ErrorMessage) values (?, ?, ?, ?, ?, ?, ?)", parameters.WordpressId, parameters.SlowtwitchId, parameters.OldUrl, parameters.OldUrlStatus, parameters.NewUrl, parameters.IsSuccess, parameters.ErrorMessage)
if err != nil {
return 0, err
} else {
id, err := result.LastInsertId()
if err != nil {
return 0, err
}
return int(id), nil
}
}

64
services/slowtwitch/get-images-and-post-html.go Normal file
View File

@ -0,0 +1,64 @@
package slowtwitch
import (
"github.com/PuerkitoBio/goquery"
"io"
"net/http"
"strings"
)
func GetImagesAndPostHtml(url string) (imagePaths []string, htmlBody string, err error) {
//EXPERIMENT START
res, err := http.Get(url)
if err != nil {
return
}
defer res.Body.Close()
// Read the HTML content
htmlContent, err := io.ReadAll(res.Body)
if err != nil {
return
}
doc, err := goquery.NewDocumentFromReader(strings.NewReader(string(htmlContent)))
if err != nil {
return
}
// Find all image tags and extract their 'src' attributes
doc.Find(".detail_text img").Each(func(i int, img *goquery.Selection) {
imgUrl, exists := img.Attr("src")
if exists {
imagePaths = append(imagePaths, imgUrl)
}
})
// Get blog html, remove first image because wordpress will handle that as a featured image
blog := doc.Find(".detail_text")
blog.Find(":first-child").Remove()
htmlBody, err = blog.Html()
return
}
/*
example of how to switch out nodes
blog.Find("img").Each(func(i int, img *goquery.Selection) {
imgUrl, exists := img.Attr("src")
if exists {
newEle := goquery.NewDocumentFromNode(&html.Node{
Type: html.ElementNode,
Data: "img",
Attr: []html.Attribute{
html.Attribute{
Key: "src",
Val: "www.slowtwitch.cloud" + imgUrl,
},
html.Attribute{
Key: "class",
Val: "class1 class2 class3",
},
},
})
img.AfterSelection(newEle.Selection)
}
img.Remove()
})*/

7
services/slowtwitch/get-post-base.go
View File

@ -12,13 +12,16 @@ type SlowtwitchPostBase struct {
Author string
Description string
DatePublished sql.NullTime
Swim bool
Bike bool
Run bool
}
func GetPostBase(id int, db *sql.DB) (SlowtwitchPostBase, error) {
var output SlowtwitchPostBase
//Get Base
row := db.QueryRow("select ID, Title, LinkOwner, Add_Date, Description from glinks_Links where ID = ?", id)
err := row.Scan(&output.Id, &output.Title, &output.Author, &output.DatePublished, &output.Description)
row := db.QueryRow("select ID, Title, LinkOwner, Add_Date, Description, (tag_swim = b'1'), (tag_bike = b'1'), (tag_run = b'1') from glinks_Links where ID = ?", id)
err := row.Scan(&output.Id, &output.Title, &output.Author, &output.DatePublished, &output.Description, &output.Swim, &output.Bike, &output.Run)
if err != nil {
return output, err

29
services/wordpress/create-image.go
View File

@ -16,27 +16,40 @@ type CreateImage struct {
type CreateImageResponse struct {
Id int `json:"id"`
Link string `json:"link"`
Slug string `json:"slug"`
}
func (parameters *CreateImage) Execute(baseUrl, user, pass string) CreateImageResponse {
func (parameters *CreateImage) Execute(baseUrl, user, pass string) (CreateImageResponse, error) {
resp, err := http.Get(parameters.Url)
utilities.CheckError(err)
if err != nil {
return CreateImageResponse{}, err
}
defer utilities.CloseBodyAndCheckError(resp.Body)
body, err := io.ReadAll(resp.Body)
utilities.CheckError(err)
if err != nil {
return CreateImageResponse{}, err
}
request, err := http.NewRequest("POST", baseUrl+"wp-json/wp/v2/media", bytes.NewReader(body))
utilities.CheckError(err)
if err != nil {
return CreateImageResponse{}, err
}
filename := GetFileName(parameters.Url)
request.Header.Set("Content-Disposition", `attachment;filename="`+filename+`"`)
request.SetBasicAuth(user, pass)
rsp, err := http.DefaultClient.Do(request)
utilities.CheckError(err)
if err != nil {
return CreateImageResponse{}, err
}
result, err := io.ReadAll(rsp.Body)
utilities.CheckError(err)
if err != nil {
return CreateImageResponse{}, err
}
var data CreateImageResponse
err = json.Unmarshal(result, &data)
utilities.CheckError(err)
return data
if err != nil {
return CreateImageResponse{}, err
}
return data, nil
}
func GetFileName(url string) string {

1
services/wordpress/create-post.go
View File

@ -14,7 +14,6 @@ type CreatePost struct {
Tags []int `json:"tags"`
Status string `json:"status"`
Categories []int `json:"categories"`
Slug string `json:"slug"`
Date string `json:"date"`
}