From 683e3a6a87e5c1aebd9fc60dd2c3abb76ad39335 Mon Sep 17 00:00:00 2001 From: Ross Trottier Date: Sun, 19 May 2024 15:00:33 -0600 Subject: [PATCH] checkpoint --- main.go | 82 +++---------- services/migration/image-result.go | 17 +++ services/migration/migrate-posts.go | 110 +++++++++++++++++- services/migration/post-result.go | 26 +++++ .../slowtwitch/get-images-and-post-html.go | 64 ++++++++++ services/slowtwitch/get-post-base.go | 7 +- services/wordpress/create-image.go | 29 +++-- services/wordpress/create-post.go | 1 - 8 files changed, 258 insertions(+), 78 deletions(-) create mode 100644 services/migration/image-result.go create mode 100644 services/migration/post-result.go create mode 100644 services/slowtwitch/get-images-and-post-html.go diff --git a/main.go b/main.go index 3f0e43f..f519b77 100644 --- a/main.go +++ b/main.go @@ -1,15 +1,10 @@ package main import ( + "database/sql" "federated.computer/wp-sync-slowtwitch/services/migration" "federated.computer/wp-sync-slowtwitch/services/slowtwitch" "fmt" - "github.com/PuerkitoBio/goquery" - "golang.org/x/net/html" - "io" - "log" - "net/http" - "strings" ) const baseUrl = "https://slowtwitch.cloud/" @@ -24,73 +19,34 @@ const federatedDbUrl = "slowtwitch.northend.network" const federatedDbPort = "3306" var appCache AppCache +var slowtwitchDB *sql.DB +var resultsDB *sql.DB func main() { - // TODO Article migration - slowtwitchDB, slowtwitchDbErr := migration.Connect(slowtwitchAdminUser, slowtwitchAdminPass, federatedDbUrl, federatedDbPort, slowtwitchDbName+"?parseTime=true") + //Connect to databases + slowtwitchDatabase, slowtwitchDbErr := migration.Connect(slowtwitchAdminUser, slowtwitchAdminPass, federatedDbUrl, federatedDbPort, slowtwitchDbName+"?parseTime=true") if slowtwitchDbErr != nil { - fmt.Println(slowtwitchDbErr) + panic("Could not connect to slowtwitch database.") + } else { + slowtwitchDB = slowtwitchDatabase } - resultsDB, resultsDBerr := migration.Connect(slowtwitchAdminUser, slowtwitchAdminPass, federatedDbUrl, federatedDbPort, migrationDbName) + resultsDatabase, resultsDBerr := migration.Connect(slowtwitchAdminUser, slowtwitchAdminPass, federatedDbUrl, federatedDbPort, migrationDbName) if resultsDBerr != nil { - fmt.Println(resultsDBerr) + panic("Could not connect to results database.") + } else { + resultsDB = resultsDatabase } - //EXPERIMENT START - res, err := http.Get("https://www.slowtwitch.com/Products/Components/SRAM_Drops_New_RED_AXS_Groupset_8950.html") - if err != nil { - log.Fatalf("http.Get -> %v", err) - } - defer res.Body.Close() - // Read the HTML content - htmlContent, err := io.ReadAll(res.Body) - if err != nil { - log.Fatalf("ioutil.ReadAll -> %v", err) - } - doc, err := goquery.NewDocumentFromReader(strings.NewReader(string(htmlContent))) - if err != nil { - log.Fatalf("goquery.NewDocumentFromReader -> %v", err) - } - // Find all image tags and extract their 'src' attributes - var imagePaths []string - doc.Find(".detail_text img").Each(func(i int, img *goquery.Selection) { - imgUrl, exists := img.Attr("src") - if exists { - log.Printf("Image URL %d: %s", i+1, slowtwitch.GetURL(imgUrl)) - imagePaths = append(imagePaths, imgUrl) - } - }) - blog := doc.Find(".detail_text") - blog.Find(":first-child").Remove() - blog.Find("img").Each(func(i int, img *goquery.Selection) { - imgUrl, exists := img.Attr("src") - if exists { - newEle := goquery.NewDocumentFromNode(&html.Node{ - Type: html.ElementNode, - Data: "img", - Attr: []html.Attribute{ - html.Attribute{ - Key: "src", - Val: "www.slowtwitch.cloud" + imgUrl, - }, - html.Attribute{ - Key: "class", - Val: "class1 class2 class3", - }, - }, - }) - img.AfterSelection(newEle.Selection) - } - - img.Remove() - }) - blogContent, err := blog.Html() + // TODO Article migration + //EXPERIMENT AREA START + imageUrls, html, err := slowtwitch.GetImagesAndPostHtml("https://www.slowtwitch.com/Products/Components/SRAM_Drops_New_RED_AXS_Groupset_8950.html") if err != nil { fmt.Println(err) } - fmt.Println(blogContent) - //First image in list will be the featured image, remove that img tag from the returned html - //images will need their src updated after upload + for i, imageUrl := range imageUrls { + fmt.Println(i, imageUrl) + } + fmt.Println(html) //EXPERIMENT END editorMigration := migration.MigrateAuthors{ SlowtwitchDatabase: slowtwitchDB, diff --git a/services/migration/image-result.go b/services/migration/image-result.go new file mode 100644 index 0000000..9937beb --- /dev/null +++ b/services/migration/image-result.go @@ -0,0 +1,17 @@ +package migration + +import "database/sql" + +type ImageResult struct { + WordpressId int + PostId int + OldUrl string + NewUrl string + IsSuccess bool + ErrorMessage string +} + +func CreateImageResult(parameters ImageResult, db *sql.DB) error { + _, err := db.Exec("insert into ImageResults (PostId, WordpressId, OldUrl, NewUrl, IsSuccess, ErrorMessage) values (?, ?, ?, ?, ?, ?)", parameters.PostId, parameters.WordpressId, parameters.OldUrl, parameters.NewUrl, parameters.IsSuccess, parameters.ErrorMessage) + return err +} diff --git a/services/migration/migrate-posts.go b/services/migration/migrate-posts.go index 7f0194c..316dc65 100644 --- a/services/migration/migrate-posts.go +++ b/services/migration/migrate-posts.go @@ -32,6 +32,17 @@ func (migration MigratePosts) Execute() { fmt.Println("Could not migrate posts:", err) return } + //get wordpress tag data, there are only 3 + tags := []string{"swim", "bike", "run"} + var wpTagData []wordpress.TagData + + for _, tag := range tags { + tagData, ok := wordpress.GetTag(tag, migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword) + if ok == false { + panic("could not get tag data from wp") + } + wpTagData = append(wpTagData, tagData) + } slowtwitchPostIdsForMigration := getPostIdsThatNeedMigration(slowtwitchPostIds, migratedPostIds) @@ -42,6 +53,7 @@ func (migration MigratePosts) Execute() { createWordpressPost := wordpress.CreatePost{ Title: postBase.Title, Excerpt: postBase.Description, + Status: "published", } if postBase.DatePublished.Valid { @@ -52,6 +64,18 @@ func (migration MigratePosts) Execute() { continue } + for _, tag := range wpTagData { + if postBase.Bike == true && tag.Name == "bike" { + createWordpressPost.Tags = append(createWordpressPost.Tags, tag.Id) + } + if postBase.Swim == true && tag.Name == "swim" { + createWordpressPost.Tags = append(createWordpressPost.Tags, tag.Id) + } + if postBase.Run == true && tag.Name == "run" { + createWordpressPost.Tags = append(createWordpressPost.Tags, tag.Id) + } + } + if err != nil { errorMessage = errorMessage + err.Error() // TODO SEND TO RESULTS DB WITH CALL @@ -89,16 +113,94 @@ func (migration MigratePosts) Execute() { // TODO SEND TO RESULTS DB WITH CALL continue } - + //Get page, parse out post data and images //Upload images to wordpress, swap out with new image urls //Submit + imagePaths, html, err := slowtwitch.GetImagesAndPostHtml(oldLink) + if err != nil { + errorMessage = errorMessage + err.Error() + // TODO SEND TO RESULTS DB WITH CALL + continue + } - //Get new link - //Store results - //Update advanced Custom Fields with images + var imageResults []ImageResult + + for i, imagePath := range imagePaths { + imageUrl := "https://www.slowtwitch.com" + imagePath + createWordpressImage := wordpress.CreateImage{ + Url: imageUrl, + } + + wordpressImage, err := createWordpressImage.Execute(migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword) + + if err != nil { + errorMessage = errorMessage + err.Error() + // TODO SEND TO RESULTS DB WITH CALL + continue + } + //first photo is the featured photo + if i == 0 { + createWordpressPost.FeaturedMedia = wordpressImage.Id + } + //begin process of recording result + imageResult := ImageResult{ + OldUrl: imageUrl, + NewUrl: wordpressImage.Link, + WordpressId: wordpressImage.Id, + IsSuccess: true, + } + imageResults = append(imageResults, imageResult) + //replace old links with new in post html + strings.ReplaceAll(html, imageUrl, wordpressImage.Link) + //create redirect + createRedirect := wordpress.CreateRedirect{ + Title: postBase.Title + "image-" + string((i + 1)), + Url: imagePath, + MatchType: "page", + ActionType: "url", + ActionCode: 301, + GroupId: 1, + ActionData: wordpress.ActionData{ + Url: "/" + wordpressImage.Slug, + }, + } + + createRedirect.Execute(migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword) + } + createWordpressPost.Content = html + post, err := createWordpressPost.Execute(migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword) + if err != nil { + errorMessage = errorMessage + err.Error() + // TODO SEND TO RESULTS DB WITH CALL + continue + } + //set up post result here to create + postResult := PostResult{ + SlowtwitchId: postId, + WordpressId: post.Id, + OldUrl: oldLink, + OldUrlStatus: linkStatus, + NewUrl: post.Link, + IsSuccess: true, + ErrorMessage: errorMessage, + } + + postResultId, err := CreatePostResult(postResult, migration.ResultsDatabase) + if err != nil { + panic("Could not record post result for Slowtwitch post:" + string(postId)) + } + for _, imageResult := range imageResults { + imageResult.PostId = postResultId + err := CreateImageResult(imageResult, migration.ResultsDatabase) + if err != nil { + fmt.Println("Error recording image result") + } + } + // TODO record redirect } //Update related posts (get from post results db) as second loop + //Update advanced Custom Fields with images } func getPostIdsThatNeedMigration(slowtwitchPostIds, migratedPostIds []int) []int { diff --git a/services/migration/post-result.go b/services/migration/post-result.go new file mode 100644 index 0000000..7cac1dd --- /dev/null +++ b/services/migration/post-result.go @@ -0,0 +1,26 @@ +package migration + +import "database/sql" + +type PostResult struct { + WordpressId int + SlowtwitchId int + OldUrl string + OldUrlStatus int + NewUrl string + IsSuccess bool + ErrorMessage string +} + +func CreatePostResult(parameters PostResult, db *sql.DB) (int, error) { + result, err := db.Exec("insert into ImageResults (WordpressId, SlowtwitchId, OldUrl, OldUrlStatus, NewUrl, IsSuccess, ErrorMessage) values (?, ?, ?, ?, ?, ?, ?)", parameters.WordpressId, parameters.SlowtwitchId, parameters.OldUrl, parameters.OldUrlStatus, parameters.NewUrl, parameters.IsSuccess, parameters.ErrorMessage) + if err != nil { + return 0, err + } else { + id, err := result.LastInsertId() + if err != nil { + return 0, err + } + return int(id), nil + } +} diff --git a/services/slowtwitch/get-images-and-post-html.go b/services/slowtwitch/get-images-and-post-html.go new file mode 100644 index 0000000..488c44c --- /dev/null +++ b/services/slowtwitch/get-images-and-post-html.go @@ -0,0 +1,64 @@ +package slowtwitch + +import ( + "github.com/PuerkitoBio/goquery" + "io" + "net/http" + "strings" +) + +func GetImagesAndPostHtml(url string) (imagePaths []string, htmlBody string, err error) { + //EXPERIMENT START + res, err := http.Get(url) + if err != nil { + return + } + defer res.Body.Close() + + // Read the HTML content + htmlContent, err := io.ReadAll(res.Body) + if err != nil { + return + } + doc, err := goquery.NewDocumentFromReader(strings.NewReader(string(htmlContent))) + if err != nil { + return + } + // Find all image tags and extract their 'src' attributes + doc.Find(".detail_text img").Each(func(i int, img *goquery.Selection) { + imgUrl, exists := img.Attr("src") + if exists { + imagePaths = append(imagePaths, imgUrl) + } + }) + // Get blog html, remove first image because wordpress will handle that as a featured image + blog := doc.Find(".detail_text") + blog.Find(":first-child").Remove() + htmlBody, err = blog.Html() + return +} + +/* +example of how to switch out nodes +blog.Find("img").Each(func(i int, img *goquery.Selection) { + imgUrl, exists := img.Attr("src") + if exists { + newEle := goquery.NewDocumentFromNode(&html.Node{ + Type: html.ElementNode, + Data: "img", + Attr: []html.Attribute{ + html.Attribute{ + Key: "src", + Val: "www.slowtwitch.cloud" + imgUrl, + }, + html.Attribute{ + Key: "class", + Val: "class1 class2 class3", + }, + }, + }) + img.AfterSelection(newEle.Selection) + } + + img.Remove() +})*/ diff --git a/services/slowtwitch/get-post-base.go b/services/slowtwitch/get-post-base.go index 9e2dacf..323d2e7 100644 --- a/services/slowtwitch/get-post-base.go +++ b/services/slowtwitch/get-post-base.go @@ -12,13 +12,16 @@ type SlowtwitchPostBase struct { Author string Description string DatePublished sql.NullTime + Swim bool + Bike bool + Run bool } func GetPostBase(id int, db *sql.DB) (SlowtwitchPostBase, error) { var output SlowtwitchPostBase //Get Base - row := db.QueryRow("select ID, Title, LinkOwner, Add_Date, Description from glinks_Links where ID = ?", id) - err := row.Scan(&output.Id, &output.Title, &output.Author, &output.DatePublished, &output.Description) + row := db.QueryRow("select ID, Title, LinkOwner, Add_Date, Description, (tag_swim = b'1'), (tag_bike = b'1'), (tag_run = b'1') from glinks_Links where ID = ?", id) + err := row.Scan(&output.Id, &output.Title, &output.Author, &output.DatePublished, &output.Description, &output.Swim, &output.Bike, &output.Run) if err != nil { return output, err diff --git a/services/wordpress/create-image.go b/services/wordpress/create-image.go index c5bd00a..18a019e 100644 --- a/services/wordpress/create-image.go +++ b/services/wordpress/create-image.go @@ -16,27 +16,40 @@ type CreateImage struct { type CreateImageResponse struct { Id int `json:"id"` Link string `json:"link"` + Slug string `json:"slug"` } -func (parameters *CreateImage) Execute(baseUrl, user, pass string) CreateImageResponse { +func (parameters *CreateImage) Execute(baseUrl, user, pass string) (CreateImageResponse, error) { resp, err := http.Get(parameters.Url) - utilities.CheckError(err) + if err != nil { + return CreateImageResponse{}, err + } defer utilities.CloseBodyAndCheckError(resp.Body) body, err := io.ReadAll(resp.Body) - utilities.CheckError(err) + if err != nil { + return CreateImageResponse{}, err + } request, err := http.NewRequest("POST", baseUrl+"wp-json/wp/v2/media", bytes.NewReader(body)) - utilities.CheckError(err) + if err != nil { + return CreateImageResponse{}, err + } filename := GetFileName(parameters.Url) request.Header.Set("Content-Disposition", `attachment;filename="`+filename+`"`) request.SetBasicAuth(user, pass) rsp, err := http.DefaultClient.Do(request) - utilities.CheckError(err) + if err != nil { + return CreateImageResponse{}, err + } result, err := io.ReadAll(rsp.Body) - utilities.CheckError(err) + if err != nil { + return CreateImageResponse{}, err + } var data CreateImageResponse err = json.Unmarshal(result, &data) - utilities.CheckError(err) - return data + if err != nil { + return CreateImageResponse{}, err + } + return data, nil } func GetFileName(url string) string { diff --git a/services/wordpress/create-post.go b/services/wordpress/create-post.go index 329c36e..57a722b 100644 --- a/services/wordpress/create-post.go +++ b/services/wordpress/create-post.go @@ -14,7 +14,6 @@ type CreatePost struct { Tags []int `json:"tags"` Status string `json:"status"` Categories []int `json:"categories"` - Slug string `json:"slug"` Date string `json:"date"` }