checkpoint

main.go

@@ -1,15 +1,10 @@
 package main
 
 import (
+	"database/sql"
 	"federated.computer/wp-sync-slowtwitch/services/migration"
 	"federated.computer/wp-sync-slowtwitch/services/slowtwitch"
 	"fmt"
-	"github.com/PuerkitoBio/goquery"
-	"golang.org/x/net/html"
-	"io"
-	"log"
-	"net/http"
-	"strings"
 )
 
 const baseUrl = "https://slowtwitch.cloud/"
@@ -24,73 +19,34 @@ const federatedDbUrl = "slowtwitch.northend.network"
 const federatedDbPort = "3306"
 
 var appCache AppCache
+var slowtwitchDB *sql.DB
+var resultsDB *sql.DB
 
 func main() {
-	// TODO Article migration
+	//Connect to databases
-	slowtwitchDB, slowtwitchDbErr := migration.Connect(slowtwitchAdminUser, slowtwitchAdminPass, federatedDbUrl, federatedDbPort, slowtwitchDbName+"?parseTime=true")
+	slowtwitchDatabase, slowtwitchDbErr := migration.Connect(slowtwitchAdminUser, slowtwitchAdminPass, federatedDbUrl, federatedDbPort, slowtwitchDbName+"?parseTime=true")
 	if slowtwitchDbErr != nil {
-		fmt.Println(slowtwitchDbErr)
+		panic("Could not connect to slowtwitch database.")
+	} else {
+		slowtwitchDB = slowtwitchDatabase
 	}
-	resultsDB, resultsDBerr := migration.Connect(slowtwitchAdminUser, slowtwitchAdminPass, federatedDbUrl, federatedDbPort, migrationDbName)
+	resultsDatabase, resultsDBerr := migration.Connect(slowtwitchAdminUser, slowtwitchAdminPass, federatedDbUrl, federatedDbPort, migrationDbName)
 	if resultsDBerr != nil {
-		fmt.Println(resultsDBerr)
+		panic("Could not connect to results database.")
+	} else {
+		resultsDB = resultsDatabase
 	}
-	//EXPERIMENT START
-	res, err := http.Get("https://www.slowtwitch.com/Products/Components/SRAM_Drops_New_RED_AXS_Groupset_8950.html")
-	if err != nil {
-		log.Fatalf("http.Get -> %v", err)
-	}
-	defer res.Body.Close()
 
-	// Read the HTML content
+	// TODO Article migration
-	htmlContent, err := io.ReadAll(res.Body)
+	//EXPERIMENT AREA START
-	if err != nil {
+	imageUrls, html, err := slowtwitch.GetImagesAndPostHtml("https://www.slowtwitch.com/Products/Components/SRAM_Drops_New_RED_AXS_Groupset_8950.html")
-		log.Fatalf("ioutil.ReadAll -> %v", err)
-	}
-	doc, err := goquery.NewDocumentFromReader(strings.NewReader(string(htmlContent)))
-	if err != nil {
-		log.Fatalf("goquery.NewDocumentFromReader -> %v", err)
-	}
-	// Find all image tags and extract their 'src' attributes
-	var imagePaths []string
-	doc.Find(".detail_text img").Each(func(i int, img *goquery.Selection) {
-		imgUrl, exists := img.Attr("src")
-		if exists {
-			log.Printf("Image URL %d: %s", i+1, slowtwitch.GetURL(imgUrl))
-			imagePaths = append(imagePaths, imgUrl)
-		}
-	})
-	blog := doc.Find(".detail_text")
-	blog.Find(":first-child").Remove()
-	blog.Find("img").Each(func(i int, img *goquery.Selection) {
-		imgUrl, exists := img.Attr("src")
-		if exists {
-			newEle := goquery.NewDocumentFromNode(&html.Node{
-				Type: html.ElementNode,
-				Data: "img",
-				Attr: []html.Attribute{
-					html.Attribute{
-						Key: "src",
-						Val: "www.slowtwitch.cloud" + imgUrl,
-					},
-					html.Attribute{
-						Key: "class",
-						Val: "class1 class2 class3",
-					},
-				},
-			})
-			img.AfterSelection(newEle.Selection)
-		}
-
-		img.Remove()
-	})
-	blogContent, err := blog.Html()
 	if err != nil {
 		fmt.Println(err)
 	}
-	fmt.Println(blogContent)
+	for i, imageUrl := range imageUrls {
-	//First image in list will be the featured image, remove that img tag from the returned html
+		fmt.Println(i, imageUrl)
-	//images will need their src updated after upload
+	}
+	fmt.Println(html)
 	//EXPERIMENT END
 	editorMigration := migration.MigrateAuthors{
 		SlowtwitchDatabase: slowtwitchDB,

services/migration/image-result.go

@@ -0,0 +1,17 @@
+package migration
+
+import "database/sql"
+
+type ImageResult struct {
+	WordpressId  int
+	PostId       int
+	OldUrl       string
+	NewUrl       string
+	IsSuccess    bool
+	ErrorMessage string
+}
+
+func CreateImageResult(parameters ImageResult, db *sql.DB) error {
+	_, err := db.Exec("insert into ImageResults (PostId, WordpressId, OldUrl, NewUrl, IsSuccess, ErrorMessage) values (?, ?, ?, ?, ?, ?)", parameters.PostId, parameters.WordpressId, parameters.OldUrl, parameters.NewUrl, parameters.IsSuccess, parameters.ErrorMessage)
+	return err
+}

services/migration/migrate-posts.go

@@ -32,6 +32,17 @@ func (migration MigratePosts) Execute() {
 		fmt.Println("Could not migrate posts:", err)
 		return
 	}
+	//get wordpress tag data, there are only 3
+	tags := []string{"swim", "bike", "run"}
+	var wpTagData []wordpress.TagData
+
+	for _, tag := range tags {
+		tagData, ok := wordpress.GetTag(tag, migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword)
+		if ok == false {
+			panic("could not get tag data from wp")
+		}
+		wpTagData = append(wpTagData, tagData)
+	}
 
 	slowtwitchPostIdsForMigration := getPostIdsThatNeedMigration(slowtwitchPostIds, migratedPostIds)
 
@@ -42,6 +53,7 @@ func (migration MigratePosts) Execute() {
 		createWordpressPost := wordpress.CreatePost{
 			Title:   postBase.Title,
 			Excerpt: postBase.Description,
+			Status:  "published",
 		}
 
 		if postBase.DatePublished.Valid {
@@ -52,6 +64,18 @@ func (migration MigratePosts) Execute() {
 			continue
 		}
 
+		for _, tag := range wpTagData {
+			if postBase.Bike == true && tag.Name == "bike" {
+				createWordpressPost.Tags = append(createWordpressPost.Tags, tag.Id)
+			}
+			if postBase.Swim == true && tag.Name == "swim" {
+				createWordpressPost.Tags = append(createWordpressPost.Tags, tag.Id)
+			}
+			if postBase.Run == true && tag.Name == "run" {
+				createWordpressPost.Tags = append(createWordpressPost.Tags, tag.Id)
+			}
+		}
+
 		if err != nil {
 			errorMessage = errorMessage + err.Error()
 			// TODO SEND TO RESULTS DB WITH CALL
@@ -89,16 +113,94 @@ func (migration MigratePosts) Execute() {
 			// TODO SEND TO RESULTS DB WITH CALL
 			continue
 		}
-		
+
 		//Get page, parse out post data and images
 		//Upload images to wordpress, swap out with new image urls
 		//Submit
+		imagePaths, html, err := slowtwitch.GetImagesAndPostHtml(oldLink)
+		if err != nil {
+			errorMessage = errorMessage + err.Error()
+			// TODO SEND TO RESULTS DB WITH CALL
+			continue
+		}
 
-		//Get new link
+		var imageResults []ImageResult
-		//Store results
+
-		//Update advanced Custom Fields with images
+		for i, imagePath := range imagePaths {
+			imageUrl := "https://www.slowtwitch.com" + imagePath
+			createWordpressImage := wordpress.CreateImage{
+				Url: imageUrl,
+			}
+
+			wordpressImage, err := createWordpressImage.Execute(migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword)
+
+			if err != nil {
+				errorMessage = errorMessage + err.Error()
+				// TODO SEND TO RESULTS DB WITH CALL
+				continue
+			}
+			//first photo is the featured photo
+			if i == 0 {
+				createWordpressPost.FeaturedMedia = wordpressImage.Id
+			}
+			//begin process of recording result
+			imageResult := ImageResult{
+				OldUrl:      imageUrl,
+				NewUrl:      wordpressImage.Link,
+				WordpressId: wordpressImage.Id,
+				IsSuccess:   true,
+			}
+			imageResults = append(imageResults, imageResult)
+			//replace old links with new in post html
+			strings.ReplaceAll(html, imageUrl, wordpressImage.Link)
+			//create redirect
+			createRedirect := wordpress.CreateRedirect{
+				Title:      postBase.Title + "image-" + string((i + 1)),
+				Url:        imagePath,
+				MatchType:  "page",
+				ActionType: "url",
+				ActionCode: 301,
+				GroupId:    1,
+				ActionData: wordpress.ActionData{
+					Url: "/" + wordpressImage.Slug,
+				},
+			}
+
+			createRedirect.Execute(migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword)
+		}
+		createWordpressPost.Content = html
+		post, err := createWordpressPost.Execute(migration.WordpressBaseUrl, migration.WordpressUser, migration.WordpressPassword)
+		if err != nil {
+			errorMessage = errorMessage + err.Error()
+			// TODO SEND TO RESULTS DB WITH CALL
+			continue
+		}
+		//set up post result here to create
+		postResult := PostResult{
+			SlowtwitchId: postId,
+			WordpressId:  post.Id,
+			OldUrl:       oldLink,
+			OldUrlStatus: linkStatus,
+			NewUrl:       post.Link,
+			IsSuccess:    true,
+			ErrorMessage: errorMessage,
+		}
+
+		postResultId, err := CreatePostResult(postResult, migration.ResultsDatabase)
+		if err != nil {
+			panic("Could not record post result for Slowtwitch post:" + string(postId))
+		}
+		for _, imageResult := range imageResults {
+			imageResult.PostId = postResultId
+			err := CreateImageResult(imageResult, migration.ResultsDatabase)
+			if err != nil {
+				fmt.Println("Error recording image result")
+			}
+		}
+		// TODO record redirect
 	}
 	//Update related posts (get from post results db) as second loop
+	//Update advanced Custom Fields with images
 }
 
 func getPostIdsThatNeedMigration(slowtwitchPostIds, migratedPostIds []int) []int {

services/migration/post-result.go

@@ -0,0 +1,26 @@
+package migration
+
+import "database/sql"
+
+type PostResult struct {
+	WordpressId  int
+	SlowtwitchId int
+	OldUrl       string
+	OldUrlStatus int
+	NewUrl       string
+	IsSuccess    bool
+	ErrorMessage string
+}
+
+func CreatePostResult(parameters PostResult, db *sql.DB) (int, error) {
+	result, err := db.Exec("insert into ImageResults (WordpressId, SlowtwitchId, OldUrl, OldUrlStatus, NewUrl, IsSuccess, ErrorMessage) values (?, ?, ?, ?, ?, ?, ?)", parameters.WordpressId, parameters.SlowtwitchId, parameters.OldUrl, parameters.OldUrlStatus, parameters.NewUrl, parameters.IsSuccess, parameters.ErrorMessage)
+	if err != nil {
+		return 0, err
+	} else {
+		id, err := result.LastInsertId()
+		if err != nil {
+			return 0, err
+		}
+		return int(id), nil
+	}
+}

services/slowtwitch/get-images-and-post-html.go

@@ -0,0 +1,64 @@
+package slowtwitch
+
+import (
+	"github.com/PuerkitoBio/goquery"
+	"io"
+	"net/http"
+	"strings"
+)
+
+func GetImagesAndPostHtml(url string) (imagePaths []string, htmlBody string, err error) {
+	//EXPERIMENT START
+	res, err := http.Get(url)
+	if err != nil {
+		return
+	}
+	defer res.Body.Close()
+
+	// Read the HTML content
+	htmlContent, err := io.ReadAll(res.Body)
+	if err != nil {
+		return
+	}
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(string(htmlContent)))
+	if err != nil {
+		return
+	}
+	// Find all image tags and extract their 'src' attributes
+	doc.Find(".detail_text img").Each(func(i int, img *goquery.Selection) {
+		imgUrl, exists := img.Attr("src")
+		if exists {
+			imagePaths = append(imagePaths, imgUrl)
+		}
+	})
+	// Get blog html, remove first image because wordpress will handle that as a featured image
+	blog := doc.Find(".detail_text")
+	blog.Find(":first-child").Remove()
+	htmlBody, err = blog.Html()
+	return
+}
+
+/*
+example of how to switch out nodes
+blog.Find("img").Each(func(i int, img *goquery.Selection) {
+	imgUrl, exists := img.Attr("src")
+	if exists {
+		newEle := goquery.NewDocumentFromNode(&html.Node{
+			Type: html.ElementNode,
+			Data: "img",
+			Attr: []html.Attribute{
+				html.Attribute{
+					Key: "src",
+					Val: "www.slowtwitch.cloud" + imgUrl,
+				},
+				html.Attribute{
+					Key: "class",
+					Val: "class1 class2 class3",
+				},
+			},
+		})
+		img.AfterSelection(newEle.Selection)
+	}
+
+	img.Remove()
+})*/

services/slowtwitch/get-post-base.go

@@ -12,13 +12,16 @@ type SlowtwitchPostBase struct {
 	Author        string
 	Description   string
 	DatePublished sql.NullTime
+	Swim          bool
+	Bike          bool
+	Run           bool
 }
 
 func GetPostBase(id int, db *sql.DB) (SlowtwitchPostBase, error) {
 	var output SlowtwitchPostBase
 	//Get Base
-	row := db.QueryRow("select ID, Title, LinkOwner, Add_Date, Description from glinks_Links where ID = ?", id)
+	row := db.QueryRow("select ID, Title, LinkOwner, Add_Date, Description, (tag_swim = b'1'), (tag_bike = b'1'), (tag_run = b'1') from glinks_Links where ID = ?", id)
-	err := row.Scan(&output.Id, &output.Title, &output.Author, &output.DatePublished, &output.Description)
+	err := row.Scan(&output.Id, &output.Title, &output.Author, &output.DatePublished, &output.Description, &output.Swim, &output.Bike, &output.Run)
 
 	if err != nil {
 		return output, err

services/wordpress/create-image.go

@@ -16,27 +16,40 @@ type CreateImage struct {
 type CreateImageResponse struct {
 	Id   int    `json:"id"`
 	Link string `json:"link"`
+	Slug string `json:"slug"`
 }
 
-func (parameters *CreateImage) Execute(baseUrl, user, pass string) CreateImageResponse {
+func (parameters *CreateImage) Execute(baseUrl, user, pass string) (CreateImageResponse, error) {
 	resp, err := http.Get(parameters.Url)
-	utilities.CheckError(err)
+	if err != nil {
+		return CreateImageResponse{}, err
+	}
 	defer utilities.CloseBodyAndCheckError(resp.Body)
 	body, err := io.ReadAll(resp.Body)
-	utilities.CheckError(err)
+	if err != nil {
+		return CreateImageResponse{}, err
+	}
 	request, err := http.NewRequest("POST", baseUrl+"wp-json/wp/v2/media", bytes.NewReader(body))
-	utilities.CheckError(err)
+	if err != nil {
+		return CreateImageResponse{}, err
+	}
 	filename := GetFileName(parameters.Url)
 	request.Header.Set("Content-Disposition", `attachment;filename="`+filename+`"`)
 	request.SetBasicAuth(user, pass)
 	rsp, err := http.DefaultClient.Do(request)
-	utilities.CheckError(err)
+	if err != nil {
+		return CreateImageResponse{}, err
+	}
 	result, err := io.ReadAll(rsp.Body)
-	utilities.CheckError(err)
+	if err != nil {
+		return CreateImageResponse{}, err
+	}
 	var data CreateImageResponse
 	err = json.Unmarshal(result, &data)
-	utilities.CheckError(err)
+	if err != nil {
-	return data
+		return CreateImageResponse{}, err
+	}
+	return data, nil
 }
 
 func GetFileName(url string) string {

services/wordpress/create-post.go

@@ -14,7 +14,6 @@ type CreatePost struct {
 	Tags          []int  `json:"tags"`
 	Status        string `json:"status"`
 	Categories    []int  `json:"categories"`
-	Slug          string `json:"slug"`
 	Date          string `json:"date"`
 }