adds scraper for game search

cmd/dealsbot/api/gog.go

@@ -17,7 +17,7 @@ type GogStruct struct {
 
 func NewGogApi() GogStruct {
 	gog := GogStruct{
-		url:      "https://www.gog.com/",
+		url:      "https://www.gog.com/en/games?priceRange=0,0&discounted=true",
 		baseUrl:  "https://www.gog.com/game/",
 		idPrefix: "gog-",
 		headers:  make(map[string]string),
@@ -67,11 +67,11 @@ func (e GogStruct) Load() error {
 					continue
 				}
 				for _, a := range t.Attr {
-					if !(a.Key == "id" && a.Val == "giveaway") {
+					if !(a.Key == "class" && a.Val == "product-tile product-tile--grid") {
 						continue
 					}
 					for _, attr := range t.Attr {
-						if attr.Key != "ng-href" {
+						if attr.Key != "href" {
 							continue
 						}
 						appID := regexAppid.FindStringSubmatch(attr.Val)
@@ -116,7 +116,7 @@ func (e GogStruct) Load() error {
 					}
 					for _, a := range t.Attr {
 						if !(a.Key == "class" && a.Val == "productcard-basics__title") {
-
+							continue
 						}
 						if tt = bodyGame.Next(); tt != html.TextToken {
 							continue

cmd/dealsbot/api/gogfront.go

@@ -0,0 +1,148 @@
+package api
+
+import (
+	"fmt"
+	"golang.org/x/net/html"
+	"net/http"
+	"regexp"
+)
+
+type GogFrontStruct struct {
+	url      string
+	baseUrl  string
+	idPrefix string
+	headers  map[string]string
+	deals    DealsMap
+}
+
+func NewGogFrontApi() GogFrontStruct {
+	gog := GogFrontStruct{
+		url:      "https://www.gog.com/",
+		baseUrl:  "https://www.gog.com/game/",
+		idPrefix: "gog-",
+		headers:  make(map[string]string),
+		deals:    make(map[string]Deal),
+	}
+	gog.headers["Accept-Language"] = "en"
+	gog.headers["User-Agent"] = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:93.0) Gecko/20100101 Firefox/93.0"
+	return gog
+}
+
+func (e GogFrontStruct) Load() error {
+	client := &http.Client{}
+	// might have to add a cookie at a later time but currently works without
+	// "Cookie", "gog_lc=GB_GBP_en-US" or "Accept-Language", "en"
+	reqStore, err := http.NewRequest("GET", e.url, nil)
+	if err != nil {
+		return err
+	}
+
+	for key, value := range e.headers {
+		reqStore.Header.Set(key, value)
+	}
+
+	resStore, err := client.Do(reqStore)
+	if err != nil {
+		return err
+	}
+	bodyStore := html.NewTokenizer(resStore.Body)
+
+	regexAppid, err := regexp.Compile(`/\w{2}/game/([-\w]+)`)
+	if err != nil {
+		return err
+	}
+
+	var appIDs []string
+	func() {
+		for {
+			tt := bodyStore.Next()
+
+			switch {
+			case tt == html.ErrorToken:
+				// file end or error
+				return
+			case tt == html.StartTagToken:
+				t := bodyStore.Token()
+				if t.Data != "a" {
+					continue
+				}
+				for _, a := range t.Attr {
+					if !(a.Key == "id" && a.Val == "giveaway") {
+						continue
+					}
+					for _, attr := range t.Attr {
+						if attr.Key != "ng-href" {
+							continue
+						}
+						appID := regexAppid.FindStringSubmatch(attr.Val)
+						if len(appID) < 1 {
+							continue
+						}
+						appIDs = append(appIDs, appID[1])
+					}
+				}
+			}
+		}
+	}()
+
+	for _, appID := range appIDs {
+		reqGame, err := http.NewRequest("GET", fmt.Sprintf("%v%v", e.baseUrl, appID), nil)
+		if err != nil {
+			return err
+		}
+
+		for key, value := range e.headers {
+			reqGame.Header.Set(key, value)
+		}
+
+		resGame, err := client.Do(reqGame)
+		if err != nil {
+			return err
+		}
+		bodyGame := html.NewTokenizer(resGame.Body)
+
+		func() {
+			for {
+				tt := bodyGame.Next()
+
+				switch {
+				case tt == html.ErrorToken:
+					// file end or error
+					return
+				case tt == html.StartTagToken:
+					t := bodyGame.Token()
+					if t.Data != "h1" {
+						continue
+					}
+					for _, a := range t.Attr {
+						if !(a.Key == "class" && a.Val == "productcard-basics__title") {
+							continue
+						}
+						if tt = bodyGame.Next(); tt != html.TextToken {
+							continue
+						}
+						id := fmt.Sprintf("%v%v", e.idPrefix, appID)
+						title := bodyGame.Token().Data
+						url := fmt.Sprintf("%v%v", e.baseUrl, appID)
+
+						e.deals[id] = Deal{
+							Id:    id,
+							Title: title,
+							Url:   url,
+						}
+					}
+				}
+			}
+		}()
+	}
+
+	return nil
+}
+
+func (e GogFrontStruct) Get() []Deal {
+	var deals []Deal
+	for _, deal := range e.deals {
+		deals = append(deals, deal)
+	}
+	return deals
+}

cmd/dealsbot/main.go

@@ -73,7 +73,7 @@ func main() {
 			select {
 			case <-ticker.C:
 				var apis []api.Api
-				apis = append(apis, api.NewUbsioftApi(), api.NewEpicApi(), api.NewSteamApi(), api.NewGogApi(), api.NewHumbleBundleApi())
+				apis = append(apis, api.NewUbsioftApi(), api.NewEpicApi(), api.NewSteamApi(), api.NewGogFrontApi(), api.NewGogApi(), api.NewHumbleBundleApi())
 				for _, a := range apis {
 					err := a.Load()
 					if err != nil {