adds header to all crawlers

cmd/dealsbot/epic.go

@@ -11,16 +11,23 @@ import (
 
 type EpicStruct struct {
 	url      string
+	baseUrl  string
 	idPrefix string
+	headers  map[string]string
 	deals    DealsMap
 }
 
 func newEpicApi() EpicStruct {
-	return EpicStruct{
+	epic := EpicStruct{
 		url:      "https://store-site-backend-static-ipv4.ak.epicgames.com/freeGamesPromotions",
+		baseUrl:  "https://store.epicgames.com/p/",
 		idPrefix: "epic-",
+		headers:  make(map[string]string),
 		deals:    make(map[string]Deal),
 	}
+	epic.headers["Accept-Language"] = "en"
+	epic.headers["User-Agent"] = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:93.0) Gecko/20100101 Firefox/93.0"
+	return epic
 }
 
 type epicApiBody struct {
@@ -75,6 +82,10 @@ func (e EpicStruct) load() error {
 		return err
 	}
 
+	for key, value := range e.headers {
+		req.Header.Set(key, value)
+	}
+
 	res, err := client.Do(req)
 	if err != nil {
 		return err
@@ -108,7 +119,7 @@ func (e EpicStruct) load() error {
 
 		id := fmt.Sprintf("%v%v", e.idPrefix, element.Id)
 		title := element.Title
-		url := fmt.Sprintf("https://store.epicgames.com/en-US/p/%v", productSlug)
+		url := fmt.Sprintf("%v%v", e.baseUrl, productSlug)
 
 		e.deals[id] = Deal{
 			Id:    id,

cmd/dealsbot/gog.go

@@ -11,16 +11,21 @@ type GogStruct struct {
 	url      string
 	baseUrl  string
 	idPrefix string
+	headers  map[string]string
 	deals    DealsMap
 }
 
 func newGogApi() GogStruct {
-	return GogStruct{
+	gog := GogStruct{
-		url:      "https://www.gog.com/en",
+		url:      "https://www.gog.com/",
-		baseUrl:  "https://www.gog.com/en/game/",
+		baseUrl:  "https://www.gog.com/game/",
 		idPrefix: "gog-",
+		headers:  make(map[string]string),
 		deals:    make(map[string]Deal),
 	}
+	gog.headers["Accept-Language"] = "en"
+	gog.headers["User-Agent"] = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:93.0) Gecko/20100101 Firefox/93.0"
+	return gog
 }
 
 func (e GogStruct) load() error {
@@ -32,13 +37,17 @@ func (e GogStruct) load() error {
 		return err
 	}
 
+	for key, value := range e.headers {
+		reqStore.Header.Set(key, value)
+	}
+
 	resStore, err := client.Do(reqStore)
 	if err != nil {
 		return err
 	}
 	bodyStore := html.NewTokenizer(resStore.Body)
 
-	regexAppid, err := regexp.Compile(`/en/game/([-\w]+)`)
+	regexAppid, err := regexp.Compile(`/\w{2}/game/([-\w]+)`)
 	if err != nil {
 		return err
 	}
@@ -82,6 +91,10 @@ func (e GogStruct) load() error {
 			return err
 		}
 
+		for key, value := range e.headers {
+			reqGame.Header.Set(key, value)
+		}
+
 		resGame, err := client.Do(reqGame)
 		if err != nil {
 			return err

cmd/dealsbot/humblebundle.go

@@ -12,16 +12,22 @@ type HumbleBundleStruct struct {
 	url      string
 	baseUrl  string
 	idPrefix string
+	headers  map[string]string
 	deals    DealsMap
 }
 
 func newHumbleBundleApi() HumbleBundleStruct {
-	return HumbleBundleStruct{
+	humbleBundle := HumbleBundleStruct{
 		url:      "https://www.humblebundle.com/",
 		baseUrl:  "https://www.humblebundle.com/store/",
 		idPrefix: "humblebundle-",
+		headers:  make(map[string]string),
 		deals:    make(map[string]Deal),
 	}
+	humbleBundle.headers["Accept-Language"] = "en"
+	humbleBundle.headers["User-Agent"] = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:93.0) Gecko/20100101 Firefox/93.0"
+	return humbleBundle
+
 }
 
 type humblebundleJsonBody struct {
@@ -49,6 +55,10 @@ func (e HumbleBundleStruct) load() error {
 		return err
 	}
 
+	for key, value := range e.headers {
+		reqStore.Header.Set(key, value)
+	}
+
 	resStore, err := client.Do(reqStore)
 	if err != nil {
 		return err

cmd/dealsbot/main.go

@@ -24,20 +24,17 @@ var (
 )
 
 func main() {
-	// translate this to go: https://dev.rievo.net/sst/feed-python
-
 	// query different sources store to db
 	// try to incorporate operagx apiUrl:
 	// - https://gx-proxy.operacdn.com/content/free-games?_limit=300&_sort=order%3AASC
 	// send messages to discord
 	// ideas:
 	// - https://github.com/TheLovinator1/discord-free-game-notifier
-	// - https://gg.deals/games/free-games/
 	// - https://gg.deals/news/free-gog-games/
 	// - origin
 	// - check ubisoft works
 
-	log.SetLevel(log.LevelDebug)
+	log.SetLevel(log.LevelInfo)
 	log.Info("starting dealsbot...")
 	log.Info("disgo version: ", disgo.Version)
 

cmd/dealsbot/steam.go

@@ -14,17 +14,22 @@ type SteamStruct struct {
 	baseUrl  string
 	apiUrl   string
 	idPrefix string
+	headers  map[string]string
 	deals    DealsMap
 }
 
 func newSteamApi() SteamStruct {
-	return SteamStruct{
+	steam := SteamStruct{
 		url:      "https://store.steampowered.com/search/results?force_infinite=1&maxprice=free&specials=1",
 		baseUrl:  "https://store.steampowered.com/app/",
 		apiUrl:   "https://store.steampowered.com/api/appdetails?appids=",
 		idPrefix: "steam-",
+		headers:  make(map[string]string),
 		deals:    make(map[string]Deal),
 	}
+	steam.headers["Accept-Language"] = "en"
+	steam.headers["User-Agent"] = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:93.0) Gecko/20100101 Firefox/93.0"
+	return steam
 }
 
 type steamApiBodyGame struct {
@@ -58,6 +63,10 @@ func (e SteamStruct) load() error {
 		return err
 	}
 
+	for key, value := range e.headers {
+		reqStore.Header.Set(key, value)
+	}
+
 	resStore, err := client.Do(reqStore)
 	if err != nil {
 		return err
@@ -105,6 +114,10 @@ func (e SteamStruct) load() error {
 			return err
 		}
 
+		for key, value := range e.headers {
+			reqApi.Header.Set(key, value)
+		}
+
 		resApi, err := client.Do(reqApi)
 		if err != nil {
 			return err

cmd/dealsbot/ubisoft.go

@@ -26,8 +26,8 @@ func newUbsioftApi() UbisoftStruct {
 	ubisoft.headers["referer"] = "https://free.ubisoft.com/"
 	ubisoft.headers["origin"] = "https://free.ubisoft.com"
 	ubisoft.headers["ubi-localecode"] = "en-US"
+	ubisoft.headers["Accept-Language"] = "en"
 	ubisoft.headers["User-Agent"] = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:93.0) Gecko/20100101 Firefox/93.0"
-
 	return ubisoft
 }
 
@@ -89,6 +89,7 @@ func (e UbisoftStruct) load() error {
 	for key, value := range e.headers {
 		req.Header.Set(key, value)
 	}
+
 	req.Header.Set("ubi-appid", appId)
 	res, err := client.Do(req)
 	if err != nil {