RievoGitAdmin/discordBots
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

adds header to all crawlers

Browse source
This commit is contained in:
Seraphim Strub 2023-03-06 19:02:40 +01:00
parent fcff591655
commit d8793a1b9f
6 changed files with 58 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
cmd/dealsbot/epic.go
View file

@ -11,16 +11,23 @@ import (
type EpicStruct struct {
url string
baseUrl string
idPrefix string
headers map[string]string
deals DealsMap
}
func newEpicApi() EpicStruct {
return EpicStruct{
epic := EpicStruct{
url: "https://store-site-backend-static-ipv4.ak.epicgames.com/freeGamesPromotions",
baseUrl: "https://store.epicgames.com/p/",
idPrefix: "epic-",
headers: make(map[string]string),
deals: make(map[string]Deal),
}
epic.headers["Accept-Language"] = "en"
epic.headers["User-Agent"] = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:93.0) Gecko/20100101 Firefox/93.0"
return epic
}
type epicApiBody struct {
@ -75,6 +82,10 @@ func (e EpicStruct) load() error {
return err
}
for key, value := range e.headers {
req.Header.Set(key, value)
}
res, err := client.Do(req)
if err != nil {
return err
@ -108,7 +119,7 @@ func (e EpicStruct) load() error {
id := fmt.Sprintf("%v%v", e.idPrefix, element.Id)
title := element.Title
url := fmt.Sprintf("https://store.epicgames.com/en-US/p/%v", productSlug)
url := fmt.Sprintf("%v%v", e.baseUrl, productSlug)
e.deals[id] = Deal{
Id: id,

21
cmd/dealsbot/gog.go
View file

@ -11,16 +11,21 @@ type GogStruct struct {
url string
baseUrl string
idPrefix string
headers map[string]string
deals DealsMap
}
func newGogApi() GogStruct {
return GogStruct{
url: "https://www.gog.com/en",
baseUrl: "https://www.gog.com/en/game/",
gog := GogStruct{
url: "https://www.gog.com/",
baseUrl: "https://www.gog.com/game/",
idPrefix: "gog-",
headers: make(map[string]string),
deals: make(map[string]Deal),
}
gog.headers["Accept-Language"] = "en"
gog.headers["User-Agent"] = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:93.0) Gecko/20100101 Firefox/93.0"
return gog
}
func (e GogStruct) load() error {
@ -32,13 +37,17 @@ func (e GogStruct) load() error {
return err
}
for key, value := range e.headers {
reqStore.Header.Set(key, value)
}
resStore, err := client.Do(reqStore)
if err != nil {
return err
}
bodyStore := html.NewTokenizer(resStore.Body)
regexAppid, err := regexp.Compile(`/en/game/([-\w]+)`)
regexAppid, err := regexp.Compile(`/\w{2}/game/([-\w]+)`)
if err != nil {
return err
}
@ -82,6 +91,10 @@ func (e GogStruct) load() error {
return err
}
for key, value := range e.headers {
reqGame.Header.Set(key, value)
}
resGame, err := client.Do(reqGame)
if err != nil {
return err

12
cmd/dealsbot/humblebundle.go
View file

@ -12,16 +12,22 @@ type HumbleBundleStruct struct {
url string
baseUrl string
idPrefix string
headers map[string]string
deals DealsMap
}
func newHumbleBundleApi() HumbleBundleStruct {
return HumbleBundleStruct{
humbleBundle := HumbleBundleStruct{
url: "https://www.humblebundle.com/",
baseUrl: "https://www.humblebundle.com/store/",
idPrefix: "humblebundle-",
headers: make(map[string]string),
deals: make(map[string]Deal),
}
humbleBundle.headers["Accept-Language"] = "en"
humbleBundle.headers["User-Agent"] = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:93.0) Gecko/20100101 Firefox/93.0"
return humbleBundle
}
type humblebundleJsonBody struct {
@ -49,6 +55,10 @@ func (e HumbleBundleStruct) load() error {
return err
}
for key, value := range e.headers {
reqStore.Header.Set(key, value)
}
resStore, err := client.Do(reqStore)
if err != nil {
return err

5
cmd/dealsbot/main.go
View file

@ -24,20 +24,17 @@ var (
)
func main() {
// translate this to go: https://dev.rievo.net/sst/feed-python
// query different sources store to db
// try to incorporate operagx apiUrl:
// - https://gx-proxy.operacdn.com/content/free-games?_limit=300&_sort=order%3AASC
// send messages to discord
// ideas:
// - https://github.com/TheLovinator1/discord-free-game-notifier
// - https://gg.deals/games/free-games/
// - https://gg.deals/news/free-gog-games/
// - origin
// - check ubisoft works
log.SetLevel(log.LevelDebug)
log.SetLevel(log.LevelInfo)
log.Info("starting dealsbot...")
log.Info("disgo version: ", disgo.Version)

15
cmd/dealsbot/steam.go
View file

@ -14,17 +14,22 @@ type SteamStruct struct {
baseUrl string
apiUrl string
idPrefix string
headers map[string]string
deals DealsMap
}
func newSteamApi() SteamStruct {
return SteamStruct{
steam := SteamStruct{
url: "https://store.steampowered.com/search/results?force_infinite=1&maxprice=free&specials=1",
baseUrl: "https://store.steampowered.com/app/",
apiUrl: "https://store.steampowered.com/api/appdetails?appids=",
idPrefix: "steam-",
headers: make(map[string]string),
deals: make(map[string]Deal),
}
steam.headers["Accept-Language"] = "en"
steam.headers["User-Agent"] = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:93.0) Gecko/20100101 Firefox/93.0"
return steam
}
type steamApiBodyGame struct {
@ -58,6 +63,10 @@ func (e SteamStruct) load() error {
return err
}
for key, value := range e.headers {
reqStore.Header.Set(key, value)
}
resStore, err := client.Do(reqStore)
if err != nil {
return err
@ -105,6 +114,10 @@ func (e SteamStruct) load() error {
return err
}
for key, value := range e.headers {
reqApi.Header.Set(key, value)
}
resApi, err := client.Do(reqApi)
if err != nil {
return err

3
cmd/dealsbot/ubisoft.go
View file

@ -26,8 +26,8 @@ func newUbsioftApi() UbisoftStruct {
ubisoft.headers["referer"] = "https://free.ubisoft.com/"
ubisoft.headers["origin"] = "https://free.ubisoft.com"
ubisoft.headers["ubi-localecode"] = "en-US"
ubisoft.headers["Accept-Language"] = "en"
ubisoft.headers["User-Agent"] = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:93.0) Gecko/20100101 Firefox/93.0"
return ubisoft
}
@ -89,6 +89,7 @@ func (e UbisoftStruct) load() error {
for key, value := range e.headers {
req.Header.Set(key, value)
}
req.Header.Set("ubi-appid", appId)
res, err := client.Do(req)
if err != nil {