feature: switch dealsbot to sqlite kv and remove defunct ubisoft api
This commit is contained in:
Seraphim Strub
parent
759aae3b54
commit
50cd000ee5
4 changed files with 35 additions and 293 deletions
|
|
@ -23,7 +23,7 @@ type SteamStruct struct {
|
|||
|
||||
func NewSteamApi(logger *slog.Logger) SteamStruct {
|
||||
steam := SteamStruct{
|
||||
url: "https://store.steampowered.com/search/results?force_infinite=1&maxprice=free&specials=1",
|
||||
url: "https://store.steampowered.com/search/results?force_infinite=1&maxprice=free&specials=1&category1=998",
|
||||
baseUrl: "https://store.steampowered.com/app/",
|
||||
apiUrl: "https://store.steampowered.com/api/appdetails?appids=",
|
||||
idPrefix: "steam-",
|
||||
|
|
|
|||
|
|
@ -1,151 +0,0 @@
|
|||
package api
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type UbisoftStruct struct {
|
||||
url string
|
||||
idPrefix string
|
||||
headers map[string]string
|
||||
deals DealsMap
|
||||
logger *slog.Logger
|
||||
}
|
||||
|
||||
func NewUbsioftApi(logger *slog.Logger) UbisoftStruct {
|
||||
ubisoft := UbisoftStruct{
|
||||
url: "https://free.ubisoft.com/configuration.js",
|
||||
idPrefix: "ubisoft-",
|
||||
headers: make(map[string]string),
|
||||
deals: make(map[string]Deal),
|
||||
logger: logger,
|
||||
}
|
||||
ubisoft.headers["referer"] = "https://free.ubisoft.com/"
|
||||
ubisoft.headers["origin"] = "https://free.ubisoft.com"
|
||||
ubisoft.headers["ubi-localecode"] = "en-US"
|
||||
ubisoft.headers["Accept-Language"] = "en"
|
||||
ubisoft.headers["User-Agent"] = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:93.0) Gecko/20100101 Firefox/93.0"
|
||||
return ubisoft
|
||||
}
|
||||
|
||||
type ubisoftApiBody struct {
|
||||
News []struct {
|
||||
Type string `json:"type"`
|
||||
Title string `json:"title"`
|
||||
MediaUrl string `json:"mediaURL"`
|
||||
Links []struct {
|
||||
Param string `json:"param"`
|
||||
} `json:"links"`
|
||||
} `json:"news"`
|
||||
}
|
||||
|
||||
func (e UbisoftStruct) Load() error {
|
||||
|
||||
appId, prodUrl, err := func() (string, string, error) {
|
||||
client := &http.Client{}
|
||||
req, err := http.NewRequest("GET", e.url, nil)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
|
||||
for key, value := range e.headers {
|
||||
req.Header.Set(key, value)
|
||||
}
|
||||
res, err := client.Do(req)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
body, err := io.ReadAll(res.Body)
|
||||
regexAppId, err := regexp.Compile(`appId:\s*'(.+)'`)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
regexProd, err := regexp.Compile(`prod:\s*'(.+)'`)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
appId := regexAppId.FindSubmatch(body)
|
||||
prodUrl := regexProd.FindSubmatch(body)
|
||||
|
||||
if len(appId) < 1 || len(prodUrl) < 1 {
|
||||
return "", "", errors.New("appid or prod url not found")
|
||||
}
|
||||
|
||||
return string(appId[1]), string(prodUrl[1]), nil
|
||||
}()
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
client := &http.Client{}
|
||||
req, err := http.NewRequest("GET", prodUrl, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for key, value := range e.headers {
|
||||
req.Header.Set(key, value)
|
||||
}
|
||||
|
||||
req.Header.Set("ubi-appid", appId)
|
||||
res, err := client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
body, err := io.ReadAll(res.Body)
|
||||
|
||||
var data ubisoftApiBody
|
||||
err = json.Unmarshal(body, &data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, news := range data.News {
|
||||
if news.Type != "freegame" {
|
||||
continue
|
||||
}
|
||||
|
||||
title := strings.TrimSpace(news.Title)
|
||||
if len(news.Links) != 1 {
|
||||
return errors.New(fmt.Sprintf("lenght of links for %v is more or less then 1", news.Title))
|
||||
}
|
||||
|
||||
regexName, err := regexp.Compile(`https://register.ubisoft.com/([^/]*)/?`)
|
||||
idFromUrl := regexName.FindStringSubmatch(news.Links[0].Param)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(idFromUrl) < 1 {
|
||||
return errors.New("could not parse url")
|
||||
}
|
||||
|
||||
id := fmt.Sprintf("%v%v", e.idPrefix, idFromUrl[1])
|
||||
url := news.Links[0].Param
|
||||
image := news.MediaUrl
|
||||
|
||||
e.deals[id] = Deal{
|
||||
Id: id,
|
||||
Title: title,
|
||||
Url: url,
|
||||
Image: image,
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e UbisoftStruct) Get() []Deal {
|
||||
var deals []Deal
|
||||
for _, deal := range e.deals {
|
||||
deals = append(deals, deal)
|
||||
}
|
||||
return deals
|
||||
}
|
||||
|
|
@ -2,16 +2,18 @@ package main
|
|||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/dgraph-io/badger/v4"
|
||||
"github.com/disgoorg/disgo"
|
||||
"github.com/disgoorg/disgo/discord"
|
||||
"github.com/disgoorg/disgo/rest"
|
||||
"github.com/disgoorg/disgo/webhook"
|
||||
"github.com/disgoorg/snowflake/v2"
|
||||
"grow.rievo.dev/discordBots"
|
||||
"grow.rievo.dev/discordBots/cmd/dealsbot/api"
|
||||
"grow.rievo.dev/discordBots/cmd/dealsbot/repository"
|
||||
"grow.rievo.dev/discordBots/pkg/db"
|
||||
"log/slog"
|
||||
"os"
|
||||
"os/signal"
|
||||
|
|
@ -19,6 +21,8 @@ import (
|
|||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
var (
|
||||
|
|
@ -39,18 +43,30 @@ func main() {
|
|||
// - https://github.com/TheLovinator1/discord-free-game-notifier
|
||||
// - https://gg.deals/news/free-gog-games/
|
||||
// - origin
|
||||
// - check ubisoft works
|
||||
// - need new ubisoft api if it even exists
|
||||
// - add more supported types for steam (bundles,software,dlc)
|
||||
|
||||
logger.Info("starting dealsbot...", slog.String("disgo version", disgo.Version))
|
||||
|
||||
client := webhook.New(webhookID, webhookToken)
|
||||
defer client.Close(context.TODO())
|
||||
|
||||
repo := repository.InitDb(logger)
|
||||
defer repo.Close()
|
||||
ctx := context.Background()
|
||||
con, err := sql.Open("sqlite", "file:db/deals.db?cache=shared")
|
||||
if err != nil {
|
||||
logger.Error("error opening db", slog.Any("error", err))
|
||||
panic(err)
|
||||
}
|
||||
defer con.Close()
|
||||
|
||||
ticker := time.NewTicker(10 * time.Minute)
|
||||
tickerGC := time.NewTicker(15 * time.Minute)
|
||||
// create tables
|
||||
if _, err := con.ExecContext(ctx, discordBots.Schema); err != nil {
|
||||
logger.Error("error creating db schema", slog.Any("error", err))
|
||||
panic(err)
|
||||
}
|
||||
query := db.New(con)
|
||||
|
||||
ticker := time.NewTicker(1 * time.Minute)
|
||||
quit := make(chan struct{})
|
||||
|
||||
go func() {
|
||||
|
|
@ -59,7 +75,6 @@ func main() {
|
|||
case <-ticker.C:
|
||||
var apis []api.Api
|
||||
apis = append(apis,
|
||||
api.NewUbsioftApi(logger),
|
||||
api.NewEpicApi(logger),
|
||||
api.NewSteamApi(logger),
|
||||
api.NewGogFrontApi(logger),
|
||||
|
|
@ -79,7 +94,11 @@ func main() {
|
|||
}
|
||||
|
||||
for _, deal := range deals {
|
||||
retrievedDeal, _ := repo.GetValue(deal.Id)
|
||||
retrievedDealJson, dbErr := query.GetItem(ctx, deal.Id)
|
||||
retrievedDeal := api.Deal{}
|
||||
if err := json.Unmarshal(retrievedDealJson.Data, &retrievedDeal); !errors.Is(dbErr, sql.ErrNoRows) && err != nil {
|
||||
logger.Error("failed unmarshalling deal", slog.Any("error", err))
|
||||
}
|
||||
|
||||
if deal.Id == retrievedDeal.Id {
|
||||
logger.Debug("deal is already published", slog.String("deal", deal.Id))
|
||||
|
|
@ -88,23 +107,20 @@ func main() {
|
|||
} else {
|
||||
logger.Info("deal is new and will be published", slog.String("deal", deal.Id))
|
||||
go sendWebhook(client, deal)
|
||||
err := repo.SetValue(deal)
|
||||
dealJson, _ := json.Marshal(deal)
|
||||
err = query.CreateItem(ctx, db.CreateItemParams{
|
||||
ID: deal.Id,
|
||||
Data: dealJson,
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
logger.Error("failed saving deal", slog.Any("error", err))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
case <-tickerGC.C:
|
||||
err := repo.RunGC()
|
||||
if err != nil && !errors.Is(err, badger.ErrNoRewrite) {
|
||||
logger.Error("GC failed", slog.Any("error", err))
|
||||
} else {
|
||||
logger.Debug("GC successful")
|
||||
}
|
||||
case <-quit:
|
||||
ticker.Stop()
|
||||
tickerGC.Stop()
|
||||
return
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,123 +0,0 @@
|
|||
package repository
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"github.com/dgraph-io/badger/v4"
|
||||
"grow.rievo.dev/discordBots/cmd/dealsbot/api"
|
||||
"log/slog"
|
||||
)
|
||||
|
||||
type Repository interface {
|
||||
GetAll() ([]api.Deal, error)
|
||||
GetValue(dealId string) api.Deal
|
||||
SetValue(deal api.Deal) error
|
||||
DeleteValue(dealId string) error
|
||||
Close() error
|
||||
}
|
||||
|
||||
type DealRepository struct {
|
||||
db *badger.DB
|
||||
logger *slog.Logger
|
||||
}
|
||||
|
||||
func InitDb(logger *slog.Logger) *DealRepository {
|
||||
opts := badger.DefaultOptions("./db")
|
||||
opts.Logger = nil
|
||||
db, err := badger.Open(opts)
|
||||
if err != nil {
|
||||
logger.Error("error opening DB", slog.Any("error", err))
|
||||
}
|
||||
return &DealRepository{db, logger}
|
||||
}
|
||||
|
||||
func (d *DealRepository) Close() error {
|
||||
return d.db.Close()
|
||||
}
|
||||
|
||||
func (d *DealRepository) RunGC() error {
|
||||
return d.db.RunValueLogGC(0.7)
|
||||
}
|
||||
|
||||
func (d *DealRepository) GetAll() ([]api.Deal, error) {
|
||||
var deals []api.Deal
|
||||
err := d.db.View(func(txn *badger.Txn) error {
|
||||
opts := badger.DefaultIteratorOptions
|
||||
opts.PrefetchSize = 10
|
||||
it := txn.NewIterator(opts)
|
||||
defer it.Close()
|
||||
for it.Rewind(); it.Valid(); it.Next() {
|
||||
item := it.Item()
|
||||
err := item.Value(func(val []byte) error {
|
||||
retrievedDeal := api.Deal{}
|
||||
err := json.Unmarshal(val, &retrievedDeal)
|
||||
deals = append(deals, retrievedDeal)
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return deals, err
|
||||
}
|
||||
|
||||
func (d *DealRepository) GetValue(dealId string) (api.Deal, error) {
|
||||
retrievedDeal := api.Deal{}
|
||||
err := d.db.View(func(txn *badger.Txn) error {
|
||||
item, err := txn.Get([]byte(dealId))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = item.Value(func(val []byte) error {
|
||||
err = json.Unmarshal(val, &retrievedDeal)
|
||||
return err
|
||||
})
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
return api.Deal{}, err
|
||||
}
|
||||
return retrievedDeal, nil
|
||||
}
|
||||
|
||||
func (d *DealRepository) SetValue(deal api.Deal) error {
|
||||
jsonBytes, err := json.Marshal(deal)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = d.db.Update(func(txn *badger.Txn) error {
|
||||
err := txn.Set([]byte(deal.Id), jsonBytes)
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *DealRepository) DeleteValue(dealId string) error {
|
||||
err := d.db.Update(func(txn *badger.Txn) error {
|
||||
err := txn.Delete([]byte(dealId))
|
||||
return err
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
func (d *DealRepository) DeleteAll() error {
|
||||
err := d.db.Update(func(txn *badger.Txn) error {
|
||||
opts := badger.DefaultIteratorOptions
|
||||
opts.PrefetchSize = 10
|
||||
it := txn.NewIterator(opts)
|
||||
defer it.Close()
|
||||
for it.Rewind(); it.Valid(); it.Next() {
|
||||
item := it.Item()
|
||||
err := txn.Delete(item.Key())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return err
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue