feature: switch dealsbot to sqlite kv and remove defunct ubisoft api

cmd/dealsbot/api/steam.go

@@ -23,7 +23,7 @@ type SteamStruct struct {
 
 func NewSteamApi(logger *slog.Logger) SteamStruct {
 	steam := SteamStruct{
-		url:      "https://store.steampowered.com/search/results?force_infinite=1&maxprice=free&specials=1",
+		url:      "https://store.steampowered.com/search/results?force_infinite=1&maxprice=free&specials=1&category1=998",
 		baseUrl:  "https://store.steampowered.com/app/",
 		apiUrl:   "https://store.steampowered.com/api/appdetails?appids=",
 		idPrefix: "steam-",

cmd/dealsbot/api/ubisoft.go

@@ -1,151 +0,0 @@
-package api
-
-import (
-	"encoding/json"
-	"errors"
-	"fmt"
-	"io"
-	"log/slog"
-	"net/http"
-	"regexp"
-	"strings"
-)
-
-type UbisoftStruct struct {
-	url      string
-	idPrefix string
-	headers  map[string]string
-	deals    DealsMap
-	logger   *slog.Logger
-}
-
-func NewUbsioftApi(logger *slog.Logger) UbisoftStruct {
-	ubisoft := UbisoftStruct{
-		url:      "https://free.ubisoft.com/configuration.js",
-		idPrefix: "ubisoft-",
-		headers:  make(map[string]string),
-		deals:    make(map[string]Deal),
-		logger:   logger,
-	}
-	ubisoft.headers["referer"] = "https://free.ubisoft.com/"
-	ubisoft.headers["origin"] = "https://free.ubisoft.com"
-	ubisoft.headers["ubi-localecode"] = "en-US"
-	ubisoft.headers["Accept-Language"] = "en"
-	ubisoft.headers["User-Agent"] = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:93.0) Gecko/20100101 Firefox/93.0"
-	return ubisoft
-}
-
-type ubisoftApiBody struct {
-	News []struct {
-		Type     string `json:"type"`
-		Title    string `json:"title"`
-		MediaUrl string `json:"mediaURL"`
-		Links    []struct {
-			Param string `json:"param"`
-		} `json:"links"`
-	} `json:"news"`
-}
-
-func (e UbisoftStruct) Load() error {
-
-	appId, prodUrl, err := func() (string, string, error) {
-		client := &http.Client{}
-		req, err := http.NewRequest("GET", e.url, nil)
-		if err != nil {
-			return "", "", err
-		}
-
-		for key, value := range e.headers {
-			req.Header.Set(key, value)
-		}
-		res, err := client.Do(req)
-		if err != nil {
-			return "", "", err
-		}
-		body, err := io.ReadAll(res.Body)
-		regexAppId, err := regexp.Compile(`appId:\s*'(.+)'`)
-		if err != nil {
-			return "", "", err
-		}
-		regexProd, err := regexp.Compile(`prod:\s*'(.+)'`)
-		if err != nil {
-			return "", "", err
-		}
-		appId := regexAppId.FindSubmatch(body)
-		prodUrl := regexProd.FindSubmatch(body)
-
-		if len(appId) < 1 || len(prodUrl) < 1 {
-			return "", "", errors.New("appid or prod url not found")
-		}
-
-		return string(appId[1]), string(prodUrl[1]), nil
-	}()
-
-	if err != nil {
-		return err
-	}
-
-	client := &http.Client{}
-	req, err := http.NewRequest("GET", prodUrl, nil)
-	if err != nil {
-		return err
-	}
-
-	for key, value := range e.headers {
-		req.Header.Set(key, value)
-	}
-
-	req.Header.Set("ubi-appid", appId)
-	res, err := client.Do(req)
-	if err != nil {
-		return err
-	}
-	body, err := io.ReadAll(res.Body)
-
-	var data ubisoftApiBody
-	err = json.Unmarshal(body, &data)
-	if err != nil {
-		return err
-	}
-
-	for _, news := range data.News {
-		if news.Type != "freegame" {
-			continue
-		}
-
-		title := strings.TrimSpace(news.Title)
-		if len(news.Links) != 1 {
-			return errors.New(fmt.Sprintf("lenght of links for %v is more or less then 1", news.Title))
-		}
-
-		regexName, err := regexp.Compile(`https://register.ubisoft.com/([^/]*)/?`)
-		idFromUrl := regexName.FindStringSubmatch(news.Links[0].Param)
-		if err != nil {
-			return err
-		}
-		if len(idFromUrl) < 1 {
-			return errors.New("could not parse url")
-		}
-
-		id := fmt.Sprintf("%v%v", e.idPrefix, idFromUrl[1])
-		url := news.Links[0].Param
-		image := news.MediaUrl
-
-		e.deals[id] = Deal{
-			Id:    id,
-			Title: title,
-			Url:   url,
-			Image: image,
-		}
-	}
-
-	return nil
-}
-
-func (e UbisoftStruct) Get() []Deal {
-	var deals []Deal
-	for _, deal := range e.deals {
-		deals = append(deals, deal)
-	}
-	return deals
-}

cmd/dealsbot/main.go

@@ -2,16 +2,18 @@ package main
 
 import (
 	"context"
+	"database/sql"
+	"encoding/json"
 	"errors"
 	"fmt"
-	"github.com/dgraph-io/badger/v4"
 	"github.com/disgoorg/disgo"
 	"github.com/disgoorg/disgo/discord"
 	"github.com/disgoorg/disgo/rest"
 	"github.com/disgoorg/disgo/webhook"
 	"github.com/disgoorg/snowflake/v2"
+	"grow.rievo.dev/discordBots"
 	"grow.rievo.dev/discordBots/cmd/dealsbot/api"
-	"grow.rievo.dev/discordBots/cmd/dealsbot/repository"
+	"grow.rievo.dev/discordBots/pkg/db"
 	"log/slog"
 	"os"
 	"os/signal"
@@ -19,6 +21,8 @@ import (
 	"strings"
 	"syscall"
 	"time"
+
+	_ "modernc.org/sqlite"
 )
 
 var (
@@ -39,18 +43,30 @@ func main() {
 	// - https://github.com/TheLovinator1/discord-free-game-notifier
 	// - https://gg.deals/news/free-gog-games/
 	// - origin
-	// - check ubisoft works
+	// - need new ubisoft api if it even exists
+	// - add more supported types for steam (bundles,software,dlc)
 
 	logger.Info("starting dealsbot...", slog.String("disgo version", disgo.Version))
 
 	client := webhook.New(webhookID, webhookToken)
 	defer client.Close(context.TODO())
 
-	repo := repository.InitDb(logger)
+	ctx := context.Background()
-	defer repo.Close()
+	con, err := sql.Open("sqlite", "file:db/deals.db?cache=shared")
+	if err != nil {
+		logger.Error("error opening db", slog.Any("error", err))
+		panic(err)
+	}
+	defer con.Close()
 
-	ticker := time.NewTicker(10 * time.Minute)
+	// create tables
-	tickerGC := time.NewTicker(15 * time.Minute)
+	if _, err := con.ExecContext(ctx, discordBots.Schema); err != nil {
+		logger.Error("error creating db schema", slog.Any("error", err))
+		panic(err)
+	}
+	query := db.New(con)
+
+	ticker := time.NewTicker(1 * time.Minute)
 	quit := make(chan struct{})
 
 	go func() {
@@ -59,7 +75,6 @@ func main() {
 			case <-ticker.C:
 				var apis []api.Api
 				apis = append(apis,
-					api.NewUbsioftApi(logger),
 					api.NewEpicApi(logger),
 					api.NewSteamApi(logger),
 					api.NewGogFrontApi(logger),
@@ -79,7 +94,11 @@ func main() {
 				}
 
 				for _, deal := range deals {
-					retrievedDeal, _ := repo.GetValue(deal.Id)
+					retrievedDealJson, dbErr := query.GetItem(ctx, deal.Id)
+					retrievedDeal := api.Deal{}
+					if err := json.Unmarshal(retrievedDealJson.Data, &retrievedDeal); !errors.Is(dbErr, sql.ErrNoRows) && err != nil {
+						logger.Error("failed unmarshalling deal", slog.Any("error", err))
+					}
 
 					if deal.Id == retrievedDeal.Id {
 						logger.Debug("deal is already published", slog.String("deal", deal.Id))
@@ -88,23 +107,20 @@ func main() {
 					} else {
 						logger.Info("deal is new and will be published", slog.String("deal", deal.Id))
 						go sendWebhook(client, deal)
-						err := repo.SetValue(deal)
+						dealJson, _ := json.Marshal(deal)
+						err = query.CreateItem(ctx, db.CreateItemParams{
+							ID:   deal.Id,
+							Data: dealJson,
+						})
+
 						if err != nil {
 							logger.Error("failed saving deal", slog.Any("error", err))
 						}
 					}
 				}
 
-			case <-tickerGC.C:
-				err := repo.RunGC()
-				if err != nil && !errors.Is(err, badger.ErrNoRewrite) {
-					logger.Error("GC failed", slog.Any("error", err))
-				} else {
-					logger.Debug("GC successful")
-				}
 			case <-quit:
 				ticker.Stop()
-				tickerGC.Stop()
 				return
 			}
 		}

cmd/dealsbot/repository/repository.go

@@ -1,123 +0,0 @@
-package repository
-
-import (
-	"encoding/json"
-	"github.com/dgraph-io/badger/v4"
-	"grow.rievo.dev/discordBots/cmd/dealsbot/api"
-	"log/slog"
-)
-
-type Repository interface {
-	GetAll() ([]api.Deal, error)
-	GetValue(dealId string) api.Deal
-	SetValue(deal api.Deal) error
-	DeleteValue(dealId string) error
-	Close() error
-}
-
-type DealRepository struct {
-	db     *badger.DB
-	logger *slog.Logger
-}
-
-func InitDb(logger *slog.Logger) *DealRepository {
-	opts := badger.DefaultOptions("./db")
-	opts.Logger = nil
-	db, err := badger.Open(opts)
-	if err != nil {
-		logger.Error("error opening DB", slog.Any("error", err))
-	}
-	return &DealRepository{db, logger}
-}
-
-func (d *DealRepository) Close() error {
-	return d.db.Close()
-}
-
-func (d *DealRepository) RunGC() error {
-	return d.db.RunValueLogGC(0.7)
-}
-
-func (d *DealRepository) GetAll() ([]api.Deal, error) {
-	var deals []api.Deal
-	err := d.db.View(func(txn *badger.Txn) error {
-		opts := badger.DefaultIteratorOptions
-		opts.PrefetchSize = 10
-		it := txn.NewIterator(opts)
-		defer it.Close()
-		for it.Rewind(); it.Valid(); it.Next() {
-			item := it.Item()
-			err := item.Value(func(val []byte) error {
-				retrievedDeal := api.Deal{}
-				err := json.Unmarshal(val, &retrievedDeal)
-				deals = append(deals, retrievedDeal)
-				return err
-			})
-			if err != nil {
-				return err
-			}
-		}
-		return nil
-	})
-	return deals, err
-}
-
-func (d *DealRepository) GetValue(dealId string) (api.Deal, error) {
-	retrievedDeal := api.Deal{}
-	err := d.db.View(func(txn *badger.Txn) error {
-		item, err := txn.Get([]byte(dealId))
-		if err != nil {
-			return err
-		}
-		err = item.Value(func(val []byte) error {
-			err = json.Unmarshal(val, &retrievedDeal)
-			return err
-		})
-		return err
-	})
-	if err != nil {
-		return api.Deal{}, err
-	}
-	return retrievedDeal, nil
-}
-
-func (d *DealRepository) SetValue(deal api.Deal) error {
-	jsonBytes, err := json.Marshal(deal)
-	if err != nil {
-		return err
-	}
-	err = d.db.Update(func(txn *badger.Txn) error {
-		err := txn.Set([]byte(deal.Id), jsonBytes)
-		return err
-	})
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-func (d *DealRepository) DeleteValue(dealId string) error {
-	err := d.db.Update(func(txn *badger.Txn) error {
-		err := txn.Delete([]byte(dealId))
-		return err
-	})
-	return err
-}
-
-func (d *DealRepository) DeleteAll() error {
-	err := d.db.Update(func(txn *badger.Txn) error {
-		opts := badger.DefaultIteratorOptions
-		opts.PrefetchSize = 10
-		it := txn.NewIterator(opts)
-		defer it.Close()
-		for it.Rewind(); it.Valid(); it.Next() {
-			item := it.Item()
-			err := txn.Delete(item.Key())
-			if err != nil {
-				return err
-			}
-		}
-		return nil
-	})
-	return err
-}