v1.5 (#9)

Dockerfile

@@ -1,10 +1,9 @@
-FROM golang:1.21rc2-alpine3.18 AS build
+FROM golang:1.21-alpine AS build
 RUN apk add --no-cache git
 WORKDIR /src/bdo-rest-api
 COPY . .
 RUN go mod download
-ARG tags=none
-RUN go build -tags $tags -o /bdo-rest-api -ldflags="-s -w" .
+RUN go build -o /bdo-rest-api -ldflags="-s -w" .
 
 FROM alpine:3.14 AS bin
 RUN addgroup --system --gid 1001 go

README.md

@@ -10,7 +10,7 @@ A scraper for Black Desert Online player in-game data with a REST API. It curren
 There are two ways to use this scraper for your needs:
 * By querying https://bdo.hemlo.cc/communityapi/v1 — this is the "official" instance hosted by me.
 * If you want to have more control over the API, host the scraper yourself using one of the following methods:
-  - As a Docker container: the image is available on [DockerHub](https://hub.docker.com/r/man90/bdo-rest-api) or you can build it yourself following instructions [here](docs/buildingDocker.md).
+  - As a Docker container: the image is available on [DockerHub](https://hub.docker.com/r/man90/bdo-rest-api).
   - Natively: build the binary from source as described in [this guide](docs/buildingFromSource.md).
 
 API documentation can be viewed [here](https://man90es.github.io/BDO-REST-API/).
@@ -19,14 +19,14 @@ API documentation can be viewed [here](https://man90es.github.io/BDO-REST-API/).
 If you host the API yourself, either via Docker or natively, you can control some of its features by executing it with flags.
 
 Available flags:
-- `-cachecap`
-	- Allows to specify response cache capacity
-	- Type: unsigned integer
-	- Default value: `10000`
 - `-cachettl`
 	- Allows to specify cache TTL in minutes
 	- Type: unsigned integer
 	- Default value: `180`
+- `-maintenancettl`
+	- Allows to limit how frequently scraper can check for maintenance end in minutes
+	- Type: unsigned integer
+	- Default value: `5`
 - `-port`
 	- Allows to specify API server's port
 	- Type: unsigned integer

cache/cache.go

@@ -0,0 +1,66 @@
+package cache
+
+import (
+	"strings"
+	"time"
+
+	goCache "github.com/patrickmn/go-cache"
+
+	"bdo-rest-api/config"
+	"bdo-rest-api/utils"
+)
+
+type cacheEntry[T any] struct {
+	data   T
+	date   time.Time
+	status int
+}
+
+type cache[T any] struct {
+	internalCache *goCache.Cache
+}
+
+func joinKeys(keys []string) string {
+	return strings.Join(keys, ",")
+}
+
+func NewCache[T any]() *cache[T] {
+	cacheTTL := config.GetCacheTTL()
+
+	return &cache[T]{
+		internalCache: goCache.New(cacheTTL, time.Hour),
+	}
+}
+
+func (c *cache[T]) AddRecord(keys []string, data T, status int) (date string, expires string) {
+	cacheTTL := config.GetCacheTTL()
+	entry := cacheEntry[T]{
+		data:   data,
+		date:   time.Now(),
+		status: status,
+	}
+
+	c.internalCache.Add(joinKeys(keys), entry, cacheTTL)
+	expirationDate := entry.date.Add(cacheTTL)
+
+	return utils.FormatDateForHeaders(entry.date), utils.FormatDateForHeaders(expirationDate)
+}
+
+func (c *cache[T]) GetRecord(keys []string) (data T, status int, date string, expires string, found bool) {
+	var anyEntry interface{}
+	var expirationDate time.Time
+
+	anyEntry, expirationDate, found = c.internalCache.GetWithExpiration(joinKeys(keys))
+
+	if !found {
+		return
+	}
+
+	entry := anyEntry.(cacheEntry[T])
+
+	return entry.data, entry.status, utils.FormatDateForHeaders(entry.date), utils.FormatDateForHeaders(expirationDate), found
+}
+
+func (c *cache[T]) GetItemCount() int {
+	return c.internalCache.ItemCount()
+}

cache/cache_test.go

@@ -0,0 +1,63 @@
+package cache
+
+import (
+	"testing"
+	"time"
+
+	"bdo-rest-api/config"
+)
+
+func init() {
+	config.SetCacheTTL(time.Second)
+}
+
+func TestCache(t *testing.T) {
+	// Create a cache instance for testing
+	testCache := NewCache[string]()
+
+	// Test AddRecord and GetRecord
+	keys := []string{"key1", "key2"}
+	data := "test data"
+	status := 200
+
+	date, expires := testCache.AddRecord(keys, data, status)
+
+	// Validate AddRecord results
+	if date == "" || expires == "" {
+		t.Error("AddRecord should return non-empty date and expires values")
+	}
+
+	// Test GetRecord for an existing record
+	returnedData, returnedStatus, returnedDate, returnedExpires, found := testCache.GetRecord(keys)
+
+	if !found {
+		t.Error("GetRecord should find the record")
+	}
+
+	// Validate GetRecord results
+	if returnedData != data || returnedStatus != status || returnedDate == "" || returnedExpires == "" {
+		t.Error("GetRecord returned unexpected values")
+	}
+
+	// Test GetItemCount
+	itemCount := testCache.GetItemCount()
+	if itemCount != 1 {
+		t.Errorf("GetItemCount should return 1, but got %d", itemCount)
+	}
+
+	// Sleep for a while to allow the cache entry to expire
+	time.Sleep(2 * time.Second)
+
+	// Test GetRecord for an expired record
+	_, _, _, _, found = testCache.GetRecord(keys)
+
+	if found {
+		t.Error("GetRecord should not find an expired record")
+	}
+
+	// Test GetItemCount after expiration
+	itemCount = testCache.GetItemCount()
+	if itemCount != 0 {
+		t.Errorf("GetItemCount should return 0 after expiration, but got %d", itemCount)
+	}
+}

config/config.go

@@ -0,0 +1,112 @@
+package config
+
+import (
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/gocolly/colly/v2"
+	"github.com/gocolly/colly/v2/proxy"
+)
+
+type config struct {
+	mu             sync.RWMutex
+	cacheTTL       time.Duration
+	maintenanceTTL time.Duration
+	port           int
+	proxyList      []string
+	proxySwitcher  colly.ProxyFunc
+	verbosity      bool
+}
+
+var instance *config
+var once sync.Once
+
+func getInstance() *config {
+	once.Do(func() {
+		instance = &config{
+			cacheTTL:       3 * time.Hour,
+			maintenanceTTL: 5 * time.Minute,
+			port:           8001,
+			proxyList:      nil,
+			verbosity:      false,
+		}
+	})
+	return instance
+}
+
+func SetCacheTTL(ttl time.Duration) {
+	getInstance().mu.Lock()
+	defer getInstance().mu.Unlock()
+	getInstance().cacheTTL = ttl
+}
+
+func GetCacheTTL() time.Duration {
+	getInstance().mu.RLock()
+	defer getInstance().mu.RUnlock()
+	return getInstance().cacheTTL
+}
+
+func SetMaintenanceStatusTTL(ttl time.Duration) {
+	getInstance().mu.Lock()
+	defer getInstance().mu.Unlock()
+	getInstance().maintenanceTTL = ttl
+}
+
+func GetMaintenanceStatusTTL() time.Duration {
+	getInstance().mu.RLock()
+	defer getInstance().mu.RUnlock()
+	return getInstance().maintenanceTTL
+}
+
+func SetPort(port int) {
+	getInstance().mu.Lock()
+	defer getInstance().mu.Unlock()
+	getInstance().port = port
+}
+
+func GetPort() int {
+	getInstance().mu.RLock()
+	defer getInstance().mu.RUnlock()
+	return getInstance().port
+}
+
+func SetProxyList(proxies []string) {
+	getInstance().mu.Lock()
+	defer getInstance().mu.Unlock()
+	getInstance().proxyList = proxies
+	getInstance().proxySwitcher, _ = proxy.RoundRobinProxySwitcher(proxies...)
+}
+
+func GetProxyList() []string {
+	getInstance().mu.RLock()
+	defer getInstance().mu.RUnlock()
+	return getInstance().proxyList
+}
+
+func GetProxySwitcher() colly.ProxyFunc {
+	getInstance().mu.RLock()
+	defer getInstance().mu.RUnlock()
+	return getInstance().proxySwitcher
+}
+
+func SetVerbosity(verbosity bool) {
+	getInstance().mu.Lock()
+	defer getInstance().mu.Unlock()
+	getInstance().verbosity = verbosity
+}
+
+func GetVerbosity() bool {
+	getInstance().mu.RLock()
+	defer getInstance().mu.RUnlock()
+	return getInstance().verbosity
+}
+
+func PrintConfig() {
+	fmt.Printf("Configuration:\n" +
+		fmt.Sprintf("\tPort:\t\t%v\n", GetPort()) +
+		fmt.Sprintf("\tProxies:\t%v\n", GetProxyList()) +
+		fmt.Sprintf("\tVerbosity:\t%v\n", GetVerbosity()) +
+		fmt.Sprintf("\tCache TTL:\t%v\n", GetCacheTTL()),
+	)
+}

config/config_test.go

@@ -0,0 +1,70 @@
+package config
+
+import (
+	"reflect"
+	"testing"
+	"time"
+)
+
+func TestConfigSingleton(t *testing.T) {
+	// Test that the instance is the same when retrieved multiple times
+	instance1 := getInstance()
+	instance2 := getInstance()
+
+	if instance1 != instance2 {
+		t.Error("Multiple instances of Config were created")
+	}
+}
+
+func TestConfigOperations(t *testing.T) {
+	// Set values
+	ttl := 5 * time.Minute
+	SetCacheTTL(ttl)
+	SetPort(8081)
+	proxies := []string{"proxy3", "proxy4"}
+	SetProxyList(proxies)
+	SetVerbosity(true)
+
+	// Test updated values
+	if GetCacheTTL() != ttl {
+		t.Error("Failed to set Cache TTL")
+	}
+	if GetPort() != 8081 {
+		t.Error("Failed to set Port")
+	}
+	if !reflect.DeepEqual(GetProxyList(), proxies) {
+		t.Error("Failed to set Proxy List")
+	}
+	if !GetVerbosity() {
+		t.Error("Failed to set Verbosity")
+	}
+}
+
+func TestConcurrency(t *testing.T) {
+	// Set values concurrently
+	go func() {
+		ttl := 5 * time.Minute
+		SetCacheTTL(ttl)
+	}()
+	go func() {
+		proxies := []string{"proxy5", "proxy6"}
+		SetProxyList(proxies)
+	}()
+	go func() {
+		SetVerbosity(true)
+	}()
+
+	// Allow goroutines to finish
+	time.Sleep(100 * time.Millisecond)
+
+	// Test updated values
+	if GetCacheTTL() == 0 {
+		t.Error("Failed to set Cache TTL concurrently")
+	}
+	if !reflect.DeepEqual(GetProxyList(), []string{"proxy5", "proxy6"}) {
+		t.Error("Failed to set Proxy List concurrently")
+	}
+	if !GetVerbosity() {
+		t.Error("Failed to set Verbosity concurrently")
+	}
+}

docs/buildingFromSource.md

@@ -3,15 +3,10 @@ Building the scraper from source may be preferable in some cases. This way, the
 
 ## Prerequisites:
 - GNU/Linux (other platforms should work as well but I haven't tested them)
-- Go compiler >=v1.15
+- Go compiler >=v1.21
 
 ## Compilation
-By default, scraped results are cached in memory and stored for 3 hours. This helps to ease the pressure on BDO servers and decreases the response time tremendously (for cached responses). Use this command to compile the app:
+Use this command to compile the app:
 ```bash
 go build
 ```
-
-If you don't want to cache scraped results (e.g., if you are 100% sure that there will be no similar requests sent to the API), you can also use this command instead:
-```bash
-go build -tags "cacheless"
-```