From 5342d3655c64e6b1a39b12a50c7797505bf8778e Mon Sep 17 00:00:00 2001 From: Max Blazejewski Date: Wed, 27 Dec 2023 10:35:10 +0100 Subject: [PATCH] v1.5 (#9) --- Dockerfile | 5 +- README.md | 10 +- cache/cache.go | 66 +++++++++ cache/cache_test.go | 63 ++++++++ config/config.go | 112 ++++++++++++++ config/config_test.go | 70 +++++++++ docs/buildingDocker.md | 19 --- docs/buildingFromSource.md | 9 +- docs/openapi.json | 50 +++++-- go.mod | 22 ++- go.sum | 138 +++--------------- handlers/GetAdventurer.go | 34 +++-- handlers/GetAdventurerSearch.go | 49 ++++--- handlers/GetGuild.go | 34 +++-- handlers/GetGuildSearch.go | 41 ++++-- handlers/giveMaintenanceResponse.go | 22 +++ handlers/handlers.go | 4 - httpServer/BuildServer.go | 7 +- httpServer/registerHandlers-cacheless.go | 28 ---- httpServer/registerHandlers.go | 36 +---- main.go | 50 +++---- middleware/SetHeaders.go | 4 +- scrapers/GetCloseTime.go | 26 ++++ scrapers/ScrapeAdventurer.go | 34 +++-- scrapers/ScrapeAdventurerSearch.go | 18 +-- scrapers/ScrapeGuild.go | 12 +- scrapers/ScrapeGuildSearch.go | 16 +- scrapers/extractProfileTarget.go | 11 ++ scrapers/scraper.go | 53 +++++++ scrapers/scrapers.go | 63 -------- translators/TranslateClassName.go | 2 + translators/TranslateMisc.go | 14 ++ utils/FormatDateForHeaders.go | 7 + ...go => ValidateAdventurerNameQueryParam.go} | 12 +- .../ValidateAdventurerNameQueryParam_test.go | 31 ++++ validators/ValidateAdventurerName_test.go | 30 ---- ...Name.go => ValidateGuildNameQueryParam.go} | 14 +- .../ValidateGuildNameQueryParam_test.go | 30 ++++ validators/ValidateGuildName_test.go | 29 ---- validators/ValidatePage.go | 8 - validators/ValidatePageQueryParam.go | 15 ++ validators/ValidatePageQueryParam_test.go | 25 ++++ validators/ValidatePage_test.go | 27 ---- validators/ValidateProfileTarget.go | 7 - validators/ValidateProfileTargetQueryParam.go | 11 ++ .../ValidateProfileTargetQueryParam_test.go | 42 ++++++ validators/ValidateProfileTarget_test.go | 35 ----- validators/ValidateRegion.go | 7 - validators/ValidateRegionQueryParam.go | 17 +++ validators/ValidateRegionQueryParam_test.go | 28 ++++ validators/ValidateRegion_test.go | 27 ---- validators/ValidateSearchType.go | 5 - validators/ValidateSearchTypeQueryParam.go | 16 ++ .../ValidateSearchTypeQueryParam_test.go | 22 +++ validators/ValidateSearchType_test.go | 26 ---- 55 files changed, 940 insertions(+), 653 deletions(-) create mode 100644 cache/cache.go create mode 100644 cache/cache_test.go create mode 100644 config/config.go create mode 100644 config/config_test.go delete mode 100644 docs/buildingDocker.md create mode 100644 handlers/giveMaintenanceResponse.go delete mode 100644 handlers/handlers.go delete mode 100644 httpServer/registerHandlers-cacheless.go create mode 100644 scrapers/GetCloseTime.go create mode 100644 scrapers/extractProfileTarget.go create mode 100644 scrapers/scraper.go delete mode 100644 scrapers/scrapers.go create mode 100644 translators/TranslateMisc.go create mode 100644 utils/FormatDateForHeaders.go rename validators/{ValidateAdventurerName.go => ValidateAdventurerNameQueryParam.go} (68%) create mode 100644 validators/ValidateAdventurerNameQueryParam_test.go delete mode 100644 validators/ValidateAdventurerName_test.go rename validators/{ValidateGuildName.go => ValidateGuildNameQueryParam.go} (70%) create mode 100644 validators/ValidateGuildNameQueryParam_test.go delete mode 100644 validators/ValidateGuildName_test.go delete mode 100644 validators/ValidatePage.go create mode 100644 validators/ValidatePageQueryParam.go create mode 100644 validators/ValidatePageQueryParam_test.go delete mode 100644 validators/ValidatePage_test.go delete mode 100644 validators/ValidateProfileTarget.go create mode 100644 validators/ValidateProfileTargetQueryParam.go create mode 100644 validators/ValidateProfileTargetQueryParam_test.go delete mode 100644 validators/ValidateProfileTarget_test.go delete mode 100644 validators/ValidateRegion.go create mode 100644 validators/ValidateRegionQueryParam.go create mode 100644 validators/ValidateRegionQueryParam_test.go delete mode 100644 validators/ValidateRegion_test.go delete mode 100644 validators/ValidateSearchType.go create mode 100644 validators/ValidateSearchTypeQueryParam.go create mode 100644 validators/ValidateSearchTypeQueryParam_test.go delete mode 100644 validators/ValidateSearchType_test.go diff --git a/Dockerfile b/Dockerfile index 1c2b5d7..6f6a637 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,9 @@ -FROM golang:1.21rc2-alpine3.18 AS build +FROM golang:1.21-alpine AS build RUN apk add --no-cache git WORKDIR /src/bdo-rest-api COPY . . RUN go mod download -ARG tags=none -RUN go build -tags $tags -o /bdo-rest-api -ldflags="-s -w" . +RUN go build -o /bdo-rest-api -ldflags="-s -w" . FROM alpine:3.14 AS bin RUN addgroup --system --gid 1001 go diff --git a/README.md b/README.md index 7a361f7..b95835c 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ A scraper for Black Desert Online player in-game data with a REST API. It curren There are two ways to use this scraper for your needs: * By querying https://bdo.hemlo.cc/communityapi/v1 — this is the "official" instance hosted by me. * If you want to have more control over the API, host the scraper yourself using one of the following methods: - - As a Docker container: the image is available on [DockerHub](https://hub.docker.com/r/man90/bdo-rest-api) or you can build it yourself following instructions [here](docs/buildingDocker.md). + - As a Docker container: the image is available on [DockerHub](https://hub.docker.com/r/man90/bdo-rest-api). - Natively: build the binary from source as described in [this guide](docs/buildingFromSource.md). API documentation can be viewed [here](https://man90es.github.io/BDO-REST-API/). @@ -19,14 +19,14 @@ API documentation can be viewed [here](https://man90es.github.io/BDO-REST-API/). If you host the API yourself, either via Docker or natively, you can control some of its features by executing it with flags. Available flags: -- `-cachecap` - - Allows to specify response cache capacity - - Type: unsigned integer - - Default value: `10000` - `-cachettl` - Allows to specify cache TTL in minutes - Type: unsigned integer - Default value: `180` +- `-maintenancettl` + - Allows to limit how frequently scraper can check for maintenance end in minutes + - Type: unsigned integer + - Default value: `5` - `-port` - Allows to specify API server's port - Type: unsigned integer diff --git a/cache/cache.go b/cache/cache.go new file mode 100644 index 0000000..7cade43 --- /dev/null +++ b/cache/cache.go @@ -0,0 +1,66 @@ +package cache + +import ( + "strings" + "time" + + goCache "github.com/patrickmn/go-cache" + + "bdo-rest-api/config" + "bdo-rest-api/utils" +) + +type cacheEntry[T any] struct { + data T + date time.Time + status int +} + +type cache[T any] struct { + internalCache *goCache.Cache +} + +func joinKeys(keys []string) string { + return strings.Join(keys, ",") +} + +func NewCache[T any]() *cache[T] { + cacheTTL := config.GetCacheTTL() + + return &cache[T]{ + internalCache: goCache.New(cacheTTL, time.Hour), + } +} + +func (c *cache[T]) AddRecord(keys []string, data T, status int) (date string, expires string) { + cacheTTL := config.GetCacheTTL() + entry := cacheEntry[T]{ + data: data, + date: time.Now(), + status: status, + } + + c.internalCache.Add(joinKeys(keys), entry, cacheTTL) + expirationDate := entry.date.Add(cacheTTL) + + return utils.FormatDateForHeaders(entry.date), utils.FormatDateForHeaders(expirationDate) +} + +func (c *cache[T]) GetRecord(keys []string) (data T, status int, date string, expires string, found bool) { + var anyEntry interface{} + var expirationDate time.Time + + anyEntry, expirationDate, found = c.internalCache.GetWithExpiration(joinKeys(keys)) + + if !found { + return + } + + entry := anyEntry.(cacheEntry[T]) + + return entry.data, entry.status, utils.FormatDateForHeaders(entry.date), utils.FormatDateForHeaders(expirationDate), found +} + +func (c *cache[T]) GetItemCount() int { + return c.internalCache.ItemCount() +} diff --git a/cache/cache_test.go b/cache/cache_test.go new file mode 100644 index 0000000..a04c2e5 --- /dev/null +++ b/cache/cache_test.go @@ -0,0 +1,63 @@ +package cache + +import ( + "testing" + "time" + + "bdo-rest-api/config" +) + +func init() { + config.SetCacheTTL(time.Second) +} + +func TestCache(t *testing.T) { + // Create a cache instance for testing + testCache := NewCache[string]() + + // Test AddRecord and GetRecord + keys := []string{"key1", "key2"} + data := "test data" + status := 200 + + date, expires := testCache.AddRecord(keys, data, status) + + // Validate AddRecord results + if date == "" || expires == "" { + t.Error("AddRecord should return non-empty date and expires values") + } + + // Test GetRecord for an existing record + returnedData, returnedStatus, returnedDate, returnedExpires, found := testCache.GetRecord(keys) + + if !found { + t.Error("GetRecord should find the record") + } + + // Validate GetRecord results + if returnedData != data || returnedStatus != status || returnedDate == "" || returnedExpires == "" { + t.Error("GetRecord returned unexpected values") + } + + // Test GetItemCount + itemCount := testCache.GetItemCount() + if itemCount != 1 { + t.Errorf("GetItemCount should return 1, but got %d", itemCount) + } + + // Sleep for a while to allow the cache entry to expire + time.Sleep(2 * time.Second) + + // Test GetRecord for an expired record + _, _, _, _, found = testCache.GetRecord(keys) + + if found { + t.Error("GetRecord should not find an expired record") + } + + // Test GetItemCount after expiration + itemCount = testCache.GetItemCount() + if itemCount != 0 { + t.Errorf("GetItemCount should return 0 after expiration, but got %d", itemCount) + } +} diff --git a/config/config.go b/config/config.go new file mode 100644 index 0000000..f1dd7be --- /dev/null +++ b/config/config.go @@ -0,0 +1,112 @@ +package config + +import ( + "fmt" + "sync" + "time" + + "github.com/gocolly/colly/v2" + "github.com/gocolly/colly/v2/proxy" +) + +type config struct { + mu sync.RWMutex + cacheTTL time.Duration + maintenanceTTL time.Duration + port int + proxyList []string + proxySwitcher colly.ProxyFunc + verbosity bool +} + +var instance *config +var once sync.Once + +func getInstance() *config { + once.Do(func() { + instance = &config{ + cacheTTL: 3 * time.Hour, + maintenanceTTL: 5 * time.Minute, + port: 8001, + proxyList: nil, + verbosity: false, + } + }) + return instance +} + +func SetCacheTTL(ttl time.Duration) { + getInstance().mu.Lock() + defer getInstance().mu.Unlock() + getInstance().cacheTTL = ttl +} + +func GetCacheTTL() time.Duration { + getInstance().mu.RLock() + defer getInstance().mu.RUnlock() + return getInstance().cacheTTL +} + +func SetMaintenanceStatusTTL(ttl time.Duration) { + getInstance().mu.Lock() + defer getInstance().mu.Unlock() + getInstance().maintenanceTTL = ttl +} + +func GetMaintenanceStatusTTL() time.Duration { + getInstance().mu.RLock() + defer getInstance().mu.RUnlock() + return getInstance().maintenanceTTL +} + +func SetPort(port int) { + getInstance().mu.Lock() + defer getInstance().mu.Unlock() + getInstance().port = port +} + +func GetPort() int { + getInstance().mu.RLock() + defer getInstance().mu.RUnlock() + return getInstance().port +} + +func SetProxyList(proxies []string) { + getInstance().mu.Lock() + defer getInstance().mu.Unlock() + getInstance().proxyList = proxies + getInstance().proxySwitcher, _ = proxy.RoundRobinProxySwitcher(proxies...) +} + +func GetProxyList() []string { + getInstance().mu.RLock() + defer getInstance().mu.RUnlock() + return getInstance().proxyList +} + +func GetProxySwitcher() colly.ProxyFunc { + getInstance().mu.RLock() + defer getInstance().mu.RUnlock() + return getInstance().proxySwitcher +} + +func SetVerbosity(verbosity bool) { + getInstance().mu.Lock() + defer getInstance().mu.Unlock() + getInstance().verbosity = verbosity +} + +func GetVerbosity() bool { + getInstance().mu.RLock() + defer getInstance().mu.RUnlock() + return getInstance().verbosity +} + +func PrintConfig() { + fmt.Printf("Configuration:\n" + + fmt.Sprintf("\tPort:\t\t%v\n", GetPort()) + + fmt.Sprintf("\tProxies:\t%v\n", GetProxyList()) + + fmt.Sprintf("\tVerbosity:\t%v\n", GetVerbosity()) + + fmt.Sprintf("\tCache TTL:\t%v\n", GetCacheTTL()), + ) +} diff --git a/config/config_test.go b/config/config_test.go new file mode 100644 index 0000000..b54a796 --- /dev/null +++ b/config/config_test.go @@ -0,0 +1,70 @@ +package config + +import ( + "reflect" + "testing" + "time" +) + +func TestConfigSingleton(t *testing.T) { + // Test that the instance is the same when retrieved multiple times + instance1 := getInstance() + instance2 := getInstance() + + if instance1 != instance2 { + t.Error("Multiple instances of Config were created") + } +} + +func TestConfigOperations(t *testing.T) { + // Set values + ttl := 5 * time.Minute + SetCacheTTL(ttl) + SetPort(8081) + proxies := []string{"proxy3", "proxy4"} + SetProxyList(proxies) + SetVerbosity(true) + + // Test updated values + if GetCacheTTL() != ttl { + t.Error("Failed to set Cache TTL") + } + if GetPort() != 8081 { + t.Error("Failed to set Port") + } + if !reflect.DeepEqual(GetProxyList(), proxies) { + t.Error("Failed to set Proxy List") + } + if !GetVerbosity() { + t.Error("Failed to set Verbosity") + } +} + +func TestConcurrency(t *testing.T) { + // Set values concurrently + go func() { + ttl := 5 * time.Minute + SetCacheTTL(ttl) + }() + go func() { + proxies := []string{"proxy5", "proxy6"} + SetProxyList(proxies) + }() + go func() { + SetVerbosity(true) + }() + + // Allow goroutines to finish + time.Sleep(100 * time.Millisecond) + + // Test updated values + if GetCacheTTL() == 0 { + t.Error("Failed to set Cache TTL concurrently") + } + if !reflect.DeepEqual(GetProxyList(), []string{"proxy5", "proxy6"}) { + t.Error("Failed to set Proxy List concurrently") + } + if !GetVerbosity() { + t.Error("Failed to set Verbosity concurrently") + } +} diff --git a/docs/buildingDocker.md b/docs/buildingDocker.md deleted file mode 100644 index 124372b..0000000 --- a/docs/buildingDocker.md +++ /dev/null @@ -1,19 +0,0 @@ -# Building Docker image -Building a Docker image yourself can be preferable over using a Docker Image from [Docker Hub](https://hub.docker.com/r/man90/bdo-rest-api) if you want to disable caching. - -## Building an image -By default, scraped results are cached in memory and stored for 3 hours. This helps to ease the pressure on BDO servers and decreases the response time tremendously (for cached responses). Use this command to compile the app: -```bash -docker build -t bdo-rest-api . -``` - -If you don't want to cache scraped results (e.g., if you are 100% sure that there will be no similar requests sent to the API), you can use this command instead: -```bash -docker build -t bdo-rest-api --build-arg tags=cacheless . -``` - -## Running a container -You can run a Docker container you just built by executing this command: -```bash -docker container run -p 8001:8001 bdo-rest-api -``` diff --git a/docs/buildingFromSource.md b/docs/buildingFromSource.md index 302233a..1940ac8 100644 --- a/docs/buildingFromSource.md +++ b/docs/buildingFromSource.md @@ -3,15 +3,10 @@ Building the scraper from source may be preferable in some cases. This way, the ## Prerequisites: - GNU/Linux (other platforms should work as well but I haven't tested them) -- Go compiler >=v1.15 +- Go compiler >=v1.21 ## Compilation -By default, scraped results are cached in memory and stored for 3 hours. This helps to ease the pressure on BDO servers and decreases the response time tremendously (for cached responses). Use this command to compile the app: +Use this command to compile the app: ```bash go build ``` - -If you don't want to cache scraped results (e.g., if you are 100% sure that there will be no similar requests sent to the API), you can also use this command instead: -```bash -go build -tags "cacheless" -``` diff --git a/docs/openapi.json b/docs/openapi.json index d84daef..89dab92 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -20,7 +20,11 @@ "description": "Only Eropean, North American and South American servers are supported. If you omit this parameter, it's assumed that you want to search on the European server.", "schema": { "type": "string", - "enum": ["EU", "NA", "SA"], + "enum": [ + "EU", + "NA", + "SA" + ], "default": "EU" } } @@ -59,7 +63,11 @@ "description": "Only Eropean, North American and South American servers are supported. You may omit this parameter for the European and North American servers.", "schema": { "type": "string", - "enum": ["EU", "NA", "SA"], + "enum": [ + "EU", + "NA", + "SA" + ], "default": "EU" } } @@ -83,7 +91,11 @@ }, "region": { "type": "string", - "enum": ["EU", "NA"] + "enum": [ + "EU", + "NA", + "SA" + ] }, "guild": { "properties": { @@ -214,14 +226,17 @@ "description": "Switch between filtering by family name and character name. If you omit this parameter, it's assumed that you want to filter by family name.", "schema": { "type": "string", - "enum": ["familyName", "characterName"], + "enum": [ + "familyName", + "characterName" + ], "example": "familyName" } }, { "name": "page", "in": "query", - "description": "This parameter is understood by the API, but you should either omit it or set to 1. Because of how player search works, there is never more than one page.", + "description": "This parameter is understood by the API, but you should either omit it or set to 1. Because of how search currently works, there is never more than one page.", "schema": { "type": "number", "default": 1 @@ -249,7 +264,11 @@ }, "region": { "type": "string", - "enum": ["EU", "NA", "SA"] + "enum": [ + "EU", + "NA", + "SA" + ] }, "guild": { "properties": { @@ -335,7 +354,11 @@ }, "region": { "type": "string", - "enum": ["EU", "NA", "SA"] + "enum": [ + "EU", + "NA", + "SA" + ] }, "createdOn": { "type": "string", @@ -416,7 +439,7 @@ { "name": "page", "in": "query", - "description": "If the results have more than one page, you can navigate through those pages using this parameter. The indexing starts with 1.", + "description": "This parameter is understood by the API, but you should either omit it or set to 1. Because of how search currently works, there is never more than one page.", "schema": { "type": "number", "default": 1 @@ -440,7 +463,11 @@ }, "region": { "type": "string", - "enum": ["EU", "NA", "SA"] + "enum": [ + "EU", + "NA", + "SA" + ] }, "createdOn": { "type": "string", @@ -461,7 +488,10 @@ }, "kind": { "type": "string", - "enum": ["Guild", "Clan"] + "enum": [ + "Guild", + "Clan" + ] }, "population": { "type": "number", diff --git a/go.mod b/go.mod index fab4958..2c6500f 100644 --- a/go.mod +++ b/go.mod @@ -1,32 +1,28 @@ module bdo-rest-api -// +heroku goVersion go1.17 -go 1.17 - -// A fork without hardcoded port filter -replace github.com/victorspringer/http-cache => github.com/octoman90/http-cache v0.0.0-20221206214057-1d8c8cb7c3dc +// +heroku goVersion go1.21 +go 1.21 require ( github.com/gocolly/colly/v2 v2.1.0 - github.com/gorilla/mux v1.8.0 - github.com/victorspringer/http-cache v0.0.0-20220131145941-ef3624e6666f + github.com/gorilla/mux v1.8.1 + github.com/patrickmn/go-cache v2.1.0+incompatible ) require ( github.com/PuerkitoBio/goquery v1.8.1 // indirect github.com/andybalholm/cascadia v1.3.2 // indirect github.com/antchfx/htmlquery v1.3.0 // indirect - github.com/antchfx/xmlquery v1.3.17 // indirect - github.com/antchfx/xpath v1.2.4 // indirect + github.com/antchfx/xmlquery v1.3.18 // indirect + github.com/antchfx/xpath v1.2.5 // indirect github.com/gobwas/glob v0.2.3 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/kennygrant/sanitize v1.2.4 // indirect github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect github.com/temoto/robotstxt v1.1.2 // indirect - golang.org/x/net v0.13.0 // indirect - golang.org/x/text v0.11.0 // indirect - google.golang.org/appengine v1.6.7 // indirect + golang.org/x/net v0.19.0 // indirect + golang.org/x/text v0.14.0 // indirect + google.golang.org/appengine v1.6.8 // indirect google.golang.org/protobuf v1.31.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 45b10a4..74d49b3 100644 --- a/go.sum +++ b/go.sum @@ -3,7 +3,6 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM= github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ= -github.com/allegro/bigcache v1.2.1/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= github.com/andybalholm/cascadia v1.2.0/go.mod h1:YCyR8vOZT9aZ1CHEd8ap0gMVm2aFgxBp0T0eFw1RUQY= github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= @@ -13,32 +12,23 @@ github.com/antchfx/htmlquery v1.2.3/go.mod h1:B0ABL+F5irhhMWg54ymEZinzMSi0Kt3I2i github.com/antchfx/htmlquery v1.3.0 h1:5I5yNFOVI+egyia5F2s/5Do2nFWxJz41Tr3DyfKD25E= github.com/antchfx/htmlquery v1.3.0/go.mod h1:zKPDVTMhfOmcwxheXUsx4rKJy8KEY/PU6eXr/2SebQ8= github.com/antchfx/xmlquery v1.2.4/go.mod h1:KQQuESaxSlqugE2ZBcM/qn+ebIpt+d+4Xx7YcSGAIrM= -github.com/antchfx/xmlquery v1.3.17 h1:d0qWjPp/D+vtRw7ivCwT5ApH/3CkQU8JOeo3245PpTk= -github.com/antchfx/xmlquery v1.3.17/go.mod h1:Afkq4JIeXut75taLSuI31ISJ/zeq+3jG7TunF7noreA= +github.com/antchfx/xmlquery v1.3.18 h1:FSQ3wMuphnPPGJOFhvc+cRQ2CT/rUj4cyQXkJcjOwz0= +github.com/antchfx/xmlquery v1.3.18/go.mod h1:Afkq4JIeXut75taLSuI31ISJ/zeq+3jG7TunF7noreA= github.com/antchfx/xpath v1.1.6/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk= github.com/antchfx/xpath v1.1.8/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk= github.com/antchfx/xpath v1.2.3/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= -github.com/antchfx/xpath v1.2.4 h1:dW1HB/JxKvGtJ9WyVGJ0sIoEcqftV3SqIstujI+B9XY= github.com/antchfx/xpath v1.2.4/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= +github.com/antchfx/xpath v1.2.5 h1:hqZ+wtQ+KIOV/S3bGZcIhpgYC26um2bZYP2KVGcR7VY= +github.com/antchfx/xpath v1.2.5/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-redis/cache v6.4.0+incompatible/go.mod h1:XNnMdvlNjcZvHjsscEozHAeOeSE5riG9Fj54meG4WT4= -github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= -github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= -github.com/gocolly/colly v1.2.0 h1:qRz9YAn8FIH0qzgNUw+HT9UN7wm1oF9OBAilwEWpyrI= github.com/gocolly/colly v1.2.0/go.mod h1:Hof5T3ZswNVsOHYmba1u03W65HDWgpV5HifSuueE0EA= github.com/gocolly/colly/v2 v2.1.0 h1:k0DuZkDoCsx51bKpRJNEmcxcp+W5N8ziuwGaSDuFoGs= github.com/gocolly/colly/v2 v2.1.0/go.mod h1:I2MuhsLjQ+Ex+IzK3afNS8/1qP3AedHOusRPcRdC5o0= @@ -65,44 +55,15 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= -github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= -github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= +github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/jawher/mow.cli v1.1.0/go.mod h1:aNaQlc7ozF3vw6IJ2dHjp2ZFiA4ozMIYY6PyuRJwlUg= github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8NzpJ3o= github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= -github.com/octoman90/http-cache v0.0.0-20221206214057-1d8c8cb7c3dc h1:rSejQS6VZ9zH+nssM2cmzpJp4FJq3R1DbPckhU5jyew= -github.com/octoman90/http-cache v0.0.0-20221206214057-1d8c8cb7c3dc/go.mod h1:D1AD6nlXv7HkIfTVd8ZWK1KQEiXYNy/LbLkx8H9tIQw= -github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= -github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= -github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= -github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU= -github.com/onsi/ginkgo/v2 v2.1.6/go.mod h1:MEH45j8TBi6u9BMogfbp0stKC5cdGjumZj5Y7AG4VIk= -github.com/onsi/ginkgo/v2 v2.3.0/go.mod h1:Eew0uilEqZmIEZr8JrvYlvOM7Rr6xzTmMV8AyFNU9d0= -github.com/onsi/ginkgo/v2 v2.4.0/go.mod h1:iHkDK1fKGcBoEHT5W7YBq4RFWaQulw+caOMkAt4OrFo= -github.com/onsi/ginkgo/v2 v2.5.0/go.mod h1:Luc4sArBICYCS8THh8v3i3i5CuSZO+RaQRaJoeNwomw= -github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= -github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= -github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= -github.com/onsi/gomega v1.20.1/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo= -github.com/onsi/gomega v1.21.1/go.mod h1:iYAIXgPSaDHak0LCMA+AWBpIKBr8WZicMxnE8luStNc= -github.com/onsi/gomega v1.22.1/go.mod h1:x6n7VNe4hw0vkyYUM4mjIXx3JbLiPaBPNgB7PRQ1tuM= -github.com/onsi/gomega v1.24.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg= -github.com/onsi/gomega v1.24.1/go.mod h1:3AOiACssS3/MajrniINInwbfOOtfZvplPzuRSmvt1jM= +github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= +github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -111,41 +72,24 @@ github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7 github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/temoto/robotstxt v1.1.1/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo= github.com/temoto/robotstxt v1.1.2 h1:W2pOjSJ6SWvldyEuiFXNxz3xZ8aiWX5LbfDiOFd7Fxg= github.com/temoto/robotstxt v1.1.2/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo= -github.com/vmihailenco/msgpack v4.0.4+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= -golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -153,80 +97,50 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.13.0 h1:Nvo8UFsZ8X3BhAC9699Z1j7XQ3rsZnUUm7jfBEk1ueY= -golang.org/x/net v0.13.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= +golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= +golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= -golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= -golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -234,20 +148,16 @@ golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= @@ -265,19 +175,7 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/handlers/GetAdventurer.go b/handlers/GetAdventurer.go index fbc28eb..90f67fe 100644 --- a/handlers/GetAdventurer.go +++ b/handlers/GetAdventurer.go @@ -4,29 +4,43 @@ import ( "encoding/json" "net/http" + "bdo-rest-api/cache" + "bdo-rest-api/models" "bdo-rest-api/scrapers" "bdo-rest-api/validators" ) +var profilesCache = cache.NewCache[models.Profile]() + func GetAdventurer(w http.ResponseWriter, r *http.Request) { - profileTargetParams, profileTargetProvided := r.URL.Query()["profileTarget"] - regionParams, regionProvided := r.URL.Query()["region"] + profileTarget, profileTargetOk := validators.ValidateProfileTargetQueryParam(r.URL.Query()["profileTarget"]) + region, regionOk := validators.ValidateRegionQueryParam(r.URL.Query()["region"]) - // Return status 400 if a required parameter is invalid - if !profileTargetProvided || !validators.ValidateProfileTarget(&profileTargetParams[0]) { + if !profileTargetOk || !regionOk { w.WriteHeader(http.StatusBadRequest) return } - // Set defaults for optional parameters - region := defaultRegion + if ok := giveMaintenanceResponse(w, region); ok { + return + } + + // Look for cached data, then run the scraper if needed + data, status, date, expires, found := profilesCache.GetRecord([]string{region, profileTarget}) + if !found { + data, status = scrapers.ScrapeAdventurer(region, profileTarget) + + if ok := giveMaintenanceResponse(w, region); ok { + return + } - if regionProvided && validators.ValidateRegion(®ionParams[0]) { - region = regionParams[0] + date, expires = profilesCache.AddRecord([]string{region, profileTarget}, data, status) } - // Run the scraper - if data, status := scrapers.ScrapeAdventurer(region, profileTargetParams[0]); status == http.StatusOK { + w.Header().Set("Date", date) + w.Header().Set("Expires", expires) + + if status == http.StatusOK { json.NewEncoder(w).Encode(data) } else { w.WriteHeader(status) diff --git a/handlers/GetAdventurerSearch.go b/handlers/GetAdventurerSearch.go index ba46f00..7babd71 100644 --- a/handlers/GetAdventurerSearch.go +++ b/handlers/GetAdventurerSearch.go @@ -2,47 +2,48 @@ package handlers import ( "encoding/json" + "fmt" "net/http" - "strconv" + "bdo-rest-api/cache" + "bdo-rest-api/models" "bdo-rest-api/scrapers" "bdo-rest-api/validators" ) +var profileSearchCache = cache.NewCache[[]models.Profile]() + func GetAdventurerSearch(w http.ResponseWriter, r *http.Request) { - regionParams, regionProvided := r.URL.Query()["region"] - searchTypeParams, searchTypeProvided := r.URL.Query()["searchType"] - pageParams, pageProvided := r.URL.Query()["page"] - queryParams, queryProvided := r.URL.Query()["query"] + page := validators.ValidatePageQueryParam(r.URL.Query()["page"]) + query, queryOk := validators.ValidateAdventurerNameQueryParam(r.URL.Query()["query"]) + region, regionOk := validators.ValidateRegionQueryParam(r.URL.Query()["region"]) + searchType := validators.ValidateSearchTypeQueryParam(r.URL.Query()["searchType"]) - // Return status 400 if a required parameter is invalid - if !queryProvided || !validators.ValidateAdventurerName(&queryParams[0]) { + if !queryOk || !regionOk { w.WriteHeader(http.StatusBadRequest) return } - // Set defaults for optional parameters - region := defaultRegion - searchType := uint8(2) - page := defaultPage - - if regionProvided && validators.ValidateRegion(®ionParams[0]) { - region = regionParams[0] + if ok := giveMaintenanceResponse(w, region); ok { + return } - if searchTypeProvided && validators.ValidateSearchType(&searchTypeParams[0]) { - searchType = map[string]uint8{ - "characterName": 1, - "familyName": 2, - }[searchTypeParams[0]] - } + // Look for cached data, then run the scraper if needed + data, status, date, expires, found := profileSearchCache.GetRecord([]string{region, query, fmt.Sprint(searchType), fmt.Sprint(page)}) + if !found { + data, status = scrapers.ScrapeAdventurerSearch(region, query, searchType, page) + + if ok := giveMaintenanceResponse(w, region); ok { + return + } - if pageProvided && validators.ValidatePage(&pageParams[0]) { - page, _ = strconv.Atoi(pageParams[0]) + date, expires = profileSearchCache.AddRecord([]string{region, query, fmt.Sprint(searchType), fmt.Sprint(page)}, data, status) } - // Run the scraper - if data, status := scrapers.ScrapeAdventurerSearch(region, queryParams[0], searchType, uint16(page)); status == http.StatusOK { + w.Header().Set("Date", date) + w.Header().Set("Expires", expires) + + if status == http.StatusOK { json.NewEncoder(w).Encode(data) } else { w.WriteHeader(status) diff --git a/handlers/GetGuild.go b/handlers/GetGuild.go index 39388be..7bd47d3 100644 --- a/handlers/GetGuild.go +++ b/handlers/GetGuild.go @@ -4,29 +4,43 @@ import ( "encoding/json" "net/http" + "bdo-rest-api/cache" + "bdo-rest-api/models" "bdo-rest-api/scrapers" "bdo-rest-api/validators" ) +var guildProfilesCache = cache.NewCache[models.GuildProfile]() + func GetGuild(w http.ResponseWriter, r *http.Request) { - regionParams, regionProvided := r.URL.Query()["region"] - nameParams, nameProvided := r.URL.Query()["guildName"] + name, nameOk := validators.ValidateGuildNameQueryParam(r.URL.Query()["guildName"]) + region, regionOk := validators.ValidateRegionQueryParam(r.URL.Query()["region"]) - // Return status 400 if a required parameter is invalid - if !nameProvided || !validators.ValidateGuildName(&nameParams[0]) { + if !nameOk || !regionOk { w.WriteHeader(http.StatusBadRequest) return } - // Set defaults for optional parameters - region := defaultRegion + if ok := giveMaintenanceResponse(w, region); ok { + return + } + + // Look for cached data, then run the scraper if needed + data, status, date, expires, found := guildProfilesCache.GetRecord([]string{region, name}) + if !found { + data, status = scrapers.ScrapeGuild(region, name) + + if ok := giveMaintenanceResponse(w, region); ok { + return + } - if regionProvided && validators.ValidateRegion(®ionParams[0]) { - region = regionParams[0] + date, expires = guildProfilesCache.AddRecord([]string{region, name}, data, status) } - // Run the scraper - if data, status := scrapers.ScrapeGuild(region, nameParams[0]); status == http.StatusOK { + w.Header().Set("Date", date) + w.Header().Set("Expires", expires) + + if status == http.StatusOK { json.NewEncoder(w).Encode(data) } else { w.WriteHeader(status) diff --git a/handlers/GetGuildSearch.go b/handlers/GetGuildSearch.go index 8eb65f8..8ee903f 100644 --- a/handlers/GetGuildSearch.go +++ b/handlers/GetGuildSearch.go @@ -2,38 +2,47 @@ package handlers import ( "encoding/json" + "fmt" "net/http" - "strconv" + "bdo-rest-api/cache" + "bdo-rest-api/models" "bdo-rest-api/scrapers" "bdo-rest-api/validators" ) +var guildSearchCache = cache.NewCache[[]models.GuildProfile]() + func GetGuildSearch(w http.ResponseWriter, r *http.Request) { - regionParams, regionProvided := r.URL.Query()["region"] - pageParams, pageProvided := r.URL.Query()["page"] - queryParams, queryProvided := r.URL.Query()["query"] + name, nameOk := validators.ValidateGuildNameQueryParam(r.URL.Query()["query"]) + page := validators.ValidatePageQueryParam(r.URL.Query()["page"]) + region, regionOk := validators.ValidateRegionQueryParam(r.URL.Query()["region"]) - // Return status 400 if a required parameter is invalid - if !queryProvided || !validators.ValidateGuildName(&queryParams[0]) { + if !nameOk || !regionOk { w.WriteHeader(http.StatusBadRequest) return } - // Set defaults for optional parameters - region := defaultRegion - page := defaultPage - - if regionProvided && validators.ValidateRegion(®ionParams[0]) { - region = regionParams[0] + if ok := giveMaintenanceResponse(w, region); ok { + return } - if pageProvided && validators.ValidatePage(&pageParams[0]) { - page, _ = strconv.Atoi(pageParams[0]) + // Look for cached data, then run the scraper if needed + data, status, date, expires, found := guildSearchCache.GetRecord([]string{region, name, fmt.Sprint(page)}) + if !found { + data, status = scrapers.ScrapeGuildSearch(region, name, page) + + if ok := giveMaintenanceResponse(w, region); ok { + return + } + + date, expires = guildSearchCache.AddRecord([]string{region, name, fmt.Sprint(page)}, data, status) } - // Run the scraper - if data, status := scrapers.ScrapeGuildSearch(region, queryParams[0], uint16(page)); status == http.StatusOK { + w.Header().Set("Date", date) + w.Header().Set("Expires", expires) + + if status == http.StatusOK { json.NewEncoder(w).Encode(data) } else { w.WriteHeader(status) diff --git a/handlers/giveMaintenanceResponse.go b/handlers/giveMaintenanceResponse.go new file mode 100644 index 0000000..4e159ea --- /dev/null +++ b/handlers/giveMaintenanceResponse.go @@ -0,0 +1,22 @@ +package handlers + +import ( + "net/http" + "time" + + "bdo-rest-api/scrapers" + "bdo-rest-api/utils" +) + +func giveMaintenanceResponse(w http.ResponseWriter, region string) (ok bool) { + isCloseTime, expires := scrapers.GetCloseTime(region) + + if !isCloseTime { + return false + } + + w.Header().Set("Date", utils.FormatDateForHeaders(time.Now())) + w.Header().Set("Expires", utils.FormatDateForHeaders(expires)) + w.WriteHeader(http.StatusServiceUnavailable) + return true +} diff --git a/handlers/handlers.go b/handlers/handlers.go deleted file mode 100644 index b7cb684..0000000 --- a/handlers/handlers.go +++ /dev/null @@ -1,4 +0,0 @@ -package handlers - -const defaultRegion = "EU" -const defaultPage = 1 diff --git a/httpServer/BuildServer.go b/httpServer/BuildServer.go index 6c9defb..fce5f2b 100644 --- a/httpServer/BuildServer.go +++ b/httpServer/BuildServer.go @@ -7,16 +7,17 @@ import ( "os" "time" + "bdo-rest-api/config" "bdo-rest-api/handlers" ) -func BuildServer(port *string, flagCacheTTL *int, flagCacheCap *int) *http.Server { +func BuildServer() *http.Server { router, err := registerHandlers(map[string]func(http.ResponseWriter, *http.Request){ "/v1/adventurer/search": handlers.GetAdventurerSearch, "/v1/guild/search": handlers.GetGuildSearch, "/v1/adventurer": handlers.GetAdventurer, "/v1/guild": handlers.GetGuild, - }, time.Duration(*flagCacheTTL)*time.Minute, *flagCacheCap) + }) if err != nil { log.Fatal(err) @@ -24,7 +25,7 @@ func BuildServer(port *string, flagCacheTTL *int, flagCacheCap *int) *http.Serve } return &http.Server{ - Addr: fmt.Sprintf("0.0.0.0:%v", *port), + Addr: fmt.Sprintf("0.0.0.0:%v", config.GetPort()), Handler: router, IdleTimeout: 60 * time.Second, ReadTimeout: 15 * time.Second, diff --git a/httpServer/registerHandlers-cacheless.go b/httpServer/registerHandlers-cacheless.go deleted file mode 100644 index bba9b27..0000000 --- a/httpServer/registerHandlers-cacheless.go +++ /dev/null @@ -1,28 +0,0 @@ -//go:build cacheless - -package httpServer - -import ( - "net/http" - "time" - - "bdo-rest-api/middleware" - - "github.com/gorilla/mux" -) - -const CacheSupport = false - -func registerHandlers(handlerMap map[string]func(http.ResponseWriter, *http.Request), ttl time.Duration, cap int) (*mux.Router, error) { - router := mux.NewRouter() - - for route, handler := range handlerMap { - router.Handle(route, - middleware.SetHeaders( - http.HandlerFunc(handler), - ), - ).Methods("GET") - } - - return router, nil -} diff --git a/httpServer/registerHandlers.go b/httpServer/registerHandlers.go index d7c9efc..08aa3f1 100644 --- a/httpServer/registerHandlers.go +++ b/httpServer/registerHandlers.go @@ -1,50 +1,20 @@ -//go:build !cacheless - package httpServer import ( "net/http" - "time" "bdo-rest-api/middleware" "github.com/gorilla/mux" - cache "github.com/victorspringer/http-cache" - "github.com/victorspringer/http-cache/adapter/memory" ) -const CacheSupport = true - -func registerHandlers(handlerMap map[string]func(http.ResponseWriter, *http.Request), ttl time.Duration, cap int) (*mux.Router, error) { - memcached, err := memory.NewAdapter( - memory.AdapterWithAlgorithm(memory.LRU), - memory.AdapterWithCapacity(cap), - ) - - if err != nil { - return nil, err - } - - cacheClient, err := cache.NewClient( - cache.ClientWithAdapter(memcached), - cache.ClientWithTTL(ttl), - cache.ClientWithRefreshKey("opn"), - cache.ClientWithStatusCodeFilter(func(code int) bool { return code != 400 }), - cache.ClientWithExpiresHeader(), - ) - - if err != nil { - return nil, err - } - +func registerHandlers(handlerMap map[string]func(http.ResponseWriter, *http.Request)) (*mux.Router, error) { router := mux.NewRouter() for route, handler := range handlerMap { router.Handle(route, - cacheClient.Middleware( - middleware.SetHeaders( - http.HandlerFunc(handler), - ), + middleware.SetHeaders( + http.HandlerFunc(handler), ), ).Methods("GET") } diff --git a/main.go b/main.go index 3edf179..6493e66 100644 --- a/main.go +++ b/main.go @@ -2,62 +2,50 @@ package main import ( "flag" - "fmt" "log" "os" + "strconv" "strings" + "time" + "bdo-rest-api/config" "bdo-rest-api/httpServer" - "bdo-rest-api/scrapers" ) func main() { - // Parse flags - flagCacheCap := flag.Int("cachecap", 1e4, "Cache capacity") flagCacheTTL := flag.Int("cachettl", 180, "Cache TTL in minutes") + flagMaintenanceTTL := flag.Int("maintenancettl", 5, "Allows to limit how frequently scraper can check for maintenance end in minutes") flagPort := flag.Int("port", 8001, "Port to catch requests on") flagProxy := flag.String("proxy", "", "Open proxy address to make requests to BDO servers") flagVerbose := flag.Bool("verbose", false, "Print out additional logs into stdout") flag.Parse() // Read port from flags and env - var port string if *flagPort == 8001 && len(os.Getenv("PORT")) > 0 { - port = os.Getenv("PORT") + port, err := strconv.Atoi(os.Getenv("PORT")) + + if nil != err { + port = 8001 + } + + config.SetPort(port) } else { - port = fmt.Sprintf("%v", *flagPort) + config.SetPort(*flagPort) } // Read proxies from flags - var proxies []string if len(*flagProxy) > 0 { - proxies = strings.Fields(*flagProxy) - } else { - proxies = strings.Fields(os.Getenv("PROXY")) - } - scrapers.PushProxies(proxies...) - - // Set scraper verbosity level according to flag - scrapers.SetVerbose(*flagVerbose) - - // Print out start info - configPrintOut := "Configuration:\n" + - fmt.Sprintf("\tPort:\t\t%v\n", port) + - fmt.Sprintf("\tProxies:\t%v\n", proxies) + - fmt.Sprintf("\tVerbosity:\t%v\n", *flagVerbose) - - if httpServer.CacheSupport { - configPrintOut += fmt.Sprintf("\tCache TTL:\t%v minutes\n", *flagCacheTTL) + - fmt.Sprintf("\tCache capacity:\t%v\n\n", *flagCacheCap) + config.SetProxyList(strings.Fields(*flagProxy)) } else { - configPrintOut += "\tCache:\tUnsupported in this build\n\n" + config.SetProxyList(strings.Fields(os.Getenv("PROXY"))) } - fmt.Printf(configPrintOut) - - // Build server - srv := httpServer.BuildServer(&port, flagCacheTTL, flagCacheCap) + config.SetCacheTTL(time.Duration(*flagCacheTTL) * time.Minute) + config.SetMaintenanceStatusTTL(time.Duration(*flagMaintenanceTTL) * time.Minute) + config.SetVerbosity(*flagVerbose) + config.PrintConfig() log.Println("Listening for requests") + srv := httpServer.BuildServer() log.Fatal(srv.ListenAndServe()) } diff --git a/middleware/SetHeaders.go b/middleware/SetHeaders.go index 59699de..2e45ec4 100644 --- a/middleware/SetHeaders.go +++ b/middleware/SetHeaders.go @@ -2,12 +2,14 @@ package middleware import ( "net/http" + "time" ) func SetHeaders(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") w.Header().Set("Access-Control-Allow-Origin", "*") + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Date", time.Now().Format(time.RFC1123Z)) next.ServeHTTP(w, r) }) diff --git a/scrapers/GetCloseTime.go b/scrapers/GetCloseTime.go new file mode 100644 index 0000000..90e3893 --- /dev/null +++ b/scrapers/GetCloseTime.go @@ -0,0 +1,26 @@ +package scrapers + +import ( + "time" + + "bdo-rest-api/config" +) + +var lastCloseTimes = map[string]time.Time{ + "EU": {}, + "SA": {}, +} + +func GetCloseTime(region string) (isCloseTime bool, expires time.Time) { + // NA and EU use one website + if region == "NA" { + region = "EU" + } + + expires = lastCloseTimes[region].Add(config.GetMaintenanceStatusTTL()) + return time.Now().Before(expires), expires +} + +func setCloseTime(region string) { + lastCloseTimes[region] = time.Now() +} diff --git a/scrapers/ScrapeAdventurer.go b/scrapers/ScrapeAdventurer.go index 3f4dc56..35e16bb 100644 --- a/scrapers/ScrapeAdventurer.go +++ b/scrapers/ScrapeAdventurer.go @@ -16,16 +16,12 @@ import ( ) func ScrapeAdventurer(region string, profileTarget string) (profile models.Profile, status int) { - c := collyFactory() + c := newScraper(region) profile.ProfileTarget = profileTarget profile.Region = region status = http.StatusNotFound - c.OnHTML(closetimeSelector, func(e *colly.HTMLElement) { - status = http.StatusServiceUnavailable - }) - c.OnHTML(`.nick`, func(e *colly.HTMLElement) { profile.FamilyName = e.Text status = http.StatusOK @@ -36,15 +32,21 @@ func ScrapeAdventurer(region string, profileTarget string) (profile models.Profi }) c.OnHTML(`.desc.guild a`, func(e *colly.HTMLElement) { - if e.Attr("href") != "javscript:void(0)" { - profile.Guild = &models.GuildProfile{ - Name: e.Text, - } + profile.Guild = &models.GuildProfile{ + Name: e.Text, } }) c.OnHTML(`.desc.guild span`, func(e *colly.HTMLElement) { - profile.Privacy = profile.Privacy | models.PrivateGuild + guildStatus := e.Text + + if region != "EU" && region != "NA" { + translators.TranslateMisc(&guildStatus) + } + + if guildStatus == "Private" { + profile.Privacy = profile.Privacy | models.PrivateGuild + } }) c.OnHTML(`.line_list .desc:not(.guild)`, func(e *colly.HTMLElement) { @@ -53,8 +55,7 @@ func ScrapeAdventurer(region string, profileTarget string) (profile models.Profi }) c.OnHTML(`.character_desc_area .character_info span:nth-child(3) em`, func(e *colly.HTMLElement) { - if e.Text != "Private" { - contributionPoints, _ := strconv.Atoi(e.Text) + if contributionPoints, err := strconv.Atoi(e.Text); err == nil { profile.ContributionPoints = uint16(contributionPoints) } else { profile.Privacy = profile.Privacy | models.PrivateContrib @@ -74,8 +75,7 @@ func ScrapeAdventurer(region string, profileTarget string) (profile models.Profi character.Main = true }) - if levelStr := e.ChildText(".character_info span:nth-child(2) em"); levelStr != "Private" { - level, _ := strconv.Atoi(levelStr) + if level, err := strconv.Atoi(e.ChildText(".character_info span:nth-child(2) em")); err == nil { character.Level = uint8(level) } else { profile.Privacy = profile.Privacy | models.PrivateLevel @@ -128,7 +128,11 @@ func ScrapeAdventurer(region string, profileTarget string) (profile models.Profi profile.Privacy = profile.Privacy | models.PrivateSpecs }) - c.Visit(fmt.Sprintf("%v/Adventure/Profile?profileTarget=%v", getSiteRoot(region), url.QueryEscape(profileTarget))) + c.Visit(fmt.Sprintf("/Profile?profileTarget=%v", url.QueryEscape(profileTarget))) + + if isCloseTime, _ := GetCloseTime(region); isCloseTime { + status = http.StatusServiceUnavailable + } return } diff --git a/scrapers/ScrapeAdventurerSearch.go b/scrapers/ScrapeAdventurerSearch.go index eccd5ac..03533c0 100644 --- a/scrapers/ScrapeAdventurerSearch.go +++ b/scrapers/ScrapeAdventurerSearch.go @@ -12,14 +12,12 @@ import ( ) func ScrapeAdventurerSearch(region string, query string, searchType uint8, page uint16) (profiles []models.Profile, status int) { - c := collyFactory() - closetime := false + c := newScraper(region) - c.OnHTML(closetimeSelector, func(e *colly.HTMLElement) { - closetime = true - }) + status = http.StatusNotFound c.OnHTML(`.box_list_area li:not(.no_result)`, func(e *colly.HTMLElement) { + status = http.StatusOK profile := models.Profile{ Region: region, FamilyName: e.ChildText(".title a"), @@ -42,7 +40,7 @@ func ScrapeAdventurerSearch(region string, query string, searchType uint8, page // Site displays the main character when searching by family name // And the searched character when searching by character name - if 2 == searchType { + if searchType == 2 { profile.Characters[0].Main = true } @@ -57,14 +55,10 @@ func ScrapeAdventurerSearch(region string, query string, searchType uint8, page profiles = append(profiles, profile) }) - c.Visit(fmt.Sprintf("%v/Adventure?region=%v&searchType=%v&searchKeyword=%v&Page=%v", getSiteRoot(region), region, searchType, query, page)) + c.Visit(fmt.Sprintf("?region=%v&searchType=%v&searchKeyword=%v&Page=%v", region, searchType, query, page)) - if closetime { + if isCloseTime, _ := GetCloseTime(region); isCloseTime { status = http.StatusServiceUnavailable - } else if len(profiles) < 1 { - status = http.StatusNotFound - } else { - status = http.StatusOK } return diff --git a/scrapers/ScrapeGuild.go b/scrapers/ScrapeGuild.go index 4a753bb..7f2b807 100644 --- a/scrapers/ScrapeGuild.go +++ b/scrapers/ScrapeGuild.go @@ -12,15 +12,11 @@ import ( ) func ScrapeGuild(region, name string) (guildProfile models.GuildProfile, status int) { - c := collyFactory() + c := newScraper(region) guildProfile.Region = region status = http.StatusNotFound - c.OnHTML(closetimeSelector, func(e *colly.HTMLElement) { - status = http.StatusServiceUnavailable - }) - c.OnHTML(`.region_info`, func(e *colly.HTMLElement) { guildProfile.Region = e.Text }) @@ -63,7 +59,11 @@ func ScrapeGuild(region, name string) (guildProfile models.GuildProfile, status guildProfile.Members = append(guildProfile.Members, member) }) - c.Visit(fmt.Sprintf("%v/Adventure/Guild/GuildProfile?guildName=%v®ion=%v", getSiteRoot(region), name, region)) + c.Visit(fmt.Sprintf("/Guild/GuildProfile?guildName=%v®ion=%v", name, region)) + + if isCloseTime, _ := GetCloseTime(region); isCloseTime { + status = http.StatusServiceUnavailable + } return } diff --git a/scrapers/ScrapeGuildSearch.go b/scrapers/ScrapeGuildSearch.go index f40babe..4c750a9 100644 --- a/scrapers/ScrapeGuildSearch.go +++ b/scrapers/ScrapeGuildSearch.go @@ -13,15 +13,13 @@ import ( ) func ScrapeGuildSearch(region, query string, page uint16) (guildProfiles []models.GuildProfile, status int) { - c := collyFactory() - closetime := false + c := newScraper(region) - c.OnHTML(closetimeSelector, func(e *colly.HTMLElement) { - closetime = true - }) + status = http.StatusNotFound c.OnHTML(`.box_list_area li:not(.no_result)`, func(e *colly.HTMLElement) { createdOn := utils.ParseDate(e.ChildText(".date")) + status = http.StatusOK guildProfile := models.GuildProfile{ Name: e.ChildText(".guild_title a"), @@ -50,14 +48,10 @@ func ScrapeGuildSearch(region, query string, page uint16) (guildProfiles []model guildProfiles = append(guildProfiles, guildProfile) }) - c.Visit(fmt.Sprintf("%v/Adventure/Guild?region=%v&page=%v&searchText=%v", getSiteRoot(region), region, page, query)) + c.Visit(fmt.Sprintf("/Guild?region=%v&page=%v&searchText=%v", region, page, query)) - if closetime { + if isCloseTime, _ := GetCloseTime(region); isCloseTime { status = http.StatusServiceUnavailable - } else if len(guildProfiles) < 1 { - status = http.StatusNotFound - } else { - status = http.StatusOK } return diff --git a/scrapers/extractProfileTarget.go b/scrapers/extractProfileTarget.go new file mode 100644 index 0000000..6976386 --- /dev/null +++ b/scrapers/extractProfileTarget.go @@ -0,0 +1,11 @@ +package scrapers + +import ( + "net/url" +) + +func extractProfileTarget(link string) string { + u, _ := url.Parse(link) + m, _ := url.ParseQuery(u.RawQuery) + return m["profileTarget"][0] +} diff --git a/scrapers/scraper.go b/scrapers/scraper.go new file mode 100644 index 0000000..16f4f6a --- /dev/null +++ b/scrapers/scraper.go @@ -0,0 +1,53 @@ +package scrapers + +import ( + "fmt" + "log" + "time" + + "bdo-rest-api/config" + + colly "github.com/gocolly/colly/v2" +) + +type scraper struct { + c *colly.Collector + region string +} + +func newScraper(region string) (s scraper) { + s.region = region + s.c = colly.NewCollector() + s.c.SetRequestTimeout(time.Minute / 2) + + if len(config.GetProxyList()) > 0 { + s.c.SetProxyFunc(config.GetProxySwitcher()) + } + + s.c.OnRequest(func(r *colly.Request) { + if config.GetVerbosity() { + log.Println("Visiting", r.URL) + } + }) + + s.OnHTML(`.closetime_wrap`, func(e *colly.HTMLElement) { + setCloseTime(region) + }) + + return +} + +func (s *scraper) OnHTML(goquerySelector string, f colly.HTMLCallback) { + s.c.OnHTML(goquerySelector, f) +} + +func (s *scraper) Visit(URL string) error { + regionPrefix := map[string]string{ + "EU": "naeu.playblackdesert.com/en-US", + "KR": "kr.playblackdesert.com/ko-KR", + "SA": "sa.playblackdesert.com/pt-BR", + "US": "naeu.playblackdesert.com/en-US", + }[s.region] + + return s.c.Visit(fmt.Sprintf("https://www.%v/Adventure%v", regionPrefix, URL)) +} diff --git a/scrapers/scrapers.go b/scrapers/scrapers.go deleted file mode 100644 index 848e764..0000000 --- a/scrapers/scrapers.go +++ /dev/null @@ -1,63 +0,0 @@ -package scrapers - -import ( - "log" - "net/url" - "time" - - "github.com/gocolly/colly/v2" - "github.com/gocolly/colly/v2/proxy" -) - -const closetimeSelector = ".closetime_wrap" - -var proxies = make([]string, 0) -var proxySwitcher colly.ProxyFunc - -func PushProxies(args ...string) { - proxies = append(proxies, args...) - proxySwitcher, _ = proxy.RoundRobinProxySwitcher(proxies...) -} - -var verbose = false - -func SetVerbose(v bool) { - verbose = v -} - -func extractProfileTarget(link string) string { - u, _ := url.Parse(link) - m, _ := url.ParseQuery(u.RawQuery) - return m["profileTarget"][0] -} - -func getSiteRoot(region string) string { - if "SA" == region { - return "https://www.sa.playblackdesert.com/pt-BR" - } - - if "KR" == region { - return "https://www.kr.playblackdesert.com/ko-KR" - } - - return "https://www.naeu.playblackdesert.com/en-US" -} - -func collyFactory() (c *colly.Collector) { - c = colly.NewCollector() - c.SetRequestTimeout(time.Minute / 2) - - if len(proxies) > 0 { - c.SetProxyFunc(proxySwitcher) - } - - c.OnRequest(func(r *colly.Request) { - if !verbose { - return - } - - log.Println("Visiting", r.URL) - }) - - return -} diff --git a/translators/TranslateClassName.go b/translators/TranslateClassName.go index 62134ad..63b1bd5 100644 --- a/translators/TranslateClassName.go +++ b/translators/TranslateClassName.go @@ -8,6 +8,7 @@ var classNameTranslationMap = map[string]string{ "Cavaleira Negra": "Dark Knight", "Corsária": "Corsair", "Domadora": "Tamer", + "Erudite": "Scholar", "Feiticeira": "Sorceress", "Guardiã": "Guardian", "Guerreiro": "Warrior", @@ -38,6 +39,7 @@ var classNameTranslationMap = map[string]string{ "샤이": "Shai", "세이지": "Sage", "소서러": "Sorceress", + "스칼라": "Scholar", "아처": "Archer", "우사": "Woosa", "워리어": "Warrior", diff --git a/translators/TranslateMisc.go b/translators/TranslateMisc.go new file mode 100644 index 0000000..f494248 --- /dev/null +++ b/translators/TranslateMisc.go @@ -0,0 +1,14 @@ +package translators + +var miscTranslationMap = map[string]string{ + "Não está alistado em nenhuma guilda.": "Not in a guild", + "Privado": "Private", + "가입된 길드가 없습니다.": "Not in a guild", + "비공개": "Private", +} + +func TranslateMisc(guildMembershipStatus *string) { + if val, ok := miscTranslationMap[*guildMembershipStatus]; ok { + *guildMembershipStatus = val + } +} diff --git a/utils/FormatDateForHeaders.go b/utils/FormatDateForHeaders.go new file mode 100644 index 0000000..912f5f1 --- /dev/null +++ b/utils/FormatDateForHeaders.go @@ -0,0 +1,7 @@ +package utils + +import "time" + +func FormatDateForHeaders(date time.Time) string { + return date.Format(time.RFC1123Z) +} diff --git a/validators/ValidateAdventurerName.go b/validators/ValidateAdventurerNameQueryParam.go similarity index 68% rename from validators/ValidateAdventurerName.go rename to validators/ValidateAdventurerNameQueryParam.go index 6615204..9d5651c 100644 --- a/validators/ValidateAdventurerName.go +++ b/validators/ValidateAdventurerNameQueryParam.go @@ -7,9 +7,13 @@ import ( // The naming policies in BDO are fucked up // This function only checks the length and allowed symbols -func ValidateAdventurerName(name *string) bool { - if len(*name) < 3 || len(*name) > 16 { - return false +func ValidateAdventurerNameQueryParam(query []string) (name string, ok bool) { + if 1 > len(query) { + return "", false + } + + if len(query[0]) < 3 || len(query[0]) > 16 { + return query[0], false } // Returns false for allowed characters @@ -38,5 +42,5 @@ func ValidateAdventurerName(name *string) bool { return true } - return strings.IndexFunc(*name, f) == -1 + return query[0], strings.IndexFunc(query[0], f) == -1 } diff --git a/validators/ValidateAdventurerNameQueryParam_test.go b/validators/ValidateAdventurerNameQueryParam_test.go new file mode 100644 index 0000000..1cd0421 --- /dev/null +++ b/validators/ValidateAdventurerNameQueryParam_test.go @@ -0,0 +1,31 @@ +package validators + +import "testing" + +func TestValidateAdventurerNameQueryParam(t *testing.T) { + tests := []struct { + expectedName string + expectedOk bool + input []string + }{ + {input: []string{"1Number"}, expectedName: "1Number", expectedOk: true}, // Starts with a number + {input: []string{"Adventurer_123"}, expectedName: "Adventurer_123", expectedOk: true}, + {input: []string{"JohnDoe"}, expectedName: "JohnDoe", expectedOk: true}, + {input: []string{"Name1", "Name2"}, expectedName: "Name1", expectedOk: true}, + {input: []string{"고대신"}, expectedName: "고대신", expectedOk: true}, // Adventurer name with Korean characters + + {input: []string{""}, expectedName: "", expectedOk: false}, // Empty adventurer name + {input: []string{"Ad"}, expectedName: "Ad", expectedOk: false}, // Too short + {input: []string{"Adventurer With Spaces"}, expectedName: "Adventurer With Spaces", expectedOk: false}, // Contains spaces + {input: []string{"AdventurerNameTooLong12345"}, expectedName: "AdventurerNameTooLong12345", expectedOk: false}, // Too long + {input: []string{"Name$"}, expectedName: "Name$", expectedOk: false}, // Contains an invalid symbol + {input: []string{}, expectedName: "", expectedOk: false}, + } + + for _, test := range tests { + name, ok := ValidateAdventurerNameQueryParam(test.input) + if name != test.expectedName || ok != test.expectedOk { + t.Errorf("Input: %v, Expected: %v %v, Got: %v %v", test.input, test.expectedName, test.expectedOk, name, ok) + } + } +} diff --git a/validators/ValidateAdventurerName_test.go b/validators/ValidateAdventurerName_test.go deleted file mode 100644 index 498507d..0000000 --- a/validators/ValidateAdventurerName_test.go +++ /dev/null @@ -1,30 +0,0 @@ -package validators - -import "testing" - -func TestValidateAdventurerName(t *testing.T) { - tests := []struct { - input string - expected bool - }{ - // Valid adventurer names - {input: "1Number", expected: true}, // Starts with a number - {input: "Adventurer_123", expected: true}, - {input: "JohnDoe", expected: true}, - {input: "고대신", expected: true}, // Adventurer name with Korean characters - - // Invalid adventurer names - {input: "", expected: false}, // Empty adventurer name - {input: "Ad", expected: false}, // Too short - {input: "Adventurer With Spaces", expected: false}, // Contains spaces - {input: "AdventurerNameTooLong12345", expected: false}, // Too long - {input: "Name$", expected: false}, // Contains an invalid symbol '$' - } - - for _, test := range tests { - result := ValidateAdventurerName(&test.input) - if result != test.expected { - t.Errorf("Input: %v, Expected: %v, Got: %v", test.input, test.expected, result) - } - } -} diff --git a/validators/ValidateGuildName.go b/validators/ValidateGuildNameQueryParam.go similarity index 70% rename from validators/ValidateGuildName.go rename to validators/ValidateGuildNameQueryParam.go index a207507..1538675 100644 --- a/validators/ValidateGuildName.go +++ b/validators/ValidateGuildNameQueryParam.go @@ -8,9 +8,15 @@ import ( // The naming policies in BDO are fucked up // This function only checks the length and allowed symbols // I also assumed that the allowed symbols are the same as for adventurer names -func ValidateGuildName(name *string) bool { - if len(*name) < 2 { - return false +func ValidateGuildNameQueryParam(query []string) (guildName string, ok bool) { + if 1 > len(query) { + return "", false + } + + guildName = strings.ToLower(query[0]) + + if len(guildName) < 2 { + return guildName, false } // Returns false for allowed characters @@ -39,5 +45,5 @@ func ValidateGuildName(name *string) bool { return true } - return strings.IndexFunc(*name, f) == -1 + return guildName, strings.IndexFunc(guildName, f) == -1 } diff --git a/validators/ValidateGuildNameQueryParam_test.go b/validators/ValidateGuildNameQueryParam_test.go new file mode 100644 index 0000000..d718563 --- /dev/null +++ b/validators/ValidateGuildNameQueryParam_test.go @@ -0,0 +1,30 @@ +package validators + +import "testing" + +func TestValidateGuildNameQueryParam(t *testing.T) { + tests := []struct { + expectedName string + expectedOk bool + input []string + }{ + {input: []string{"1NumberGuild"}, expectedName: "1numberguild", expectedOk: true}, // Contains a number + {input: []string{"Adventure_Guild"}, expectedName: "adventure_guild", expectedOk: true}, + {input: []string{"FirstGuild", "SecondGuild"}, expectedName: "firstguild", expectedOk: true}, + {input: []string{"MyGuild"}, expectedName: "myguild", expectedOk: true}, + {input: []string{"고대신"}, expectedName: "고대신", expectedOk: true}, // Guild name with Korean characters + + {input: []string{""}, expectedName: "", expectedOk: false}, // Empty guild name + {input: []string{"A Guild With Spaces"}, expectedName: "a guild with spaces", expectedOk: false}, // Contains spaces + {input: []string{"Some$"}, expectedName: "some$", expectedOk: false}, // Contains an invalid symbol + {input: []string{"x"}, expectedName: "x", expectedOk: false}, // Too short + {input: []string{}, expectedName: "", expectedOk: false}, + } + + for _, test := range tests { + name, ok := ValidateGuildNameQueryParam(test.input) + if name != test.expectedName || ok != test.expectedOk { + t.Errorf("Input: %v, Expected: %v %v, Got: %v %v", test.input, test.expectedName, test.expectedOk, name, ok) + } + } +} diff --git a/validators/ValidateGuildName_test.go b/validators/ValidateGuildName_test.go deleted file mode 100644 index d0279c3..0000000 --- a/validators/ValidateGuildName_test.go +++ /dev/null @@ -1,29 +0,0 @@ -package validators - -import "testing" - -func TestValidateGuildName(t *testing.T) { - tests := []struct { - input string - expected bool - }{ - // Valid guild names - {input: "1NumberGuild", expected: true}, // Contains a number - {input: "Adventure_Guild", expected: true}, - {input: "MyGuild", expected: true}, - {input: "고대신", expected: true}, // Guild name with Korean characters - - // Invalid guild names - {input: "", expected: false}, // Empty guild name - {input: "A Guild With Spaces", expected: false}, // Contains spaces - {input: "Some$", expected: false}, // Contains an invalid symbol '$' - {input: "x", expected: false}, // Too short - } - - for _, test := range tests { - result := ValidateGuildName(&test.input) - if result != test.expected { - t.Errorf("Input: %v, Expected: %v, Got: %v", test.input, test.expected, result) - } - } -} diff --git a/validators/ValidatePage.go b/validators/ValidatePage.go deleted file mode 100644 index 0e6ae22..0000000 --- a/validators/ValidatePage.go +++ /dev/null @@ -1,8 +0,0 @@ -package validators - -import "strconv" - -func ValidatePage(p *string) bool { - page, ok := strconv.Atoi(*p) - return ok == nil && page > 0 -} diff --git a/validators/ValidatePageQueryParam.go b/validators/ValidatePageQueryParam.go new file mode 100644 index 0000000..6a24efd --- /dev/null +++ b/validators/ValidatePageQueryParam.go @@ -0,0 +1,15 @@ +package validators + +import "strconv" + +func ValidatePageQueryParam(query []string) (page uint16) { + if 1 > len(query) { + return 1 + } + + if page, err := strconv.Atoi(query[0]); nil == err { + return uint16(max(page, 1)) + } + + return 1 +} diff --git a/validators/ValidatePageQueryParam_test.go b/validators/ValidatePageQueryParam_test.go new file mode 100644 index 0000000..3d23fce --- /dev/null +++ b/validators/ValidatePageQueryParam_test.go @@ -0,0 +1,25 @@ +package validators + +import ( + "testing" +) + +func TestValidatePageQueryParam(t *testing.T) { + tests := []struct { + input []string + expected uint16 + }{ + {input: []string{}, expected: 1}, + {input: []string{"5"}, expected: 5}, + {input: []string{"invalid"}, expected: 1}, + {input: []string{"32767"}, expected: 32767}, + {input: []string{"-7"}, expected: 1}, + } + + for _, test := range tests { + result := ValidatePageQueryParam(test.input) + if result != test.expected { + t.Errorf("For input %v, expected %d, but got %d", test.input, test.expected, result) + } + } +} diff --git a/validators/ValidatePage_test.go b/validators/ValidatePage_test.go deleted file mode 100644 index f9d7016..0000000 --- a/validators/ValidatePage_test.go +++ /dev/null @@ -1,27 +0,0 @@ -package validators - -import "testing" - -func TestValidatePage(t *testing.T) { - tests := []struct { - input string - expected bool - }{ - // Valid pages - {input: "1", expected: true}, - {input: "10", expected: true}, - {input: "999", expected: true}, - - // Invalid pages - {input: "-5", expected: false}, - {input: "0", expected: false}, - {input: "abc", expected: false}, - } - - for _, test := range tests { - result := ValidatePage(&test.input) - if result != test.expected { - t.Errorf("Input: %v, Expected: %v, Got: %v", test.input, test.expected, result) - } - } -} diff --git a/validators/ValidateProfileTarget.go b/validators/ValidateProfileTarget.go deleted file mode 100644 index 89a2708..0000000 --- a/validators/ValidateProfileTarget.go +++ /dev/null @@ -1,7 +0,0 @@ -package validators - -// Check that the length is at least 150 characters -// I don't actually know how long it should be, but the length varies -func ValidateProfileTarget(profileTarget *string) bool { - return len(*profileTarget) >= 150 -} diff --git a/validators/ValidateProfileTargetQueryParam.go b/validators/ValidateProfileTargetQueryParam.go new file mode 100644 index 0000000..6a7cca1 --- /dev/null +++ b/validators/ValidateProfileTargetQueryParam.go @@ -0,0 +1,11 @@ +package validators + +// Check that the length is at least 150 characters +// I don't actually know how long it should be, but the length varies +func ValidateProfileTargetQueryParam(query []string) (profileTarget string, ok bool) { + if 1 > len(query) { + return "", false + } + + return query[0], len(query[0]) >= 150 +} diff --git a/validators/ValidateProfileTargetQueryParam_test.go b/validators/ValidateProfileTargetQueryParam_test.go new file mode 100644 index 0000000..7fea75b --- /dev/null +++ b/validators/ValidateProfileTargetQueryParam_test.go @@ -0,0 +1,42 @@ +package validators + +import "testing" + +func TestValidateProfileTargetQueryParam(t *testing.T) { + tests := []struct { + expectedOk bool + expectedPT string + input []string + }{ + // Valid profile targets with lengths >= 150 + {input: []string{repeat("A", 150)}, expectedPT: repeat("A", 150), expectedOk: true}, + {input: []string{repeat("A", 200)}, expectedPT: repeat("A", 200), expectedOk: true}, + + // Invalid profile targets with lengths < 150 + {input: []string{""}, expectedPT: "", expectedOk: false}, + {input: []string{"Short"}, expectedPT: "Short", expectedOk: false}, + {input: []string{repeat("A", 149)}, expectedPT: repeat("A", 149), expectedOk: false}, + + // Query param not provided + {input: []string{}, expectedPT: "", expectedOk: false}, + + // Several profileTargets provided + {input: []string{repeat("A", 150), repeat("B", 150)}, expectedPT: repeat("A", 150), expectedOk: true}, + } + + for _, test := range tests { + pT, ok := ValidateProfileTargetQueryParam(test.input) + if pT != test.expectedPT || ok != test.expectedOk { + t.Errorf("Input: %v, Expected: %v %v, Got: %v %v", test.input, test.expectedPT, test.expectedOk, pT, ok) + } + } +} + +// Helper function to repeat a character n times. +func repeat(s string, n int) string { + var result string + for i := 0; i < n; i++ { + result += s + } + return result +} diff --git a/validators/ValidateProfileTarget_test.go b/validators/ValidateProfileTarget_test.go deleted file mode 100644 index 5aa2ff0..0000000 --- a/validators/ValidateProfileTarget_test.go +++ /dev/null @@ -1,35 +0,0 @@ -package validators - -import "testing" - -func TestValidateProfileTarget(t *testing.T) { - tests := []struct { - input string - expected bool - }{ - // Valid profile targets with lengths >= 150 - {input: repeat("A", 150), expected: true}, // 150-character string - {input: repeat("A", 200), expected: true}, // 200-character string - - // Invalid profile targets with lengths < 150 - {input: "", expected: false}, - {input: "Short", expected: false}, - {input: repeat("A", 149), expected: false}, // 149-character string - } - - for _, test := range tests { - result := ValidateProfileTarget(&test.input) - if result != test.expected { - t.Errorf("Input: %v, Expected: %v, Got: %v", test.input, test.expected, result) - } - } -} - -// Helper function to repeat a character n times. -func repeat(s string, n int) string { - var result string - for i := 0; i < n; i++ { - result += s - } - return result -} diff --git a/validators/ValidateRegion.go b/validators/ValidateRegion.go deleted file mode 100644 index 296fc34..0000000 --- a/validators/ValidateRegion.go +++ /dev/null @@ -1,7 +0,0 @@ -package validators - -func ValidateRegion(r *string) bool { - // TODO: Readd KR region once the translations are ready - // return *r == "EU" || *r == "NA" || *r == "SA" || *r == "KR" - return *r == "EU" || *r == "NA" || *r == "SA" -} diff --git a/validators/ValidateRegionQueryParam.go b/validators/ValidateRegionQueryParam.go new file mode 100644 index 0000000..b7bc81f --- /dev/null +++ b/validators/ValidateRegionQueryParam.go @@ -0,0 +1,17 @@ +package validators + +import ( + "slices" + "strings" +) + +func ValidateRegionQueryParam(query []string) (region string, ok bool) { + if 1 > len(query) { + return "EU", true + } + + region = strings.ToUpper(query[0]) + + // TODO: Add KR region once the translations are ready + return region, slices.Contains([]string{"EU", "NA", "SA"}, region) +} diff --git a/validators/ValidateRegionQueryParam_test.go b/validators/ValidateRegionQueryParam_test.go new file mode 100644 index 0000000..1ac6142 --- /dev/null +++ b/validators/ValidateRegionQueryParam_test.go @@ -0,0 +1,28 @@ +package validators + +import ( + "testing" +) + +func TestValidateRegionQueryParameter(t *testing.T) { + tests := []struct { + expectedOk bool + expectedRegion string + input []string + }{ + {input: []string{}, expectedRegion: "EU", expectedOk: true}, + {input: []string{"NA"}, expectedRegion: "NA", expectedOk: true}, + {input: []string{"na"}, expectedRegion: "NA", expectedOk: true}, + {input: []string{"SA"}, expectedRegion: "SA", expectedOk: true}, + {input: []string{"EU"}, expectedRegion: "EU", expectedOk: true}, + {input: []string{"KR"}, expectedRegion: "KR", expectedOk: false}, + {input: []string{"NA", "SA"}, expectedRegion: "NA", expectedOk: true}, // Takes the first region in case of multiple regions + } + + for _, test := range tests { + result, ok := ValidateRegionQueryParam(test.input) + if result != test.expectedRegion || ok != test.expectedOk { + t.Errorf("For input %v, expected %v %v, but got %v %v", test.input, test.expectedRegion, test.expectedOk, result, ok) + } + } +} diff --git a/validators/ValidateRegion_test.go b/validators/ValidateRegion_test.go deleted file mode 100644 index 78202df..0000000 --- a/validators/ValidateRegion_test.go +++ /dev/null @@ -1,27 +0,0 @@ -package validators - -import "testing" - -func TestValidateRegion(t *testing.T) { - tests := []struct { - input string - expected bool - }{ - // Valid regions - {input: "EU", expected: true}, - {input: "NA", expected: true}, - {input: "SA", expected: true}, - - // Invalid regions - {input: "", expected: false}, - {input: "JP", expected: false}, - {input: "KR", expected: false}, - } - - for _, test := range tests { - result := ValidateRegion(&test.input) - if result != test.expected { - t.Errorf("Input: %v, Expected: %v, Got: %v", test.input, test.expected, result) - } - } -} diff --git a/validators/ValidateSearchType.go b/validators/ValidateSearchType.go deleted file mode 100644 index e845fb8..0000000 --- a/validators/ValidateSearchType.go +++ /dev/null @@ -1,5 +0,0 @@ -package validators - -func ValidateSearchType(s *string) bool { - return *s == "characterName" || *s == "familyName" -} diff --git a/validators/ValidateSearchTypeQueryParam.go b/validators/ValidateSearchTypeQueryParam.go new file mode 100644 index 0000000..664b9b4 --- /dev/null +++ b/validators/ValidateSearchTypeQueryParam.go @@ -0,0 +1,16 @@ +package validators + +func ValidateSearchTypeQueryParam(query []string) (searchType uint8) { + if 1 > len(query) { + return 2 + } + + if query[0] == "characterName" { + return map[string]uint8{ + "characterName": 1, + "familyName": 2, + }[query[0]] + } + + return 2 +} diff --git a/validators/ValidateSearchTypeQueryParam_test.go b/validators/ValidateSearchTypeQueryParam_test.go new file mode 100644 index 0000000..bb019e8 --- /dev/null +++ b/validators/ValidateSearchTypeQueryParam_test.go @@ -0,0 +1,22 @@ +package validators + +import "testing" + +func TestValidateSearchTypeQueryParam(t *testing.T) { + tests := []struct { + expected uint8 + input []string + }{ + {input: []string{}, expected: 2}, + {input: []string{"characterName"}, expected: 1}, + {input: []string{"invalidType"}, expected: 2}, + {input: []string{"familyName"}, expected: 2}, + } + + for _, test := range tests { + result := ValidateSearchTypeQueryParam(test.input) + if result != test.expected { + t.Errorf("For input %v, expected %v, but got %v", test.input, test.expected, result) + } + } +} diff --git a/validators/ValidateSearchType_test.go b/validators/ValidateSearchType_test.go deleted file mode 100644 index 2059e0a..0000000 --- a/validators/ValidateSearchType_test.go +++ /dev/null @@ -1,26 +0,0 @@ -package validators - -import "testing" - -func TestValidateSearchType(t *testing.T) { - tests := []struct { - input string - expected bool - }{ - // Valid search types - {input: "characterName", expected: true}, - {input: "familyName", expected: true}, - - // Invalid search types - {input: "", expected: false}, - {input: "invalidType", expected: false}, - {input: "someOtherType", expected: false}, - } - - for _, test := range tests { - result := ValidateSearchType(&test.input) - if result != test.expected { - t.Errorf("Input: %v, Expected: %v, Got: %v", test.input, test.expected, result) - } - } -}