feat(47): add initial gitlab scraper with basic repo count metric (#72)

* feat: gitlab scraper first pass WIP

* feat: fix workspace errors and introduce genqlient

* chore: fix bearerauth and add new queries.

* chore: fix broken graphql queries and genqclient

* chore: merge changes from main branch

* feat: cleanup unworking code for small batch delivery and run checks

* chore: fix up config yaml to include gitlab configuration

* chore: fix genqlient make command

* chore: remove main.go & go files at root of repository

* chore: re-add ds_store back to ignore

* chore: update alpine image as .2 is removed from docker

---------

Co-authored-by: shwaddell28 <shwaddell28@gmail.com>
Co-authored-by: Adriel Perkins <adrielp@liatrio.com>

.gitignore

@@ -26,3 +26,5 @@ dist/
 # MacOS
 .DS_Store
 
+# JetBrains IDEs
+.idea/

Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.18.2 as cacerts
+FROM alpine:3.18.3 as cacerts
 RUN apk --update add --no-cache ca-certificates
 
 FROM scratch

Makefile.Common

@@ -9,7 +9,8 @@ else
 	.SHELLFLAGS = -o pipefail -c
 endif
 
-GENQLIENT_DIR = $(shell find . -type f -name 'genqlient.yaml' -exec dirname {} \;)
+#GENQLIENT_DIRS = $(shell find . -type f -name 'genqlient.yaml' -exec dirname {} \;)
+GENQLIENT_DIRS = $(shell find . -type f -name 'genqlient.yaml' -exec dirname {} \;)
 
 # SRC_ROOT is the top of the source tree.
 SRC_ROOT := $(shell git rev-parse --show-toplevel)
@@ -37,6 +38,9 @@ fmt:
 
 .PHONY: genqlient
 genqlient:
-	cd $(GENQLIENT_DIR) && \
+	for dir in $(GENQLIENT_DIRS); do \
+		pushd $$dir && \
 		[[ -f genqlient.yaml && -f genqlient.graphql && -f schema.graphql ]] && \
-		go run github.com/Khan/genqlient@latest
+		go run github.com/Khan/genqlient@latest && \
+		popd; \
+	done

config/config.yaml

@@ -5,16 +5,20 @@ extensions:
         endpoint: 0.0.0.0:1777
     zpages:
         endpoint: 0.0.0.0:55679
-  ## Uncomment below if you want to enable authentication to GitHub
-  ## Presume that GH_USER and GH_PAT are set as env variables.
-  #basicauth/github:
-  #    client_auth:
-  #        username: ${env:GH_USER}
-  #        password: ${env:GH_PAT}
-  #basicauth/grafana:
-  #  client_auth:
-  #    username: ${env:GRAF_USER}
-  #    password: ${env:GRAF_PAT}
+    ## Uncomment below if you want to enable authentication to GitHub
+    ## Presume that GH_USER and GH_PAT are set as env variables.
+    #basicauth/gitlab:
+    #    client_auth:
+    #        username: "${env:GITLAB_USER}"
+    #        password: ${env:GITLAB_PAT}
+    ## Uncomment below if you want to enable authentication to Grafana
+    #basicauth/grafana:
+    #  client_auth:
+    #    username: ${env:GRAF_USER}
+    #    password: ${env:GRAF_PAT}
+    ## Uncomment below if you want to enable authentication to GitLab
+    #bearertokenauth/gitlab:
+    #  token: ${env:GL_PAT}
 
 receivers:
     otlp:
@@ -38,10 +42,17 @@ receivers:
         scrapers:
             github:
                 github_org: ${env:GITHUB_ORG}
-              ## Uncomment if you want to view private authentication required
-              ## GitHub data or and decrease rate limiting.
-              #auth:
-              #    authenticator: basicauth/github
+                search_query: org:${env.GITHUB_ORG}
+            ## Uncomment if you want to view private authentication required
+            ## GitHub data or and decrease rate limiting.
+            #auth:
+            #    authenticator: basicauth/github
+            gitlab:
+                gitlab_org: ${env:GL_ORG}
+            ## Uncomment if you want to view private authentication required
+            ## GitHub data or and decrease rate limiting.
+            #auth:
+            #  authenticator: bearertokenauth/gitlab
 
 processors:
     batch:
@@ -64,16 +75,17 @@ exporters:
 
 service:
     ## Add basicauth/* if you want to use basic auth extensions
+    ## Add bearertokenauth/* if you want to use bearer token auth extensions
     extensions: [health_check, pprof, zpages]
     pipelines:
         metrics:
             receivers: [otlp, prometheus, gitprovider]
-            processors: [resourcedetection,batch]
+            processors: [resourcedetection, batch]
             exporters: [logging]
 
         traces:
             receivers: [otlp]
-            processors: [resourcedetection,batch]
+            processors: [resourcedetection, batch]
             exporters: [logging]
 
     telemetry:

config/manifest.yaml

@@ -13,6 +13,7 @@ extensions:
     - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/basicauthextension v0.83.0
     - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckextension v0.83.0
     - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/pprofextension v0.83.0
+    - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/extension/bearertokenauthextension v0.83.0
 
 exporters:
     - gomod: go.opentelemetry.io/collector/exporter/loggingexporter v0.83.0

pkg/receiver/gitproviderreceiver/factory.go

@@ -16,6 +16,7 @@ import (
 	"github.com/liatrio/liatrio-otel-collector/pkg/receiver/gitproviderreceiver/internal"
 	"github.com/liatrio/liatrio-otel-collector/pkg/receiver/gitproviderreceiver/internal/metadata"
 	"github.com/liatrio/liatrio-otel-collector/pkg/receiver/gitproviderreceiver/internal/scraper/githubscraper"
+	"github.com/liatrio/liatrio-otel-collector/pkg/receiver/gitproviderreceiver/internal/scraper/gitlabscraper"
 )
 
 // This file implements a factory for the git provider receiver
@@ -28,6 +29,7 @@ import (
 var (
 	scraperFactories = map[string]internal.ScraperFactory{
 		githubscraper.TypeStr: &githubscraper.Factory{},
+		gitlabscraper.TypeStr: &gitlabscraper.Factory{},
 	}
 
 	errConfigNotValid = errors.New("configuration is not valid for the git provider receiver")

pkg/receiver/gitproviderreceiver/internal/scraper/gitlabscraper/config.go

@@ -0,0 +1,20 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package gitlabscraper
+
+import (
+	"go.opentelemetry.io/collector/config/confighttp"
+
+	"github.com/liatrio/liatrio-otel-collector/pkg/receiver/gitproviderreceiver/internal"
+	"github.com/liatrio/liatrio-otel-collector/pkg/receiver/gitproviderreceiver/internal/metadata"
+)
+
+// Config relating to GitLab Metric Scraper.
+type Config struct {
+	metadata.MetricsBuilderConfig `mapstructure:",squash"`
+	confighttp.HTTPClientSettings `mapstructure:",squash"`
+	internal.ScraperConfig
+	// GitLabOrg is the name of the GitLab organization to srape (gitlab scraper only)
+	GitLabOrg string `mapstructure:"gitlab_org"`
+}

pkg/receiver/gitproviderreceiver/internal/scraper/gitlabscraper/factory.go

@@ -0,0 +1,50 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package gitlabscraper
+
+import (
+	"context"
+	"time"
+
+	"go.opentelemetry.io/collector/config/confighttp"
+	"go.opentelemetry.io/collector/receiver"
+	"go.opentelemetry.io/collector/receiver/scraperhelper"
+
+	"github.com/liatrio/liatrio-otel-collector/pkg/receiver/gitproviderreceiver/internal"
+	"github.com/liatrio/liatrio-otel-collector/pkg/receiver/gitproviderreceiver/internal/metadata"
+)
+
+// This file implements factory for the GitLab Scraper as part of the  Git Provider Receiver
+
+const (
+	// TypeStr is the value of "type" key in configuration.
+	TypeStr            = "gitlab"
+	defaultHTTPTimeout = 15 * time.Second
+)
+
+type Factory struct{}
+
+func (f *Factory) CreateDefaultConfig() internal.Config {
+	return &Config{
+		MetricsBuilderConfig: metadata.DefaultMetricsBuilderConfig(),
+		HTTPClientSettings: confighttp.HTTPClientSettings{
+			Timeout: defaultHTTPTimeout,
+		},
+	}
+}
+
+func (f *Factory) CreateMetricsScraper(
+	ctx context.Context,
+	params receiver.CreateSettings,
+	cfg internal.Config,
+) (scraperhelper.Scraper, error) {
+	conf := cfg.(*Config)
+	s := newGitLabScraper(ctx, params, conf)
+
+	return scraperhelper.NewScraper(
+		TypeStr,
+		s.scrape,
+		scraperhelper.WithStart(s.start),
+	)
+}