watchdog: migrate to Cobra and Viper for config management; search /etc for configs
Signed-off-by: NotAShelf <raf@notashelf.dev> Change-Id: I65dbf466cb030dccc7025585d6282bd26a6a6964
This commit is contained in:
parent
951ed9c36f
commit
f988174145
GPG key ID:
29D95B64378DB4BF
9 changed files with 187 additions and 61 deletions
|
|
@ -1,15 +1,92 @@
|
|||
package watchdog
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"log"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/spf13/viper"
|
||||
"notashelf.dev/watchdog/internal/config"
|
||||
)
|
||||
|
||||
func Main() {
|
||||
configPath := flag.String("config", "config.yaml", "path to config file")
|
||||
flag.Parse()
|
||||
var (
|
||||
cfgFile string
|
||||
cfg *config.Config
|
||||
)
|
||||
|
||||
if err := Run(*configPath); err != nil {
|
||||
log.Fatalf("Error: %v", err)
|
||||
var rootCmd = &cobra.Command{
|
||||
Use: "watchdog",
|
||||
Short: "Privacy-first web analytics with Prometheus metrics",
|
||||
Long: `Watchdog is a lightweight, privacy-first analytics system that aggregates web traffic data.`,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
return Run(cfg)
|
||||
},
|
||||
}
|
||||
|
||||
func init() {
|
||||
cobra.OnInitialize(initConfig)
|
||||
|
||||
rootCmd.PersistentFlags().
|
||||
StringVar(&cfgFile, "config", "", "config file (default: config.yaml)")
|
||||
rootCmd.PersistentFlags().String("listen-addr", "", "server listen address (overrides config)")
|
||||
rootCmd.PersistentFlags().String("metrics-path", "", "metrics endpoint path (overrides config)")
|
||||
rootCmd.PersistentFlags().
|
||||
String("ingestion-path", "", "ingestion endpoint path (overrides config)")
|
||||
|
||||
// Bind flags to viper
|
||||
viper.BindPFlag("server.listen_addr", rootCmd.PersistentFlags().Lookup("listen-addr"))
|
||||
viper.BindPFlag("server.metrics_path", rootCmd.PersistentFlags().Lookup("metrics-path"))
|
||||
viper.BindPFlag("server.ingestion_path", rootCmd.PersistentFlags().Lookup("ingestion-path"))
|
||||
}
|
||||
|
||||
func initConfig() {
|
||||
if cfgFile != "" {
|
||||
// Use config file from the flag
|
||||
viper.SetConfigFile(cfgFile)
|
||||
} else {
|
||||
// Search for config in current directory
|
||||
viper.AddConfigPath(".")
|
||||
viper.AddConfigPath("/etc/watchdog")
|
||||
viper.SetConfigName("config")
|
||||
viper.SetConfigType("yaml")
|
||||
}
|
||||
|
||||
// Environment variables
|
||||
viper.SetEnvPrefix("WATCHDOG")
|
||||
viper.AutomaticEnv()
|
||||
viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
|
||||
|
||||
// Read config file
|
||||
if err := viper.ReadInConfig(); err != nil {
|
||||
if _, ok := err.(viper.ConfigFileNotFoundError); ok {
|
||||
fmt.Fprintf(
|
||||
os.Stderr,
|
||||
"Config file not found, using defaults and environment variables\n",
|
||||
)
|
||||
} else {
|
||||
fmt.Fprintf(os.Stderr, "Error reading config file: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// Unmarshal into config struct
|
||||
cfg = &config.Config{}
|
||||
if err := viper.Unmarshal(cfg); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Failed to parse config: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// Validate config
|
||||
if err := cfg.Validate(); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Config validation failed: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func Main() {
|
||||
if err := rootCmd.Execute(); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -21,13 +21,7 @@ import (
|
|||
"notashelf.dev/watchdog/internal/normalize"
|
||||
)
|
||||
|
||||
func Run(configPath string) error {
|
||||
// Load configuration
|
||||
cfg, err := config.Load(configPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to load config: %w", err)
|
||||
}
|
||||
|
||||
func Run(cfg *config.Config) error {
|
||||
log.Printf("Loaded config for domains: %v", cfg.Site.Domains)
|
||||
|
||||
// Initialize components
|
||||
|
|
@ -35,7 +29,7 @@ func Run(configPath string) error {
|
|||
pathRegistry := aggregate.NewPathRegistry(cfg.Limits.MaxPaths)
|
||||
refRegistry := normalize.NewReferrerRegistry(cfg.Limits.MaxSources)
|
||||
eventRegistry := aggregate.NewCustomEventRegistry(cfg.Limits.MaxCustomEvents)
|
||||
metricsAgg := aggregate.NewMetricsAggregator(pathRegistry, eventRegistry, *cfg)
|
||||
metricsAgg := aggregate.NewMetricsAggregator(pathRegistry, eventRegistry, cfg)
|
||||
|
||||
// HLL state persistence is handled automatically if salt_rotation is configured
|
||||
if cfg.Site.SaltRotation != "" {
|
||||
|
|
@ -48,7 +42,7 @@ func Run(configPath string) error {
|
|||
|
||||
// Create HTTP handlers
|
||||
ingestionHandler := api.NewIngestionHandler(
|
||||
*cfg,
|
||||
cfg,
|
||||
pathNormalizer,
|
||||
pathRegistry,
|
||||
refRegistry,
|
||||
|
|
|
|||
13
go.mod
13
go.mod
|
|
@ -7,6 +7,8 @@ require gopkg.in/yaml.v3 v3.0.1
|
|||
require (
|
||||
github.com/axiomhq/hyperloglog v0.2.6
|
||||
github.com/prometheus/client_golang v1.23.2
|
||||
github.com/spf13/cobra v1.10.2
|
||||
github.com/spf13/viper v1.21.0
|
||||
golang.org/x/net v0.51.0
|
||||
)
|
||||
|
||||
|
|
@ -14,13 +16,24 @@ require (
|
|||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/dgryski/go-metro v0.0.0-20250106013310-edb8663e5e33 // indirect
|
||||
github.com/fsnotify/fsnotify v1.9.0 // indirect
|
||||
github.com/go-viper/mapstructure/v2 v2.5.0 // indirect
|
||||
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||
github.com/kamstrup/intmap v0.5.2 // indirect
|
||||
github.com/kylelemons/godebug v1.1.0 // indirect
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
|
||||
github.com/prometheus/client_model v0.6.2 // indirect
|
||||
github.com/prometheus/common v0.67.5 // indirect
|
||||
github.com/prometheus/procfs v0.20.1 // indirect
|
||||
github.com/sagikazarmark/locafero v0.12.0 // indirect
|
||||
github.com/spf13/afero v1.15.0 // indirect
|
||||
github.com/spf13/cast v1.10.0 // indirect
|
||||
github.com/spf13/pflag v1.0.10 // indirect
|
||||
github.com/subosito/gotenv v1.6.0 // indirect
|
||||
go.yaml.in/yaml/v2 v2.4.3 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
golang.org/x/sys v0.41.0 // indirect
|
||||
golang.org/x/text v0.34.0 // indirect
|
||||
google.golang.org/protobuf v1.36.11 // indirect
|
||||
)
|
||||
|
|
|
|||
60
go.sum
60
go.sum
|
|
@ -4,39 +4,81 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
|||
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dgryski/go-metro v0.0.0-20250106013310-edb8663e5e33 h1:ucRHb6/lvW/+mTEIGbvhcYU3S8+uSNkuMjx/qZFfhtM=
|
||||
github.com/dgryski/go-metro v0.0.0-20250106013310-edb8663e5e33/go.mod h1:c9O8+fpSOX1DM8cPNSkX/qsBWdkD4yd2dpciOWQjpBw=
|
||||
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
|
||||
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
|
||||
github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
|
||||
github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
|
||||
github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro=
|
||||
github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
||||
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||
github.com/kamstrup/intmap v0.5.2 h1:qnwBm1mh4XAnW9W9Ue9tZtTff8pS6+s6iKF6JRIV2Dk=
|
||||
github.com/kamstrup/intmap v0.5.2/go.mod h1:gWUVWHKzWj8xpJVFf5GC0O26bWmv3GqdnIX/LMT6Aq4=
|
||||
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
|
||||
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
|
||||
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
|
||||
github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
|
||||
github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
|
||||
github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
|
||||
github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
|
||||
github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
|
||||
github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs=
|
||||
github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA=
|
||||
github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTUGI4=
|
||||
github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw=
|
||||
github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg=
|
||||
github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
|
||||
github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc=
|
||||
github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo=
|
||||
go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
|
||||
go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
|
||||
github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
|
||||
github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
|
||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/sagikazarmark/locafero v0.12.0 h1:/NQhBAkUb4+fH1jivKHWusDYFjMOOKU88eegjfxfHb4=
|
||||
github.com/sagikazarmark/locafero v0.12.0/go.mod h1:sZh36u/YSZ918v0Io+U9ogLYQJ9tLLBmM4eneO6WwsI=
|
||||
github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I=
|
||||
github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg=
|
||||
github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY=
|
||||
github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo=
|
||||
github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU=
|
||||
github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4=
|
||||
github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
|
||||
github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU=
|
||||
github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
|
||||
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
|
||||
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
||||
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
||||
go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0=
|
||||
go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8=
|
||||
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
|
||||
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
||||
golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo=
|
||||
golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y=
|
||||
golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k=
|
||||
golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||
google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc=
|
||||
google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU=
|
||||
golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk=
|
||||
golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
|
||||
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
|
||||
google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ var prometheusLabelPattern = regexp.MustCompile(`^[a-zA-Z0-9_/:.-]*$`)
|
|||
type MetricsAggregator struct {
|
||||
pathRegistry *PathRegistry
|
||||
eventRegistry *CustomEventRegistry
|
||||
cfg config.Config
|
||||
cfg *config.Config
|
||||
pageviews *prometheus.CounterVec
|
||||
customEvents *prometheus.CounterVec
|
||||
pathOverflow prometheus.Counter
|
||||
|
|
@ -30,7 +30,7 @@ type MetricsAggregator struct {
|
|||
func NewMetricsAggregator(
|
||||
pathRegistry *PathRegistry,
|
||||
eventRegistry *CustomEventRegistry,
|
||||
cfg config.Config,
|
||||
cfg *config.Config,
|
||||
) *MetricsAggregator {
|
||||
// Build label names based on what's enabled in config
|
||||
labels := []string{"path"} // path is always included
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ func TestMetricsAggregator_RecordPageview(t *testing.T) {
|
|||
},
|
||||
}
|
||||
|
||||
agg := NewMetricsAggregator(registry, NewCustomEventRegistry(100), cfg)
|
||||
agg := NewMetricsAggregator(registry, NewCustomEventRegistry(100), &cfg)
|
||||
|
||||
// Record pageview with all dimensions
|
||||
agg.RecordPageview("/home", "US", "desktop", "google.com", "")
|
||||
|
|
@ -51,7 +51,7 @@ func TestMetricsAggregator_RecordPageview_MinimalDimensions(t *testing.T) {
|
|||
},
|
||||
}
|
||||
|
||||
agg := NewMetricsAggregator(registry, NewCustomEventRegistry(100), cfg)
|
||||
agg := NewMetricsAggregator(registry, NewCustomEventRegistry(100), &cfg)
|
||||
|
||||
// Record pageview with only path
|
||||
agg.RecordPageview("/home", "", "", "", "")
|
||||
|
|
@ -78,7 +78,7 @@ func TestMetricsAggregator_PathOverflow(t *testing.T) {
|
|||
},
|
||||
}
|
||||
|
||||
agg := NewMetricsAggregator(registry, NewCustomEventRegistry(100), cfg)
|
||||
agg := NewMetricsAggregator(registry, NewCustomEventRegistry(100), &cfg)
|
||||
|
||||
// Add two paths successfully
|
||||
registry.Add("/path1")
|
||||
|
|
@ -114,7 +114,7 @@ func TestMetricsAggregator_RecordCustomEvent(t *testing.T) {
|
|||
},
|
||||
}
|
||||
|
||||
agg := NewMetricsAggregator(registry, NewCustomEventRegistry(100), cfg)
|
||||
agg := NewMetricsAggregator(registry, NewCustomEventRegistry(100), &cfg)
|
||||
|
||||
// Record custom event
|
||||
agg.RecordCustomEvent("signup")
|
||||
|
|
@ -141,7 +141,7 @@ func TestMetricsAggregator_RecordCustomEvent_MultipleEvents(t *testing.T) {
|
|||
},
|
||||
}
|
||||
|
||||
agg := NewMetricsAggregator(registry, NewCustomEventRegistry(100), cfg)
|
||||
agg := NewMetricsAggregator(registry, NewCustomEventRegistry(100), &cfg)
|
||||
|
||||
// Record multiple events
|
||||
agg.RecordCustomEvent("signup")
|
||||
|
|
@ -170,7 +170,7 @@ func TestMetricsAggregator_MustRegister(t *testing.T) {
|
|||
}
|
||||
|
||||
promRegistry := prometheus.NewRegistry()
|
||||
agg := NewMetricsAggregator(registry, NewCustomEventRegistry(100), cfg)
|
||||
agg := NewMetricsAggregator(registry, NewCustomEventRegistry(100), &cfg)
|
||||
|
||||
// Register metrics
|
||||
agg.MustRegister(promRegistry)
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ import (
|
|||
|
||||
// Handles incoming analytics events
|
||||
type IngestionHandler struct {
|
||||
cfg config.Config
|
||||
cfg *config.Config
|
||||
pathNorm *normalize.PathNormalizer
|
||||
pathRegistry *aggregate.PathRegistry
|
||||
refRegistry *normalize.ReferrerRegistry
|
||||
|
|
@ -25,7 +25,7 @@ type IngestionHandler struct {
|
|||
|
||||
// Creates a new ingestion handler
|
||||
func NewIngestionHandler(
|
||||
cfg config.Config,
|
||||
cfg *config.Config,
|
||||
pathNorm *normalize.PathNormalizer,
|
||||
pathRegistry *aggregate.PathRegistry,
|
||||
refRegistry *normalize.ReferrerRegistry,
|
||||
|
|
|
|||
|
|
@ -41,10 +41,10 @@ func TestIngestionHandler_Pageview(t *testing.T) {
|
|||
metricsAgg := aggregate.NewMetricsAggregator(
|
||||
pathRegistry,
|
||||
aggregate.NewCustomEventRegistry(100),
|
||||
cfg,
|
||||
&cfg,
|
||||
)
|
||||
|
||||
handler := NewIngestionHandler(cfg, pathNorm, pathRegistry, refRegistry, metricsAgg)
|
||||
handler := NewIngestionHandler(&cfg, pathNorm, pathRegistry, refRegistry, metricsAgg)
|
||||
|
||||
body := `{"d":"example.com","p":"/home?query=1","r":"https://google.com","w":1920}`
|
||||
req := httptest.NewRequest("POST", "/api/event", bytes.NewBufferString(body))
|
||||
|
|
@ -87,10 +87,10 @@ func TestIngestionHandler_CustomEvent(t *testing.T) {
|
|||
metricsAgg := aggregate.NewMetricsAggregator(
|
||||
pathRegistry,
|
||||
aggregate.NewCustomEventRegistry(100),
|
||||
cfg,
|
||||
&cfg,
|
||||
)
|
||||
|
||||
handler := NewIngestionHandler(cfg, pathNorm, pathRegistry, refRegistry, metricsAgg)
|
||||
handler := NewIngestionHandler(&cfg, pathNorm, pathRegistry, refRegistry, metricsAgg)
|
||||
|
||||
body := `{"d":"example.com","p":"/signup","e":"signup"}`
|
||||
req := httptest.NewRequest("POST", "/api/event", bytes.NewBufferString(body))
|
||||
|
|
@ -125,10 +125,10 @@ func TestIngestionHandler_WrongDomain(t *testing.T) {
|
|||
metricsAgg := aggregate.NewMetricsAggregator(
|
||||
pathRegistry,
|
||||
aggregate.NewCustomEventRegistry(100),
|
||||
cfg,
|
||||
&cfg,
|
||||
)
|
||||
|
||||
handler := NewIngestionHandler(cfg, pathNorm, pathRegistry, refRegistry, metricsAgg)
|
||||
handler := NewIngestionHandler(&cfg, pathNorm, pathRegistry, refRegistry, metricsAgg)
|
||||
|
||||
body := `{"d":"wrong.com","p":"/home"}`
|
||||
req := httptest.NewRequest("POST", "/api/event", bytes.NewBufferString(body))
|
||||
|
|
@ -159,10 +159,10 @@ func TestIngestionHandler_MethodNotAllowed(t *testing.T) {
|
|||
metricsAgg := aggregate.NewMetricsAggregator(
|
||||
pathRegistry,
|
||||
aggregate.NewCustomEventRegistry(100),
|
||||
cfg,
|
||||
&cfg,
|
||||
)
|
||||
|
||||
handler := NewIngestionHandler(cfg, pathNorm, pathRegistry, refRegistry, metricsAgg)
|
||||
handler := NewIngestionHandler(&cfg, pathNorm, pathRegistry, refRegistry, metricsAgg)
|
||||
|
||||
req := httptest.NewRequest("GET", "/api/event", nil)
|
||||
w := httptest.NewRecorder()
|
||||
|
|
@ -190,10 +190,10 @@ func TestIngestionHandler_InvalidJSON(t *testing.T) {
|
|||
metricsAgg := aggregate.NewMetricsAggregator(
|
||||
pathRegistry,
|
||||
aggregate.NewCustomEventRegistry(100),
|
||||
cfg,
|
||||
&cfg,
|
||||
)
|
||||
|
||||
handler := NewIngestionHandler(cfg, pathNorm, pathRegistry, refRegistry, metricsAgg)
|
||||
handler := NewIngestionHandler(&cfg, pathNorm, pathRegistry, refRegistry, metricsAgg)
|
||||
|
||||
body := `{invalid json}`
|
||||
req := httptest.NewRequest("POST", "/api/event", bytes.NewBufferString(body))
|
||||
|
|
@ -229,10 +229,10 @@ func TestIngestionHandler_DeviceClassification(t *testing.T) {
|
|||
metricsAgg := aggregate.NewMetricsAggregator(
|
||||
pathRegistry,
|
||||
aggregate.NewCustomEventRegistry(100),
|
||||
cfg,
|
||||
&cfg,
|
||||
)
|
||||
|
||||
handler := NewIngestionHandler(cfg, pathNorm, pathRegistry, refRegistry, metricsAgg)
|
||||
handler := NewIngestionHandler(&cfg, pathNorm, pathRegistry, refRegistry, metricsAgg)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
|
|
|
|||
|
|
@ -39,14 +39,14 @@ func TestEndToEnd_BasicFlow(t *testing.T) {
|
|||
pathRegistry := aggregate.NewPathRegistry(cfg.Limits.MaxPaths)
|
||||
refRegistry := normalize.NewReferrerRegistry(cfg.Limits.MaxSources)
|
||||
eventRegistry := aggregate.NewCustomEventRegistry(cfg.Limits.MaxCustomEvents)
|
||||
metricsAgg := aggregate.NewMetricsAggregator(pathRegistry, eventRegistry, *cfg)
|
||||
metricsAgg := aggregate.NewMetricsAggregator(pathRegistry, eventRegistry, cfg)
|
||||
|
||||
// Register metrics
|
||||
promRegistry := prometheus.NewRegistry()
|
||||
metricsAgg.MustRegister(promRegistry)
|
||||
|
||||
// Create handlers
|
||||
ingestionHandler := api.NewIngestionHandler(*cfg, pathNorm, pathRegistry, refRegistry, metricsAgg)
|
||||
ingestionHandler := api.NewIngestionHandler(cfg, pathNorm, pathRegistry, refRegistry, metricsAgg)
|
||||
metricsHandler := promhttp.HandlerFor(promRegistry, promhttp.HandlerOpts{})
|
||||
|
||||
// Test pageview ingestion
|
||||
|
|
@ -152,12 +152,12 @@ func TestEndToEnd_CardinalityLimits(t *testing.T) {
|
|||
pathRegistry := aggregate.NewPathRegistry(cfg.Limits.MaxPaths)
|
||||
refRegistry := normalize.NewReferrerRegistry(cfg.Limits.MaxSources)
|
||||
eventRegistry := aggregate.NewCustomEventRegistry(cfg.Limits.MaxCustomEvents)
|
||||
metricsAgg := aggregate.NewMetricsAggregator(pathRegistry, eventRegistry, cfg)
|
||||
metricsAgg := aggregate.NewMetricsAggregator(pathRegistry, eventRegistry, &cfg)
|
||||
|
||||
promRegistry := prometheus.NewRegistry()
|
||||
metricsAgg.MustRegister(promRegistry)
|
||||
|
||||
handler := api.NewIngestionHandler(cfg, pathNorm, pathRegistry, refRegistry, metricsAgg)
|
||||
handler := api.NewIngestionHandler(&cfg, pathNorm, pathRegistry, refRegistry, metricsAgg)
|
||||
|
||||
// Send more paths than limit
|
||||
t.Run("PathOverflow", func(t *testing.T) {
|
||||
|
|
@ -200,11 +200,11 @@ func TestEndToEnd_GracefulShutdown(t *testing.T) {
|
|||
pathRegistry := aggregate.NewPathRegistry(cfg.Limits.MaxPaths)
|
||||
refRegistry := normalize.NewReferrerRegistry(cfg.Limits.MaxSources)
|
||||
eventRegistry := aggregate.NewCustomEventRegistry(cfg.Limits.MaxCustomEvents)
|
||||
metricsAgg := aggregate.NewMetricsAggregator(pathRegistry, eventRegistry, cfg)
|
||||
metricsAgg := aggregate.NewMetricsAggregator(pathRegistry, eventRegistry, &cfg)
|
||||
|
||||
// Send some events to populate HLL
|
||||
pathNorm := normalize.NewPathNormalizer(cfg.Site.Path)
|
||||
handler := api.NewIngestionHandler(cfg, pathNorm, pathRegistry, refRegistry, metricsAgg)
|
||||
handler := api.NewIngestionHandler(&cfg, pathNorm, pathRegistry, refRegistry, metricsAgg)
|
||||
|
||||
for i := 0; i < 10; i++ {
|
||||
event := `{"d":"test.example.com","p":"/","r":"","w":1920}`
|
||||
|
|
@ -247,9 +247,9 @@ func TestEndToEnd_InvalidRequests(t *testing.T) {
|
|||
pathRegistry := aggregate.NewPathRegistry(cfg.Limits.MaxPaths)
|
||||
refRegistry := normalize.NewReferrerRegistry(cfg.Limits.MaxSources)
|
||||
eventRegistry := aggregate.NewCustomEventRegistry(cfg.Limits.MaxCustomEvents)
|
||||
metricsAgg := aggregate.NewMetricsAggregator(pathRegistry, eventRegistry, cfg)
|
||||
metricsAgg := aggregate.NewMetricsAggregator(pathRegistry, eventRegistry, &cfg)
|
||||
|
||||
handler := api.NewIngestionHandler(cfg, pathNorm, pathRegistry, refRegistry, metricsAgg)
|
||||
handler := api.NewIngestionHandler(&cfg, pathNorm, pathRegistry, refRegistry, metricsAgg)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
|
|
@ -327,9 +327,9 @@ func TestEndToEnd_RateLimiting(t *testing.T) {
|
|||
pathRegistry := aggregate.NewPathRegistry(cfg.Limits.MaxPaths)
|
||||
refRegistry := normalize.NewReferrerRegistry(cfg.Limits.MaxSources)
|
||||
eventRegistry := aggregate.NewCustomEventRegistry(cfg.Limits.MaxCustomEvents)
|
||||
metricsAgg := aggregate.NewMetricsAggregator(pathRegistry, eventRegistry, cfg)
|
||||
metricsAgg := aggregate.NewMetricsAggregator(pathRegistry, eventRegistry, &cfg)
|
||||
|
||||
handler := api.NewIngestionHandler(cfg, pathNorm, pathRegistry, refRegistry, metricsAgg)
|
||||
handler := api.NewIngestionHandler(&cfg, pathNorm, pathRegistry, refRegistry, metricsAgg)
|
||||
|
||||
// Send requests until rate limited
|
||||
rateLimited := false
|
||||
|
|
@ -376,9 +376,9 @@ func TestEndToEnd_CORS(t *testing.T) {
|
|||
pathRegistry := aggregate.NewPathRegistry(cfg.Limits.MaxPaths)
|
||||
refRegistry := normalize.NewReferrerRegistry(cfg.Limits.MaxSources)
|
||||
eventRegistry := aggregate.NewCustomEventRegistry(cfg.Limits.MaxCustomEvents)
|
||||
metricsAgg := aggregate.NewMetricsAggregator(pathRegistry, eventRegistry, cfg)
|
||||
metricsAgg := aggregate.NewMetricsAggregator(pathRegistry, eventRegistry, &cfg)
|
||||
|
||||
handler := api.NewIngestionHandler(cfg, pathNorm, pathRegistry, refRegistry, metricsAgg)
|
||||
handler := api.NewIngestionHandler(&cfg, pathNorm, pathRegistry, refRegistry, metricsAgg)
|
||||
|
||||
// Test OPTIONS preflight
|
||||
t.Run("Preflight", func(t *testing.T) {
|
||||
|
|
@ -435,9 +435,9 @@ func BenchmarkIngestionThroughput(b *testing.B) {
|
|||
pathRegistry := aggregate.NewPathRegistry(cfg.Limits.MaxPaths)
|
||||
refRegistry := normalize.NewReferrerRegistry(cfg.Limits.MaxSources)
|
||||
eventRegistry := aggregate.NewCustomEventRegistry(cfg.Limits.MaxCustomEvents)
|
||||
metricsAgg := aggregate.NewMetricsAggregator(pathRegistry, eventRegistry, cfg)
|
||||
metricsAgg := aggregate.NewMetricsAggregator(pathRegistry, eventRegistry, &cfg)
|
||||
|
||||
handler := api.NewIngestionHandler(cfg, pathNorm, pathRegistry, refRegistry, metricsAgg)
|
||||
handler := api.NewIngestionHandler(&cfg, pathNorm, pathRegistry, refRegistry, metricsAgg)
|
||||
|
||||
event := `{"d":"test.example.com","p":"/","r":"https://google.com","w":1920}`
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue