From 52488277cbd7b6e933ce050c39b3b3d776d49fec Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Wed, 14 Jun 2023 11:21:54 +0200 Subject: [PATCH] create thalos-tools cli app and have the old bench command as a subcommand. --- Makefile | 8 +-- cmd/bench/main.go | 122 ------------------------------------------ cmd/tools/bench.go | 128 +++++++++++++++++++++++++++++++++++++++++++++ cmd/tools/main.go | 23 ++++++++ go.mod | 3 ++ go.sum | 8 +++ 6 files changed, 166 insertions(+), 126 deletions(-) delete mode 100644 cmd/bench/main.go create mode 100644 cmd/tools/bench.go create mode 100644 cmd/tools/main.go diff --git a/Makefile b/Makefile index 0742c77..b06bd31 100644 --- a/Makefile +++ b/Makefile @@ -7,17 +7,17 @@ PREFIX=/usr/local BINDIR=$(PREFIX)/bin CFGDIR=$(PREFIX)/etc/thalos -.PHONY: build build/$(PROGRAM) build/benchmark test +.PHONY: build build/$(PROGRAM) build/thalos-tools test build: build/$(PROGRAM) build/$(PROGRAM) : $(GO) build $(GOBUILDFLAGS) -o $@ cmd/thalos/main.go -build-benchmark : build/benchmark +build-tools : build/thalos-tools -build/benchmark : - $(GO) build $(GOBUILDFLAGS) -o $@ cmd/bench/main.go +build/thalos-tools : + $(GO) build $(GOBUILDFLAGS) -o $@ cmd/tools/main.go cmd/tools/bench.go install: build install -D build/$(PROGRAM) $(DESTDIR)$(BINDIR)/$(PROGRAM) diff --git a/cmd/bench/main.go b/cmd/bench/main.go deleted file mode 100644 index e44064a..0000000 --- a/cmd/bench/main.go +++ /dev/null @@ -1,122 +0,0 @@ -package main - -import ( - "context" - "flag" - "fmt" - "os" - "os/signal" - "time" - - "github.com/eosswedenorg/thalos/api" - "github.com/eosswedenorg/thalos/api/message" - _ "github.com/eosswedenorg/thalos/api/message/json" - api_redis "github.com/eosswedenorg/thalos/api/redis" - - "github.com/redis/go-redis/v9" - log "github.com/sirupsen/logrus" -) - -var ( - interval time.Duration - chain_id string - - redis_prefix string - redis_url string - redis_db int -) - -func init() { - flag.DurationVar(&interval, "interval", time.Minute, "How often the benchmark results should be displayed.") - flag.StringVar(&chain_id, "chain_id", "1064487b3cd1a897ce03ae5b6a865651747e2e152090f99c1d19d44e01aea5a4", "") - flag.StringVar(&redis_prefix, "prefix", "ship", "") - - flag.StringVar(&redis_url, "redis-url", "127.0.0.1:6379", "host:port to the redis server") - flag.IntVar(&redis_db, "redis-db", 0, "What redis database we should connect to.") -} - -func main() { - var counter int = 0 - - flag.Parse() - - log.WithFields(log.Fields{ - "url": redis_url, - "prefix": redis_prefix, - "chain_id": chain_id, - "database": redis_db, - }).Info("Connecting to redis") - - // Create redis client - rdb := redis.NewClient(&redis.Options{ - Addr: redis_url, - DB: redis_db, - }) - - status := rdb.Ping(context.Background()) - - if status.Err() != nil { - log.Fatal("cant connect to redis: ", status.Err()) - return - } - - log.Println("Connected to redis") - - log.WithFields(log.Fields{ - "interval": interval, - }).Info("Starting benchmark") - - sub := api_redis.NewSubscriber(context.Background(), rdb, api_redis.Namespace{ - Prefix: redis_prefix, - ChainID: chain_id, - }) - - codec, err := message.GetCodec("json") - if err != nil { - log.Fatal(err) - return - } - - client := api.NewClient(sub, codec.Decoder) - - client.OnAction = func(act message.ActionTrace) { - counter++ - } - - // Subscribe to all actions - if err = client.Subscribe(api.ActionChannel{}.Channel()); err != nil { - log.Fatal(err) - return - } - - go func() { - t := time.Now() - sig := make(chan os.Signal, 1) - signal.Notify(sig, os.Interrupt) - - for { - select { - case <-sig: - fmt.Println("Got interrupt") - client.Close() - return - case now := <-time.After(interval): - elapsed := now.Sub(t) - t = now - - log.WithFields(log.Fields{ - "num_messages": counter, - "elapsed": elapsed, - "msg_per_sec": float64(counter) / elapsed.Seconds(), - "msg_per_ms": float64(counter) / float64(elapsed.Milliseconds()), - "msg_per_min": float64(counter) / elapsed.Minutes(), - }).Info("Benchmark results") - - counter = 0 - } - } - }() - - // Read stuff. - client.Run() -} diff --git a/cmd/tools/bench.go b/cmd/tools/bench.go new file mode 100644 index 0000000..aa5d0fc --- /dev/null +++ b/cmd/tools/bench.go @@ -0,0 +1,128 @@ +package main + +import ( + "context" + "fmt" + "os" + "os/signal" + "time" + + "github.com/spf13/cobra" + + "github.com/eosswedenorg/thalos/api" + "github.com/eosswedenorg/thalos/api/message" + _ "github.com/eosswedenorg/thalos/api/message/json" + api_redis "github.com/eosswedenorg/thalos/api/redis" + + "github.com/redis/go-redis/v9" + log "github.com/sirupsen/logrus" +) + +var ( + interval time.Duration + chain_id string + + redis_prefix string + redis_url string + redis_db int +) + +var benchCmd = &cobra.Command{ + Use: "bench", + Short: "Run a benchmark against a thalos node", + Example: "thalos-tools bench -u 192.168.0.123:6379 --redis-db 1 --chain_id my_id -i 5m", + Run: func(cmd *cobra.Command, args []string) { + var counter int = 0 + + log.WithFields(log.Fields{ + "url": redis_url, + "prefix": redis_prefix, + "chain_id": chain_id, + "database": redis_db, + }).Info("Connecting to redis") + + // Create redis client + rdb := redis.NewClient(&redis.Options{ + Addr: redis_url, + DB: redis_db, + }) + + status := rdb.Ping(context.Background()) + + if status.Err() != nil { + log.Fatal("cant connect to redis: ", status.Err()) + return + } + + log.Println("Connected to redis") + + log.WithFields(log.Fields{ + "interval": interval, + }).Info("Starting benchmark") + + sub := api_redis.NewSubscriber(context.Background(), rdb, api_redis.Namespace{ + Prefix: redis_prefix, + ChainID: chain_id, + }) + + codec, err := message.GetCodec("json") + if err != nil { + log.Fatal(err) + return + } + + client := api.NewClient(sub, codec.Decoder) + + client.OnAction = func(act message.ActionTrace) { + counter++ + } + + // Subscribe to all actions + if err = client.Subscribe(api.ActionChannel{}.Channel()); err != nil { + log.Fatal(err) + return + } + + go func() { + t := time.Now() + sig := make(chan os.Signal, 1) + signal.Notify(sig, os.Interrupt) + + for { + select { + case <-sig: + fmt.Println("Got interrupt") + client.Close() + return + case now := <-time.After(interval): + elapsed := now.Sub(t) + t = now + + log.WithFields(log.Fields{ + "num_messages": counter, + "elapsed": elapsed, + "msg_per_sec": float64(counter) / elapsed.Seconds(), + "msg_per_ms": float64(counter) / float64(elapsed.Milliseconds()), + "msg_per_min": float64(counter) / elapsed.Minutes(), + }).Info("Benchmark results") + + counter = 0 + } + } + }() + + // Read stuff. + client.Run() + }, +} + +func init() { + benchCmd.Flags().DurationVarP(&interval, "interval", "i", time.Minute, "How often the benchmark results should be displayed.") + benchCmd.Flags().StringVar(&chain_id, "chain_id", "1064487b3cd1a897ce03ae5b6a865651747e2e152090f99c1d19d44e01aea5a4", "") + benchCmd.Flags().StringVar(&redis_prefix, "prefix", "ship", "") + + benchCmd.Flags().StringVarP(&redis_url, "redis-url", "u", "127.0.0.1:6379", "host:port to the redis server") + benchCmd.Flags().IntVar(&redis_db, "redis-db", 0, "What redis database we should connect to.") + + rootCmd.AddCommand(benchCmd) +} diff --git a/cmd/tools/main.go b/cmd/tools/main.go new file mode 100644 index 0000000..6625e77 --- /dev/null +++ b/cmd/tools/main.go @@ -0,0 +1,23 @@ +package main + +import ( + "os" + + "github.com/spf13/cobra" + + log "github.com/sirupsen/logrus" +) + +var VersionString string = "dev" + +var rootCmd = &cobra.Command{ + Use: os.Args[0], + Short: "Collection of tools for dealing with the thalos application", + Version: VersionString, +} + +func main() { + if err := rootCmd.Execute(); err != nil { + log.WithError(err).Fatal("Application error") + } +} diff --git a/go.mod b/go.mod index f477733..f614e01 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( github.com/pborman/getopt/v2 v2.1.0 github.com/redis/go-redis/v9 v9.0.5 github.com/sirupsen/logrus v1.9.0 + github.com/spf13/cobra v1.7.0 github.com/stretchr/testify v1.8.2 gopkg.in/yaml.v3 v3.0.1 ) @@ -28,6 +29,7 @@ require ( github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/go-telegram-bot-api/telegram-bot-api v4.6.4+incompatible // indirect github.com/gorilla/websocket v1.5.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.16.5 // indirect github.com/logrusorgru/aurora v2.0.3+incompatible // indirect @@ -37,6 +39,7 @@ require ( github.com/onsi/gomega v1.27.6 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect github.com/streamingfast/logging v0.0.0-20221209193439-bff11742bf4c // indirect github.com/technoweenie/multipartstreamer v1.0.1 // indirect github.com/tidwall/gjson v1.14.4 // indirect diff --git a/go.sum b/go.sum index 7cf243e..bccea53 100644 --- a/go.sum +++ b/go.sum @@ -14,6 +14,7 @@ github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -64,6 +65,8 @@ github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWm github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jarcoal/httpmock v1.2.0 h1:gSvTxxFR/MEMfsGrvRbdfpRUMBStovlSRLw0Ep1bwwc= github.com/jordan-wright/email v4.0.1-0.20210109023952-943e75fe5223+incompatible h1:jdpOPRN1zP63Td1hDQbZW73xKmzDvZHzVdNYxhnTMDA= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -129,8 +132,13 @@ github.com/redis/go-redis/v9 v9.0.5 h1:CuQcn5HIEeK7BgElubPP8CGtE0KakrnbBSTLjathl github.com/redis/go-redis/v9 v9.0.5/go.mod h1:WqMKv5vnQbRuZstUwxQI195wHy+t4PuXDOjzMvcuQHk= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.1 h1:geMPLpDpQOgVyCg5z5GoRwLHepNdb71NXb67XFkP+Eg= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= +github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/streamingfast/logging v0.0.0-20221209193439-bff11742bf4c h1:dV1ye/S2PiW9uIWvLtMrxWoTLcZS+yhjZDSKEV102Ho= github.com/streamingfast/logging v0.0.0-20221209193439-bff11742bf4c/go.mod h1:VlduQ80JcGJSargkRU4Sg9Xo63wZD/l8A5NC/Uo1/uU= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=