1
0
Fork 0
mirror of https://github.com/eosswedenorg/thalos synced 2026-07-03 11:53:41 +02:00

tools: cleanup and refactor to make code more readable.

This commit is contained in:
Henrik Hautakoski 2024-06-19 21:45:45 +02:00
parent ea5b2b8fc2
commit 8b8867394f
6 changed files with 341 additions and 328 deletions

View file

@ -18,101 +18,108 @@ import (
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
) )
var benchCmd = &cobra.Command{ func CreateBenchCmd() *cobra.Command {
Use: "bench", cmd := &cobra.Command{
Short: "Run a benchmark against a thalos node", Use: "bench",
Run: func(cmd *cobra.Command, args []string) { Short: "Run a benchmark against a thalos node",
counter := 0 Run: func(cmd *cobra.Command, args []string) {
interval, _ := cmd.Flags().GetDuration("interval") counter := 0
interval, _ := cmd.Flags().GetDuration("interval")
url, _ := cmd.Flags().GetString("redis-url") url, _ := cmd.Flags().GetString("redis-url")
user, _ := cmd.Flags().GetString("redis-user") user, _ := cmd.Flags().GetString("redis-user")
pw, _ := cmd.Flags().GetString("redis-pw") pw, _ := cmd.Flags().GetString("redis-pw")
prefix, _ := cmd.Flags().GetString("prefix") prefix, _ := cmd.Flags().GetString("prefix")
chain_id, _ := cmd.Flags().GetString("chain_id") chain_id, _ := cmd.Flags().GetString("chain_id")
db, _ := cmd.Flags().GetInt("redis-db") db, _ := cmd.Flags().GetInt("redis-db")
log.WithFields(log.Fields{ log.WithFields(log.Fields{
"url": url, "url": url,
"prefix": prefix, "prefix": prefix,
"chain_id": chain_id, "chain_id": chain_id,
"database": db, "database": db,
}).Info("Connecting to redis") }).Info("Connecting to redis")
// Create redis client // Create redis client
rdb := redis.NewClient(&redis.Options{ rdb := redis.NewClient(&redis.Options{
Addr: url, Addr: url,
Username: user, Username: user,
Password: pw, Password: pw,
DB: db, DB: db,
}) })
if err := rdb.Ping(context.Background()).Err(); err != nil { if err := rdb.Ping(context.Background()).Err(); err != nil {
log.WithError(err).Fatal("Failed to connect to redis") log.WithError(err).Fatal("Failed to connect to redis")
return return
} }
log.Println("Connected to redis") log.Println("Connected to redis")
log.WithFields(log.Fields{ log.WithFields(log.Fields{
"interval": interval, "interval": interval,
}).Info("Starting benchmark") }).Info("Starting benchmark")
sub := api_redis.NewSubscriber(context.Background(), rdb, api_redis.Namespace{ sub := api_redis.NewSubscriber(context.Background(), rdb, api_redis.Namespace{
Prefix: prefix, Prefix: prefix,
ChainID: chain_id, ChainID: chain_id,
}) })
codec, err := message.GetCodec("json") codec, err := message.GetCodec("json")
if err != nil { if err != nil {
log.WithError(err).Fatal("Failed to get codec") log.WithError(err).Fatal("Failed to get codec")
return return
} }
client := api.NewClient(sub, codec.Decoder) client := api.NewClient(sub, codec.Decoder)
// Subscribe to all actions // Subscribe to all actions
if err = client.Subscribe(api.ActionChannel{}.Channel()); err != nil { if err = client.Subscribe(api.ActionChannel{}.Channel()); err != nil {
log.WithError(err).Fatal("Failed to subscribe to channels") log.WithError(err).Fatal("Failed to subscribe to channels")
return return
} }
go func() { go func() {
for t := range client.Channel() { for t := range client.Channel() {
switch err := t.(type) { switch err := t.(type) {
case message.ActionTrace: case message.ActionTrace:
counter++ counter++
case error: case error:
log.WithError(err).Error("Error when reading stream") log.WithError(err).Error("Error when reading stream")
}
}
}()
t := time.Now()
sig := make(chan os.Signal, 1)
signal.Notify(sig, os.Interrupt)
// Read stuff.
for {
select {
case <-sig:
fmt.Println("Got interrupt")
client.Close()
return
case now := <-time.After(interval):
elapsed := now.Sub(t)
t = now
log.WithFields(log.Fields{
"num_messages": counter,
"elapsed": elapsed,
"msg_per_sec": float64(counter) / elapsed.Seconds(),
"msg_per_ms": float64(counter) / float64(elapsed.Milliseconds()),
"msg_per_min": float64(counter) / elapsed.Minutes(),
}).Info("Benchmark results")
counter = 0
} }
} }
}() },
}
t := time.Now() cmd.Flags().AddFlagSet(RedisFlags())
sig := make(chan os.Signal, 1) cmd.Flags().DurationP("interval", "i", time.Minute, "How often the benchmark results should be displayed.")
signal.Notify(sig, os.Interrupt)
// Read stuff. return cmd
for {
select {
case <-sig:
fmt.Println("Got interrupt")
client.Close()
return
case now := <-time.After(interval):
elapsed := now.Sub(t)
t = now
log.WithFields(log.Fields{
"num_messages": counter,
"elapsed": elapsed,
"msg_per_sec": float64(counter) / elapsed.Seconds(),
"msg_per_ms": float64(counter) / float64(elapsed.Milliseconds()),
"msg_per_min": float64(counter) / elapsed.Minutes(),
}).Info("Benchmark results")
counter = 0
}
}
},
} }

14
cmd/tools/flags.go Normal file
View file

@ -0,0 +1,14 @@
package main
import "github.com/spf13/pflag"
func RedisFlags() *pflag.FlagSet {
set := pflag.FlagSet{}
set.String("redis-url", "127.0.0.1:6379", "host:port to the redis server")
set.String("redis-user", "", "User to use when authenticating to the server")
set.String("redis-pw", "", "Password to use when authenticating to the server")
set.Int("redis-db", 0, "What redis database we should connect to.")
set.String("prefix", "ship", "redis prefix")
set.String("chain_id", "1064487b3cd1a897ce03ae5b6a865651747e2e152090f99c1d19d44e01aea5a4", "chain id")
return &set
}

View file

@ -1,28 +1,15 @@
package main package main
import ( import (
"time"
_ "github.com/eosswedenorg/thalos/internal/log" _ "github.com/eosswedenorg/thalos/internal/log"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/spf13/cobra" "github.com/spf13/cobra"
"github.com/spf13/pflag"
) )
var VersionString string = "dev" var VersionString string = "dev"
var rootCmd *cobra.Command func main() {
rootCmd := &cobra.Command{
func init() {
redisFlags := pflag.FlagSet{}
redisFlags.String("redis-url", "127.0.0.1:6379", "host:port to the redis server")
redisFlags.String("redis-user", "", "User to use when authenticating to the server")
redisFlags.String("redis-pw", "", "Password to use when authenticating to the server")
redisFlags.Int("redis-db", 0, "What redis database we should connect to.")
redisFlags.String("prefix", "ship", "redis prefix")
redisFlags.String("chain_id", "1064487b3cd1a897ce03ae5b6a865651747e2e152090f99c1d19d44e01aea5a4", "chain id")
rootCmd = &cobra.Command{
Use: "thalos-tools", Use: "thalos-tools",
Short: "Collection of tools for dealing with the thalos application", Short: "Collection of tools for dealing with the thalos application",
FParseErrWhitelist: cobra.FParseErrWhitelist{ FParseErrWhitelist: cobra.FParseErrWhitelist{
@ -31,33 +18,13 @@ func init() {
Version: VersionString, Version: VersionString,
} }
benchCmd.Flags().AddFlagSet(&redisFlags)
benchCmd.Flags().DurationP("interval", "i", time.Minute, "How often the benchmark results should be displayed.")
validateCmd.Flags().AddFlagSet(&redisFlags)
MockPublisherCmd.Flags().AddFlagSet(&redisFlags)
MockPublisherCmd.Flags().String("codec", "json", "codec to use")
RedisACLCmd.Flags().String("default-pw", "", "Password to use for the default account, if not provided a random one will be generated")
RedisACLCmd.Flags().String("client", "thalos-client", "Thalos client account name")
RedisACLCmd.Flags().String("client-pw", "", "Password to use for the thalos client account, if not provided a random one will be generated")
RedisACLCmd.Flags().String("server", "thalos", "Thalos account name")
RedisACLCmd.Flags().String("server-pw", "", "Password to use for the thalos server account, if not provided a random one will be generated")
RedisACLCmd.Flags().String("prefix", "ship", "Redis key prefix")
RedisACLCmd.Flags().Bool("cleartext", false, "If passwords should be hashed or left in cleartext.")
RedisACLCmd.Flags().String("file", "", "Where the config should be written to (default: standard out)")
RedisACLCmd.Flags().Uint("pass-len", 32, "The length of generated passwords")
rootCmd.AddCommand( rootCmd.AddCommand(
validateCmd, CreateValidateCmd(),
benchCmd, CreateBenchCmd(),
RedisACLCmd, CreateRedisACLCmd(),
MockPublisherCmd, CreateMockPublisherCmd(),
) )
}
func main() {
if err := rootCmd.Execute(); err != nil { if err := rootCmd.Execute(); err != nil {
log.Fatal(err) log.Fatal(err)
} }

View file

@ -16,100 +16,106 @@ import (
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
) )
var MockPublisherCmd = &cobra.Command{ func CreateMockPublisherCmd() *cobra.Command {
Use: "mock_publisher", cmd := &cobra.Command{
Short: "Run a publisher that mocks messages to a redis server. tries to send as many messages as possible", Use: "mock_publisher",
Run: func(cmd *cobra.Command, args []string) { Short: "Run a publisher that mocks messages to a redis server. tries to send as many messages as possible",
url, _ := cmd.Flags().GetString("redis-url") Run: func(cmd *cobra.Command, args []string) {
user, _ := cmd.Flags().GetString("redis-user") url, _ := cmd.Flags().GetString("redis-url")
pw, _ := cmd.Flags().GetString("redis-pw") user, _ := cmd.Flags().GetString("redis-user")
prefix, _ := cmd.Flags().GetString("prefix") pw, _ := cmd.Flags().GetString("redis-pw")
chain_id, _ := cmd.Flags().GetString("chain_id") prefix, _ := cmd.Flags().GetString("prefix")
db, _ := cmd.Flags().GetInt("redis-db") chain_id, _ := cmd.Flags().GetString("chain_id")
db, _ := cmd.Flags().GetInt("redis-db")
// Create redis client // Create redis client
rdb := redis.NewClient(&redis.Options{ rdb := redis.NewClient(&redis.Options{
Addr: url, Addr: url,
Username: user, Username: user,
Password: pw, Password: pw,
DB: db, DB: db,
}) })
codecArg, _ := cmd.Flags().GetString("codec") codecArg, _ := cmd.Flags().GetString("codec")
codec, err := message.GetCodec(codecArg) codec, err := message.GetCodec(codecArg)
if err != nil { if err != nil {
log.WithError(err).Fatal("Failed to get codec") log.WithError(err).Fatal("Failed to get codec")
return return
} }
log.WithFields(log.Fields{ log.WithFields(log.Fields{
"url": url, "url": url,
"prefix": prefix, "prefix": prefix,
"chain_id": chain_id, "chain_id": chain_id,
"database": db, "database": db,
}).Info("Starting mock publisher") }).Info("Starting mock publisher")
ns := api_redis.Namespace{ ns := api_redis.Namespace{
Prefix: prefix, Prefix: prefix,
ChainID: chain_id, ChainID: chain_id,
} }
publisher := redis_driver.NewPublisher(context.Background(), rdb, ns) publisher := redis_driver.NewPublisher(context.Background(), rdb, ns)
msg := message.ActionTrace{ msg := message.ActionTrace{
TxID: "401e8a7e5deb18a2a69fc6559f49509a155f4355c85efb69c1c1fab5b60ee532", TxID: "401e8a7e5deb18a2a69fc6559f49509a155f4355c85efb69c1c1fab5b60ee532",
BlockNum: 18237917, BlockNum: 18237917,
Timestamp: time.Date(2014, 3, 22, 11, 36, 43, 0, time.UTC), Timestamp: time.Date(2014, 3, 22, 11, 36, 43, 0, time.UTC),
Receipt: &message.ActionReceipt{ Receipt: &message.ActionReceipt{
Receiver: "acc1", Receiver: "acc1",
ActDigest: "4c5c08be612e937564fc526ebb5fadf34ae8c2a571fe9d7cdb3ffcdfc53b0e8d", ActDigest: "4c5c08be612e937564fc526ebb5fadf34ae8c2a571fe9d7cdb3ffcdfc53b0e8d",
GlobalSequence: 12314, GlobalSequence: 12314,
RecvSequence: 237187239, RecvSequence: 237187239,
AuthSequence: []message.AccountAuthSequence{ AuthSequence: []message.AccountAuthSequence{
{
Account: "acc1",
Sequence: 2732863,
},
{
Account: "acc2",
Sequence: 263762,
},
},
CodeSequence: 2327832,
ABISequence: 12376189,
},
Name: "fake",
Contract: "fake",
Receiver: "acc1",
Data: map[string]interface{}{
"one": 238771832,
"two": "str",
},
Authorization: []message.PermissionLevel{
{ {
Account: "acc1", Actor: "acc1",
Sequence: 2732863, Permission: "active",
}, },
{ {
Account: "acc2", Actor: "acc2",
Sequence: 263762, Permission: "owner",
}, },
}, },
CodeSequence: 2327832, Except: "err",
ABISequence: 12376189, Error: 2,
}, Return: []byte{0xbe, 0xef},
Name: "fake", }
Contract: "fake",
Receiver: "acc1",
Data: map[string]interface{}{
"one": 238771832,
"two": "str",
},
Authorization: []message.PermissionLevel{
{
Actor: "acc1",
Permission: "active",
},
{
Actor: "acc2",
Permission: "owner",
},
},
Except: "err",
Error: 2,
Return: []byte{0xbe, 0xef},
}
payload, err := codec.Encoder(msg) payload, err := codec.Encoder(msg)
if err != nil { if err != nil {
log.WithError(err).Fatal("Failed to encode message") log.WithError(err).Fatal("Failed to encode message")
return return
} }
channel := api.ActionChannel{}.Channel() channel := api.ActionChannel{}.Channel()
for { for {
_ = publisher.Write(channel, payload) _ = publisher.Write(channel, payload)
publisher.Flush() publisher.Flush()
} }
}, },
}
cmd.Flags().AddFlagSet(RedisFlags())
cmd.Flags().String("codec", "json", "codec to use")
return cmd
} }

View file

@ -43,7 +43,6 @@ func NewUser(name, password string, pass_len uint) User {
} }
func (u *User) GetPassword() string { func (u *User) GetPassword() string {
if u.Hash { if u.Hash {
return "#" + hash(u.Password) return "#" + hash(u.Password)
} }
@ -99,60 +98,74 @@ user {{.client}} on {{.clientpw}} resetchannels &{{.prefix}}::* -@all +subscribe
}) })
} }
var RedisACLCmd = &cobra.Command{ func CreateRedisACLCmd() *cobra.Command {
Use: "redis-acl", cmd := &cobra.Command{
Short: "create a users.acl file", Use: "redis-acl",
Run: func(cmd *cobra.Command, args []string) { Short: "create a users.acl file",
var err error Run: func(cmd *cobra.Command, args []string) {
out := os.Stdout var err error
out := os.Stdout
rnd = rand.New(rand.NewSource(time.Now().UnixNano())) rnd = rand.New(rand.NewSource(time.Now().UnixNano()))
flagDefUserPw, _ := cmd.Flags().GetString("default-pw") flagDefUserPw, _ := cmd.Flags().GetString("default-pw")
flagServer, _ := cmd.Flags().GetString("server") flagServer, _ := cmd.Flags().GetString("server")
flagServerPw, _ := cmd.Flags().GetString("server-pw") flagServerPw, _ := cmd.Flags().GetString("server-pw")
flagClient, _ := cmd.Flags().GetString("client") flagClient, _ := cmd.Flags().GetString("client")
flagClientPw, _ := cmd.Flags().GetString("client-pw") flagClientPw, _ := cmd.Flags().GetString("client-pw")
flagPrefix, _ := cmd.Flags().GetString("prefix") flagPrefix, _ := cmd.Flags().GetString("prefix")
flagPassLen, _ := cmd.Flags().GetUint("pass-len") flagPassLen, _ := cmd.Flags().GetUint("pass-len")
defaultUser := NewUser("default", flagDefUserPw, flagPassLen) defaultUser := NewUser("default", flagDefUserPw, flagPassLen)
serverUser := NewUser(flagServer, flagServerPw, flagPassLen) serverUser := NewUser(flagServer, flagServerPw, flagPassLen)
clientUser := NewUser(flagClient, flagClientPw, flagPassLen) clientUser := NewUser(flagClient, flagClientPw, flagPassLen)
atleastOneGeneratedPw := defaultUser.Generated || serverUser.Generated || clientUser.Generated atleastOneGeneratedPw := defaultUser.Generated || serverUser.Generated || clientUser.Generated
cleartext, _ := cmd.Flags().GetBool("cleartext") cleartext, _ := cmd.Flags().GetBool("cleartext")
if !cleartext { if !cleartext {
if atleastOneGeneratedPw { if atleastOneGeneratedPw {
println("Passwords") println("Passwords")
}
defaultUser.PrintIfGeneratedPW()
serverUser.PrintIfGeneratedPW()
clientUser.PrintIfGeneratedPW()
defaultUser.Hash = true
serverUser.Hash = true
clientUser.Hash = true
} }
defaultUser.PrintIfGeneratedPW() filename, _ := cmd.Flags().GetString("file")
serverUser.PrintIfGeneratedPW() if len(filename) > 0 {
clientUser.PrintIfGeneratedPW() out, err = os.Create(filename)
if err != nil {
log.WithError(err).Fatal("Failed to create output file")
return
}
defer out.Close()
} else if !cleartext && atleastOneGeneratedPw {
fmt.Println()
}
defaultUser.Hash = true err = writeTemplate(out, defaultUser, serverUser, clientUser, flagPrefix)
serverUser.Hash = true
clientUser.Hash = true
}
filename, _ := cmd.Flags().GetString("file")
if len(filename) > 0 {
out, err = os.Create(filename)
if err != nil { if err != nil {
log.WithError(err).Fatal("Failed to create output file") log.WithError(err).Fatal("Failed to writte config")
return return
} }
defer out.Close() },
} else if !cleartext && atleastOneGeneratedPw { }
fmt.Println()
}
err = writeTemplate(out, defaultUser, serverUser, clientUser, flagPrefix) cmd.Flags().String("default-pw", "", "Password to use for the default account, if not provided a random one will be generated")
if err != nil { cmd.Flags().String("client", "thalos-client", "Thalos client account name")
log.WithError(err).Fatal("Failed to writte config") cmd.Flags().String("client-pw", "", "Password to use for the thalos client account, if not provided a random one will be generated")
return cmd.Flags().String("server", "thalos", "Thalos account name")
} cmd.Flags().String("server-pw", "", "Password to use for the thalos server account, if not provided a random one will be generated")
}, cmd.Flags().String("prefix", "ship", "Redis key prefix")
cmd.Flags().Bool("cleartext", false, "If passwords should be hashed or left in cleartext.")
cmd.Flags().String("file", "", "Where the config should be written to (default: standard out)")
cmd.Flags().Uint("pass-len", 32, "The length of generated passwords")
return cmd
} }

View file

@ -18,101 +18,107 @@ import (
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
) )
var validateCmd = &cobra.Command{ func CreateValidateCmd() *cobra.Command {
Use: "validate", cmd := &cobra.Command{
Short: "Validate a thalos server by following action traces and makes sure that blocks arrive in order.", Use: "validate",
Run: func(cmd *cobra.Command, args []string) { Short: "Validate a thalos server by following action traces and makes sure that blocks arrive in order.",
status_duration := time.Second * 10 Run: func(cmd *cobra.Command, args []string) {
status_duration := time.Second * 10
url, _ := cmd.Flags().GetString("redis-url") url, _ := cmd.Flags().GetString("redis-url")
prefix, _ := cmd.Flags().GetString("prefix") prefix, _ := cmd.Flags().GetString("prefix")
chain_id, _ := cmd.Flags().GetString("chain_id") chain_id, _ := cmd.Flags().GetString("chain_id")
db, _ := cmd.Flags().GetInt("redis-db") db, _ := cmd.Flags().GetInt("redis-db")
log.WithFields(log.Fields{ log.WithFields(log.Fields{
"url": url, "url": url,
"prefix": prefix, "prefix": prefix,
"chain_id": chain_id, "chain_id": chain_id,
"database": db, "database": db,
}).Info("Connecting to redis") }).Info("Connecting to redis")
// Create redis client // Create redis client
rdb := redis.NewClient(&redis.Options{ rdb := redis.NewClient(&redis.Options{
Addr: url, Addr: url,
DB: db, DB: db,
}) })
if err := rdb.Ping(context.Background()).Err(); err != nil { if err := rdb.Ping(context.Background()).Err(); err != nil {
log.WithError(err).Fatal("Failed to connect to redis") log.WithError(err).Fatal("Failed to connect to redis")
return return
} }
log.Println("Connected to redis") log.Println("Connected to redis")
log.Info("Starting validation, following the stream") log.Info("Starting validation, following the stream")
sub := api_redis.NewSubscriber(context.Background(), rdb, api_redis.Namespace{ sub := api_redis.NewSubscriber(context.Background(), rdb, api_redis.Namespace{
Prefix: prefix, Prefix: prefix,
ChainID: chain_id, ChainID: chain_id,
}) })
codec, err := message.GetCodec("json") codec, err := message.GetCodec("json")
if err != nil { if err != nil {
log.WithError(err).Fatal("Failed to get codec") log.WithError(err).Fatal("Failed to get codec")
return return
} }
client := api.NewClient(sub, codec.Decoder) client := api.NewClient(sub, codec.Decoder)
// Subscribe to all actions // Subscribe to all actions
if err = client.Subscribe(api.ActionChannel{}.Channel()); err != nil { if err = client.Subscribe(api.ActionChannel{}.Channel()); err != nil {
log.WithError(err).Fatal("Failed to subscribe to channels") log.WithError(err).Fatal("Failed to subscribe to channels")
return return
} }
block_num := uint32(0) block_num := uint32(0)
timeout := time.Second * 5 timeout := time.Second * 5
timer := time.NewTicker(timeout) timer := time.NewTicker(timeout)
go func() { go func() {
for t := range client.Channel() { for t := range client.Channel() {
switch msg := t.(type) { switch msg := t.(type) {
case error: case error:
log.WithError(msg).Error("Error when reading stream") log.WithError(msg).Error("Error when reading stream")
case message.ActionTrace: case message.ActionTrace:
if block_num > 0 { if block_num > 0 {
diff := int32(msg.BlockNum - block_num) diff := int32(msg.BlockNum - block_num)
if diff < 0 || diff > 1 { if diff < 0 || diff > 1 {
log.WithFields(log.Fields{ log.WithFields(log.Fields{
"current_block": block_num, "current_block": block_num,
"block": msg.BlockNum, "block": msg.BlockNum,
"diff": diff, "diff": diff,
}).Warn("Invalid") }).Warn("Invalid")
}
} }
block_num = msg.BlockNum
timer.Reset(timeout)
} }
block_num = msg.BlockNum }
timer.Reset(timeout) }()
sig := make(chan os.Signal, 1)
signal.Notify(sig, os.Interrupt)
for {
select {
case <-sig:
fmt.Println("Got interrupt")
client.Close()
return
case <-timer.C:
log.WithField("duration", timeout).
Warn("Did not get any messages during the defined duration")
case <-time.After(status_duration):
log.WithFields(log.Fields{
"current_block": block_num,
}).Info("Status")
} }
} }
}() },
}
sig := make(chan os.Signal, 1) cmd.Flags().AddFlagSet(RedisFlags())
signal.Notify(sig, os.Interrupt)
for { return cmd
select {
case <-sig:
fmt.Println("Got interrupt")
client.Close()
return
case <-timer.C:
log.WithField("duration", timeout).
Warn("Did not get any messages during the defined duration")
case <-time.After(status_duration):
log.WithFields(log.Fields{
"current_block": block_num,
}).Info("Status")
}
}
},
} }