From 0d1bec6a6246da3d4d5056a23fb7b8c40239f612 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 18 Feb 2024 10:43:58 +0100 Subject: [PATCH 01/15] Adding internal/config/builder.go --- internal/config/builder.go | 102 +++++++++++++++++++++ internal/config/builder_test.go | 151 ++++++++++++++++++++++++++++++++ 2 files changed, 253 insertions(+) create mode 100644 internal/config/builder.go create mode 100644 internal/config/builder_test.go diff --git a/internal/config/builder.go b/internal/config/builder.go new file mode 100644 index 0000000..0090cc2 --- /dev/null +++ b/internal/config/builder.go @@ -0,0 +1,102 @@ +package config + +import ( + "errors" + "io" + "os" + + "github.com/mitchellh/mapstructure" + "github.com/spf13/pflag" + "github.com/spf13/viper" +) + +// This is a simple module that encapsulate the creation +// of a config object and can override values from cli flags. + +type Builder struct { + in io.Reader + flags *pflag.FlagSet + binds map[string]string +} + +func NewBuilder() *Builder { + return &Builder{ + binds: map[string]string{}, + } +} + +// Set the config file to read +func (b *Builder) SetConfigFile(filename string) *Builder { + file, _ := os.Open(filename) + return b.SetSource(file) +} + +// Set the source to read +func (b *Builder) SetSource(in io.Reader) *Builder { + b.in = in + return b +} + +// Set all flags that the builder should use. +func (b *Builder) SetFlags(flags *pflag.FlagSet) *Builder { + b.flags = flags + return b +} + +// Add a flag to the builder. +func (b *Builder) AddFlag(flag *pflag.Flag) *Builder { + b.flags.AddFlag(flag) + return b +} + +// Build the config object from file, cli-flags +func (b *Builder) Build() (*Config, error) { + if b.in == nil { + return nil, errors.New("Config not set") + } + + conf := New() + + v := viper.New() + v.SetConfigType("yaml") + + if b.flags != nil { + // bind flags in viper. + for key, flagname := range b.binds { + flag := b.flags.Lookup(flagname) + if flag == nil { + continue + } + + if err := v.BindPFlag(key, flag); err != nil { + return nil, err + } + } + } + + // Read config and unmarshal + if err := v.ReadConfig(b.in); err != nil { + return nil, err + } + + decoders := mapstructure.ComposeDecodeHookFunc( + mapstructure.TextUnmarshallerHookFunc(), + mapstructure.StringToTimeDurationHookFunc(), + mapstructure.StringToSliceHookFunc(","), + decodeShorthandShipConfig, + ) + + err := v.Unmarshal(&conf, viper.DecodeHook(decoders)) + if err != nil { + return nil, err + } + + // Call custom handler. + if b.flags != nil { + if err := conf.ReadCliFlags(b.flags); err != nil { + return nil, err + } + } + + return &conf, nil +} diff --git a/internal/config/builder_test.go b/internal/config/builder_test.go new file mode 100644 index 0000000..e7ceaf5 --- /dev/null +++ b/internal/config/builder_test.go @@ -0,0 +1,151 @@ +package config + +import ( + "bytes" + "testing" + "time" + + "github.com/eosswedenorg/thalos/internal/log" + "github.com/spf13/pflag" + "github.com/stretchr/testify/require" + + shipclient "github.com/eosswedenorg-go/antelope-ship-client" +) + +func TestBuilder(t *testing.T) { + expected := Config{ + Name: "ship-reader-1", + Api: "http://127.0.0.1:8080", + MessageCodec: "mojibake", + Log: log.Config{ + Filename: "some_file.log", + Directory: "/path/to/whatever", + MaxFileSize: 200, + MaxTime: 30 * time.Minute, + }, + Ship: ShipConfig{ + Url: "127.0.0.1:8089", + StartBlockNum: 23671836, + EndBlockNum: 23872222, + IrreversibleOnly: true, + MaxMessagesInFlight: 1337, + }, + Telegram: TelegramConfig{ + Id: "110201543:AAHdqTcvCH1vGWJxfSeofSAs0K5PALDsaw", + Channel: -123456789, + }, + Redis: RedisConfig{ + Addr: "localhost:6379", + User: "myuser", + Password: "passwd", + DB: 4, + Prefix: "some::ship", + }, + } + + builder := NewBuilder() + builder.SetSource(bytes.NewBuffer([]byte(` +name: "ship-reader-1" +api: "http://127.0.0.1:8080" +message_codec: "mojibake" +log: + filename: some_file.log + directory: /path/to/whatever + maxtime: 30m + maxfilesize: 200b +ship: + url: "127.0.0.1:8089" + irreversible_only: true + max_messages_in_flight: 1337 + start_block_num: 23671836 + end_block_num: 23872222 +telegram: + id: "110201543:AAHdqTcvCH1vGWJxfSeofSAs0K5PALDsaw" + channel: -123456789 +redis: + addr: "localhost:6379" + user: "myuser" + password: "passwd" + db: 4 + prefix: "some::ship" +`))) + + cfg, err := builder.Build() + + require.NoError(t, err) + require.Equal(t, &expected, cfg) +} + +func TestBuilder_NilSource(t *testing.T) { + cfg, err := NewBuilder().Build() + require.Nil(t, cfg) + require.EqualError(t, err, "Config not set") +} + +func TestBuilder_WithShorthandShipUrl(t *testing.T) { + expected := Config{ + Name: "ship-reader-1", + Api: "http://127.0.0.1:8080", + MessageCodec: "json", + Log: log.Config{ + MaxFileSize: 10 * 1000 * 1000, // 10 mb + MaxTime: time.Hour * 24, + }, + Ship: ShipConfig{ + Url: "127.0.0.1:8089", + StartBlockNum: shipclient.NULL_BLOCK_NUMBER, + EndBlockNum: shipclient.NULL_BLOCK_NUMBER, + MaxMessagesInFlight: 10, + IrreversibleOnly: false, + }, + Telegram: TelegramConfig{ + Id: "110201543:AAHdqTcvCH1vGWJxfSeofSAs0K5PALDsaw", + Channel: -123456789, + }, + Redis: RedisConfig{ + Addr: "localhost:6379", + Password: "passwd", + DB: 4, + Prefix: "some::ship", + }, + } + + builder := NewBuilder() + builder.SetSource(bytes.NewBuffer([]byte(` +name: "ship-reader-1" +api: "http://127.0.0.1:8080" +ship: "127.0.0.1:8089" +telegram: + id: "110201543:AAHdqTcvCH1vGWJxfSeofSAs0K5PALDsaw" + channel: -123456789 +redis: + addr: "localhost:6379" + password: "passwd" + db: 4 + prefix: "some::ship" +`))) + + cfg, err := builder.Build() + + require.NoError(t, err) + require.Equal(t, &expected, cfg) +} + +func TestBuilder_Flags(t *testing.T) { + flags := pflag.FlagSet{} + flags.StringP("log", "l", "", "") + + require.NoError(t, flags.Set("log", "/path/to/logs")) + + cfg, err := NewBuilder(). + SetSource(bytes.NewReader([]byte(``))). + SetFlags(&flags). + Build() + + expected := New() + expected.Log.Filename = "logs" + expected.Log.Directory = "/path/to" + + require.NoError(t, err) + require.Equal(t, &expected, cfg) +} From cad74a4d8433671d53ba03c085f7e0271dacc7f3 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 18 Feb 2024 10:51:39 +0100 Subject: [PATCH 02/15] cmd/thalos/server.go: use the new config.Builder --- cmd/thalos/server.go | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/cmd/thalos/server.go b/cmd/thalos/server.go index 4170296..1a5c95b 100644 --- a/cmd/thalos/server.go +++ b/cmd/thalos/server.go @@ -153,7 +153,7 @@ func initAbiManager(api *eos.API, store cache.Store, chain_id string) *abi.AbiMa return abi.NewAbiManager(cache, api) } -func stateLoader(conf config.Config, chainInfo *eos.InfoResp, cache *cache.Cache, current_block_no_cache bool) StateLoader { +func stateLoader(conf *config.Config, chainInfo *eos.InfoResp, cache *cache.Cache, current_block_no_cache bool) StateLoader { return func(state *State) { var source string @@ -198,23 +198,16 @@ func stateSaver(cache *cache.Cache) StateSaver { } } -func ReadConfig(cfg *config.Config, flags *pflag.FlagSet) error { +func GetConfig(flags *pflag.FlagSet) (*config.Config, error) { filename, err := flags.GetString("config") if err != nil { - return err + return nil, err } - // Read file first. - if err := cfg.ReadFile(filename); err != nil { - return err - } - - // Then override any cli flags - if err := cfg.ReadCliFlags(flags); err != nil { - return err - } - - return nil + return config.NewBuilder(). + SetConfigFile(filename). + SetFlags(flags). + Build() } func serverCmd(cmd *cobra.Command, args []string) { @@ -234,8 +227,8 @@ func serverCmd(cmd *cobra.Command, args []string) { } // Parse config - conf := config.New() - if err = ReadConfig(&conf, cmd.Flags()); err != nil { + conf, err := GetConfig(cmd.Flags()) + if err != nil { log.WithError(err).Fatal("Failed to read config") return } @@ -354,7 +347,7 @@ func serverCmd(cmd *cobra.Command, args []string) { ) // Run the application - run(&conf, shClient, processor) + run(conf, shClient, processor) // Close the processor properly processor.Close() From b4c305d8ead681f7ba18d90f6d173ff331d3b1f3 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 18 Feb 2024 10:52:06 +0100 Subject: [PATCH 03/15] Remove internal/config/file.go --- internal/config/config_test.go | 110 --------------------------------- internal/config/file.go | 37 ----------- 2 files changed, 147 deletions(-) delete mode 100644 internal/config/file.go diff --git a/internal/config/config_test.go b/internal/config/config_test.go index ceba4e2..c919a18 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -36,113 +36,3 @@ func TestNew(t *testing.T) { require.Equal(t, expected, New()) } - -func TestRead(t *testing.T) { - expected := Config{ - Name: "ship-reader-1", - Api: "http://127.0.0.1:8080", - MessageCodec: "mojibake", - Log: log.Config{ - Filename: "some_file.log", - Directory: "/path/to/whatever", - MaxFileSize: 200, - MaxTime: 30 * time.Minute, - }, - Ship: ShipConfig{ - Url: "127.0.0.1:8089", - StartBlockNum: 23671836, - EndBlockNum: 23872222, - IrreversibleOnly: true, - MaxMessagesInFlight: 1337, - }, - Telegram: TelegramConfig{ - Id: "110201543:AAHdqTcvCH1vGWJxfSeofSAs0K5PALDsaw", - Channel: -123456789, - }, - Redis: RedisConfig{ - Addr: "localhost:6379", - User: "myuser", - Password: "passwd", - DB: 4, - Prefix: "some::ship", - }, - } - - cfg := Config{} - err := cfg.Read([]byte(` -name: "ship-reader-1" -api: "http://127.0.0.1:8080" -message_codec: "mojibake" -log: - filename: some_file.log - directory: /path/to/whatever - maxtime: 30m - maxfilesize: 200b -ship: - url: "127.0.0.1:8089" - irreversible_only: true - max_messages_in_flight: 1337 - start_block_num: 23671836 - end_block_num: 23872222 -telegram: - id: "110201543:AAHdqTcvCH1vGWJxfSeofSAs0K5PALDsaw" - channel: -123456789 -redis: - addr: "localhost:6379" - user: "myuser" - password: "passwd" - db: 4 - prefix: "some::ship" -`)) - - require.NoError(t, err) - require.Equal(t, expected, cfg) -} - -func TestReadShorthandShipUrl(t *testing.T) { - expected := Config{ - Name: "ship-reader-1", - Api: "http://127.0.0.1:8080", - MessageCodec: "json", - Log: log.Config{ - MaxFileSize: 10 * 1000 * 1000, // 10 mb - MaxTime: time.Hour * 24, - }, - Ship: ShipConfig{ - Url: "127.0.0.1:8089", - StartBlockNum: shipclient.NULL_BLOCK_NUMBER, - EndBlockNum: shipclient.NULL_BLOCK_NUMBER, - MaxMessagesInFlight: 10, - IrreversibleOnly: false, - }, - Telegram: TelegramConfig{ - Id: "110201543:AAHdqTcvCH1vGWJxfSeofSAs0K5PALDsaw", - Channel: -123456789, - }, - Redis: RedisConfig{ - Addr: "localhost:6379", - Password: "passwd", - DB: 4, - Prefix: "some::ship", - }, - } - - cfg := New() - - err := cfg.Read([]byte(` -name: "ship-reader-1" -api: "http://127.0.0.1:8080" -ship: "127.0.0.1:8089" -telegram: - id: "110201543:AAHdqTcvCH1vGWJxfSeofSAs0K5PALDsaw" - channel: -123456789 -redis: - addr: "localhost:6379" - password: "passwd" - db: 4 - prefix: "some::ship" -`)) - - require.NoError(t, err) - require.Equal(t, expected, cfg) -} diff --git a/internal/config/file.go b/internal/config/file.go deleted file mode 100644 index c14b00d..0000000 --- a/internal/config/file.go +++ /dev/null @@ -1,37 +0,0 @@ -package config - -import ( - "bytes" - "os" - - "github.com/mitchellh/mapstructure" - "github.com/spf13/viper" -) - -// Read values from file -func (cfg *Config) ReadFile(filename string) error { - bytes, err := os.ReadFile(filename) - if err != nil { - return err - } - - return cfg.Read(bytes) -} - -func (cfg *Config) Read(in []byte) error { - v := viper.New() - v.SetConfigType("yaml") - - if err := v.ReadConfig(bytes.NewBuffer(in)); err != nil { - return err - } - - decoders := mapstructure.ComposeDecodeHookFunc( - mapstructure.TextUnmarshallerHookFunc(), - mapstructure.StringToTimeDurationHookFunc(), - mapstructure.StringToSliceHookFunc(","), - decodeShorthandShipConfig, - ) - - return v.Unmarshal(cfg, viper.DecodeHook(decoders)) -} From 7033240000735e47d4b77c0981e04d6c24bc49e6 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 18 Feb 2024 10:55:15 +0100 Subject: [PATCH 04/15] internal/config/cli.go: Rename Config.ReadCliFlags to ovverideCliFlags and make it a function instead of method. --- internal/config/builder.go | 2 +- internal/config/cli.go | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/internal/config/builder.go b/internal/config/builder.go index 0090cc2..b9c0e75 100644 --- a/internal/config/builder.go +++ b/internal/config/builder.go @@ -93,7 +93,7 @@ func (b *Builder) Build() (*Config, error) { // Call custom handler. if b.flags != nil { - if err := conf.ReadCliFlags(b.flags); err != nil { + if err := overrideCliFlags(&conf, b.flags); err != nil { return nil, err } } diff --git a/internal/config/cli.go b/internal/config/cli.go index d5bb6ed..b56f6a0 100644 --- a/internal/config/cli.go +++ b/internal/config/cli.go @@ -6,8 +6,7 @@ import ( "github.com/spf13/pflag" ) -// Read cli flag values into the config -func (cfg *Config) ReadCliFlags(flags *pflag.FlagSet) error { +func overrideCliFlags(cfg *Config, flags *pflag.FlagSet) error { logFile, _ := flags.GetString("log") if len(logFile) > 0 { cfg.Log.Directory = path.Dir(logFile) From cb207b2f3385df281aa7bfcadacba914ba740c69 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 18 Feb 2024 11:00:58 +0100 Subject: [PATCH 05/15] cmd/thalos/main.go: Add start-block and end-block flags. --- cmd/thalos/main.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmd/thalos/main.go b/cmd/thalos/main.go index 994cdd2..c31c956 100644 --- a/cmd/thalos/main.go +++ b/cmd/thalos/main.go @@ -34,6 +34,9 @@ func init() { flags.StringP("pid", "p", "", "Where to write process id") flags.BoolP("no-state-cache", "n", false, "Force the application to take start block from config/api") + flags.Int("start-block", 0, "Start to stream from this block (default: config value, cache, head from api)") + flags.Int("end-block", 0, "Stop streaming when this block is reached") + rootCmd.PersistentFlags().AddFlagSet(&flags) } From 117f1b50b4730499c822eff93308b076fd3d0af8 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 18 Feb 2024 11:02:11 +0100 Subject: [PATCH 06/15] internal/config/builder.go: bind start-block, end-block flags to config --- internal/config/builder.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/internal/config/builder.go b/internal/config/builder.go index b9c0e75..35f0056 100644 --- a/internal/config/builder.go +++ b/internal/config/builder.go @@ -21,7 +21,10 @@ type Builder struct { func NewBuilder() *Builder { return &Builder{ - binds: map[string]string{}, + binds: map[string]string{ + "ship.start_block_num": "start-block", + "ship.end_block_num": "end-block", + }, } } From beb5b6cf0465e2a4beed72adfb72f50ffa876f5f Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 18 Feb 2024 11:44:19 +0100 Subject: [PATCH 07/15] cmd/thalos/main.go: move cli flags to internal/config/cli.go as it is easier to write tests if we can get a hold of the flags. --- cmd/thalos/main.go | 14 ++------------ internal/config/builder_test.go | 6 ++---- internal/config/cli.go | 14 ++++++++++++++ 3 files changed, 18 insertions(+), 16 deletions(-) diff --git a/cmd/thalos/main.go b/cmd/thalos/main.go index c31c956..b2d8f33 100644 --- a/cmd/thalos/main.go +++ b/cmd/thalos/main.go @@ -1,9 +1,9 @@ package main import ( + "github.com/eosswedenorg/thalos/internal/config" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" - "github.com/spf13/pflag" ) var VersionString string = "dev" @@ -27,17 +27,7 @@ func init() { `) rootCmd.SetVersionTemplate(`{{with .Name}}{{printf "%s " .}}{{end}}{{printf "v%s" .Version}}` + "\n") - flags := pflag.FlagSet{} - flags.StringP("config", "c", "./config.yml", "Config file to read") - flags.StringP("level", "L", "info", "Log level to use") - flags.StringP("log", "l", "", "Path to log file (default: print to stdout/stderr)") - flags.StringP("pid", "p", "", "Where to write process id") - flags.BoolP("no-state-cache", "n", false, "Force the application to take start block from config/api") - - flags.Int("start-block", 0, "Start to stream from this block (default: config value, cache, head from api)") - flags.Int("end-block", 0, "Stop streaming when this block is reached") - - rootCmd.PersistentFlags().AddFlagSet(&flags) + rootCmd.PersistentFlags().AddFlagSet(config.GetFlags()) } func main() { diff --git a/internal/config/builder_test.go b/internal/config/builder_test.go index e7ceaf5..8ff339f 100644 --- a/internal/config/builder_test.go +++ b/internal/config/builder_test.go @@ -6,7 +6,6 @@ import ( "time" "github.com/eosswedenorg/thalos/internal/log" - "github.com/spf13/pflag" "github.com/stretchr/testify/require" shipclient "github.com/eosswedenorg-go/antelope-ship-client" @@ -132,14 +131,13 @@ redis: } func TestBuilder_Flags(t *testing.T) { - flags := pflag.FlagSet{} - flags.StringP("log", "l", "", "") + flags := GetFlags() require.NoError(t, flags.Set("log", "/path/to/logs")) cfg, err := NewBuilder(). SetSource(bytes.NewReader([]byte(``))). - SetFlags(&flags). + SetFlags(flags). Build() expected := New() diff --git a/internal/config/cli.go b/internal/config/cli.go index b56f6a0..0993601 100644 --- a/internal/config/cli.go +++ b/internal/config/cli.go @@ -6,6 +6,20 @@ import ( "github.com/spf13/pflag" ) +func GetFlags() *pflag.FlagSet { + flags := pflag.FlagSet{} + flags.StringP("config", "c", "./config.yml", "Config file to read") + flags.StringP("level", "L", "info", "Log level to use") + flags.StringP("log", "l", "", "Path to log file (default: print to stdout/stderr)") + flags.StringP("pid", "p", "", "Where to write process id") + flags.BoolP("no-state-cache", "n", false, "Force the application to take start block from config/api") + + flags.Int("start-block", 0, "Start to stream from this block (default: config value, cache, head from api)") + flags.Int("end-block", 0, "Stop streaming when this block is reached") + + return &flags +} + func overrideCliFlags(cfg *Config, flags *pflag.FlagSet) error { logFile, _ := flags.GetString("log") if len(logFile) > 0 { From 2db0a64bd4715d7e50c2deb5a36e9402ad062eb8 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 18 Feb 2024 15:45:10 +0100 Subject: [PATCH 08/15] internal/config: add additional flags for config fields. --- internal/config/builder.go | 27 +++++++++++++++-- internal/config/builder_test.go | 51 ++++++++++++++++++++++++++++++--- internal/config/cli.go | 38 ++++++++++++++++++++++-- 3 files changed, 107 insertions(+), 9 deletions(-) diff --git a/internal/config/builder.go b/internal/config/builder.go index 35f0056..32ded36 100644 --- a/internal/config/builder.go +++ b/internal/config/builder.go @@ -22,8 +22,31 @@ type Builder struct { func NewBuilder() *Builder { return &Builder{ binds: map[string]string{ - "ship.start_block_num": "start-block", - "ship.end_block_num": "end-block", + "api": "url", + "message_codec": "codec", + + // Redis + "redis.addr": "redis-addr", + "redis.user": "redis-user", + "redis.password": "redis-password", + "redis.db": "redis-db", + "redis.prefix": "redis-prefix", + + // Telegram + "telegram.id": "telegram-id", + "telegram.channel": "telegram-channel", + + // Log + "log.maxfilesize": "log-max-filesize", + "log.maxtime": "log-max-time", + + // Ship + "ship.url": "ship-url", + "ship.start_block_num": "start-block", + "ship.end_block_num": "end-block", + "ship.irreversible_only": "irreversible-only", + "ship.max_messages_in_flight": "max-msg-in-flight", + "ship.chain": "chain", }, } } diff --git a/internal/config/builder_test.go b/internal/config/builder_test.go index 8ff339f..01e1c67 100644 --- a/internal/config/builder_test.go +++ b/internal/config/builder_test.go @@ -133,16 +133,59 @@ redis: func TestBuilder_Flags(t *testing.T) { flags := GetFlags() - require.NoError(t, flags.Set("log", "/path/to/logs")) + require.NoError(t, flags.Set("url", "https://myapi")) + require.NoError(t, flags.Set("codec", "binary")) + require.NoError(t, flags.Set("redis-addr", "154.223.38.15:6380")) + require.NoError(t, flags.Set("redis-user", "myuser")) + require.NoError(t, flags.Set("redis-password", "secret123")) + require.NoError(t, flags.Set("redis-db", "3")) + require.NoError(t, flags.Set("redis-prefix", "custom-prefix")) + require.NoError(t, flags.Set("telegram-id", "72983126312982618")) + require.NoError(t, flags.Set("telegram-channel", "-293492332")) + require.NoError(t, flags.Set("log", "/path/to/logs/mylog")) + require.NoError(t, flags.Set("log-max-filesize", "25mb")) + require.NoError(t, flags.Set("log-max-time", "10m")) + require.NoError(t, flags.Set("ship-url", "ws://myship.com:7823")) + require.NoError(t, flags.Set("start-block", "7327833")) + require.NoError(t, flags.Set("end-block", "329408392")) + require.NoError(t, flags.Set("irreversible-only", "true")) + require.NoError(t, flags.Set("max-msg-in-flight", "98")) + require.NoError(t, flags.Set("chain", "wax")) cfg, err := NewBuilder(). SetSource(bytes.NewReader([]byte(``))). SetFlags(flags). Build() - expected := New() - expected.Log.Filename = "logs" - expected.Log.Directory = "/path/to" + expected := Config{ + Api: "https://myapi", + MessageCodec: "binary", + Log: log.Config{ + Filename: "mylog", + Directory: "/path/to/logs", + MaxFileSize: 25 * 1000 * 1000, // 25 mb + MaxTime: time.Minute * 10, + }, + Ship: ShipConfig{ + Url: "ws://myship.com:7823", + StartBlockNum: 7327833, + EndBlockNum: 329408392, + MaxMessagesInFlight: 98, + IrreversibleOnly: true, + Chain: "wax", + }, + Telegram: TelegramConfig{ + Id: "72983126312982618", + Channel: -293492332, + }, + Redis: RedisConfig{ + Addr: "154.223.38.15:6380", + User: "myuser", + Password: "secret123", + DB: 3, + Prefix: "custom-prefix", + }, + } require.NoError(t, err) require.Equal(t, &expected, cfg) diff --git a/internal/config/cli.go b/internal/config/cli.go index 0993601..5baecdb 100644 --- a/internal/config/cli.go +++ b/internal/config/cli.go @@ -2,20 +2,52 @@ package config import ( "path" + "time" + shipclient "github.com/eosswedenorg-go/antelope-ship-client" "github.com/spf13/pflag" ) func GetFlags() *pflag.FlagSet { flags := pflag.FlagSet{} + + // Cli only flags flags.StringP("config", "c", "./config.yml", "Config file to read") flags.StringP("level", "L", "info", "Log level to use") - flags.StringP("log", "l", "", "Path to log file (default: print to stdout/stderr)") flags.StringP("pid", "p", "", "Where to write process id") flags.BoolP("no-state-cache", "n", false, "Force the application to take start block from config/api") - flags.Int("start-block", 0, "Start to stream from this block (default: config value, cache, head from api)") - flags.Int("end-block", 0, "Stop streaming when this block is reached") + // Generic + flags.StringP("url", "u", "", "Url to antelope api") + flags.String("codec", "json", "Codec used to send messages") + + // Redis + flags.String("redis-addr", "127.0.0.1:6379", "host:port to redis server") + flags.String("redis-user", "", "Redis username") + flags.String("redis-password", "", "Redis password") + flags.Int("redis-db", 0, "Redis database") + flags.String("redis-prefix", "ship", "Redis channel prefix") + + // Telegram + flags.String("telegram-id", "", "Id of telegram bot") + flags.Int64("telegram-channel", 0, "Telegram channel to send notifications to") + + // Log + flags.StringP("log", "l", "", "Path to log file (default: print to stdout/stderr)") + flags.String("log-max-filesize", "10mb", "Max filesize for logfile to rotate") + flags.Duration("log-max-time", time.Hour*24, "Max time for logfile to rotate") + + // Ship + flags.String("ship-url", "ws://127.0.0.1:8080", "Url to ship node") + flags.Uint32("start-block", shipclient.NULL_BLOCK_NUMBER, "Start to stream from this block") + flags.Uint32("end-block", shipclient.NULL_BLOCK_NUMBER, "Stop streaming when this block is reached") + + flags.Lookup("start-block").DefValue = "config value, cache, head from api" + flags.Lookup("end-block").DefValue = "none" + + flags.Bool("irreversible-only", false, "Only stream irreversible blocks from ship") + flags.Int("max-msg-in-flight", 10, "Maximum messages that can be sent from SHIP without acknowledgement") + flags.String("chain", "", "ChainID used in channel namespace, can be any string (default from api)") return &flags } From 4a4489e2be3972ea62dcfe516578c8ee71642c28 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 18 Feb 2024 16:04:02 +0100 Subject: [PATCH 09/15] internal/config/builder_test.go: adding test for config with flags --- internal/config/builder_test.go | 73 +++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/internal/config/builder_test.go b/internal/config/builder_test.go index 01e1c67..377c457 100644 --- a/internal/config/builder_test.go +++ b/internal/config/builder_test.go @@ -75,6 +75,79 @@ redis: require.Equal(t, &expected, cfg) } +func TestBuilder_ConfigWithFlags(t *testing.T) { + expected := Config{ + Name: "ship-reader-1", + Api: "https://api.example.com", + MessageCodec: "msgpack", + Log: log.Config{ + Filename: "mylog.log", + Directory: "/var/log", + MaxFileSize: 200, + MaxTime: 30 * time.Minute, + }, + Ship: ShipConfig{ + Url: "127.0.0.1:8089", + StartBlockNum: 23671836, + EndBlockNum: 23872222, + IrreversibleOnly: true, + MaxMessagesInFlight: 1337, + }, + Telegram: TelegramConfig{ + Id: "110201543:AAHdqTcvCH1vGWJxfSeofSAs0K5PALDsaw", + Channel: -123456789, + }, + Redis: RedisConfig{ + Addr: "localhost:6379", + User: "userfromcli", + Password: "passwd", + DB: 4, + Prefix: "some::ship", + }, + } + + builder := NewBuilder() + builder.SetSource(bytes.NewBuffer([]byte(` +name: "ship-reader-1" +api: "http://127.0.0.1:8080" +message_codec: "mojibake" +log: + filename: some_file.log + directory: /path/to/whatever + maxtime: 30m + maxfilesize: 200b +ship: + url: "127.0.0.1:8089" + irreversible_only: true + max_messages_in_flight: 1337 + start_block_num: 23671836 + end_block_num: 23872222 +telegram: + id: "110201543:AAHdqTcvCH1vGWJxfSeofSAs0K5PALDsaw" + channel: -123456789 +redis: + addr: "localhost:6379" + user: "myuser" + password: "passwd" + db: 4 + prefix: "some::ship" +`))) + + flags := GetFlags() + + require.NoError(t, flags.Set("url", "https://api.example.com")) + require.NoError(t, flags.Set("codec", "msgpack")) + require.NoError(t, flags.Set("log", "/var/log/mylog.log")) + require.NoError(t, flags.Set("redis-user", "userfromcli")) + + builder.SetFlags(flags) + + cfg, err := builder.Build() + + require.NoError(t, err) + require.Equal(t, &expected, cfg) +} + func TestBuilder_NilSource(t *testing.T) { cfg, err := NewBuilder().Build() require.Nil(t, cfg) From e54a4fa929f5a31963701915a21b1aeefa105365 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 18 Feb 2024 16:08:18 +0100 Subject: [PATCH 10/15] internal/config: remove shorthand ship config (where it is possible to have "ship" key only contain a url string instead of struct) It makes parsing harder and i don't think anyone uses it and its not that useful tbh. --- internal/config/builder.go | 1 - internal/config/builder_test.go | 51 --------------------------------- internal/config/config.go | 32 ++++----------------- 3 files changed, 6 insertions(+), 78 deletions(-) diff --git a/internal/config/builder.go b/internal/config/builder.go index 32ded36..d2ab2bf 100644 --- a/internal/config/builder.go +++ b/internal/config/builder.go @@ -109,7 +109,6 @@ func (b *Builder) Build() (*Config, error) { mapstructure.TextUnmarshallerHookFunc(), mapstructure.StringToTimeDurationHookFunc(), mapstructure.StringToSliceHookFunc(","), - decodeShorthandShipConfig, ) err := v.Unmarshal(&conf, viper.DecodeHook(decoders)) diff --git a/internal/config/builder_test.go b/internal/config/builder_test.go index 377c457..3f52548 100644 --- a/internal/config/builder_test.go +++ b/internal/config/builder_test.go @@ -7,8 +7,6 @@ import ( "github.com/eosswedenorg/thalos/internal/log" "github.com/stretchr/testify/require" - - shipclient "github.com/eosswedenorg-go/antelope-ship-client" ) func TestBuilder(t *testing.T) { @@ -154,55 +152,6 @@ func TestBuilder_NilSource(t *testing.T) { require.EqualError(t, err, "Config not set") } -func TestBuilder_WithShorthandShipUrl(t *testing.T) { - expected := Config{ - Name: "ship-reader-1", - Api: "http://127.0.0.1:8080", - MessageCodec: "json", - Log: log.Config{ - MaxFileSize: 10 * 1000 * 1000, // 10 mb - MaxTime: time.Hour * 24, - }, - Ship: ShipConfig{ - Url: "127.0.0.1:8089", - StartBlockNum: shipclient.NULL_BLOCK_NUMBER, - EndBlockNum: shipclient.NULL_BLOCK_NUMBER, - MaxMessagesInFlight: 10, - IrreversibleOnly: false, - }, - Telegram: TelegramConfig{ - Id: "110201543:AAHdqTcvCH1vGWJxfSeofSAs0K5PALDsaw", - Channel: -123456789, - }, - Redis: RedisConfig{ - Addr: "localhost:6379", - Password: "passwd", - DB: 4, - Prefix: "some::ship", - }, - } - - builder := NewBuilder() - builder.SetSource(bytes.NewBuffer([]byte(` -name: "ship-reader-1" -api: "http://127.0.0.1:8080" -ship: "127.0.0.1:8089" -telegram: - id: "110201543:AAHdqTcvCH1vGWJxfSeofSAs0K5PALDsaw" - channel: -123456789 -redis: - addr: "localhost:6379" - password: "passwd" - db: 4 - prefix: "some::ship" -`))) - - cfg, err := builder.Build() - - require.NoError(t, err) - require.Equal(t, &expected, cfg) -} - func TestBuilder_Flags(t *testing.T) { flags := GetFlags() diff --git a/internal/config/config.go b/internal/config/config.go index e137352..8edacdb 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1,7 +1,6 @@ package config import ( - "reflect" "time" "github.com/eosswedenorg/thalos/internal/log" @@ -52,34 +51,15 @@ func New() Config { MaxFileSize: 10 * 1000 * 1000, // 10 mb MaxTime: time.Hour * 24, }, - Ship: defaultShipConfig(""), + Ship: ShipConfig{ + StartBlockNum: shipclient.NULL_BLOCK_NUMBER, + EndBlockNum: shipclient.NULL_BLOCK_NUMBER, + MaxMessagesInFlight: 10, + IrreversibleOnly: false, + }, Redis: RedisConfig{ Addr: "localhost:6379", Prefix: "ship", }, } } - -func defaultShipConfig(url string) ShipConfig { - return ShipConfig{ - Url: url, - StartBlockNum: shipclient.NULL_BLOCK_NUMBER, - EndBlockNum: shipclient.NULL_BLOCK_NUMBER, - MaxMessagesInFlight: 10, - IrreversibleOnly: false, - } -} - -// mapstructure DecodeHook that can parse a shorthand ship config (only string instead of struct.) -func decodeShorthandShipConfig(from reflect.Value, to reflect.Value) (interface{}, error) { - shipType := reflect.TypeOf(ShipConfig{}) - - // If to is a struct and is assignable to a ShipConfig and from is a string. - // Then we treat the from value as ShipConfig.Url - if to.Kind() == reflect.Struct && to.Type().AssignableTo(shipType) && from.Kind() == reflect.String { - return defaultShipConfig(from.String()), nil - } - - // If not, decode as normal. - return from.Interface(), nil -} From 69a36e016c599bc77807a32b9ad96b0a49929cdc Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 18 Feb 2024 16:11:32 +0100 Subject: [PATCH 11/15] internal/config: skip creating a Config struct with default values. those should be set by flags or viper in the builder --- internal/config/builder.go | 2 +- internal/config/config.go | 25 ---------------------- internal/config/config_test.go | 38 ---------------------------------- 3 files changed, 1 insertion(+), 64 deletions(-) delete mode 100644 internal/config/config_test.go diff --git a/internal/config/builder.go b/internal/config/builder.go index d2ab2bf..6a55976 100644 --- a/internal/config/builder.go +++ b/internal/config/builder.go @@ -81,7 +81,7 @@ func (b *Builder) Build() (*Config, error) { return nil, errors.New("Config not set") } - conf := New() + conf := Config{} v := viper.New() v.SetConfigType("yaml") diff --git a/internal/config/config.go b/internal/config/config.go index 8edacdb..41777df 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1,11 +1,7 @@ package config import ( - "time" - "github.com/eosswedenorg/thalos/internal/log" - - shipclient "github.com/eosswedenorg-go/antelope-ship-client" ) type RedisConfig struct { @@ -42,24 +38,3 @@ type Config struct { Telegram TelegramConfig `yaml:"telegram" mapstructure:"telegram"` } - -// Create a new Config object with default values -func New() Config { - return Config{ - MessageCodec: "json", - Log: log.Config{ - MaxFileSize: 10 * 1000 * 1000, // 10 mb - MaxTime: time.Hour * 24, - }, - Ship: ShipConfig{ - StartBlockNum: shipclient.NULL_BLOCK_NUMBER, - EndBlockNum: shipclient.NULL_BLOCK_NUMBER, - MaxMessagesInFlight: 10, - IrreversibleOnly: false, - }, - Redis: RedisConfig{ - Addr: "localhost:6379", - Prefix: "ship", - }, - } -} diff --git a/internal/config/config_test.go b/internal/config/config_test.go deleted file mode 100644 index c919a18..0000000 --- a/internal/config/config_test.go +++ /dev/null @@ -1,38 +0,0 @@ -package config - -import ( - "testing" - "time" - - "github.com/eosswedenorg/thalos/internal/log" - "github.com/stretchr/testify/require" - - shipclient "github.com/eosswedenorg-go/antelope-ship-client" -) - -func TestNew(t *testing.T) { - expected := Config{ - MessageCodec: "json", - - Log: log.Config{ - MaxFileSize: 10 * 1000 * 1000, // 10 mb - MaxTime: time.Hour * 24, - }, - - Ship: ShipConfig{ - StartBlockNum: shipclient.NULL_BLOCK_NUMBER, - EndBlockNum: shipclient.NULL_BLOCK_NUMBER, - MaxMessagesInFlight: 10, - IrreversibleOnly: false, - }, - - Redis: RedisConfig{ - Addr: "localhost:6379", - Password: "", - DB: 0, - Prefix: "ship", - }, - } - - require.Equal(t, expected, New()) -} From 53baae8a7ff629e65963602e0cb6c955c2b36f20 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 18 Feb 2024 17:40:04 +0100 Subject: [PATCH 12/15] internal/config/cli.go: move flags that are not bound to config to cmd/thalos/main.go --- cmd/thalos/main.go | 9 +++++++++ internal/config/cli.go | 6 ------ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/cmd/thalos/main.go b/cmd/thalos/main.go index b2d8f33..fc55eef 100644 --- a/cmd/thalos/main.go +++ b/cmd/thalos/main.go @@ -4,6 +4,7 @@ import ( "github.com/eosswedenorg/thalos/internal/config" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" + "github.com/spf13/pflag" ) var VersionString string = "dev" @@ -27,6 +28,14 @@ func init() { `) rootCmd.SetVersionTemplate(`{{with .Name}}{{printf "%s " .}}{{end}}{{printf "v%s" .Version}}` + "\n") + flags := pflag.FlagSet{} + + flags.StringP("config", "c", "./config.yml", "Config file to read") + flags.StringP("level", "L", "info", "Log level to use") + flags.StringP("pid", "p", "", "Where to write process id") + flags.BoolP("no-state-cache", "n", false, "Force the application to take start block from config/api") + + rootCmd.PersistentFlags().AddFlagSet(&flags) rootCmd.PersistentFlags().AddFlagSet(config.GetFlags()) } diff --git a/internal/config/cli.go b/internal/config/cli.go index 5baecdb..c5b3c95 100644 --- a/internal/config/cli.go +++ b/internal/config/cli.go @@ -11,12 +11,6 @@ import ( func GetFlags() *pflag.FlagSet { flags := pflag.FlagSet{} - // Cli only flags - flags.StringP("config", "c", "./config.yml", "Config file to read") - flags.StringP("level", "L", "info", "Log level to use") - flags.StringP("pid", "p", "", "Where to write process id") - flags.BoolP("no-state-cache", "n", false, "Force the application to take start block from config/api") - // Generic flags.StringP("url", "u", "", "Url to antelope api") flags.String("codec", "json", "Codec used to send messages") From 7f1f186aa02cbad418c0585e791bb07e3b3bcfc2 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 18 Feb 2024 18:00:26 +0100 Subject: [PATCH 13/15] nternal/config/cli.go: move overrideCliFlags() to cmd/thalos/server.go as its pretty specific code. --- cmd/thalos/server.go | 14 +++++++++++++- internal/config/builder.go | 7 ------- internal/config/builder_test.go | 8 ++------ internal/config/cli.go | 10 ---------- 4 files changed, 15 insertions(+), 24 deletions(-) diff --git a/cmd/thalos/server.go b/cmd/thalos/server.go index 1a5c95b..5103b5a 100644 --- a/cmd/thalos/server.go +++ b/cmd/thalos/server.go @@ -7,6 +7,7 @@ import ( "io" "os" "os/signal" + "path" "syscall" "time" @@ -204,10 +205,21 @@ func GetConfig(flags *pflag.FlagSet) (*config.Config, error) { return nil, err } - return config.NewBuilder(). + cfg, err := config.NewBuilder(). SetConfigFile(filename). SetFlags(flags). Build() + if err != nil { + return nil, err + } + + logFile, _ := flags.GetString("log") + if len(logFile) > 0 { + cfg.Log.Directory = path.Dir(logFile) + cfg.Log.Filename = path.Base(logFile) + } + + return cfg, nil } func serverCmd(cmd *cobra.Command, args []string) { diff --git a/internal/config/builder.go b/internal/config/builder.go index 6a55976..5257e3d 100644 --- a/internal/config/builder.go +++ b/internal/config/builder.go @@ -116,12 +116,5 @@ func (b *Builder) Build() (*Config, error) { return nil, err } - // Call custom handler. - if b.flags != nil { - if err := overrideCliFlags(&conf, b.flags); err != nil { - return nil, err - } - } - return &conf, nil } diff --git a/internal/config/builder_test.go b/internal/config/builder_test.go index 3f52548..9bc1589 100644 --- a/internal/config/builder_test.go +++ b/internal/config/builder_test.go @@ -110,8 +110,8 @@ name: "ship-reader-1" api: "http://127.0.0.1:8080" message_codec: "mojibake" log: - filename: some_file.log - directory: /path/to/whatever + filename: mylog.log + directory: /var/log maxtime: 30m maxfilesize: 200b ship: @@ -135,7 +135,6 @@ redis: require.NoError(t, flags.Set("url", "https://api.example.com")) require.NoError(t, flags.Set("codec", "msgpack")) - require.NoError(t, flags.Set("log", "/var/log/mylog.log")) require.NoError(t, flags.Set("redis-user", "userfromcli")) builder.SetFlags(flags) @@ -164,7 +163,6 @@ func TestBuilder_Flags(t *testing.T) { require.NoError(t, flags.Set("redis-prefix", "custom-prefix")) require.NoError(t, flags.Set("telegram-id", "72983126312982618")) require.NoError(t, flags.Set("telegram-channel", "-293492332")) - require.NoError(t, flags.Set("log", "/path/to/logs/mylog")) require.NoError(t, flags.Set("log-max-filesize", "25mb")) require.NoError(t, flags.Set("log-max-time", "10m")) require.NoError(t, flags.Set("ship-url", "ws://myship.com:7823")) @@ -183,8 +181,6 @@ func TestBuilder_Flags(t *testing.T) { Api: "https://myapi", MessageCodec: "binary", Log: log.Config{ - Filename: "mylog", - Directory: "/path/to/logs", MaxFileSize: 25 * 1000 * 1000, // 25 mb MaxTime: time.Minute * 10, }, diff --git a/internal/config/cli.go b/internal/config/cli.go index c5b3c95..d831b7a 100644 --- a/internal/config/cli.go +++ b/internal/config/cli.go @@ -1,7 +1,6 @@ package config import ( - "path" "time" shipclient "github.com/eosswedenorg-go/antelope-ship-client" @@ -45,12 +44,3 @@ func GetFlags() *pflag.FlagSet { return &flags } - -func overrideCliFlags(cfg *Config, flags *pflag.FlagSet) error { - logFile, _ := flags.GetString("log") - if len(logFile) > 0 { - cfg.Log.Directory = path.Dir(logFile) - cfg.Log.Filename = path.Base(logFile) - } - return nil -} From 146ea99298f3e745e2002d582648285a56f9ece3 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 18 Feb 2024 18:03:38 +0100 Subject: [PATCH 14/15] cmd/thalos/server.go: in GetConfig() should override "no-state-cache" flag if "start-block" flag is set. --- cmd/thalos/server.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/cmd/thalos/server.go b/cmd/thalos/server.go index 5103b5a..3a98d24 100644 --- a/cmd/thalos/server.go +++ b/cmd/thalos/server.go @@ -219,6 +219,13 @@ func GetConfig(flags *pflag.FlagSet) (*config.Config, error) { cfg.Log.Filename = path.Base(logFile) } + // If start-block is provided, we should set no-state-cache to true. + if startBlock := flags.Lookup("start-block"); startBlock != nil && startBlock.Changed { + if err := flags.Set("no-state-cache", "true"); err != nil { + return cfg, nil + } + } + return cfg, nil } @@ -226,8 +233,6 @@ func serverCmd(cmd *cobra.Command, args []string) { var err error var chainInfo *eos.InfoResp - skip_currentblock_cache, _ := cmd.Flags().GetBool("no-state-cache") - // Write PID file pidFile, err := cmd.Flags().GetString("pid") if err != nil { @@ -245,6 +250,8 @@ func serverCmd(cmd *cobra.Command, args []string) { return } + skip_currentblock_cache, _ := cmd.Flags().GetBool("no-state-cache") + flagLevel, _ := cmd.Flags().GetString("level") lvl, err := log.ParseLevel(flagLevel) if err == nil { From fb3b35cbe93c50329171746d29082a6b3a365f4d Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 18 Feb 2024 18:09:12 +0100 Subject: [PATCH 15/15] cmd/thalos/server.go: in stateLoader(): set "cli" as source if start block is set via cli flag. --- cmd/thalos/server.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/cmd/thalos/server.go b/cmd/thalos/server.go index 3a98d24..aa10fac 100644 --- a/cmd/thalos/server.go +++ b/cmd/thalos/server.go @@ -154,7 +154,7 @@ func initAbiManager(api *eos.API, store cache.Store, chain_id string) *abi.AbiMa return abi.NewAbiManager(cache, api) } -func stateLoader(conf *config.Config, chainInfo *eos.InfoResp, cache *cache.Cache, current_block_no_cache bool) StateLoader { +func stateLoader(conf *config.Config, start_block_flag *pflag.Flag, chainInfo *eos.InfoResp, cache *cache.Cache, current_block_no_cache bool) StateLoader { return func(state *State) { var source string @@ -168,7 +168,13 @@ func stateLoader(conf *config.Config, chainInfo *eos.InfoResp, cache *cache.Cach if current_block_no_cache || err != nil { // Set from config if we have a sane value. if conf.Ship.StartBlockNum != shipclient.NULL_BLOCK_NUMBER { - source = "config" + + if start_block_flag != nil && start_block_flag.Changed { + source = "cli" + } else { + source = "config" + } + state.CurrentBlock = conf.Ship.StartBlockNum } else { // Otherwise, set from api. @@ -355,7 +361,7 @@ func serverCmd(cmd *cobra.Command, args []string) { processor := SpawnProccessor( shClient, - stateLoader(conf, chainInfo, cache, skip_currentblock_cache), + stateLoader(conf, cmd.Flags().Lookup("start-block"), chainInfo, cache, skip_currentblock_cache), stateSaver(cache), driver.NewPublisher(context.Background(), rdb, api_redis.Namespace{ Prefix: conf.Redis.Prefix,