From 31c7ba6a4b20984bf77a04d191e363c5ff221364 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Tue, 22 Aug 2023 16:22:03 +0200 Subject: [PATCH 001/291] Improved code documentation. --- app/abi/cache.go | 4 ++++ app/abi/manager.go | 4 ++++ app/log/RotatingFile.go | 5 +++++ app/log/config.go | 16 ++++++++++++---- app/ship_processor.go | 20 +++++++++++++++++--- app/types/size.go | 5 +++-- 6 files changed, 45 insertions(+), 9 deletions(-) diff --git a/app/abi/cache.go b/app/abi/cache.go index ac595f0..3b54084 100644 --- a/app/abi/cache.go +++ b/app/abi/cache.go @@ -8,12 +8,14 @@ import ( redis_cache "github.com/go-redis/cache/v9" ) +// Cache represents a abi cache in redis. type Cache struct { c *redis_cache.Cache ctx context.Context prefix string } +// Create a new cache func NewCache(prefix string, options *redis_cache.Options) *Cache { return &Cache{ c: redis_cache.New(options), @@ -22,12 +24,14 @@ func NewCache(prefix string, options *redis_cache.Options) *Cache { } } +// Get an ABI from the cache using the contract account name as the key. func (cache *Cache) Get(account string) (*eos.ABI, error) { var v eos.ABI err := cache.c.Get(cache.ctx, cache.key(account), &v) return &v, err } +// Set an ABI in the cache. func (cache *Cache) Set(account string, abi *eos.ABI, ttl time.Duration) error { return cache.c.Set(&redis_cache.Item{ Ctx: cache.ctx, diff --git a/app/abi/manager.go b/app/abi/manager.go index 2040a3a..79df1bd 100644 --- a/app/abi/manager.go +++ b/app/abi/manager.go @@ -10,12 +10,14 @@ import ( "github.com/redis/go-redis/v9" ) +// AbiManager handles an ABI cache that fetches the ABI from an API on cache miss. type AbiManager struct { cache *Cache api *eos.API ctx context.Context } +// Create a new ABI Manager func NewAbiManager(rdb *redis.Client, api *eos.API, id string) *AbiManager { // Init abi cache cache := NewCache("thalos::cache::"+id+"::abi", &redis_cache.Options{ @@ -36,6 +38,8 @@ func (mgr *AbiManager) SetAbi(account eos.AccountName, abi *eos.ABI) error { return mgr.cache.Set(string(account), abi, time.Hour) } +// Get an ABI from the cache, on cache miss it is fetched from the +// API, gets cached and then returned to the user func (mgr *AbiManager) GetAbi(account eos.AccountName) (*eos.ABI, error) { key := string(account) diff --git a/app/log/RotatingFile.go b/app/log/RotatingFile.go index 4654747..bbedd9f 100644 --- a/app/log/RotatingFile.go +++ b/app/log/RotatingFile.go @@ -8,6 +8,8 @@ import ( "time" ) +// Rotating file represents a file that can be rotated when either the file +// becomes to large or to old, whatever comes first type RotatingFile struct { fd *os.File size int64 @@ -21,6 +23,7 @@ func open(filename string) (*os.File, error) { return os.OpenFile(filename, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0o666) } +// Open a new rotating file. func NewRotatingFile(filename string, maxSize int64, maxAge time.Duration) (*RotatingFile, error) { if err := os.MkdirAll(path.Dir(filename), 0o766); err != nil && !os.IsExist(err) { return nil, err @@ -46,6 +49,7 @@ func NewRotatingFile(filename string, maxSize int64, maxAge time.Duration) (*Rot }, nil } +// Open a new rotating file using a config struct. func NewRotatingFileFromConfig(config Config, suffix string) (*RotatingFile, error) { if len(suffix) > 0 { suffix = "_" + suffix @@ -62,6 +66,7 @@ func (w *RotatingFile) newFilename(name string) string { return fmt.Sprintf("%s-%s%s", name, time.Now().Format(w.format), ext) } +// Get the filename func (w RotatingFile) GetFilename() string { return path.Base(w.fd.Name()) } diff --git a/app/log/config.go b/app/log/config.go index 54c3b55..16498ce 100644 --- a/app/log/config.go +++ b/app/log/config.go @@ -7,11 +7,19 @@ import ( "github.com/eosswedenorg/thalos/app/types" ) +// Config represents configuration parameters for a log. type Config struct { - Filename string `yaml:"filename"` - Directory string `yaml:"directory"` - MaxFileSize types.Size `yaml:"maxfilesize"` - MaxTime time.Duration `yaml:"maxtime"` + // Filename where the log is stored. + Filename string `yaml:"filename"` + + // Directory where the log files are stored. + Directory string `yaml:"directory"` + + // Maximum filesize, the log is rotated when this size is exceeded. + MaxFileSize types.Size `yaml:"maxfilesize"` + + // Maximum lifetime of the file before it is rotated. + MaxTime time.Duration `yaml:"maxtime"` } func (c Config) GetFilename() string { diff --git a/app/ship_processor.go b/app/ship_processor.go index 7ce37f5..96ebcef 100644 --- a/app/ship_processor.go +++ b/app/ship_processor.go @@ -28,11 +28,20 @@ func logDecoratedEncoder(encoder message.Encoder) message.Encoder { } } +// A ShipProcessor will consume messages from a ship stream, convert the messages into +// thalos specfic ones, encode them and finally post them to an api.Writer type ShipProcessor struct { - abi *abi.AbiManager - writer api.Writer + // The ship stream to process. shipStream *shipclient.Stream - encode message.Encoder + + // Abi manager used for cacheing + abi *abi.AbiManager + + // Writer to send messages to. + writer api.Writer + + // Encoder used to encode messages + encode message.Encoder // Keep track of the current block we have processed. current_block uint32 @@ -41,6 +50,7 @@ type ShipProcessor struct { syscontract eos.AccountName } +// SpawnProcessor creates a new ShipProccessor that consumes the shipclient.Stream passed to it. func SpawnProccessor(shipStream *shipclient.Stream, writer api.Writer, abi *abi.AbiManager, codec message.Codec) *ShipProcessor { processor := &ShipProcessor{ abi: abi, @@ -84,6 +94,7 @@ func decode(abi *eos.ABI, act *ship.Action, v any) error { return json.Unmarshal(jsondata, v) } +// updateAbiFromAction updates the contract abi based on the ship.Action passed. func (processor *ShipProcessor) updateAbiFromAction(act *ship.Action) error { ABI, err := processor.abi.GetAbi(processor.syscontract) if err != nil { @@ -111,10 +122,12 @@ func (processor *ShipProcessor) updateAbiFromAction(act *ship.Action) error { return processor.abi.SetAbi(set_abi.Account, &contract_abi) } +// Get the current block. func (processor *ShipProcessor) GetCurrentBlock() uint32 { return processor.current_block } +// Callback function called by shipclient.Stream when a new block arrives. func (processor *ShipProcessor) processBlock(block *ship.GetBlocksResultV0) { processor.current_block = block.ThisBlock.BlockNum @@ -261,6 +274,7 @@ func (processor *ShipProcessor) processBlock(block *ship.GetBlocksResultV0) { } } +// Close closes the writer assciated with the processor. func (processor *ShipProcessor) Close() error { return processor.writer.Close() } diff --git a/app/types/size.go b/app/types/size.go index a39b80d..c872c09 100644 --- a/app/types/size.go +++ b/app/types/size.go @@ -6,9 +6,10 @@ import ( ) // Size is an alias of int64 that can handle sizes represented -// in human readable strings like "200mb", "20 GB" etc +// in human readable strings like "200mb", "20 GB" etc. -type Size int64 // Size in bytes. +// The value is in bytes. +type Size int64 // Parse a string into number of bytes stored in a int64 func (s *Size) Parse(value string) error { From 9f61bbd316660f3235ec20b44be699fb5bd4888c Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Tue, 22 Aug 2023 16:24:09 +0200 Subject: [PATCH 002/291] cmd/tools/flags.go: adding Redis User and Password flags. --- cmd/tools/flags.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/cmd/tools/flags.go b/cmd/tools/flags.go index ede7ede..30c5a94 100644 --- a/cmd/tools/flags.go +++ b/cmd/tools/flags.go @@ -15,6 +15,16 @@ var redisUrlFlag = &cli.StringFlag{ Usage: "host:port to the redis server", } +var redisUserFlag = &cli.StringFlag{ + Name: "redis-user", + Usage: "User to use when authenticating to the server", +} + +var redisPasswordFlag = &cli.StringFlag{ + Name: "redis-pw", + Usage: "Password to use when authenticating to the server", +} + var redisDbFlag = &cli.IntFlag{ Name: "redis-db", Value: 0, From 1ba1422c6f264f381e9ccf5073c05a86674fb25f Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Tue, 22 Aug 2023 16:24:43 +0200 Subject: [PATCH 003/291] cmd/tools/bench.go: Support Redis username and password flags. --- cmd/tools/bench.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/cmd/tools/bench.go b/cmd/tools/bench.go index 4657f85..5d4dcc2 100644 --- a/cmd/tools/bench.go +++ b/cmd/tools/bench.go @@ -23,6 +23,8 @@ var benchCmd = &cli.Command{ Usage: "Run a benchmark against a thalos node", Flags: []cli.Flag{ redisUrlFlag, + redisUserFlag, + redisPasswordFlag, redisDbFlag, redisPrefixFlag, chainIdFlag, @@ -46,8 +48,10 @@ var benchCmd = &cli.Command{ // Create redis client rdb := redis.NewClient(&redis.Options{ - Addr: ctx.String("redis-url"), - DB: ctx.Int("redis-db"), + Addr: ctx.String("redis-url"), + Username: ctx.String("redis-user"), + Password: ctx.String("redis-pw"), + DB: ctx.Int("redis-db"), }) if err := rdb.Ping(context.Background()).Err(); err != nil { From 0f5c94f8dee1e1039c6304e22171ca386155de40 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Wed, 23 Aug 2023 11:23:58 +0200 Subject: [PATCH 004/291] Adding cmd/tools/redis-acl.go --- cmd/tools/main.go | 1 + cmd/tools/redis-acl.go | 171 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 172 insertions(+) create mode 100644 cmd/tools/redis-acl.go diff --git a/cmd/tools/main.go b/cmd/tools/main.go index 0dab953..48ac549 100644 --- a/cmd/tools/main.go +++ b/cmd/tools/main.go @@ -18,6 +18,7 @@ func main() { Commands: []*cli.Command{ validateCmd, benchCmd, + RedisACLCmd, }, } diff --git a/cmd/tools/redis-acl.go b/cmd/tools/redis-acl.go new file mode 100644 index 0000000..049f7ce --- /dev/null +++ b/cmd/tools/redis-acl.go @@ -0,0 +1,171 @@ +package main + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "math/rand" + "os" + "text/template" + "time" + + "github.com/urfave/cli/v2" +) + +// Helper struct representing a redis user. +type User struct { + // Username + Name string + + // Password + Password string + + // True if password was generated, false if not. + Generated bool +} + +func NewUser(name, password string) User { + if len(password) < 1 { + return User{ + Name: name, + Password: randomString(32), + Generated: true, + } + } + return User{Name: name, Password: password} +} + +func (u *User) Hash() { + u.Password = "#" + hash(u.Password) +} + +func (u User) Print() { + fmt.Println(u.Name+":", u.Password) +} + +func (u User) PrintIfGeneratedPW() { + if u.Generated { + u.Print() + } +} + +func randomString(length int) string { + charset := "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUWXYZ0123456789" + out := "" + for i := 0; i < length; i++ { + idx := rand.Intn(len(charset)) + out += string(charset[idx]) + } + return out +} + +func hash(str string) string { + data := sha256.Sum256([]byte(str)) + return hex.EncodeToString(data[:]) +} + +func writeTemplate(w io.Writer, defUser, serverUser, clientUser User, prefix string) error { + tmplStr := `# Created by thalos-tools on {{.timestamp}} +user default on {{.defaultpw}} ~* &* +@all +user {{.server}} on {{.serverpw}} resetchannels ~{{.prefix}}::* &{{.prefix}}::* -@all +get +publish +set +user {{.client}} on {{.clientpw}} resetchannels &{{.prefix}}::* -@all +subscribe +` + + tmpl, err := template.New("acl").Parse(tmplStr) + if err != nil { + return err + } + + return tmpl.Execute(w, map[string]string{ + "defaultpw": defUser.Password, + "client": clientUser.Name, + "clientpw": clientUser.Password, + "server": serverUser.Name, + "serverpw": serverUser.Password, + "prefix": prefix, + "timestamp": time.Now().Format(time.UnixDate), + }) +} + +var RedisACLCmd = &cli.Command{ + Name: "redis-acl", + Usage: "create a users.acl file", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "default-pw", + Usage: "Password to use for the default account, if not provided a random one will be generated", + }, + &cli.StringFlag{ + Name: "client", + Value: "thalos-client", + Usage: "Thalos client account name", + }, + &cli.StringFlag{ + Name: "client-pw", + Usage: "Password to use for the thalos client account, if not provided a random one will be generated", + }, + &cli.StringFlag{ + Name: "server", + Value: "thalos", + Usage: "Thalos account name", + }, + &cli.StringFlag{ + Name: "server-pw", + Usage: "Password to use for the thalos server account, if not provided a random one will be generated", + }, + &cli.StringFlag{ + Name: "prefix", + Value: "ship", + Usage: "Redis key prefix", + }, + &cli.BoolFlag{ + Name: "cleartext", + Usage: "If passwords should be hashed or left in cleartext.", + }, + &cli.StringFlag{ + Name: "file", + DefaultText: "Standard out", + Usage: "Where the config should be written to", + }, + }, + Action: func(ctx *cli.Context) error { + var err error + var out *os.File = os.Stdout + + rand.Seed(time.Now().Unix()) + + defaultUser := NewUser("default", ctx.String("default-pw")) + serverUser := NewUser(ctx.String("server"), ctx.String("server-pw")) + clientUser := NewUser(ctx.String("client"), ctx.String("client-pw")) + + atleastOneGeneratedPw := defaultUser.Generated || serverUser.Generated || clientUser.Generated + + if !ctx.Bool("cleartext") { + if atleastOneGeneratedPw { + println("Passwords") + } + + defaultUser.PrintIfGeneratedPW() + serverUser.PrintIfGeneratedPW() + clientUser.PrintIfGeneratedPW() + + defaultUser.Hash() + serverUser.Hash() + clientUser.Hash() + } + + filename := ctx.String("file") + if len(filename) > 0 { + out, err = os.Create(filename) + if err != nil { + return err + } + defer out.Close() + } else if !ctx.Bool("cleartext") && atleastOneGeneratedPw { + fmt.Println() + } + + return writeTemplate(out, defaultUser, serverUser, clientUser, ctx.String("prefix")) + }, +} From 16ab7d1fcf4be4f957f6a203e54d171c3fcc120c Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Wed, 30 Aug 2023 15:50:40 +0200 Subject: [PATCH 005/291] cmd/thalos/main.go: Fix some log lines to use structured logging. --- cmd/thalos/main.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/cmd/thalos/main.go b/cmd/thalos/main.go index 087fef9..c8d2473 100644 --- a/cmd/thalos/main.go +++ b/cmd/thalos/main.go @@ -69,7 +69,10 @@ func readerLoop(processor *app.ShipProcessor) { connectOp := func() error { recon_cnt++ - log.Infof("Connecting to ship at: %s (Try %d)", conf.Ship.Url, recon_cnt) + log.WithFields(log.Fields{ + "url": conf.Ship.Url, + "try": recon_cnt, + }).Info("Connecting to ship") if err := shClient.Connect(conf.Ship.Url); err != nil { return err @@ -108,7 +111,10 @@ func readerLoop(processor *app.ShipProcessor) { } recon_cnt = 0 - log.Infof("Connected, Start: %d, End: %d", shClient.StartBlock, shClient.EndBlock) + log.WithFields(log.Fields{ + "start": shClient.StartBlock, + "end": shClient.EndBlock, + }).Info("Connected to ship") if err := shClient.Run(); err != nil { @@ -183,7 +189,7 @@ func main() { // Write PID file if len(*pidFile) > 0 { - log.Infof("Writing pid to: %s", *pidFile) + log.WithField("file", *pidFile).Info("Writing pid to file") err = pid.Save(*pidFile) if err != nil { log.WithError(err).Fatal("failed to write pid file") @@ -258,7 +264,7 @@ func main() { return } - log.Printf("Get chain info from api at: %s", conf.Api) + log.WithField("api", conf.Api).Info("Get chain info from api") eosClient := eos.New(conf.Api) chainInfo, err = eosClient.GetInfo(context.Background()) if err != nil { From fb6cfb9fa64c1bd92f1628109131384205836916 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Wed, 30 Aug 2023 16:00:09 +0200 Subject: [PATCH 006/291] app/ship_processor.go: Need to call log.Dup() otherwise things like log.Level will not be copied. --- app/ship_processor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/ship_processor.go b/app/ship_processor.go index 96ebcef..7d2fe39 100644 --- a/app/ship_processor.go +++ b/app/ship_processor.go @@ -149,7 +149,7 @@ func (processor *ShipProcessor) processBlock(block *ship.GetBlocksResultV0) { if block.Traces != nil && len(block.Traces.Elem) > 0 { for _, trace := range block.Traces.AsTransactionTracesV0() { - logger := log.WithField("tx_id", trace.ID.String()) + logger := log.WithField("tx_id", trace.ID.String()).Dup() transaction := message.TransactionTrace{ ID: trace.ID.String(), From b364ce56d88f1d632d4a45b242f6c540c7723e00 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Wed, 30 Aug 2023 16:06:57 +0200 Subject: [PATCH 007/291] cmd/thalos/main.go: Add "level" cli flag to make it possible for user to specify log level. --- cmd/thalos/main.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/cmd/thalos/main.go b/cmd/thalos/main.go index c8d2473..020681b 100644 --- a/cmd/thalos/main.go +++ b/cmd/thalos/main.go @@ -163,6 +163,14 @@ func getChain(def string) string { return def } +func LogLevels() []string { + list := []string{} + for _, lvl := range log.AllLevels { + list = append(list, lvl.String()) + } + return list +} + func main() { var err error var chainInfo *eos.InfoResp @@ -174,6 +182,7 @@ func main() { configFile := getopt.StringLong("config", 'c', "./config.yml", "Config file to read", "file") pidFile := getopt.StringLong("pid", 'p', "", "Where to write process id", "file") logFile := getopt.StringLong("log", 'l', "", "Path to log file", "file") + logLevel := getopt.EnumLong("level", 'L', LogLevels(), "info", "Log level to use") getopt.Parse() @@ -210,6 +219,14 @@ func main() { conf.Log.Filename = path.Base(*logFile) } + lvl, err := log.ParseLevel(*logLevel) + if err == nil { + log.WithField("value", lvl).Info("Setting log level") + log.SetLevel(lvl) + } else { + log.WithError(err).Warn("Failed to parse level") + } + if len(conf.Log.Filename) > 0 { stdWriter, err := NewRotatingFileFromConfig(conf.Log, "info") if err != nil { From cc7ea62487042c614fa5ad9459b6cb6bcd4569fc Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Thu, 12 Oct 2023 20:51:00 +0200 Subject: [PATCH 008/291] app/config/config.go: use os.ReadFile instead of ioutil. --- app/config/config.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/config/config.go b/app/config/config.go index 18f9a00..e3050a2 100644 --- a/app/config/config.go +++ b/app/config/config.go @@ -1,7 +1,7 @@ package config import ( - "io/ioutil" + "os" "time" "github.com/eosswedenorg/thalos/app/log" @@ -88,7 +88,7 @@ func (ship *ShipConfig) UnmarshalYAML(value *yaml.Node) error { } func Load(filename string) (*Config, error) { - bytes, err := ioutil.ReadFile(filename) + bytes, err := os.ReadFile(filename) if err != nil { return nil, err } From 7cdf2f113d81163b6905fafc718e56aff20ae638 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Fri, 20 Oct 2023 18:00:15 +0200 Subject: [PATCH 009/291] Update packages. --- go.mod | 28 ++++++++++++++-------------- go.sum | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 14 deletions(-) diff --git a/go.mod b/go.mod index 2def7f1..431088e 100644 --- a/go.mod +++ b/go.mod @@ -5,15 +5,15 @@ go 1.18 require ( github.com/cenkalti/backoff/v4 v4.2.1 github.com/docker/go-units v0.5.0 - github.com/eoscanada/eos-go v0.10.3-0.20230413154640-bb75101dc2f6 + github.com/eoscanada/eos-go v0.10.3-0.20230609180026-5d4ee54fe15e github.com/eosswedenorg-go/antelope-ship-client v0.2.6 github.com/eosswedenorg-go/pid v1.0.1 - github.com/eosswedenorg/thalos/api v0.0.0-00010101000000-000000000000 + github.com/eosswedenorg/thalos/api v0.1.2 github.com/go-redis/cache/v9 v9.0.0 - github.com/go-redis/redismock/v9 v9.0.3 + github.com/go-redis/redismock/v9 v9.2.0 github.com/nikoksr/notify v0.41.0 github.com/pborman/getopt/v2 v2.1.0 - github.com/redis/go-redis/v9 v9.0.5 + github.com/redis/go-redis/v9 v9.2.1 github.com/sirupsen/logrus v1.9.3 github.com/stretchr/testify v1.8.4 github.com/urfave/cli/v2 v2.25.7 @@ -23,7 +23,7 @@ require ( require ( github.com/blendle/zapdriver v1.3.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/eosswedenorg-go/jsontime v0.0.0-20230509125027-08422d6236c7 // indirect @@ -31,7 +31,7 @@ require ( github.com/go-telegram-bot-api/telegram-bot-api v4.6.4+incompatible // indirect github.com/gorilla/websocket v1.5.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.16.6 // indirect + github.com/klauspost/compress v1.17.1 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/logrusorgru/aurora v2.0.3+incompatible // indirect github.com/mitchellh/go-testing-interface v1.14.1 // indirect @@ -43,22 +43,22 @@ require ( github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/streamingfast/logging v0.0.0-20230608130331-f22c91403091 // indirect github.com/technoweenie/multipartstreamer v1.0.1 // indirect - github.com/tidwall/gjson v1.14.4 // indirect + github.com/tidwall/gjson v1.17.0 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect github.com/ugorji/go/codec v1.2.11 // indirect github.com/vmihailenco/go-tinylfu v0.2.2 // indirect - github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect + github.com/vmihailenco/msgpack/v5 v5.4.0 // indirect github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/goleak v1.2.1 // indirect - go.uber.org/multierr v1.8.0 // indirect - go.uber.org/zap v1.24.0 // indirect - golang.org/x/crypto v0.10.0 // indirect - golang.org/x/sync v0.3.0 // indirect - golang.org/x/sys v0.9.0 // indirect - golang.org/x/term v0.9.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.26.0 // indirect + golang.org/x/crypto v0.14.0 // indirect + golang.org/x/sync v0.4.0 // indirect + golang.org/x/sys v0.13.0 // indirect + golang.org/x/term v0.13.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect ) diff --git a/go.sum b/go.sum index 4619816..f498e2a 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,4 @@ +contrib.go.opencensus.io/exporter/stackdriver v0.12.6/go.mod h1:8x999/OcIPy5ivx/wDiV7Gx4D+VUPODf0mWRGRc5kSk= github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a h1:HbKu58rmZpUGpz5+4FfNmIU+FmZg2P3Xaj2v2bfNWmk= github.com/alicebob/miniredis/v2 v2.30.2 h1:lc1UAUT9ZA7h4srlfBmBt2aorm5Yftk9nBjxz7EyY9I= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= @@ -5,7 +6,9 @@ github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZx github.com/blendle/zapdriver v1.3.1 h1:C3dydBOWYRiOk+B8X9IVZ5IOe+7cl+tGOexN4QqHfpE= github.com/blendle/zapdriver v1.3.1/go.mod h1:mdXfREi6u5MArG4j9fewC+FGnXaBR+T4Ox4J2u4eHCc= github.com/bsm/ginkgo/v2 v2.7.0 h1:ItPMPH90RbmZJt5GtkcNvIRuGEdwlBItdNVoyzaNQao= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= github.com/bsm/gomega v1.26.0 h1:LhQm+AFcgV2M0WyKroMASzAzCAJVpAxQXv4SaI9a69Y= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -16,6 +19,8 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.3 h1:qMCsGGgs+MAzDFyp9LpAe1Lqy/fY/qCovCm0qnXZOBM= +github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -26,6 +31,8 @@ github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4 github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/eoscanada/eos-go v0.10.3-0.20230413154640-bb75101dc2f6 h1:93LUOgAmRkmz8DF2V62GBAFm+7JgWA15zI1uYukBeRk= github.com/eoscanada/eos-go v0.10.3-0.20230413154640-bb75101dc2f6/go.mod h1:L3avCf8OkDrjlUeNy9DdoV67TCmDNj2dSlc5Xp3DNNk= +github.com/eoscanada/eos-go v0.10.3-0.20230609180026-5d4ee54fe15e h1:C5V5vgv5ZFf51RW+QmBLR0eOZHvIlLdapAaIWE1pD+Q= +github.com/eoscanada/eos-go v0.10.3-0.20230609180026-5d4ee54fe15e/go.mod h1:L3avCf8OkDrjlUeNy9DdoV67TCmDNj2dSlc5Xp3DNNk= github.com/eosswedenorg-go/antelope-ship-client v0.2.6 h1:0wPF9TC867eG/+rEzJd0L+TtSOrP09YQzcbDU050FoA= github.com/eosswedenorg-go/antelope-ship-client v0.2.6/go.mod h1:kZ/4gkAIdAq4/WiZlVaSONpELcDCMJQJMmlikLUGCb8= github.com/eosswedenorg-go/jsontime v0.0.0-20230509125027-08422d6236c7 h1:rLPu++RHaxg4WmUOXeWYioZuafWs0PVcYuvzOWbOJjk= @@ -41,6 +48,8 @@ github.com/go-redis/cache/v9 v9.0.0 h1:0thdtFo0xJi0/WXbRVu8B066z8OvVymXTJGaXrVWn github.com/go-redis/cache/v9 v9.0.0/go.mod h1:cMwi1N8ASBOufbIvk7cdXe2PbPjK/WMRL95FFHWsSgI= github.com/go-redis/redismock/v9 v9.0.3 h1:mtHQi2l51lCmXIbTRTqb1EiHYe9tL5Yk5oorlSJJqR0= github.com/go-redis/redismock/v9 v9.0.3/go.mod h1:F6tJRfnU8R/NZ0E+Gjvoluk14MqMC5ueSZX6vVQypc0= +github.com/go-redis/redismock/v9 v9.2.0 h1:ZrMYQeKPECZPjOj5u9eyOjg8Nnb0BS9lkVIZ6IpsKLw= +github.com/go-redis/redismock/v9 v9.2.0/go.mod h1:18KHfGDK4Y6c2R0H38EUGWAdc7ZQS9gfYxc94k7rWT0= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-telegram-bot-api/telegram-bot-api v4.6.4+incompatible h1:2cauKuaELYAEARXRkq2LrJ0yDDv1rW7+wrTEdVL3uaU= github.com/go-telegram-bot-api/telegram-bot-api v4.6.4+incompatible/go.mod h1:qf9acutJ8cwBUhm1bqgz6Bei9/C/c93FPDljKWwsOgM= @@ -73,6 +82,8 @@ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHm github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.16.6 h1:91SKEy4K37vkp255cJ8QesJhjyRO0hn9i9G0GoUwLsk= github.com/klauspost/compress v1.16.6/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.17.1 h1:NE3C767s2ak2bweCZo3+rdP4U/HoyVXLv/X9f2gPS5g= +github.com/klauspost/compress v1.17.1/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= @@ -131,6 +142,8 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/redis/go-redis/v9 v9.0.0-rc.4/go.mod h1:Vo3EsyWnicKnSKCA7HhgnvnyA74wOA69Cd2Meli5mmA= github.com/redis/go-redis/v9 v9.0.5 h1:CuQcn5HIEeK7BgElubPP8CGtE0KakrnbBSTLjathl5o= github.com/redis/go-redis/v9 v9.0.5/go.mod h1:WqMKv5vnQbRuZstUwxQI195wHy+t4PuXDOjzMvcuQHk= +github.com/redis/go-redis/v9 v9.2.1 h1:WlYJg71ODF0dVspZZCpYmoF1+U1Jjk9Rwd7pq6QmlCg= +github.com/redis/go-redis/v9 v9.2.1/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= @@ -154,10 +167,13 @@ github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcU github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/technoweenie/multipartstreamer v1.0.1 h1:XRztA5MXiR1TIRHxH2uNxXxaIkKQDeX7m2XsSOlQEnM= github.com/technoweenie/multipartstreamer v1.0.1/go.mod h1:jNVxdtShOxzAsukZwTSw6MDx5eUJoiEBsSvzDU9uzog= +github.com/teris-io/shortid v0.0.0-20171029131806-771a37caa5cf/go.mod h1:M8agBzgqHIhgj7wEn9/0hJUZcrvt9VY+Ln+S1I5Mha0= github.com/test-go/testify v1.1.4 h1:Tf9lntrKUMHiXQ07qBScBTSA0dhYQlu83hswqelv1iE= github.com/test-go/testify v1.1.4/go.mod h1:rH7cfJo/47vWGdi4GPj16x3/t1xGOj2YxzmNQzk2ghU= github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM= github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.17.0 h1:/Jocvlh98kcTfpN2+JzGQWQcqrPQwDrVEMApx/M5ZwM= +github.com/tidwall/gjson v1.17.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= @@ -172,6 +188,8 @@ github.com/vmihailenco/go-tinylfu v0.2.2/go.mod h1:CutYi2Q9puTxfcolkliPq4npPuofg github.com/vmihailenco/msgpack/v5 v5.3.4/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU= github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= +github.com/vmihailenco/msgpack/v5 v5.4.0 h1:hRM0digJwyR6vll33NNAwCFguy5JuBD6jxDmQP3l608= +github.com/vmihailenco/msgpack/v5 v5.4.0/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok= github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= @@ -192,10 +210,15 @@ go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/ go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8= go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= +go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= +go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -204,6 +227,8 @@ golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM= golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= +golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -238,6 +263,8 @@ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ= +golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -267,6 +294,8 @@ golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -275,6 +304,8 @@ golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.9.0 h1:GRRCnKYhdQrD8kfRAdQ6Zcw1P0OcELxGLKJvtjVMZ28= golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo= +golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -283,6 +314,7 @@ golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -324,3 +356,4 @@ gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= From 1356b58b5588edc0833f3e80ada4b34853004c75 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Fri, 20 Oct 2023 18:24:31 +0200 Subject: [PATCH 010/291] Drop support for golang 1.18 and 1.19, only support the 2 latest major releases. --- .github/workflows/release.yml | 2 +- .github/workflows/test.yml | 2 +- README.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e376c07..cf13230 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -19,7 +19,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v3 with: - go-version: 1.18 + go-version: 1.21 - name: compile id: compile diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e334239..679909e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,7 +10,7 @@ jobs: strategy: fail-fast: false matrix: - go-version: ["1.18", "1.19", "1.20"] + go-version: ["1.20", "1.21"] runs-on: ubuntu-latest name: Test (go v${{ matrix.go-version }}) steps: diff --git a/README.md b/README.md index aa3ac84..85700c0 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Consult the [documentation](https://thalos.waxsweden.org/docs) for more informat ## Compiling -You will need golang version `1.18` or later to compile the source. +You will need golang version `1.20` or later to compile the source. Compile using make: From 809f42f5afde4ca5e214fa7223dd0b7a2bd1c615 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 22 Oct 2023 20:26:10 +0200 Subject: [PATCH 011/291] Adding cmd/tools/mock_publisher.go --- cmd/tools/main.go | 1 + cmd/tools/mock_publisher.go | 107 ++++++++++++++++++++++++++++++++++++ 2 files changed, 108 insertions(+) create mode 100644 cmd/tools/mock_publisher.go diff --git a/cmd/tools/main.go b/cmd/tools/main.go index 48ac549..4670591 100644 --- a/cmd/tools/main.go +++ b/cmd/tools/main.go @@ -19,6 +19,7 @@ func main() { validateCmd, benchCmd, RedisACLCmd, + MockPublisherCmd, }, } diff --git a/cmd/tools/mock_publisher.go b/cmd/tools/mock_publisher.go new file mode 100644 index 0000000..4a335c1 --- /dev/null +++ b/cmd/tools/mock_publisher.go @@ -0,0 +1,107 @@ +package main + +import ( + "context" + "time" + + "github.com/urfave/cli/v2" + + "github.com/eosswedenorg/thalos/api" + "github.com/eosswedenorg/thalos/api/message" + _ "github.com/eosswedenorg/thalos/api/message/json" + api_redis "github.com/eosswedenorg/thalos/api/redis" + + "github.com/redis/go-redis/v9" +) + +var MockPublisherCmd = &cli.Command{ + Name: "mock_publisher", + Usage: "Run a publisher that mocks messages to a redis server. tries to send as many messages as possible", + Flags: []cli.Flag{ + redisUrlFlag, + redisUserFlag, + redisPasswordFlag, + redisDbFlag, + redisPrefixFlag, + chainIdFlag, + &cli.StringFlag{ + Name: "codec", + Value: "json", + }, + }, + Action: func(ctx *cli.Context) error { + // Create redis client + rdb := redis.NewClient(&redis.Options{ + Addr: ctx.String("redis-url"), + Username: ctx.String("redis-user"), + Password: ctx.String("redis-pw"), + DB: ctx.Int("redis-db"), + }) + + codec, err := message.GetCodec(ctx.String("codec")) + if err != nil { + return err + } + + ns := api_redis.Namespace{ + Prefix: ctx.String("redis-prefix"), + ChainID: ctx.String("chain_id"), + } + publisher := api_redis.NewPublisher(context.Background(), rdb, ns) + + msg := message.ActionTrace{ + TxID: "401e8a7e5deb18a2a69fc6559f49509a155f4355c85efb69c1c1fab5b60ee532", + BlockNum: 18237917, + Timestamp: time.Date(2014, 3, 22, 11, 36, 43, 0, time.UTC), + Receipt: &message.ActionReceipt{ + Receiver: "acc1", + ActDigest: "4c5c08be612e937564fc526ebb5fadf34ae8c2a571fe9d7cdb3ffcdfc53b0e8d", + GlobalSequence: 12314, + RecvSequence: 237187239, + AuthSequence: []message.AccountAuthSequence{ + { + Account: "acc1", + Sequence: 2732863, + }, + { + Account: "acc2", + Sequence: 263762, + }, + }, + CodeSequence: 2327832, + ABISequence: 12376189, + }, + Name: "fake", + Contract: "fake", + Receiver: "acc1", + Data: map[string]interface{}{ + "one": 238771832, + "two": "str", + }, + Authorization: []message.PermissionLevel{ + { + Actor: "acc1", + Permission: "active", + }, + { + Actor: "acc2", + Permission: "owner", + }, + }, + Except: "err", + Error: 2, + Return: []byte{0xbe, 0xef}, + } + + payload, err := codec.Encoder(msg) + if err != nil { + return err + } + channel := api.ActionChannel{}.Channel() + + for { + _ = publisher.Write(channel, payload) + publisher.Flush() + } + }, +} From 1ced6146b08d67c18c8bd50880927d4f2643f5a2 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 22 Oct 2023 20:32:49 +0200 Subject: [PATCH 012/291] Makefile: fix inclusion of source files for thalos-tools --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index d4acb70..ba8b768 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,7 @@ build/$(PROGRAM) : tools : build/thalos-tools build/thalos-tools : - $(GO) build $(GOBUILDFLAGS) -o $@ cmd/tools/main.go cmd/tools/flags.go cmd/tools/bench.go cmd/tools/validate.go + $(GO) build $(GOBUILDFLAGS) -o $@ $(shell find cmd/tools -type f -name *.go) install: build tools install -D build/$(PROGRAM) $(DESTDIR)$(BINDIR)/$(PROGRAM) From c592c35e7c51fb128749acb6413561c9cd6da737 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 22 Oct 2023 20:34:38 +0200 Subject: [PATCH 013/291] go.mod: cleanup dependencies. --- go.mod | 1 - go.sum | 36 ------------------------------------ 2 files changed, 37 deletions(-) diff --git a/go.mod b/go.mod index 431088e..013b0fc 100644 --- a/go.mod +++ b/go.mod @@ -51,7 +51,6 @@ require ( github.com/vmihailenco/msgpack/v5 v5.4.0 // indirect github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect - go.uber.org/atomic v1.11.0 // indirect go.uber.org/goleak v1.2.1 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.26.0 // indirect diff --git a/go.sum b/go.sum index f498e2a..2bd4ff0 100644 --- a/go.sum +++ b/go.sum @@ -1,13 +1,9 @@ -contrib.go.opencensus.io/exporter/stackdriver v0.12.6/go.mod h1:8x999/OcIPy5ivx/wDiV7Gx4D+VUPODf0mWRGRc5kSk= github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a h1:HbKu58rmZpUGpz5+4FfNmIU+FmZg2P3Xaj2v2bfNWmk= github.com/alicebob/miniredis/v2 v2.30.2 h1:lc1UAUT9ZA7h4srlfBmBt2aorm5Yftk9nBjxz7EyY9I= -github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/blendle/zapdriver v1.3.1 h1:C3dydBOWYRiOk+B8X9IVZ5IOe+7cl+tGOexN4QqHfpE= github.com/blendle/zapdriver v1.3.1/go.mod h1:mdXfREi6u5MArG4j9fewC+FGnXaBR+T4Ox4J2u4eHCc= -github.com/bsm/ginkgo/v2 v2.7.0 h1:ItPMPH90RbmZJt5GtkcNvIRuGEdwlBItdNVoyzaNQao= github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= -github.com/bsm/gomega v1.26.0 h1:LhQm+AFcgV2M0WyKroMASzAzCAJVpAxQXv4SaI9a69Y= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= @@ -17,8 +13,6 @@ github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= -github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cpuguy83/go-md2man/v2 v2.0.3 h1:qMCsGGgs+MAzDFyp9LpAe1Lqy/fY/qCovCm0qnXZOBM= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= @@ -29,8 +23,6 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/r github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/eoscanada/eos-go v0.10.3-0.20230413154640-bb75101dc2f6 h1:93LUOgAmRkmz8DF2V62GBAFm+7JgWA15zI1uYukBeRk= -github.com/eoscanada/eos-go v0.10.3-0.20230413154640-bb75101dc2f6/go.mod h1:L3avCf8OkDrjlUeNy9DdoV67TCmDNj2dSlc5Xp3DNNk= github.com/eoscanada/eos-go v0.10.3-0.20230609180026-5d4ee54fe15e h1:C5V5vgv5ZFf51RW+QmBLR0eOZHvIlLdapAaIWE1pD+Q= github.com/eoscanada/eos-go v0.10.3-0.20230609180026-5d4ee54fe15e/go.mod h1:L3avCf8OkDrjlUeNy9DdoV67TCmDNj2dSlc5Xp3DNNk= github.com/eosswedenorg-go/antelope-ship-client v0.2.6 h1:0wPF9TC867eG/+rEzJd0L+TtSOrP09YQzcbDU050FoA= @@ -46,8 +38,6 @@ github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbS github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-redis/cache/v9 v9.0.0 h1:0thdtFo0xJi0/WXbRVu8B066z8OvVymXTJGaXrVWnN0= github.com/go-redis/cache/v9 v9.0.0/go.mod h1:cMwi1N8ASBOufbIvk7cdXe2PbPjK/WMRL95FFHWsSgI= -github.com/go-redis/redismock/v9 v9.0.3 h1:mtHQi2l51lCmXIbTRTqb1EiHYe9tL5Yk5oorlSJJqR0= -github.com/go-redis/redismock/v9 v9.0.3/go.mod h1:F6tJRfnU8R/NZ0E+Gjvoluk14MqMC5ueSZX6vVQypc0= github.com/go-redis/redismock/v9 v9.2.0 h1:ZrMYQeKPECZPjOj5u9eyOjg8Nnb0BS9lkVIZ6IpsKLw= github.com/go-redis/redismock/v9 v9.2.0/go.mod h1:18KHfGDK4Y6c2R0H38EUGWAdc7ZQS9gfYxc94k7rWT0= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= @@ -80,8 +70,6 @@ github.com/jordan-wright/email v4.0.1-0.20210109023952-943e75fe5223+incompatible github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.16.6 h1:91SKEy4K37vkp255cJ8QesJhjyRO0hn9i9G0GoUwLsk= -github.com/klauspost/compress v1.16.6/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/compress v1.17.1 h1:NE3C767s2ak2bweCZo3+rdP4U/HoyVXLv/X9f2gPS5g= github.com/klauspost/compress v1.17.1/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -140,8 +128,6 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/redis/go-redis/v9 v9.0.0-rc.4/go.mod h1:Vo3EsyWnicKnSKCA7HhgnvnyA74wOA69Cd2Meli5mmA= -github.com/redis/go-redis/v9 v9.0.5 h1:CuQcn5HIEeK7BgElubPP8CGtE0KakrnbBSTLjathl5o= -github.com/redis/go-redis/v9 v9.0.5/go.mod h1:WqMKv5vnQbRuZstUwxQI195wHy+t4PuXDOjzMvcuQHk= github.com/redis/go-redis/v9 v9.2.1 h1:WlYJg71ODF0dVspZZCpYmoF1+U1Jjk9Rwd7pq6QmlCg= github.com/redis/go-redis/v9 v9.2.1/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= @@ -167,11 +153,8 @@ github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcU github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/technoweenie/multipartstreamer v1.0.1 h1:XRztA5MXiR1TIRHxH2uNxXxaIkKQDeX7m2XsSOlQEnM= github.com/technoweenie/multipartstreamer v1.0.1/go.mod h1:jNVxdtShOxzAsukZwTSw6MDx5eUJoiEBsSvzDU9uzog= -github.com/teris-io/shortid v0.0.0-20171029131806-771a37caa5cf/go.mod h1:M8agBzgqHIhgj7wEn9/0hJUZcrvt9VY+Ln+S1I5Mha0= github.com/test-go/testify v1.1.4 h1:Tf9lntrKUMHiXQ07qBScBTSA0dhYQlu83hswqelv1iE= github.com/test-go/testify v1.1.4/go.mod h1:rH7cfJo/47vWGdi4GPj16x3/t1xGOj2YxzmNQzk2ghU= -github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM= -github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/gjson v1.17.0 h1:/Jocvlh98kcTfpN2+JzGQWQcqrPQwDrVEMApx/M5ZwM= github.com/tidwall/gjson v1.17.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= @@ -186,8 +169,6 @@ github.com/urfave/cli/v2 v2.25.7/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6S github.com/vmihailenco/go-tinylfu v0.2.2 h1:H1eiG6HM36iniK6+21n9LLpzx1G9R3DJa2UjUjbynsI= github.com/vmihailenco/go-tinylfu v0.2.2/go.mod h1:CutYi2Q9puTxfcolkliPq4npPuofg9N9t8JVrjzwa3Q= github.com/vmihailenco/msgpack/v5 v5.3.4/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= -github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU= -github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= github.com/vmihailenco/msgpack/v5 v5.4.0 h1:hRM0digJwyR6vll33NNAwCFguy5JuBD6jxDmQP3l608= github.com/vmihailenco/msgpack/v5 v5.4.0/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok= github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= @@ -201,22 +182,15 @@ github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5t github.com/yuin/gopher-lua v1.1.0 h1:BojcDhfyDWgU2f2TOzYK/g5p2gxMrku8oupLDqlnSqE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= -go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8= -go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= -go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= -go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -225,8 +199,6 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= -golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM= -golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -261,8 +233,6 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= -golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ= golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -292,8 +262,6 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= -golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -302,8 +270,6 @@ golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= -golang.org/x/term v0.9.0 h1:GRRCnKYhdQrD8kfRAdQ6Zcw1P0OcELxGLKJvtjVMZ28= -golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo= golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -313,7 +279,6 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58= golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -356,4 +321,3 @@ gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= From 1bc52d7b3d260a31bc0b8e103da2de4d37180610 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 22 Oct 2023 20:35:39 +0200 Subject: [PATCH 014/291] go.mod: Require at least go 1.20 --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 013b0fc..538d2e4 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/eosswedenorg/thalos -go 1.18 +go 1.20 require ( github.com/cenkalti/backoff/v4 v4.2.1 From d6d9fa5ca03891031982a05b7b8a9ac92a26f515 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 22 Oct 2023 20:37:23 +0200 Subject: [PATCH 015/291] api/go.mod: require at least go 1.20 --- api/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/go.mod b/api/go.mod index 8090fbf..2831b7d 100644 --- a/api/go.mod +++ b/api/go.mod @@ -1,6 +1,6 @@ module github.com/eosswedenorg/thalos/api -go 1.18 +go 1.20 require ( github.com/alicebob/miniredis/v2 v2.30.2 From 23d05e55125b29c1cc3214e50eeaa32994b59eec Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Tue, 31 Oct 2023 16:58:40 +0100 Subject: [PATCH 016/291] Adding app/cache/store.go --- app/cache/store.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 app/cache/store.go diff --git a/app/cache/store.go b/app/cache/store.go new file mode 100644 index 0000000..309333f --- /dev/null +++ b/app/cache/store.go @@ -0,0 +1,15 @@ +package cache + +import "time" + +type Store interface { + // Set an item in the store. + Set(key string, value any, TTL time.Duration) error + + // Get an item from the store. + // returns an error if key is not found or there is other problems. + Get(key string, value any) error + + // Check if a key exist in the store. + Has(key string) bool +} From ed1009062dd3fb390517fab383a14a6145c74514 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Tue, 31 Oct 2023 16:59:08 +0100 Subject: [PATCH 017/291] Adding app/cache/memory_store.go --- app/cache/memory_store.go | 55 +++++++++++++++++++++ app/cache/memory_store_test.go | 87 ++++++++++++++++++++++++++++++++++ 2 files changed, 142 insertions(+) create mode 100644 app/cache/memory_store.go create mode 100644 app/cache/memory_store_test.go diff --git a/app/cache/memory_store.go b/app/cache/memory_store.go new file mode 100644 index 0000000..7c3bd9f --- /dev/null +++ b/app/cache/memory_store.go @@ -0,0 +1,55 @@ +package cache + +import ( + "fmt" + "reflect" + "time" +) + +var now = time.Now + +type memoryStoreItem struct { + value any + expired time.Time +} + +type MemoryStore struct { + data map[string]memoryStoreItem +} + +func NewMemoryStore() *MemoryStore { + return &MemoryStore{make(map[string]memoryStoreItem)} +} + +func (s *MemoryStore) Get(key string, value any) error { + if item, ok := s.data[key]; ok { + + if item.expired.Before(now()) { + delete(s.data, key) + return fmt.Errorf("key: %s does not exist", key) + } + + v := reflect.ValueOf(value) + if v.Kind() != reflect.Pointer { + return fmt.Errorf("value must be of pointer type, '%s' passed", v.Kind().String()) + } + + v.Elem().Set(reflect.ValueOf(item.value)) + + return nil + } + return fmt.Errorf("key: %s does not exist", key) +} + +func (s *MemoryStore) Has(key string) bool { + _, hit := s.data[key] + return hit +} + +func (s *MemoryStore) Set(key string, value any, ttl time.Duration) error { + s.data[key] = memoryStoreItem{ + value: value, + expired: now().Add(ttl), + } + return nil +} diff --git a/app/cache/memory_store_test.go b/app/cache/memory_store_test.go new file mode 100644 index 0000000..d6bc6bd --- /dev/null +++ b/app/cache/memory_store_test.go @@ -0,0 +1,87 @@ +package cache + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +type memoryTestItem struct { + String string + Num uint32 + Float float32 +} + +func TestMemoryStore_Set(t *testing.T) { + now = func() time.Time { return time.Unix(1581315270, 0) } + + item := memoryTestItem{ + String: "MyString", + Num: 23, + Float: 3.14, + } + + expected := map[string]memoryStoreItem{ + "key1": { + value: item, + expired: now().Add(time.Hour), + }, + } + + store := NewMemoryStore() + err := store.Set("key1", item, time.Hour) + assert.NoError(t, err) + + assert.Equal(t, expected, store.data) +} + +func TestMemoryStore_GetMiss(t *testing.T) { + store := NewMemoryStore() + + var v any + err := store.Get("Key2", &v) + assert.Error(t, err) +} + +func TestMemoryStore_GetHit(t *testing.T) { + expected := memoryTestItem{ + String: "MyString", + Num: 23, + Float: 3.14, + } + + store := NewMemoryStore() + err := store.Set("key1", expected, time.Hour) + assert.NoError(t, err) + + var actual memoryTestItem + err = store.Get("key1", &actual) + assert.NoError(t, err) + assert.Equal(t, expected, actual) +} + +func TestMemoryStore_GetNonPointer(t *testing.T) { + expected := memoryTestItem{ + String: "MyString", + Num: 23, + Float: 3.14, + } + + store := NewMemoryStore() + err := store.Set("key1", expected, time.Hour) + assert.NoError(t, err) + + var actual string + err = store.Get("key1", actual) + assert.EqualError(t, err, "value must be of pointer type, 'string' passed") +} + +func TestMemoryStore_Has(t *testing.T) { + store := NewMemoryStore() + err := store.Set("key1", "value", time.Hour) + assert.NoError(t, err) + + assert.True(t, store.Has("key1")) + assert.False(t, store.Has("key2")) +} From c6cb26d543a7e42290efa51463daf1571e6213bb Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Tue, 31 Oct 2023 16:59:26 +0100 Subject: [PATCH 018/291] Adding app/cache/redis_store.go --- app/cache/redis_store.go | 37 ++++++++++++ app/cache/redis_store_test.go | 103 ++++++++++++++++++++++++++++++++++ 2 files changed, 140 insertions(+) create mode 100644 app/cache/redis_store.go create mode 100644 app/cache/redis_store_test.go diff --git a/app/cache/redis_store.go b/app/cache/redis_store.go new file mode 100644 index 0000000..5016a15 --- /dev/null +++ b/app/cache/redis_store.go @@ -0,0 +1,37 @@ +package cache + +import ( + "context" + "time" + + "github.com/go-redis/cache/v9" +) + +type RedisStore struct { + c *cache.Cache + ctx context.Context +} + +func NewRedisStore(options *cache.Options) *RedisStore { + return &RedisStore{ + c: cache.New(options), + ctx: context.Background(), + } +} + +func (s *RedisStore) Get(key string, value interface{}) error { + return s.c.Get(s.ctx, key, value) +} + +func (s *RedisStore) Has(key string) bool { + return s.c.Exists(s.ctx, key) +} + +func (s *RedisStore) Set(key string, value any, ttl time.Duration) error { + return s.c.Set(&cache.Item{ + Ctx: s.ctx, + Key: key, + Value: value, + TTL: ttl, + }) +} diff --git a/app/cache/redis_store_test.go b/app/cache/redis_store_test.go new file mode 100644 index 0000000..34e3f34 --- /dev/null +++ b/app/cache/redis_store_test.go @@ -0,0 +1,103 @@ +package cache + +import ( + "testing" + "time" + + "github.com/go-redis/redismock/v9" + + redis_cache "github.com/go-redis/cache/v9" + "github.com/stretchr/testify/assert" +) + +type testItem struct { + Num uint32 + Name string +} + +func TestRedisStore_Set(t *testing.T) { + client, mock := redismock.NewClientMock() + + store := NewRedisStore(&redis_cache.Options{ + Redis: client, + }) + + expected := testItem{ + Num: 24, + Name: "Some Name", + } + + bytes, err := store.c.Marshal(expected) + assert.NoError(t, err) + + mock.ExpectSet("mykey", bytes, time.Minute).SetVal("OK") + + err = store.Set("mykey", expected, time.Minute) + assert.NoError(t, err) + + assert.NoError(t, mock.ExpectationsWereMet()) +} + +func TestRedisStore_GetMiss(t *testing.T) { + client, mock := redismock.NewClientMock() + + store := NewRedisStore(&redis_cache.Options{ + Redis: client, + }) + + mock.ExpectGet("mykey").SetErr(redis_cache.ErrCacheMiss) + + expected := testItem{} + err := store.Get("mykey", &expected) + assert.ErrorIs(t, err, redis_cache.ErrCacheMiss) + assert.NoError(t, mock.ExpectationsWereMet()) +} + +func TestRedisStore_GetHit(t *testing.T) { + client, mock := redismock.NewClientMock() + + store := NewRedisStore(&redis_cache.Options{ + Redis: client, + }) + + expected := testItem{ + Num: 42, + Name: "MyName", + } + + bytes, err := store.c.Marshal(expected) + assert.NoError(t, err) + + mock.ExpectSet("mykey2", bytes, time.Second*20).SetVal("OK") + mock.ExpectGet("mykey2").SetVal(string(bytes)) + + err = store.Set("mykey2", expected, time.Second*20) + assert.NoError(t, err) + + actual := testItem{} + err = store.Get("mykey2", &actual) + assert.NoError(t, err) + assert.Equal(t, expected, actual) + assert.NoError(t, mock.ExpectationsWereMet()) +} + +func TestRedisStore_Has(t *testing.T) { + client, mock := redismock.NewClientMock() + + store := NewRedisStore(&redis_cache.Options{ + Redis: client, + }) + + bytes, err := store.c.Marshal("value") + assert.NoError(t, err) + + mock.ExpectSet("key1", bytes, time.Minute*15).SetVal("OK") + mock.ExpectGet("key1").SetVal(string(bytes)) + mock.ExpectGet("key2").RedisNil() + + err = store.Set("key1", "value", time.Minute*15) + assert.NoError(t, err) + assert.True(t, store.Has("key1")) + assert.False(t, store.Has("key2")) + assert.NoError(t, mock.ExpectationsWereMet()) +} From 2eb62db117aeec4a09774e2602925ba9dae4d1bd Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Wed, 1 Nov 2023 21:23:48 +0100 Subject: [PATCH 019/291] Adding app/cache/cache.go --- app/cache/cache.go | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 app/cache/cache.go diff --git a/app/cache/cache.go b/app/cache/cache.go new file mode 100644 index 0000000..290019b --- /dev/null +++ b/app/cache/cache.go @@ -0,0 +1,30 @@ +package cache + +import ( + "time" +) + +type Cache struct { + store Store + prefix string +} + +// Create a new cache +func NewCache(prefix string, store Store) *Cache { + return &Cache{ + store: store, + prefix: prefix, + } +} + +func (cache *Cache) Get(key string, value any) error { + return cache.store.Get(cache.key(key), value) +} + +func (cache *Cache) Set(key string, value any, ttl time.Duration) error { + return cache.store.Set(cache.key(key), value, ttl) +} + +func (cache *Cache) key(key string) string { + return cache.prefix + "::" + key +} From 424ae2fc40eb915a33259448d2a6387416f94974 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Wed, 1 Nov 2023 21:24:59 +0100 Subject: [PATCH 020/291] app/abi: rework manager to use new cache struct. --- app/abi/cache.go | 46 ----------- app/abi/cache_test.go | 158 ------------------------------------ app/abi/manager.go | 28 +++---- app/abi/manager_test.go | 173 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 182 insertions(+), 223 deletions(-) delete mode 100644 app/abi/cache.go delete mode 100644 app/abi/cache_test.go create mode 100644 app/abi/manager_test.go diff --git a/app/abi/cache.go b/app/abi/cache.go deleted file mode 100644 index 3b54084..0000000 --- a/app/abi/cache.go +++ /dev/null @@ -1,46 +0,0 @@ -package abi - -import ( - "context" - "time" - - eos "github.com/eoscanada/eos-go" - redis_cache "github.com/go-redis/cache/v9" -) - -// Cache represents a abi cache in redis. -type Cache struct { - c *redis_cache.Cache - ctx context.Context - prefix string -} - -// Create a new cache -func NewCache(prefix string, options *redis_cache.Options) *Cache { - return &Cache{ - c: redis_cache.New(options), - ctx: context.Background(), - prefix: prefix, - } -} - -// Get an ABI from the cache using the contract account name as the key. -func (cache *Cache) Get(account string) (*eos.ABI, error) { - var v eos.ABI - err := cache.c.Get(cache.ctx, cache.key(account), &v) - return &v, err -} - -// Set an ABI in the cache. -func (cache *Cache) Set(account string, abi *eos.ABI, ttl time.Duration) error { - return cache.c.Set(&redis_cache.Item{ - Ctx: cache.ctx, - Key: cache.key(account), - Value: *abi, - TTL: ttl, - }) -} - -func (cache *Cache) key(account string) string { - return cache.prefix + "::" + account -} diff --git a/app/abi/cache_test.go b/app/abi/cache_test.go deleted file mode 100644 index da74e74..0000000 --- a/app/abi/cache_test.go +++ /dev/null @@ -1,158 +0,0 @@ -package abi - -import ( - "strings" - "testing" - "time" - - eos "github.com/eoscanada/eos-go" - redis_cache "github.com/go-redis/cache/v9" - "github.com/go-redis/redismock/v9" - - "github.com/stretchr/testify/assert" -) - -var abiString = ` -{ - "version": "eosio::abi/1.0", - "types": [{ - "new_type_name": "new_type_name_1", - "type": "name" - }], - "structs": [ - { - "name": "struct_name_1", - "base": "struct_name_2", - "fields": [ - {"name":"struct_1_field_1", "type":"new_type_name_1"}, - {"name":"struct_1_field_2", "type":"struct_name_3"}, - {"name":"struct_1_field_3", "type":"string?"}, - {"name":"struct_1_field_4", "type":"string?"}, - {"name":"struct_1_field_5", "type":"struct_name_4[]"} - ] - },{ - "name": "struct_name_2", - "base": "", - "fields": [ - {"name":"struct_2_field_1", "type":"string"} - ] - },{ - "name": "struct_name_3", - "base": "", - "fields": [ - {"name":"struct_3_field_1", "type":"string"} - ] - },{ - "name": "struct_name_4", - "base": "", - "fields": [ - {"name":"struct_4_field_1", "type":"string"} - ] - } - ], - "actions": [{ - "name": "action_name_1", - "type": "struct_name_1", - "ricardian_contract": "" - }], - "tables": [{ - "name": "table_name_1", - "index_type": "i64", - "key_names": [ - "key_name_1", - "key_name_2" - ], - "key_types": [ - "string", - "int" - ], - "type": "struct_name_1" - } - ] -} -` - -func TestGetSet(t *testing.T) { - client, mock := redismock.NewClientMock() - - c := NewCache("thalos::cache::test", &redis_cache.Options{ - Redis: client, - // Cache 10k keys for 1 minute. - LocalCache: redis_cache.NewTinyLFU(10000, time.Minute), - }) - - abi, err := eos.NewABI(strings.NewReader(abiString)) - assert.NoError(t, err) - - bytes, _ := c.c.Marshal(*abi) - - mock.ExpectSet("thalos::cache::test::testaccount", bytes, time.Minute).SetVal("OK") - - err = c.Set("testaccount", abi, time.Minute) - assert.NoError(t, err) - - c_abi, err := c.Get("testaccount") - assert.NoError(t, err) - - assert.Equal(t, c_abi.Version, "eosio::abi/1.0") - - // Types - assert.Equal(t, c_abi.Types[0].NewTypeName, "new_type_name_1") - assert.Equal(t, c_abi.Types[0].Type, "name") - - // Structs - assert.Equal(t, c_abi.Structs[0].Name, "struct_name_1") - assert.Equal(t, c_abi.Structs[0].Base, "struct_name_2") - assert.Equal(t, c_abi.Structs[0].Fields[0].Name, "struct_1_field_1") - assert.Equal(t, c_abi.Structs[0].Fields[0].Type, "new_type_name_1") - assert.Equal(t, c_abi.Structs[0].Fields[1].Name, "struct_1_field_2") - assert.Equal(t, c_abi.Structs[0].Fields[1].Type, "struct_name_3") - assert.Equal(t, c_abi.Structs[0].Fields[2].Name, "struct_1_field_3") - assert.Equal(t, c_abi.Structs[0].Fields[2].Type, "string?") - assert.Equal(t, c_abi.Structs[0].Fields[3].Name, "struct_1_field_4") - assert.Equal(t, c_abi.Structs[0].Fields[3].Type, "string?") - assert.Equal(t, c_abi.Structs[0].Fields[4].Name, "struct_1_field_5") - assert.Equal(t, c_abi.Structs[0].Fields[4].Type, "struct_name_4[]") - - assert.Equal(t, c_abi.Structs[1].Name, "struct_name_2") - assert.Equal(t, c_abi.Structs[1].Base, "") - assert.Equal(t, c_abi.Structs[1].Fields[0].Name, "struct_2_field_1") - assert.Equal(t, c_abi.Structs[1].Fields[0].Type, "string") - - assert.Equal(t, c_abi.Structs[2].Name, "struct_name_3") - assert.Equal(t, c_abi.Structs[2].Base, "") - assert.Equal(t, c_abi.Structs[2].Fields[0].Name, "struct_3_field_1") - assert.Equal(t, c_abi.Structs[2].Fields[0].Type, "string") - - assert.Equal(t, c_abi.Structs[3].Name, "struct_name_4") - assert.Equal(t, c_abi.Structs[3].Base, "") - assert.Equal(t, c_abi.Structs[3].Fields[0].Name, "struct_4_field_1") - assert.Equal(t, c_abi.Structs[3].Fields[0].Type, "string") - - // Actions - assert.Equal(t, c_abi.Actions[0].Name, eos.ActN("action_name_1")) - assert.Equal(t, c_abi.Actions[0].Type, "struct_name_1") - assert.Equal(t, c_abi.Actions[0].RicardianContract, "") - - // Tables - assert.Equal(t, c_abi.Tables[0].Name, eos.TableName("table_name_1")) - assert.Equal(t, c_abi.Tables[0].Type, "struct_name_1") - assert.Equal(t, c_abi.Tables[0].IndexType, "i64") - assert.Equal(t, c_abi.Tables[0].KeyNames[0], "key_name_1") - assert.Equal(t, c_abi.Tables[0].KeyNames[1], "key_name_2") - assert.Equal(t, c_abi.Tables[0].KeyTypes[0], "string") - assert.Equal(t, c_abi.Tables[0].KeyTypes[1], "int") -} - -func TestCacheMiss(t *testing.T) { - client, _ := redismock.NewClientMock() - - c := NewCache("thalos::cache::test", &redis_cache.Options{ - Redis: client, - // Cache 10k keys for 1 minute. - LocalCache: redis_cache.NewTinyLFU(10000, time.Minute), - }) - - _, err := c.Get("nonexist") - assert.Error(t, err) -} diff --git a/app/abi/manager.go b/app/abi/manager.go index 79df1bd..d8f3682 100644 --- a/app/abi/manager.go +++ b/app/abi/manager.go @@ -6,26 +6,18 @@ import ( "time" eos "github.com/eoscanada/eos-go" - redis_cache "github.com/go-redis/cache/v9" - "github.com/redis/go-redis/v9" + "github.com/eosswedenorg/thalos/app/cache" ) // AbiManager handles an ABI cache that fetches the ABI from an API on cache miss. type AbiManager struct { - cache *Cache + cache *cache.Cache api *eos.API ctx context.Context } // Create a new ABI Manager -func NewAbiManager(rdb *redis.Client, api *eos.API, id string) *AbiManager { - // Init abi cache - cache := NewCache("thalos::cache::"+id+"::abi", &redis_cache.Options{ - Redis: rdb, - // Cache 10k keys for 10 minutes. - LocalCache: redis_cache.NewTinyLFU(10000, 10*time.Minute), - }) - +func NewAbiManager(cache *cache.Cache, api *eos.API) *AbiManager { return &AbiManager{ cache: cache, api: api, @@ -35,26 +27,24 @@ func NewAbiManager(rdb *redis.Client, api *eos.API, id string) *AbiManager { // Set or update an ABI in the cache. func (mgr *AbiManager) SetAbi(account eos.AccountName, abi *eos.ABI) error { - return mgr.cache.Set(string(account), abi, time.Hour) + return mgr.cache.Set(string(account), *abi, time.Hour) } // Get an ABI from the cache, on cache miss it is fetched from the // API, gets cached and then returned to the user func (mgr *AbiManager) GetAbi(account eos.AccountName) (*eos.ABI, error) { - key := string(account) - - abi, err := mgr.cache.Get(key) - if err != nil { + var abi eos.ABI + if err := mgr.cache.Get(string(account), &abi); err != nil { resp, err := mgr.api.GetABI(mgr.ctx, account) if err != nil { return nil, fmt.Errorf("api: %s", err) } - abi = &resp.ABI + abi = resp.ABI - err = mgr.SetAbi(account, abi) + err = mgr.SetAbi(account, &abi) if err != nil { return nil, fmt.Errorf("cache: %s", err) } } - return abi, nil + return &abi, nil } diff --git a/app/abi/manager_test.go b/app/abi/manager_test.go new file mode 100644 index 0000000..661c8ec --- /dev/null +++ b/app/abi/manager_test.go @@ -0,0 +1,173 @@ +package abi + +import ( + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + + eos "github.com/eoscanada/eos-go" + + "github.com/eosswedenorg/thalos/app/cache" + "github.com/stretchr/testify/assert" +) + +var abiString = ` +{ + "version": "eosio::abi/1.0", + "types": [{ + "new_type_name": "new_type_name_1", + "type": "name" + }], + "structs": [ + { + "name": "struct_name_1", + "base": "struct_name_2", + "fields": [ + {"name":"struct_1_field_1", "type":"new_type_name_1"}, + {"name":"struct_1_field_2", "type":"struct_name_3"}, + {"name":"struct_1_field_3", "type":"string?"}, + {"name":"struct_1_field_4", "type":"string?"}, + {"name":"struct_1_field_5", "type":"struct_name_4[]"} + ] + },{ + "name": "struct_name_2", + "base": "", + "fields": [ + {"name":"struct_2_field_1", "type":"string"} + ] + },{ + "name": "struct_name_3", + "base": "", + "fields": [ + {"name":"struct_3_field_1", "type":"string"} + ] + },{ + "name": "struct_name_4", + "base": "", + "fields": [ + {"name":"struct_4_field_1", "type":"string"} + ] + } + ], + "actions": [{ + "name": "action_name_1", + "type": "struct_name_1", + "ricardian_contract": "" + }], + "tables": [{ + "name": "table_name_1", + "index_type": "i64", + "key_names": [ + "key_name_1", + "key_name_2" + ], + "key_types": [ + "string", + "int" + ], + "type": "struct_name_1" + } + ] +} +` + +func assert_abi(t *testing.T, abi *eos.ABI) { + assert.Equal(t, abi.Version, "eosio::abi/1.0") + + // Types + assert.Equal(t, abi.Types[0].NewTypeName, "new_type_name_1") + assert.Equal(t, abi.Types[0].Type, "name") + + // Structs + assert.Equal(t, abi.Structs[0].Name, "struct_name_1") + assert.Equal(t, abi.Structs[0].Base, "struct_name_2") + assert.Equal(t, abi.Structs[0].Fields[0].Name, "struct_1_field_1") + assert.Equal(t, abi.Structs[0].Fields[0].Type, "new_type_name_1") + assert.Equal(t, abi.Structs[0].Fields[1].Name, "struct_1_field_2") + assert.Equal(t, abi.Structs[0].Fields[1].Type, "struct_name_3") + assert.Equal(t, abi.Structs[0].Fields[2].Name, "struct_1_field_3") + assert.Equal(t, abi.Structs[0].Fields[2].Type, "string?") + assert.Equal(t, abi.Structs[0].Fields[3].Name, "struct_1_field_4") + assert.Equal(t, abi.Structs[0].Fields[3].Type, "string?") + assert.Equal(t, abi.Structs[0].Fields[4].Name, "struct_1_field_5") + assert.Equal(t, abi.Structs[0].Fields[4].Type, "struct_name_4[]") + + assert.Equal(t, abi.Structs[1].Name, "struct_name_2") + assert.Equal(t, abi.Structs[1].Base, "") + assert.Equal(t, abi.Structs[1].Fields[0].Name, "struct_2_field_1") + assert.Equal(t, abi.Structs[1].Fields[0].Type, "string") + + assert.Equal(t, abi.Structs[2].Name, "struct_name_3") + assert.Equal(t, abi.Structs[2].Base, "") + assert.Equal(t, abi.Structs[2].Fields[0].Name, "struct_3_field_1") + assert.Equal(t, abi.Structs[2].Fields[0].Type, "string") + + assert.Equal(t, abi.Structs[3].Name, "struct_name_4") + assert.Equal(t, abi.Structs[3].Base, "") + assert.Equal(t, abi.Structs[3].Fields[0].Name, "struct_4_field_1") + assert.Equal(t, abi.Structs[3].Fields[0].Type, "string") + + // Actions + assert.Equal(t, abi.Actions[0].Name, eos.ActN("action_name_1")) + assert.Equal(t, abi.Actions[0].Type, "struct_name_1") + assert.Equal(t, abi.Actions[0].RicardianContract, "") + + // Tables + assert.Equal(t, abi.Tables[0].Name, eos.TableName("table_name_1")) + assert.Equal(t, abi.Tables[0].Type, "struct_name_1") + assert.Equal(t, abi.Tables[0].IndexType, "i64") + assert.Equal(t, abi.Tables[0].KeyNames[0], "key_name_1") + assert.Equal(t, abi.Tables[0].KeyNames[1], "key_name_2") + assert.Equal(t, abi.Tables[0].KeyTypes[0], "string") + assert.Equal(t, abi.Tables[0].KeyTypes[1], "int") +} + +func mockAPI(handler http.HandlerFunc) (*eos.API, *httptest.Server) { + server := httptest.NewServer(handler) + + return &eos.API{ + HttpClient: server.Client(), + BaseURL: strings.TrimRight(server.URL, "/"), + Compress: eos.CompressionZlib, + Header: make(http.Header), + }, server +} + +func TestManager_GetAbiFromCache(t *testing.T) { + cache := cache.NewCache("thalos::cache::abi::test", cache.NewMemoryStore()) + + api, _ := mockAPI(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + })) + + mgr := NewAbiManager(cache, api) + + abi, err := eos.NewABI(strings.NewReader(abiString)) + assert.NoError(t, err) + + err = mgr.SetAbi("testaccount", abi) + assert.NoError(t, err) + + c_abi, err := mgr.GetAbi("testaccount") + assert.NoError(t, err) + assert_abi(t, c_abi) +} + +func TestManager_GetAbiFromAPI(t *testing.T) { + cache := cache.NewCache("thalos::cache::abi::test", cache.NewMemoryStore()) + + api, _ := mockAPI(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body := fmt.Sprintf(`{"account_name": "testaccount", "abi": %s}`, abiString) + + _, err := w.Write([]byte(body)) + assert.NoError(t, err) + })) + + mgr := NewAbiManager(cache, api) + + c_abi, err := mgr.GetAbi("testaccount") + assert.NoError(t, err) + + assert_abi(t, c_abi) +} From 2acae14ff8f75cf252f0ca0a1e74b4a6a61011a2 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Wed, 1 Nov 2023 21:25:54 +0100 Subject: [PATCH 021/291] cmd/thalos/main.go: use new cache struct and abi manager. --- cmd/thalos/main.go | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/cmd/thalos/main.go b/cmd/thalos/main.go index 020681b..d590c02 100644 --- a/cmd/thalos/main.go +++ b/cmd/thalos/main.go @@ -21,8 +21,10 @@ import ( api_redis "github.com/eosswedenorg/thalos/api/redis" "github.com/eosswedenorg/thalos/app" "github.com/eosswedenorg/thalos/app/abi" + . "github.com/eosswedenorg/thalos/app/cache" "github.com/eosswedenorg/thalos/app/config" . "github.com/eosswedenorg/thalos/app/log" + redis_cache "github.com/go-redis/cache/v9" "github.com/nikoksr/notify" "github.com/nikoksr/notify/service/telegram" "github.com/pborman/getopt/v2" @@ -44,6 +46,10 @@ var VersionString string = "dev" var exit chan bool +var cache *Cache + +var cacheStore Store + func readerLoop(processor *app.ShipProcessor) { running = true recon_cnt := 0 @@ -171,6 +177,11 @@ func LogLevels() []string { return list } +func initAbiManger(api *eos.API, chain_id string) *abi.AbiManager { + cache := NewCache("thalos::cache::abi::"+chain_id, cacheStore) + return abi.NewAbiManager(cache, api) +} + func main() { var err error var chainInfo *eos.InfoResp @@ -281,6 +292,16 @@ func main() { return } + // Setup cache storage + cacheStore = NewRedisStore(&redis_cache.Options{ + Redis: rdb, + // Cache 10k keys for 10 minutes. + LocalCache: redis_cache.NewTinyLFU(10000, 10*time.Minute), + }) + + // Setup general cache + cache = NewCache("thalos::cache::instance::"+conf.Name, cacheStore) + log.WithField("api", conf.Api).Info("Get chain info from api") eosClient := eos.New(conf.Api) chainInfo, err = eosClient.GetInfo(context.Background()) @@ -318,7 +339,7 @@ func main() { Prefix: conf.Redis.Prefix, ChainID: chain_id, }), - abi.NewAbiManager(rdb, eosClient, chain_id), + initAbiManger(eosClient, chain_id), codec, ) From 64459eca10b58ea6932102634a41c53747bfdf0d Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Wed, 1 Nov 2023 21:46:00 +0100 Subject: [PATCH 022/291] app/ship_processor.go: typo fix. --- app/ship_processor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/ship_processor.go b/app/ship_processor.go index 7d2fe39..1e5abb1 100644 --- a/app/ship_processor.go +++ b/app/ship_processor.go @@ -274,7 +274,7 @@ func (processor *ShipProcessor) processBlock(block *ship.GetBlocksResultV0) { } } -// Close closes the writer assciated with the processor. +// Close closes the writer associated with the processor. func (processor *ShipProcessor) Close() error { return processor.writer.Close() } From 98a49538e2e515adac3c5762ee829cdecbfa3c27 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Wed, 1 Nov 2023 21:46:15 +0100 Subject: [PATCH 023/291] Adding app/state.go --- app/state.go | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 app/state.go diff --git a/app/state.go b/app/state.go new file mode 100644 index 0000000..b0c358e --- /dev/null +++ b/app/state.go @@ -0,0 +1,7 @@ +package app + +// State represents thalos runtime state +type State struct { + // Last processed block + CurrentBlock uint32 +} From 310f96219f09ba772fa5090077690dc3ec32092f Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Wed, 1 Nov 2023 21:47:21 +0100 Subject: [PATCH 024/291] app/ship_processor.go: remove current_block and use state.CurrentBlock instead. --- app/ship_processor.go | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/app/ship_processor.go b/app/ship_processor.go index 1e5abb1..c2fde70 100644 --- a/app/ship_processor.go +++ b/app/ship_processor.go @@ -43,8 +43,8 @@ type ShipProcessor struct { // Encoder used to encode messages encode message.Encoder - // Keep track of the current block we have processed. - current_block uint32 + // Internal state + state State // System contract ("eosio" per default) syscontract eos.AccountName @@ -53,12 +53,14 @@ type ShipProcessor struct { // SpawnProcessor creates a new ShipProccessor that consumes the shipclient.Stream passed to it. func SpawnProccessor(shipStream *shipclient.Stream, writer api.Writer, abi *abi.AbiManager, codec message.Codec) *ShipProcessor { processor := &ShipProcessor{ - abi: abi, - writer: writer, - shipStream: shipStream, - encode: logDecoratedEncoder(codec.Encoder), - syscontract: eos.AccountName("eosio"), - current_block: shipStream.StartBlock, + abi: abi, + writer: writer, + shipStream: shipStream, + encode: logDecoratedEncoder(codec.Encoder), + syscontract: eos.AccountName("eosio"), + state: State{ + CurrentBlock: shipStream.StartBlock, + }, } // Attach handlers @@ -124,15 +126,15 @@ func (processor *ShipProcessor) updateAbiFromAction(act *ship.Action) error { // Get the current block. func (processor *ShipProcessor) GetCurrentBlock() uint32 { - return processor.current_block + return processor.state.CurrentBlock } // Callback function called by shipclient.Stream when a new block arrives. func (processor *ShipProcessor) processBlock(block *ship.GetBlocksResultV0) { - processor.current_block = block.ThisBlock.BlockNum + processor.state.CurrentBlock = block.ThisBlock.BlockNum if block.ThisBlock.BlockNum%100 == 0 { - log.Infof("Current: %d, Head: %d", processor.current_block, block.Head.BlockNum) + log.Infof("Current: %d, Head: %d", processor.state.CurrentBlock, block.Head.BlockNum) } if block.ThisBlock.BlockNum%10 == 0 { From 1ae3da425c2e0f0e5b32db7d10a5510a3a2a652f Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sat, 4 Nov 2023 13:21:30 +0100 Subject: [PATCH 025/291] app/state.go: Adding StateLoader and StateSaver function types. --- app/state.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/app/state.go b/app/state.go index b0c358e..2edb323 100644 --- a/app/state.go +++ b/app/state.go @@ -5,3 +5,11 @@ type State struct { // Last processed block CurrentBlock uint32 } + +type ( + // StateLoader is a function that loads a state. + StateLoader func(*State) + + // StateSaver is a function that saves a state. + StateSaver func(State) error +) From 9946bd59e11282ca23ebdc012364f70f6083e359 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sat, 4 Nov 2023 13:24:15 +0100 Subject: [PATCH 026/291] app/ship_processor.go: implement StateLoader and StateSaver. --- app/ship_processor.go | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/app/ship_processor.go b/app/ship_processor.go index c2fde70..8db05a1 100644 --- a/app/ship_processor.go +++ b/app/ship_processor.go @@ -43,6 +43,9 @@ type ShipProcessor struct { // Encoder used to encode messages encode message.Encoder + // Function for saving state. + saver StateSaver + // Internal state state State @@ -51,18 +54,18 @@ type ShipProcessor struct { } // SpawnProcessor creates a new ShipProccessor that consumes the shipclient.Stream passed to it. -func SpawnProccessor(shipStream *shipclient.Stream, writer api.Writer, abi *abi.AbiManager, codec message.Codec) *ShipProcessor { +func SpawnProccessor(shipStream *shipclient.Stream, loader StateLoader, saver StateSaver, writer api.Writer, abi *abi.AbiManager, codec message.Codec) *ShipProcessor { processor := &ShipProcessor{ + saver: saver, abi: abi, writer: writer, shipStream: shipStream, encode: logDecoratedEncoder(codec.Encoder), syscontract: eos.AccountName("eosio"), - state: State{ - CurrentBlock: shipStream.StartBlock, - }, } + loader(&processor.state) + // Attach handlers shipStream.BlockHandler = processor.processBlock @@ -274,6 +277,11 @@ func (processor *ShipProcessor) processBlock(block *ship.GetBlocksResultV0) { if err != nil { log.WithError(err).Error("Failed to send messages") } + + err = processor.saver(processor.state) + if err != nil { + log.WithError(err).Error("Failed to save state") + } } // Close closes the writer associated with the processor. From fa48a79610abc3cc54b71e2dd10a847036251bed Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sat, 4 Nov 2023 13:24:51 +0100 Subject: [PATCH 027/291] cmd/thalos/main.go: adding stateLoader and stateSaver functions for passing to processor. --- cmd/thalos/main.go | 48 ++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 40 insertions(+), 8 deletions(-) diff --git a/cmd/thalos/main.go b/cmd/thalos/main.go index d590c02..df9c814 100644 --- a/cmd/thalos/main.go +++ b/cmd/thalos/main.go @@ -182,6 +182,44 @@ func initAbiManger(api *eos.API, chain_id string) *abi.AbiManager { return abi.NewAbiManager(cache, api) } +func stateLoader(chainInfo *eos.InfoResp) app.StateLoader { + return func(state *app.State) { + var source string + + // Load state from cache. + err := cache.Get("state", &state) + + // on error (cache miss) set current block from config/api + if err != nil { + // Set from config if we have a sane value. + if conf.Ship.StartBlockNum != shipclient.NULL_BLOCK_NUMBER { + source = "config" + state.CurrentBlock = conf.Ship.StartBlockNum + } else { + // Otherwise, set from api. + if conf.Ship.IrreversibleOnly { + source = "api (LIB)" + state.CurrentBlock = uint32(chainInfo.LastIrreversibleBlockNum) + } else { + source = "api (HEAD)" + state.CurrentBlock = uint32(chainInfo.HeadBlockNum) + } + } + } else { + source = "cache" + } + + log.WithFields(log.Fields{ + "block": state.CurrentBlock, + "source": source, + }).Info("Starting from block") + } +} + +func stateSaver(state app.State) error { + return cache.Set("state", state, 0) +} + func main() { var err error var chainInfo *eos.InfoResp @@ -310,14 +348,6 @@ func main() { return } - if conf.Ship.StartBlockNum == shipclient.NULL_BLOCK_NUMBER { - if conf.Ship.IrreversibleOnly { - conf.Ship.StartBlockNum = uint32(chainInfo.LastIrreversibleBlockNum) - } else { - conf.Ship.StartBlockNum = uint32(chainInfo.HeadBlockNum) - } - } - shClient = shipclient.NewStream(func(s *shipclient.Stream) { s.StartBlock = conf.Ship.StartBlockNum s.EndBlock = conf.Ship.EndBlockNum @@ -335,6 +365,8 @@ func main() { processor := app.SpawnProccessor( shClient, + stateLoader(chainInfo), + stateSaver, api_redis.NewPublisher(context.Background(), rdb, api_redis.Namespace{ Prefix: conf.Redis.Prefix, ChainID: chain_id, From 1daf38bd3d92cc5c8fa3f0b505b2ef96c7d2667d Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sat, 4 Nov 2023 13:32:32 +0100 Subject: [PATCH 028/291] cmd/thalos/main.go: adding "-n" flag to force current block from config/api. --- cmd/thalos/main.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/cmd/thalos/main.go b/cmd/thalos/main.go index df9c814..60d7de7 100644 --- a/cmd/thalos/main.go +++ b/cmd/thalos/main.go @@ -182,15 +182,16 @@ func initAbiManger(api *eos.API, chain_id string) *abi.AbiManager { return abi.NewAbiManager(cache, api) } -func stateLoader(chainInfo *eos.InfoResp) app.StateLoader { +func stateLoader(chainInfo *eos.InfoResp, current_block_no_cache bool) app.StateLoader { return func(state *app.State) { var source string // Load state from cache. err := cache.Get("state", &state) - // on error (cache miss) set current block from config/api - if err != nil { + // on error (cache miss) or if current_block_no_cache is set. + // set current block from config/api + if current_block_no_cache || err != nil { // Set from config if we have a sane value. if conf.Ship.StartBlockNum != shipclient.NULL_BLOCK_NUMBER { source = "config" @@ -232,6 +233,7 @@ func main() { pidFile := getopt.StringLong("pid", 'p', "", "Where to write process id", "file") logFile := getopt.StringLong("log", 'l', "", "Path to log file", "file") logLevel := getopt.EnumLong("level", 'L', LogLevels(), "info", "Log level to use") + skip_currentblock_cache := getopt.Bool('n', "Force the application to take start block from config/api") getopt.Parse() @@ -365,7 +367,7 @@ func main() { processor := app.SpawnProccessor( shClient, - stateLoader(chainInfo), + stateLoader(chainInfo, *skip_currentblock_cache), stateSaver, api_redis.NewPublisher(context.Background(), rdb, api_redis.Namespace{ Prefix: conf.Redis.Prefix, From 7ee70c00d7488883d0022d6fa9f9a79d049894ba Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sat, 4 Nov 2023 13:34:00 +0100 Subject: [PATCH 029/291] cmd/thalos/main.go: fix help text for version flag. --- cmd/thalos/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/thalos/main.go b/cmd/thalos/main.go index 020681b..5c6cf4d 100644 --- a/cmd/thalos/main.go +++ b/cmd/thalos/main.go @@ -178,7 +178,7 @@ func main() { exit = make(chan bool) showHelp := getopt.BoolLong("help", 'h', "display this help text") - showVersion := getopt.BoolLong("version", 'v', "display this help text") + showVersion := getopt.BoolLong("version", 'v', "display the version") configFile := getopt.StringLong("config", 'c', "./config.yml", "Config file to read", "file") pidFile := getopt.StringLong("pid", 'p', "", "Where to write process id", "file") logFile := getopt.StringLong("log", 'l', "", "Path to log file", "file") From 2a906e8818f9b100146a7f2f35d38e02fe3d0cc1 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sat, 4 Nov 2023 13:42:42 +0100 Subject: [PATCH 030/291] Version 0.2.1 --- Makefile | 2 +- debian/changelog | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index ba8b768..d7f64d4 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ GO=go GOLDFLAGS=-v -s -w -X main.VersionString=$(PROGRAM_VERSION) GOBUILDFLAGS=-v -p $(shell nproc) -ldflags="$(GOLDFLAGS)" PROGRAM=thalos-server -PROGRAM_VERSION=0.2.0 +PROGRAM_VERSION=0.2.1 PREFIX=/usr/local BINDIR=$(PREFIX)/bin CFGDIR=$(PREFIX)/etc/thalos diff --git a/debian/changelog b/debian/changelog index 646a2b6..5478e97 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,14 @@ +thalos (0.2.1) bionic focal jammy; urgency=medium + + * new tool: publisher mock (writes test data to redis channels.) + * golang: Drop support for 1.18, 1.19 + * new cli flag: "--level" to specify log level. + * new tool: redis ACL config generator. + * tools: adding flags for redis username/password + * code documentation + + -- Henrik Hautakoski Sat, 04 Nov 2023 13:35:06 +0100 + thalos (0.2.0) bionic focal jammy; urgency=medium * Update to use antelope-ship-client v0.2.5 From ba7f11742033e57df33ce6cede9556ad8371e07b Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 5 Nov 2023 20:40:10 +0100 Subject: [PATCH 031/291] .github/workflows/release.yml: use go version 1.21 for ubuntu packages. --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index cf13230..2f505c8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -72,7 +72,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v3 with: - go-version: 1.18 + go-version: 1.21 - name: Install build dependencies run: | From 1166f3ac5f82efff4047177c3d7677f0b851be31 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Mon, 6 Nov 2023 16:51:08 +0100 Subject: [PATCH 032/291] go mod: update github.com/eosswedenorg-go/antelope-ship-client to v0.2.7 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 538d2e4..99d479c 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/cenkalti/backoff/v4 v4.2.1 github.com/docker/go-units v0.5.0 github.com/eoscanada/eos-go v0.10.3-0.20230609180026-5d4ee54fe15e - github.com/eosswedenorg-go/antelope-ship-client v0.2.6 + github.com/eosswedenorg-go/antelope-ship-client v0.2.7 github.com/eosswedenorg-go/pid v1.0.1 github.com/eosswedenorg/thalos/api v0.1.2 github.com/go-redis/cache/v9 v9.0.0 diff --git a/go.sum b/go.sum index 2bd4ff0..6301e05 100644 --- a/go.sum +++ b/go.sum @@ -25,8 +25,8 @@ github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4 github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/eoscanada/eos-go v0.10.3-0.20230609180026-5d4ee54fe15e h1:C5V5vgv5ZFf51RW+QmBLR0eOZHvIlLdapAaIWE1pD+Q= github.com/eoscanada/eos-go v0.10.3-0.20230609180026-5d4ee54fe15e/go.mod h1:L3avCf8OkDrjlUeNy9DdoV67TCmDNj2dSlc5Xp3DNNk= -github.com/eosswedenorg-go/antelope-ship-client v0.2.6 h1:0wPF9TC867eG/+rEzJd0L+TtSOrP09YQzcbDU050FoA= -github.com/eosswedenorg-go/antelope-ship-client v0.2.6/go.mod h1:kZ/4gkAIdAq4/WiZlVaSONpELcDCMJQJMmlikLUGCb8= +github.com/eosswedenorg-go/antelope-ship-client v0.2.7 h1:ZGrXlGtMBqIV1TCCRjZAk9/RAA7pROO/Q/xN+baG45M= +github.com/eosswedenorg-go/antelope-ship-client v0.2.7/go.mod h1:kZ/4gkAIdAq4/WiZlVaSONpELcDCMJQJMmlikLUGCb8= github.com/eosswedenorg-go/jsontime v0.0.0-20230509125027-08422d6236c7 h1:rLPu++RHaxg4WmUOXeWYioZuafWs0PVcYuvzOWbOJjk= github.com/eosswedenorg-go/jsontime v0.0.0-20230509125027-08422d6236c7/go.mod h1:eNUkVOymzgl0lViUhmm08PkutzqLnOQ6Dr+RUnf+Mq0= github.com/eosswedenorg-go/pid v1.0.1 h1:W4AEnnNwb041SpNR1uTZ/KbJ0OTA5eqiqIR1Q5Ah6A0= From a94fbe191a1d08c7f9f8847a09eb79690535b3f1 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Mon, 6 Nov 2023 17:13:01 +0100 Subject: [PATCH 033/291] go mod: update dependencies --- go.mod | 22 ++++++++++++---------- go.sum | 43 ++++++++++++++++++++++--------------------- 2 files changed, 34 insertions(+), 31 deletions(-) diff --git a/go.mod b/go.mod index 99d479c..ae2f241 100644 --- a/go.mod +++ b/go.mod @@ -8,12 +8,12 @@ require ( github.com/eoscanada/eos-go v0.10.3-0.20230609180026-5d4ee54fe15e github.com/eosswedenorg-go/antelope-ship-client v0.2.7 github.com/eosswedenorg-go/pid v1.0.1 - github.com/eosswedenorg/thalos/api v0.1.2 + github.com/eosswedenorg/thalos/api v0.2.1 github.com/go-redis/cache/v9 v9.0.0 github.com/go-redis/redismock/v9 v9.2.0 github.com/nikoksr/notify v0.41.0 github.com/pborman/getopt/v2 v2.1.0 - github.com/redis/go-redis/v9 v9.2.1 + github.com/redis/go-redis/v9 v9.3.0 github.com/sirupsen/logrus v1.9.3 github.com/stretchr/testify v1.8.4 github.com/urfave/cli/v2 v2.25.7 @@ -27,17 +27,18 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/eosswedenorg-go/jsontime v0.0.0-20230509125027-08422d6236c7 // indirect - github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-telegram-bot-api/telegram-bot-api v4.6.4+incompatible // indirect - github.com/gorilla/websocket v1.5.0 // indirect + github.com/google/go-cmp v0.6.0 // indirect + github.com/gorilla/websocket v1.5.1 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.17.1 // indirect + github.com/klauspost/compress v1.17.2 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/logrusorgru/aurora v2.0.3+incompatible // indirect github.com/mitchellh/go-testing-interface v1.14.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/onsi/gomega v1.27.6 // indirect + github.com/onsi/gomega v1.29.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect @@ -48,15 +49,16 @@ require ( github.com/tidwall/pretty v1.2.1 // indirect github.com/ugorji/go/codec v1.2.11 // indirect github.com/vmihailenco/go-tinylfu v0.2.2 // indirect - github.com/vmihailenco/msgpack/v5 v5.4.0 // indirect + github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect - go.uber.org/goleak v1.2.1 // indirect + go.uber.org/goleak v1.3.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.26.0 // indirect golang.org/x/crypto v0.14.0 // indirect - golang.org/x/sync v0.4.0 // indirect - golang.org/x/sys v0.13.0 // indirect + golang.org/x/net v0.17.0 // indirect + golang.org/x/sync v0.5.0 // indirect + golang.org/x/sys v0.14.0 // indirect golang.org/x/term v0.13.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect ) diff --git a/go.sum b/go.sum index 6301e05..a7b8a14 100644 --- a/go.sum +++ b/go.sum @@ -33,8 +33,8 @@ github.com/eosswedenorg-go/pid v1.0.1 h1:W4AEnnNwb041SpNR1uTZ/KbJ0OTA5eqiqIR1Q5A github.com/eosswedenorg-go/pid v1.0.1/go.mod h1:wiOB/JXGt4YA3+T0j0xmCGSc3Jxzb7Ti/Ftli1fgWu4= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= -github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-redis/cache/v9 v9.0.0 h1:0thdtFo0xJi0/WXbRVu8B066z8OvVymXTJGaXrVWnN0= github.com/go-redis/cache/v9 v9.0.0/go.mod h1:cMwi1N8ASBOufbIvk7cdXe2PbPjK/WMRL95FFHWsSgI= @@ -57,12 +57,13 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= -github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= +github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/jarcoal/httpmock v1.2.0 h1:gSvTxxFR/MEMfsGrvRbdfpRUMBStovlSRLw0Ep1bwwc= @@ -70,8 +71,8 @@ github.com/jordan-wright/email v4.0.1-0.20210109023952-943e75fe5223+incompatible github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.17.1 h1:NE3C767s2ak2bweCZo3+rdP4U/HoyVXLv/X9f2gPS5g= -github.com/klauspost/compress v1.17.1/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4= +github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= @@ -117,8 +118,8 @@ github.com/onsi/gomega v1.22.1/go.mod h1:x6n7VNe4hw0vkyYUM4mjIXx3JbLiPaBPNgB7PRQ github.com/onsi/gomega v1.24.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg= github.com/onsi/gomega v1.24.1/go.mod h1:3AOiACssS3/MajrniINInwbfOOtfZvplPzuRSmvt1jM= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= -github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE= -github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg= +github.com/onsi/gomega v1.29.0 h1:KIA/t2t5UBzoirT4H9tsML45GEbo3ouUnBHsCfD2tVg= +github.com/onsi/gomega v1.29.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= github.com/pborman/getopt/v2 v2.1.0 h1:eNfR+r+dWLdWmV8g5OlpyrTYHkhVNxHBdN2cCrJmOEA= github.com/pborman/getopt/v2 v2.1.0/go.mod h1:4NtW75ny4eBw9fO1bhtNdYTlZKYX5/tBLtsOpwKIKd0= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= @@ -128,8 +129,8 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/redis/go-redis/v9 v9.0.0-rc.4/go.mod h1:Vo3EsyWnicKnSKCA7HhgnvnyA74wOA69Cd2Meli5mmA= -github.com/redis/go-redis/v9 v9.2.1 h1:WlYJg71ODF0dVspZZCpYmoF1+U1Jjk9Rwd7pq6QmlCg= -github.com/redis/go-redis/v9 v9.2.1/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= +github.com/redis/go-redis/v9 v9.3.0 h1:RiVDjmig62jIWp7Kk4XVLs0hzV6pI3PyTnnL0cnn0u0= +github.com/redis/go-redis/v9 v9.3.0/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= @@ -169,8 +170,8 @@ github.com/urfave/cli/v2 v2.25.7/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6S github.com/vmihailenco/go-tinylfu v0.2.2 h1:H1eiG6HM36iniK6+21n9LLpzx1G9R3DJa2UjUjbynsI= github.com/vmihailenco/go-tinylfu v0.2.2/go.mod h1:CutYi2Q9puTxfcolkliPq4npPuofg9N9t8JVrjzwa3Q= github.com/vmihailenco/msgpack/v5 v5.3.4/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= -github.com/vmihailenco/msgpack/v5 v5.4.0 h1:hRM0digJwyR6vll33NNAwCFguy5JuBD6jxDmQP3l608= -github.com/vmihailenco/msgpack/v5 v5.4.0/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok= +github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IUPn0Bjt8= +github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok= github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= @@ -183,8 +184,8 @@ github.com/yuin/gopher-lua v1.1.0 h1:BojcDhfyDWgU2f2TOzYK/g5p2gxMrku8oupLDqlnSqE go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= -go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= -go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -226,15 +227,16 @@ golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= -golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ= -golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= +golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -257,13 +259,12 @@ golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= +golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= From 24bf44a175a40d95e332f1c02dd936e924e1062a Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sat, 25 Nov 2023 15:12:08 +0100 Subject: [PATCH 034/291] move writer interface and redis publish from api module as these are for the server side. --- {api => app/driver}/redis/publisher.go | 1 + {api => app/driver}/redis/publisher_test.go | 1 + {api => app/driver}/writer.go | 6 ++++-- app/ship_processor.go | 5 +++-- cmd/thalos/main.go | 3 ++- 5 files changed, 11 insertions(+), 5 deletions(-) rename {api => app/driver}/redis/publisher.go (93%) rename {api => app/driver}/redis/publisher_test.go (94%) rename {api => app/driver}/writer.go (83%) diff --git a/api/redis/publisher.go b/app/driver/redis/publisher.go similarity index 93% rename from api/redis/publisher.go rename to app/driver/redis/publisher.go index f37d02b..623d67b 100644 --- a/api/redis/publisher.go +++ b/app/driver/redis/publisher.go @@ -4,6 +4,7 @@ import ( "context" "github.com/eosswedenorg/thalos/api" + . "github.com/eosswedenorg/thalos/api/redis" "github.com/redis/go-redis/v9" ) diff --git a/api/redis/publisher_test.go b/app/driver/redis/publisher_test.go similarity index 94% rename from api/redis/publisher_test.go rename to app/driver/redis/publisher_test.go index fa8e4ab..460daaf 100644 --- a/api/redis/publisher_test.go +++ b/app/driver/redis/publisher_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/eosswedenorg/thalos/api" + . "github.com/eosswedenorg/thalos/api/redis" "github.com/go-redis/redismock/v9" "github.com/stretchr/testify/assert" diff --git a/api/writer.go b/app/driver/writer.go similarity index 83% rename from api/writer.go rename to app/driver/writer.go index 4cfc8b2..e9c2f73 100644 --- a/api/writer.go +++ b/app/driver/writer.go @@ -1,4 +1,6 @@ -package api +package driver + +import "github.com/eosswedenorg/thalos/api" // Writer interface defines the required methods // to send messages over an channel. @@ -7,7 +9,7 @@ package api type Writer interface { // Write writes a message over a channel. // The message may or may not be buffered depending on the implementation. - Write(channel Channel, payload []byte) error + Write(channel api.Channel, payload []byte) error // Flush writes any buffered messages to the channel. // If the implementation does not support buffering. this is a noop. diff --git a/app/ship_processor.go b/app/ship_processor.go index 7d2fe39..a0d8fc4 100644 --- a/app/ship_processor.go +++ b/app/ship_processor.go @@ -7,6 +7,7 @@ import ( "github.com/eosswedenorg/thalos/api" "github.com/eosswedenorg/thalos/api/message" "github.com/eosswedenorg/thalos/app/abi" + "github.com/eosswedenorg/thalos/app/driver" log "github.com/sirupsen/logrus" @@ -38,7 +39,7 @@ type ShipProcessor struct { abi *abi.AbiManager // Writer to send messages to. - writer api.Writer + writer driver.Writer // Encoder used to encode messages encode message.Encoder @@ -51,7 +52,7 @@ type ShipProcessor struct { } // SpawnProcessor creates a new ShipProccessor that consumes the shipclient.Stream passed to it. -func SpawnProccessor(shipStream *shipclient.Stream, writer api.Writer, abi *abi.AbiManager, codec message.Codec) *ShipProcessor { +func SpawnProccessor(shipStream *shipclient.Stream, writer driver.Writer, abi *abi.AbiManager, codec message.Codec) *ShipProcessor { processor := &ShipProcessor{ abi: abi, writer: writer, diff --git a/cmd/thalos/main.go b/cmd/thalos/main.go index 5c6cf4d..431d75d 100644 --- a/cmd/thalos/main.go +++ b/cmd/thalos/main.go @@ -22,6 +22,7 @@ import ( "github.com/eosswedenorg/thalos/app" "github.com/eosswedenorg/thalos/app/abi" "github.com/eosswedenorg/thalos/app/config" + driver "github.com/eosswedenorg/thalos/app/driver/redis" . "github.com/eosswedenorg/thalos/app/log" "github.com/nikoksr/notify" "github.com/nikoksr/notify/service/telegram" @@ -314,7 +315,7 @@ func main() { processor := app.SpawnProccessor( shClient, - api_redis.NewPublisher(context.Background(), rdb, api_redis.Namespace{ + driver.NewPublisher(context.Background(), rdb, api_redis.Namespace{ Prefix: conf.Redis.Prefix, ChainID: chain_id, }), From 7b1b500b22b320bf75c9b56bf36f9f985cb4f1ac Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Wed, 6 Dec 2023 15:30:09 +0100 Subject: [PATCH 035/291] cmd/thalos/main.go: have running variable be set to true on initialization. No point setting it in readerLoop() as its only used to exit the application, as soon as its set to false. the application should exit as soon as possible. --- cmd/thalos/main.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cmd/thalos/main.go b/cmd/thalos/main.go index 431d75d..4278365 100644 --- a/cmd/thalos/main.go +++ b/cmd/thalos/main.go @@ -39,14 +39,13 @@ var conf *config.Config var shClient *shipclient.Stream -var running bool = false +var running bool = true var VersionString string = "dev" var exit chan bool func readerLoop(processor *app.ShipProcessor) { - running = true recon_cnt := 0 exp := &backoff.ExponentialBackOff{ From eeab11c44b5e63cf4b7e6a5e3bc9d79f2bfb8da0 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Wed, 6 Dec 2023 15:32:33 +0100 Subject: [PATCH 036/291] app/abi/manager.go: its probably a good thing (tm) to set a context timout when getting the abi from the api. --- app/abi/manager.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/abi/manager.go b/app/abi/manager.go index 79df1bd..09e9256 100644 --- a/app/abi/manager.go +++ b/app/abi/manager.go @@ -45,7 +45,9 @@ func (mgr *AbiManager) GetAbi(account eos.AccountName) (*eos.ABI, error) { abi, err := mgr.cache.Get(key) if err != nil { - resp, err := mgr.api.GetABI(mgr.ctx, account) + ctx, cancel := context.WithTimeout(mgr.ctx, time.Second) + defer cancel() + resp, err := mgr.api.GetABI(ctx, account) if err != nil { return nil, fmt.Errorf("api: %s", err) } From 8b3202e4d377c9148b914aace44ce305c367ca2a Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 17 Dec 2023 17:51:32 +0100 Subject: [PATCH 037/291] app/cache/store.go: make interface context aware. --- app/cache/store.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/app/cache/store.go b/app/cache/store.go index 309333f..1b69639 100644 --- a/app/cache/store.go +++ b/app/cache/store.go @@ -1,15 +1,18 @@ package cache -import "time" +import ( + "context" + "time" +) type Store interface { // Set an item in the store. - Set(key string, value any, TTL time.Duration) error + Set(ctx context.Context, key string, value any, TTL time.Duration) error // Get an item from the store. // returns an error if key is not found or there is other problems. - Get(key string, value any) error + Get(ctx context.Context, key string, value any) error // Check if a key exist in the store. - Has(key string) bool + Has(ctx context.Context, key string) bool } From 610648141799bd2df9b41f211f967d0bd0a92711 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 17 Dec 2023 17:51:50 +0100 Subject: [PATCH 038/291] app/cache: implement store interface. --- app/cache/memory_store.go | 7 ++++--- app/cache/memory_store_test.go | 19 ++++++++++--------- app/cache/redis_store.go | 18 ++++++++---------- app/cache/redis_store_test.go | 15 ++++++++------- 4 files changed, 30 insertions(+), 29 deletions(-) diff --git a/app/cache/memory_store.go b/app/cache/memory_store.go index 7c3bd9f..0dbc1be 100644 --- a/app/cache/memory_store.go +++ b/app/cache/memory_store.go @@ -1,6 +1,7 @@ package cache import ( + "context" "fmt" "reflect" "time" @@ -21,7 +22,7 @@ func NewMemoryStore() *MemoryStore { return &MemoryStore{make(map[string]memoryStoreItem)} } -func (s *MemoryStore) Get(key string, value any) error { +func (s *MemoryStore) Get(ctx context.Context, key string, value any) error { if item, ok := s.data[key]; ok { if item.expired.Before(now()) { @@ -41,12 +42,12 @@ func (s *MemoryStore) Get(key string, value any) error { return fmt.Errorf("key: %s does not exist", key) } -func (s *MemoryStore) Has(key string) bool { +func (s *MemoryStore) Has(ctx context.Context, key string) bool { _, hit := s.data[key] return hit } -func (s *MemoryStore) Set(key string, value any, ttl time.Duration) error { +func (s *MemoryStore) Set(ctx context.Context, key string, value any, ttl time.Duration) error { s.data[key] = memoryStoreItem{ value: value, expired: now().Add(ttl), diff --git a/app/cache/memory_store_test.go b/app/cache/memory_store_test.go index d6bc6bd..9c949e3 100644 --- a/app/cache/memory_store_test.go +++ b/app/cache/memory_store_test.go @@ -1,6 +1,7 @@ package cache import ( + "context" "testing" "time" @@ -30,7 +31,7 @@ func TestMemoryStore_Set(t *testing.T) { } store := NewMemoryStore() - err := store.Set("key1", item, time.Hour) + err := store.Set(context.Background(), "key1", item, time.Hour) assert.NoError(t, err) assert.Equal(t, expected, store.data) @@ -40,7 +41,7 @@ func TestMemoryStore_GetMiss(t *testing.T) { store := NewMemoryStore() var v any - err := store.Get("Key2", &v) + err := store.Get(context.Background(), "Key2", &v) assert.Error(t, err) } @@ -52,11 +53,11 @@ func TestMemoryStore_GetHit(t *testing.T) { } store := NewMemoryStore() - err := store.Set("key1", expected, time.Hour) + err := store.Set(context.Background(), "key1", expected, time.Hour) assert.NoError(t, err) var actual memoryTestItem - err = store.Get("key1", &actual) + err = store.Get(context.Background(), "key1", &actual) assert.NoError(t, err) assert.Equal(t, expected, actual) } @@ -69,19 +70,19 @@ func TestMemoryStore_GetNonPointer(t *testing.T) { } store := NewMemoryStore() - err := store.Set("key1", expected, time.Hour) + err := store.Set(context.Background(), "key1", expected, time.Hour) assert.NoError(t, err) var actual string - err = store.Get("key1", actual) + err = store.Get(context.Background(), "key1", actual) assert.EqualError(t, err, "value must be of pointer type, 'string' passed") } func TestMemoryStore_Has(t *testing.T) { store := NewMemoryStore() - err := store.Set("key1", "value", time.Hour) + err := store.Set(context.Background(), "key1", "value", time.Hour) assert.NoError(t, err) - assert.True(t, store.Has("key1")) - assert.False(t, store.Has("key2")) + assert.True(t, store.Has(context.Background(), "key1")) + assert.False(t, store.Has(context.Background(), "key2")) } diff --git a/app/cache/redis_store.go b/app/cache/redis_store.go index 5016a15..54d7f4d 100644 --- a/app/cache/redis_store.go +++ b/app/cache/redis_store.go @@ -8,28 +8,26 @@ import ( ) type RedisStore struct { - c *cache.Cache - ctx context.Context + c *cache.Cache } func NewRedisStore(options *cache.Options) *RedisStore { return &RedisStore{ - c: cache.New(options), - ctx: context.Background(), + c: cache.New(options), } } -func (s *RedisStore) Get(key string, value interface{}) error { - return s.c.Get(s.ctx, key, value) +func (s *RedisStore) Get(ctx context.Context, key string, value interface{}) error { + return s.c.Get(ctx, key, value) } -func (s *RedisStore) Has(key string) bool { - return s.c.Exists(s.ctx, key) +func (s *RedisStore) Has(ctx context.Context, key string) bool { + return s.c.Exists(ctx, key) } -func (s *RedisStore) Set(key string, value any, ttl time.Duration) error { +func (s *RedisStore) Set(ctx context.Context, key string, value any, ttl time.Duration) error { return s.c.Set(&cache.Item{ - Ctx: s.ctx, + Ctx: ctx, Key: key, Value: value, TTL: ttl, diff --git a/app/cache/redis_store_test.go b/app/cache/redis_store_test.go index 34e3f34..f12cc33 100644 --- a/app/cache/redis_store_test.go +++ b/app/cache/redis_store_test.go @@ -1,6 +1,7 @@ package cache import ( + "context" "testing" "time" @@ -32,7 +33,7 @@ func TestRedisStore_Set(t *testing.T) { mock.ExpectSet("mykey", bytes, time.Minute).SetVal("OK") - err = store.Set("mykey", expected, time.Minute) + err = store.Set(context.Background(), "mykey", expected, time.Minute) assert.NoError(t, err) assert.NoError(t, mock.ExpectationsWereMet()) @@ -48,7 +49,7 @@ func TestRedisStore_GetMiss(t *testing.T) { mock.ExpectGet("mykey").SetErr(redis_cache.ErrCacheMiss) expected := testItem{} - err := store.Get("mykey", &expected) + err := store.Get(context.Background(), "mykey", &expected) assert.ErrorIs(t, err, redis_cache.ErrCacheMiss) assert.NoError(t, mock.ExpectationsWereMet()) } @@ -71,11 +72,11 @@ func TestRedisStore_GetHit(t *testing.T) { mock.ExpectSet("mykey2", bytes, time.Second*20).SetVal("OK") mock.ExpectGet("mykey2").SetVal(string(bytes)) - err = store.Set("mykey2", expected, time.Second*20) + err = store.Set(context.Background(), "mykey2", expected, time.Second*20) assert.NoError(t, err) actual := testItem{} - err = store.Get("mykey2", &actual) + err = store.Get(context.Background(), "mykey2", &actual) assert.NoError(t, err) assert.Equal(t, expected, actual) assert.NoError(t, mock.ExpectationsWereMet()) @@ -95,9 +96,9 @@ func TestRedisStore_Has(t *testing.T) { mock.ExpectGet("key1").SetVal(string(bytes)) mock.ExpectGet("key2").RedisNil() - err = store.Set("key1", "value", time.Minute*15) + err = store.Set(context.Background(), "key1", "value", time.Minute*15) assert.NoError(t, err) - assert.True(t, store.Has("key1")) - assert.False(t, store.Has("key2")) + assert.True(t, store.Has(context.Background(), "key1")) + assert.False(t, store.Has(context.Background(), "key2")) assert.NoError(t, mock.ExpectationsWereMet()) } From 5a5e4bfd2e428a6638f3442040a9d8ea5e1843d3 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 17 Dec 2023 17:57:49 +0100 Subject: [PATCH 039/291] app/cache/cache.go: make context aware and pass context down to store. --- app/cache/cache.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/app/cache/cache.go b/app/cache/cache.go index 290019b..3cce5f5 100644 --- a/app/cache/cache.go +++ b/app/cache/cache.go @@ -1,6 +1,7 @@ package cache import ( + "context" "time" ) @@ -17,12 +18,12 @@ func NewCache(prefix string, store Store) *Cache { } } -func (cache *Cache) Get(key string, value any) error { - return cache.store.Get(cache.key(key), value) +func (cache *Cache) Get(ctx context.Context, key string, value any) error { + return cache.store.Get(ctx, cache.key(key), value) } -func (cache *Cache) Set(key string, value any, ttl time.Duration) error { - return cache.store.Set(cache.key(key), value, ttl) +func (cache *Cache) Set(ctx context.Context, key string, value any, ttl time.Duration) error { + return cache.store.Set(ctx, cache.key(key), value, ttl) } func (cache *Cache) key(key string) string { From 1d5e28a38f5a224365f04bfaf13f76a43089ad5c Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 17 Dec 2023 17:58:06 +0100 Subject: [PATCH 040/291] app/abi/manager.go: pass context to cache. --- app/abi/manager.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/app/abi/manager.go b/app/abi/manager.go index ba0fa5a..43be262 100644 --- a/app/abi/manager.go +++ b/app/abi/manager.go @@ -27,14 +27,16 @@ func NewAbiManager(cache *cache.Cache, api *eos.API) *AbiManager { // Set or update an ABI in the cache. func (mgr *AbiManager) SetAbi(account eos.AccountName, abi *eos.ABI) error { - return mgr.cache.Set(string(account), *abi, time.Hour) + ctx, cancel := context.WithTimeout(mgr.ctx, time.Millisecond*500) + defer cancel() + return mgr.cache.Set(ctx, string(account), *abi, time.Hour) } // Get an ABI from the cache, on cache miss it is fetched from the // API, gets cached and then returned to the user func (mgr *AbiManager) GetAbi(account eos.AccountName) (*eos.ABI, error) { var abi eos.ABI - if err := mgr.cache.Get(string(account), &abi); err != nil { + if err := mgr.cacheGet(account, &abi); err != nil { ctx, cancel := context.WithTimeout(mgr.ctx, time.Second) defer cancel() resp, err := mgr.api.GetABI(ctx, account) @@ -50,3 +52,9 @@ func (mgr *AbiManager) GetAbi(account eos.AccountName) (*eos.ABI, error) { } return &abi, nil } + +func (mgr *AbiManager) cacheGet(account eos.AccountName, value any) error { + ctx, cancel := context.WithTimeout(mgr.ctx, time.Millisecond*500) + defer cancel() + return mgr.cache.Get(ctx, string(account), value) +} From 41ab39d4a361655b65012555e1e43fe9f97c09a7 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 17 Dec 2023 17:58:42 +0100 Subject: [PATCH 041/291] cmd/thalos/main.go: pass context to cache. --- cmd/thalos/main.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/cmd/thalos/main.go b/cmd/thalos/main.go index ef41f25..77049fe 100644 --- a/cmd/thalos/main.go +++ b/cmd/thalos/main.go @@ -187,7 +187,9 @@ func stateLoader(chainInfo *eos.InfoResp, current_block_no_cache bool) app.State var source string // Load state from cache. - err := cache.Get("state", &state) + ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*500) + err := cache.Get(ctx, "state", &state) + cancel() // on error (cache miss) or if current_block_no_cache is set. // set current block from config/api @@ -218,7 +220,9 @@ func stateLoader(chainInfo *eos.InfoResp, current_block_no_cache bool) app.State } func stateSaver(state app.State) error { - return cache.Set("state", state, 0) + ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*500) + defer cancel() + return cache.Set(ctx, "state", state, 0) } func main() { From 578f9e83d7d6892092a017b980fd651ce7e30593 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 17 Dec 2023 19:37:08 +0100 Subject: [PATCH 042/291] cmd/tools/mock_publisher.go: fix correct imports. --- cmd/tools/mock_publisher.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/tools/mock_publisher.go b/cmd/tools/mock_publisher.go index 4a335c1..7dcb05d 100644 --- a/cmd/tools/mock_publisher.go +++ b/cmd/tools/mock_publisher.go @@ -10,6 +10,7 @@ import ( "github.com/eosswedenorg/thalos/api/message" _ "github.com/eosswedenorg/thalos/api/message/json" api_redis "github.com/eosswedenorg/thalos/api/redis" + redis_driver "github.com/eosswedenorg/thalos/app/driver/redis" "github.com/redis/go-redis/v9" ) @@ -47,7 +48,7 @@ var MockPublisherCmd = &cli.Command{ Prefix: ctx.String("redis-prefix"), ChainID: ctx.String("chain_id"), } - publisher := api_redis.NewPublisher(context.Background(), rdb, ns) + publisher := redis_driver.NewPublisher(context.Background(), rdb, ns) msg := message.ActionTrace{ TxID: "401e8a7e5deb18a2a69fc6559f49509a155f4355c85efb69c1c1fab5b60ee532", From 0bfcc4e49216903b5d861061094362c1284aa0b8 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 17 Dec 2023 19:38:02 +0100 Subject: [PATCH 043/291] Version 0.2.2 --- debian/changelog | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/debian/changelog b/debian/changelog index 5478e97..6744a4b 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,13 @@ +thalos (0.2.2) bionic focal jammy; urgency=medium + + * Current processing state (block number) is now cached and loaded on start. + * cli: new `-n` flag that forces the application to take start block from config/api instead of cache. + * bug: the ABI Cache now has a timeout for the api call on cache miss fixing + a bug where the application would hang if api did not respond. + * Update to use antelope-ship-client v0.2.7 + + -- Henrik Hautakoski Sun, 17 Dec 2023 19:37:30 +0100 + thalos (0.2.1) bionic focal jammy; urgency=medium * new tool: publisher mock (writes test data to redis channels.) From 2586ed772468414c212bcaf29ada3cb5d4f58a90 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 28 Dec 2023 16:50:30 +0000 Subject: [PATCH 044/291] build(deps): bump golang.org/x/crypto from 0.14.0 to 0.17.0 Bumps [golang.org/x/crypto](https://github.com/golang/crypto) from 0.14.0 to 0.17.0. - [Commits](https://github.com/golang/crypto/compare/v0.14.0...v0.17.0) --- updated-dependencies: - dependency-name: golang.org/x/crypto dependency-type: indirect ... Signed-off-by: dependabot[bot] --- go.mod | 6 +++--- go.sum | 14 +++++++------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index ae2f241..6987e44 100644 --- a/go.mod +++ b/go.mod @@ -55,11 +55,11 @@ require ( go.uber.org/goleak v1.3.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.26.0 // indirect - golang.org/x/crypto v0.14.0 // indirect + golang.org/x/crypto v0.17.0 // indirect golang.org/x/net v0.17.0 // indirect golang.org/x/sync v0.5.0 // indirect - golang.org/x/sys v0.14.0 // indirect - golang.org/x/term v0.13.0 // indirect + golang.org/x/sys v0.15.0 // indirect + golang.org/x/term v0.15.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect ) diff --git a/go.sum b/go.sum index a7b8a14..effd88d 100644 --- a/go.sum +++ b/go.sum @@ -200,8 +200,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= -golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= -golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= +golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -263,16 +263,16 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= -golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= -golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= -golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= +golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -280,7 +280,7 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= From 8ffd86daac78edd1568cdd68d09ca95ac7fd7394 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 7 Jan 2024 14:31:11 +0100 Subject: [PATCH 045/291] api/message: for codec implementations, define a function instead of global variables. --- api/message/json/codec.go | 20 +++++++++------- api/message/json/codec_test.go | 8 +++---- api/message/msgpack/codec.go | 39 ++++++++++++++++--------------- api/message/msgpack/codec_test.go | 12 +++++----- 4 files changed, 41 insertions(+), 38 deletions(-) diff --git a/api/message/json/codec.go b/api/message/json/codec.go index fb6b0f7..1921f88 100644 --- a/api/message/json/codec.go +++ b/api/message/json/codec.go @@ -7,17 +7,19 @@ import ( "github.com/eosswedenorg/thalos/api/message" ) -var ( - json_codec = jsontime.ConfigWithCustomTimeFormat - encoder = json_codec.Marshal - decoder = json_codec.Unmarshal -) +func createCodec() message.Codec { + json_codec := jsontime.ConfigWithCustomTimeFormat + + return message.Codec{ + Encoder: json_codec.Marshal, + Decoder: json_codec.Unmarshal, + } +} func init() { + // Set timeformat used by eos api. jsontime.SetDefaultTimeFormat("2006-01-02T15:04:05.000", time.UTC) - message.RegisterCodec("json", message.Codec{ - Encoder: encoder, - Decoder: decoder, - }) + // Register the json codec. + message.RegisterCodec("json", createCodec()) } diff --git a/api/message/json/codec_test.go b/api/message/json/codec_test.go index 074d7d5..b657eb4 100644 --- a/api/message/json/codec_test.go +++ b/api/message/json/codec_test.go @@ -45,7 +45,7 @@ func TestJson_EncodeActionTrace(t *testing.T) { expected := `{"tx_id":"ed3b8e853647971cf8296f004c3a1aeac255f082b2cb3c12cc3222e2d7c174ab","blocknum":267372365,"blocktimestamp":"2003-03-21T17:23:09.500","receipt":{"receiver":"account2","act_digest":"0a2c71dba327cf13a107d3a4f91c9c98f510a8fbbb483b233e222033f13a3e36","global_sequence":2329381932,"recv_sequence":22,"auth_sequence":[{"account":"account1","sequence":1382772}],"code_sequence":1122,"abi_sequence":12352},"name":"transfer","contract":"eosio","receiver":"account2","data":{"from":"account1","quantity":"1000.0000 WAX","to":"account2"},"authorization":[{"actor":"account1","permission":"active"}],"except":"errstr","error":2,"return":"3q2+7w=="}` - data, err := encoder(msg) + data, err := createCodec().Encoder(msg) assert.NoError(t, err) assert.Equal(t, expected, string(data)) } @@ -88,7 +88,7 @@ func TestJson_DecodeActionTrace(t *testing.T) { input := `{"tx_id":"952989f7464237b6cf9926e533ecd331df6794ed07564bd052bc368cbd65b4bc","blocknum":8723971,"blocktimestamp":"2024-06-21T08:08:26.500","receipt":{"receiver":"account2","act_digest":"f2f682847fd5bf00fb315b075dc00b4ff0ce18776758077b86a233dea49dc047","global_sequence":287328,"recv_sequence":42,"auth_sequence":[{"account":"account1","sequence":271877283}],"code_sequence":237823,"abi_sequence":68323},"name":"transfer","contract":"eosio","receiver":"account2","data":{"from":"account1","quantity":"1000.0000 WAX","to":"account2"},"authorization":[{"actor":"account1","permission":"active"}],"except":"errstr","error":2,"return":"3q2+7w=="}` msg := message.ActionTrace{} - err := decoder([]byte(input), &msg) + err := createCodec().Decoder([]byte(input), &msg) assert.NoError(t, err) assert.Equal(t, expected, msg) } @@ -155,7 +155,7 @@ func TestJson_EncodeTransactionTrace(t *testing.T) { expected := `{"id":"ed04516bdd1194aa5f0ab4c8c5445eec542c17f45a85bb3e9e4bc33e1a2486f8","blocknum":283781923,"blocktimestamp":"2029-02-08T15:10:05.500","status":"executed","cpu_usage_us":442,"net_usage_words":16,"elapsed":22,"net_usage":128,"scheduled":true,"action_traces":[{"tx_id":"ed04516bdd1194aa5f0ab4c8c5445eec542c17f45a85bb3e9e4bc33e1a2486f8","blocknum":283781923,"blocktimestamp":"2029-02-08T15:10:05.500","name":"mine","contract":"coolgame","receiver":"actor01","data":{"equipment_id":1234,"location_id":5445453},"authorization":[{"actor":"actor01","permission":"active"}],"except":"","error":0,"return":"CPE="},{"tx_id":"ed04516bdd1194aa5f0ab4c8c5445eec542c17f45a85bb3e9e4bc33e1a2486f8","blocknum":283781923,"blocktimestamp":"2029-02-08T15:10:05.500","name":"addpoints","contract":"coolgame","receiver":"coolgame","data":{"points":"1023.0423 SCAM"},"authorization":[{"actor":"coolgame","permission":"usrpoints"}],"except":"some error string","error":2,"return":"/wI="}],"except":"errstr","error":2}` - data, err := encoder(msg) + data, err := createCodec().Encoder(msg) assert.NoError(t, err) assert.Equal(t, expected, string(data)) } @@ -235,7 +235,7 @@ func TestJson_DecodeTransactionTrace(t *testing.T) { } msg := message.TransactionTrace{} - err := decoder([]byte(input), &msg) + err := createCodec().Decoder([]byte(input), &msg) assert.NoError(t, err) assert.Equal(t, expected, msg) } diff --git a/api/message/msgpack/codec.go b/api/message/msgpack/codec.go index 783635c..2e684a6 100644 --- a/api/message/msgpack/codec.go +++ b/api/message/msgpack/codec.go @@ -8,27 +8,28 @@ import ( "github.com/eosswedenorg/thalos/api/message" ) -var mh codec.MsgpackHandle - -func encode(a any) ([]byte, error) { - var b []byte - return b, codec.NewEncoderBytes(&b, &mh).Encode(a) -} - -func decode(b []byte, a any) error { - return codec.NewDecoderBytes(b, &mh).Decode(a) -} - -func init() { - mh.MapType = reflect.TypeOf(map[string]any(nil)) - mh.Canonical = true +func createCodec() message.Codec { + // Create handler. + handle := codec.MsgpackHandle{} + handle.MapType = reflect.TypeOf(map[string]any(nil)) + handle.Canonical = true // Wierd name but this is needed for the newest version of msgpack // that has support for time and string datatypes etc. - mh.WriteExt = true + handle.WriteExt = true - message.RegisterCodec("msgpack", message.Codec{ - Encoder: encode, - Decoder: decode, - }) + return message.Codec{ + Encoder: func(a any) ([]byte, error) { + var b []byte + return b, codec.NewEncoderBytes(&b, &handle).Encode(a) + }, + Decoder: func(b []byte, a any) error { + return codec.NewDecoderBytes(b, &handle).Decode(a) + }, + } +} + +func init() { + // Register codec. + message.RegisterCodec("msgpack", createCodec()) } diff --git a/api/message/msgpack/codec_test.go b/api/message/msgpack/codec_test.go index 2348257..71d47c7 100644 --- a/api/message/msgpack/codec_test.go +++ b/api/message/msgpack/codec_test.go @@ -51,7 +51,7 @@ func TestMsgpack_EncodeActionTrace(t *testing.T) { Return: []byte{0xde, 0xad, 0xbe, 0xef}, } - data, err := encode(msg) + data, err := createCodec().Encoder(msg) assert.NoError(t, err) expected := []byte{ @@ -177,7 +177,7 @@ func TestMsgpack_DecodeActionTrace(t *testing.T) { } res := message.ActionTrace{} - err := decode(data, &res) + err := createCodec().Decoder(data, &res) assert.NoError(t, err) assert.Equal(t, expected, res) @@ -190,7 +190,7 @@ func TestMsgpack_EncodeHeartbeat(t *testing.T) { LastIrreversibleBlockNum: 1236, } - data, err := encode(msg) + data, err := createCodec().Encoder(msg) assert.NoError(t, err) assert.Equal(t, data, []byte{0x83, 0xa8, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x6e, 0x75, 0x6d, 0xcd, 0x4, 0xd2, 0xad, 0x68, 0x65, 0x61, 0x64, 0x5f, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x6e, 0x75, 0x6d, 0xcd, 0x4, 0xd3, 0xba, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x69, 0x72, 0x72, 0x65, 0x76, 0x65, 0x72, 0x73, 0x69, 0x62, 0x6c, 0x65, 0x5f, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x6e, 0x75, 0x6d, 0xcd, 0x4, 0xd4}) @@ -206,7 +206,7 @@ func TestMsgpack_DecodeHeartbeat(t *testing.T) { } msg := message.HeartBeat{} - err := decode(data, &msg) + err := createCodec().Decoder(data, &msg) assert.NoError(t, err) assert.Equal(t, expected, msg) } @@ -265,7 +265,7 @@ func TestMsgpack_EncodeTransactionTrace(t *testing.T) { Error: 2, } - data, err := encode(msg) + data, err := createCodec().Encoder(msg) assert.NoError(t, err) expected := []byte{ @@ -406,7 +406,7 @@ func TestMsgpack_DecodeTransactionTrace(t *testing.T) { } res := message.TransactionTrace{} - err := decode(data, &res) + err := createCodec().Decoder(data, &res) assert.NoError(t, err) assert.Equal(t, expected, res) From f537c6f95ff767c95e5f81851a820d38594626b5 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 7 Jan 2024 14:33:22 +0100 Subject: [PATCH 046/291] app/ship_processor.go: remove redundant information to log call. --- app/ship_processor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/ship_processor.go b/app/ship_processor.go index 6f88894..3fe0901 100644 --- a/app/ship_processor.go +++ b/app/ship_processor.go @@ -248,7 +248,7 @@ func (processor *ShipProcessor) processBlock(block *ship.GetBlocksResultV0) { } } else { logger.WithField("contract", act_trace.Act.Account). - WithError(err).Errorf("Failed to get abi for contract %s", act_trace.Act.Account) + WithError(err).Error("Failed to get abi for contract") } payload, err := processor.encode(act) From dc3e59ad233eb232f3b5e45c114d9565bdc03911 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 7 Jan 2024 14:33:46 +0100 Subject: [PATCH 047/291] api/message/json/codec.go: minor fix. --- api/message/json/codec.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/message/json/codec.go b/api/message/json/codec.go index 1921f88..d04e103 100644 --- a/api/message/json/codec.go +++ b/api/message/json/codec.go @@ -17,7 +17,7 @@ func createCodec() message.Codec { } func init() { - // Set timeformat used by eos api. + // Set timeformat used by SHIP api jsontime.SetDefaultTimeFormat("2006-01-02T15:04:05.000", time.UTC) // Register the json codec. From a55367844e54bc4c76b14e4309246e601d86423f Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 7 Jan 2024 14:34:56 +0100 Subject: [PATCH 048/291] LICENSE: Update year. --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index 05ea739..e3754ba 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2022-2023 Sw/eden +Copyright (c) 2022-2024 Sw/eden Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal From 8dbf411b36098dc0c8f7c713b4c8ab9837ee9960 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 7 Jan 2024 19:11:47 +0100 Subject: [PATCH 049/291] api/channel.go: adding TableDelta channel --- api/channel.go | 15 +++++++++++++++ api/channel_test.go | 18 ++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/api/channel.go b/api/channel.go index a843f54..b722037 100644 --- a/api/channel.go +++ b/api/channel.go @@ -66,3 +66,18 @@ func (a ActionChannel) Channel() Channel { return ch } + +// Table deltas +type TableDeltaChannel struct { + Name string +} + +func (td TableDeltaChannel) Channel() Channel { + ch := Channel{"tabledeltas"} + + if len(td.Name) > 0 { + ch.Append("name", td.Name) + } + + return ch +} diff --git a/api/channel_test.go b/api/channel_test.go index 58bc597..caaf4e7 100644 --- a/api/channel_test.go +++ b/api/channel_test.go @@ -129,3 +129,21 @@ func TestAction_Channel(t *testing.T) { }) } } + +func TestTableDelta_Channel(t *testing.T) { + tests := []struct { + name string + action TableDeltaChannel + want Channel + }{ + {"Empty", TableDeltaChannel{}, Channel{"tabledeltas"}}, + {"Contract", TableDeltaChannel{Name: "delta_name"}, Channel{"tabledeltas", "name", "delta_name"}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.action.Channel(); !got.Is(tt.want) { + t.Errorf("TableDeltaChannel.String() = %v, want %v", got, tt.want) + } + }) + } +} From 8321c1633a9751e87a8b4757b52272311c60235f Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 7 Jan 2024 19:25:57 +0100 Subject: [PATCH 050/291] api/channel.go: Adding rollback channel. --- api/channel.go | 1 + 1 file changed, 1 insertion(+) diff --git a/api/channel.go b/api/channel.go index a843f54..ffe2fba 100644 --- a/api/channel.go +++ b/api/channel.go @@ -45,6 +45,7 @@ func (c Channel) Is(other Channel) bool { var ( TransactionChannel = Channel{"transactions"} HeartbeatChannel = Channel{"heartbeat"} + RollbackChannel = Channel{"rollback"} ) // Action Channel From 5b5b28669a4474a29761b226ed1df1d6d7f276fc Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 7 Jan 2024 19:26:30 +0100 Subject: [PATCH 051/291] api/message/types.go: Adding RollbackMessage struct --- api/message/types.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/api/message/types.go b/api/message/types.go index 5a2111e..081a517 100644 --- a/api/message/types.go +++ b/api/message/types.go @@ -80,3 +80,8 @@ func (act ActionTrace) GetData() (map[string]any, error) { } return nil, errors.New("failed to convert data to map") } + +type RollbackMessage struct { + OldBlockNum uint32 `json:"old_block" msgpack:"old_block"` + NewBlockNum uint32 `json:"new_block" msgpack:"new_block"` +} From a8490f85c4fd11d54d2fa2c402151ba8caba0900 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 7 Jan 2024 19:28:59 +0100 Subject: [PATCH 052/291] app/ship_processor.go: Implement support to send rollback messages on forks. --- app/ship_processor.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/app/ship_processor.go b/app/ship_processor.go index 3fe0901..2aae7b4 100644 --- a/app/ship_processor.go +++ b/app/ship_processor.go @@ -135,6 +135,19 @@ func (processor *ShipProcessor) GetCurrentBlock() uint32 { // Callback function called by shipclient.Stream when a new block arrives. func (processor *ShipProcessor) processBlock(block *ship.GetBlocksResultV0) { + // Check to see if we have a microfork and post a message to + // the rollback channel in that case. + if processor.state.CurrentBlock > 0 && block.ThisBlock.BlockNum < processor.state.CurrentBlock { + log.WithField("old_block", processor.state.CurrentBlock). + WithField("new_block", block.ThisBlock.BlockNum). + Warn("Fork detected, old_block is greater than new_block") + + processor.encodeQueue(api.RollbackChannel, message.RollbackMessage{ + OldBlockNum: processor.state.CurrentBlock, + NewBlockNum: block.ThisBlock.BlockNum, + }) + } + processor.state.CurrentBlock = block.ThisBlock.BlockNum if block.ThisBlock.BlockNum%100 == 0 { From ebeef44c31614a8c429a41d01e40ef58f0f57e62 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Mon, 15 Jan 2024 22:29:27 +0100 Subject: [PATCH 053/291] cmd/tools/redis-acl.go: should not call rand.Seed(), instead create a new rand.Rand object. --- cmd/tools/redis-acl.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cmd/tools/redis-acl.go b/cmd/tools/redis-acl.go index 049f7ce..390042a 100644 --- a/cmd/tools/redis-acl.go +++ b/cmd/tools/redis-acl.go @@ -13,6 +13,8 @@ import ( "github.com/urfave/cli/v2" ) +var rnd *rand.Rand + // Helper struct representing a redis user. type User struct { // Username @@ -54,7 +56,7 @@ func randomString(length int) string { charset := "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUWXYZ0123456789" out := "" for i := 0; i < length; i++ { - idx := rand.Intn(len(charset)) + idx := rnd.Intn(len(charset)) out += string(charset[idx]) } return out @@ -133,7 +135,7 @@ var RedisACLCmd = &cli.Command{ var err error var out *os.File = os.Stdout - rand.Seed(time.Now().Unix()) + rnd = rand.New(rand.NewSource(time.Now().Unix())) defaultUser := NewUser("default", ctx.String("default-pw")) serverUser := NewUser(ctx.String("server"), ctx.String("server-pw")) From 1f3e4a0fd94e39717d7aaf0bded39bf9f0f07058 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Mon, 15 Jan 2024 22:30:43 +0100 Subject: [PATCH 054/291] cmd/tools/redis-acl.go: Make timestamp used as the random seed have nanoseconds precision. --- cmd/tools/redis-acl.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/tools/redis-acl.go b/cmd/tools/redis-acl.go index 390042a..4f2f2aa 100644 --- a/cmd/tools/redis-acl.go +++ b/cmd/tools/redis-acl.go @@ -135,7 +135,7 @@ var RedisACLCmd = &cli.Command{ var err error var out *os.File = os.Stdout - rnd = rand.New(rand.NewSource(time.Now().Unix())) + rnd = rand.New(rand.NewSource(time.Now().UnixNano())) defaultUser := NewUser("default", ctx.String("default-pw")) serverUser := NewUser(ctx.String("server"), ctx.String("server-pw")) From 3866d5382ca8ad7e943e3a230ccf457ded27b7d9 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Mon, 15 Jan 2024 23:16:33 +0100 Subject: [PATCH 055/291] app/ship_processor.go: fix a bug where GlobalSequence was passed as ActionTrace.Receipt.RecvSequence --- app/ship_processor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/ship_processor.go b/app/ship_processor.go index 2aae7b4..3e85648 100644 --- a/app/ship_processor.go +++ b/app/ship_processor.go @@ -231,7 +231,7 @@ func (processor *ShipProcessor) processBlock(block *ship.GetBlocksResultV0) { Receiver: receipt.Receiver.String(), ActDigest: receipt.ActDigest.String(), GlobalSequence: receipt.GlobalSequence, - RecvSequence: receipt.GlobalSequence, + RecvSequence: receipt.RecvSequence, CodeSequence: uint32(receipt.CodeSequence), ABISequence: uint32(receipt.ABISequence), } From dc6dd6ae701d097c83aad128eae44d2b768d70e5 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 7 Jan 2024 19:12:39 +0100 Subject: [PATCH 056/291] api/message/types.go: Adding TableDelta and TableDeltaRow structs --- api/message/types.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/api/message/types.go b/api/message/types.go index 5a2111e..f9442e0 100644 --- a/api/message/types.go +++ b/api/message/types.go @@ -80,3 +80,16 @@ func (act ActionTrace) GetData() (map[string]any, error) { } return nil, errors.New("failed to convert data to map") } + +type TableDeltaRow struct { + Present bool `json:"present" msgpack:"present"` + Data []byte `json:"data" msgpack:"data"` + RawData []byte `json:"raw_data" msgpack:"raw_data"` +} + +type TableDelta struct { + BlockNum uint32 `json:"blocknum" msgpack:"blocknum"` + Timestamp time.Time `json:"blocktimestamp" msgpack:"blocktimestamp"` + Name string `json:"name" msgpack:"name"` + Rows []TableDeltaRow `json:"rows" msgpack:"rows"` +} From b3961f4eecf1bdea5bd3171f16cb3cf9f594c6d6 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 21 Jan 2024 12:52:17 +0100 Subject: [PATCH 057/291] api/message/types.go: Update TableDeltaRow --- api/message/types.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/message/types.go b/api/message/types.go index f9442e0..6d75bd6 100644 --- a/api/message/types.go +++ b/api/message/types.go @@ -82,9 +82,9 @@ func (act ActionTrace) GetData() (map[string]any, error) { } type TableDeltaRow struct { - Present bool `json:"present" msgpack:"present"` - Data []byte `json:"data" msgpack:"data"` - RawData []byte `json:"raw_data" msgpack:"raw_data"` + Present bool `json:"present" msgpack:"present"` + Data map[string]any `json:"data" msgpack:"data"` + RawData []byte `json:"raw_data" msgpack:"raw_data"` } type TableDelta struct { From 2268d19b08e57f5f27ceaeaf02c2f98f234b2537 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 21 Jan 2024 13:13:03 +0100 Subject: [PATCH 058/291] app/ship_processor.go: Store abi from ship in processor. --- app/ship_processor.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/app/ship_processor.go b/app/ship_processor.go index 3fe0901..65711cb 100644 --- a/app/ship_processor.go +++ b/app/ship_processor.go @@ -52,6 +52,9 @@ type ShipProcessor struct { // System contract ("eosio" per default) syscontract eos.AccountName + + // ABI Returned from SHIP + shipABI *eos.ABI } // SpawnProcessor creates a new ShipProccessor that consumes the shipclient.Stream passed to it. @@ -69,6 +72,7 @@ func SpawnProccessor(shipStream *shipclient.Stream, loader StateLoader, saver St // Attach handlers shipStream.BlockHandler = processor.processBlock + shipStream.InitHandler = processor.initHandler // Needed because if nil, traces will not be included in the response from ship. shipStream.TraceHandler = func([]*ship.TransactionTraceV0) {} @@ -76,6 +80,10 @@ func SpawnProccessor(shipStream *shipclient.Stream, loader StateLoader, saver St return processor } +func (processor *ShipProcessor) initHandler(abi *eos.ABI) { + processor.shipABI = abi +} + func (processor *ShipProcessor) queueMessage(channel api.Channel, payload []byte) bool { err := processor.writer.Write(channel, payload) if err != nil { From 5249dc1f50503c7ab35c83c7f4e40e0cbc6b0fb5 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 21 Jan 2024 13:14:39 +0100 Subject: [PATCH 059/291] app/ship_processor.go: process table deltas. --- app/ship_processor.go | 48 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/app/ship_processor.go b/app/ship_processor.go index 65711cb..6e37513 100644 --- a/app/ship_processor.go +++ b/app/ship_processor.go @@ -76,6 +76,7 @@ func SpawnProccessor(shipStream *shipclient.Stream, loader StateLoader, saver St // Needed because if nil, traces will not be included in the response from ship. shipStream.TraceHandler = func([]*ship.TransactionTraceV0) {} + shipStream.TableDeltaHandler = func([]*ship.TableDeltaV0) {} return processor } @@ -282,6 +283,53 @@ func (processor *ShipProcessor) processBlock(block *ship.GetBlocksResultV0) { } } + // Process deltas + for _, delta := range block.Deltas.AsTableDeltasV0() { + + logger := log.WithField("type", "table_delta").WithField("table", delta.Name).Dup() + + rows := []message.TableDeltaRow{} + for _, row := range delta.Rows { + + msg := message.TableDeltaRow{ + Present: row.Present, + RawData: row.Data, + } + + if processor.shipABI != nil { + v, err := processor.shipABI.DecodeTableRowTyped(delta.Name, row.Data) + if err == nil { + err = json.Unmarshal(v, &msg.Data) + if err != nil { + logger.WithError(err).Error("Failed to decode json") + } + } else { + logger.Error("Failed to decode table delta") + } + } else { + logger.Warn("No SHIP ABI present") + } + + rows = append(rows, msg) + } + + message := message.TableDelta{ + BlockNum: block.Block.BlockNumber(), + Timestamp: block.Block.Timestamp.Time.UTC(), + Name: delta.Name, + Rows: rows, + } + + channels := []api.Channel{ + api.TableDeltaChannel{}.Channel(), + api.TableDeltaChannel{Name: delta.Name}.Channel(), + } + + for _, channel := range channels { + processor.encodeQueue(channel, message) + } + } + err := processor.writer.Flush() if err != nil { log.WithError(err).Error("Failed to send messages") From a3624d131e56cc71aba3cd0c3024549f63306046 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 21 Jan 2024 14:06:08 +0100 Subject: [PATCH 060/291] app/ship_processor.go: improve logging info. --- app/ship_processor.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/app/ship_processor.go b/app/ship_processor.go index 96d9ee1..1f0abf1 100644 --- a/app/ship_processor.go +++ b/app/ship_processor.go @@ -173,11 +173,13 @@ func (processor *ShipProcessor) processBlock(block *ship.GetBlocksResultV0) { processor.encodeQueue(api.HeartbeatChannel, hb) } + mainLogger := log.WithField("block", block.ThisBlock.BlockNum).Dup() + // Process traces if block.Traces != nil && len(block.Traces.Elem) > 0 { for _, trace := range block.Traces.AsTransactionTracesV0() { - logger := log.WithField("tx_id", trace.ID.String()).Dup() + logger := mainLogger.WithField("type", "trace").WithField("tx_id", trace.ID.String()).Dup() transaction := message.TransactionTrace{ ID: trace.ID.String(), @@ -299,7 +301,7 @@ func (processor *ShipProcessor) processBlock(block *ship.GetBlocksResultV0) { // Process deltas for _, delta := range block.Deltas.AsTableDeltasV0() { - logger := log.WithField("type", "table_delta").WithField("table", delta.Name).Dup() + logger := mainLogger.WithField("type", "table_delta").WithField("table", delta.Name).Dup() rows := []message.TableDeltaRow{} for _, row := range delta.Rows { From 062bdae620bf13743c17c0d674fa2ed62d4bd3b1 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 21 Jan 2024 14:30:31 +0100 Subject: [PATCH 061/291] Version 0.3.0 --- Makefile | 2 +- debian/changelog | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index d7f64d4..b8178f0 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ GO=go GOLDFLAGS=-v -s -w -X main.VersionString=$(PROGRAM_VERSION) GOBUILDFLAGS=-v -p $(shell nproc) -ldflags="$(GOLDFLAGS)" PROGRAM=thalos-server -PROGRAM_VERSION=0.2.1 +PROGRAM_VERSION=0.3.0 PREFIX=/usr/local BINDIR=$(PREFIX)/bin CFGDIR=$(PREFIX)/etc/thalos diff --git a/debian/changelog b/debian/changelog index 6744a4b..53c240a 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,15 @@ +thalos (0.3.0) bionic focal jammy; urgency=medium + + * Feature: Table Delta message (issue https://github.com/eosswedenorg/thalos/issues/31) + * Feature: Rollback message (issue https://github.com/eosswedenorg/thalos/issues/30) + * Bug: Fix a bug where GlobalSequence was passed as ActionTrace.Receipt.RecvSequence + * Bug: fixed random password generator in redis-acl tool to not produce + the same password if executed during the same second. + + * Security: update golang.org/x/crypto from 0.14.0 to 0.17.0 + + -- Henrik Hautakoski Sun, 21 Jan 2024 14:29:23 +0100 + thalos (0.2.2) bionic focal jammy; urgency=medium * Current processing state (block number) is now cached and loaded on start. From 9e47ae54e7a11cd80f3fe18f1ba5b823e66a1a7c Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Mon, 22 Jan 2024 13:51:24 +0100 Subject: [PATCH 062/291] debian/patches/0001-fix-config-logpath.patch: fix correct path. --- debian/patches/0001-fix-config-logpath.patch | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/patches/0001-fix-config-logpath.patch b/debian/patches/0001-fix-config-logpath.patch index 801fd2b..d78e5d9 100644 --- a/debian/patches/0001-fix-config-logpath.patch +++ b/debian/patches/0001-fix-config-logpath.patch @@ -7,6 +7,6 @@ index 9f4272a..8daac0a 100644 filename: thalos.log # Directory to store the logfiles in. - directory: logs -+ directory: /var/log ++ directory: /var/log/thalos # Format to rename log files when rotating time_format: 2006-01-02_150405 \ No newline at end of file From a2491aa355c2c9fca4a782e2c516df3280487552 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Mon, 22 Jan 2024 14:39:14 +0100 Subject: [PATCH 063/291] install.sh: improve error checking. --- install.sh | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/install.sh b/install.sh index a7126e1..fe4a358 100755 --- a/install.sh +++ b/install.sh @@ -7,10 +7,28 @@ fi INSTALL_DIR=$1 -echo "Installing application in: $INSTALL_DIR" +echo -e "\033[34m-\033[0m Installing application in: $INSTALL_DIR" + +PROGRAMS=(make go) + +missing_prog=0 +for prog in ${PROGRAMS[@]}; do + CMD=$(which $prog) + if [ -z "$CMD" ]; then + echo -e "\033[31m!!\033[0m Failed to locate $prog, please install this program" + missing_prog=1 + fi +done + +if [ $missing_prog -ne 0 ]; then + exit 1 +fi mkdir -p "$INSTALL_DIR"/{bin,logs} - make -e DESTDIR=$INSTALL_DIR PREFIX= CFGDIR= install install-scripts +if [ $? -ne 0 ]; then + echo -e "\033[31m!!\033[0m Installation failed" + exit 1 +fi -echo "Done" +echo -e "\033[32m*\033[0m Done" From 02923aadc6f3560173df4cc7e5e083a0f91f0ae4 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Mon, 22 Jan 2024 14:43:02 +0100 Subject: [PATCH 064/291] install.sh: remove $INSTALL_DIR if installation failed. --- install.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/install.sh b/install.sh index fe4a358..a1d4c12 100755 --- a/install.sh +++ b/install.sh @@ -27,6 +27,7 @@ fi mkdir -p "$INSTALL_DIR"/{bin,logs} make -e DESTDIR=$INSTALL_DIR PREFIX= CFGDIR= install install-scripts if [ $? -ne 0 ]; then + rm -fr "$INSTALL_DIR" echo -e "\033[31m!!\033[0m Installation failed" exit 1 fi From 6da433dc4d68ff33d59fc16dda78643317849de1 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Mon, 22 Jan 2024 14:47:56 +0100 Subject: [PATCH 065/291] Version 0.3.1 --- Makefile | 2 +- debian/changelog | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index b8178f0..1bedf74 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ GO=go GOLDFLAGS=-v -s -w -X main.VersionString=$(PROGRAM_VERSION) GOBUILDFLAGS=-v -p $(shell nproc) -ldflags="$(GOLDFLAGS)" PROGRAM=thalos-server -PROGRAM_VERSION=0.3.0 +PROGRAM_VERSION=0.3.1 PREFIX=/usr/local BINDIR=$(PREFIX)/bin CFGDIR=$(PREFIX)/etc/thalos diff --git a/debian/changelog b/debian/changelog index 53c240a..2701dcf 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,10 @@ +thalos (0.3.1) bionic focal jammy; urgency=medium + + * Bug: fix incorrect log path in debian packages. + * Improved install.sh script. + + -- Henrik Hautakoski Mon, 22 Jan 2024 14:46:48 +0100 + thalos (0.3.0) bionic focal jammy; urgency=medium * Feature: Table Delta message (issue https://github.com/eosswedenorg/thalos/issues/31) From 2b252ea24af10930b1401e2ec7c37452404d168a Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Thu, 1 Feb 2024 11:27:42 +0100 Subject: [PATCH 066/291] Documentation --- app/cache/memory_store.go | 7 ++++++- app/cache/store.go | 1 + app/ship_processor.go | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/app/cache/memory_store.go b/app/cache/memory_store.go index 0dbc1be..a198a70 100644 --- a/app/cache/memory_store.go +++ b/app/cache/memory_store.go @@ -7,10 +7,15 @@ import ( "time" ) +// Store time function in a variable. +// Makes it easy to travel in time when testing. var now = time.Now type memoryStoreItem struct { - value any + // Actual value stored. + value any + + // Cache expiration time. expired time.Time } diff --git a/app/cache/store.go b/app/cache/store.go index 1b69639..a9d85e8 100644 --- a/app/cache/store.go +++ b/app/cache/store.go @@ -5,6 +5,7 @@ import ( "time" ) +// Interface to a cache storage. type Store interface { // Set an item in the store. Set(ctx context.Context, key string, value any, TTL time.Duration) error diff --git a/app/ship_processor.go b/app/ship_processor.go index 1f0abf1..b750cb5 100644 --- a/app/ship_processor.go +++ b/app/ship_processor.go @@ -74,7 +74,7 @@ func SpawnProccessor(shipStream *shipclient.Stream, loader StateLoader, saver St shipStream.BlockHandler = processor.processBlock shipStream.InitHandler = processor.initHandler - // Needed because if nil, traces will not be included in the response from ship. + // Needed because if nil, traces/table deltas will not be included in the response from ship. shipStream.TraceHandler = func([]*ship.TransactionTraceV0) {} shipStream.TableDeltaHandler = func([]*ship.TableDeltaV0) {} From 569eefd07a32f4c17b21cf22d8b11d190fea6075 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Fri, 2 Feb 2024 16:43:02 +0100 Subject: [PATCH 067/291] go.mod: update dependancies --- go.mod | 26 +++++++++++++------------- go.sum | 48 ++++++++++++++++++++++++------------------------ 2 files changed, 37 insertions(+), 37 deletions(-) diff --git a/go.mod b/go.mod index 6987e44..92490a9 100644 --- a/go.mod +++ b/go.mod @@ -5,18 +5,18 @@ go 1.20 require ( github.com/cenkalti/backoff/v4 v4.2.1 github.com/docker/go-units v0.5.0 - github.com/eoscanada/eos-go v0.10.3-0.20230609180026-5d4ee54fe15e + github.com/eoscanada/eos-go v0.10.3-0.20231109144819-59afdfa3a37d github.com/eosswedenorg-go/antelope-ship-client v0.2.7 github.com/eosswedenorg-go/pid v1.0.1 - github.com/eosswedenorg/thalos/api v0.2.1 + github.com/eosswedenorg/thalos/api v0.3.0 github.com/go-redis/cache/v9 v9.0.0 github.com/go-redis/redismock/v9 v9.2.0 github.com/nikoksr/notify v0.41.0 github.com/pborman/getopt/v2 v2.1.0 - github.com/redis/go-redis/v9 v9.3.0 + github.com/redis/go-redis/v9 v9.4.0 github.com/sirupsen/logrus v1.9.3 github.com/stretchr/testify v1.8.4 - github.com/urfave/cli/v2 v2.25.7 + github.com/urfave/cli/v2 v2.27.1 gopkg.in/yaml.v3 v3.0.1 ) @@ -32,13 +32,13 @@ require ( github.com/google/go-cmp v0.6.0 // indirect github.com/gorilla/websocket v1.5.1 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.17.2 // indirect + github.com/klauspost/compress v1.17.5 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/logrusorgru/aurora v2.0.3+incompatible // indirect github.com/mitchellh/go-testing-interface v1.14.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/onsi/gomega v1.29.0 // indirect + github.com/onsi/gomega v1.31.1 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect @@ -47,19 +47,19 @@ require ( github.com/tidwall/gjson v1.17.0 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect - github.com/ugorji/go/codec v1.2.11 // indirect + github.com/ugorji/go/codec v1.2.12 // indirect github.com/vmihailenco/go-tinylfu v0.2.2 // indirect github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect - github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect + github.com/xrash/smetrics v0.0.0-20231213231151-1d8dd44e695e // indirect go.uber.org/goleak v1.3.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.26.0 // indirect - golang.org/x/crypto v0.17.0 // indirect - golang.org/x/net v0.17.0 // indirect - golang.org/x/sync v0.5.0 // indirect - golang.org/x/sys v0.15.0 // indirect - golang.org/x/term v0.15.0 // indirect + golang.org/x/crypto v0.18.0 // indirect + golang.org/x/net v0.20.0 // indirect + golang.org/x/sync v0.6.0 // indirect + golang.org/x/sys v0.16.0 // indirect + golang.org/x/term v0.16.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect ) diff --git a/go.sum b/go.sum index effd88d..9b4a4af 100644 --- a/go.sum +++ b/go.sum @@ -23,8 +23,8 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/r github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/eoscanada/eos-go v0.10.3-0.20230609180026-5d4ee54fe15e h1:C5V5vgv5ZFf51RW+QmBLR0eOZHvIlLdapAaIWE1pD+Q= -github.com/eoscanada/eos-go v0.10.3-0.20230609180026-5d4ee54fe15e/go.mod h1:L3avCf8OkDrjlUeNy9DdoV67TCmDNj2dSlc5Xp3DNNk= +github.com/eoscanada/eos-go v0.10.3-0.20231109144819-59afdfa3a37d h1:vK5PijzcJaUPOhgWvY9lL99H9t3lrRNHx2IDHqS0ILc= +github.com/eoscanada/eos-go v0.10.3-0.20231109144819-59afdfa3a37d/go.mod h1:L3avCf8OkDrjlUeNy9DdoV67TCmDNj2dSlc5Xp3DNNk= github.com/eosswedenorg-go/antelope-ship-client v0.2.7 h1:ZGrXlGtMBqIV1TCCRjZAk9/RAA7pROO/Q/xN+baG45M= github.com/eosswedenorg-go/antelope-ship-client v0.2.7/go.mod h1:kZ/4gkAIdAq4/WiZlVaSONpELcDCMJQJMmlikLUGCb8= github.com/eosswedenorg-go/jsontime v0.0.0-20230509125027-08422d6236c7 h1:rLPu++RHaxg4WmUOXeWYioZuafWs0PVcYuvzOWbOJjk= @@ -71,8 +71,8 @@ github.com/jordan-wright/email v4.0.1-0.20210109023952-943e75fe5223+incompatible github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4= -github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.17.5 h1:d4vBd+7CHydUqpFBgUEKkSdtSugf9YFmSkvUYPquI5E= +github.com/klauspost/compress v1.17.5/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= @@ -118,8 +118,8 @@ github.com/onsi/gomega v1.22.1/go.mod h1:x6n7VNe4hw0vkyYUM4mjIXx3JbLiPaBPNgB7PRQ github.com/onsi/gomega v1.24.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg= github.com/onsi/gomega v1.24.1/go.mod h1:3AOiACssS3/MajrniINInwbfOOtfZvplPzuRSmvt1jM= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= -github.com/onsi/gomega v1.29.0 h1:KIA/t2t5UBzoirT4H9tsML45GEbo3ouUnBHsCfD2tVg= -github.com/onsi/gomega v1.29.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= +github.com/onsi/gomega v1.31.1 h1:KYppCUK+bUgAZwHOu7EXVBKyQA6ILvOESHkn/tgoqvo= +github.com/onsi/gomega v1.31.1/go.mod h1:y40C95dwAD1Nz36SsEnxvfFe8FFfNxzI5eJ0EYGyAy0= github.com/pborman/getopt/v2 v2.1.0 h1:eNfR+r+dWLdWmV8g5OlpyrTYHkhVNxHBdN2cCrJmOEA= github.com/pborman/getopt/v2 v2.1.0/go.mod h1:4NtW75ny4eBw9fO1bhtNdYTlZKYX5/tBLtsOpwKIKd0= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= @@ -129,8 +129,8 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/redis/go-redis/v9 v9.0.0-rc.4/go.mod h1:Vo3EsyWnicKnSKCA7HhgnvnyA74wOA69Cd2Meli5mmA= -github.com/redis/go-redis/v9 v9.3.0 h1:RiVDjmig62jIWp7Kk4XVLs0hzV6pI3PyTnnL0cnn0u0= -github.com/redis/go-redis/v9 v9.3.0/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= +github.com/redis/go-redis/v9 v9.4.0 h1:Yzoz33UZw9I/mFhx4MNrB6Fk+XHO1VukNcCa1+lwyKk= +github.com/redis/go-redis/v9 v9.4.0/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= @@ -163,10 +163,10 @@ github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JT github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= -github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= -github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= -github.com/urfave/cli/v2 v2.25.7 h1:VAzn5oq403l5pHjc4OhD54+XGO9cdKVL/7lDjF+iKUs= -github.com/urfave/cli/v2 v2.25.7/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= +github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= +github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/urfave/cli/v2 v2.27.1 h1:8xSQ6szndafKVRmfyeUMxkNUJQMjL1F2zmsZ+qHpfho= +github.com/urfave/cli/v2 v2.27.1/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= github.com/vmihailenco/go-tinylfu v0.2.2 h1:H1eiG6HM36iniK6+21n9LLpzx1G9R3DJa2UjUjbynsI= github.com/vmihailenco/go-tinylfu v0.2.2/go.mod h1:CutYi2Q9puTxfcolkliPq4npPuofg9N9t8JVrjzwa3Q= github.com/vmihailenco/msgpack/v5 v5.3.4/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= @@ -174,8 +174,8 @@ github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IU github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok= github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= -github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= -github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= +github.com/xrash/smetrics v0.0.0-20231213231151-1d8dd44e695e h1:+SOyEddqYF09QP7vr7CgJ1eti3pY9Fn3LHO1M1r/0sI= +github.com/xrash/smetrics v0.0.0-20231213231151-1d8dd44e695e/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= @@ -200,8 +200,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= -golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= -golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= +golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -227,16 +227,16 @@ golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= +golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= -golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -263,16 +263,16 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= +golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= -golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= -golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= +golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE= +golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= From e93cc1d139791894cd2a4c42426fd40d9fdd0df8 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Fri, 2 Feb 2024 16:52:31 +0100 Subject: [PATCH 068/291] tools: rename redis prefix flag to just prefix. --- cmd/tools/bench.go | 6 +++--- cmd/tools/flags.go | 10 +++++----- cmd/tools/mock_publisher.go | 4 ++-- cmd/tools/validate.go | 6 +++--- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/cmd/tools/bench.go b/cmd/tools/bench.go index 5d4dcc2..ae23a55 100644 --- a/cmd/tools/bench.go +++ b/cmd/tools/bench.go @@ -26,7 +26,7 @@ var benchCmd = &cli.Command{ redisUserFlag, redisPasswordFlag, redisDbFlag, - redisPrefixFlag, + prefixFlag, chainIdFlag, &cli.DurationFlag{ Name: "interval", @@ -41,7 +41,7 @@ var benchCmd = &cli.Command{ log.WithFields(log.Fields{ "url": ctx.String("redis-url"), - "prefix": ctx.String("redis-prefix"), + "prefix": ctx.String("prefix"), "chain_id": ctx.String("chain_id"), "database": ctx.Int("redis-db"), }).Info("Connecting to redis") @@ -65,7 +65,7 @@ var benchCmd = &cli.Command{ }).Info("Starting benchmark") sub := api_redis.NewSubscriber(context.Background(), rdb, api_redis.Namespace{ - Prefix: ctx.String("redis-prefix"), + Prefix: ctx.String("prefix"), ChainID: ctx.String("chain_id"), }) diff --git a/cmd/tools/flags.go b/cmd/tools/flags.go index 30c5a94..24c0ee5 100644 --- a/cmd/tools/flags.go +++ b/cmd/tools/flags.go @@ -4,11 +4,6 @@ import ( "github.com/urfave/cli/v2" ) -var redisPrefixFlag = &cli.StringFlag{ - Name: "prefix", - Value: "ship", -} - var redisUrlFlag = &cli.StringFlag{ Name: "redis-url", Value: "127.0.0.1:6379", @@ -31,6 +26,11 @@ var redisDbFlag = &cli.IntFlag{ Usage: "What redis database we should connect to.", } +var prefixFlag = &cli.StringFlag{ + Name: "prefix", + Value: "ship", +} + var chainIdFlag = &cli.StringFlag{ Name: "chain_id", Value: "1064487b3cd1a897ce03ae5b6a865651747e2e152090f99c1d19d44e01aea5a4", diff --git a/cmd/tools/mock_publisher.go b/cmd/tools/mock_publisher.go index 7dcb05d..9920468 100644 --- a/cmd/tools/mock_publisher.go +++ b/cmd/tools/mock_publisher.go @@ -23,7 +23,7 @@ var MockPublisherCmd = &cli.Command{ redisUserFlag, redisPasswordFlag, redisDbFlag, - redisPrefixFlag, + prefixFlag, chainIdFlag, &cli.StringFlag{ Name: "codec", @@ -45,7 +45,7 @@ var MockPublisherCmd = &cli.Command{ } ns := api_redis.Namespace{ - Prefix: ctx.String("redis-prefix"), + Prefix: ctx.String("prefix"), ChainID: ctx.String("chain_id"), } publisher := redis_driver.NewPublisher(context.Background(), rdb, ns) diff --git a/cmd/tools/validate.go b/cmd/tools/validate.go index 1f2aa5b..061e35d 100644 --- a/cmd/tools/validate.go +++ b/cmd/tools/validate.go @@ -55,7 +55,7 @@ var validateCmd = &cli.Command{ Flags: []cli.Flag{ redisUrlFlag, redisDbFlag, - redisPrefixFlag, + prefixFlag, chainIdFlag, }, Action: func(ctx *cli.Context) error { @@ -64,7 +64,7 @@ var validateCmd = &cli.Command{ log.WithFields(log.Fields{ "url": ctx.String("redis-url"), - "prefix": ctx.String("redis-prefix"), + "prefix": ctx.String("prefix"), "chain_id": ctx.String("chain_id"), "database": ctx.Int("redis-db"), }).Info("Connecting to redis") @@ -84,7 +84,7 @@ var validateCmd = &cli.Command{ log.Info("Starting validation, following the stream") sub := api_redis.NewSubscriber(context.Background(), rdb, api_redis.Namespace{ - Prefix: ctx.String("redis-prefix"), + Prefix: ctx.String("prefix"), ChainID: ctx.String("chain_id"), }) From cb14978118756526852549b6486b7817922b2821 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Fri, 2 Feb 2024 16:52:50 +0100 Subject: [PATCH 069/291] cmd/tools/mock_publisher.go: add log message before starting. --- cmd/tools/mock_publisher.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/cmd/tools/mock_publisher.go b/cmd/tools/mock_publisher.go index 9920468..7d69e1a 100644 --- a/cmd/tools/mock_publisher.go +++ b/cmd/tools/mock_publisher.go @@ -13,6 +13,7 @@ import ( redis_driver "github.com/eosswedenorg/thalos/app/driver/redis" "github.com/redis/go-redis/v9" + log "github.com/sirupsen/logrus" ) var MockPublisherCmd = &cli.Command{ @@ -44,6 +45,13 @@ var MockPublisherCmd = &cli.Command{ return err } + log.WithFields(log.Fields{ + "url": ctx.String("redis-url"), + "prefix": ctx.String("prefix"), + "chain_id": ctx.String("chain_id"), + "database": ctx.Int("redis-db"), + }).Info("Starting mock publisher") + ns := api_redis.Namespace{ Prefix: ctx.String("prefix"), ChainID: ctx.String("chain_id"), From e69ff9781fd6c85c2a3976f246c546662843bad8 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Fri, 2 Feb 2024 17:05:29 +0100 Subject: [PATCH 070/291] api: update dependancies --- api/go.mod | 8 ++++---- api/go.sum | 25 ++++++++++--------------- 2 files changed, 14 insertions(+), 19 deletions(-) diff --git a/api/go.mod b/api/go.mod index 2831b7d..fe492ef 100644 --- a/api/go.mod +++ b/api/go.mod @@ -5,10 +5,10 @@ go 1.20 require ( github.com/alicebob/miniredis/v2 v2.30.2 github.com/eosswedenorg-go/jsontime v0.0.0-20230509125027-08422d6236c7 - github.com/go-redis/redismock/v9 v9.0.3 - github.com/redis/go-redis/v9 v9.0.5 - github.com/stretchr/testify v1.8.2 - github.com/ugorji/go/codec v1.2.11 + github.com/go-redis/redismock/v9 v9.2.0 + github.com/redis/go-redis/v9 v9.4.0 + github.com/stretchr/testify v1.8.4 + github.com/ugorji/go/codec v1.2.12 ) require ( diff --git a/api/go.sum b/api/go.sum index 54abd34..7c06dcc 100644 --- a/api/go.sum +++ b/api/go.sum @@ -2,8 +2,8 @@ github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a h1:HbKu58rmZp github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc= github.com/alicebob/miniredis/v2 v2.30.2 h1:lc1UAUT9ZA7h4srlfBmBt2aorm5Yftk9nBjxz7EyY9I= github.com/alicebob/miniredis/v2 v2.30.2/go.mod h1:b25qWj4fCEsBeAAR2mlb0ufImGC6uH3VlUfb/HS5zKg= -github.com/bsm/ginkgo/v2 v2.7.0 h1:ItPMPH90RbmZJt5GtkcNvIRuGEdwlBItdNVoyzaNQao= -github.com/bsm/gomega v1.26.0 h1:LhQm+AFcgV2M0WyKroMASzAzCAJVpAxQXv4SaI9a69Y= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= @@ -17,8 +17,8 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cu github.com/eosswedenorg-go/jsontime v0.0.0-20230509125027-08422d6236c7 h1:rLPu++RHaxg4WmUOXeWYioZuafWs0PVcYuvzOWbOJjk= github.com/eosswedenorg-go/jsontime v0.0.0-20230509125027-08422d6236c7/go.mod h1:eNUkVOymzgl0lViUhmm08PkutzqLnOQ6Dr+RUnf+Mq0= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= -github.com/go-redis/redismock/v9 v9.0.3 h1:mtHQi2l51lCmXIbTRTqb1EiHYe9tL5Yk5oorlSJJqR0= -github.com/go-redis/redismock/v9 v9.0.3/go.mod h1:F6tJRfnU8R/NZ0E+Gjvoluk14MqMC5ueSZX6vVQypc0= +github.com/go-redis/redismock/v9 v9.2.0 h1:ZrMYQeKPECZPjOj5u9eyOjg8Nnb0BS9lkVIZ6IpsKLw= +github.com/go-redis/redismock/v9 v9.2.0/go.mod h1:18KHfGDK4Y6c2R0H38EUGWAdc7ZQS9gfYxc94k7rWT0= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -33,18 +33,14 @@ github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/redis/go-redis/v9 v9.0.5 h1:CuQcn5HIEeK7BgElubPP8CGtE0KakrnbBSTLjathl5o= -github.com/redis/go-redis/v9 v9.0.5/go.mod h1:WqMKv5vnQbRuZstUwxQI195wHy+t4PuXDOjzMvcuQHk= +github.com/redis/go-redis/v9 v9.4.0 h1:Yzoz33UZw9I/mFhx4MNrB6Fk+XHO1VukNcCa1+lwyKk= +github.com/redis/go-redis/v9 v9.4.0/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= -github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= -github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= +github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/yuin/gopher-lua v1.1.0 h1:BojcDhfyDWgU2f2TOzYK/g5p2gxMrku8oupLDqlnSqE= github.com/yuin/gopher-lua v1.1.0/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw= golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw= @@ -54,6 +50,5 @@ golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From 0d8ecaea367f73c3e34600edc85a99db1b35ce35 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sat, 3 Feb 2024 13:39:25 +0100 Subject: [PATCH 071/291] api/client.go: some comments. --- api/client.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/api/client.go b/api/client.go index 023499f..a52e6c5 100644 --- a/api/client.go +++ b/api/client.go @@ -14,6 +14,7 @@ type Client struct { reader Reader decoder message.Decoder + // waitgroup for worker threads. wg sync.WaitGroup OnError func(error) @@ -42,6 +43,7 @@ func (c *Client) worker(channel Channel, h handler) { } } +// Action handler func (c *Client) actHandler(payload []byte) { var act message.ActionTrace if err := c.decoder(payload, &act); err != nil { @@ -53,6 +55,7 @@ func (c *Client) actHandler(payload []byte) { c.OnAction(act) } +// HeartBeat handler func (c *Client) hbHandler(payload []byte) { var hb message.HeartBeat if err := c.decoder(payload, &hb); err != nil { @@ -87,6 +90,7 @@ func (c *Client) Subscribe(channel Channel) error { } func (c *Client) Run() { + // Just wait for workers to complete. c.wg.Wait() } From bd742e678ae2d1cdc93e4dd53a1627f33ec70b5e Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sat, 3 Feb 2024 13:40:25 +0100 Subject: [PATCH 072/291] api/message/json/codec_test.go: add tests for table delta messages. --- api/message/json/codec_test.go | 65 ++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/api/message/json/codec_test.go b/api/message/json/codec_test.go index b657eb4..7e0aec0 100644 --- a/api/message/json/codec_test.go +++ b/api/message/json/codec_test.go @@ -239,3 +239,68 @@ func TestJson_DecodeTransactionTrace(t *testing.T) { assert.NoError(t, err) assert.Equal(t, expected, msg) } + +func TestJson_EncodeTableDelta(t *testing.T) { + msg := message.TableDelta{ + BlockNum: 2736712863, + Timestamp: time.Unix(1724791632, 0).UTC(), + Name: "contract_row", + Rows: []message.TableDeltaRow{ + { + Present: true, + Data: map[string]any{ + "string": "value", + "int": 1234, + }, + RawData: []byte{0x23, 0x54, 0xF2, 0xab, 0xeb}, + }, + { + Present: false, + Data: map[string]any{ + "string": "value", + "int": 1234, + }, + RawData: []byte{0x23, 0xe2, 0x3c, 0xb9}, + }, + }, + } + + expected := `{"blocknum":2736712863,"blocktimestamp":"2024-08-27T20:47:12.000","name":"contract_row","rows":[{"present":true,"data":{"int":1234,"string":"value"},"raw_data":"I1Tyq+s="},{"present":false,"data":{"int":1234,"string":"value"},"raw_data":"I+I8uQ=="}]}` + + data, err := createCodec().Encoder(msg) + assert.NoError(t, err) + assert.Equal(t, expected, string(data)) +} + +func TestJson_DecodeTableDelta(t *testing.T) { + data := `{"blocknum":2787282732,"blocktimestamp":"2014-02-18T04:20:43.500","name":"contract_row","rows":[{"present":true,"data":{"id":2,"name":"some_name"},"raw_data":"XbosHA1WDbI="},{"present":false,"data":{"id":1234,"name":"some_other_name"},"raw_data":"KNQA1g=="}]}` + + expected := message.TableDelta{ + BlockNum: 2787282732, + Timestamp: time.Date(2014, time.February, 18, 4, 20, 43, 500000000, time.UTC), + Name: "contract_row", + Rows: []message.TableDeltaRow{ + { + Present: true, + Data: map[string]any{ + "name": "some_name", + "id": float64(2), + }, + RawData: []byte{0x5d, 0xba, 0x2c, 0x1c, 0x0d, 0x56, 0x0d, 0xb2}, + }, + { + Present: false, + Data: map[string]any{ + "name": "some_other_name", + "id": float64(1234), + }, + RawData: []byte{0x28, 0xd4, 0x00, 0xd6}, + }, + }, + } + + actual := message.TableDelta{} + err := createCodec().Decoder([]byte(data), &actual) + assert.NoError(t, err) + assert.Equal(t, expected, actual) +} From 9143c1e06bfee9ec9c252ba23165383a72c8a4fd Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sat, 3 Feb 2024 14:06:47 +0100 Subject: [PATCH 073/291] api/message/msgpack/codec_test.go: adding test for table delta messages. --- api/message/msgpack/codec_test.go | 69 +++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/api/message/msgpack/codec_test.go b/api/message/msgpack/codec_test.go index 71d47c7..35be0d1 100644 --- a/api/message/msgpack/codec_test.go +++ b/api/message/msgpack/codec_test.go @@ -411,3 +411,72 @@ func TestMsgpack_DecodeTransactionTrace(t *testing.T) { assert.Equal(t, expected, res) } + +func TestMsgpack_EncodeTableDelta(t *testing.T) { + msg := message.TableDelta{ + BlockNum: 6347293, + Timestamp: time.Date(1998, time.December, 4, 8, 54, 35, 500, time.UTC), + Name: "contract_row", + Rows: []message.TableDeltaRow{ + { + Present: true, + Data: map[string]any{ + "id": 2213, + "name": "Freddie Mercury", + "band": "Queen", + }, + RawData: []byte{0x23, 0x13, 0xe2}, + }, + { + Present: false, + Data: map[string]any{ + "id": 27182, + "name": "Eddie Van Halen", + "band": "Van Halen", + }, + RawData: []byte{0xfe, 0x4e, 0x52, 0x05}, + }, + }, + } + + expected := "\x84\xa8blocknum\xce\x00`\xda\x1d\xaeblocktimestamp\xd7\xff\x00\x00\a\xd06g\xa3K\xa4name\xaccontract_row\xa4rows\x92\x83\xa4data\x83\xa4band\xa5Queen\xa2id\xd1\b\xa5\xa4name\xafFreddie Mercury\xa7presentèraw_data\xc4\x03#\x13⃤data\x83\xa4band\xa9Van Halen\xa2id\xd1j.\xa4name\xafEddie Van Halen\xa7present¨raw_data\xc4\x04\xfeNR\x05" + + actual, err := createCodec().Encoder(msg) + assert.NoError(t, err) + assert.Equal(t, expected, string(actual)) +} + +func TestMsgpack_DecodeTableDelta(t *testing.T) { + data := []byte("\x84\xa8blocknum\xce\x00`\xda\x1d\xaeblocktimestamp\xd7\xff\x00\x00\a\xd06g\xa3K\xa4name\xaccontract_row\xa4rows\x92\x83\xa4data\x83\xa4band\xa5Queen\xa2id\xd1\b\xa5\xa4name\xafFreddie Mercury\xa7presentèraw_data\xc4\x03#\x13⃤data\x83\xa4band\xa9Van Halen\xa2id\xd1j.\xa4name\xafEddie Van Halen\xa7present¨raw_data\xc4\x04\xfeNR\x05") + + expected := message.TableDelta{ + BlockNum: 6347293, + Timestamp: time.Date(1998, time.December, 4, 8, 54, 35, 500, time.UTC), + Name: "contract_row", + Rows: []message.TableDeltaRow{ + { + Present: true, + Data: map[string]any{ + "id": int64(2213), + "name": "Freddie Mercury", + "band": "Queen", + }, + RawData: []byte{0x23, 0x13, 0xe2}, + }, + { + Present: false, + Data: map[string]any{ + "id": int64(27182), + "name": "Eddie Van Halen", + "band": "Van Halen", + }, + RawData: []byte{0xfe, 0x4e, 0x52, 0x05}, + }, + }, + } + actual := message.TableDelta{} + err := createCodec().Decoder(data, &actual) + assert.NoError(t, err) + + assert.Equal(t, expected, actual) +} From e7ad37c38a26214d1ea0835392b62bf79d8147ab Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 4 Feb 2024 11:32:28 +0100 Subject: [PATCH 074/291] api/client.go: add handler for table delta --- api/client.go | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/api/client.go b/api/client.go index a52e6c5..e76b2cc 100644 --- a/api/client.go +++ b/api/client.go @@ -17,9 +17,10 @@ type Client struct { // waitgroup for worker threads. wg sync.WaitGroup - OnError func(error) - OnAction func(message.ActionTrace) - OnHeartbeat func(message.HeartBeat) + OnError func(error) + OnAction func(message.ActionTrace) + OnHeartbeat func(message.HeartBeat) + OnTableDelta func(message.TableDelta) } func NewClient(reader Reader, decoder message.Decoder) *Client { @@ -55,6 +56,18 @@ func (c *Client) actHandler(payload []byte) { c.OnAction(act) } +// TableDelta handler +func (c *Client) tableDeltaHandler(payload []byte) { + td := message.TableDelta{} + if err := c.decoder(payload, &td); err != nil { + if c.OnError != nil { + c.OnError(err) + } + return + } + c.OnTableDelta(td) +} + // HeartBeat handler func (c *Client) hbHandler(payload []byte) { var hb message.HeartBeat @@ -75,6 +88,8 @@ func (c *Client) Subscribe(channel Channel) error { handler = c.hbHandler case ActionChannel{}.Channel().Type(): handler = c.actHandler + case TableDeltaChannel{}.Channel().Type(): + handler = c.tableDeltaHandler default: return fmt.Errorf("invalid channel type. %s", t) } From bcd4a93c46497e701fdbe1d989200c38a79e2dbc Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 4 Feb 2024 14:30:59 +0100 Subject: [PATCH 075/291] api/client_test.go: add test for calling Subscribe with nil handler. --- api/client_test.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/api/client_test.go b/api/client_test.go index 8fae00e..2b28b11 100644 --- a/api/client_test.go +++ b/api/client_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/eosswedenorg/thalos/api/message" + "github.com/stretchr/testify/assert" ) type mockReader struct{} @@ -48,3 +49,13 @@ func TestClient_Subscribe(t *testing.T) { }) } } + +func TestClient_SubscribeWithNilHandler(t *testing.T) { + client := NewClient(nil, nil) + client.OnAction = mockActionHandler + client.OnHeartbeat = mockHbHandler + + err := client.Subscribe(TableDeltaChannel{Name: "name"}.Channel()) + + assert.Error(t, err) +} From 35a9706954638cd1879f7ad291a5631f5dd57d88 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 4 Feb 2024 14:33:23 +0100 Subject: [PATCH 076/291] api/client.go: in Subscribe() handle nil handler correctly. --- api/client.go | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/api/client.go b/api/client.go index e76b2cc..e029e7f 100644 --- a/api/client.go +++ b/api/client.go @@ -2,6 +2,7 @@ package api import ( "fmt" + "reflect" "sync" "github.com/eosswedenorg/thalos/api/message" @@ -81,24 +82,30 @@ func (c *Client) hbHandler(payload []byte) { } func (c *Client) Subscribe(channel Channel) error { - var handler handler + handlers := map[string]struct { + handler handler + callback any + }{ + HeartbeatChannel.Type(): {c.hbHandler, c.OnHeartbeat}, + ActionChannel{}.Channel().Type(): {c.actHandler, c.OnAction}, + TableDeltaChannel{}.Channel().Type(): {c.tableDeltaHandler, c.OnTableDelta}, + } - switch t := channel.Type(); t { - case HeartbeatChannel.Type(): - handler = c.hbHandler - case ActionChannel{}.Channel().Type(): - handler = c.actHandler - case TableDeltaChannel{}.Channel().Type(): - handler = c.tableDeltaHandler - default: - return fmt.Errorf("invalid channel type. %s", t) + h, ok := handlers[channel.Type()] + + if !ok { + return fmt.Errorf("invalid channel type. %s", channel.Type()) + } + + if h.callback == nil || reflect.ValueOf(h.callback).IsNil() { + return fmt.Errorf("please set an handler before calling Subscribe") } // Start a worker for this channel. c.wg.Add(1) go func() { defer c.wg.Done() - c.worker(channel, handler) + c.worker(channel, h.handler) }() return nil From 728b03422fc19440514c8fab1f9231b1c755ed1b Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 4 Feb 2024 14:48:54 +0100 Subject: [PATCH 077/291] api/redis/subscriber.go: adding some comments. --- api/redis/subscriber.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/api/redis/subscriber.go b/api/redis/subscriber.go index 9a8e8fe..1a1c91a 100644 --- a/api/redis/subscriber.go +++ b/api/redis/subscriber.go @@ -11,8 +11,10 @@ import ( ) type Subscriber struct { - sub *redis.PubSub - ctx context.Context + sub *redis.PubSub + ctx context.Context + + // Mutex for channels map. mu sync.RWMutex timeout time.Duration channels map[string]chan []byte @@ -94,11 +96,15 @@ func (s *Subscriber) Read(channel api.Channel) ([]byte, error) { } func (s *Subscriber) Close() error { + // Close redis pubsub. err := s.sub.Close() + // Close all go channels, this will make Read() unblock. for _, ch := range s.channels { close(ch) } + + // Clear the channel map of old channels. s.mu.Lock() s.channels = make(map[string]chan []byte) s.mu.Unlock() From 38f885f0dfceec748ce7c212afa9bf0c4e91fb5d Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 4 Feb 2024 15:54:10 +0100 Subject: [PATCH 078/291] api/client.go: refactor decoding in handler functions to a helper function --- api/client.go | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/api/client.go b/api/client.go index e029e7f..4b5a9ff 100644 --- a/api/client.go +++ b/api/client.go @@ -45,40 +45,40 @@ func (c *Client) worker(channel Channel, h handler) { } } -// Action handler -func (c *Client) actHandler(payload []byte) { - var act message.ActionTrace - if err := c.decoder(payload, &act); err != nil { +// Helper method to decode a message and call OnError on error. +// Returns true if successfull. false otherwise +func (c *Client) decode(payload []byte, msg any) bool { + if err := c.decoder(payload, msg); err != nil { if c.OnError != nil { c.OnError(err) } - return + return false + } + return true +} + +// Action handler +func (c *Client) actHandler(payload []byte) { + var act message.ActionTrace + if ok := c.decode(payload, &act); ok { + c.OnAction(act) } - c.OnAction(act) } // TableDelta handler func (c *Client) tableDeltaHandler(payload []byte) { td := message.TableDelta{} - if err := c.decoder(payload, &td); err != nil { - if c.OnError != nil { - c.OnError(err) - } - return + if ok := c.decode(payload, &td); ok { + c.OnTableDelta(td) } - c.OnTableDelta(td) } // HeartBeat handler func (c *Client) hbHandler(payload []byte) { var hb message.HeartBeat - if err := c.decoder(payload, &hb); err != nil { - if c.OnError != nil { - c.OnError(err) - } - return + if ok := c.decode(payload, &hb); ok { + c.OnHeartbeat(hb) } - c.OnHeartbeat(hb) } func (c *Client) Subscribe(channel Channel) error { From 021c71f50ba9e47b9f4477d2c9932df9eb3a8144 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 4 Feb 2024 16:05:01 +0100 Subject: [PATCH 079/291] api/client.go: add Transaction callback. --- api/client.go | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/api/client.go b/api/client.go index 4b5a9ff..ae6130d 100644 --- a/api/client.go +++ b/api/client.go @@ -18,10 +18,11 @@ type Client struct { // waitgroup for worker threads. wg sync.WaitGroup - OnError func(error) - OnAction func(message.ActionTrace) - OnHeartbeat func(message.HeartBeat) - OnTableDelta func(message.TableDelta) + OnError func(error) + OnTransaction func(message.TransactionTrace) + OnAction func(message.ActionTrace) + OnHeartbeat func(message.HeartBeat) + OnTableDelta func(message.TableDelta) } func NewClient(reader Reader, decoder message.Decoder) *Client { @@ -57,6 +58,14 @@ func (c *Client) decode(payload []byte, msg any) bool { return true } +// Transaction handler +func (c *Client) transactionHandler(payload []byte) { + var trans message.TransactionTrace + if ok := c.decode(payload, &trans); ok { + c.OnTransaction(trans) + } +} + // Action handler func (c *Client) actHandler(payload []byte) { var act message.ActionTrace @@ -86,6 +95,7 @@ func (c *Client) Subscribe(channel Channel) error { handler handler callback any }{ + TransactionChannel.Type(): {c.transactionHandler, c.OnTransaction}, HeartbeatChannel.Type(): {c.hbHandler, c.OnHeartbeat}, ActionChannel{}.Channel().Type(): {c.actHandler, c.OnAction}, TableDeltaChannel{}.Channel().Type(): {c.tableDeltaHandler, c.OnTableDelta}, From 9d79bb8315103768d923d20ea608f93ebc350a37 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 4 Feb 2024 16:55:21 +0100 Subject: [PATCH 080/291] api/client.go: add rollback callback --- api/client.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/api/client.go b/api/client.go index ae6130d..a426d3d 100644 --- a/api/client.go +++ b/api/client.go @@ -19,6 +19,7 @@ type Client struct { wg sync.WaitGroup OnError func(error) + OnRollback func(message.RollbackMessage) OnTransaction func(message.TransactionTrace) OnAction func(message.ActionTrace) OnHeartbeat func(message.HeartBeat) @@ -58,6 +59,14 @@ func (c *Client) decode(payload []byte, msg any) bool { return true } +// Rollback handler +func (c *Client) rollbackHandler(payload []byte) { + var rb message.RollbackMessage + if ok := c.decode(payload, &rb); ok { + c.OnRollback(rb) + } +} + // Transaction handler func (c *Client) transactionHandler(payload []byte) { var trans message.TransactionTrace @@ -95,6 +104,7 @@ func (c *Client) Subscribe(channel Channel) error { handler handler callback any }{ + RollbackChannel.Type(): {c.rollbackHandler, c.OnRollback}, TransactionChannel.Type(): {c.transactionHandler, c.OnTransaction}, HeartbeatChannel.Type(): {c.hbHandler, c.OnHeartbeat}, ActionChannel{}.Channel().Type(): {c.actHandler, c.OnAction}, From a4954ab949607d8d2285e901a5427c81bdfd1a6e Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 4 Feb 2024 16:56:59 +0100 Subject: [PATCH 081/291] api/client_test.go: Adding test for reading rollback messages. Because these don't happen that often on a real ship node. Test as much as possible with unit tests. --- api/client_test.go | 43 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/api/client_test.go b/api/client_test.go index 2b28b11..6fa9d8b 100644 --- a/api/client_test.go +++ b/api/client_test.go @@ -1,16 +1,28 @@ package api import ( + "bytes" + "io" "testing" "github.com/eosswedenorg/thalos/api/message" + _ "github.com/eosswedenorg/thalos/api/message/json" "github.com/stretchr/testify/assert" ) -type mockReader struct{} +type mockReader struct { + r io.Reader +} func (m mockReader) Read(channel Channel) ([]byte, error) { - return []byte{}, nil + if m.r != nil { + b, err := io.ReadAll(m.r) + if err == nil && len(b) < 1 { + err = io.EOF + } + return b, err + } + return []byte{}, io.EOF } func (m mockReader) Close() error { @@ -59,3 +71,30 @@ func TestClient_SubscribeWithNilHandler(t *testing.T) { assert.Error(t, err) } + +func TestClient_ReadRollback(t *testing.T) { + called := false + expected := message.RollbackMessage{ + OldBlockNum: 1000, + NewBlockNum: 50, + } + + codec, err := message.GetCodec("json") + assert.NoError(t, err) + + payload, err := codec.Encoder(expected) + assert.NoError(t, err) + + client := NewClient(mockReader{bytes.NewReader(payload)}, codec.Decoder) + client.OnRollback = func(rb message.RollbackMessage) { + assert.Equal(t, rb, expected) + called = true + } + + err = client.Subscribe(RollbackChannel) + assert.NoError(t, err) + + client.Run() + + assert.True(t, called, "Rollback callback not called when it should have been") +} From 133af980a3c90ec4808ff929e86201fc9b5a3806 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 4 Feb 2024 21:55:33 +0100 Subject: [PATCH 082/291] api/client.go: Rework to use a channel instead of callback. --- api/client.go | 82 +++++++++++++++++++++------------------- api/client_test.go | 31 ++------------- cmd/tools/bench.go | 61 +++++++++++++++--------------- cmd/tools/validate.go | 88 ++++++++++++++++++------------------------- 4 files changed, 114 insertions(+), 148 deletions(-) diff --git a/api/client.go b/api/client.go index a426d3d..146b4aa 100644 --- a/api/client.go +++ b/api/client.go @@ -2,15 +2,15 @@ package api import ( "fmt" - "reflect" "sync" + "time" "github.com/eosswedenorg/thalos/api/message" ) type handler func([]byte) -// Client reads and decodes messages from a reader and provides callback functions. +// Client reads and decodes messages from a reader and posts thems to a go channel type Client struct { reader Reader decoder message.Decoder @@ -18,18 +18,26 @@ type Client struct { // waitgroup for worker threads. wg sync.WaitGroup - OnError func(error) - OnRollback func(message.RollbackMessage) - OnTransaction func(message.TransactionTrace) - OnAction func(message.ActionTrace) - OnHeartbeat func(message.HeartBeat) - OnTableDelta func(message.TableDelta) + // Channel for messages and errors + channel chan any } func NewClient(reader Reader, decoder message.Decoder) *Client { return &Client{ reader: reader, decoder: decoder, + channel: make(chan any), + } +} + +func (c *Client) Channel() <-chan any { + return c.channel +} + +func (c *Client) post(msg any) { + select { + case <-time.After(time.Second): + case c.channel <- msg: } } @@ -37,9 +45,7 @@ func (c *Client) worker(channel Channel, h handler) { for { payload, err := c.reader.Read(channel) if err != nil { - if c.OnError != nil { - c.OnError(err) - } + c.post(err) return } @@ -47,13 +53,11 @@ func (c *Client) worker(channel Channel, h handler) { } } -// Helper method to decode a message and call OnError on error. +// Helper method to decode a message and post and error on the channel if it fails. // Returns true if successfull. false otherwise func (c *Client) decode(payload []byte, msg any) bool { if err := c.decoder(payload, msg); err != nil { - if c.OnError != nil { - c.OnError(err) - } + c.post(err) return false } return true @@ -63,7 +67,7 @@ func (c *Client) decode(payload []byte, msg any) bool { func (c *Client) rollbackHandler(payload []byte) { var rb message.RollbackMessage if ok := c.decode(payload, &rb); ok { - c.OnRollback(rb) + c.post(rb) } } @@ -71,7 +75,7 @@ func (c *Client) rollbackHandler(payload []byte) { func (c *Client) transactionHandler(payload []byte) { var trans message.TransactionTrace if ok := c.decode(payload, &trans); ok { - c.OnTransaction(trans) + c.post(trans) } } @@ -79,7 +83,7 @@ func (c *Client) transactionHandler(payload []byte) { func (c *Client) actHandler(payload []byte) { var act message.ActionTrace if ok := c.decode(payload, &act); ok { - c.OnAction(act) + c.post(act) } } @@ -87,7 +91,7 @@ func (c *Client) actHandler(payload []byte) { func (c *Client) tableDeltaHandler(payload []byte) { td := message.TableDelta{} if ok := c.decode(payload, &td); ok { - c.OnTableDelta(td) + c.post(td) } } @@ -95,37 +99,33 @@ func (c *Client) tableDeltaHandler(payload []byte) { func (c *Client) hbHandler(payload []byte) { var hb message.HeartBeat if ok := c.decode(payload, &hb); ok { - c.OnHeartbeat(hb) + c.post(hb) } } func (c *Client) Subscribe(channel Channel) error { - handlers := map[string]struct { - handler handler - callback any - }{ - RollbackChannel.Type(): {c.rollbackHandler, c.OnRollback}, - TransactionChannel.Type(): {c.transactionHandler, c.OnTransaction}, - HeartbeatChannel.Type(): {c.hbHandler, c.OnHeartbeat}, - ActionChannel{}.Channel().Type(): {c.actHandler, c.OnAction}, - TableDeltaChannel{}.Channel().Type(): {c.tableDeltaHandler, c.OnTableDelta}, - } + var handler handler - h, ok := handlers[channel.Type()] - - if !ok { + switch channel.Type() { + case RollbackChannel.Type(): + handler = c.rollbackHandler + case TransactionChannel.Type(): + handler = c.transactionHandler + case HeartbeatChannel.Type(): + handler = c.hbHandler + case ActionChannel{}.Channel().Type(): + handler = c.actHandler + case TableDeltaChannel{}.Channel().Type(): + handler = c.tableDeltaHandler + default: return fmt.Errorf("invalid channel type. %s", channel.Type()) } - if h.callback == nil || reflect.ValueOf(h.callback).IsNil() { - return fmt.Errorf("please set an handler before calling Subscribe") - } - // Start a worker for this channel. c.wg.Add(1) go func() { defer c.wg.Done() - c.worker(channel, h.handler) + c.worker(channel, handler) }() return nil @@ -137,5 +137,9 @@ func (c *Client) Run() { } func (c *Client) Close() error { - return c.reader.Close() + err := c.reader.Close() + // Wait for all goroutines before closing channel. + c.wg.Wait() + close(c.channel) + return err } diff --git a/api/client_test.go b/api/client_test.go index 6fa9d8b..784eee6 100644 --- a/api/client_test.go +++ b/api/client_test.go @@ -33,12 +33,6 @@ func mockDecoder([]byte, any) error { return nil } -func mockHbHandler(message.HeartBeat) { -} - -func mockActionHandler(message.ActionTrace) { -} - func TestClient_Subscribe(t *testing.T) { tests := []struct { name string @@ -48,13 +42,12 @@ func TestClient_Subscribe(t *testing.T) { {"Channel", Channel{}, true}, {"ActionChannel", ActionChannel{}.Channel(), false}, {"HeartbeatChannel", HeartbeatChannel, false}, - {"TransactionChannel", TransactionChannel, true}, + {"TransactionChannel", TransactionChannel, false}, + {"InvalidChannel", Channel{"random_type"}, true}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { c := NewClient(&mockReader{}, mockDecoder) - c.OnHeartbeat = mockHbHandler - c.OnAction = mockActionHandler if err := c.Subscribe(tt.channel); (err != nil) != tt.wantErr { t.Errorf("Client.Subscribe() error = %v, wantErr %v", err, tt.wantErr) } @@ -62,18 +55,7 @@ func TestClient_Subscribe(t *testing.T) { } } -func TestClient_SubscribeWithNilHandler(t *testing.T) { - client := NewClient(nil, nil) - client.OnAction = mockActionHandler - client.OnHeartbeat = mockHbHandler - - err := client.Subscribe(TableDeltaChannel{Name: "name"}.Channel()) - - assert.Error(t, err) -} - func TestClient_ReadRollback(t *testing.T) { - called := false expected := message.RollbackMessage{ OldBlockNum: 1000, NewBlockNum: 50, @@ -86,15 +68,10 @@ func TestClient_ReadRollback(t *testing.T) { assert.NoError(t, err) client := NewClient(mockReader{bytes.NewReader(payload)}, codec.Decoder) - client.OnRollback = func(rb message.RollbackMessage) { - assert.Equal(t, rb, expected) - called = true - } err = client.Subscribe(RollbackChannel) assert.NoError(t, err) - client.Run() - - assert.True(t, called, "Rollback callback not called when it should have been") + actual := <-client.Channel() + assert.Equal(t, expected, actual) } diff --git a/cmd/tools/bench.go b/cmd/tools/bench.go index ae23a55..31149f9 100644 --- a/cmd/tools/bench.go +++ b/cmd/tools/bench.go @@ -76,46 +76,47 @@ var benchCmd = &cli.Command{ client := api.NewClient(sub, codec.Decoder) - client.OnAction = func(act message.ActionTrace) { - counter++ - } - // Subscribe to all actions if err = client.Subscribe(api.ActionChannel{}.Channel()); err != nil { return err } go func() { - t := time.Now() - sig := make(chan os.Signal, 1) - signal.Notify(sig, os.Interrupt) - - for { - select { - case <-sig: - fmt.Println("Got interrupt") - client.Close() - return - case now := <-time.After(interval): - elapsed := now.Sub(t) - t = now - - log.WithFields(log.Fields{ - "num_messages": counter, - "elapsed": elapsed, - "msg_per_sec": float64(counter) / elapsed.Seconds(), - "msg_per_ms": float64(counter) / float64(elapsed.Milliseconds()), - "msg_per_min": float64(counter) / elapsed.Minutes(), - }).Info("Benchmark results") - - counter = 0 + for t := range client.Channel() { + switch err := t.(type) { + case message.ActionTrace: + counter++ + case error: + log.WithError(err).Error("Error when reading stream") } } }() - // Read stuff. - client.Run() + t := time.Now() + sig := make(chan os.Signal, 1) + signal.Notify(sig, os.Interrupt) - return nil + // Read stuff. + for { + select { + case <-sig: + fmt.Println("Got interrupt") + client.Close() + return nil + case now := <-time.After(interval): + elapsed := now.Sub(t) + t = now + + log.WithFields(log.Fields{ + "num_messages": counter, + "elapsed": elapsed, + "msg_per_sec": float64(counter) / elapsed.Seconds(), + "msg_per_ms": float64(counter) / float64(elapsed.Milliseconds()), + "msg_per_min": float64(counter) / elapsed.Minutes(), + }).Info("Benchmark results") + + counter = 0 + } + } }, } diff --git a/cmd/tools/validate.go b/cmd/tools/validate.go index 061e35d..e696da8 100644 --- a/cmd/tools/validate.go +++ b/cmd/tools/validate.go @@ -18,37 +18,6 @@ import ( log "github.com/sirupsen/logrus" ) -type Tester struct { - block_num uint32 - timeout time.Duration - timer *time.Ticker -} - -func NewTester(timeout time.Duration) *Tester { - return &Tester{ - block_num: 0, - timeout: timeout, - timer: time.NewTicker(timeout), - } -} - -func (t *Tester) OnAction(act message.ActionTrace) { - if t.block_num > 0 { - var diff int32 = int32(act.BlockNum - t.block_num) - if diff < 0 || diff > 1 { - log.WithFields(log.Fields{ - "current_block": t.block_num, - "block": act.BlockNum, - "diff": diff, - }).Warn("Invalid") - } - } - - t.block_num = act.BlockNum - - t.timer.Reset(t.timeout) -} - var validateCmd = &cli.Command{ Name: "validate", Usage: "Validate a thalos server by following action traces and makes sure that blocks arrive in order.", @@ -59,7 +28,6 @@ var validateCmd = &cli.Command{ chainIdFlag, }, Action: func(ctx *cli.Context) error { - tester := NewTester(time.Second * 5) status_duration := time.Second * 10 log.WithFields(log.Fields{ @@ -94,37 +62,53 @@ var validateCmd = &cli.Command{ } client := api.NewClient(sub, codec.Decoder) - client.OnAction = tester.OnAction // Subscribe to all actions if err = client.Subscribe(api.ActionChannel{}.Channel()); err != nil { return err } - go func() { - sig := make(chan os.Signal, 1) - signal.Notify(sig, os.Interrupt) + block_num := uint32(0) + timeout := time.Second * 5 + timer := time.NewTicker(timeout) - for { - select { - case <-sig: - fmt.Println("Got interrupt") - client.Close() - return - case <-tester.timer.C: - log.WithField("duration", tester.timeout). - Warn("Did not get any messages during the defined duration") - case <-time.After(status_duration): - log.WithFields(log.Fields{ - "current_block": tester.block_num, - }).Info("Status") + go func() { + for t := range client.Channel() { + switch msg := t.(type) { + case message.ActionTrace: + if block_num > 0 { + var diff int32 = int32(msg.BlockNum - block_num) + if diff < 0 || diff > 1 { + log.WithFields(log.Fields{ + "current_block": block_num, + "block": msg.BlockNum, + "diff": diff, + }).Warn("Invalid") + } + } + block_num = msg.BlockNum + timer.Reset(timeout) } } }() - // Read stuff. - client.Run() + sig := make(chan os.Signal, 1) + signal.Notify(sig, os.Interrupt) - return nil + for { + select { + case <-sig: + fmt.Println("Got interrupt") + client.Close() + return nil + case <-timer.C: + log.WithField("duration", timeout). + Warn("Did not get any messages during the defined duration") + case <-time.After(status_duration): + log.WithFields(log.Fields{ + "current_block": block_num, + }).Info("Status") + } + } }, } From 816d405d31e3ef6d104f2facd4351f6db6287404 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 4 Feb 2024 22:11:37 +0100 Subject: [PATCH 083/291] api/client.go: make Subscribe support a list of channels as argument. --- api/client.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/api/client.go b/api/client.go index 146b4aa..db79dc7 100644 --- a/api/client.go +++ b/api/client.go @@ -103,7 +103,7 @@ func (c *Client) hbHandler(payload []byte) { } } -func (c *Client) Subscribe(channel Channel) error { +func (c *Client) sub(channel Channel) error { var handler handler switch channel.Type() { @@ -131,6 +131,15 @@ func (c *Client) Subscribe(channel Channel) error { return nil } +func (c *Client) Subscribe(channels ...Channel) error { + for _, ch := range channels { + if err := c.sub(ch); err != nil { + return err + } + } + return nil +} + func (c *Client) Run() { // Just wait for workers to complete. c.wg.Wait() From b854c1dfa794f7643c7282d8a952626f582cf20c Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Wed, 7 Feb 2024 17:26:24 +0100 Subject: [PATCH 084/291] api/redis/subscriber.go: in Read() return io.EOF if redis connection is closed or channel returns a empty byte slice. --- api/redis/subscriber.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/api/redis/subscriber.go b/api/redis/subscriber.go index 1a1c91a..2387360 100644 --- a/api/redis/subscriber.go +++ b/api/redis/subscriber.go @@ -2,6 +2,7 @@ package redis import ( "context" + "io" "sync" "time" @@ -82,6 +83,10 @@ func (s *Subscriber) Read(channel api.Channel) ([]byte, error) { // Subscribe and insert it. err = s.sub.Subscribe(s.ctx, key) if err != nil { + // Closed redis client is considered an EOF. + if err == redis.ErrClosed { + err = io.EOF + } return nil, err } @@ -92,7 +97,12 @@ func (s *Subscriber) Read(channel api.Channel) ([]byte, error) { s.mu.Unlock() } - return <-ch, nil + data := <-ch + // Zero length data is considered an EOF + if len(data) == 0 { + return nil, io.EOF + } + return data, nil } func (s *Subscriber) Close() error { From d789b6a29478f045f6131c55579e4cfc3798ada9 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Wed, 7 Feb 2024 17:29:54 +0100 Subject: [PATCH 085/291] api/redis/subscriber.go: in worker() no need to spawn a goroutine when sending to the channel. --- api/redis/subscriber.go | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/api/redis/subscriber.go b/api/redis/subscriber.go index 2387360..07218ba 100644 --- a/api/redis/subscriber.go +++ b/api/redis/subscriber.go @@ -48,16 +48,6 @@ func NewSubscriber(ctx context.Context, client *redis.Client, ns Namespace, opti return sub } -// forward forwards a message to the channel. -// as writes to a unbuffered channel will block until it's read. -// We run select on it and discard the message if no read happends during timeout -func forward(msg redis.Message, ch chan<- []byte, timeout time.Duration) { - select { - case <-time.After(timeout): - case ch <- []byte(msg.Payload): - } -} - // worker reads messages from redis pubsub and forwards them to // correct channels. func (s *Subscriber) worker() { @@ -65,7 +55,10 @@ func (s *Subscriber) worker() { // Route message to correct channel. s.mu.RLock() if ch, ok := s.channels[msg.Channel]; ok { - go forward(*msg, ch, s.timeout) + select { + case <-time.After(s.timeout): + case ch <- []byte(msg.Payload): + } } s.mu.RUnlock() } From daa89cf3728bf56d660ff41f4d168337f3204479 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Wed, 7 Feb 2024 17:30:19 +0100 Subject: [PATCH 086/291] api/reader.go: Document io.EOF --- api/reader.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/api/reader.go b/api/reader.go index 5310932..21b4be7 100644 --- a/api/reader.go +++ b/api/reader.go @@ -8,10 +8,13 @@ type Reader interface { // Read a message from a channel. // Read may block until a message is ready or an error occured. // + // io.EOF is returned from a reader when there is no more data to be read. + // If Read returns io.EOF all subsequent calls must also return io.EOF + // // This function should be designed to handle concurrent calls. eg. thread safe. Read(channel Channel) ([]byte, error) // Close closes the reader - // Any blocked Read operations will be unblocked. + // Any blocked Read operations will be unblocked and return io.EOF Close() error } From 7feee11fadbde3f1571dce5939c2788528a7e21f Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Wed, 7 Feb 2024 17:31:19 +0100 Subject: [PATCH 087/291] api/client.go: Don't report io.EOF as an error. --- api/client.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/api/client.go b/api/client.go index db79dc7..ad9a3d5 100644 --- a/api/client.go +++ b/api/client.go @@ -2,6 +2,7 @@ package api import ( "fmt" + "io" "sync" "time" @@ -45,7 +46,11 @@ func (c *Client) worker(channel Channel, h handler) { for { payload, err := c.reader.Read(channel) if err != nil { - c.post(err) + // Dont report EOF as an error because it is used + // by readers to signal an graceful end of input. + if err != io.EOF { + c.post(err) + } return } From 729860cf763420611b3f4760ecc886e164f3295f Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Wed, 7 Feb 2024 17:31:44 +0100 Subject: [PATCH 088/291] cmd/tools/validate.go: log errors from client. --- cmd/tools/validate.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/tools/validate.go b/cmd/tools/validate.go index e696da8..96645e5 100644 --- a/cmd/tools/validate.go +++ b/cmd/tools/validate.go @@ -75,6 +75,8 @@ var validateCmd = &cli.Command{ go func() { for t := range client.Channel() { switch msg := t.(type) { + case error: + log.WithError(msg).Error("Error when reading stream") case message.ActionTrace: if block_num > 0 { var diff int32 = int32(msg.BlockNum - block_num) From d862788b8ddeb87f69b7e497892862f84048ffc2 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Wed, 7 Feb 2024 17:56:19 +0100 Subject: [PATCH 089/291] Add api/README.md --- api/README.md | 104 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 api/README.md diff --git a/api/README.md b/api/README.md new file mode 100644 index 0000000..b39cea6 --- /dev/null +++ b/api/README.md @@ -0,0 +1,104 @@ +# Thalos Golang API + +## Usage + +The api is designed with callback functions that are called when messages arrive + +## Example + +```go +package main + +import ( + "context" + "fmt" + "os" + "os/signal" + + "github.com/eosswedenorg/thalos/api" + "github.com/eosswedenorg/thalos/api/message" + _ "github.com/eosswedenorg/thalos/api/message/json" + api_redis "github.com/eosswedenorg/thalos/api/redis" + "github.com/redis/go-redis/v9" +) + +func main() { + // Create redis client + rdb := redis.NewClient(&redis.Options{}) + + sub := api_redis.NewSubscriber(context.Background(), rdb, api_redis.Namespace{ + Prefix: "ship", + ChainID: "1064487b3cd1a897ce03ae5b6a865651747e2e152090f99c1d19d44e01aea5a4", // Wax mainnet. + }) + + // Create client + codec, err := message.GetCodec("json") + if err != nil { + fmt.Println("Failed to get json codec") + return + } + + client := api.NewClient(sub, codec.Decoder) + + // Subscribe to some channels. + err = client.Subscribe( + api.TransactionChannel, + api.ActionChannel{Contract: "eosio"}.Channel(), + api.ActionChannel{Name: "mine"}.Channel(), + api.HeartbeatChannel, + api.TableDeltaChannel{}.Channel(), + ) + + if err != nil { + fmt.Println(err) + return + } + + // Wait for interrupt in a go routine and close the client. + go func() { + sig := make(chan os.Signal) + signal.Notify(sig, os.Interrupt) + + <-sig + fmt.Println("Got interrupt") + + client.Close() + }() + + // Read messages + for t := range client.Channel() { + switch msg := t.(type) { + case error: + fmt.Println("Error:", msg) + case message.TransactionTrace: + fmt.Println("Transaction", msg.BlockNum, msg.ID) + fmt.Println(msg) + fmt.Println("---") + case message.HeartBeat : + fmt.Println("Heartbeat") + fmt.Println(msg) + fmt.Println("---") + } + } +} + + + +``` + +## Message channels and types + +There are several types of channels to subscribe to aswell with their respectivly +message types. + +NOTE: this is not the same as an go channel. all messages will be posted to the same go channel that can +be accessed by `client.Channel()` + +| Channel type | Message type | Description | +| -------------------- | ------------------ | -------------------------------------------------------------------------------------------------------------------- | +| - | `error` | Posted if an error occured on the client. There is no channel to subscribe to. error messages will always be posted. | +| `HeartbeatChannel` | `HeartBeat` | Heartbeat message. Used to know if thalos is there or not if messages are not posted frequently on real channels. | +| `RollbackChannel` | `RollbackMessage` | This message is posted if the chain has experienced a microfork. | +| `TransactionChannel` | `TransactionTrace` | Information about an transaction | +| `ActionChannel` | `ActionTrace` | Information about an action | +| `TableDeltaChannel` | `TableDelta` | Information about an table change | From f08b6a3e45c5b61ee0429349e4f9d43fe0a4a6ba Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Wed, 7 Feb 2024 19:31:13 +0100 Subject: [PATCH 090/291] debian/control: update email. --- debian/control | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/control b/debian/control index 7fa3acd..0db0f29 100644 --- a/debian/control +++ b/debian/control @@ -6,7 +6,7 @@ Standards-Version: 4.5.0 Vcs-Git: https://github.com/eosswedenorg/thalos.git Vcs-Browser: https://github.com/eosswedenorg/thalos Priority: optional -Maintainer: Henrik Hautakoski +Maintainer: Henrik Hautakoski Package: thalos Section: utils From f99bbec5ff843517b0bd643d767075b7d2c56cec Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Wed, 7 Feb 2024 19:31:31 +0100 Subject: [PATCH 091/291] Version 1.0.0 --- Makefile | 2 +- debian/changelog | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 1bedf74..d65f3c1 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ GO=go GOLDFLAGS=-v -s -w -X main.VersionString=$(PROGRAM_VERSION) GOBUILDFLAGS=-v -p $(shell nproc) -ldflags="$(GOLDFLAGS)" PROGRAM=thalos-server -PROGRAM_VERSION=0.3.1 +PROGRAM_VERSION=1.0.0 PREFIX=/usr/local BINDIR=$(PREFIX)/bin CFGDIR=$(PREFIX)/etc/thalos diff --git a/debian/changelog b/debian/changelog index 2701dcf..8dee4ee 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,18 @@ +thalos (1.0.0) bionic focal jammy; urgency=medium + + * Improved code documentation. + * Updated dependancies. + * Small fixes for thalos tools. + + Go API: + + * Rewrite client api to provide a go channel instead of callback functions + * Support Transaction messages + * Support Rollback messages + * Support TableDelta messages + + -- Henrik Hautakoski Wed, 07 Feb 2024 19:32:10 +0100 + thalos (0.3.1) bionic focal jammy; urgency=medium * Bug: fix incorrect log path in debian packages. From 655dd730d7ac7697a6ecfc14c2ed203353728031 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sat, 10 Feb 2024 18:52:14 +0100 Subject: [PATCH 092/291] switch github.com/pborman/getopt to github.com/urfave/cli for handling cli flags --- Makefile | 2 +- cmd/thalos/main.go | 421 ++++++------------------------------------- cmd/thalos/server.go | 365 +++++++++++++++++++++++++++++++++++++ go.mod | 1 - go.sum | 2 - 5 files changed, 422 insertions(+), 369 deletions(-) create mode 100644 cmd/thalos/server.go diff --git a/Makefile b/Makefile index d65f3c1..57567ad 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,7 @@ CFGDIR=$(PREFIX)/etc/thalos build: build/$(PROGRAM) build/$(PROGRAM) : - $(GO) build $(GOBUILDFLAGS) -o $@ cmd/thalos/main.go + $(GO) build $(GOBUILDFLAGS) -o $@ cmd/thalos/main.go cmd/thalos/server.go tools : build/thalos-tools diff --git a/cmd/thalos/main.go b/cmd/thalos/main.go index 77049fe..7b749ce 100644 --- a/cmd/thalos/main.go +++ b/cmd/thalos/main.go @@ -1,389 +1,80 @@ package main import ( - "context" - "errors" "fmt" - "io" "os" - "os/signal" - "path" - "syscall" - "time" - "github.com/cenkalti/backoff/v4" - eos "github.com/eoscanada/eos-go" - shipclient "github.com/eosswedenorg-go/antelope-ship-client" - "github.com/eosswedenorg-go/pid" - "github.com/eosswedenorg/thalos/api/message" - _ "github.com/eosswedenorg/thalos/api/message/json" - _ "github.com/eosswedenorg/thalos/api/message/msgpack" - api_redis "github.com/eosswedenorg/thalos/api/redis" - "github.com/eosswedenorg/thalos/app" - "github.com/eosswedenorg/thalos/app/abi" - . "github.com/eosswedenorg/thalos/app/cache" - "github.com/eosswedenorg/thalos/app/config" - driver "github.com/eosswedenorg/thalos/app/driver/redis" - . "github.com/eosswedenorg/thalos/app/log" - redis_cache "github.com/go-redis/cache/v9" - "github.com/nikoksr/notify" - "github.com/nikoksr/notify/service/telegram" - "github.com/pborman/getopt/v2" - "github.com/redis/go-redis/v9" log "github.com/sirupsen/logrus" + "github.com/urfave/cli/v2" ) -// --------------------------- -// Global variables -// --------------------------- - -var conf *config.Config - -var shClient *shipclient.Stream - -var running bool = true - var VersionString string = "dev" -var exit chan bool - -var cache *Cache - -var cacheStore Store - -func readerLoop(processor *app.ShipProcessor) { - recon_cnt := 0 - - exp := &backoff.ExponentialBackOff{ - InitialInterval: time.Second, - RandomizationFactor: 0.25, - Multiplier: 2, - MaxInterval: 10 * time.Minute, - MaxElapsedTime: 0, - Stop: -1, - Clock: backoff.SystemClock, - } - exp.Reset() - - log.WithFields(log.Fields{ - "initial_interval": exp.InitialInterval, - "max_interval": exp.MaxInterval, - "randomization_factor": exp.RandomizationFactor, - "multiplier": exp.Multiplier, - }).Info("Connecting with Exponential Backoff") - - connectOp := func() error { - recon_cnt++ - - log.WithFields(log.Fields{ - "url": conf.Ship.Url, - "try": recon_cnt, - }).Info("Connecting to ship") - - if err := shClient.Connect(conf.Ship.Url); err != nil { - return err - } - - // Set stream client start block to processors current block - // Both values should be the same on first connect, but when reconnecting - // We don't want to start from the beginning - shClient.StartBlock = processor.GetCurrentBlock() - - return shClient.SendBlocksRequest() - } - - for running { - - err := backoff.RetryNotify(connectOp, exp, func(err error, d time.Duration) { - if recon_cnt >= 3 { - msg := fmt.Sprintf("Failed to connect to ship at '%s'", conf.Ship.Url) - if err := notify.Send(context.Background(), conf.Name, msg); err != nil { - log.WithError(err).Error("Failed to send notification") - } - recon_cnt = 0 - } - - log.WithError(err).Error("Failed to connect to SHIP") - - log.WithFields(log.Fields{ - "reconn_at": time.Now().Add(d), - "reconn_in": d, - }).Info("Reconnecting in ", d) - }) - if err != nil { - log.WithError(err).Error("Failed to connect to SHIP") - running = false - continue - } - - recon_cnt = 0 - log.WithFields(log.Fields{ - "start": shClient.StartBlock, - "end": shClient.EndBlock, - }).Info("Connected to ship") - - if err := shClient.Run(); err != nil { - - if errors.Is(err, shipclient.ErrEndBlockReached) { - exit <- true - log.Info("Endblock reached.") - break - } - - log.WithError(err).Error("Failed to read from ship") - } - } -} - -func run(processor *app.ShipProcessor) { - // Spawn reader loop in another thread. - go readerLoop(processor) - - // Create interrupt channel. - signals := make(chan os.Signal, 1) - - // Register signal channel to receive signals from the os. - signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM) - - // Wait for interrupt - select { - case sig := <-signals: - log.WithField("signal", sig).Info("Signal received") - - // Cleanly close the connection by sending a close message. - err := shClient.Shutdown() - if err != nil { - log.WithError(err).Info("failed to send close message to ship server") - } - case <-exit: - // Do nothing, just exit. - } - - running = false -} - -func getChain(def string) string { - if len(conf.Ship.Chain) > 0 { - return conf.Ship.Chain - } - return def -} - -func LogLevels() []string { - list := []string{} - for _, lvl := range log.AllLevels { - list = append(list, lvl.String()) - } - return list -} - -func initAbiManger(api *eos.API, chain_id string) *abi.AbiManager { - cache := NewCache("thalos::cache::abi::"+chain_id, cacheStore) - return abi.NewAbiManager(cache, api) -} - -func stateLoader(chainInfo *eos.InfoResp, current_block_no_cache bool) app.StateLoader { - return func(state *app.State) { - var source string - - // Load state from cache. - ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*500) - err := cache.Get(ctx, "state", &state) - cancel() - - // on error (cache miss) or if current_block_no_cache is set. - // set current block from config/api - if current_block_no_cache || err != nil { - // Set from config if we have a sane value. - if conf.Ship.StartBlockNum != shipclient.NULL_BLOCK_NUMBER { - source = "config" - state.CurrentBlock = conf.Ship.StartBlockNum - } else { - // Otherwise, set from api. - if conf.Ship.IrreversibleOnly { - source = "api (LIB)" - state.CurrentBlock = uint32(chainInfo.LastIrreversibleBlockNum) - } else { - source = "api (HEAD)" - state.CurrentBlock = uint32(chainInfo.HeadBlockNum) - } - } - } else { - source = "cache" - } - - log.WithFields(log.Fields{ - "block": state.CurrentBlock, - "source": source, - }).Info("Starting from block") - } -} - -func stateSaver(state app.State) error { - ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*500) - defer cancel() - return cache.Set(ctx, "state", state, 0) -} - func main() { - var err error - var chainInfo *eos.InfoResp + cli.AppHelpTemplate = `Usage: {{.HelpName}} [options] - exit = make(chan bool) + {{range .VisibleFlags}}{{.}} + {{end}}` - showHelp := getopt.BoolLong("help", 'h', "display this help text") - showVersion := getopt.BoolLong("version", 'v', "display the version") - configFile := getopt.StringLong("config", 'c', "./config.yml", "Config file to read", "file") - pidFile := getopt.StringLong("pid", 'p', "", "Where to write process id", "file") - logFile := getopt.StringLong("log", 'l', "", "Path to log file", "file") - logLevel := getopt.EnumLong("level", 'L', LogLevels(), "info", "Log level to use") - skip_currentblock_cache := getopt.Bool('n', "Force the application to take start block from config/api") - - getopt.Parse() - - if *showHelp { - getopt.Usage() - return + cli.HelpFlag = &cli.BoolFlag{ + Name: "help", + Aliases: []string{"h"}, + Usage: "display this help text", + DisableDefaultText: true, } - if *showVersion { - fmt.Println(VersionString) - return + cli.VersionPrinter = func(cCtx *cli.Context) { + fmt.Printf("Version %s\n", VersionString) } - // Write PID file - if len(*pidFile) > 0 { - log.WithField("file", *pidFile).Info("Writing pid to file") - err = pid.Save(*pidFile) - if err != nil { - log.WithError(err).Fatal("failed to write pid file") - return - } + cli.VersionFlag = &cli.BoolFlag{ + Name: "version", + Aliases: []string{"v"}, + Usage: "display the version", + DisableDefaultText: true, } - // Parse config - conf, err = config.Load(*configFile) - if err != nil { - log.WithError(err).Fatal("failed to read config file") - return + app := &cli.App{ + Version: VersionString, + Args: true, + UseShortOptionHandling: true, + HideHelpCommand: true, + Flags: []cli.Flag{ + &cli.PathFlag{ + Name: "config", + Aliases: []string{"c"}, + Value: "./config.yml", + Usage: "Config `file` to read", + TakesFile: true, + }, + &cli.StringFlag{ + Name: "level", + Aliases: []string{"L"}, + Usage: "Log level to use", + Value: "info", + }, + &cli.PathFlag{ + Name: "log", + Aliases: []string{"l"}, + Usage: "Path to log `file`", + TakesFile: true, + }, + &cli.BoolFlag{ + Name: "n", + Usage: "Force the application to take start block from config/api", + DisableDefaultText: true, + }, + &cli.StringFlag{ + Name: "pid", + Aliases: []string{"p"}, + Usage: "`file` to save process id to", + TakesFile: true, + }, + }, + Action: serverCmd, } - // If log file is given on the commandline, override config values. - if len(*logFile) > 0 { - conf.Log.Directory = path.Dir(*logFile) - conf.Log.Filename = path.Base(*logFile) + if err := app.Run(os.Args); err != nil { + log.WithError(err).Fatal("Application error") } - - lvl, err := log.ParseLevel(*logLevel) - if err == nil { - log.WithField("value", lvl).Info("Setting log level") - log.SetLevel(lvl) - } else { - log.WithError(err).Warn("Failed to parse level") - } - - if len(conf.Log.Filename) > 0 { - stdWriter, err := NewRotatingFileFromConfig(conf.Log, "info") - if err != nil { - log.WithError(err).Fatal("Failed to open info log") - return - } - errWriter, err := NewRotatingFileFromConfig(conf.Log, "error") - if err != nil { - log.WithError(err).Fatal("Failed to open error log") - return - } - - log.WithFields(log.Fields{ - "maxfilesize": conf.Log.MaxFileSize, - "maxage": conf.Log.MaxTime, - "directory": conf.Log.GetDirectory(), - "info_filename": stdWriter.GetFilename(), - "error_filename": errWriter.GetFilename(), - }).Info("Logging to file") - - log.SetOutput(io.Discard) - log.AddHook(MakeStdHook(stdWriter)) - log.AddHook(MakeErrorHook(errWriter)) - } - - // Init telegram notification service - if len(conf.Telegram.Id) > 0 { - - telegram, err := telegram.New(conf.Telegram.Id) - if err != nil { - log.WithError(err).Fatal("Failed to initialize telegram") - return - } - - telegram.AddReceivers(conf.Telegram.Channel) - - // Register services in notification manager - notify.UseServices(telegram) - } - - // Connect to redis - rdb := redis.NewClient(&redis.Options{ - Addr: conf.Redis.Addr, - Username: conf.Redis.User, - Password: conf.Redis.Password, - DB: conf.Redis.DB, - }) - - err = rdb.Ping(context.Background()).Err() - if err != nil { - log.WithError(err).Fatal("Failed to connect to redis") - return - } - - // Setup cache storage - cacheStore = NewRedisStore(&redis_cache.Options{ - Redis: rdb, - // Cache 10k keys for 10 minutes. - LocalCache: redis_cache.NewTinyLFU(10000, 10*time.Minute), - }) - - // Setup general cache - cache = NewCache("thalos::cache::instance::"+conf.Name, cacheStore) - - log.WithField("api", conf.Api).Info("Get chain info from api") - eosClient := eos.New(conf.Api) - chainInfo, err = eosClient.GetInfo(context.Background()) - if err != nil { - log.WithError(err).Fatal("Failed to get info") - return - } - - shClient = shipclient.NewStream(func(s *shipclient.Stream) { - s.StartBlock = conf.Ship.StartBlockNum - s.EndBlock = conf.Ship.EndBlockNum - s.IrreversibleOnly = conf.Ship.IrreversibleOnly - }) - - // Get codec - codec, err := message.GetCodec(conf.MessageCodec) - if err != nil { - log.WithError(err).Fatal("Failed to load codec") - return - } - - chain_id := getChain(chainInfo.ChainID.String()) - - processor := app.SpawnProccessor( - shClient, - stateLoader(chainInfo, *skip_currentblock_cache), - stateSaver, - driver.NewPublisher(context.Background(), rdb, api_redis.Namespace{ - Prefix: conf.Redis.Prefix, - ChainID: chain_id, - }), - initAbiManger(eosClient, chain_id), - codec, - ) - - // Run the application - run(processor) - - // Close the processor properly - processor.Close() } diff --git a/cmd/thalos/server.go b/cmd/thalos/server.go new file mode 100644 index 0000000..12211a7 --- /dev/null +++ b/cmd/thalos/server.go @@ -0,0 +1,365 @@ +package main + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "os/signal" + "path" + "syscall" + "time" + + "github.com/cenkalti/backoff/v4" + eos "github.com/eoscanada/eos-go" + shipclient "github.com/eosswedenorg-go/antelope-ship-client" + "github.com/eosswedenorg-go/pid" + "github.com/eosswedenorg/thalos/api/message" + _ "github.com/eosswedenorg/thalos/api/message/json" + _ "github.com/eosswedenorg/thalos/api/message/msgpack" + api_redis "github.com/eosswedenorg/thalos/api/redis" + "github.com/eosswedenorg/thalos/app" + "github.com/eosswedenorg/thalos/app/abi" + . "github.com/eosswedenorg/thalos/app/cache" + "github.com/eosswedenorg/thalos/app/config" + driver "github.com/eosswedenorg/thalos/app/driver/redis" + . "github.com/eosswedenorg/thalos/app/log" + redis_cache "github.com/go-redis/cache/v9" + "github.com/nikoksr/notify" + "github.com/nikoksr/notify/service/telegram" + "github.com/redis/go-redis/v9" + log "github.com/sirupsen/logrus" + "github.com/urfave/cli/v2" +) + +// --------------------------- +// Global variables +// --------------------------- + +var conf *config.Config + +var shClient *shipclient.Stream + +var running bool = true + +var exit chan bool + +var cache *Cache + +var cacheStore Store + +func readerLoop(processor *app.ShipProcessor) { + recon_cnt := 0 + + exp := &backoff.ExponentialBackOff{ + InitialInterval: time.Second, + RandomizationFactor: 0.25, + Multiplier: 2, + MaxInterval: 10 * time.Minute, + MaxElapsedTime: 0, + Stop: -1, + Clock: backoff.SystemClock, + } + exp.Reset() + + log.WithFields(log.Fields{ + "initial_interval": exp.InitialInterval, + "max_interval": exp.MaxInterval, + "randomization_factor": exp.RandomizationFactor, + "multiplier": exp.Multiplier, + }).Info("Connecting with Exponential Backoff") + + connectOp := func() error { + recon_cnt++ + + log.WithFields(log.Fields{ + "url": conf.Ship.Url, + "try": recon_cnt, + }).Info("Connecting to ship") + + if err := shClient.Connect(conf.Ship.Url); err != nil { + return err + } + + // Set stream client start block to processors current block + // Both values should be the same on first connect, but when reconnecting + // We don't want to start from the beginning + shClient.StartBlock = processor.GetCurrentBlock() + + return shClient.SendBlocksRequest() + } + + for running { + + err := backoff.RetryNotify(connectOp, exp, func(err error, d time.Duration) { + if recon_cnt >= 3 { + msg := fmt.Sprintf("Failed to connect to ship at '%s'", conf.Ship.Url) + if err := notify.Send(context.Background(), conf.Name, msg); err != nil { + log.WithError(err).Error("Failed to send notification") + } + recon_cnt = 0 + } + + log.WithError(err).Error("Failed to connect to SHIP") + + log.WithFields(log.Fields{ + "reconn_at": time.Now().Add(d), + "reconn_in": d, + }).Info("Reconnecting in ", d) + }) + if err != nil { + log.WithError(err).Error("Failed to connect to SHIP") + running = false + continue + } + + recon_cnt = 0 + log.WithFields(log.Fields{ + "start": shClient.StartBlock, + "end": shClient.EndBlock, + }).Info("Connected to ship") + + if err := shClient.Run(); err != nil { + + if errors.Is(err, shipclient.ErrEndBlockReached) { + exit <- true + log.Info("Endblock reached.") + break + } + + log.WithError(err).Error("Failed to read from ship") + } + } +} + +func run(processor *app.ShipProcessor) { + // Spawn reader loop in another thread. + go readerLoop(processor) + + // Create interrupt channel. + signals := make(chan os.Signal, 1) + + // Register signal channel to receive signals from the os. + signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM) + + // Wait for interrupt + select { + case sig := <-signals: + log.WithField("signal", sig).Info("Signal received") + + // Cleanly close the connection by sending a close message. + err := shClient.Shutdown() + if err != nil { + log.WithError(err).Info("failed to send close message to ship server") + } + case <-exit: + // Do nothing, just exit. + } + + running = false +} + +func getChain(def string) string { + if len(conf.Ship.Chain) > 0 { + return conf.Ship.Chain + } + return def +} + +func LogLevels() []string { + list := []string{} + for _, lvl := range log.AllLevels { + list = append(list, lvl.String()) + } + return list +} + +func initAbiManger(api *eos.API, chain_id string) *abi.AbiManager { + cache := NewCache("thalos::cache::abi::"+chain_id, cacheStore) + return abi.NewAbiManager(cache, api) +} + +func stateLoader(chainInfo *eos.InfoResp, current_block_no_cache bool) app.StateLoader { + return func(state *app.State) { + var source string + + // Load state from cache. + ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*500) + err := cache.Get(ctx, "state", &state) + cancel() + + // on error (cache miss) or if current_block_no_cache is set. + // set current block from config/api + if current_block_no_cache || err != nil { + // Set from config if we have a sane value. + if conf.Ship.StartBlockNum != shipclient.NULL_BLOCK_NUMBER { + source = "config" + state.CurrentBlock = conf.Ship.StartBlockNum + } else { + // Otherwise, set from api. + if conf.Ship.IrreversibleOnly { + source = "api (LIB)" + state.CurrentBlock = uint32(chainInfo.LastIrreversibleBlockNum) + } else { + source = "api (HEAD)" + state.CurrentBlock = uint32(chainInfo.HeadBlockNum) + } + } + } else { + source = "cache" + } + + log.WithFields(log.Fields{ + "block": state.CurrentBlock, + "source": source, + }).Info("Starting from block") + } +} + +func stateSaver(state app.State) error { + ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*500) + defer cancel() + return cache.Set(ctx, "state", state, 0) +} + +func serverCmd(ctx *cli.Context) error { + var err error + var chainInfo *eos.InfoResp + + exit = make(chan bool) + + skip_currentblock_cache := ctx.Bool("n") + + // Write PID file + pidFile := ctx.String("pid") + if len(pidFile) > 0 { + log.WithField("file", pidFile).Info("Writing pid to file") + if err = pid.Save(pidFile); err != nil { + return err + } + } + + // Parse config + conf, err = config.Load(ctx.Path("config")) + if err != nil { + return err + } + + // If log file is given on the commandline, override config values. + logFile := ctx.Path("log") + if len(logFile) > 0 { + conf.Log.Directory = path.Dir(logFile) + conf.Log.Filename = path.Base(logFile) + } + + lvl, err := log.ParseLevel(ctx.String("level")) + if err == nil { + log.WithField("value", lvl).Info("Setting log level") + log.SetLevel(lvl) + } else { + log.WithError(err).Warn("Failed to parse level") + } + + if len(conf.Log.Filename) > 0 { + stdWriter, err := NewRotatingFileFromConfig(conf.Log, "info") + if err != nil { + return err + } + errWriter, err := NewRotatingFileFromConfig(conf.Log, "error") + if err != nil { + return err + } + + log.WithFields(log.Fields{ + "maxfilesize": conf.Log.MaxFileSize, + "maxage": conf.Log.MaxTime, + "directory": conf.Log.GetDirectory(), + "info_filename": stdWriter.GetFilename(), + "error_filename": errWriter.GetFilename(), + }).Info("Logging to file") + + log.SetOutput(io.Discard) + log.AddHook(MakeStdHook(stdWriter)) + log.AddHook(MakeErrorHook(errWriter)) + } + + // Init telegram notification service + if len(conf.Telegram.Id) > 0 { + + telegram, err := telegram.New(conf.Telegram.Id) + if err != nil { + return err + } + + telegram.AddReceivers(conf.Telegram.Channel) + + // Register services in notification manager + notify.UseServices(telegram) + } + + // Connect to redis + rdb := redis.NewClient(&redis.Options{ + Addr: conf.Redis.Addr, + Username: conf.Redis.User, + Password: conf.Redis.Password, + DB: conf.Redis.DB, + }) + + err = rdb.Ping(context.Background()).Err() + if err != nil { + return err + } + + // Setup cache storage + cacheStore = NewRedisStore(&redis_cache.Options{ + Redis: rdb, + // Cache 10k keys for 10 minutes. + LocalCache: redis_cache.NewTinyLFU(10000, 10*time.Minute), + }) + + // Setup general cache + cache = NewCache("thalos::cache::instance::"+conf.Name, cacheStore) + + log.WithField("api", conf.Api).Info("Get chain info from api") + eosClient := eos.New(conf.Api) + chainInfo, err = eosClient.GetInfo(context.Background()) + if err != nil { + return err + } + + shClient = shipclient.NewStream(func(s *shipclient.Stream) { + s.StartBlock = conf.Ship.StartBlockNum + s.EndBlock = conf.Ship.EndBlockNum + s.IrreversibleOnly = conf.Ship.IrreversibleOnly + }) + + // Get codec + codec, err := message.GetCodec(conf.MessageCodec) + if err != nil { + log.WithError(err) + return err + } + + chain_id := getChain(chainInfo.ChainID.String()) + + processor := app.SpawnProccessor( + shClient, + stateLoader(chainInfo, skip_currentblock_cache), + stateSaver, + driver.NewPublisher(context.Background(), rdb, api_redis.Namespace{ + Prefix: conf.Redis.Prefix, + ChainID: chain_id, + }), + initAbiManger(eosClient, chain_id), + codec, + ) + + // Run the application + run(processor) + + // Close the processor properly + processor.Close() + + return nil +} diff --git a/go.mod b/go.mod index 92490a9..722fa33 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,6 @@ require ( github.com/go-redis/cache/v9 v9.0.0 github.com/go-redis/redismock/v9 v9.2.0 github.com/nikoksr/notify v0.41.0 - github.com/pborman/getopt/v2 v2.1.0 github.com/redis/go-redis/v9 v9.4.0 github.com/sirupsen/logrus v1.9.3 github.com/stretchr/testify v1.8.4 diff --git a/go.sum b/go.sum index 9b4a4af..5fe7d82 100644 --- a/go.sum +++ b/go.sum @@ -120,8 +120,6 @@ github.com/onsi/gomega v1.24.1/go.mod h1:3AOiACssS3/MajrniINInwbfOOtfZvplPzuRSmv github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/onsi/gomega v1.31.1 h1:KYppCUK+bUgAZwHOu7EXVBKyQA6ILvOESHkn/tgoqvo= github.com/onsi/gomega v1.31.1/go.mod h1:y40C95dwAD1Nz36SsEnxvfFe8FFfNxzI5eJ0EYGyAy0= -github.com/pborman/getopt/v2 v2.1.0 h1:eNfR+r+dWLdWmV8g5OlpyrTYHkhVNxHBdN2cCrJmOEA= -github.com/pborman/getopt/v2 v2.1.0/go.mod h1:4NtW75ny4eBw9fO1bhtNdYTlZKYX5/tBLtsOpwKIKd0= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= From e5e15a7645e813351dbc205463fcbb1434dbee11 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 11 Feb 2024 14:29:01 +0100 Subject: [PATCH 093/291] app/config/config.go: rework api. --- app/config/config.go | 23 +++++++++++------------ app/config/config_test.go | 21 +++++++++++---------- cmd/thalos/server.go | 6 +++--- 3 files changed, 25 insertions(+), 25 deletions(-) diff --git a/app/config/config.go b/app/config/config.go index e3050a2..3cc3ebc 100644 --- a/app/config/config.go +++ b/app/config/config.go @@ -46,8 +46,8 @@ type Config struct { Telegram TelegramConfig `yaml:"telegram"` } -func Parse(data []byte) (*Config, error) { - cfg := Config{ +func New() Config { + return Config{ MessageCodec: "json", Log: log.Config{ MaxFileSize: 10 * 1000 * 1000, // 10 mb @@ -60,15 +60,10 @@ func Parse(data []byte) (*Config, error) { IrreversibleOnly: false, }, Redis: RedisConfig{ - Addr: "localhost:6379", - Password: "", - DB: 0, - Prefix: "ship", + Addr: "localhost:6379", + Prefix: "ship", }, } - - err := yaml.Unmarshal(data, &cfg) - return &cfg, err } func (ship *ShipConfig) UnmarshalYAML(value *yaml.Node) error { @@ -87,11 +82,15 @@ func (ship *ShipConfig) UnmarshalYAML(value *yaml.Node) error { return err } -func Load(filename string) (*Config, error) { +func (cfg *Config) ReadYAML(data []byte) error { + return yaml.Unmarshal(data, cfg) +} + +func (cfg *Config) ReadFile(filename string) error { bytes, err := os.ReadFile(filename) if err != nil { - return nil, err + return err } - return Parse(bytes) + return cfg.ReadYAML(bytes) } diff --git a/app/config/config_test.go b/app/config/config_test.go index d9aad83..f3737a7 100644 --- a/app/config/config_test.go +++ b/app/config/config_test.go @@ -10,7 +10,7 @@ import ( shipclient "github.com/eosswedenorg-go/antelope-ship-client" ) -func TestParse_Default(t *testing.T) { +func TestNew(t *testing.T) { expected := Config{ MessageCodec: "json", @@ -34,12 +34,10 @@ func TestParse_Default(t *testing.T) { }, } - cfg, err := Parse([]byte(``)) - require.NoError(t, err) - require.Equal(t, cfg, &expected) + require.Equal(t, New(), expected) } -func TestParse(t *testing.T) { +func TestReadYAML(t *testing.T) { expected := Config{ Name: "ship-reader-1", Api: "http://127.0.0.1:8080", @@ -70,7 +68,8 @@ func TestParse(t *testing.T) { }, } - cfg, err := Parse([]byte(` + cfg := Config{} + err := cfg.ReadYAML([]byte(` name: "ship-reader-1" api: "http://127.0.0.1:8080" message_codec: "mojibake" @@ -97,10 +96,10 @@ redis: `)) require.NoError(t, err) - require.Equal(t, cfg, &expected) + require.Equal(t, cfg, expected) } -func TestParseShorthandShipUrl(t *testing.T) { +func TestReadYAMLShorthandShipUrl(t *testing.T) { expected := Config{ Name: "ship-reader-1", Api: "http://127.0.0.1:8080", @@ -128,7 +127,9 @@ func TestParseShorthandShipUrl(t *testing.T) { }, } - cfg, err := Parse([]byte(` + cfg := New() + + err := cfg.ReadYAML([]byte(` name: "ship-reader-1" api: "http://127.0.0.1:8080" ship: "127.0.0.1:8089" @@ -143,5 +144,5 @@ redis: `)) require.NoError(t, err) - require.Equal(t, cfg, &expected) + require.Equal(t, cfg, expected) } diff --git a/cmd/thalos/server.go b/cmd/thalos/server.go index 12211a7..eb6d6d9 100644 --- a/cmd/thalos/server.go +++ b/cmd/thalos/server.go @@ -37,7 +37,7 @@ import ( // Global variables // --------------------------- -var conf *config.Config +var conf config.Config var shClient *shipclient.Stream @@ -241,8 +241,8 @@ func serverCmd(ctx *cli.Context) error { } // Parse config - conf, err = config.Load(ctx.Path("config")) - if err != nil { + conf = config.New() + if err = conf.ReadFile(ctx.Path("config")); err != nil { return err } From a36c3f7853f3a9fe541a7d72a2c0f96500fafdb6 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Mon, 12 Feb 2024 14:55:02 +0100 Subject: [PATCH 094/291] cmd/thalos/server.go: wrap errors returned to main (to give more context) --- cmd/thalos/server.go | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/cmd/thalos/server.go b/cmd/thalos/server.go index eb6d6d9..b9c51d5 100644 --- a/cmd/thalos/server.go +++ b/cmd/thalos/server.go @@ -236,7 +236,7 @@ func serverCmd(ctx *cli.Context) error { if len(pidFile) > 0 { log.WithField("file", pidFile).Info("Writing pid to file") if err = pid.Save(pidFile); err != nil { - return err + return fmt.Errorf("pid: %s", err) } } @@ -264,11 +264,11 @@ func serverCmd(ctx *cli.Context) error { if len(conf.Log.Filename) > 0 { stdWriter, err := NewRotatingFileFromConfig(conf.Log, "info") if err != nil { - return err + return fmt.Errorf("log: %s", err) } errWriter, err := NewRotatingFileFromConfig(conf.Log, "error") if err != nil { - return err + return fmt.Errorf("log: %s", err) } log.WithFields(log.Fields{ @@ -289,7 +289,7 @@ func serverCmd(ctx *cli.Context) error { telegram, err := telegram.New(conf.Telegram.Id) if err != nil { - return err + return fmt.Errorf("telegram: %s", err) } telegram.AddReceivers(conf.Telegram.Channel) @@ -308,7 +308,7 @@ func serverCmd(ctx *cli.Context) error { err = rdb.Ping(context.Background()).Err() if err != nil { - return err + return fmt.Errorf("redis: %s", err) } // Setup cache storage @@ -325,7 +325,7 @@ func serverCmd(ctx *cli.Context) error { eosClient := eos.New(conf.Api) chainInfo, err = eosClient.GetInfo(context.Background()) if err != nil { - return err + return fmt.Errorf("eosapi: %s", err) } shClient = shipclient.NewStream(func(s *shipclient.Stream) { @@ -337,8 +337,7 @@ func serverCmd(ctx *cli.Context) error { // Get codec codec, err := message.GetCodec(conf.MessageCodec) if err != nil { - log.WithError(err) - return err + return fmt.Errorf("codec: %s", err) } chain_id := getChain(chainInfo.ChainID.String()) From 57cc55b60136d7d8f4ce60e1258fd82c7273fbd9 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Mon, 12 Feb 2024 15:08:19 +0100 Subject: [PATCH 095/291] app/config/config.go: refactor yaml and file methods into their own files. --- app/config/config.go | 32 -------------------------------- app/config/file.go | 14 ++++++++++++++ app/config/yaml.go | 25 +++++++++++++++++++++++++ 3 files changed, 39 insertions(+), 32 deletions(-) create mode 100644 app/config/file.go create mode 100644 app/config/yaml.go diff --git a/app/config/config.go b/app/config/config.go index 3cc3ebc..091edd9 100644 --- a/app/config/config.go +++ b/app/config/config.go @@ -1,13 +1,10 @@ package config import ( - "os" "time" "github.com/eosswedenorg/thalos/app/log" - "gopkg.in/yaml.v3" - shipclient "github.com/eosswedenorg-go/antelope-ship-client" ) @@ -65,32 +62,3 @@ func New() Config { }, } } - -func (ship *ShipConfig) UnmarshalYAML(value *yaml.Node) error { - var err error - - if value.Kind == yaml.ScalarNode { - ship.Url = value.Value - } else { - type ShipConfigRaw ShipConfig - raw := ShipConfigRaw(*ship) - if err = value.Decode(&raw); err == nil { - *ship = ShipConfig(raw) - } - } - - return err -} - -func (cfg *Config) ReadYAML(data []byte) error { - return yaml.Unmarshal(data, cfg) -} - -func (cfg *Config) ReadFile(filename string) error { - bytes, err := os.ReadFile(filename) - if err != nil { - return err - } - - return cfg.ReadYAML(bytes) -} diff --git a/app/config/file.go b/app/config/file.go new file mode 100644 index 0000000..b1b4863 --- /dev/null +++ b/app/config/file.go @@ -0,0 +1,14 @@ +package config + +import ( + "os" +) + +func (cfg *Config) ReadFile(filename string) error { + bytes, err := os.ReadFile(filename) + if err != nil { + return err + } + + return cfg.ReadYAML(bytes) +} diff --git a/app/config/yaml.go b/app/config/yaml.go new file mode 100644 index 0000000..b1d7db8 --- /dev/null +++ b/app/config/yaml.go @@ -0,0 +1,25 @@ +package config + +import ( + "gopkg.in/yaml.v3" +) + +func (ship *ShipConfig) UnmarshalYAML(value *yaml.Node) error { + var err error + + if value.Kind == yaml.ScalarNode { + ship.Url = value.Value + } else { + type ShipConfigRaw ShipConfig + raw := ShipConfigRaw(*ship) + if err = value.Decode(&raw); err == nil { + *ship = ShipConfig(raw) + } + } + + return err +} + +func (cfg *Config) ReadYAML(data []byte) error { + return yaml.Unmarshal(data, cfg) +} From 49faaf532561008403d7262db23aeaaff7242170 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Mon, 12 Feb 2024 15:09:17 +0100 Subject: [PATCH 096/291] Adding app/config/cli.go --- app/config/cli.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 app/config/cli.go diff --git a/app/config/cli.go b/app/config/cli.go new file mode 100644 index 0000000..1ff3b2b --- /dev/null +++ b/app/config/cli.go @@ -0,0 +1,17 @@ +package config + +import ( + "path" + + "github.com/urfave/cli/v2" +) + +func (cfg *Config) ReadCliFlags(ctx *cli.Context) error { + logFile := ctx.Path("log") + if len(logFile) > 0 { + cfg.Log.Directory = path.Dir(logFile) + cfg.Log.Filename = path.Base(logFile) + } + + return nil +} From 4a5cff20cd7c9dc72722b8bf5f0b3dc57f1268f6 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Mon, 12 Feb 2024 15:13:09 +0100 Subject: [PATCH 097/291] app/config: Adding description to methods. --- app/config/cli.go | 1 + app/config/config.go | 1 + app/config/file.go | 1 + app/config/yaml.go | 1 + 4 files changed, 4 insertions(+) diff --git a/app/config/cli.go b/app/config/cli.go index 1ff3b2b..8fa44bc 100644 --- a/app/config/cli.go +++ b/app/config/cli.go @@ -6,6 +6,7 @@ import ( "github.com/urfave/cli/v2" ) +// Read cli flag values into the config func (cfg *Config) ReadCliFlags(ctx *cli.Context) error { logFile := ctx.Path("log") if len(logFile) > 0 { diff --git a/app/config/config.go b/app/config/config.go index 091edd9..1e71e33 100644 --- a/app/config/config.go +++ b/app/config/config.go @@ -43,6 +43,7 @@ type Config struct { Telegram TelegramConfig `yaml:"telegram"` } +// Create a new Config object with default values func New() Config { return Config{ MessageCodec: "json", diff --git a/app/config/file.go b/app/config/file.go index b1b4863..d47444e 100644 --- a/app/config/file.go +++ b/app/config/file.go @@ -4,6 +4,7 @@ import ( "os" ) +// Read values from file func (cfg *Config) ReadFile(filename string) error { bytes, err := os.ReadFile(filename) if err != nil { diff --git a/app/config/yaml.go b/app/config/yaml.go index b1d7db8..60a2097 100644 --- a/app/config/yaml.go +++ b/app/config/yaml.go @@ -20,6 +20,7 @@ func (ship *ShipConfig) UnmarshalYAML(value *yaml.Node) error { return err } +// Read YAML config data func (cfg *Config) ReadYAML(data []byte) error { return yaml.Unmarshal(data, cfg) } From 457e0e4eebbd3bb315ca5b51dc7507ba42099fa8 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Mon, 12 Feb 2024 15:14:21 +0100 Subject: [PATCH 098/291] cmd/thalos/server.go: define and use ReadConfig() that utilizes Config.ReadFile and Config.ReadCliFlags() methods --- cmd/thalos/server.go | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/cmd/thalos/server.go b/cmd/thalos/server.go index b9c51d5..e348cb9 100644 --- a/cmd/thalos/server.go +++ b/cmd/thalos/server.go @@ -7,7 +7,6 @@ import ( "io" "os" "os/signal" - "path" "syscall" "time" @@ -223,6 +222,20 @@ func stateSaver(state app.State) error { return cache.Set(ctx, "state", state, 0) } +func ReadConfig(cfg *config.Config, ctx *cli.Context) error { + // Read file first. + if err := cfg.ReadFile(ctx.Path("config")); err != nil { + return err + } + + // Then override any cli flags + if err := cfg.ReadCliFlags(ctx); err != nil { + return err + } + + return nil +} + func serverCmd(ctx *cli.Context) error { var err error var chainInfo *eos.InfoResp @@ -242,15 +255,8 @@ func serverCmd(ctx *cli.Context) error { // Parse config conf = config.New() - if err = conf.ReadFile(ctx.Path("config")); err != nil { - return err - } - - // If log file is given on the commandline, override config values. - logFile := ctx.Path("log") - if len(logFile) > 0 { - conf.Log.Directory = path.Dir(logFile) - conf.Log.Filename = path.Base(logFile) + if err = ReadConfig(&conf, ctx); err != nil { + return fmt.Errorf("config: %s", err) } lvl, err := log.ParseLevel(ctx.String("level")) From e723e817863b2cef012c5224d0164fa07a7714a7 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Tue, 13 Feb 2024 22:18:55 +0100 Subject: [PATCH 099/291] go mod: update dependancies --- go.mod | 15 +++++++-------- go.sum | 27 +++++++++++++-------------- 2 files changed, 20 insertions(+), 22 deletions(-) diff --git a/go.mod b/go.mod index 722fa33..9079d42 100644 --- a/go.mod +++ b/go.mod @@ -6,9 +6,9 @@ require ( github.com/cenkalti/backoff/v4 v4.2.1 github.com/docker/go-units v0.5.0 github.com/eoscanada/eos-go v0.10.3-0.20231109144819-59afdfa3a37d - github.com/eosswedenorg-go/antelope-ship-client v0.2.7 + github.com/eosswedenorg-go/antelope-ship-client v0.2.8 github.com/eosswedenorg-go/pid v1.0.1 - github.com/eosswedenorg/thalos/api v0.3.0 + github.com/eosswedenorg/thalos/api v1.0.0 github.com/go-redis/cache/v9 v9.0.0 github.com/go-redis/redismock/v9 v9.2.0 github.com/nikoksr/notify v0.41.0 @@ -28,10 +28,9 @@ require ( github.com/eosswedenorg-go/jsontime v0.0.0-20230509125027-08422d6236c7 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-telegram-bot-api/telegram-bot-api v4.6.4+incompatible // indirect - github.com/google/go-cmp v0.6.0 // indirect github.com/gorilla/websocket v1.5.1 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.17.5 // indirect + github.com/klauspost/compress v1.17.6 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/logrusorgru/aurora v2.0.3+incompatible // indirect github.com/mitchellh/go-testing-interface v1.14.1 // indirect @@ -54,11 +53,11 @@ require ( go.uber.org/goleak v1.3.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.26.0 // indirect - golang.org/x/crypto v0.18.0 // indirect - golang.org/x/net v0.20.0 // indirect + golang.org/x/crypto v0.19.0 // indirect + golang.org/x/net v0.21.0 // indirect golang.org/x/sync v0.6.0 // indirect - golang.org/x/sys v0.16.0 // indirect - golang.org/x/term v0.16.0 // indirect + golang.org/x/sys v0.17.0 // indirect + golang.org/x/term v0.17.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect ) diff --git a/go.sum b/go.sum index 5fe7d82..7003b7b 100644 --- a/go.sum +++ b/go.sum @@ -25,8 +25,8 @@ github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4 github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/eoscanada/eos-go v0.10.3-0.20231109144819-59afdfa3a37d h1:vK5PijzcJaUPOhgWvY9lL99H9t3lrRNHx2IDHqS0ILc= github.com/eoscanada/eos-go v0.10.3-0.20231109144819-59afdfa3a37d/go.mod h1:L3avCf8OkDrjlUeNy9DdoV67TCmDNj2dSlc5Xp3DNNk= -github.com/eosswedenorg-go/antelope-ship-client v0.2.7 h1:ZGrXlGtMBqIV1TCCRjZAk9/RAA7pROO/Q/xN+baG45M= -github.com/eosswedenorg-go/antelope-ship-client v0.2.7/go.mod h1:kZ/4gkAIdAq4/WiZlVaSONpELcDCMJQJMmlikLUGCb8= +github.com/eosswedenorg-go/antelope-ship-client v0.2.8 h1:xqtRrijqVcOgLeh5foXjNoqOXkU/w1l6T4unwLgCrP0= +github.com/eosswedenorg-go/antelope-ship-client v0.2.8/go.mod h1:YcOEgcsZs9a7MjFjRZiX6Qpgc+c0a09PPE4mg8I6IU4= github.com/eosswedenorg-go/jsontime v0.0.0-20230509125027-08422d6236c7 h1:rLPu++RHaxg4WmUOXeWYioZuafWs0PVcYuvzOWbOJjk= github.com/eosswedenorg-go/jsontime v0.0.0-20230509125027-08422d6236c7/go.mod h1:eNUkVOymzgl0lViUhmm08PkutzqLnOQ6Dr+RUnf+Mq0= github.com/eosswedenorg-go/pid v1.0.1 h1:W4AEnnNwb041SpNR1uTZ/KbJ0OTA5eqiqIR1Q5Ah6A0= @@ -59,7 +59,6 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= @@ -71,8 +70,8 @@ github.com/jordan-wright/email v4.0.1-0.20210109023952-943e75fe5223+incompatible github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.17.5 h1:d4vBd+7CHydUqpFBgUEKkSdtSugf9YFmSkvUYPquI5E= -github.com/klauspost/compress v1.17.5/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= +github.com/klauspost/compress v1.17.6 h1:60eq2E/jlfwQXtvZEeBUYADs+BwKBWURIY+Gj2eRGjI= +github.com/klauspost/compress v1.17.6/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= @@ -198,8 +197,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= -golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= -golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= +golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -225,8 +224,8 @@ golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= -golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= -golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= +golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -261,16 +260,16 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= -golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= -golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE= -golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= +golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -319,4 +318,4 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o= +gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU= From 775760ec322b631a67d988ef2c5a7f0f0a8b6c01 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Tue, 13 Feb 2024 23:12:58 +0100 Subject: [PATCH 100/291] cmd/thalos/server.go: get rid of exit channel. its sufficient to just use running flag. --- cmd/thalos/server.go | 26 ++++++++------------------ 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/cmd/thalos/server.go b/cmd/thalos/server.go index e348cb9..e0a2976 100644 --- a/cmd/thalos/server.go +++ b/cmd/thalos/server.go @@ -42,8 +42,6 @@ var shClient *shipclient.Stream var running bool = true -var exit chan bool - var cache *Cache var cacheStore Store @@ -109,8 +107,7 @@ func readerLoop(processor *app.ShipProcessor) { }) if err != nil { log.WithError(err).Error("Failed to connect to SHIP") - running = false - continue + return } recon_cnt = 0 @@ -122,9 +119,8 @@ func readerLoop(processor *app.ShipProcessor) { if err := shClient.Run(); err != nil { if errors.Is(err, shipclient.ErrEndBlockReached) { - exit <- true log.Info("Endblock reached.") - break + return } log.WithError(err).Error("Failed to read from ship") @@ -143,17 +139,13 @@ func run(processor *app.ShipProcessor) { signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM) // Wait for interrupt - select { - case sig := <-signals: - log.WithField("signal", sig).Info("Signal received") + sig := <-signals + log.WithField("signal", sig).Info("Signal received") - // Cleanly close the connection by sending a close message. - err := shClient.Shutdown() - if err != nil { - log.WithError(err).Info("failed to send close message to ship server") - } - case <-exit: - // Do nothing, just exit. + // Cleanly close the connection by sending a close message. + err := shClient.Shutdown() + if err != nil { + log.WithError(err).Info("failed to send close message to ship server") } running = false @@ -240,8 +232,6 @@ func serverCmd(ctx *cli.Context) error { var err error var chainInfo *eos.InfoResp - exit = make(chan bool) - skip_currentblock_cache := ctx.Bool("n") // Write PID file From afb90af1db3a08e10c9938dde368ce45cbb3a3f2 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Tue, 13 Feb 2024 23:14:47 +0100 Subject: [PATCH 101/291] cmd/thalos/server.go: in run() set running flag to false before calling shClient.Shutdown() so readerLoop() exists correctly. --- cmd/thalos/server.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cmd/thalos/server.go b/cmd/thalos/server.go index e0a2976..412f007 100644 --- a/cmd/thalos/server.go +++ b/cmd/thalos/server.go @@ -143,12 +143,11 @@ func run(processor *app.ShipProcessor) { log.WithField("signal", sig).Info("Signal received") // Cleanly close the connection by sending a close message. + running = false err := shClient.Shutdown() if err != nil { log.WithError(err).Info("failed to send close message to ship server") } - - running = false } func getChain(def string) string { From 9974bfe3fd22804c911599b3530b857e5ee1c0a5 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Wed, 14 Feb 2024 13:00:33 +0100 Subject: [PATCH 102/291] rename app folder to internal. --- cmd/thalos/server.go | 24 +++++++++---------- cmd/tools/main.go | 2 +- cmd/tools/mock_publisher.go | 2 +- {app => internal}/abi/manager.go | 2 +- {app => internal}/abi/manager_test.go | 2 +- {app => internal}/cache/cache.go | 0 {app => internal}/cache/memory_store.go | 0 {app => internal}/cache/memory_store_test.go | 0 {app => internal}/cache/redis_store.go | 0 {app => internal}/cache/redis_store_test.go | 0 {app => internal}/cache/store.go | 0 {app => internal}/config/cli.go | 0 {app => internal}/config/config.go | 2 +- {app => internal}/config/config_test.go | 2 +- {app => internal}/config/file.go | 0 {app => internal}/config/yaml.go | 0 {app => internal}/driver/redis/publisher.go | 0 .../driver/redis/publisher_test.go | 0 {app => internal}/driver/writer.go | 0 {app => internal}/log/HookWriter.go | 0 {app => internal}/log/RotatingFile.go | 0 {app => internal}/log/config.go | 2 +- {app => internal}/log/config_test.go | 0 {app => internal}/log/init.go | 0 {app => internal/server}/ship_processor.go | 6 ++--- {app => internal/server}/state.go | 2 +- {app => internal}/types/size.go | 0 {app => internal}/types/size_test.go | 0 28 files changed, 23 insertions(+), 23 deletions(-) rename {app => internal}/abi/manager.go (96%) rename {app => internal}/abi/manager_test.go (99%) rename {app => internal}/cache/cache.go (100%) rename {app => internal}/cache/memory_store.go (100%) rename {app => internal}/cache/memory_store_test.go (100%) rename {app => internal}/cache/redis_store.go (100%) rename {app => internal}/cache/redis_store_test.go (100%) rename {app => internal}/cache/store.go (100%) rename {app => internal}/config/cli.go (100%) rename {app => internal}/config/config.go (96%) rename {app => internal}/config/config_test.go (98%) rename {app => internal}/config/file.go (100%) rename {app => internal}/config/yaml.go (100%) rename {app => internal}/driver/redis/publisher.go (100%) rename {app => internal}/driver/redis/publisher_test.go (100%) rename {app => internal}/driver/writer.go (100%) rename {app => internal}/log/HookWriter.go (100%) rename {app => internal}/log/RotatingFile.go (100%) rename {app => internal}/log/config.go (93%) rename {app => internal}/log/config_test.go (100%) rename {app => internal}/log/init.go (100%) rename {app => internal/server}/ship_processor.go (98%) rename {app => internal/server}/state.go (94%) rename {app => internal}/types/size.go (100%) rename {app => internal}/types/size_test.go (100%) diff --git a/cmd/thalos/server.go b/cmd/thalos/server.go index 412f007..7be4b29 100644 --- a/cmd/thalos/server.go +++ b/cmd/thalos/server.go @@ -18,12 +18,12 @@ import ( _ "github.com/eosswedenorg/thalos/api/message/json" _ "github.com/eosswedenorg/thalos/api/message/msgpack" api_redis "github.com/eosswedenorg/thalos/api/redis" - "github.com/eosswedenorg/thalos/app" - "github.com/eosswedenorg/thalos/app/abi" - . "github.com/eosswedenorg/thalos/app/cache" - "github.com/eosswedenorg/thalos/app/config" - driver "github.com/eosswedenorg/thalos/app/driver/redis" - . "github.com/eosswedenorg/thalos/app/log" + "github.com/eosswedenorg/thalos/internal/abi" + . "github.com/eosswedenorg/thalos/internal/cache" + "github.com/eosswedenorg/thalos/internal/config" + driver "github.com/eosswedenorg/thalos/internal/driver/redis" + . "github.com/eosswedenorg/thalos/internal/log" + . "github.com/eosswedenorg/thalos/internal/server" redis_cache "github.com/go-redis/cache/v9" "github.com/nikoksr/notify" "github.com/nikoksr/notify/service/telegram" @@ -46,7 +46,7 @@ var cache *Cache var cacheStore Store -func readerLoop(processor *app.ShipProcessor) { +func readerLoop(processor *ShipProcessor) { recon_cnt := 0 exp := &backoff.ExponentialBackOff{ @@ -128,7 +128,7 @@ func readerLoop(processor *app.ShipProcessor) { } } -func run(processor *app.ShipProcessor) { +func run(processor *ShipProcessor) { // Spawn reader loop in another thread. go readerLoop(processor) @@ -170,8 +170,8 @@ func initAbiManger(api *eos.API, chain_id string) *abi.AbiManager { return abi.NewAbiManager(cache, api) } -func stateLoader(chainInfo *eos.InfoResp, current_block_no_cache bool) app.StateLoader { - return func(state *app.State) { +func stateLoader(chainInfo *eos.InfoResp, current_block_no_cache bool) StateLoader { + return func(state *State) { var source string // Load state from cache. @@ -207,7 +207,7 @@ func stateLoader(chainInfo *eos.InfoResp, current_block_no_cache bool) app.State } } -func stateSaver(state app.State) error { +func stateSaver(state State) error { ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*500) defer cancel() return cache.Set(ctx, "state", state, 0) @@ -337,7 +337,7 @@ func serverCmd(ctx *cli.Context) error { chain_id := getChain(chainInfo.ChainID.String()) - processor := app.SpawnProccessor( + processor := SpawnProccessor( shClient, stateLoader(chainInfo, skip_currentblock_cache), stateSaver, diff --git a/cmd/tools/main.go b/cmd/tools/main.go index 4670591..bf8a396 100644 --- a/cmd/tools/main.go +++ b/cmd/tools/main.go @@ -5,7 +5,7 @@ import ( "github.com/urfave/cli/v2" - _ "github.com/eosswedenorg/thalos/app/log" + _ "github.com/eosswedenorg/thalos/internal/log" log "github.com/sirupsen/logrus" ) diff --git a/cmd/tools/mock_publisher.go b/cmd/tools/mock_publisher.go index 7d69e1a..421c6e4 100644 --- a/cmd/tools/mock_publisher.go +++ b/cmd/tools/mock_publisher.go @@ -10,7 +10,7 @@ import ( "github.com/eosswedenorg/thalos/api/message" _ "github.com/eosswedenorg/thalos/api/message/json" api_redis "github.com/eosswedenorg/thalos/api/redis" - redis_driver "github.com/eosswedenorg/thalos/app/driver/redis" + redis_driver "github.com/eosswedenorg/thalos/internal/driver/redis" "github.com/redis/go-redis/v9" log "github.com/sirupsen/logrus" diff --git a/app/abi/manager.go b/internal/abi/manager.go similarity index 96% rename from app/abi/manager.go rename to internal/abi/manager.go index 43be262..2441e01 100644 --- a/app/abi/manager.go +++ b/internal/abi/manager.go @@ -6,7 +6,7 @@ import ( "time" eos "github.com/eoscanada/eos-go" - "github.com/eosswedenorg/thalos/app/cache" + "github.com/eosswedenorg/thalos/internal/cache" ) // AbiManager handles an ABI cache that fetches the ABI from an API on cache miss. diff --git a/app/abi/manager_test.go b/internal/abi/manager_test.go similarity index 99% rename from app/abi/manager_test.go rename to internal/abi/manager_test.go index 661c8ec..7ecfc75 100644 --- a/app/abi/manager_test.go +++ b/internal/abi/manager_test.go @@ -9,7 +9,7 @@ import ( eos "github.com/eoscanada/eos-go" - "github.com/eosswedenorg/thalos/app/cache" + "github.com/eosswedenorg/thalos/internal/cache" "github.com/stretchr/testify/assert" ) diff --git a/app/cache/cache.go b/internal/cache/cache.go similarity index 100% rename from app/cache/cache.go rename to internal/cache/cache.go diff --git a/app/cache/memory_store.go b/internal/cache/memory_store.go similarity index 100% rename from app/cache/memory_store.go rename to internal/cache/memory_store.go diff --git a/app/cache/memory_store_test.go b/internal/cache/memory_store_test.go similarity index 100% rename from app/cache/memory_store_test.go rename to internal/cache/memory_store_test.go diff --git a/app/cache/redis_store.go b/internal/cache/redis_store.go similarity index 100% rename from app/cache/redis_store.go rename to internal/cache/redis_store.go diff --git a/app/cache/redis_store_test.go b/internal/cache/redis_store_test.go similarity index 100% rename from app/cache/redis_store_test.go rename to internal/cache/redis_store_test.go diff --git a/app/cache/store.go b/internal/cache/store.go similarity index 100% rename from app/cache/store.go rename to internal/cache/store.go diff --git a/app/config/cli.go b/internal/config/cli.go similarity index 100% rename from app/config/cli.go rename to internal/config/cli.go diff --git a/app/config/config.go b/internal/config/config.go similarity index 96% rename from app/config/config.go rename to internal/config/config.go index 1e71e33..790d647 100644 --- a/app/config/config.go +++ b/internal/config/config.go @@ -3,7 +3,7 @@ package config import ( "time" - "github.com/eosswedenorg/thalos/app/log" + "github.com/eosswedenorg/thalos/internal/log" shipclient "github.com/eosswedenorg-go/antelope-ship-client" ) diff --git a/app/config/config_test.go b/internal/config/config_test.go similarity index 98% rename from app/config/config_test.go rename to internal/config/config_test.go index f3737a7..d798436 100644 --- a/app/config/config_test.go +++ b/internal/config/config_test.go @@ -4,7 +4,7 @@ import ( "testing" "time" - "github.com/eosswedenorg/thalos/app/log" + "github.com/eosswedenorg/thalos/internal/log" "github.com/stretchr/testify/require" shipclient "github.com/eosswedenorg-go/antelope-ship-client" diff --git a/app/config/file.go b/internal/config/file.go similarity index 100% rename from app/config/file.go rename to internal/config/file.go diff --git a/app/config/yaml.go b/internal/config/yaml.go similarity index 100% rename from app/config/yaml.go rename to internal/config/yaml.go diff --git a/app/driver/redis/publisher.go b/internal/driver/redis/publisher.go similarity index 100% rename from app/driver/redis/publisher.go rename to internal/driver/redis/publisher.go diff --git a/app/driver/redis/publisher_test.go b/internal/driver/redis/publisher_test.go similarity index 100% rename from app/driver/redis/publisher_test.go rename to internal/driver/redis/publisher_test.go diff --git a/app/driver/writer.go b/internal/driver/writer.go similarity index 100% rename from app/driver/writer.go rename to internal/driver/writer.go diff --git a/app/log/HookWriter.go b/internal/log/HookWriter.go similarity index 100% rename from app/log/HookWriter.go rename to internal/log/HookWriter.go diff --git a/app/log/RotatingFile.go b/internal/log/RotatingFile.go similarity index 100% rename from app/log/RotatingFile.go rename to internal/log/RotatingFile.go diff --git a/app/log/config.go b/internal/log/config.go similarity index 93% rename from app/log/config.go rename to internal/log/config.go index 16498ce..b5114f8 100644 --- a/app/log/config.go +++ b/internal/log/config.go @@ -4,7 +4,7 @@ import ( "path" "time" - "github.com/eosswedenorg/thalos/app/types" + "github.com/eosswedenorg/thalos/internal/types" ) // Config represents configuration parameters for a log. diff --git a/app/log/config_test.go b/internal/log/config_test.go similarity index 100% rename from app/log/config_test.go rename to internal/log/config_test.go diff --git a/app/log/init.go b/internal/log/init.go similarity index 100% rename from app/log/init.go rename to internal/log/init.go diff --git a/app/ship_processor.go b/internal/server/ship_processor.go similarity index 98% rename from app/ship_processor.go rename to internal/server/ship_processor.go index b750cb5..d510c99 100644 --- a/app/ship_processor.go +++ b/internal/server/ship_processor.go @@ -1,4 +1,4 @@ -package app +package server import ( "encoding/hex" @@ -6,8 +6,8 @@ import ( "github.com/eosswedenorg/thalos/api" "github.com/eosswedenorg/thalos/api/message" - "github.com/eosswedenorg/thalos/app/abi" - "github.com/eosswedenorg/thalos/app/driver" + "github.com/eosswedenorg/thalos/internal/abi" + "github.com/eosswedenorg/thalos/internal/driver" log "github.com/sirupsen/logrus" diff --git a/app/state.go b/internal/server/state.go similarity index 94% rename from app/state.go rename to internal/server/state.go index 2edb323..d8312ef 100644 --- a/app/state.go +++ b/internal/server/state.go @@ -1,4 +1,4 @@ -package app +package server // State represents thalos runtime state type State struct { diff --git a/app/types/size.go b/internal/types/size.go similarity index 100% rename from app/types/size.go rename to internal/types/size.go diff --git a/app/types/size_test.go b/internal/types/size_test.go similarity index 100% rename from app/types/size_test.go rename to internal/types/size_test.go From 7108299550642dcf62e32be0dadcc1daf7efec96 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Wed, 14 Feb 2024 13:01:34 +0100 Subject: [PATCH 103/291] cmd/thalos/main.go: minor cleanup in VersionPrinter --- cmd/thalos/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/thalos/main.go b/cmd/thalos/main.go index 7b749ce..961dbb1 100644 --- a/cmd/thalos/main.go +++ b/cmd/thalos/main.go @@ -23,8 +23,8 @@ func main() { DisableDefaultText: true, } - cli.VersionPrinter = func(cCtx *cli.Context) { - fmt.Printf("Version %s\n", VersionString) + cli.VersionPrinter = func(ctx *cli.Context) { + fmt.Printf("Version %s\n", ctx.App.Version) } cli.VersionFlag = &cli.BoolFlag{ From b853bc026ed5a55d8d89db890f88e8403afd7a65 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Wed, 14 Feb 2024 21:03:37 +0100 Subject: [PATCH 104/291] cmd/thalos/server.go: remove global variables. --- cmd/thalos/server.go | 69 ++++++++++++++++++-------------------------- 1 file changed, 28 insertions(+), 41 deletions(-) diff --git a/cmd/thalos/server.go b/cmd/thalos/server.go index 7be4b29..a642420 100644 --- a/cmd/thalos/server.go +++ b/cmd/thalos/server.go @@ -19,6 +19,7 @@ import ( _ "github.com/eosswedenorg/thalos/api/message/msgpack" api_redis "github.com/eosswedenorg/thalos/api/redis" "github.com/eosswedenorg/thalos/internal/abi" + "github.com/eosswedenorg/thalos/internal/cache" . "github.com/eosswedenorg/thalos/internal/cache" "github.com/eosswedenorg/thalos/internal/config" driver "github.com/eosswedenorg/thalos/internal/driver/redis" @@ -32,21 +33,7 @@ import ( "github.com/urfave/cli/v2" ) -// --------------------------- -// Global variables -// --------------------------- - -var conf config.Config - -var shClient *shipclient.Stream - -var running bool = true - -var cache *Cache - -var cacheStore Store - -func readerLoop(processor *ShipProcessor) { +func readerLoop(conf *config.Config, running *bool, shClient *shipclient.Stream, processor *ShipProcessor) { recon_cnt := 0 exp := &backoff.ExponentialBackOff{ @@ -87,7 +74,7 @@ func readerLoop(processor *ShipProcessor) { return shClient.SendBlocksRequest() } - for running { + for *running { err := backoff.RetryNotify(connectOp, exp, func(err error, d time.Duration) { if recon_cnt >= 3 { @@ -128,9 +115,11 @@ func readerLoop(processor *ShipProcessor) { } } -func run(processor *ShipProcessor) { +func run(conf *config.Config, shClient *shipclient.Stream, processor *ShipProcessor) { + running := true + // Spawn reader loop in another thread. - go readerLoop(processor) + go readerLoop(conf, &running, shClient, processor) // Create interrupt channel. signals := make(chan os.Signal, 1) @@ -150,13 +139,6 @@ func run(processor *ShipProcessor) { } } -func getChain(def string) string { - if len(conf.Ship.Chain) > 0 { - return conf.Ship.Chain - } - return def -} - func LogLevels() []string { list := []string{} for _, lvl := range log.AllLevels { @@ -165,12 +147,12 @@ func LogLevels() []string { return list } -func initAbiManger(api *eos.API, chain_id string) *abi.AbiManager { - cache := NewCache("thalos::cache::abi::"+chain_id, cacheStore) +func initAbiManager(api *eos.API, store cache.Store, chain_id string) *abi.AbiManager { + cache := NewCache("thalos::cache::abi::"+chain_id, store) return abi.NewAbiManager(cache, api) } -func stateLoader(chainInfo *eos.InfoResp, current_block_no_cache bool) StateLoader { +func stateLoader(conf config.Config, chainInfo *eos.InfoResp, cache *cache.Cache, current_block_no_cache bool) StateLoader { return func(state *State) { var source string @@ -207,10 +189,12 @@ func stateLoader(chainInfo *eos.InfoResp, current_block_no_cache bool) StateLoad } } -func stateSaver(state State) error { - ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*500) - defer cancel() - return cache.Set(ctx, "state", state, 0) +func stateSaver(cache *cache.Cache) StateSaver { + return func(state State) error { + ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*500) + defer cancel() + return cache.Set(ctx, "state", state, 0) + } } func ReadConfig(cfg *config.Config, ctx *cli.Context) error { @@ -243,7 +227,7 @@ func serverCmd(ctx *cli.Context) error { } // Parse config - conf = config.New() + conf := config.New() if err = ReadConfig(&conf, ctx); err != nil { return fmt.Errorf("config: %s", err) } @@ -307,14 +291,14 @@ func serverCmd(ctx *cli.Context) error { } // Setup cache storage - cacheStore = NewRedisStore(&redis_cache.Options{ + cacheStore := NewRedisStore(&redis_cache.Options{ Redis: rdb, // Cache 10k keys for 10 minutes. LocalCache: redis_cache.NewTinyLFU(10000, 10*time.Minute), }) // Setup general cache - cache = NewCache("thalos::cache::instance::"+conf.Name, cacheStore) + cache := NewCache("thalos::cache::instance::"+conf.Name, cacheStore) log.WithField("api", conf.Api).Info("Get chain info from api") eosClient := eos.New(conf.Api) @@ -323,7 +307,7 @@ func serverCmd(ctx *cli.Context) error { return fmt.Errorf("eosapi: %s", err) } - shClient = shipclient.NewStream(func(s *shipclient.Stream) { + shClient := shipclient.NewStream(func(s *shipclient.Stream) { s.StartBlock = conf.Ship.StartBlockNum s.EndBlock = conf.Ship.EndBlockNum s.IrreversibleOnly = conf.Ship.IrreversibleOnly @@ -335,22 +319,25 @@ func serverCmd(ctx *cli.Context) error { return fmt.Errorf("codec: %s", err) } - chain_id := getChain(chainInfo.ChainID.String()) + chain_id := conf.Ship.Chain + if len(chain_id) < 1 { + chain_id = chainInfo.ChainID.String() + } processor := SpawnProccessor( shClient, - stateLoader(chainInfo, skip_currentblock_cache), - stateSaver, + stateLoader(conf, chainInfo, cache, skip_currentblock_cache), + stateSaver(cache), driver.NewPublisher(context.Background(), rdb, api_redis.Namespace{ Prefix: conf.Redis.Prefix, ChainID: chain_id, }), - initAbiManger(eosClient, chain_id), + initAbiManager(eosClient, cacheStore, chain_id), codec, ) // Run the application - run(processor) + run(&conf, shClient, processor) // Close the processor properly processor.Close() From fad70e19b9fda81f8b2660a2a273217d0d4e2050 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sat, 17 Feb 2024 14:32:07 +0100 Subject: [PATCH 105/291] app/config/config_test.go: fix parameter order to require.Equal() --- app/config/config_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/config/config_test.go b/app/config/config_test.go index f3737a7..5f6230c 100644 --- a/app/config/config_test.go +++ b/app/config/config_test.go @@ -34,7 +34,7 @@ func TestNew(t *testing.T) { }, } - require.Equal(t, New(), expected) + require.Equal(t, expected, New()) } func TestReadYAML(t *testing.T) { @@ -96,7 +96,7 @@ redis: `)) require.NoError(t, err) - require.Equal(t, cfg, expected) + require.Equal(t, expected, cfg) } func TestReadYAMLShorthandShipUrl(t *testing.T) { @@ -144,5 +144,5 @@ redis: `)) require.NoError(t, err) - require.Equal(t, cfg, expected) + require.Equal(t, expected, cfg) } From 59480533d3c2df6377d5fd5b41a5997619042ad2 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sat, 17 Feb 2024 13:45:05 +0100 Subject: [PATCH 106/291] refactor cli to use cobra. --- app/config/cli.go | 7 ++- cmd/thalos/main.go | 96 ++++++++++++------------------------- cmd/thalos/server.go | 55 +++++++++++++-------- cmd/tools/bench.go | 64 ++++++++++++------------- cmd/tools/flags.go | 37 -------------- cmd/tools/main.go | 62 ++++++++++++++++++------ cmd/tools/mock_publisher.go | 59 +++++++++++------------ cmd/tools/redis-acl.go | 78 +++++++++++------------------- cmd/tools/validate.go | 48 ++++++++++--------- go.mod | 7 ++- go.sum | 12 ++--- 11 files changed, 236 insertions(+), 289 deletions(-) delete mode 100644 cmd/tools/flags.go diff --git a/app/config/cli.go b/app/config/cli.go index 8fa44bc..d5bb6ed 100644 --- a/app/config/cli.go +++ b/app/config/cli.go @@ -3,16 +3,15 @@ package config import ( "path" - "github.com/urfave/cli/v2" + "github.com/spf13/pflag" ) // Read cli flag values into the config -func (cfg *Config) ReadCliFlags(ctx *cli.Context) error { - logFile := ctx.Path("log") +func (cfg *Config) ReadCliFlags(flags *pflag.FlagSet) error { + logFile, _ := flags.GetString("log") if len(logFile) > 0 { cfg.Log.Directory = path.Dir(logFile) cfg.Log.Filename = path.Base(logFile) } - return nil } diff --git a/cmd/thalos/main.go b/cmd/thalos/main.go index 7b749ce..994cdd2 100644 --- a/cmd/thalos/main.go +++ b/cmd/thalos/main.go @@ -1,80 +1,44 @@ package main import ( - "fmt" - "os" - log "github.com/sirupsen/logrus" - "github.com/urfave/cli/v2" + "github.com/spf13/cobra" + "github.com/spf13/pflag" ) var VersionString string = "dev" -func main() { - cli.AppHelpTemplate = `Usage: {{.HelpName}} [options] +var rootCmd *cobra.Command - {{range .VisibleFlags}}{{.}} - {{end}}` - - cli.HelpFlag = &cli.BoolFlag{ - Name: "help", - Aliases: []string{"h"}, - Usage: "display this help text", - DisableDefaultText: true, - } - - cli.VersionPrinter = func(cCtx *cli.Context) { - fmt.Printf("Version %s\n", VersionString) - } - - cli.VersionFlag = &cli.BoolFlag{ - Name: "version", - Aliases: []string{"v"}, - Usage: "display the version", - DisableDefaultText: true, - } - - app := &cli.App{ - Version: VersionString, - Args: true, - UseShortOptionHandling: true, - HideHelpCommand: true, - Flags: []cli.Flag{ - &cli.PathFlag{ - Name: "config", - Aliases: []string{"c"}, - Value: "./config.yml", - Usage: "Config `file` to read", - TakesFile: true, - }, - &cli.StringFlag{ - Name: "level", - Aliases: []string{"L"}, - Usage: "Log level to use", - Value: "info", - }, - &cli.PathFlag{ - Name: "log", - Aliases: []string{"l"}, - Usage: "Path to log `file`", - TakesFile: true, - }, - &cli.BoolFlag{ - Name: "n", - Usage: "Force the application to take start block from config/api", - DisableDefaultText: true, - }, - &cli.StringFlag{ - Name: "pid", - Aliases: []string{"p"}, - Usage: "`file` to save process id to", - TakesFile: true, - }, +func init() { + rootCmd = &cobra.Command{ + Use: "thalos-server", + FParseErrWhitelist: cobra.FParseErrWhitelist{ + UnknownFlags: true, }, - Action: serverCmd, + Version: VersionString, + Run: serverCmd, } - if err := app.Run(os.Args); err != nil { - log.WithError(err).Fatal("Application error") + rootCmd.SetHelpTemplate( + `{{ .Use | trimTrailingWhitespaces}} v{{.Version}} + +{{if or .Runnable .HasSubCommands}}{{.UsageString}}{{end}} +`) + rootCmd.SetVersionTemplate(`{{with .Name}}{{printf "%s " .}}{{end}}{{printf "v%s" .Version}}` + "\n") + + flags := pflag.FlagSet{} + flags.StringP("config", "c", "./config.yml", "Config file to read") + flags.StringP("level", "L", "info", "Log level to use") + flags.StringP("log", "l", "", "Path to log file (default: print to stdout/stderr)") + flags.StringP("pid", "p", "", "Where to write process id") + flags.BoolP("no-state-cache", "n", false, "Force the application to take start block from config/api") + + rootCmd.PersistentFlags().AddFlagSet(&flags) +} + +func main() { + if err := rootCmd.Execute(); err != nil { + log.Fatal(err) } } diff --git a/cmd/thalos/server.go b/cmd/thalos/server.go index e348cb9..bae8a48 100644 --- a/cmd/thalos/server.go +++ b/cmd/thalos/server.go @@ -29,7 +29,8 @@ import ( "github.com/nikoksr/notify/service/telegram" "github.com/redis/go-redis/v9" log "github.com/sirupsen/logrus" - "github.com/urfave/cli/v2" + "github.com/spf13/cobra" + "github.com/spf13/pflag" ) // --------------------------- @@ -222,44 +223,52 @@ func stateSaver(state app.State) error { return cache.Set(ctx, "state", state, 0) } -func ReadConfig(cfg *config.Config, ctx *cli.Context) error { +func ReadConfig(cfg *config.Config, flags *pflag.FlagSet) error { + filename, err := flags.GetString("config") + if err != nil { + return err + } + // Read file first. - if err := cfg.ReadFile(ctx.Path("config")); err != nil { + if err := cfg.ReadFile(filename); err != nil { return err } // Then override any cli flags - if err := cfg.ReadCliFlags(ctx); err != nil { + if err := cfg.ReadCliFlags(flags); err != nil { return err } return nil } -func serverCmd(ctx *cli.Context) error { +func serverCmd(cmd *cobra.Command, args []string) { var err error var chainInfo *eos.InfoResp exit = make(chan bool) - skip_currentblock_cache := ctx.Bool("n") + skip_currentblock_cache, _ := cmd.Flags().GetBool("no-state-cache") // Write PID file - pidFile := ctx.String("pid") - if len(pidFile) > 0 { + pidFile, err := cmd.Flags().GetString("pid") + if err != nil { log.WithField("file", pidFile).Info("Writing pid to file") if err = pid.Save(pidFile); err != nil { - return fmt.Errorf("pid: %s", err) + log.WithError(err).Fatal("Failed to write pid") + return } } // Parse config conf = config.New() - if err = ReadConfig(&conf, ctx); err != nil { - return fmt.Errorf("config: %s", err) + if err = ReadConfig(&conf, cmd.Flags()); err != nil { + log.WithError(err).Fatal("Failed to read config") + return } - lvl, err := log.ParseLevel(ctx.String("level")) + flagLevel, _ := cmd.Flags().GetString("level") + lvl, err := log.ParseLevel(flagLevel) if err == nil { log.WithField("value", lvl).Info("Setting log level") log.SetLevel(lvl) @@ -268,13 +277,17 @@ func serverCmd(ctx *cli.Context) error { } if len(conf.Log.Filename) > 0 { + fmt.Println(conf.Log.Filename) + stdWriter, err := NewRotatingFileFromConfig(conf.Log, "info") if err != nil { - return fmt.Errorf("log: %s", err) + log.WithError(err).Fatal("Failed to set standard log file") + return } errWriter, err := NewRotatingFileFromConfig(conf.Log, "error") if err != nil { - return fmt.Errorf("log: %s", err) + log.WithError(err).Fatal("Failed to set error log file") + return } log.WithFields(log.Fields{ @@ -295,7 +308,8 @@ func serverCmd(ctx *cli.Context) error { telegram, err := telegram.New(conf.Telegram.Id) if err != nil { - return fmt.Errorf("telegram: %s", err) + log.WithError(err).Fatal("Failed to initialize telegram notifier") + return } telegram.AddReceivers(conf.Telegram.Channel) @@ -314,7 +328,8 @@ func serverCmd(ctx *cli.Context) error { err = rdb.Ping(context.Background()).Err() if err != nil { - return fmt.Errorf("redis: %s", err) + log.WithError(err).Fatal("Failed to connect to redis") + return } // Setup cache storage @@ -331,7 +346,8 @@ func serverCmd(ctx *cli.Context) error { eosClient := eos.New(conf.Api) chainInfo, err = eosClient.GetInfo(context.Background()) if err != nil { - return fmt.Errorf("eosapi: %s", err) + log.WithError(err).Fatal("Failed to call eos api") + return } shClient = shipclient.NewStream(func(s *shipclient.Stream) { @@ -343,7 +359,8 @@ func serverCmd(ctx *cli.Context) error { // Get codec codec, err := message.GetCodec(conf.MessageCodec) if err != nil { - return fmt.Errorf("codec: %s", err) + log.WithError(err).Fatal("Failed to initialze codec") + return } chain_id := getChain(chainInfo.ChainID.String()) @@ -365,6 +382,4 @@ func serverCmd(ctx *cli.Context) error { // Close the processor properly processor.Close() - - return nil } diff --git a/cmd/tools/bench.go b/cmd/tools/bench.go index 31149f9..31af016 100644 --- a/cmd/tools/bench.go +++ b/cmd/tools/bench.go @@ -7,7 +7,7 @@ import ( "os/signal" "time" - "github.com/urfave/cli/v2" + "github.com/spf13/cobra" "github.com/eosswedenorg/thalos/api" "github.com/eosswedenorg/thalos/api/message" @@ -18,44 +18,38 @@ import ( log "github.com/sirupsen/logrus" ) -var benchCmd = &cli.Command{ - Name: "bench", - Usage: "Run a benchmark against a thalos node", - Flags: []cli.Flag{ - redisUrlFlag, - redisUserFlag, - redisPasswordFlag, - redisDbFlag, - prefixFlag, - chainIdFlag, - &cli.DurationFlag{ - Name: "interval", - Aliases: []string{"i"}, - Value: time.Minute, - Usage: "How often the benchmark results should be displayed.", - }, - }, - Action: func(ctx *cli.Context) error { +var benchCmd = &cobra.Command{ + Use: "bench", + Short: "Run a benchmark against a thalos node", + Run: func(cmd *cobra.Command, args []string) { var counter int = 0 - interval := ctx.Duration("interval") + interval, _ := cmd.Flags().GetDuration("interval") + + url, _ := cmd.Flags().GetString("redis-url") + user, _ := cmd.Flags().GetString("redis-user") + pw, _ := cmd.Flags().GetString("redis-pw") + prefix, _ := cmd.Flags().GetString("prefix") + chain_id, _ := cmd.Flags().GetString("chain_id") + db, _ := cmd.Flags().GetInt("redis-db") log.WithFields(log.Fields{ - "url": ctx.String("redis-url"), - "prefix": ctx.String("prefix"), - "chain_id": ctx.String("chain_id"), - "database": ctx.Int("redis-db"), + "url": url, + "prefix": prefix, + "chain_id": chain_id, + "database": db, }).Info("Connecting to redis") // Create redis client rdb := redis.NewClient(&redis.Options{ - Addr: ctx.String("redis-url"), - Username: ctx.String("redis-user"), - Password: ctx.String("redis-pw"), - DB: ctx.Int("redis-db"), + Addr: url, + Username: user, + Password: pw, + DB: db, }) if err := rdb.Ping(context.Background()).Err(); err != nil { - return err + log.WithError(err).Fatal("Failed to connect to redis") + return } log.Println("Connected to redis") @@ -65,20 +59,22 @@ var benchCmd = &cli.Command{ }).Info("Starting benchmark") sub := api_redis.NewSubscriber(context.Background(), rdb, api_redis.Namespace{ - Prefix: ctx.String("prefix"), - ChainID: ctx.String("chain_id"), + Prefix: prefix, + ChainID: chain_id, }) codec, err := message.GetCodec("json") if err != nil { - return err + log.WithError(err).Fatal("Failed to get codec") + return } client := api.NewClient(sub, codec.Decoder) // Subscribe to all actions if err = client.Subscribe(api.ActionChannel{}.Channel()); err != nil { - return err + log.WithError(err).Fatal("Failed to subscribe to channels") + return } go func() { @@ -102,7 +98,7 @@ var benchCmd = &cli.Command{ case <-sig: fmt.Println("Got interrupt") client.Close() - return nil + return case now := <-time.After(interval): elapsed := now.Sub(t) t = now diff --git a/cmd/tools/flags.go b/cmd/tools/flags.go deleted file mode 100644 index 24c0ee5..0000000 --- a/cmd/tools/flags.go +++ /dev/null @@ -1,37 +0,0 @@ -package main - -import ( - "github.com/urfave/cli/v2" -) - -var redisUrlFlag = &cli.StringFlag{ - Name: "redis-url", - Value: "127.0.0.1:6379", - Usage: "host:port to the redis server", -} - -var redisUserFlag = &cli.StringFlag{ - Name: "redis-user", - Usage: "User to use when authenticating to the server", -} - -var redisPasswordFlag = &cli.StringFlag{ - Name: "redis-pw", - Usage: "Password to use when authenticating to the server", -} - -var redisDbFlag = &cli.IntFlag{ - Name: "redis-db", - Value: 0, - Usage: "What redis database we should connect to.", -} - -var prefixFlag = &cli.StringFlag{ - Name: "prefix", - Value: "ship", -} - -var chainIdFlag = &cli.StringFlag{ - Name: "chain_id", - Value: "1064487b3cd1a897ce03ae5b6a865651747e2e152090f99c1d19d44e01aea5a4", -} diff --git a/cmd/tools/main.go b/cmd/tools/main.go index 4670591..aac073e 100644 --- a/cmd/tools/main.go +++ b/cmd/tools/main.go @@ -1,29 +1,63 @@ package main import ( - "os" - - "github.com/urfave/cli/v2" + "time" _ "github.com/eosswedenorg/thalos/app/log" log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + "github.com/spf13/pflag" ) var VersionString string = "dev" -func main() { - app := &cli.App{ - Usage: "Collection of tools for dealing with the thalos application", - Version: VersionString, - Commands: []*cli.Command{ - validateCmd, - benchCmd, - RedisACLCmd, - MockPublisherCmd, +var rootCmd *cobra.Command + +func init() { + redisFlags := pflag.FlagSet{} + redisFlags.String("redis-url", "127.0.0.1:6379", "host:port to the redis server") + redisFlags.String("redis-user", "", "User to use when authenticating to the server") + redisFlags.String("redis-pw", "", "Password to use when authenticating to the server") + redisFlags.Int("redis-db", 0, "What redis database we should connect to.") + redisFlags.String("prefix", "ship", "redis prefix") + redisFlags.String("chain_id", "1064487b3cd1a897ce03ae5b6a865651747e2e152090f99c1d19d44e01aea5a4", "chain id") + + rootCmd = &cobra.Command{ + Use: "thalos-tools", + Short: "Collection of tools for dealing with the thalos application", + FParseErrWhitelist: cobra.FParseErrWhitelist{ + UnknownFlags: true, }, + Version: VersionString, } - if err := app.Run(os.Args); err != nil { - log.WithError(err).Fatal("Application error") + benchCmd.Flags().AddFlagSet(&redisFlags) + benchCmd.Flags().DurationP("interval", "i", time.Minute, "How often the benchmark results should be displayed.") + + validateCmd.Flags().AddFlagSet(&redisFlags) + + MockPublisherCmd.Flags().AddFlagSet(&redisFlags) + MockPublisherCmd.Flags().String("codec", "json", "codec to use") + + RedisACLCmd.Flags().String("default-pw", "", "Password to use for the default account, if not provided a random one will be generated") + RedisACLCmd.Flags().String("client", "thalos-client", "Thalos client account name") + RedisACLCmd.Flags().String("client-pw", "", "Password to use for the thalos client account, if not provided a random one will be generated") + RedisACLCmd.Flags().String("server", "thalos", "Thalos account name") + RedisACLCmd.Flags().String("server-pw", "", "Password to use for the thalos server account, if not provided a random one will be generated") + RedisACLCmd.Flags().String("prefix", "ship", "Redis key prefix") + RedisACLCmd.Flags().Bool("cleartext", false, "If passwords should be hashed or left in cleartext.") + RedisACLCmd.Flags().String("file", "", "Where the config should be written to (default: standard out)") + + rootCmd.AddCommand( + validateCmd, + benchCmd, + RedisACLCmd, + MockPublisherCmd, + ) +} + +func main() { + if err := rootCmd.Execute(); err != nil { + log.Fatal(err) } } diff --git a/cmd/tools/mock_publisher.go b/cmd/tools/mock_publisher.go index 7d69e1a..a45b280 100644 --- a/cmd/tools/mock_publisher.go +++ b/cmd/tools/mock_publisher.go @@ -4,7 +4,7 @@ import ( "context" "time" - "github.com/urfave/cli/v2" + "github.com/spf13/cobra" "github.com/eosswedenorg/thalos/api" "github.com/eosswedenorg/thalos/api/message" @@ -16,45 +16,43 @@ import ( log "github.com/sirupsen/logrus" ) -var MockPublisherCmd = &cli.Command{ - Name: "mock_publisher", - Usage: "Run a publisher that mocks messages to a redis server. tries to send as many messages as possible", - Flags: []cli.Flag{ - redisUrlFlag, - redisUserFlag, - redisPasswordFlag, - redisDbFlag, - prefixFlag, - chainIdFlag, - &cli.StringFlag{ - Name: "codec", - Value: "json", - }, - }, - Action: func(ctx *cli.Context) error { +var MockPublisherCmd = &cobra.Command{ + Use: "mock_publisher", + Short: "Run a publisher that mocks messages to a redis server. tries to send as many messages as possible", + Run: func(cmd *cobra.Command, args []string) { + url, _ := cmd.Flags().GetString("redis-url") + user, _ := cmd.Flags().GetString("redis-user") + pw, _ := cmd.Flags().GetString("redis-pw") + prefix, _ := cmd.Flags().GetString("prefix") + chain_id, _ := cmd.Flags().GetString("chain_id") + db, _ := cmd.Flags().GetInt("redis-db") + // Create redis client rdb := redis.NewClient(&redis.Options{ - Addr: ctx.String("redis-url"), - Username: ctx.String("redis-user"), - Password: ctx.String("redis-pw"), - DB: ctx.Int("redis-db"), + Addr: url, + Username: user, + Password: pw, + DB: db, }) - codec, err := message.GetCodec(ctx.String("codec")) + codecArg, _ := cmd.Flags().GetString("codec") + + codec, err := message.GetCodec(codecArg) if err != nil { - return err + log.WithError(err).Fatal("Failed to get codec") + return } log.WithFields(log.Fields{ - "url": ctx.String("redis-url"), - "prefix": ctx.String("prefix"), - "chain_id": ctx.String("chain_id"), - "database": ctx.Int("redis-db"), + "url": url, + "prefix": prefix, + "chain_id": chain_id, + "database": db, }).Info("Starting mock publisher") ns := api_redis.Namespace{ - Prefix: ctx.String("prefix"), - ChainID: ctx.String("chain_id"), + Prefix: prefix, + ChainID: chain_id, } publisher := redis_driver.NewPublisher(context.Background(), rdb, ns) @@ -104,7 +102,8 @@ var MockPublisherCmd = &cli.Command{ payload, err := codec.Encoder(msg) if err != nil { - return err + log.WithError(err).Fatal("Failed to encode message") + return } channel := api.ActionChannel{}.Channel() diff --git a/cmd/tools/redis-acl.go b/cmd/tools/redis-acl.go index 4f2f2aa..25307b1 100644 --- a/cmd/tools/redis-acl.go +++ b/cmd/tools/redis-acl.go @@ -10,7 +10,8 @@ import ( "text/template" "time" - "github.com/urfave/cli/v2" + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" ) var rnd *rand.Rand @@ -90,60 +91,30 @@ user {{.client}} on {{.clientpw}} resetchannels &{{.prefix}}::* -@all +subscribe }) } -var RedisACLCmd = &cli.Command{ - Name: "redis-acl", - Usage: "create a users.acl file", - Flags: []cli.Flag{ - &cli.StringFlag{ - Name: "default-pw", - Usage: "Password to use for the default account, if not provided a random one will be generated", - }, - &cli.StringFlag{ - Name: "client", - Value: "thalos-client", - Usage: "Thalos client account name", - }, - &cli.StringFlag{ - Name: "client-pw", - Usage: "Password to use for the thalos client account, if not provided a random one will be generated", - }, - &cli.StringFlag{ - Name: "server", - Value: "thalos", - Usage: "Thalos account name", - }, - &cli.StringFlag{ - Name: "server-pw", - Usage: "Password to use for the thalos server account, if not provided a random one will be generated", - }, - &cli.StringFlag{ - Name: "prefix", - Value: "ship", - Usage: "Redis key prefix", - }, - &cli.BoolFlag{ - Name: "cleartext", - Usage: "If passwords should be hashed or left in cleartext.", - }, - &cli.StringFlag{ - Name: "file", - DefaultText: "Standard out", - Usage: "Where the config should be written to", - }, - }, - Action: func(ctx *cli.Context) error { +var RedisACLCmd = &cobra.Command{ + Use: "redis-acl", + Short: "create a users.acl file", + Run: func(cmd *cobra.Command, args []string) { var err error var out *os.File = os.Stdout rnd = rand.New(rand.NewSource(time.Now().UnixNano())) - defaultUser := NewUser("default", ctx.String("default-pw")) - serverUser := NewUser(ctx.String("server"), ctx.String("server-pw")) - clientUser := NewUser(ctx.String("client"), ctx.String("client-pw")) + flagDefUserPw, _ := cmd.Flags().GetString("default-pw") + flagServer, _ := cmd.Flags().GetString("server") + flagServerPw, _ := cmd.Flags().GetString("server-pw") + flagClient, _ := cmd.Flags().GetString("client") + flagClientPw, _ := cmd.Flags().GetString("client-pw") + flagPrefix, _ := cmd.Flags().GetString("prefix") + + defaultUser := NewUser("default", flagDefUserPw) + serverUser := NewUser(flagServer, flagServerPw) + clientUser := NewUser(flagClient, flagClientPw) atleastOneGeneratedPw := defaultUser.Generated || serverUser.Generated || clientUser.Generated - if !ctx.Bool("cleartext") { + cleartext, _ := cmd.Flags().GetBool("cleartext") + if !cleartext { if atleastOneGeneratedPw { println("Passwords") } @@ -157,17 +128,22 @@ var RedisACLCmd = &cli.Command{ clientUser.Hash() } - filename := ctx.String("file") + filename, _ := cmd.Flags().GetString("file") if len(filename) > 0 { out, err = os.Create(filename) if err != nil { - return err + log.WithError(err).Fatal("Failed to create output file") + return } defer out.Close() - } else if !ctx.Bool("cleartext") && atleastOneGeneratedPw { + } else if !cleartext && atleastOneGeneratedPw { fmt.Println() } - return writeTemplate(out, defaultUser, serverUser, clientUser, ctx.String("prefix")) + err = writeTemplate(out, defaultUser, serverUser, clientUser, flagPrefix) + if err != nil { + log.WithError(err).Fatal("Failed to writte config") + return + } }, } diff --git a/cmd/tools/validate.go b/cmd/tools/validate.go index 96645e5..a0de15d 100644 --- a/cmd/tools/validate.go +++ b/cmd/tools/validate.go @@ -7,7 +7,7 @@ import ( "os/signal" "time" - "github.com/urfave/cli/v2" + "github.com/spf13/cobra" "github.com/eosswedenorg/thalos/api" "github.com/eosswedenorg/thalos/api/message" @@ -18,33 +18,33 @@ import ( log "github.com/sirupsen/logrus" ) -var validateCmd = &cli.Command{ - Name: "validate", - Usage: "Validate a thalos server by following action traces and makes sure that blocks arrive in order.", - Flags: []cli.Flag{ - redisUrlFlag, - redisDbFlag, - prefixFlag, - chainIdFlag, - }, - Action: func(ctx *cli.Context) error { +var validateCmd = &cobra.Command{ + Use: "validate", + Short: "Validate a thalos server by following action traces and makes sure that blocks arrive in order.", + Run: func(cmd *cobra.Command, args []string) { status_duration := time.Second * 10 + url, _ := cmd.Flags().GetString("redis-url") + prefix, _ := cmd.Flags().GetString("prefix") + chain_id, _ := cmd.Flags().GetString("chain_id") + db, _ := cmd.Flags().GetInt("redis-db") + log.WithFields(log.Fields{ - "url": ctx.String("redis-url"), - "prefix": ctx.String("prefix"), - "chain_id": ctx.String("chain_id"), - "database": ctx.Int("redis-db"), + "url": url, + "prefix": prefix, + "chain_id": chain_id, + "database": db, }).Info("Connecting to redis") // Create redis client rdb := redis.NewClient(&redis.Options{ - Addr: ctx.String("redis-url"), - DB: ctx.Int("redis-db"), + Addr: url, + DB: db, }) if err := rdb.Ping(context.Background()).Err(); err != nil { - return err + log.WithError(err).Fatal("Failed to connect to redis") + return } log.Println("Connected to redis") @@ -52,20 +52,22 @@ var validateCmd = &cli.Command{ log.Info("Starting validation, following the stream") sub := api_redis.NewSubscriber(context.Background(), rdb, api_redis.Namespace{ - Prefix: ctx.String("prefix"), - ChainID: ctx.String("chain_id"), + Prefix: prefix, + ChainID: chain_id, }) codec, err := message.GetCodec("json") if err != nil { - return err + log.WithError(err).Fatal("Failed to get codec") + return } client := api.NewClient(sub, codec.Decoder) // Subscribe to all actions if err = client.Subscribe(api.ActionChannel{}.Channel()); err != nil { - return err + log.WithError(err).Fatal("Failed to subscribe to channels") + return } block_num := uint32(0) @@ -102,7 +104,7 @@ var validateCmd = &cli.Command{ case <-sig: fmt.Println("Got interrupt") client.Close() - return nil + return case <-timer.C: log.WithField("duration", timeout). Warn("Did not get any messages during the defined duration") diff --git a/go.mod b/go.mod index 9079d42..1c28f41 100644 --- a/go.mod +++ b/go.mod @@ -14,21 +14,22 @@ require ( github.com/nikoksr/notify v0.41.0 github.com/redis/go-redis/v9 v9.4.0 github.com/sirupsen/logrus v1.9.3 + github.com/spf13/cobra v1.8.0 + github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.8.4 - github.com/urfave/cli/v2 v2.27.1 gopkg.in/yaml.v3 v3.0.1 ) require ( github.com/blendle/zapdriver v1.3.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/eosswedenorg-go/jsontime v0.0.0-20230509125027-08422d6236c7 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-telegram-bot-api/telegram-bot-api v4.6.4+incompatible // indirect github.com/gorilla/websocket v1.5.1 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.17.6 // indirect github.com/kr/pretty v0.3.1 // indirect @@ -39,7 +40,6 @@ require ( github.com/onsi/gomega v1.31.1 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/streamingfast/logging v0.0.0-20230608130331-f22c91403091 // indirect github.com/technoweenie/multipartstreamer v1.0.1 // indirect github.com/tidwall/gjson v1.17.0 // indirect @@ -49,7 +49,6 @@ require ( github.com/vmihailenco/go-tinylfu v0.2.2 // indirect github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect - github.com/xrash/smetrics v0.0.0-20231213231151-1d8dd44e695e // indirect go.uber.org/goleak v1.3.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.26.0 // indirect diff --git a/go.sum b/go.sum index 7003b7b..1998506 100644 --- a/go.sum +++ b/go.sum @@ -13,7 +13,6 @@ github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/cpuguy83/go-md2man/v2 v2.0.3 h1:qMCsGGgs+MAzDFyp9LpAe1Lqy/fY/qCovCm0qnXZOBM= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -65,6 +64,8 @@ github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/ github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jarcoal/httpmock v1.2.0 h1:gSvTxxFR/MEMfsGrvRbdfpRUMBStovlSRLw0Ep1bwwc= github.com/jordan-wright/email v4.0.1-0.20210109023952-943e75fe5223+incompatible h1:jdpOPRN1zP63Td1hDQbZW73xKmzDvZHzVdNYxhnTMDA= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -131,10 +132,13 @@ github.com/redis/go-redis/v9 v9.4.0/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0 github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= -github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= +github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/streamingfast/logging v0.0.0-20230608130331-f22c91403091 h1:RN5mrigyirb8anBEtdjtHFIufXdacyTi6i4KBfeNXeo= github.com/streamingfast/logging v0.0.0-20230608130331-f22c91403091/go.mod h1:VlduQ80JcGJSargkRU4Sg9Xo63wZD/l8A5NC/Uo1/uU= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -162,8 +166,6 @@ github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= -github.com/urfave/cli/v2 v2.27.1 h1:8xSQ6szndafKVRmfyeUMxkNUJQMjL1F2zmsZ+qHpfho= -github.com/urfave/cli/v2 v2.27.1/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= github.com/vmihailenco/go-tinylfu v0.2.2 h1:H1eiG6HM36iniK6+21n9LLpzx1G9R3DJa2UjUjbynsI= github.com/vmihailenco/go-tinylfu v0.2.2/go.mod h1:CutYi2Q9puTxfcolkliPq4npPuofg9N9t8JVrjzwa3Q= github.com/vmihailenco/msgpack/v5 v5.3.4/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= @@ -171,8 +173,6 @@ github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IU github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok= github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= -github.com/xrash/smetrics v0.0.0-20231213231151-1d8dd44e695e h1:+SOyEddqYF09QP7vr7CgJ1eti3pY9Fn3LHO1M1r/0sI= -github.com/xrash/smetrics v0.0.0-20231213231151-1d8dd44e695e/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= From d43ef7f7ffbbf08dfcc19f1d724981b85f665331 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sat, 17 Feb 2024 15:17:12 +0100 Subject: [PATCH 107/291] app/types/size.go: Add UnmarshalText --- app/types/size.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/types/size.go b/app/types/size.go index c872c09..adb7708 100644 --- a/app/types/size.go +++ b/app/types/size.go @@ -34,3 +34,7 @@ func (s Size) String() string { func (s *Size) UnmarshalYAML(value *yaml.Node) error { return s.Parse(value.Value) } + +func (s *Size) UnmarshalText(text []byte) error { + return s.Parse(string(text)) +} From 106e88f2f4deaa70e2098d42e80d01d9bb3bae6f Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sat, 17 Feb 2024 16:30:54 +0100 Subject: [PATCH 108/291] app/config/config.go: add mapstructure tags. --- app/config/config.go | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/app/config/config.go b/app/config/config.go index 1e71e33..b181080 100644 --- a/app/config/config.go +++ b/app/config/config.go @@ -17,30 +17,30 @@ type RedisConfig struct { } type TelegramConfig struct { - Id string `yaml:"id"` - Channel int64 `yaml:"channel"` + Id string `yaml:"id" mapstructure:"id"` + Channel int64 `yaml:"channel" mapstructure:"channel"` } type ShipConfig struct { - Url string `yaml:"url"` - IrreversibleOnly bool `yaml:"irreversible_only"` - MaxMessagesInFlight uint32 `yaml:"max_messages_in_flight"` - StartBlockNum uint32 `yaml:"start_block_num"` - EndBlockNum uint32 `yaml:"end_block_num"` - Chain string `yaml:"chain"` + Url string `yaml:"url" mapstructure:"url"` + IrreversibleOnly bool `yaml:"irreversible_only" mapstructure:"irreversible_only"` + MaxMessagesInFlight uint32 `yaml:"max_messages_in_flight" mapstructure:"max_messages_in_flight"` + StartBlockNum uint32 `yaml:"start_block_num" mapstructure:"start_block_num"` + EndBlockNum uint32 `yaml:"end_block_num" mapstructure:"end_block_num"` + Chain string `yaml:"chain" mapstructure:"chain"` } type Config struct { - Name string `yaml:"name"` - Ship ShipConfig `yaml:"ship"` - Api string `yaml:"api"` + Name string `yaml:"name" mapstructure:"name"` + Ship ShipConfig `yaml:"ship" mapstructure:"ship"` + Api string `yaml:"api" mapstructure:"api"` - Log log.Config `yaml:"log"` + Log log.Config `yaml:"log" mapstructure:"log"` - Redis RedisConfig `yaml:"redis"` - MessageCodec string `yaml:"message_codec"` + Redis RedisConfig `yaml:"redis" mapstructure:"redis"` + MessageCodec string `yaml:"message_codec" mapstructure:"message_codec"` - Telegram TelegramConfig `yaml:"telegram"` + Telegram TelegramConfig `yaml:"telegram" mapstructure:"telegram"` } // Create a new Config object with default values From b517943fee88ad373407b336168223823808ca1f Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sat, 17 Feb 2024 16:31:14 +0100 Subject: [PATCH 109/291] app/log/config.go: add mapstructure tags --- app/log/config.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/log/config.go b/app/log/config.go index 16498ce..7179bdb 100644 --- a/app/log/config.go +++ b/app/log/config.go @@ -10,16 +10,16 @@ import ( // Config represents configuration parameters for a log. type Config struct { // Filename where the log is stored. - Filename string `yaml:"filename"` + Filename string `yaml:"filename" mapstructure:"filename"` // Directory where the log files are stored. - Directory string `yaml:"directory"` + Directory string `yaml:"directory" mapstructure:"directory"` // Maximum filesize, the log is rotated when this size is exceeded. - MaxFileSize types.Size `yaml:"maxfilesize"` + MaxFileSize types.Size `yaml:"maxfilesize" mapstructure:"maxfilesize"` // Maximum lifetime of the file before it is rotated. - MaxTime time.Duration `yaml:"maxtime"` + MaxTime time.Duration `yaml:"maxtime" mapstructure:"maxtime"` } func (c Config) GetFilename() string { From dbaa520160a1cf0340f5102b23af543472e31f91 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sat, 17 Feb 2024 16:37:16 +0100 Subject: [PATCH 110/291] app/config/config.go: adding decodeShorhandShipConfig() for mapstructure decoding. --- app/config/config.go | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/app/config/config.go b/app/config/config.go index b181080..1ec5591 100644 --- a/app/config/config.go +++ b/app/config/config.go @@ -1,6 +1,7 @@ package config import ( + "reflect" "time" "github.com/eosswedenorg/thalos/app/log" @@ -51,15 +52,34 @@ func New() Config { MaxFileSize: 10 * 1000 * 1000, // 10 mb MaxTime: time.Hour * 24, }, - Ship: ShipConfig{ - StartBlockNum: shipclient.NULL_BLOCK_NUMBER, - EndBlockNum: shipclient.NULL_BLOCK_NUMBER, - MaxMessagesInFlight: 10, - IrreversibleOnly: false, - }, + Ship: defaultShipConfig(""), Redis: RedisConfig{ Addr: "localhost:6379", Prefix: "ship", }, } } + +func defaultShipConfig(url string) ShipConfig { + return ShipConfig{ + Url: url, + StartBlockNum: shipclient.NULL_BLOCK_NUMBER, + EndBlockNum: shipclient.NULL_BLOCK_NUMBER, + MaxMessagesInFlight: 10, + IrreversibleOnly: false, + } +} + +// mapstructure DecodeHook that can parse a shorthand ship config (only string instead of struct.) +func decodeShorthandShipConfig(from reflect.Value, to reflect.Value) (interface{}, error) { + shipType := reflect.TypeOf(ShipConfig{}) + + // If to is a struct and is assignable to a ShipConfig and from is a string. + // Then we treat the from value as ShipConfig.Url + if to.Kind() == reflect.Struct && to.Type().AssignableTo(shipType) && from.Kind() == reflect.String { + return defaultShipConfig(from.String()), nil + } + + // If not, decode as normal. + return from.Interface(), nil +} From a0b88d19918fff266d3f69b15c9ffa4b96e62f60 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sat, 17 Feb 2024 16:43:20 +0100 Subject: [PATCH 111/291] app/config/file.go: define Read() that uses viper to read the config. --- app/config/config_test.go | 8 ++++---- app/config/file.go | 24 +++++++++++++++++++++++- go.mod | 19 ++++++++++++++++--- go.sum | 37 ++++++++++++++++++++++++++++++++----- 4 files changed, 75 insertions(+), 13 deletions(-) diff --git a/app/config/config_test.go b/app/config/config_test.go index 5f6230c..e78f93b 100644 --- a/app/config/config_test.go +++ b/app/config/config_test.go @@ -37,7 +37,7 @@ func TestNew(t *testing.T) { require.Equal(t, expected, New()) } -func TestReadYAML(t *testing.T) { +func TestRead(t *testing.T) { expected := Config{ Name: "ship-reader-1", Api: "http://127.0.0.1:8080", @@ -69,7 +69,7 @@ func TestReadYAML(t *testing.T) { } cfg := Config{} - err := cfg.ReadYAML([]byte(` + err := cfg.Read([]byte(` name: "ship-reader-1" api: "http://127.0.0.1:8080" message_codec: "mojibake" @@ -99,7 +99,7 @@ redis: require.Equal(t, expected, cfg) } -func TestReadYAMLShorthandShipUrl(t *testing.T) { +func TestReadShorthandShipUrl(t *testing.T) { expected := Config{ Name: "ship-reader-1", Api: "http://127.0.0.1:8080", @@ -129,7 +129,7 @@ func TestReadYAMLShorthandShipUrl(t *testing.T) { cfg := New() - err := cfg.ReadYAML([]byte(` + err := cfg.Read([]byte(` name: "ship-reader-1" api: "http://127.0.0.1:8080" ship: "127.0.0.1:8089" diff --git a/app/config/file.go b/app/config/file.go index d47444e..c14b00d 100644 --- a/app/config/file.go +++ b/app/config/file.go @@ -1,7 +1,11 @@ package config import ( + "bytes" "os" + + "github.com/mitchellh/mapstructure" + "github.com/spf13/viper" ) // Read values from file @@ -11,5 +15,23 @@ func (cfg *Config) ReadFile(filename string) error { return err } - return cfg.ReadYAML(bytes) + return cfg.Read(bytes) +} + +func (cfg *Config) Read(in []byte) error { + v := viper.New() + v.SetConfigType("yaml") + + if err := v.ReadConfig(bytes.NewBuffer(in)); err != nil { + return err + } + + decoders := mapstructure.ComposeDecodeHookFunc( + mapstructure.TextUnmarshallerHookFunc(), + mapstructure.StringToTimeDurationHookFunc(), + mapstructure.StringToSliceHookFunc(","), + decodeShorthandShipConfig, + ) + + return v.Unmarshal(cfg, viper.DecodeHook(decoders)) } diff --git a/go.mod b/go.mod index 1c28f41..186759b 100644 --- a/go.mod +++ b/go.mod @@ -11,11 +11,13 @@ require ( github.com/eosswedenorg/thalos/api v1.0.0 github.com/go-redis/cache/v9 v9.0.0 github.com/go-redis/redismock/v9 v9.2.0 + github.com/mitchellh/mapstructure v1.5.0 github.com/nikoksr/notify v0.41.0 github.com/redis/go-redis/v9 v9.4.0 github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.8.0 github.com/spf13/pflag v1.0.5 + github.com/spf13/viper v1.18.2 github.com/stretchr/testify v1.8.4 gopkg.in/yaml.v3 v3.0.1 ) @@ -23,24 +25,32 @@ require ( require ( github.com/blendle/zapdriver v1.3.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/eosswedenorg-go/jsontime v0.0.0-20230509125027-08422d6236c7 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-telegram-bot-api/telegram-bot-api v4.6.4+incompatible // indirect github.com/gorilla/websocket v1.5.1 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.17.6 // indirect - github.com/kr/pretty v0.3.1 // indirect github.com/logrusorgru/aurora v2.0.3+incompatible // indirect + github.com/magiconair/properties v1.8.7 // indirect github.com/mitchellh/go-testing-interface v1.14.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/onsi/gomega v1.31.1 // indirect + github.com/pelletier/go-toml/v2 v2.1.0 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/sagikazarmark/locafero v0.4.0 // indirect + github.com/sagikazarmark/slog-shim v0.1.0 // indirect + github.com/sourcegraph/conc v0.3.0 // indirect + github.com/spf13/afero v1.11.0 // indirect + github.com/spf13/cast v1.6.0 // indirect github.com/streamingfast/logging v0.0.0-20230608130331-f22c91403091 // indirect + github.com/subosito/gotenv v1.6.0 // indirect github.com/technoweenie/multipartstreamer v1.0.1 // indirect github.com/tidwall/gjson v1.17.0 // indirect github.com/tidwall/match v1.1.1 // indirect @@ -53,11 +63,14 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.26.0 // indirect golang.org/x/crypto v0.19.0 // indirect + golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect golang.org/x/net v0.21.0 // indirect golang.org/x/sync v0.6.0 // indirect golang.org/x/sys v0.17.0 // indirect golang.org/x/term v0.17.0 // indirect + golang.org/x/text v0.14.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect + gopkg.in/ini.v1 v1.67.0 // indirect ) replace github.com/eosswedenorg/thalos/api => ./api diff --git a/go.sum b/go.sum index 1998506..7ccd99c 100644 --- a/go.sum +++ b/go.sum @@ -16,8 +16,9 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= @@ -30,6 +31,7 @@ github.com/eosswedenorg-go/jsontime v0.0.0-20230509125027-08422d6236c7 h1:rLPu++ github.com/eosswedenorg-go/jsontime v0.0.0-20230509125027-08422d6236c7/go.mod h1:eNUkVOymzgl0lViUhmm08PkutzqLnOQ6Dr+RUnf+Mq0= github.com/eosswedenorg-go/pid v1.0.1 h1:W4AEnnNwb041SpNR1uTZ/KbJ0OTA5eqiqIR1Q5Ah6A0= github.com/eosswedenorg-go/pid v1.0.1/go.mod h1:wiOB/JXGt4YA3+T0j0xmCGSc3Jxzb7Ti/Ftli1fgWu4= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= @@ -62,6 +64,8 @@ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/ github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= @@ -77,15 +81,18 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= -github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8= github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mitchellh/go-testing-interface v1.14.1 h1:jrgshOhYAUVNMAJiKbEu7EqAwgJJ2JqpQmpLJOu07cU= github.com/mitchellh/go-testing-interface v1.14.1/go.mod h1:gfgS7OtZj6MA4U1UrDRp04twqAjfvlZyCfX3sDjEym8= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -120,25 +127,38 @@ github.com/onsi/gomega v1.24.1/go.mod h1:3AOiACssS3/MajrniINInwbfOOtfZvplPzuRSmv github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/onsi/gomega v1.31.1 h1:KYppCUK+bUgAZwHOu7EXVBKyQA6ILvOESHkn/tgoqvo= github.com/onsi/gomega v1.31.1/go.mod h1:y40C95dwAD1Nz36SsEnxvfFe8FFfNxzI5eJ0EYGyAy0= -github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= +github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/redis/go-redis/v9 v9.0.0-rc.4/go.mod h1:Vo3EsyWnicKnSKCA7HhgnvnyA74wOA69Cd2Meli5mmA= github.com/redis/go-redis/v9 v9.4.0 h1:Yzoz33UZw9I/mFhx4MNrB6Fk+XHO1VukNcCa1+lwyKk= github.com/redis/go-redis/v9 v9.4.0/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= -github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= +github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= +github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= +github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= +github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= +github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= +github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= +github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= +github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.18.2 h1:LUXCnvUvSM6FXAsj6nnfc8Q2tp1dIgUfY9Kc8GsSOiQ= +github.com/spf13/viper v1.18.2/go.mod h1:EKmWIqdnk5lOcmR72yw6hS+8OPYcwD0jteitLMVB+yk= github.com/streamingfast/logging v0.0.0-20230608130331-f22c91403091 h1:RN5mrigyirb8anBEtdjtHFIufXdacyTi6i4KBfeNXeo= github.com/streamingfast/logging v0.0.0-20230608130331-f22c91403091/go.mod h1:VlduQ80JcGJSargkRU4Sg9Xo63wZD/l8A5NC/Uo1/uU= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -153,6 +173,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/technoweenie/multipartstreamer v1.0.1 h1:XRztA5MXiR1TIRHxH2uNxXxaIkKQDeX7m2XsSOlQEnM= github.com/technoweenie/multipartstreamer v1.0.1/go.mod h1:jNVxdtShOxzAsukZwTSw6MDx5eUJoiEBsSvzDU9uzog= github.com/test-go/testify v1.1.4 h1:Tf9lntrKUMHiXQ07qBScBTSA0dhYQlu83hswqelv1iE= @@ -199,6 +221,8 @@ golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= +golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -278,6 +302,7 @@ golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -307,6 +332,8 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= From 33efb36901aae8d97251a906b10084462fca7026 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sat, 17 Feb 2024 16:43:57 +0100 Subject: [PATCH 112/291] Remove app/config/yaml.go as we use viper and mapstructure to parse. --- app/config/yaml.go | 26 -------------------------- 1 file changed, 26 deletions(-) delete mode 100644 app/config/yaml.go diff --git a/app/config/yaml.go b/app/config/yaml.go deleted file mode 100644 index 60a2097..0000000 --- a/app/config/yaml.go +++ /dev/null @@ -1,26 +0,0 @@ -package config - -import ( - "gopkg.in/yaml.v3" -) - -func (ship *ShipConfig) UnmarshalYAML(value *yaml.Node) error { - var err error - - if value.Kind == yaml.ScalarNode { - ship.Url = value.Value - } else { - type ShipConfigRaw ShipConfig - raw := ShipConfigRaw(*ship) - if err = value.Decode(&raw); err == nil { - *ship = ShipConfig(raw) - } - } - - return err -} - -// Read YAML config data -func (cfg *Config) ReadYAML(data []byte) error { - return yaml.Unmarshal(data, cfg) -} From 0d1bec6a6246da3d4d5056a23fb7b8c40239f612 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 18 Feb 2024 10:43:58 +0100 Subject: [PATCH 113/291] Adding internal/config/builder.go --- internal/config/builder.go | 102 +++++++++++++++++++++ internal/config/builder_test.go | 151 ++++++++++++++++++++++++++++++++ 2 files changed, 253 insertions(+) create mode 100644 internal/config/builder.go create mode 100644 internal/config/builder_test.go diff --git a/internal/config/builder.go b/internal/config/builder.go new file mode 100644 index 0000000..0090cc2 --- /dev/null +++ b/internal/config/builder.go @@ -0,0 +1,102 @@ +package config + +import ( + "errors" + "io" + "os" + + "github.com/mitchellh/mapstructure" + "github.com/spf13/pflag" + "github.com/spf13/viper" +) + +// This is a simple module that encapsulate the creation +// of a config object and can override values from cli flags. + +type Builder struct { + in io.Reader + flags *pflag.FlagSet + binds map[string]string +} + +func NewBuilder() *Builder { + return &Builder{ + binds: map[string]string{}, + } +} + +// Set the config file to read +func (b *Builder) SetConfigFile(filename string) *Builder { + file, _ := os.Open(filename) + return b.SetSource(file) +} + +// Set the source to read +func (b *Builder) SetSource(in io.Reader) *Builder { + b.in = in + return b +} + +// Set all flags that the builder should use. +func (b *Builder) SetFlags(flags *pflag.FlagSet) *Builder { + b.flags = flags + return b +} + +// Add a flag to the builder. +func (b *Builder) AddFlag(flag *pflag.Flag) *Builder { + b.flags.AddFlag(flag) + return b +} + +// Build the config object from file, cli-flags +func (b *Builder) Build() (*Config, error) { + if b.in == nil { + return nil, errors.New("Config not set") + } + + conf := New() + + v := viper.New() + v.SetConfigType("yaml") + + if b.flags != nil { + // bind flags in viper. + for key, flagname := range b.binds { + flag := b.flags.Lookup(flagname) + if flag == nil { + continue + } + + if err := v.BindPFlag(key, flag); err != nil { + return nil, err + } + } + } + + // Read config and unmarshal + if err := v.ReadConfig(b.in); err != nil { + return nil, err + } + + decoders := mapstructure.ComposeDecodeHookFunc( + mapstructure.TextUnmarshallerHookFunc(), + mapstructure.StringToTimeDurationHookFunc(), + mapstructure.StringToSliceHookFunc(","), + decodeShorthandShipConfig, + ) + + err := v.Unmarshal(&conf, viper.DecodeHook(decoders)) + if err != nil { + return nil, err + } + + // Call custom handler. + if b.flags != nil { + if err := conf.ReadCliFlags(b.flags); err != nil { + return nil, err + } + } + + return &conf, nil +} diff --git a/internal/config/builder_test.go b/internal/config/builder_test.go new file mode 100644 index 0000000..e7ceaf5 --- /dev/null +++ b/internal/config/builder_test.go @@ -0,0 +1,151 @@ +package config + +import ( + "bytes" + "testing" + "time" + + "github.com/eosswedenorg/thalos/internal/log" + "github.com/spf13/pflag" + "github.com/stretchr/testify/require" + + shipclient "github.com/eosswedenorg-go/antelope-ship-client" +) + +func TestBuilder(t *testing.T) { + expected := Config{ + Name: "ship-reader-1", + Api: "http://127.0.0.1:8080", + MessageCodec: "mojibake", + Log: log.Config{ + Filename: "some_file.log", + Directory: "/path/to/whatever", + MaxFileSize: 200, + MaxTime: 30 * time.Minute, + }, + Ship: ShipConfig{ + Url: "127.0.0.1:8089", + StartBlockNum: 23671836, + EndBlockNum: 23872222, + IrreversibleOnly: true, + MaxMessagesInFlight: 1337, + }, + Telegram: TelegramConfig{ + Id: "110201543:AAHdqTcvCH1vGWJxfSeofSAs0K5PALDsaw", + Channel: -123456789, + }, + Redis: RedisConfig{ + Addr: "localhost:6379", + User: "myuser", + Password: "passwd", + DB: 4, + Prefix: "some::ship", + }, + } + + builder := NewBuilder() + builder.SetSource(bytes.NewBuffer([]byte(` +name: "ship-reader-1" +api: "http://127.0.0.1:8080" +message_codec: "mojibake" +log: + filename: some_file.log + directory: /path/to/whatever + maxtime: 30m + maxfilesize: 200b +ship: + url: "127.0.0.1:8089" + irreversible_only: true + max_messages_in_flight: 1337 + start_block_num: 23671836 + end_block_num: 23872222 +telegram: + id: "110201543:AAHdqTcvCH1vGWJxfSeofSAs0K5PALDsaw" + channel: -123456789 +redis: + addr: "localhost:6379" + user: "myuser" + password: "passwd" + db: 4 + prefix: "some::ship" +`))) + + cfg, err := builder.Build() + + require.NoError(t, err) + require.Equal(t, &expected, cfg) +} + +func TestBuilder_NilSource(t *testing.T) { + cfg, err := NewBuilder().Build() + require.Nil(t, cfg) + require.EqualError(t, err, "Config not set") +} + +func TestBuilder_WithShorthandShipUrl(t *testing.T) { + expected := Config{ + Name: "ship-reader-1", + Api: "http://127.0.0.1:8080", + MessageCodec: "json", + Log: log.Config{ + MaxFileSize: 10 * 1000 * 1000, // 10 mb + MaxTime: time.Hour * 24, + }, + Ship: ShipConfig{ + Url: "127.0.0.1:8089", + StartBlockNum: shipclient.NULL_BLOCK_NUMBER, + EndBlockNum: shipclient.NULL_BLOCK_NUMBER, + MaxMessagesInFlight: 10, + IrreversibleOnly: false, + }, + Telegram: TelegramConfig{ + Id: "110201543:AAHdqTcvCH1vGWJxfSeofSAs0K5PALDsaw", + Channel: -123456789, + }, + Redis: RedisConfig{ + Addr: "localhost:6379", + Password: "passwd", + DB: 4, + Prefix: "some::ship", + }, + } + + builder := NewBuilder() + builder.SetSource(bytes.NewBuffer([]byte(` +name: "ship-reader-1" +api: "http://127.0.0.1:8080" +ship: "127.0.0.1:8089" +telegram: + id: "110201543:AAHdqTcvCH1vGWJxfSeofSAs0K5PALDsaw" + channel: -123456789 +redis: + addr: "localhost:6379" + password: "passwd" + db: 4 + prefix: "some::ship" +`))) + + cfg, err := builder.Build() + + require.NoError(t, err) + require.Equal(t, &expected, cfg) +} + +func TestBuilder_Flags(t *testing.T) { + flags := pflag.FlagSet{} + flags.StringP("log", "l", "", "") + + require.NoError(t, flags.Set("log", "/path/to/logs")) + + cfg, err := NewBuilder(). + SetSource(bytes.NewReader([]byte(``))). + SetFlags(&flags). + Build() + + expected := New() + expected.Log.Filename = "logs" + expected.Log.Directory = "/path/to" + + require.NoError(t, err) + require.Equal(t, &expected, cfg) +} From cad74a4d8433671d53ba03c085f7e0271dacc7f3 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 18 Feb 2024 10:51:39 +0100 Subject: [PATCH 114/291] cmd/thalos/server.go: use the new config.Builder --- cmd/thalos/server.go | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/cmd/thalos/server.go b/cmd/thalos/server.go index 4170296..1a5c95b 100644 --- a/cmd/thalos/server.go +++ b/cmd/thalos/server.go @@ -153,7 +153,7 @@ func initAbiManager(api *eos.API, store cache.Store, chain_id string) *abi.AbiMa return abi.NewAbiManager(cache, api) } -func stateLoader(conf config.Config, chainInfo *eos.InfoResp, cache *cache.Cache, current_block_no_cache bool) StateLoader { +func stateLoader(conf *config.Config, chainInfo *eos.InfoResp, cache *cache.Cache, current_block_no_cache bool) StateLoader { return func(state *State) { var source string @@ -198,23 +198,16 @@ func stateSaver(cache *cache.Cache) StateSaver { } } -func ReadConfig(cfg *config.Config, flags *pflag.FlagSet) error { +func GetConfig(flags *pflag.FlagSet) (*config.Config, error) { filename, err := flags.GetString("config") if err != nil { - return err + return nil, err } - // Read file first. - if err := cfg.ReadFile(filename); err != nil { - return err - } - - // Then override any cli flags - if err := cfg.ReadCliFlags(flags); err != nil { - return err - } - - return nil + return config.NewBuilder(). + SetConfigFile(filename). + SetFlags(flags). + Build() } func serverCmd(cmd *cobra.Command, args []string) { @@ -234,8 +227,8 @@ func serverCmd(cmd *cobra.Command, args []string) { } // Parse config - conf := config.New() - if err = ReadConfig(&conf, cmd.Flags()); err != nil { + conf, err := GetConfig(cmd.Flags()) + if err != nil { log.WithError(err).Fatal("Failed to read config") return } @@ -354,7 +347,7 @@ func serverCmd(cmd *cobra.Command, args []string) { ) // Run the application - run(&conf, shClient, processor) + run(conf, shClient, processor) // Close the processor properly processor.Close() From b4c305d8ead681f7ba18d90f6d173ff331d3b1f3 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 18 Feb 2024 10:52:06 +0100 Subject: [PATCH 115/291] Remove internal/config/file.go --- internal/config/config_test.go | 110 --------------------------------- internal/config/file.go | 37 ----------- 2 files changed, 147 deletions(-) delete mode 100644 internal/config/file.go diff --git a/internal/config/config_test.go b/internal/config/config_test.go index ceba4e2..c919a18 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -36,113 +36,3 @@ func TestNew(t *testing.T) { require.Equal(t, expected, New()) } - -func TestRead(t *testing.T) { - expected := Config{ - Name: "ship-reader-1", - Api: "http://127.0.0.1:8080", - MessageCodec: "mojibake", - Log: log.Config{ - Filename: "some_file.log", - Directory: "/path/to/whatever", - MaxFileSize: 200, - MaxTime: 30 * time.Minute, - }, - Ship: ShipConfig{ - Url: "127.0.0.1:8089", - StartBlockNum: 23671836, - EndBlockNum: 23872222, - IrreversibleOnly: true, - MaxMessagesInFlight: 1337, - }, - Telegram: TelegramConfig{ - Id: "110201543:AAHdqTcvCH1vGWJxfSeofSAs0K5PALDsaw", - Channel: -123456789, - }, - Redis: RedisConfig{ - Addr: "localhost:6379", - User: "myuser", - Password: "passwd", - DB: 4, - Prefix: "some::ship", - }, - } - - cfg := Config{} - err := cfg.Read([]byte(` -name: "ship-reader-1" -api: "http://127.0.0.1:8080" -message_codec: "mojibake" -log: - filename: some_file.log - directory: /path/to/whatever - maxtime: 30m - maxfilesize: 200b -ship: - url: "127.0.0.1:8089" - irreversible_only: true - max_messages_in_flight: 1337 - start_block_num: 23671836 - end_block_num: 23872222 -telegram: - id: "110201543:AAHdqTcvCH1vGWJxfSeofSAs0K5PALDsaw" - channel: -123456789 -redis: - addr: "localhost:6379" - user: "myuser" - password: "passwd" - db: 4 - prefix: "some::ship" -`)) - - require.NoError(t, err) - require.Equal(t, expected, cfg) -} - -func TestReadShorthandShipUrl(t *testing.T) { - expected := Config{ - Name: "ship-reader-1", - Api: "http://127.0.0.1:8080", - MessageCodec: "json", - Log: log.Config{ - MaxFileSize: 10 * 1000 * 1000, // 10 mb - MaxTime: time.Hour * 24, - }, - Ship: ShipConfig{ - Url: "127.0.0.1:8089", - StartBlockNum: shipclient.NULL_BLOCK_NUMBER, - EndBlockNum: shipclient.NULL_BLOCK_NUMBER, - MaxMessagesInFlight: 10, - IrreversibleOnly: false, - }, - Telegram: TelegramConfig{ - Id: "110201543:AAHdqTcvCH1vGWJxfSeofSAs0K5PALDsaw", - Channel: -123456789, - }, - Redis: RedisConfig{ - Addr: "localhost:6379", - Password: "passwd", - DB: 4, - Prefix: "some::ship", - }, - } - - cfg := New() - - err := cfg.Read([]byte(` -name: "ship-reader-1" -api: "http://127.0.0.1:8080" -ship: "127.0.0.1:8089" -telegram: - id: "110201543:AAHdqTcvCH1vGWJxfSeofSAs0K5PALDsaw" - channel: -123456789 -redis: - addr: "localhost:6379" - password: "passwd" - db: 4 - prefix: "some::ship" -`)) - - require.NoError(t, err) - require.Equal(t, expected, cfg) -} diff --git a/internal/config/file.go b/internal/config/file.go deleted file mode 100644 index c14b00d..0000000 --- a/internal/config/file.go +++ /dev/null @@ -1,37 +0,0 @@ -package config - -import ( - "bytes" - "os" - - "github.com/mitchellh/mapstructure" - "github.com/spf13/viper" -) - -// Read values from file -func (cfg *Config) ReadFile(filename string) error { - bytes, err := os.ReadFile(filename) - if err != nil { - return err - } - - return cfg.Read(bytes) -} - -func (cfg *Config) Read(in []byte) error { - v := viper.New() - v.SetConfigType("yaml") - - if err := v.ReadConfig(bytes.NewBuffer(in)); err != nil { - return err - } - - decoders := mapstructure.ComposeDecodeHookFunc( - mapstructure.TextUnmarshallerHookFunc(), - mapstructure.StringToTimeDurationHookFunc(), - mapstructure.StringToSliceHookFunc(","), - decodeShorthandShipConfig, - ) - - return v.Unmarshal(cfg, viper.DecodeHook(decoders)) -} From 7033240000735e47d4b77c0981e04d6c24bc49e6 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 18 Feb 2024 10:55:15 +0100 Subject: [PATCH 116/291] internal/config/cli.go: Rename Config.ReadCliFlags to ovverideCliFlags and make it a function instead of method. --- internal/config/builder.go | 2 +- internal/config/cli.go | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/internal/config/builder.go b/internal/config/builder.go index 0090cc2..b9c0e75 100644 --- a/internal/config/builder.go +++ b/internal/config/builder.go @@ -93,7 +93,7 @@ func (b *Builder) Build() (*Config, error) { // Call custom handler. if b.flags != nil { - if err := conf.ReadCliFlags(b.flags); err != nil { + if err := overrideCliFlags(&conf, b.flags); err != nil { return nil, err } } diff --git a/internal/config/cli.go b/internal/config/cli.go index d5bb6ed..b56f6a0 100644 --- a/internal/config/cli.go +++ b/internal/config/cli.go @@ -6,8 +6,7 @@ import ( "github.com/spf13/pflag" ) -// Read cli flag values into the config -func (cfg *Config) ReadCliFlags(flags *pflag.FlagSet) error { +func overrideCliFlags(cfg *Config, flags *pflag.FlagSet) error { logFile, _ := flags.GetString("log") if len(logFile) > 0 { cfg.Log.Directory = path.Dir(logFile) From cb207b2f3385df281aa7bfcadacba914ba740c69 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 18 Feb 2024 11:00:58 +0100 Subject: [PATCH 117/291] cmd/thalos/main.go: Add start-block and end-block flags. --- cmd/thalos/main.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmd/thalos/main.go b/cmd/thalos/main.go index 994cdd2..c31c956 100644 --- a/cmd/thalos/main.go +++ b/cmd/thalos/main.go @@ -34,6 +34,9 @@ func init() { flags.StringP("pid", "p", "", "Where to write process id") flags.BoolP("no-state-cache", "n", false, "Force the application to take start block from config/api") + flags.Int("start-block", 0, "Start to stream from this block (default: config value, cache, head from api)") + flags.Int("end-block", 0, "Stop streaming when this block is reached") + rootCmd.PersistentFlags().AddFlagSet(&flags) } From 117f1b50b4730499c822eff93308b076fd3d0af8 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 18 Feb 2024 11:02:11 +0100 Subject: [PATCH 118/291] internal/config/builder.go: bind start-block, end-block flags to config --- internal/config/builder.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/internal/config/builder.go b/internal/config/builder.go index b9c0e75..35f0056 100644 --- a/internal/config/builder.go +++ b/internal/config/builder.go @@ -21,7 +21,10 @@ type Builder struct { func NewBuilder() *Builder { return &Builder{ - binds: map[string]string{}, + binds: map[string]string{ + "ship.start_block_num": "start-block", + "ship.end_block_num": "end-block", + }, } } From beb5b6cf0465e2a4beed72adfb72f50ffa876f5f Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 18 Feb 2024 11:44:19 +0100 Subject: [PATCH 119/291] cmd/thalos/main.go: move cli flags to internal/config/cli.go as it is easier to write tests if we can get a hold of the flags. --- cmd/thalos/main.go | 14 ++------------ internal/config/builder_test.go | 6 ++---- internal/config/cli.go | 14 ++++++++++++++ 3 files changed, 18 insertions(+), 16 deletions(-) diff --git a/cmd/thalos/main.go b/cmd/thalos/main.go index c31c956..b2d8f33 100644 --- a/cmd/thalos/main.go +++ b/cmd/thalos/main.go @@ -1,9 +1,9 @@ package main import ( + "github.com/eosswedenorg/thalos/internal/config" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" - "github.com/spf13/pflag" ) var VersionString string = "dev" @@ -27,17 +27,7 @@ func init() { `) rootCmd.SetVersionTemplate(`{{with .Name}}{{printf "%s " .}}{{end}}{{printf "v%s" .Version}}` + "\n") - flags := pflag.FlagSet{} - flags.StringP("config", "c", "./config.yml", "Config file to read") - flags.StringP("level", "L", "info", "Log level to use") - flags.StringP("log", "l", "", "Path to log file (default: print to stdout/stderr)") - flags.StringP("pid", "p", "", "Where to write process id") - flags.BoolP("no-state-cache", "n", false, "Force the application to take start block from config/api") - - flags.Int("start-block", 0, "Start to stream from this block (default: config value, cache, head from api)") - flags.Int("end-block", 0, "Stop streaming when this block is reached") - - rootCmd.PersistentFlags().AddFlagSet(&flags) + rootCmd.PersistentFlags().AddFlagSet(config.GetFlags()) } func main() { diff --git a/internal/config/builder_test.go b/internal/config/builder_test.go index e7ceaf5..8ff339f 100644 --- a/internal/config/builder_test.go +++ b/internal/config/builder_test.go @@ -6,7 +6,6 @@ import ( "time" "github.com/eosswedenorg/thalos/internal/log" - "github.com/spf13/pflag" "github.com/stretchr/testify/require" shipclient "github.com/eosswedenorg-go/antelope-ship-client" @@ -132,14 +131,13 @@ redis: } func TestBuilder_Flags(t *testing.T) { - flags := pflag.FlagSet{} - flags.StringP("log", "l", "", "") + flags := GetFlags() require.NoError(t, flags.Set("log", "/path/to/logs")) cfg, err := NewBuilder(). SetSource(bytes.NewReader([]byte(``))). - SetFlags(&flags). + SetFlags(flags). Build() expected := New() diff --git a/internal/config/cli.go b/internal/config/cli.go index b56f6a0..0993601 100644 --- a/internal/config/cli.go +++ b/internal/config/cli.go @@ -6,6 +6,20 @@ import ( "github.com/spf13/pflag" ) +func GetFlags() *pflag.FlagSet { + flags := pflag.FlagSet{} + flags.StringP("config", "c", "./config.yml", "Config file to read") + flags.StringP("level", "L", "info", "Log level to use") + flags.StringP("log", "l", "", "Path to log file (default: print to stdout/stderr)") + flags.StringP("pid", "p", "", "Where to write process id") + flags.BoolP("no-state-cache", "n", false, "Force the application to take start block from config/api") + + flags.Int("start-block", 0, "Start to stream from this block (default: config value, cache, head from api)") + flags.Int("end-block", 0, "Stop streaming when this block is reached") + + return &flags +} + func overrideCliFlags(cfg *Config, flags *pflag.FlagSet) error { logFile, _ := flags.GetString("log") if len(logFile) > 0 { From 2db0a64bd4715d7e50c2deb5a36e9402ad062eb8 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 18 Feb 2024 15:45:10 +0100 Subject: [PATCH 120/291] internal/config: add additional flags for config fields. --- internal/config/builder.go | 27 +++++++++++++++-- internal/config/builder_test.go | 51 ++++++++++++++++++++++++++++++--- internal/config/cli.go | 38 ++++++++++++++++++++++-- 3 files changed, 107 insertions(+), 9 deletions(-) diff --git a/internal/config/builder.go b/internal/config/builder.go index 35f0056..32ded36 100644 --- a/internal/config/builder.go +++ b/internal/config/builder.go @@ -22,8 +22,31 @@ type Builder struct { func NewBuilder() *Builder { return &Builder{ binds: map[string]string{ - "ship.start_block_num": "start-block", - "ship.end_block_num": "end-block", + "api": "url", + "message_codec": "codec", + + // Redis + "redis.addr": "redis-addr", + "redis.user": "redis-user", + "redis.password": "redis-password", + "redis.db": "redis-db", + "redis.prefix": "redis-prefix", + + // Telegram + "telegram.id": "telegram-id", + "telegram.channel": "telegram-channel", + + // Log + "log.maxfilesize": "log-max-filesize", + "log.maxtime": "log-max-time", + + // Ship + "ship.url": "ship-url", + "ship.start_block_num": "start-block", + "ship.end_block_num": "end-block", + "ship.irreversible_only": "irreversible-only", + "ship.max_messages_in_flight": "max-msg-in-flight", + "ship.chain": "chain", }, } } diff --git a/internal/config/builder_test.go b/internal/config/builder_test.go index 8ff339f..01e1c67 100644 --- a/internal/config/builder_test.go +++ b/internal/config/builder_test.go @@ -133,16 +133,59 @@ redis: func TestBuilder_Flags(t *testing.T) { flags := GetFlags() - require.NoError(t, flags.Set("log", "/path/to/logs")) + require.NoError(t, flags.Set("url", "https://myapi")) + require.NoError(t, flags.Set("codec", "binary")) + require.NoError(t, flags.Set("redis-addr", "154.223.38.15:6380")) + require.NoError(t, flags.Set("redis-user", "myuser")) + require.NoError(t, flags.Set("redis-password", "secret123")) + require.NoError(t, flags.Set("redis-db", "3")) + require.NoError(t, flags.Set("redis-prefix", "custom-prefix")) + require.NoError(t, flags.Set("telegram-id", "72983126312982618")) + require.NoError(t, flags.Set("telegram-channel", "-293492332")) + require.NoError(t, flags.Set("log", "/path/to/logs/mylog")) + require.NoError(t, flags.Set("log-max-filesize", "25mb")) + require.NoError(t, flags.Set("log-max-time", "10m")) + require.NoError(t, flags.Set("ship-url", "ws://myship.com:7823")) + require.NoError(t, flags.Set("start-block", "7327833")) + require.NoError(t, flags.Set("end-block", "329408392")) + require.NoError(t, flags.Set("irreversible-only", "true")) + require.NoError(t, flags.Set("max-msg-in-flight", "98")) + require.NoError(t, flags.Set("chain", "wax")) cfg, err := NewBuilder(). SetSource(bytes.NewReader([]byte(``))). SetFlags(flags). Build() - expected := New() - expected.Log.Filename = "logs" - expected.Log.Directory = "/path/to" + expected := Config{ + Api: "https://myapi", + MessageCodec: "binary", + Log: log.Config{ + Filename: "mylog", + Directory: "/path/to/logs", + MaxFileSize: 25 * 1000 * 1000, // 25 mb + MaxTime: time.Minute * 10, + }, + Ship: ShipConfig{ + Url: "ws://myship.com:7823", + StartBlockNum: 7327833, + EndBlockNum: 329408392, + MaxMessagesInFlight: 98, + IrreversibleOnly: true, + Chain: "wax", + }, + Telegram: TelegramConfig{ + Id: "72983126312982618", + Channel: -293492332, + }, + Redis: RedisConfig{ + Addr: "154.223.38.15:6380", + User: "myuser", + Password: "secret123", + DB: 3, + Prefix: "custom-prefix", + }, + } require.NoError(t, err) require.Equal(t, &expected, cfg) diff --git a/internal/config/cli.go b/internal/config/cli.go index 0993601..5baecdb 100644 --- a/internal/config/cli.go +++ b/internal/config/cli.go @@ -2,20 +2,52 @@ package config import ( "path" + "time" + shipclient "github.com/eosswedenorg-go/antelope-ship-client" "github.com/spf13/pflag" ) func GetFlags() *pflag.FlagSet { flags := pflag.FlagSet{} + + // Cli only flags flags.StringP("config", "c", "./config.yml", "Config file to read") flags.StringP("level", "L", "info", "Log level to use") - flags.StringP("log", "l", "", "Path to log file (default: print to stdout/stderr)") flags.StringP("pid", "p", "", "Where to write process id") flags.BoolP("no-state-cache", "n", false, "Force the application to take start block from config/api") - flags.Int("start-block", 0, "Start to stream from this block (default: config value, cache, head from api)") - flags.Int("end-block", 0, "Stop streaming when this block is reached") + // Generic + flags.StringP("url", "u", "", "Url to antelope api") + flags.String("codec", "json", "Codec used to send messages") + + // Redis + flags.String("redis-addr", "127.0.0.1:6379", "host:port to redis server") + flags.String("redis-user", "", "Redis username") + flags.String("redis-password", "", "Redis password") + flags.Int("redis-db", 0, "Redis database") + flags.String("redis-prefix", "ship", "Redis channel prefix") + + // Telegram + flags.String("telegram-id", "", "Id of telegram bot") + flags.Int64("telegram-channel", 0, "Telegram channel to send notifications to") + + // Log + flags.StringP("log", "l", "", "Path to log file (default: print to stdout/stderr)") + flags.String("log-max-filesize", "10mb", "Max filesize for logfile to rotate") + flags.Duration("log-max-time", time.Hour*24, "Max time for logfile to rotate") + + // Ship + flags.String("ship-url", "ws://127.0.0.1:8080", "Url to ship node") + flags.Uint32("start-block", shipclient.NULL_BLOCK_NUMBER, "Start to stream from this block") + flags.Uint32("end-block", shipclient.NULL_BLOCK_NUMBER, "Stop streaming when this block is reached") + + flags.Lookup("start-block").DefValue = "config value, cache, head from api" + flags.Lookup("end-block").DefValue = "none" + + flags.Bool("irreversible-only", false, "Only stream irreversible blocks from ship") + flags.Int("max-msg-in-flight", 10, "Maximum messages that can be sent from SHIP without acknowledgement") + flags.String("chain", "", "ChainID used in channel namespace, can be any string (default from api)") return &flags } From 4a4489e2be3972ea62dcfe516578c8ee71642c28 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 18 Feb 2024 16:04:02 +0100 Subject: [PATCH 121/291] internal/config/builder_test.go: adding test for config with flags --- internal/config/builder_test.go | 73 +++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/internal/config/builder_test.go b/internal/config/builder_test.go index 01e1c67..377c457 100644 --- a/internal/config/builder_test.go +++ b/internal/config/builder_test.go @@ -75,6 +75,79 @@ redis: require.Equal(t, &expected, cfg) } +func TestBuilder_ConfigWithFlags(t *testing.T) { + expected := Config{ + Name: "ship-reader-1", + Api: "https://api.example.com", + MessageCodec: "msgpack", + Log: log.Config{ + Filename: "mylog.log", + Directory: "/var/log", + MaxFileSize: 200, + MaxTime: 30 * time.Minute, + }, + Ship: ShipConfig{ + Url: "127.0.0.1:8089", + StartBlockNum: 23671836, + EndBlockNum: 23872222, + IrreversibleOnly: true, + MaxMessagesInFlight: 1337, + }, + Telegram: TelegramConfig{ + Id: "110201543:AAHdqTcvCH1vGWJxfSeofSAs0K5PALDsaw", + Channel: -123456789, + }, + Redis: RedisConfig{ + Addr: "localhost:6379", + User: "userfromcli", + Password: "passwd", + DB: 4, + Prefix: "some::ship", + }, + } + + builder := NewBuilder() + builder.SetSource(bytes.NewBuffer([]byte(` +name: "ship-reader-1" +api: "http://127.0.0.1:8080" +message_codec: "mojibake" +log: + filename: some_file.log + directory: /path/to/whatever + maxtime: 30m + maxfilesize: 200b +ship: + url: "127.0.0.1:8089" + irreversible_only: true + max_messages_in_flight: 1337 + start_block_num: 23671836 + end_block_num: 23872222 +telegram: + id: "110201543:AAHdqTcvCH1vGWJxfSeofSAs0K5PALDsaw" + channel: -123456789 +redis: + addr: "localhost:6379" + user: "myuser" + password: "passwd" + db: 4 + prefix: "some::ship" +`))) + + flags := GetFlags() + + require.NoError(t, flags.Set("url", "https://api.example.com")) + require.NoError(t, flags.Set("codec", "msgpack")) + require.NoError(t, flags.Set("log", "/var/log/mylog.log")) + require.NoError(t, flags.Set("redis-user", "userfromcli")) + + builder.SetFlags(flags) + + cfg, err := builder.Build() + + require.NoError(t, err) + require.Equal(t, &expected, cfg) +} + func TestBuilder_NilSource(t *testing.T) { cfg, err := NewBuilder().Build() require.Nil(t, cfg) From e54a4fa929f5a31963701915a21b1aeefa105365 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 18 Feb 2024 16:08:18 +0100 Subject: [PATCH 122/291] internal/config: remove shorthand ship config (where it is possible to have "ship" key only contain a url string instead of struct) It makes parsing harder and i don't think anyone uses it and its not that useful tbh. --- internal/config/builder.go | 1 - internal/config/builder_test.go | 51 --------------------------------- internal/config/config.go | 32 ++++----------------- 3 files changed, 6 insertions(+), 78 deletions(-) diff --git a/internal/config/builder.go b/internal/config/builder.go index 32ded36..d2ab2bf 100644 --- a/internal/config/builder.go +++ b/internal/config/builder.go @@ -109,7 +109,6 @@ func (b *Builder) Build() (*Config, error) { mapstructure.TextUnmarshallerHookFunc(), mapstructure.StringToTimeDurationHookFunc(), mapstructure.StringToSliceHookFunc(","), - decodeShorthandShipConfig, ) err := v.Unmarshal(&conf, viper.DecodeHook(decoders)) diff --git a/internal/config/builder_test.go b/internal/config/builder_test.go index 377c457..3f52548 100644 --- a/internal/config/builder_test.go +++ b/internal/config/builder_test.go @@ -7,8 +7,6 @@ import ( "github.com/eosswedenorg/thalos/internal/log" "github.com/stretchr/testify/require" - - shipclient "github.com/eosswedenorg-go/antelope-ship-client" ) func TestBuilder(t *testing.T) { @@ -154,55 +152,6 @@ func TestBuilder_NilSource(t *testing.T) { require.EqualError(t, err, "Config not set") } -func TestBuilder_WithShorthandShipUrl(t *testing.T) { - expected := Config{ - Name: "ship-reader-1", - Api: "http://127.0.0.1:8080", - MessageCodec: "json", - Log: log.Config{ - MaxFileSize: 10 * 1000 * 1000, // 10 mb - MaxTime: time.Hour * 24, - }, - Ship: ShipConfig{ - Url: "127.0.0.1:8089", - StartBlockNum: shipclient.NULL_BLOCK_NUMBER, - EndBlockNum: shipclient.NULL_BLOCK_NUMBER, - MaxMessagesInFlight: 10, - IrreversibleOnly: false, - }, - Telegram: TelegramConfig{ - Id: "110201543:AAHdqTcvCH1vGWJxfSeofSAs0K5PALDsaw", - Channel: -123456789, - }, - Redis: RedisConfig{ - Addr: "localhost:6379", - Password: "passwd", - DB: 4, - Prefix: "some::ship", - }, - } - - builder := NewBuilder() - builder.SetSource(bytes.NewBuffer([]byte(` -name: "ship-reader-1" -api: "http://127.0.0.1:8080" -ship: "127.0.0.1:8089" -telegram: - id: "110201543:AAHdqTcvCH1vGWJxfSeofSAs0K5PALDsaw" - channel: -123456789 -redis: - addr: "localhost:6379" - password: "passwd" - db: 4 - prefix: "some::ship" -`))) - - cfg, err := builder.Build() - - require.NoError(t, err) - require.Equal(t, &expected, cfg) -} - func TestBuilder_Flags(t *testing.T) { flags := GetFlags() diff --git a/internal/config/config.go b/internal/config/config.go index e137352..8edacdb 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1,7 +1,6 @@ package config import ( - "reflect" "time" "github.com/eosswedenorg/thalos/internal/log" @@ -52,34 +51,15 @@ func New() Config { MaxFileSize: 10 * 1000 * 1000, // 10 mb MaxTime: time.Hour * 24, }, - Ship: defaultShipConfig(""), + Ship: ShipConfig{ + StartBlockNum: shipclient.NULL_BLOCK_NUMBER, + EndBlockNum: shipclient.NULL_BLOCK_NUMBER, + MaxMessagesInFlight: 10, + IrreversibleOnly: false, + }, Redis: RedisConfig{ Addr: "localhost:6379", Prefix: "ship", }, } } - -func defaultShipConfig(url string) ShipConfig { - return ShipConfig{ - Url: url, - StartBlockNum: shipclient.NULL_BLOCK_NUMBER, - EndBlockNum: shipclient.NULL_BLOCK_NUMBER, - MaxMessagesInFlight: 10, - IrreversibleOnly: false, - } -} - -// mapstructure DecodeHook that can parse a shorthand ship config (only string instead of struct.) -func decodeShorthandShipConfig(from reflect.Value, to reflect.Value) (interface{}, error) { - shipType := reflect.TypeOf(ShipConfig{}) - - // If to is a struct and is assignable to a ShipConfig and from is a string. - // Then we treat the from value as ShipConfig.Url - if to.Kind() == reflect.Struct && to.Type().AssignableTo(shipType) && from.Kind() == reflect.String { - return defaultShipConfig(from.String()), nil - } - - // If not, decode as normal. - return from.Interface(), nil -} From 69a36e016c599bc77807a32b9ad96b0a49929cdc Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 18 Feb 2024 16:11:32 +0100 Subject: [PATCH 123/291] internal/config: skip creating a Config struct with default values. those should be set by flags or viper in the builder --- internal/config/builder.go | 2 +- internal/config/config.go | 25 ---------------------- internal/config/config_test.go | 38 ---------------------------------- 3 files changed, 1 insertion(+), 64 deletions(-) delete mode 100644 internal/config/config_test.go diff --git a/internal/config/builder.go b/internal/config/builder.go index d2ab2bf..6a55976 100644 --- a/internal/config/builder.go +++ b/internal/config/builder.go @@ -81,7 +81,7 @@ func (b *Builder) Build() (*Config, error) { return nil, errors.New("Config not set") } - conf := New() + conf := Config{} v := viper.New() v.SetConfigType("yaml") diff --git a/internal/config/config.go b/internal/config/config.go index 8edacdb..41777df 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1,11 +1,7 @@ package config import ( - "time" - "github.com/eosswedenorg/thalos/internal/log" - - shipclient "github.com/eosswedenorg-go/antelope-ship-client" ) type RedisConfig struct { @@ -42,24 +38,3 @@ type Config struct { Telegram TelegramConfig `yaml:"telegram" mapstructure:"telegram"` } - -// Create a new Config object with default values -func New() Config { - return Config{ - MessageCodec: "json", - Log: log.Config{ - MaxFileSize: 10 * 1000 * 1000, // 10 mb - MaxTime: time.Hour * 24, - }, - Ship: ShipConfig{ - StartBlockNum: shipclient.NULL_BLOCK_NUMBER, - EndBlockNum: shipclient.NULL_BLOCK_NUMBER, - MaxMessagesInFlight: 10, - IrreversibleOnly: false, - }, - Redis: RedisConfig{ - Addr: "localhost:6379", - Prefix: "ship", - }, - } -} diff --git a/internal/config/config_test.go b/internal/config/config_test.go deleted file mode 100644 index c919a18..0000000 --- a/internal/config/config_test.go +++ /dev/null @@ -1,38 +0,0 @@ -package config - -import ( - "testing" - "time" - - "github.com/eosswedenorg/thalos/internal/log" - "github.com/stretchr/testify/require" - - shipclient "github.com/eosswedenorg-go/antelope-ship-client" -) - -func TestNew(t *testing.T) { - expected := Config{ - MessageCodec: "json", - - Log: log.Config{ - MaxFileSize: 10 * 1000 * 1000, // 10 mb - MaxTime: time.Hour * 24, - }, - - Ship: ShipConfig{ - StartBlockNum: shipclient.NULL_BLOCK_NUMBER, - EndBlockNum: shipclient.NULL_BLOCK_NUMBER, - MaxMessagesInFlight: 10, - IrreversibleOnly: false, - }, - - Redis: RedisConfig{ - Addr: "localhost:6379", - Password: "", - DB: 0, - Prefix: "ship", - }, - } - - require.Equal(t, expected, New()) -} From 53baae8a7ff629e65963602e0cb6c955c2b36f20 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 18 Feb 2024 17:40:04 +0100 Subject: [PATCH 124/291] internal/config/cli.go: move flags that are not bound to config to cmd/thalos/main.go --- cmd/thalos/main.go | 9 +++++++++ internal/config/cli.go | 6 ------ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/cmd/thalos/main.go b/cmd/thalos/main.go index b2d8f33..fc55eef 100644 --- a/cmd/thalos/main.go +++ b/cmd/thalos/main.go @@ -4,6 +4,7 @@ import ( "github.com/eosswedenorg/thalos/internal/config" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" + "github.com/spf13/pflag" ) var VersionString string = "dev" @@ -27,6 +28,14 @@ func init() { `) rootCmd.SetVersionTemplate(`{{with .Name}}{{printf "%s " .}}{{end}}{{printf "v%s" .Version}}` + "\n") + flags := pflag.FlagSet{} + + flags.StringP("config", "c", "./config.yml", "Config file to read") + flags.StringP("level", "L", "info", "Log level to use") + flags.StringP("pid", "p", "", "Where to write process id") + flags.BoolP("no-state-cache", "n", false, "Force the application to take start block from config/api") + + rootCmd.PersistentFlags().AddFlagSet(&flags) rootCmd.PersistentFlags().AddFlagSet(config.GetFlags()) } diff --git a/internal/config/cli.go b/internal/config/cli.go index 5baecdb..c5b3c95 100644 --- a/internal/config/cli.go +++ b/internal/config/cli.go @@ -11,12 +11,6 @@ import ( func GetFlags() *pflag.FlagSet { flags := pflag.FlagSet{} - // Cli only flags - flags.StringP("config", "c", "./config.yml", "Config file to read") - flags.StringP("level", "L", "info", "Log level to use") - flags.StringP("pid", "p", "", "Where to write process id") - flags.BoolP("no-state-cache", "n", false, "Force the application to take start block from config/api") - // Generic flags.StringP("url", "u", "", "Url to antelope api") flags.String("codec", "json", "Codec used to send messages") From 7f1f186aa02cbad418c0585e791bb07e3b3bcfc2 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 18 Feb 2024 18:00:26 +0100 Subject: [PATCH 125/291] nternal/config/cli.go: move overrideCliFlags() to cmd/thalos/server.go as its pretty specific code. --- cmd/thalos/server.go | 14 +++++++++++++- internal/config/builder.go | 7 ------- internal/config/builder_test.go | 8 ++------ internal/config/cli.go | 10 ---------- 4 files changed, 15 insertions(+), 24 deletions(-) diff --git a/cmd/thalos/server.go b/cmd/thalos/server.go index 1a5c95b..5103b5a 100644 --- a/cmd/thalos/server.go +++ b/cmd/thalos/server.go @@ -7,6 +7,7 @@ import ( "io" "os" "os/signal" + "path" "syscall" "time" @@ -204,10 +205,21 @@ func GetConfig(flags *pflag.FlagSet) (*config.Config, error) { return nil, err } - return config.NewBuilder(). + cfg, err := config.NewBuilder(). SetConfigFile(filename). SetFlags(flags). Build() + if err != nil { + return nil, err + } + + logFile, _ := flags.GetString("log") + if len(logFile) > 0 { + cfg.Log.Directory = path.Dir(logFile) + cfg.Log.Filename = path.Base(logFile) + } + + return cfg, nil } func serverCmd(cmd *cobra.Command, args []string) { diff --git a/internal/config/builder.go b/internal/config/builder.go index 6a55976..5257e3d 100644 --- a/internal/config/builder.go +++ b/internal/config/builder.go @@ -116,12 +116,5 @@ func (b *Builder) Build() (*Config, error) { return nil, err } - // Call custom handler. - if b.flags != nil { - if err := overrideCliFlags(&conf, b.flags); err != nil { - return nil, err - } - } - return &conf, nil } diff --git a/internal/config/builder_test.go b/internal/config/builder_test.go index 3f52548..9bc1589 100644 --- a/internal/config/builder_test.go +++ b/internal/config/builder_test.go @@ -110,8 +110,8 @@ name: "ship-reader-1" api: "http://127.0.0.1:8080" message_codec: "mojibake" log: - filename: some_file.log - directory: /path/to/whatever + filename: mylog.log + directory: /var/log maxtime: 30m maxfilesize: 200b ship: @@ -135,7 +135,6 @@ redis: require.NoError(t, flags.Set("url", "https://api.example.com")) require.NoError(t, flags.Set("codec", "msgpack")) - require.NoError(t, flags.Set("log", "/var/log/mylog.log")) require.NoError(t, flags.Set("redis-user", "userfromcli")) builder.SetFlags(flags) @@ -164,7 +163,6 @@ func TestBuilder_Flags(t *testing.T) { require.NoError(t, flags.Set("redis-prefix", "custom-prefix")) require.NoError(t, flags.Set("telegram-id", "72983126312982618")) require.NoError(t, flags.Set("telegram-channel", "-293492332")) - require.NoError(t, flags.Set("log", "/path/to/logs/mylog")) require.NoError(t, flags.Set("log-max-filesize", "25mb")) require.NoError(t, flags.Set("log-max-time", "10m")) require.NoError(t, flags.Set("ship-url", "ws://myship.com:7823")) @@ -183,8 +181,6 @@ func TestBuilder_Flags(t *testing.T) { Api: "https://myapi", MessageCodec: "binary", Log: log.Config{ - Filename: "mylog", - Directory: "/path/to/logs", MaxFileSize: 25 * 1000 * 1000, // 25 mb MaxTime: time.Minute * 10, }, diff --git a/internal/config/cli.go b/internal/config/cli.go index c5b3c95..d831b7a 100644 --- a/internal/config/cli.go +++ b/internal/config/cli.go @@ -1,7 +1,6 @@ package config import ( - "path" "time" shipclient "github.com/eosswedenorg-go/antelope-ship-client" @@ -45,12 +44,3 @@ func GetFlags() *pflag.FlagSet { return &flags } - -func overrideCliFlags(cfg *Config, flags *pflag.FlagSet) error { - logFile, _ := flags.GetString("log") - if len(logFile) > 0 { - cfg.Log.Directory = path.Dir(logFile) - cfg.Log.Filename = path.Base(logFile) - } - return nil -} From 146ea99298f3e745e2002d582648285a56f9ece3 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 18 Feb 2024 18:03:38 +0100 Subject: [PATCH 126/291] cmd/thalos/server.go: in GetConfig() should override "no-state-cache" flag if "start-block" flag is set. --- cmd/thalos/server.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/cmd/thalos/server.go b/cmd/thalos/server.go index 5103b5a..3a98d24 100644 --- a/cmd/thalos/server.go +++ b/cmd/thalos/server.go @@ -219,6 +219,13 @@ func GetConfig(flags *pflag.FlagSet) (*config.Config, error) { cfg.Log.Filename = path.Base(logFile) } + // If start-block is provided, we should set no-state-cache to true. + if startBlock := flags.Lookup("start-block"); startBlock != nil && startBlock.Changed { + if err := flags.Set("no-state-cache", "true"); err != nil { + return cfg, nil + } + } + return cfg, nil } @@ -226,8 +233,6 @@ func serverCmd(cmd *cobra.Command, args []string) { var err error var chainInfo *eos.InfoResp - skip_currentblock_cache, _ := cmd.Flags().GetBool("no-state-cache") - // Write PID file pidFile, err := cmd.Flags().GetString("pid") if err != nil { @@ -245,6 +250,8 @@ func serverCmd(cmd *cobra.Command, args []string) { return } + skip_currentblock_cache, _ := cmd.Flags().GetBool("no-state-cache") + flagLevel, _ := cmd.Flags().GetString("level") lvl, err := log.ParseLevel(flagLevel) if err == nil { From fb3b35cbe93c50329171746d29082a6b3a365f4d Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 18 Feb 2024 18:09:12 +0100 Subject: [PATCH 127/291] cmd/thalos/server.go: in stateLoader(): set "cli" as source if start block is set via cli flag. --- cmd/thalos/server.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/cmd/thalos/server.go b/cmd/thalos/server.go index 3a98d24..aa10fac 100644 --- a/cmd/thalos/server.go +++ b/cmd/thalos/server.go @@ -154,7 +154,7 @@ func initAbiManager(api *eos.API, store cache.Store, chain_id string) *abi.AbiMa return abi.NewAbiManager(cache, api) } -func stateLoader(conf *config.Config, chainInfo *eos.InfoResp, cache *cache.Cache, current_block_no_cache bool) StateLoader { +func stateLoader(conf *config.Config, start_block_flag *pflag.Flag, chainInfo *eos.InfoResp, cache *cache.Cache, current_block_no_cache bool) StateLoader { return func(state *State) { var source string @@ -168,7 +168,13 @@ func stateLoader(conf *config.Config, chainInfo *eos.InfoResp, cache *cache.Cach if current_block_no_cache || err != nil { // Set from config if we have a sane value. if conf.Ship.StartBlockNum != shipclient.NULL_BLOCK_NUMBER { - source = "config" + + if start_block_flag != nil && start_block_flag.Changed { + source = "cli" + } else { + source = "config" + } + state.CurrentBlock = conf.Ship.StartBlockNum } else { // Otherwise, set from api. @@ -355,7 +361,7 @@ func serverCmd(cmd *cobra.Command, args []string) { processor := SpawnProccessor( shClient, - stateLoader(conf, chainInfo, cache, skip_currentblock_cache), + stateLoader(conf, cmd.Flags().Lookup("start-block"), chainInfo, cache, skip_currentblock_cache), stateSaver(cache), driver.NewPublisher(context.Background(), rdb, api_redis.Namespace{ Prefix: conf.Redis.Prefix, From 368d3f36bf3152c5cfd50d5e075c3f61455bd67a Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 18 Feb 2024 18:10:46 +0100 Subject: [PATCH 128/291] cmd/thalos/server.go: remove debug call to Println() --- cmd/thalos/server.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/cmd/thalos/server.go b/cmd/thalos/server.go index 4170296..dcf2f76 100644 --- a/cmd/thalos/server.go +++ b/cmd/thalos/server.go @@ -250,8 +250,6 @@ func serverCmd(cmd *cobra.Command, args []string) { } if len(conf.Log.Filename) > 0 { - fmt.Println(conf.Log.Filename) - stdWriter, err := NewRotatingFileFromConfig(conf.Log, "info") if err != nil { log.WithError(err).Fatal("Failed to set standard log file") From 432dfab4100f817781caebb199d337f0cfc826de Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 18 Feb 2024 18:39:55 +0100 Subject: [PATCH 129/291] cmd/thalos/server.go: only fetch chain info from api once (or zero for code paths that does not need it). --- cmd/thalos/server.go | 41 ++++++++++++++++++++++++++++++----------- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/cmd/thalos/server.go b/cmd/thalos/server.go index dcf2f76..e5afc23 100644 --- a/cmd/thalos/server.go +++ b/cmd/thalos/server.go @@ -153,7 +153,7 @@ func initAbiManager(api *eos.API, store cache.Store, chain_id string) *abi.AbiMa return abi.NewAbiManager(cache, api) } -func stateLoader(conf config.Config, chainInfo *eos.InfoResp, cache *cache.Cache, current_block_no_cache bool) StateLoader { +func stateLoader(conf config.Config, chainInfo func() *eos.InfoResp, cache *cache.Cache, current_block_no_cache bool) StateLoader { return func(state *State) { var source string @@ -173,10 +173,10 @@ func stateLoader(conf config.Config, chainInfo *eos.InfoResp, cache *cache.Cache // Otherwise, set from api. if conf.Ship.IrreversibleOnly { source = "api (LIB)" - state.CurrentBlock = uint32(chainInfo.LastIrreversibleBlockNum) + state.CurrentBlock = uint32(chainInfo().LastIrreversibleBlockNum) } else { source = "api (HEAD)" - state.CurrentBlock = uint32(chainInfo.HeadBlockNum) + state.CurrentBlock = uint32(chainInfo().HeadBlockNum) } } } else { @@ -217,9 +217,32 @@ func ReadConfig(cfg *config.Config, flags *pflag.FlagSet) error { return nil } +// "Clever" way to make sure we only call the api once. +// Store a info pointer outside the returned closure. +// that pointer will live as long as the closure lives. +// and inside the closure we will reference the pointer and only +// call the api if it is nil. +func chainInfoOnce(api *eos.API) func() *eos.InfoResp { + var info *eos.InfoResp + return func() *eos.InfoResp { + if info == nil { + + log.WithField("api", api.BaseURL).Info("Get chain info from api") + + result, err := api.GetInfo(context.Background()) + if err != nil { + log.WithError(err).Fatal("Failed to call eos api") + return nil + } + + info = result + } + return info + } +} + func serverCmd(cmd *cobra.Command, args []string) { var err error - var chainInfo *eos.InfoResp skip_currentblock_cache, _ := cmd.Flags().GetBool("no-state-cache") @@ -313,13 +336,7 @@ func serverCmd(cmd *cobra.Command, args []string) { // Setup general cache cache := NewCache("thalos::cache::instance::"+conf.Name, cacheStore) - log.WithField("api", conf.Api).Info("Get chain info from api") eosClient := eos.New(conf.Api) - chainInfo, err = eosClient.GetInfo(context.Background()) - if err != nil { - log.WithError(err).Fatal("Failed to call eos api") - return - } shClient := shipclient.NewStream(func(s *shipclient.Stream) { s.StartBlock = conf.Ship.StartBlockNum @@ -334,9 +351,11 @@ func serverCmd(cmd *cobra.Command, args []string) { return } + chainInfo := chainInfoOnce(eosClient) + chain_id := conf.Ship.Chain if len(chain_id) < 1 { - chain_id = chainInfo.ChainID.String() + chain_id = chainInfo().ChainID.String() } processor := SpawnProccessor( From 53c7053177ff811a0c1e34e61633b1ff81685f4d Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 18 Feb 2024 19:57:03 +0100 Subject: [PATCH 130/291] .github/workflows/test.yml: update actions --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 679909e..ca634f7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,10 +14,10 @@ jobs: runs-on: ubuntu-latest name: Test (go v${{ matrix.go-version }}) steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v5 with: go-version: ${{ matrix.go-version }} From 6cc26c269c1de87f00163125f7e977c318aa6c8a Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 18 Feb 2024 19:59:12 +0100 Subject: [PATCH 131/291] .github/workflows/release.yml: update actions --- .github/workflows/release.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2f505c8..e974e70 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -14,10 +14,10 @@ jobs: name: Crosscompile - ${{matrix.os}}-${{matrix.arch}} runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v5 with: go-version: 1.21 @@ -67,10 +67,10 @@ jobs: name: Package - ${{matrix.os}} runs-on: ${{matrix.os}} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v5 with: go-version: 1.21 From 1799fe4ebff15b8a055e2159d4cc0bca16ecf162 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 18 Feb 2024 20:00:24 +0100 Subject: [PATCH 132/291] README.md: update build command --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 85700c0..e96fd70 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ $ make or using go directly if you dont have make installed. ```shell -$ go build -o build/thalos-server cmd/main/main.go +$ go build -o build/thalos-server cmd/thalos/*.go ``` ## Author From fb62e106675d6f556c0406384a2cf56ab480e482 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 18 Feb 2024 20:06:51 +0100 Subject: [PATCH 133/291] cmd/thalos/server.go: Set context timeout to eos api call. --- cmd/thalos/server.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cmd/thalos/server.go b/cmd/thalos/server.go index 65d04ec..41f87b7 100644 --- a/cmd/thalos/server.go +++ b/cmd/thalos/server.go @@ -247,7 +247,10 @@ func chainInfoOnce(api *eos.API) func() *eos.InfoResp { log.WithField("api", api.BaseURL).Info("Get chain info from api") - result, err := api.GetInfo(context.Background()) + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + + result, err := api.GetInfo(ctx) if err != nil { log.WithError(err).Fatal("Failed to call eos api") return nil From 1658b3ea99ed2657c0b5e73c92b9eb8887f542f7 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 18 Feb 2024 20:07:35 +0100 Subject: [PATCH 134/291] cmd/thalos/server.go: parse config before doing anything else. --- cmd/thalos/server.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/cmd/thalos/server.go b/cmd/thalos/server.go index 41f87b7..23b5243 100644 --- a/cmd/thalos/server.go +++ b/cmd/thalos/server.go @@ -265,6 +265,13 @@ func chainInfoOnce(api *eos.API) func() *eos.InfoResp { func serverCmd(cmd *cobra.Command, args []string) { var err error + // Parse config + conf, err := GetConfig(cmd.Flags()) + if err != nil { + log.WithError(err).Fatal("Failed to read config") + return + } + // Write PID file pidFile, err := cmd.Flags().GetString("pid") if err != nil { @@ -275,13 +282,6 @@ func serverCmd(cmd *cobra.Command, args []string) { } } - // Parse config - conf, err := GetConfig(cmd.Flags()) - if err != nil { - log.WithError(err).Fatal("Failed to read config") - return - } - skip_currentblock_cache, _ := cmd.Flags().GetBool("no-state-cache") flagLevel, _ := cmd.Flags().GetString("level") From 9dcb8c4f0cc159991def022215c3a558fdaf2375 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 18 Feb 2024 20:22:43 +0100 Subject: [PATCH 135/291] cmd/thalos/server.go: move redis connect code to it's own function and log connection attempt. --- cmd/thalos/server.go | 37 ++++++++++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/cmd/thalos/server.go b/cmd/thalos/server.go index 23b5243..9191666 100644 --- a/cmd/thalos/server.go +++ b/cmd/thalos/server.go @@ -8,6 +8,7 @@ import ( "os" "os/signal" "path" + "strings" "syscall" "time" @@ -262,6 +263,32 @@ func chainInfoOnce(api *eos.API) func() *eos.InfoResp { } } +func ConnectRedis(conf *config.RedisConfig) (*redis.Client, error) { + logEntry := log.WithFields(log.Fields{ + "addr": conf.Addr, + "db": conf.DB, + }) + + if len(conf.User) > 0 { + logEntry = logEntry.WithField("user", conf.User) + } + + if len(conf.Password) > 0 { + logEntry = logEntry.WithField("password", strings.Repeat("*", len(conf.Password))) + } + + logEntry.Info("Connecting to redis") + + rdb := redis.NewClient(&redis.Options{ + Addr: conf.Addr, + Username: conf.User, + Password: conf.Password, + DB: conf.DB, + }) + + return rdb, rdb.Ping(context.Background()).Err() +} + func serverCmd(cmd *cobra.Command, args []string) { var err error @@ -333,15 +360,7 @@ func serverCmd(cmd *cobra.Command, args []string) { notify.UseServices(telegram) } - // Connect to redis - rdb := redis.NewClient(&redis.Options{ - Addr: conf.Redis.Addr, - Username: conf.Redis.User, - Password: conf.Redis.Password, - DB: conf.Redis.DB, - }) - - err = rdb.Ping(context.Background()).Err() + rdb, err := ConnectRedis(&conf.Redis) if err != nil { log.WithError(err).Fatal("Failed to connect to redis") return From 3cb25a5717d51837644fd4e14601a72d1612f878 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 18 Feb 2024 20:28:02 +0100 Subject: [PATCH 136/291] cmd/thalos/server.go: Don't log error if we get a (normal) close message from ship. --- cmd/thalos/server.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cmd/thalos/server.go b/cmd/thalos/server.go index 9191666..a9b38b3 100644 --- a/cmd/thalos/server.go +++ b/cmd/thalos/server.go @@ -15,6 +15,7 @@ import ( "github.com/cenkalti/backoff/v4" eos "github.com/eoscanada/eos-go" shipclient "github.com/eosswedenorg-go/antelope-ship-client" + shipws "github.com/eosswedenorg-go/antelope-ship-client/websocket" "github.com/eosswedenorg-go/pid" "github.com/eosswedenorg/thalos/api/message" _ "github.com/eosswedenorg/thalos/api/message/json" @@ -113,6 +114,11 @@ func readerLoop(conf *config.Config, running *bool, shClient *shipclient.Stream, return } + if shipws.IsCloseError(err) { + log.Info("SHIP Connection closed") + return + } + log.WithError(err).Error("Failed to read from ship") } } From bc46e516d9f8a808a6769817c53a6518ae82fcf1 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 18 Feb 2024 20:35:23 +0100 Subject: [PATCH 137/291] cmd/thalos/server.go: prefix ship connection logs with "ship client" and cleanup the messages abit. --- cmd/thalos/server.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/cmd/thalos/server.go b/cmd/thalos/server.go index a9b38b3..683bbe5 100644 --- a/cmd/thalos/server.go +++ b/cmd/thalos/server.go @@ -56,7 +56,7 @@ func readerLoop(conf *config.Config, running *bool, shClient *shipclient.Stream, "max_interval": exp.MaxInterval, "randomization_factor": exp.RandomizationFactor, "multiplier": exp.Multiplier, - }).Info("Connecting with Exponential Backoff") + }).Info("ship client: Connecting with Exponential Backoff") connectOp := func() error { recon_cnt++ @@ -64,7 +64,7 @@ func readerLoop(conf *config.Config, running *bool, shClient *shipclient.Stream, log.WithFields(log.Fields{ "url": conf.Ship.Url, "try": recon_cnt, - }).Info("Connecting to ship") + }).Info("ship client: Connecting") if err := shClient.Connect(conf.Ship.Url); err != nil { return err @@ -89,15 +89,15 @@ func readerLoop(conf *config.Config, running *bool, shClient *shipclient.Stream, recon_cnt = 0 } - log.WithError(err).Error("Failed to connect to SHIP") + log.WithError(err).Error("ship client: Failed to connect") log.WithFields(log.Fields{ "reconn_at": time.Now().Add(d), "reconn_in": d, - }).Info("Reconnecting in ", d) + }).Info("ship client: Reconnecting in ", d) }) if err != nil { - log.WithError(err).Error("Failed to connect to SHIP") + log.WithError(err).Error("ship client:Failed to connect") return } @@ -105,21 +105,21 @@ func readerLoop(conf *config.Config, running *bool, shClient *shipclient.Stream, log.WithFields(log.Fields{ "start": shClient.StartBlock, "end": shClient.EndBlock, - }).Info("Connected to ship") + }).Info("ship client: Connected to ship") if err := shClient.Run(); err != nil { if errors.Is(err, shipclient.ErrEndBlockReached) { - log.Info("Endblock reached.") + log.Info("ship client: Endblock reached.") return } if shipws.IsCloseError(err) { - log.Info("SHIP Connection closed") + log.Info("ship client: Connection closed normally") return } - log.WithError(err).Error("Failed to read from ship") + log.WithError(err).Error("ship client: Failed to read message") } } } From de7c1b160c02056d806850f999c16341e7880686 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Mon, 19 Feb 2024 13:48:16 +0100 Subject: [PATCH 138/291] debian/thalos.postinst: indent fix --- debian/thalos.postinst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/thalos.postinst b/debian/thalos.postinst index bad0440..9c31f4b 100644 --- a/debian/thalos.postinst +++ b/debian/thalos.postinst @@ -2,8 +2,8 @@ set -e if [ "$1" = 'configure' ]; then - adduser --force-badname --system --home /nonexistent \ - --group --no-create-home --quiet thalos || true + adduser --force-badname --system --home /nonexistent \ + --group --no-create-home --quiet thalos || true # Create log directory mkdir -p /var/log/thalos From 39135ab781186bb0ba27a02effe18a4997ae624e Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Mon, 19 Feb 2024 15:02:18 +0100 Subject: [PATCH 139/291] makefile: adding build-deb target. --- Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile b/Makefile index 57567ad..a9d2b82 100644 --- a/Makefile +++ b/Makefile @@ -28,6 +28,9 @@ install: build tools install-scripts: install -m 755 -t $(DESTDIR) scripts/start.sh scripts/stop.sh +build-deb: + dpkg-buildpackage -b -us -uc + test: $(GO) test -v ./... From a2a9e106a36fcc7bc02f5a006aa4be4d8b5b2a5f Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Mon, 19 Feb 2024 15:07:17 +0100 Subject: [PATCH 140/291] .github/workflows/release.yml: use make to build debian package --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e974e70..5867d10 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -82,7 +82,7 @@ jobs: - name: Package id: package run: | - dpkg-buildpackage -b -us -uc + make build-deb DEB_FILE=$(ls ../*.deb | head -1) echo "deb_filename=$DEB_FILE" >> "$GITHUB_OUTPUT" echo "deb_name=$(basename $DEB_FILE)" >> "$GITHUB_OUTPUT" From e046484d9b6a360abc21577cab5d97dd0e597fb7 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Mon, 19 Feb 2024 14:46:22 +0100 Subject: [PATCH 141/291] Version 1.1.0-rc1 --- Makefile | 2 +- debian/changelog | 19 ++++++++++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index a9d2b82..82e1cad 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ GO=go GOLDFLAGS=-v -s -w -X main.VersionString=$(PROGRAM_VERSION) GOBUILDFLAGS=-v -p $(shell nproc) -ldflags="$(GOLDFLAGS)" PROGRAM=thalos-server -PROGRAM_VERSION=1.0.0 +PROGRAM_VERSION=1.0.0-rc1 PREFIX=/usr/local BINDIR=$(PREFIX)/bin CFGDIR=$(PREFIX)/etc/thalos diff --git a/debian/changelog b/debian/changelog index 8dee4ee..a320dc1 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,20 @@ +thalos (1.1.0~rc1) bionic focal jammy; urgency=medium + + * Adding flags for almost all config values. + * Improved disconnect code for ship client. + - Application now waits for ship to reply with a close message before exiting + the application. + - The application now recognizes an close error and no longer reports is as + an actual error to the log. + * Application only calls "GetInfo" from the antelope API once and only if it actually needs the information. + * CI: update actions/checkout to version 4 + * CI: update actions/setup-go to version 5 + * golang: update github.com/eosswedenorg-go/antelope-ship-client to v0.2.7 + * golang: switched github.com/pborman/getopt for github.com/spf13/cobra + * golang: use github.com/spf13/viper to handle configuration. + + -- Henrik Hautakoski Mon, 19 Feb 2024 14:27:40 +0100 + thalos (1.0.0) bionic focal jammy; urgency=medium * Improved code documentation. @@ -90,4 +107,4 @@ thalos-server (0.1.0) bionic focal jammy; urgency=medium Initial release. - -- Henrik Hautakoski Sun, 14 May 2023 18:17:35 +0200 \ No newline at end of file + -- Henrik Hautakoski Sun, 14 May 2023 18:17:35 +0200 From cd54b770a509b471be243f280f973e01b33efaa3 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Tue, 20 Feb 2024 19:28:06 +0100 Subject: [PATCH 142/291] internal/log/config.go: adding FileTimestampFormat field --- internal/log/config.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/log/config.go b/internal/log/config.go index 60c5715..2c50cc0 100644 --- a/internal/log/config.go +++ b/internal/log/config.go @@ -15,6 +15,9 @@ type Config struct { // Directory where the log files are stored. Directory string `yaml:"directory" mapstructure:"directory"` + // Timestamp format when rotation files. + FileTimestampFormat string `yaml:"file_timestamp_format" mapstructure:"file_timestamp_format"` + // Maximum filesize, the log is rotated when this size is exceeded. MaxFileSize types.Size `yaml:"maxfilesize" mapstructure:"maxfilesize"` From bcdaa9cea4f7f84efec54b390edd1cc2be158bf8 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Tue, 20 Feb 2024 19:43:27 +0100 Subject: [PATCH 143/291] internal/log/RotatingFileOptions.go: adding implementation for the functional option pattern. --- internal/log/RotatingFileOptions.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 internal/log/RotatingFileOptions.go diff --git a/internal/log/RotatingFileOptions.go b/internal/log/RotatingFileOptions.go new file mode 100644 index 0000000..c9d7422 --- /dev/null +++ b/internal/log/RotatingFileOptions.go @@ -0,0 +1,17 @@ +package log + +import "time" + +type RotatingFileOption func(*RotatingFile) + +func WithMaxSize(value int64) RotatingFileOption { + return func(f *RotatingFile) { + f.maxSize = value + } +} + +func WithMaxAge(value time.Duration) RotatingFileOption { + return func(f *RotatingFile) { + f.maxAge = value + } +} From 90b10f39ae69bb4e65a7b3a23cdfd1ec093fbd68 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Wed, 21 Feb 2024 17:22:06 +0100 Subject: [PATCH 144/291] internal/log/RotatingFile.go: use option functions. --- internal/log/RotatingFile.go | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/internal/log/RotatingFile.go b/internal/log/RotatingFile.go index bbedd9f..475042c 100644 --- a/internal/log/RotatingFile.go +++ b/internal/log/RotatingFile.go @@ -24,7 +24,7 @@ func open(filename string) (*os.File, error) { } // Open a new rotating file. -func NewRotatingFile(filename string, maxSize int64, maxAge time.Duration) (*RotatingFile, error) { +func NewRotatingFile(filename string, opts ...RotatingFileOption) (*RotatingFile, error) { if err := os.MkdirAll(path.Dir(filename), 0o766); err != nil && !os.IsExist(err) { return nil, err } @@ -39,14 +39,18 @@ func NewRotatingFile(filename string, maxSize int64, maxAge time.Duration) (*Rot return nil, err } - return &RotatingFile{ - fd: fd, - size: stat.Size(), - maxSize: maxSize, - ts: time.Now(), - maxAge: maxAge, - format: "2006-01-02_150405", - }, nil + file := &RotatingFile{ + fd: fd, + size: stat.Size(), + ts: time.Now(), + format: "2006-01-02_150405", + } + + for _, opt := range opts { + opt(file) + } + + return file, nil } // Open a new rotating file using a config struct. @@ -55,7 +59,12 @@ func NewRotatingFileFromConfig(config Config, suffix string) (*RotatingFile, err suffix = "_" + suffix } - return NewRotatingFile(config.GetFilePath()+suffix+".log", int64(config.MaxFileSize), config.MaxTime) + filename := config.GetFilePath() + suffix + ".log" + + return NewRotatingFile(filename, func(f *RotatingFile) { + f.maxAge = config.MaxTime + f.maxSize = int64(config.MaxFileSize) + }) } func (w *RotatingFile) newFilename(name string) string { From a66f1804da6018dad3a86faf542c447fd1569e03 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Wed, 21 Feb 2024 17:28:47 +0100 Subject: [PATCH 145/291] internal/log/RotatingFile.go: directories should have 755 as unix permission bits. --- internal/log/RotatingFile.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/log/RotatingFile.go b/internal/log/RotatingFile.go index 475042c..1d60582 100644 --- a/internal/log/RotatingFile.go +++ b/internal/log/RotatingFile.go @@ -25,7 +25,7 @@ func open(filename string) (*os.File, error) { // Open a new rotating file. func NewRotatingFile(filename string, opts ...RotatingFileOption) (*RotatingFile, error) { - if err := os.MkdirAll(path.Dir(filename), 0o766); err != nil && !os.IsExist(err) { + if err := os.MkdirAll(path.Dir(filename), 0o755); err != nil && !os.IsExist(err) { return nil, err } From d6a7bcd2bbcde83b857a27a21031a04acc0d869b Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Wed, 21 Feb 2024 17:29:31 +0100 Subject: [PATCH 146/291] internal/log/RotatingFileOptions.go: add WithTimestampFormat function --- internal/log/RotatingFileOptions.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/internal/log/RotatingFileOptions.go b/internal/log/RotatingFileOptions.go index c9d7422..be7b4c3 100644 --- a/internal/log/RotatingFileOptions.go +++ b/internal/log/RotatingFileOptions.go @@ -4,6 +4,14 @@ import "time" type RotatingFileOption func(*RotatingFile) +func WithTimestampFormat(value string) RotatingFileOption { + return func(f *RotatingFile) { + if len(value) > 0 { + f.format = value + } + } +} + func WithMaxSize(value int64) RotatingFileOption { return func(f *RotatingFile) { f.maxSize = value From aa43fc003f5ee8fc3b33df9410164ce24fbcfab2 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Thu, 22 Feb 2024 10:08:50 +0100 Subject: [PATCH 147/291] internal/log/RotatingFile.go: in NewRotatingFileFromConfig() set timestamp format. --- internal/log/RotatingFile.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/internal/log/RotatingFile.go b/internal/log/RotatingFile.go index 1d60582..540a783 100644 --- a/internal/log/RotatingFile.go +++ b/internal/log/RotatingFile.go @@ -61,10 +61,11 @@ func NewRotatingFileFromConfig(config Config, suffix string) (*RotatingFile, err filename := config.GetFilePath() + suffix + ".log" - return NewRotatingFile(filename, func(f *RotatingFile) { - f.maxAge = config.MaxTime - f.maxSize = int64(config.MaxFileSize) - }) + return NewRotatingFile(filename, + WithMaxAge(config.MaxTime), + WithMaxSize(int64(config.MaxFileSize)), + WithTimestampFormat(config.FileTimestampFormat), + ) } func (w *RotatingFile) newFilename(name string) string { From aaf490456c19bfe1df5e7d9ee84f80cf9d44f731 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Thu, 22 Feb 2024 22:35:44 +0100 Subject: [PATCH 148/291] internal/config: adding --log-file-timestamp flag. --- internal/config/builder.go | 5 +++-- internal/config/builder_test.go | 25 +++++++++++++++---------- internal/config/cli.go | 1 + 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/internal/config/builder.go b/internal/config/builder.go index 5257e3d..3bcbbd3 100644 --- a/internal/config/builder.go +++ b/internal/config/builder.go @@ -37,8 +37,9 @@ func NewBuilder() *Builder { "telegram.channel": "telegram-channel", // Log - "log.maxfilesize": "log-max-filesize", - "log.maxtime": "log-max-time", + "log.maxfilesize": "log-max-filesize", + "log.maxtime": "log-max-time", + "log.file_timestamp_format": "log-file-timestamp", // Ship "ship.url": "ship-url", diff --git a/internal/config/builder_test.go b/internal/config/builder_test.go index 9bc1589..24f7751 100644 --- a/internal/config/builder_test.go +++ b/internal/config/builder_test.go @@ -15,10 +15,11 @@ func TestBuilder(t *testing.T) { Api: "http://127.0.0.1:8080", MessageCodec: "mojibake", Log: log.Config{ - Filename: "some_file.log", - Directory: "/path/to/whatever", - MaxFileSize: 200, - MaxTime: 30 * time.Minute, + Filename: "some_file.log", + Directory: "/path/to/whatever", + MaxFileSize: 200, + MaxTime: 30 * time.Minute, + FileTimestampFormat: "20060102@150405", }, Ship: ShipConfig{ Url: "127.0.0.1:8089", @@ -50,6 +51,7 @@ log: directory: /path/to/whatever maxtime: 30m maxfilesize: 200b + file_timestamp_format: 20060102@150405 ship: url: "127.0.0.1:8089" irreversible_only: true @@ -79,10 +81,11 @@ func TestBuilder_ConfigWithFlags(t *testing.T) { Api: "https://api.example.com", MessageCodec: "msgpack", Log: log.Config{ - Filename: "mylog.log", - Directory: "/var/log", - MaxFileSize: 200, - MaxTime: 30 * time.Minute, + Filename: "mylog.log", + Directory: "/var/log", + MaxFileSize: 200, + MaxTime: 30 * time.Minute, + FileTimestampFormat: "2006-01-02_150405", }, Ship: ShipConfig{ Url: "127.0.0.1:8089", @@ -165,6 +168,7 @@ func TestBuilder_Flags(t *testing.T) { require.NoError(t, flags.Set("telegram-channel", "-293492332")) require.NoError(t, flags.Set("log-max-filesize", "25mb")) require.NoError(t, flags.Set("log-max-time", "10m")) + require.NoError(t, flags.Set("log-file-timestamp", "0102-15:04:05")) require.NoError(t, flags.Set("ship-url", "ws://myship.com:7823")) require.NoError(t, flags.Set("start-block", "7327833")) require.NoError(t, flags.Set("end-block", "329408392")) @@ -181,8 +185,9 @@ func TestBuilder_Flags(t *testing.T) { Api: "https://myapi", MessageCodec: "binary", Log: log.Config{ - MaxFileSize: 25 * 1000 * 1000, // 25 mb - MaxTime: time.Minute * 10, + MaxFileSize: 25 * 1000 * 1000, // 25 mb + MaxTime: time.Minute * 10, + FileTimestampFormat: "0102-15:04:05", }, Ship: ShipConfig{ Url: "ws://myship.com:7823", diff --git a/internal/config/cli.go b/internal/config/cli.go index d831b7a..bace210 100644 --- a/internal/config/cli.go +++ b/internal/config/cli.go @@ -29,6 +29,7 @@ func GetFlags() *pflag.FlagSet { flags.StringP("log", "l", "", "Path to log file (default: print to stdout/stderr)") flags.String("log-max-filesize", "10mb", "Max filesize for logfile to rotate") flags.Duration("log-max-time", time.Hour*24, "Max time for logfile to rotate") + flags.String("log-file-timestamp", "2006-01-02_150405", "Timestamp format to use when rotating log files") // Ship flags.String("ship-url", "ws://127.0.0.1:8080", "Url to ship node") From f7d020b0945e0888182a98c828aa704d3f6b8290 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Wed, 28 Feb 2024 23:26:32 +0100 Subject: [PATCH 149/291] update dependencies --- go.mod | 15 +++++++-------- go.sum | 29 ++++++++++++++--------------- 2 files changed, 21 insertions(+), 23 deletions(-) diff --git a/go.mod b/go.mod index 186759b..3645aa9 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/go-redis/redismock/v9 v9.2.0 github.com/mitchellh/mapstructure v1.5.0 github.com/nikoksr/notify v0.41.0 - github.com/redis/go-redis/v9 v9.4.0 + github.com/redis/go-redis/v9 v9.5.1 github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.8.0 github.com/spf13/pflag v1.0.5 @@ -34,14 +34,14 @@ require ( github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.17.6 // indirect + github.com/klauspost/compress v1.17.7 // indirect github.com/logrusorgru/aurora v2.0.3+incompatible // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mitchellh/go-testing-interface v1.14.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/onsi/gomega v1.31.1 // indirect - github.com/pelletier/go-toml/v2 v2.1.0 // indirect + github.com/pelletier/go-toml/v2 v2.1.1 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect @@ -52,18 +52,17 @@ require ( github.com/streamingfast/logging v0.0.0-20230608130331-f22c91403091 // indirect github.com/subosito/gotenv v1.6.0 // indirect github.com/technoweenie/multipartstreamer v1.0.1 // indirect - github.com/tidwall/gjson v1.17.0 // indirect + github.com/tidwall/gjson v1.17.1 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect github.com/ugorji/go/codec v1.2.12 // indirect github.com/vmihailenco/go-tinylfu v0.2.2 // indirect github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect - go.uber.org/goleak v1.3.0 // indirect go.uber.org/multierr v1.11.0 // indirect - go.uber.org/zap v1.26.0 // indirect - golang.org/x/crypto v0.19.0 // indirect - golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect + go.uber.org/zap v1.27.0 // indirect + golang.org/x/crypto v0.20.0 // indirect + golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 // indirect golang.org/x/net v0.21.0 // indirect golang.org/x/sync v0.6.0 // indirect golang.org/x/sys v0.17.0 // indirect diff --git a/go.sum b/go.sum index 7ccd99c..8809c73 100644 --- a/go.sum +++ b/go.sum @@ -75,8 +75,8 @@ github.com/jordan-wright/email v4.0.1-0.20210109023952-943e75fe5223+incompatible github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.17.6 h1:60eq2E/jlfwQXtvZEeBUYADs+BwKBWURIY+Gj2eRGjI= -github.com/klauspost/compress v1.17.6/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= +github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg= +github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= @@ -127,8 +127,8 @@ github.com/onsi/gomega v1.24.1/go.mod h1:3AOiACssS3/MajrniINInwbfOOtfZvplPzuRSmv github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/onsi/gomega v1.31.1 h1:KYppCUK+bUgAZwHOu7EXVBKyQA6ILvOESHkn/tgoqvo= github.com/onsi/gomega v1.31.1/go.mod h1:y40C95dwAD1Nz36SsEnxvfFe8FFfNxzI5eJ0EYGyAy0= -github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= -github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= +github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI= +github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -136,8 +136,8 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/redis/go-redis/v9 v9.0.0-rc.4/go.mod h1:Vo3EsyWnicKnSKCA7HhgnvnyA74wOA69Cd2Meli5mmA= -github.com/redis/go-redis/v9 v9.4.0 h1:Yzoz33UZw9I/mFhx4MNrB6Fk+XHO1VukNcCa1+lwyKk= -github.com/redis/go-redis/v9 v9.4.0/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= +github.com/redis/go-redis/v9 v9.5.1 h1:H1X4D3yHPaYrkL5X06Wh6xNVM/pX0Ft4RV0vMGvLBh8= +github.com/redis/go-redis/v9 v9.5.1/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -179,8 +179,8 @@ github.com/technoweenie/multipartstreamer v1.0.1 h1:XRztA5MXiR1TIRHxH2uNxXxaIkKQ github.com/technoweenie/multipartstreamer v1.0.1/go.mod h1:jNVxdtShOxzAsukZwTSw6MDx5eUJoiEBsSvzDU9uzog= github.com/test-go/testify v1.1.4 h1:Tf9lntrKUMHiXQ07qBScBTSA0dhYQlu83hswqelv1iE= github.com/test-go/testify v1.1.4/go.mod h1:rH7cfJo/47vWGdi4GPj16x3/t1xGOj2YxzmNQzk2ghU= -github.com/tidwall/gjson v1.17.0 h1:/Jocvlh98kcTfpN2+JzGQWQcqrPQwDrVEMApx/M5ZwM= -github.com/tidwall/gjson v1.17.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.17.1 h1:wlYEnwqAHgzmhNUFfw7Xalt2JzQvsMx2Se4PcoFCT/U= +github.com/tidwall/gjson v1.17.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= @@ -204,25 +204,24 @@ go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= -go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= -go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= -go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= -golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo= -golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= -golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= -golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= +golang.org/x/crypto v0.20.0 h1:jmAMJJZXr5KiCw05dfYK9QnqaqKLYXijU23lsEdcQqg= +golang.org/x/crypto v0.20.0/go.mod h1:Xwo95rrVNIoSMx9wa1JroENMToLWn3RNVrTBpLHgZPQ= +golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 h1:LfspQV/FYTatPTr/3HzIcmiUFH7PGP+OQ6mgDYo3yuQ= +golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= From 79d2496185d8ef5dba4951b4ea009736eb2f27b1 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Wed, 28 Feb 2024 23:33:03 +0100 Subject: [PATCH 150/291] Version 1.1.0-rc2 --- Makefile | 2 +- debian/changelog | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 82e1cad..384a113 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ GO=go GOLDFLAGS=-v -s -w -X main.VersionString=$(PROGRAM_VERSION) GOBUILDFLAGS=-v -p $(shell nproc) -ldflags="$(GOLDFLAGS)" PROGRAM=thalos-server -PROGRAM_VERSION=1.0.0-rc1 +PROGRAM_VERSION=1.1.0-rc2 PREFIX=/usr/local BINDIR=$(PREFIX)/bin CFGDIR=$(PREFIX)/etc/thalos diff --git a/debian/changelog b/debian/changelog index a320dc1..b156561 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,11 @@ +thalos (1.1.0~rc2) bionic focal jammy; urgency=medium + + * Adding `log.file_timestamp_format` config field + * Added cli flag `--log-file-timestamp` + * Directory where log files are stored is created with correct permissions. + + -- Henrik Hautakoski Wed, 28 Feb 2024 23:16:44 +0100 + thalos (1.1.0~rc1) bionic focal jammy; urgency=medium * Adding flags for almost all config values. From 700d7d0168f4b0485b5c021e7e2881f79597e9f9 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Thu, 29 Feb 2024 14:41:56 +0100 Subject: [PATCH 151/291] api/message/types.go: Adding FirstReceiver to ActionTrace --- api/message/types.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/api/message/types.go b/api/message/types.go index d8948e3..b012b79 100644 --- a/api/message/types.go +++ b/api/message/types.go @@ -64,8 +64,10 @@ type ActionTrace struct { // Contract account. Contract string `json:"contract" msgpack:"contract"` - Receiver string `json:"receiver" msgpack:"receiver"` - Data interface{} `json:"data" msgpack:"data"` + Receiver string `json:"receiver" msgpack:"receiver"` + FirstReceiver bool `json:"first_receiver" msgpack:"first_receiver"` + + Data interface{} `json:"data" msgpack:"data"` Authorization []PermissionLevel `json:"authorization" msgpack:"authorization"` From b3570e7e9625c8c3b984a6ca8970e5b9278966f3 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Thu, 29 Feb 2024 14:43:23 +0100 Subject: [PATCH 152/291] internal/server/ship_processor.go: set FirstReceiver on ActionTrace messages. --- internal/server/ship_processor.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/internal/server/ship_processor.go b/internal/server/ship_processor.go index d510c99..c15c4ae 100644 --- a/internal/server/ship_processor.go +++ b/internal/server/ship_processor.go @@ -228,12 +228,13 @@ func (processor *ShipProcessor) processBlock(block *ship.GetBlocksResultV0) { } act := message.ActionTrace{ - TxID: trace.ID.String(), - BlockNum: block.Block.BlockNumber(), - Timestamp: block.Block.Timestamp.Time.UTC(), - Name: act_trace.Act.Name.String(), - Contract: act_trace.Act.Account.String(), - Receiver: act_trace.Receiver.String(), + TxID: trace.ID.String(), + BlockNum: block.Block.BlockNumber(), + Timestamp: block.Block.Timestamp.Time.UTC(), + Name: act_trace.Act.Name.String(), + Contract: act_trace.Act.Account.String(), + Receiver: act_trace.Receiver.String(), + FirstReceiver: act_trace.Act.Account.String() == act_trace.Receiver.String(), } if act_trace.Receipt != nil { From 212a8c892cb0c3daac53e87576111f8f23b22450 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Thu, 29 Feb 2024 16:15:54 +0100 Subject: [PATCH 153/291] api/message/json/codec_test.go: fix tests. --- api/message/json/codec_test.go | 60 +++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 27 deletions(-) diff --git a/api/message/json/codec_test.go b/api/message/json/codec_test.go index 7e0aec0..590720c 100644 --- a/api/message/json/codec_test.go +++ b/api/message/json/codec_test.go @@ -10,12 +10,13 @@ import ( func TestJson_EncodeActionTrace(t *testing.T) { msg := message.ActionTrace{ - TxID: "ed3b8e853647971cf8296f004c3a1aeac255f082b2cb3c12cc3222e2d7c174ab", - BlockNum: 267372365, - Timestamp: time.Unix(1048267389, int64(time.Millisecond)*500).UTC(), - Name: "transfer", - Contract: "eosio", - Receiver: "account2", + TxID: "ed3b8e853647971cf8296f004c3a1aeac255f082b2cb3c12cc3222e2d7c174ab", + BlockNum: 267372365, + Timestamp: time.Unix(1048267389, int64(time.Millisecond)*500).UTC(), + Name: "transfer", + Contract: "eosio", + Receiver: "account2", + FirstReceiver: true, Receipt: &message.ActionReceipt{ Receiver: "account2", ActDigest: "0a2c71dba327cf13a107d3a4f91c9c98f510a8fbbb483b233e222033f13a3e36", @@ -43,7 +44,7 @@ func TestJson_EncodeActionTrace(t *testing.T) { Return: []byte{0xde, 0xad, 0xbe, 0xef}, } - expected := `{"tx_id":"ed3b8e853647971cf8296f004c3a1aeac255f082b2cb3c12cc3222e2d7c174ab","blocknum":267372365,"blocktimestamp":"2003-03-21T17:23:09.500","receipt":{"receiver":"account2","act_digest":"0a2c71dba327cf13a107d3a4f91c9c98f510a8fbbb483b233e222033f13a3e36","global_sequence":2329381932,"recv_sequence":22,"auth_sequence":[{"account":"account1","sequence":1382772}],"code_sequence":1122,"abi_sequence":12352},"name":"transfer","contract":"eosio","receiver":"account2","data":{"from":"account1","quantity":"1000.0000 WAX","to":"account2"},"authorization":[{"actor":"account1","permission":"active"}],"except":"errstr","error":2,"return":"3q2+7w=="}` + expected := `{"tx_id":"ed3b8e853647971cf8296f004c3a1aeac255f082b2cb3c12cc3222e2d7c174ab","blocknum":267372365,"blocktimestamp":"2003-03-21T17:23:09.500","receipt":{"receiver":"account2","act_digest":"0a2c71dba327cf13a107d3a4f91c9c98f510a8fbbb483b233e222033f13a3e36","global_sequence":2329381932,"recv_sequence":22,"auth_sequence":[{"account":"account1","sequence":1382772}],"code_sequence":1122,"abi_sequence":12352},"name":"transfer","contract":"eosio","receiver":"account2","first_receiver":true,"data":{"from":"account1","quantity":"1000.0000 WAX","to":"account2"},"authorization":[{"actor":"account1","permission":"active"}],"except":"errstr","error":2,"return":"3q2+7w=="}` data, err := createCodec().Encoder(msg) assert.NoError(t, err) @@ -52,12 +53,13 @@ func TestJson_EncodeActionTrace(t *testing.T) { func TestJson_DecodeActionTrace(t *testing.T) { expected := message.ActionTrace{ - TxID: "952989f7464237b6cf9926e533ecd331df6794ed07564bd052bc368cbd65b4bc", - BlockNum: 8723971, - Timestamp: time.Unix(1718957306, int64(time.Millisecond)*500).UTC(), - Name: "transfer", - Contract: "eosio", - Receiver: "account2", + TxID: "952989f7464237b6cf9926e533ecd331df6794ed07564bd052bc368cbd65b4bc", + BlockNum: 8723971, + Timestamp: time.Unix(1718957306, int64(time.Millisecond)*500).UTC(), + Name: "transfer", + Contract: "eosio", + Receiver: "account2", + FirstReceiver: true, Receipt: &message.ActionReceipt{ Receiver: "account2", ActDigest: "f2f682847fd5bf00fb315b075dc00b4ff0ce18776758077b86a233dea49dc047", @@ -85,7 +87,7 @@ func TestJson_DecodeActionTrace(t *testing.T) { Return: []byte{0xde, 0xad, 0xbe, 0xef}, } - input := `{"tx_id":"952989f7464237b6cf9926e533ecd331df6794ed07564bd052bc368cbd65b4bc","blocknum":8723971,"blocktimestamp":"2024-06-21T08:08:26.500","receipt":{"receiver":"account2","act_digest":"f2f682847fd5bf00fb315b075dc00b4ff0ce18776758077b86a233dea49dc047","global_sequence":287328,"recv_sequence":42,"auth_sequence":[{"account":"account1","sequence":271877283}],"code_sequence":237823,"abi_sequence":68323},"name":"transfer","contract":"eosio","receiver":"account2","data":{"from":"account1","quantity":"1000.0000 WAX","to":"account2"},"authorization":[{"actor":"account1","permission":"active"}],"except":"errstr","error":2,"return":"3q2+7w=="}` + input := `{"tx_id":"952989f7464237b6cf9926e533ecd331df6794ed07564bd052bc368cbd65b4bc","blocknum":8723971,"blocktimestamp":"2024-06-21T08:08:26.500","receipt":{"receiver":"account2","act_digest":"f2f682847fd5bf00fb315b075dc00b4ff0ce18776758077b86a233dea49dc047","global_sequence":287328,"recv_sequence":42,"auth_sequence":[{"account":"account1","sequence":271877283}],"code_sequence":237823,"abi_sequence":68323},"name":"transfer","contract":"eosio","receiver":"account2","first_receiver":true,"data":{"from":"account1","quantity":"1000.0000 WAX","to":"account2"},"authorization":[{"actor":"account1","permission":"active"}],"except":"errstr","error":2,"return":"3q2+7w=="}` msg := message.ActionTrace{} err := createCodec().Decoder([]byte(input), &msg) @@ -129,12 +131,13 @@ func TestJson_EncodeTransactionTrace(t *testing.T) { Return: []byte{0x08, 0xf1}, }, { - TxID: tx_id, - BlockNum: block_num, - Timestamp: ts, - Receiver: "coolgame", - Contract: "coolgame", - Name: "addpoints", + TxID: tx_id, + BlockNum: block_num, + Timestamp: ts, + Receiver: "coolgame", + Contract: "coolgame", + FirstReceiver: true, + Name: "addpoints", Authorization: []message.PermissionLevel{ { Actor: "coolgame", @@ -153,7 +156,7 @@ func TestJson_EncodeTransactionTrace(t *testing.T) { Error: 2, } - expected := `{"id":"ed04516bdd1194aa5f0ab4c8c5445eec542c17f45a85bb3e9e4bc33e1a2486f8","blocknum":283781923,"blocktimestamp":"2029-02-08T15:10:05.500","status":"executed","cpu_usage_us":442,"net_usage_words":16,"elapsed":22,"net_usage":128,"scheduled":true,"action_traces":[{"tx_id":"ed04516bdd1194aa5f0ab4c8c5445eec542c17f45a85bb3e9e4bc33e1a2486f8","blocknum":283781923,"blocktimestamp":"2029-02-08T15:10:05.500","name":"mine","contract":"coolgame","receiver":"actor01","data":{"equipment_id":1234,"location_id":5445453},"authorization":[{"actor":"actor01","permission":"active"}],"except":"","error":0,"return":"CPE="},{"tx_id":"ed04516bdd1194aa5f0ab4c8c5445eec542c17f45a85bb3e9e4bc33e1a2486f8","blocknum":283781923,"blocktimestamp":"2029-02-08T15:10:05.500","name":"addpoints","contract":"coolgame","receiver":"coolgame","data":{"points":"1023.0423 SCAM"},"authorization":[{"actor":"coolgame","permission":"usrpoints"}],"except":"some error string","error":2,"return":"/wI="}],"except":"errstr","error":2}` + expected := `{"id":"ed04516bdd1194aa5f0ab4c8c5445eec542c17f45a85bb3e9e4bc33e1a2486f8","blocknum":283781923,"blocktimestamp":"2029-02-08T15:10:05.500","status":"executed","cpu_usage_us":442,"net_usage_words":16,"elapsed":22,"net_usage":128,"scheduled":true,"action_traces":[{"tx_id":"ed04516bdd1194aa5f0ab4c8c5445eec542c17f45a85bb3e9e4bc33e1a2486f8","blocknum":283781923,"blocktimestamp":"2029-02-08T15:10:05.500","name":"mine","contract":"coolgame","receiver":"actor01","first_receiver":false,"data":{"equipment_id":1234,"location_id":5445453},"authorization":[{"actor":"actor01","permission":"active"}],"except":"","error":0,"return":"CPE="},{"tx_id":"ed04516bdd1194aa5f0ab4c8c5445eec542c17f45a85bb3e9e4bc33e1a2486f8","blocknum":283781923,"blocktimestamp":"2029-02-08T15:10:05.500","name":"addpoints","contract":"coolgame","receiver":"coolgame","first_receiver":true,"data":{"points":"1023.0423 SCAM"},"authorization":[{"actor":"coolgame","permission":"usrpoints"}],"except":"some error string","error":2,"return":"/wI="}],"except":"errstr","error":2}` data, err := createCodec().Encoder(msg) assert.NoError(t, err) @@ -165,7 +168,7 @@ func TestJson_DecodeTransactionTrace(t *testing.T) { ts := time.Unix(1730755743, int64(time.Millisecond)*500).UTC() block_num := uint32(2378197231) - input := `{"id":"f58bf8a0137fcea644dbc2b0cc5b6a017a848cd33b2e924703e7e3c6d1ca0c2e","blocknum":2378197231,"blocktimestamp":"2024-11-04T21:29:03.500","status":"executed","cpu_usage_us":442,"net_usage_words":16,"elapsed":22,"net_usage":128,"scheduled":true,"action_traces":[{"tx_id":"f58bf8a0137fcea644dbc2b0cc5b6a017a848cd33b2e924703e7e3c6d1ca0c2e","blocknum":2378197231,"blocktimestamp":"2024-11-04T21:29:03.500","receipt":{"receiver":"actor01","act_digest":"bf68a227a617aa9138215d31a2847005175f01151c6feeaeb6513fe2d090ff3a","global_sequence":47322,"recv_sequence":23289,"auth_sequence":[{"account":"actor01","sequence":56637232}],"code_sequence":98272,"abi_sequence":6823762},"name":"mine","contract":"","receiver":"actor01","data":{"equipment_id":1234,"location_id":5445453},"authorization":[{"actor":"actor01","permission":"active"}],"except":"","error":2,"return":"AQI="},{"tx_id":"f58bf8a0137fcea644dbc2b0cc5b6a017a848cd33b2e924703e7e3c6d1ca0c2e","blocknum":2378197231,"blocktimestamp":"2024-11-04T21:29:03.500","name":"addpoints","contract":"","receiver":"coolgame","data":{"points":"1023.0423 SCAM"},"authorization":[{"actor":"coolgame","permission":"usrpoints"}],"except":"","error":2,"return":"CPE="}],"except":"errstr","error":2}` + input := `{"id":"f58bf8a0137fcea644dbc2b0cc5b6a017a848cd33b2e924703e7e3c6d1ca0c2e","blocknum":2378197231,"blocktimestamp":"2024-11-04T21:29:03.500","status":"executed","cpu_usage_us":442,"net_usage_words":16,"elapsed":22,"net_usage":128,"scheduled":true,"action_traces":[{"tx_id":"f58bf8a0137fcea644dbc2b0cc5b6a017a848cd33b2e924703e7e3c6d1ca0c2e","blocknum":2378197231,"blocktimestamp":"2024-11-04T21:29:03.500","receipt":{"receiver":"actor01","act_digest":"bf68a227a617aa9138215d31a2847005175f01151c6feeaeb6513fe2d090ff3a","global_sequence":47322,"recv_sequence":23289,"auth_sequence":[{"account":"actor01","sequence":56637232}],"code_sequence":98272,"abi_sequence":6823762},"name":"mine","contract":"coolgame","receiver":"actor01","first_receiver":false,"data":{"equipment_id":1234,"location_id":5445453},"authorization":[{"actor":"actor01","permission":"active"}],"except":"","error":2,"return":"AQI="},{"tx_id":"f58bf8a0137fcea644dbc2b0cc5b6a017a848cd33b2e924703e7e3c6d1ca0c2e","blocknum":2378197231,"blocktimestamp":"2024-11-04T21:29:03.500","name":"addpoints","contract":"coolgame","receiver":"coolgame","first_receiver":true,"data":{"points":"1023.0423 SCAM"},"authorization":[{"actor":"coolgame","permission":"usrpoints"}],"except":"","error":2,"return":"CPE="}],"except":"errstr","error":2}` expected := message.TransactionTrace{ Timestamp: ts, @@ -183,6 +186,7 @@ func TestJson_DecodeTransactionTrace(t *testing.T) { BlockNum: block_num, Timestamp: ts, Receiver: "actor01", + Contract: "coolgame", Name: "mine", Receipt: &message.ActionReceipt{ Receiver: "actor01", @@ -212,11 +216,13 @@ func TestJson_DecodeTransactionTrace(t *testing.T) { Return: []byte{0x01, 0x02}, }, { - TxID: tx_id, - BlockNum: block_num, - Timestamp: ts, - Receiver: "coolgame", - Name: "addpoints", + TxID: tx_id, + BlockNum: block_num, + Timestamp: ts, + Receiver: "coolgame", + Contract: "coolgame", + FirstReceiver: true, + Name: "addpoints", Authorization: []message.PermissionLevel{ { Actor: "coolgame", From cbe791d7dfd9754924a66528e52b6b7fddfde5e9 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Thu, 29 Feb 2024 19:11:25 +0100 Subject: [PATCH 154/291] api/message/msgpack/codec_test.go: fix tests. --- api/message/msgpack/codec_test.go | 241 +++++++++++++++++------------- 1 file changed, 140 insertions(+), 101 deletions(-) diff --git a/api/message/msgpack/codec_test.go b/api/message/msgpack/codec_test.go index 35be0d1..0a11805 100644 --- a/api/message/msgpack/codec_test.go +++ b/api/message/msgpack/codec_test.go @@ -55,7 +55,7 @@ func TestMsgpack_EncodeActionTrace(t *testing.T) { assert.NoError(t, err) expected := []byte{ - 0x8c, 0xad, 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, + 0x8d, 0xad, 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x91, 0x82, 0xa5, 0x61, 0x63, 0x74, 0x6f, 0x72, 0xa6, 0x6d, 0x79, 0x67, 0x61, 0x6d, 0x65, 0xaa, 0x70, @@ -85,6 +85,8 @@ func TestMsgpack_EncodeActionTrace(t *testing.T) { 0x6e, 0x74, 0x32, 0xa5, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x02, 0xa6, 0x65, 0x78, 0x63, 0x65, 0x70, 0x74, 0xa6, 0x65, 0x72, 0x72, 0x73, 0x74, 0x72, + 0xae, 0x66, 0x69, 0x72, 0x73, 0x74, 0x5f, 0x72, + 0x65, 0x63, 0x65, 0x69, 0x76, 0x65, 0x72, 0xc2, 0xa4, 0x6e, 0x61, 0x6d, 0x65, 0xa8, 0x73, 0x65, 0x6c, 0x6c, 0x69, 0x74, 0x65, 0x6d, 0xa7, 0x72, 0x65, 0x63, 0x65, 0x69, 0x70, 0x74, 0x87, 0xac, @@ -193,11 +195,29 @@ func TestMsgpack_EncodeHeartbeat(t *testing.T) { data, err := createCodec().Encoder(msg) assert.NoError(t, err) - assert.Equal(t, data, []byte{0x83, 0xa8, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x6e, 0x75, 0x6d, 0xcd, 0x4, 0xd2, 0xad, 0x68, 0x65, 0x61, 0x64, 0x5f, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x6e, 0x75, 0x6d, 0xcd, 0x4, 0xd3, 0xba, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x69, 0x72, 0x72, 0x65, 0x76, 0x65, 0x72, 0x73, 0x69, 0x62, 0x6c, 0x65, 0x5f, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x6e, 0x75, 0x6d, 0xcd, 0x4, 0xd4}) + assert.Equal(t, data, []byte{ + 0x83, 0xa8, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x6e, + 0x75, 0x6d, 0xcd, 0x04, 0xd2, 0xad, 0x68, 0x65, + 0x61, 0x64, 0x5f, 0x62, 0x6c, 0x6f, 0x63, 0x6b, + 0x6e, 0x75, 0x6d, 0xcd, 0x04, 0xd3, 0xba, 0x6c, + 0x61, 0x73, 0x74, 0x5f, 0x69, 0x72, 0x72, 0x65, + 0x76, 0x65, 0x72, 0x73, 0x69, 0x62, 0x6c, 0x65, + 0x5f, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x6e, 0x75, + 0x6d, 0xcd, 0x04, 0xd4, + }) } func TestMsgpack_DecodeHeartbeat(t *testing.T) { - data := []byte{0x83, 0xa8, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x6e, 0x75, 0x6d, 0xcd, 0x03, 0xe8, 0xad, 0x68, 0x65, 0x61, 0x64, 0x5f, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x6e, 0x75, 0x6d, 0xcd, 0x0b, 0xb8, 0xba, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x69, 0x72, 0x72, 0x65, 0x76, 0x65, 0x72, 0x73, 0x69, 0x62, 0x6c, 0x65, 0x5f, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x6e, 0x75, 0x6d, 0xcd, 0x04, 0x06} + data := []byte{ + 0x83, 0xa8, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x6e, + 0x75, 0x6d, 0xcd, 0x03, 0xe8, 0xad, 0x68, 0x65, + 0x61, 0x64, 0x5f, 0x62, 0x6c, 0x6f, 0x63, 0x6b, + 0x6e, 0x75, 0x6d, 0xcd, 0x0b, 0xb8, 0xba, 0x6c, + 0x61, 0x73, 0x74, 0x5f, 0x69, 0x72, 0x72, 0x65, + 0x76, 0x65, 0x72, 0x73, 0x69, 0x62, 0x6c, 0x65, + 0x5f, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x6e, 0x75, + 0x6d, 0xcd, 0x04, 0x06, + } expected := message.HeartBeat{ BlockNum: 1000, @@ -269,104 +289,123 @@ func TestMsgpack_EncodeTransactionTrace(t *testing.T) { assert.NoError(t, err) expected := []byte{ - 0x8c, 0xad, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x5f, 0x74, 0x72, 0x61, 0x63, 0x65, 0x73, 0x92, - 0x8c, 0xad, 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, - 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x91, - 0x82, 0xa5, 0x61, 0x63, 0x74, 0x6f, 0x72, 0xaa, - 0x61, 0x63, 0x74, 0x6f, 0x72, 0x31, 0x32, 0x31, - 0x32, 0x33, 0xaa, 0x70, 0x65, 0x72, 0x6d, 0x69, - 0x73, 0x73, 0x69, 0x6f, 0x6e, 0xa6, 0x61, 0x63, - 0x74, 0x69, 0x76, 0x65, 0xa8, 0x62, 0x6c, 0x6f, - 0x63, 0x6b, 0x6e, 0x75, 0x6d, 0x00, 0xae, 0x62, - 0x6c, 0x6f, 0x63, 0x6b, 0x74, 0x69, 0x6d, 0x65, - 0x73, 0x74, 0x61, 0x6d, 0x70, 0xc0, 0xa8, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x61, 0x63, 0x74, 0xa7, - 0x73, 0x6b, 0x6a, 0x64, 0x68, 0x32, 0x33, 0xa4, - 0x64, 0x61, 0x74, 0x61, 0x81, 0xa3, 0x6b, 0x65, - 0x79, 0xa5, 0x76, 0x61, 0x6c, 0x75, 0x65, 0xa5, - 0x65, 0x72, 0x72, 0x6f, 0x72, 0x00, 0xa6, 0x65, - 0x78, 0x63, 0x65, 0x70, 0x74, 0xa0, 0xa4, 0x6e, - 0x61, 0x6d, 0x65, 0xa8, 0x73, 0x65, 0x6c, 0x6c, - 0x69, 0x74, 0x65, 0x6d, 0xa7, 0x72, 0x65, 0x63, - 0x65, 0x69, 0x70, 0x74, 0x87, 0xac, 0x61, 0x62, - 0x69, 0x5f, 0x73, 0x65, 0x71, 0x75, 0x65, 0x6e, - 0x63, 0x65, 0xce, 0x00, 0x48, 0xf8, 0x98, 0xaa, - 0x61, 0x63, 0x74, 0x5f, 0x64, 0x69, 0x67, 0x65, - 0x73, 0x74, 0xd9, 0x40, 0x36, 0x37, 0x36, 0x63, - 0x35, 0x33, 0x33, 0x36, 0x64, 0x65, 0x39, 0x64, - 0x35, 0x32, 0x38, 0x64, 0x34, 0x35, 0x36, 0x62, - 0x30, 0x31, 0x65, 0x39, 0x37, 0x35, 0x62, 0x31, - 0x30, 0x30, 0x36, 0x65, 0x32, 0x62, 0x64, 0x63, - 0x38, 0x36, 0x63, 0x38, 0x64, 0x35, 0x36, 0x36, - 0x33, 0x33, 0x30, 0x33, 0x32, 0x31, 0x65, 0x39, - 0x62, 0x33, 0x30, 0x39, 0x62, 0x36, 0x33, 0x34, - 0x66, 0x35, 0x32, 0x33, 0xad, 0x61, 0x75, 0x74, - 0x68, 0x5f, 0x73, 0x65, 0x71, 0x75, 0x65, 0x6e, - 0x63, 0x65, 0x91, 0x82, 0xa7, 0x61, 0x63, 0x63, - 0x6f, 0x75, 0x6e, 0x74, 0xaa, 0x61, 0x63, 0x74, - 0x6f, 0x72, 0x31, 0x32, 0x31, 0x32, 0x33, 0xa8, - 0x73, 0x65, 0x71, 0x75, 0x65, 0x6e, 0x63, 0x65, - 0xce, 0x00, 0x08, 0x5c, 0x39, 0xad, 0x63, 0x6f, - 0x64, 0x65, 0x5f, 0x73, 0x65, 0x71, 0x75, 0x65, - 0x6e, 0x63, 0x65, 0xce, 0x01, 0x61, 0xe8, 0x79, - 0xaf, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x5f, - 0x73, 0x65, 0x71, 0x75, 0x65, 0x6e, 0x63, 0x65, - 0xce, 0x02, 0x12, 0xd7, 0x5d, 0xa8, 0x72, 0x65, - 0x63, 0x65, 0x69, 0x76, 0x65, 0x72, 0xa5, 0x65, - 0x6f, 0x73, 0x69, 0x6f, 0xad, 0x72, 0x65, 0x63, - 0x76, 0x5f, 0x73, 0x65, 0x71, 0x75, 0x65, 0x6e, - 0x63, 0x65, 0xce, 0x22, 0x81, 0x80, 0x7a, 0xa8, - 0x72, 0x65, 0x63, 0x65, 0x69, 0x76, 0x65, 0x72, - 0xa5, 0x65, 0x6f, 0x73, 0x69, 0x6f, 0xa6, 0x72, - 0x65, 0x74, 0x75, 0x72, 0x6e, 0xc0, 0xa5, 0x74, - 0x78, 0x5f, 0x69, 0x64, 0xa0, 0x8b, 0xad, 0x61, - 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x91, 0x82, 0xa5, 0x61, - 0x63, 0x74, 0x6f, 0x72, 0xa9, 0x61, 0x63, 0x74, - 0x6f, 0x72, 0x31, 0x34, 0x38, 0x32, 0xaa, 0x70, - 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6f, - 0x6e, 0xa6, 0x61, 0x63, 0x74, 0x69, 0x76, 0x65, - 0xa8, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x6e, 0x75, - 0x6d, 0x00, 0xae, 0x62, 0x6c, 0x6f, 0x63, 0x6b, - 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, - 0x70, 0xc0, 0xa8, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x61, 0x63, 0x74, 0xa7, 0x73, 0x6b, 0x6a, 0x64, - 0x68, 0x32, 0x34, 0xa4, 0x64, 0x61, 0x74, 0x61, - 0xc0, 0xa5, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x00, - 0xa6, 0x65, 0x78, 0x63, 0x65, 0x70, 0x74, 0xa0, - 0xa4, 0x6e, 0x61, 0x6d, 0x65, 0xa8, 0x73, 0x65, - 0x6c, 0x6c, 0x69, 0x74, 0x65, 0x6d, 0xa8, 0x72, - 0x65, 0x63, 0x65, 0x69, 0x76, 0x65, 0x72, 0xa5, - 0x65, 0x6f, 0x73, 0x69, 0x6f, 0xa6, 0x72, 0x65, - 0x74, 0x75, 0x72, 0x6e, 0xc0, 0xa5, 0x74, 0x78, - 0x5f, 0x69, 0x64, 0xa0, 0xa8, 0x62, 0x6c, 0x6f, - 0x63, 0x6b, 0x6e, 0x75, 0x6d, 0xce, 0xa3, 0x3d, - 0x9b, 0xc5, 0xae, 0x62, 0x6c, 0x6f, 0x63, 0x6b, - 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, - 0x70, 0xd7, 0xff, 0x77, 0x35, 0x94, 0x00, 0x65, - 0x4e, 0x19, 0xff, 0xac, 0x63, 0x70, 0x75, 0x5f, - 0x75, 0x73, 0x61, 0x67, 0x65, 0x5f, 0x75, 0x73, - 0x17, 0xa7, 0x65, 0x6c, 0x61, 0x70, 0x73, 0x65, - 0x64, 0x04, 0xa5, 0x65, 0x72, 0x72, 0x6f, 0x72, - 0x02, 0xa6, 0x65, 0x78, 0x63, 0x65, 0x70, 0x74, - 0xa9, 0x65, 0x78, 0x63, 0x65, 0x70, 0x74, 0x73, - 0x74, 0x72, 0xa2, 0x69, 0x64, 0xd9, 0x40, 0x65, - 0x64, 0x63, 0x30, 0x36, 0x64, 0x63, 0x65, 0x36, - 0x33, 0x32, 0x30, 0x34, 0x35, 0x39, 0x66, 0x64, - 0x36, 0x34, 0x34, 0x37, 0x35, 0x36, 0x39, 0x37, - 0x32, 0x30, 0x34, 0x38, 0x64, 0x61, 0x34, 0x35, - 0x33, 0x62, 0x32, 0x38, 0x31, 0x36, 0x62, 0x30, - 0x61, 0x34, 0x33, 0x34, 0x63, 0x33, 0x37, 0x64, - 0x64, 0x66, 0x66, 0x64, 0x65, 0x33, 0x36, 0x37, - 0x37, 0x38, 0x64, 0x63, 0x61, 0x62, 0x33, 0xa9, - 0x6e, 0x65, 0x74, 0x5f, 0x75, 0x73, 0x61, 0x67, - 0x65, 0xcc, 0x80, 0xaf, 0x6e, 0x65, 0x74, 0x5f, - 0x75, 0x73, 0x61, 0x67, 0x65, 0x5f, 0x77, 0x6f, - 0x72, 0x64, 0x73, 0x10, 0xa9, 0x73, 0x63, 0x68, - 0x65, 0x64, 0x75, 0x6c, 0x65, 0x64, 0xc3, 0xa6, - 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0xa9, 0x73, - 0x6f, 0x66, 0x74, 0x5f, 0x66, 0x61, 0x69, 0x6c, + 0x8c, 0xad, 0x61, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x5f, 0x74, 0x72, 0x61, 0x63, 0x65, + 0x73, 0x92, 0x8d, 0xad, 0x61, 0x75, 0x74, + 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x91, 0x82, 0xa5, 0x61, + 0x63, 0x74, 0x6f, 0x72, 0xaa, 0x61, 0x63, + 0x74, 0x6f, 0x72, 0x31, 0x32, 0x31, 0x32, + 0x33, 0xaa, 0x70, 0x65, 0x72, 0x6d, 0x69, + 0x73, 0x73, 0x69, 0x6f, 0x6e, 0xa6, 0x61, + 0x63, 0x74, 0x69, 0x76, 0x65, 0xa8, 0x62, + 0x6c, 0x6f, 0x63, 0x6b, 0x6e, 0x75, 0x6d, + 0x00, 0xae, 0x62, 0x6c, 0x6f, 0x63, 0x6b, + 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, + 0x6d, 0x70, 0xc0, 0xa8, 0x63, 0x6f, 0x6e, + 0x74, 0x72, 0x61, 0x63, 0x74, 0xa7, 0x73, + 0x6b, 0x6a, 0x64, 0x68, 0x32, 0x33, 0xa4, + 0x64, 0x61, 0x74, 0x61, 0x81, 0xa3, 0x6b, + 0x65, 0x79, 0xa5, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0xa5, 0x65, 0x72, 0x72, 0x6f, 0x72, + 0x00, 0xa6, 0x65, 0x78, 0x63, 0x65, 0x70, + 0x74, 0xa0, 0xae, 0x66, 0x69, 0x72, 0x73, + 0x74, 0x5f, 0x72, 0x65, 0x63, 0x65, 0x69, + 0x76, 0x65, 0x72, 0xc2, 0xa4, 0x6e, 0x61, + 0x6d, 0x65, 0xa8, 0x73, 0x65, 0x6c, 0x6c, + 0x69, 0x74, 0x65, 0x6d, 0xa7, 0x72, 0x65, + 0x63, 0x65, 0x69, 0x70, 0x74, 0x87, 0xac, + 0x61, 0x62, 0x69, 0x5f, 0x73, 0x65, 0x71, + 0x75, 0x65, 0x6e, 0x63, 0x65, 0xce, 0x00, + 0x48, 0xf8, 0x98, 0xaa, 0x61, 0x63, 0x74, + 0x5f, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, + 0xd9, 0x40, 0x36, 0x37, 0x36, 0x63, 0x35, + 0x33, 0x33, 0x36, 0x64, 0x65, 0x39, 0x64, + 0x35, 0x32, 0x38, 0x64, 0x34, 0x35, 0x36, + 0x62, 0x30, 0x31, 0x65, 0x39, 0x37, 0x35, + 0x62, 0x31, 0x30, 0x30, 0x36, 0x65, 0x32, + 0x62, 0x64, 0x63, 0x38, 0x36, 0x63, 0x38, + 0x64, 0x35, 0x36, 0x36, 0x33, 0x33, 0x30, + 0x33, 0x32, 0x31, 0x65, 0x39, 0x62, 0x33, + 0x30, 0x39, 0x62, 0x36, 0x33, 0x34, 0x66, + 0x35, 0x32, 0x33, 0xad, 0x61, 0x75, 0x74, + 0x68, 0x5f, 0x73, 0x65, 0x71, 0x75, 0x65, + 0x6e, 0x63, 0x65, 0x91, 0x82, 0xa7, 0x61, + 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0xaa, + 0x61, 0x63, 0x74, 0x6f, 0x72, 0x31, 0x32, + 0x31, 0x32, 0x33, 0xa8, 0x73, 0x65, 0x71, + 0x75, 0x65, 0x6e, 0x63, 0x65, 0xce, 0x00, + 0x08, 0x5c, 0x39, 0xad, 0x63, 0x6f, 0x64, + 0x65, 0x5f, 0x73, 0x65, 0x71, 0x75, 0x65, + 0x6e, 0x63, 0x65, 0xce, 0x01, 0x61, 0xe8, + 0x79, 0xaf, 0x67, 0x6c, 0x6f, 0x62, 0x61, + 0x6c, 0x5f, 0x73, 0x65, 0x71, 0x75, 0x65, + 0x6e, 0x63, 0x65, 0xce, 0x02, 0x12, 0xd7, + 0x5d, 0xa8, 0x72, 0x65, 0x63, 0x65, 0x69, + 0x76, 0x65, 0x72, 0xa5, 0x65, 0x6f, 0x73, + 0x69, 0x6f, 0xad, 0x72, 0x65, 0x63, 0x76, + 0x5f, 0x73, 0x65, 0x71, 0x75, 0x65, 0x6e, + 0x63, 0x65, 0xce, 0x22, 0x81, 0x80, 0x7a, + 0xa8, 0x72, 0x65, 0x63, 0x65, 0x69, 0x76, + 0x65, 0x72, 0xa5, 0x65, 0x6f, 0x73, 0x69, + 0x6f, 0xa6, 0x72, 0x65, 0x74, 0x75, 0x72, + 0x6e, 0xc0, 0xa5, 0x74, 0x78, 0x5f, 0x69, + 0x64, 0xa0, 0x8c, 0xad, 0x61, 0x75, 0x74, + 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x91, 0x82, 0xa5, 0x61, + 0x63, 0x74, 0x6f, 0x72, 0xa9, 0x61, 0x63, + 0x74, 0x6f, 0x72, 0x31, 0x34, 0x38, 0x32, + 0xaa, 0x70, 0x65, 0x72, 0x6d, 0x69, 0x73, + 0x73, 0x69, 0x6f, 0x6e, 0xa6, 0x61, 0x63, + 0x74, 0x69, 0x76, 0x65, 0xa8, 0x62, 0x6c, + 0x6f, 0x63, 0x6b, 0x6e, 0x75, 0x6d, 0x00, + 0xae, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x74, + 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, + 0x70, 0xc0, 0xa8, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x61, 0x63, 0x74, 0xa7, 0x73, 0x6b, + 0x6a, 0x64, 0x68, 0x32, 0x34, 0xa4, 0x64, + 0x61, 0x74, 0x61, 0xc0, 0xa5, 0x65, 0x72, + 0x72, 0x6f, 0x72, 0x00, 0xa6, 0x65, 0x78, + 0x63, 0x65, 0x70, 0x74, 0xa0, 0xae, 0x66, + 0x69, 0x72, 0x73, 0x74, 0x5f, 0x72, 0x65, + 0x63, 0x65, 0x69, 0x76, 0x65, 0x72, 0xc2, + 0xa4, 0x6e, 0x61, 0x6d, 0x65, 0xa8, 0x73, + 0x65, 0x6c, 0x6c, 0x69, 0x74, 0x65, 0x6d, + 0xa8, 0x72, 0x65, 0x63, 0x65, 0x69, 0x76, + 0x65, 0x72, 0xa5, 0x65, 0x6f, 0x73, 0x69, + 0x6f, 0xa6, 0x72, 0x65, 0x74, 0x75, 0x72, + 0x6e, 0xc0, 0xa5, 0x74, 0x78, 0x5f, 0x69, + 0x64, 0xa0, 0xa8, 0x62, 0x6c, 0x6f, 0x63, + 0x6b, 0x6e, 0x75, 0x6d, 0xce, 0xa3, 0x3d, + 0x9b, 0xc5, 0xae, 0x62, 0x6c, 0x6f, 0x63, + 0x6b, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, + 0x61, 0x6d, 0x70, 0xd7, 0xff, 0x77, 0x35, + 0x94, 0x00, 0x65, 0x4e, 0x19, 0xff, 0xac, + 0x63, 0x70, 0x75, 0x5f, 0x75, 0x73, 0x61, + 0x67, 0x65, 0x5f, 0x75, 0x73, 0x17, 0xa7, + 0x65, 0x6c, 0x61, 0x70, 0x73, 0x65, 0x64, + 0x04, 0xa5, 0x65, 0x72, 0x72, 0x6f, 0x72, + 0x02, 0xa6, 0x65, 0x78, 0x63, 0x65, 0x70, + 0x74, 0xa9, 0x65, 0x78, 0x63, 0x65, 0x70, + 0x74, 0x73, 0x74, 0x72, 0xa2, 0x69, 0x64, + 0xd9, 0x40, 0x65, 0x64, 0x63, 0x30, 0x36, + 0x64, 0x63, 0x65, 0x36, 0x33, 0x32, 0x30, + 0x34, 0x35, 0x39, 0x66, 0x64, 0x36, 0x34, + 0x34, 0x37, 0x35, 0x36, 0x39, 0x37, 0x32, + 0x30, 0x34, 0x38, 0x64, 0x61, 0x34, 0x35, + 0x33, 0x62, 0x32, 0x38, 0x31, 0x36, 0x62, + 0x30, 0x61, 0x34, 0x33, 0x34, 0x63, 0x33, + 0x37, 0x64, 0x64, 0x66, 0x66, 0x64, 0x65, + 0x33, 0x36, 0x37, 0x37, 0x38, 0x64, 0x63, + 0x61, 0x62, 0x33, 0xa9, 0x6e, 0x65, 0x74, + 0x5f, 0x75, 0x73, 0x61, 0x67, 0x65, 0xcc, + 0x80, 0xaf, 0x6e, 0x65, 0x74, 0x5f, 0x75, + 0x73, 0x61, 0x67, 0x65, 0x5f, 0x77, 0x6f, + 0x72, 0x64, 0x73, 0x10, 0xa9, 0x73, 0x63, + 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x64, + 0xc3, 0xa6, 0x73, 0x74, 0x61, 0x74, 0x75, + 0x73, 0xa9, 0x73, 0x6f, 0x66, 0x74, 0x5f, + 0x66, 0x61, 0x69, 0x6c, } assert.Equal(t, expected, data) } From 5f2ae2ea520e9458e999bf8489f23e520eb0d866 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Thu, 29 Feb 2024 19:33:24 +0100 Subject: [PATCH 155/291] Makefile: make test command also test api. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 384a113..c69ebab 100644 --- a/Makefile +++ b/Makefile @@ -33,6 +33,6 @@ build-deb: test: $(GO) test -v ./... - + cd api; $(GO) test -v ./... clean : $(RM) -fr build From 83d57a09c4510890fa1d41f3cacf18be00a36eb9 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Fri, 1 Mar 2024 16:46:22 +0100 Subject: [PATCH 156/291] Version 1.1.0 --- Makefile | 2 +- debian/changelog | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index c69ebab..1637057 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ GO=go GOLDFLAGS=-v -s -w -X main.VersionString=$(PROGRAM_VERSION) GOBUILDFLAGS=-v -p $(shell nproc) -ldflags="$(GOLDFLAGS)" PROGRAM=thalos-server -PROGRAM_VERSION=1.1.0-rc2 +PROGRAM_VERSION=1.1.0 PREFIX=/usr/local BINDIR=$(PREFIX)/bin CFGDIR=$(PREFIX)/etc/thalos diff --git a/debian/changelog b/debian/changelog index b156561..89e0c9a 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,10 @@ +thalos (1.1.0) bionic focal jammy; urgency=medium + + * Adding `ActionTrace.FirstReceiver` flag, that is `true` + only if receiver is the same as contract name. + + -- Henrik Hautakoski Fri, 01 Mar 2024 16:41:25 +0100 + thalos (1.1.0~rc2) bionic focal jammy; urgency=medium * Adding `log.file_timestamp_format` config field From 4c843f16bf4db612aa56589ef3bda80fcd2f41f3 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Tue, 5 Mar 2024 15:00:04 +0100 Subject: [PATCH 157/291] api/README.md: update. --- api/README.md | 83 +++++++++++++++++++++++++-------------------------- 1 file changed, 40 insertions(+), 43 deletions(-) diff --git a/api/README.md b/api/README.md index b39cea6..0748531 100644 --- a/api/README.md +++ b/api/README.md @@ -2,7 +2,7 @@ ## Usage -The api is designed with callback functions that are called when messages arrive +The api is designed with go channels. ## Example @@ -31,59 +31,56 @@ func main() { ChainID: "1064487b3cd1a897ce03ae5b6a865651747e2e152090f99c1d19d44e01aea5a4", // Wax mainnet. }) - // Create client - codec, err := message.GetCodec("json") - if err != nil { - fmt.Println("Failed to get json codec") - return - } + // Create thalos client + codec, err := message.GetCodec("json") + if err != nil { + fmt.Println("Failed to get json codec") + return + } - client := api.NewClient(sub, codec.Decoder) + client := api.NewClient(sub, codec.Decoder) // Subscribe to some channels. - err = client.Subscribe( - api.TransactionChannel, - api.ActionChannel{Contract: "eosio"}.Channel(), - api.ActionChannel{Name: "mine"}.Channel(), - api.HeartbeatChannel, - api.TableDeltaChannel{}.Channel(), - ) + err = client.Subscribe( + api.TransactionChannel, + api.ActionChannel{Contract: "eosio"}.Channel(), + api.ActionChannel{Name: "mine"}.Channel(), + api.HeartbeatChannel, + api.TableDeltaChannel{}.Channel(), + ) - if err != nil { - fmt.Println(err) - return - } + if err != nil { + fmt.Println(err) + return + } // Wait for interrupt in a go routine and close the client. - go func() { - sig := make(chan os.Signal) - signal.Notify(sig, os.Interrupt) + go func() { + sig := make(chan os.Signal) + signal.Notify(sig, os.Interrupt) - <-sig - fmt.Println("Got interrupt") + <-sig + fmt.Println("Got interrupt") - client.Close() - }() + client.Close() + }() // Read messages - for t := range client.Channel() { - switch msg := t.(type) { - case error: - fmt.Println("Error:", msg) - case message.TransactionTrace: - fmt.Println("Transaction", msg.BlockNum, msg.ID) - fmt.Println(msg) - fmt.Println("---") - case message.HeartBeat : - fmt.Println("Heartbeat") - fmt.Println(msg) - fmt.Println("---") - } - } + for t := range client.Channel() { + switch msg := t.(type) { + case error: + fmt.Println("Error:", msg) + case message.TransactionTrace: + fmt.Println("Transaction", msg.BlockNum, msg.ID) + fmt.Println(msg) + fmt.Println("---") + case message.HeartBeat : + fmt.Println("Heartbeat") + fmt.Println(msg) + fmt.Println("---") + } + } } - - - ``` ## Message channels and types From aeac190c947ec45133f06fc6586553c488c6eae7 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Thu, 7 Mar 2024 18:09:14 +0100 Subject: [PATCH 158/291] Minor fixes. --- api/client.go | 14 +++++++------- cmd/thalos/server.go | 7 +++---- cmd/tools/bench.go | 2 +- cmd/tools/redis-acl.go | 2 +- cmd/tools/validate.go | 2 +- internal/server/ship_processor.go | 8 ++++---- 6 files changed, 17 insertions(+), 18 deletions(-) diff --git a/api/client.go b/api/client.go index ad9a3d5..9157af0 100644 --- a/api/client.go +++ b/api/client.go @@ -109,19 +109,19 @@ func (c *Client) hbHandler(payload []byte) { } func (c *Client) sub(channel Channel) error { - var handler handler + var h handler switch channel.Type() { case RollbackChannel.Type(): - handler = c.rollbackHandler + h = c.rollbackHandler case TransactionChannel.Type(): - handler = c.transactionHandler + h = c.transactionHandler case HeartbeatChannel.Type(): - handler = c.hbHandler + h = c.hbHandler case ActionChannel{}.Channel().Type(): - handler = c.actHandler + h = c.actHandler case TableDeltaChannel{}.Channel().Type(): - handler = c.tableDeltaHandler + h = c.tableDeltaHandler default: return fmt.Errorf("invalid channel type. %s", channel.Type()) } @@ -130,7 +130,7 @@ func (c *Client) sub(channel Channel) error { c.wg.Add(1) go func() { defer c.wg.Done() - c.worker(channel, handler) + c.worker(channel, h) }() return nil diff --git a/cmd/thalos/server.go b/cmd/thalos/server.go index 683bbe5..712cc45 100644 --- a/cmd/thalos/server.go +++ b/cmd/thalos/server.go @@ -23,7 +23,6 @@ import ( api_redis "github.com/eosswedenorg/thalos/api/redis" "github.com/eosswedenorg/thalos/internal/abi" "github.com/eosswedenorg/thalos/internal/cache" - . "github.com/eosswedenorg/thalos/internal/cache" "github.com/eosswedenorg/thalos/internal/config" driver "github.com/eosswedenorg/thalos/internal/driver/redis" . "github.com/eosswedenorg/thalos/internal/log" @@ -157,7 +156,7 @@ func LogLevels() []string { } func initAbiManager(api *eos.API, store cache.Store, chain_id string) *abi.AbiManager { - cache := NewCache("thalos::cache::abi::"+chain_id, store) + cache := cache.NewCache("thalos::cache::abi::"+chain_id, store) return abi.NewAbiManager(cache, api) } @@ -373,14 +372,14 @@ func serverCmd(cmd *cobra.Command, args []string) { } // Setup cache storage - cacheStore := NewRedisStore(&redis_cache.Options{ + cacheStore := cache.NewRedisStore(&redis_cache.Options{ Redis: rdb, // Cache 10k keys for 10 minutes. LocalCache: redis_cache.NewTinyLFU(10000, 10*time.Minute), }) // Setup general cache - cache := NewCache("thalos::cache::instance::"+conf.Name, cacheStore) + cache := cache.NewCache("thalos::cache::instance::"+conf.Name, cacheStore) eosClient := eos.New(conf.Api) diff --git a/cmd/tools/bench.go b/cmd/tools/bench.go index 31af016..cb96c8d 100644 --- a/cmd/tools/bench.go +++ b/cmd/tools/bench.go @@ -22,7 +22,7 @@ var benchCmd = &cobra.Command{ Use: "bench", Short: "Run a benchmark against a thalos node", Run: func(cmd *cobra.Command, args []string) { - var counter int = 0 + counter := 0 interval, _ := cmd.Flags().GetDuration("interval") url, _ := cmd.Flags().GetString("redis-url") diff --git a/cmd/tools/redis-acl.go b/cmd/tools/redis-acl.go index 25307b1..c1c5857 100644 --- a/cmd/tools/redis-acl.go +++ b/cmd/tools/redis-acl.go @@ -96,7 +96,7 @@ var RedisACLCmd = &cobra.Command{ Short: "create a users.acl file", Run: func(cmd *cobra.Command, args []string) { var err error - var out *os.File = os.Stdout + out := os.Stdout rnd = rand.New(rand.NewSource(time.Now().UnixNano())) diff --git a/cmd/tools/validate.go b/cmd/tools/validate.go index a0de15d..5c040b6 100644 --- a/cmd/tools/validate.go +++ b/cmd/tools/validate.go @@ -81,7 +81,7 @@ var validateCmd = &cobra.Command{ log.WithError(msg).Error("Error when reading stream") case message.ActionTrace: if block_num > 0 { - var diff int32 = int32(msg.BlockNum - block_num) + diff := int32(msg.BlockNum - block_num) if diff < 0 || diff > 1 { log.WithFields(log.Fields{ "current_block": block_num, diff --git a/internal/server/ship_processor.go b/internal/server/ship_processor.go index c15c4ae..49470a6 100644 --- a/internal/server/ship_processor.go +++ b/internal/server/ship_processor.go @@ -120,7 +120,7 @@ func (processor *ShipProcessor) updateAbiFromAction(act *ship.Action) error { Abi string Account eos.AccountName }{} - if err := decode(ABI, act, &set_abi); err != nil { + if err = decode(ABI, act, &set_abi); err != nil { return err } @@ -184,7 +184,7 @@ func (processor *ShipProcessor) processBlock(block *ship.GetBlocksResultV0) { transaction := message.TransactionTrace{ ID: trace.ID.String(), BlockNum: block.Block.BlockNumber(), - Timestamp: block.Block.Timestamp.Time.UTC(), + Timestamp: block.Block.Timestamp.UTC(), Status: trace.Status.String(), CPUUsageUS: trace.CPUUsageUS, NetUsage: trace.NetUsage, @@ -230,7 +230,7 @@ func (processor *ShipProcessor) processBlock(block *ship.GetBlocksResultV0) { act := message.ActionTrace{ TxID: trace.ID.String(), BlockNum: block.Block.BlockNumber(), - Timestamp: block.Block.Timestamp.Time.UTC(), + Timestamp: block.Block.Timestamp.UTC(), Name: act_trace.Act.Name.String(), Contract: act_trace.Act.Account.String(), Receiver: act_trace.Receiver.String(), @@ -331,7 +331,7 @@ func (processor *ShipProcessor) processBlock(block *ship.GetBlocksResultV0) { message := message.TableDelta{ BlockNum: block.Block.BlockNumber(), - Timestamp: block.Block.Timestamp.Time.UTC(), + Timestamp: block.Block.Timestamp.UTC(), Name: delta.Name, Rows: rows, } From 2d6d4e84ff7e5b82118f26c13895585e6b6c2839 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Thu, 7 Mar 2024 19:37:58 +0100 Subject: [PATCH 159/291] .github/workflows/test.yml: run tests on alpine linux. --- .github/workflows/test.yml | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ca634f7..f33a3c4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -25,4 +25,23 @@ jobs: run: go test -v ./... - name: Test API - run: cd api; go test -v ./... \ No newline at end of file + run: cd api; go test -v ./... + + test-alpine: + strategy: + fail-fast: false + matrix: + tag: ["1.20-alpine3.19", "1.21-alpine3.19"] + runs-on: ubuntu-latest + name: Test alpine (${{ matrix.tag }}) + container: + image: golang:${{ matrix.tag }} + steps: + - uses: actions/checkout@v4 + + - name: Test + run: go test -v ./... + + - name: Test API + run: cd api; go test -v ./... + From 10f47c33352e59818edfcfb45e3459960d9ce13e Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Thu, 7 Mar 2024 20:50:37 +0100 Subject: [PATCH 160/291] Update README.md Adding test badge. --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index e96fd70..4a265dc 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # Thalos +[![Test](https://github.com/eosswedenorg/thalos/actions/workflows/test.yml/badge.svg)](https://github.com/eosswedenorg/thalos/actions/workflows/test.yml) + Thalos is a application that makes it easy for users to stream blockchain data from an Antelope SHIP node. Consult the [documentation](https://thalos.waxsweden.org/docs) for more information. @@ -21,4 +23,4 @@ $ go build -o build/thalos-server cmd/thalos/*.go ``` ## Author -Henrik Hautakoski - [henrik@eossweden.org](mailto:henrik@eossweden.org) \ No newline at end of file +Henrik Hautakoski - [henrik@eossweden.org](mailto:henrik@eossweden.org) From 9a6119ca93aee5df3f2c3208b3a0d1fdb9810156 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Thu, 7 Mar 2024 20:01:03 +0100 Subject: [PATCH 161/291] .github/workflows/release.yml: build musl linked binaries. --- .github/workflows/release.yml | 57 ++++++++++++++++++++++++++++++++++- 1 file changed, 56 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5867d10..5a93045 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -59,6 +59,61 @@ jobs: asset_path: build/bundle.tar.gz asset_content_type: application/tar+gzip + # Build thalos binaries that are linked with musl libc. + musl: + strategy: + fail-fast: false + matrix: + arch: [ 386, amd64, arm, arm64 ] + runs-on: ubuntu-latest + name: Build musl (${{ matrix.arch }}) + container: + image: golang:1.21-alpine3.19 + steps: + - uses: actions/checkout@v4 + + - name: install dependencies + run: apk add make + + - name: compile + id: compile + run: | + mkdir -p build/bundle/{bin,logs} + GOARCH=${{matrix.arch}} make -e DESTDIR=build/bundle PREFIX= CFGDIR= install install-scripts + tar -C build/bundle -z -cf build/bundle.tar.gz . + echo "version=$(sed -n 's/.*PROGRAM_VERSION\s*=\s*//p' Makefile)" >> "$GITHUB_OUTPUT" + + - name: Upload thalos-server + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ github.event.release.upload_url }} + asset_name: thalos-server-${{steps.compile.outputs.version}}-linux-${{matrix.arch}}-musl + asset_path: build/thalos-server + asset_content_type: application/octal-stream + + - name: Upload thalos-tools + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ github.event.release.upload_url }} + asset_name: thalos-tools-${{steps.compile.outputs.version}}-linux-${{matrix.arch}}-musl + asset_path: build/thalos-tools + asset_content_type: application/octal-stream + + - name: Upload bundle + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ github.event.release.upload_url }} + asset_name: thalos-${{steps.compile.outputs.version}}-linux-${{matrix.arch}}-musl.tar.gz + asset_path: build/bundle.tar.gz + asset_content_type: application/tar+gzip + + package-ubuntu: strategy: fail-fast: false @@ -121,4 +176,4 @@ jobs: upload_url: ${{ github.event.release.upload_url }} asset_name: ${{ steps.package.outputs.info_name }} asset_path: ${{ steps.package.outputs.info_filename }} - asset_content_type: text/plain \ No newline at end of file + asset_content_type: text/plain From 69b31eae1afef50d43c9ad97d1ca4964fdcdaf39 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Thu, 7 Mar 2024 23:15:25 +0100 Subject: [PATCH 162/291] cmd/tools/redis-acl.go: enable ping command for server user. --- cmd/tools/redis-acl.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/tools/redis-acl.go b/cmd/tools/redis-acl.go index c1c5857..4b11609 100644 --- a/cmd/tools/redis-acl.go +++ b/cmd/tools/redis-acl.go @@ -71,7 +71,7 @@ func hash(str string) string { func writeTemplate(w io.Writer, defUser, serverUser, clientUser User, prefix string) error { tmplStr := `# Created by thalos-tools on {{.timestamp}} user default on {{.defaultpw}} ~* &* +@all -user {{.server}} on {{.serverpw}} resetchannels ~{{.prefix}}::* &{{.prefix}}::* -@all +get +publish +set +user {{.server}} on {{.serverpw}} resetchannels ~{{.prefix}}::* &{{.prefix}}::* -@all +ping +get +publish +set user {{.client}} on {{.clientpw}} resetchannels &{{.prefix}}::* -@all +subscribe ` From 72b4f0a32727492936dc3dc5b5d90548e2185757 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Thu, 7 Mar 2024 23:29:31 +0100 Subject: [PATCH 163/291] cmd/tools/redis-acl.go: fix correct syntax for cleartext passwords. --- cmd/tools/redis-acl.go | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/cmd/tools/redis-acl.go b/cmd/tools/redis-acl.go index 4b11609..7a83263 100644 --- a/cmd/tools/redis-acl.go +++ b/cmd/tools/redis-acl.go @@ -26,6 +26,9 @@ type User struct { // True if password was generated, false if not. Generated bool + + // True if password should be hashed, false otherwise. + Hash bool } func NewUser(name, password string) User { @@ -34,13 +37,19 @@ func NewUser(name, password string) User { Name: name, Password: randomString(32), Generated: true, + Hash: false, } } return User{Name: name, Password: password} } -func (u *User) Hash() { - u.Password = "#" + hash(u.Password) +func (u *User) GetPassword() string { + + if u.Hash { + return "#" + hash(u.Password) + } + + return ">" + u.Password } func (u User) Print() { @@ -81,11 +90,11 @@ user {{.client}} on {{.clientpw}} resetchannels &{{.prefix}}::* -@all +subscribe } return tmpl.Execute(w, map[string]string{ - "defaultpw": defUser.Password, + "defaultpw": defUser.GetPassword(), "client": clientUser.Name, - "clientpw": clientUser.Password, + "clientpw": clientUser.GetPassword(), "server": serverUser.Name, - "serverpw": serverUser.Password, + "serverpw": serverUser.GetPassword(), "prefix": prefix, "timestamp": time.Now().Format(time.UnixDate), }) @@ -123,9 +132,9 @@ var RedisACLCmd = &cobra.Command{ serverUser.PrintIfGeneratedPW() clientUser.PrintIfGeneratedPW() - defaultUser.Hash() - serverUser.Hash() - clientUser.Hash() + defaultUser.Hash = true + serverUser.Hash = true + clientUser.Hash = true } filename, _ := cmd.Flags().GetString("file") From bf97f2e97abde523d00b050c58fbbedce227f0ad Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Thu, 14 Mar 2024 00:26:17 +0100 Subject: [PATCH 164/291] cmd/tools/redis-acl.go: adding flag for password length. --- cmd/tools/main.go | 1 + cmd/tools/redis-acl.go | 16 ++++++++-------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/cmd/tools/main.go b/cmd/tools/main.go index ec477d0..abb206d 100644 --- a/cmd/tools/main.go +++ b/cmd/tools/main.go @@ -47,6 +47,7 @@ func init() { RedisACLCmd.Flags().String("prefix", "ship", "Redis key prefix") RedisACLCmd.Flags().Bool("cleartext", false, "If passwords should be hashed or left in cleartext.") RedisACLCmd.Flags().String("file", "", "Where the config should be written to (default: standard out)") + RedisACLCmd.Flags().Uint("pass-len", 32, "The length of generated passwords") rootCmd.AddCommand( validateCmd, diff --git a/cmd/tools/redis-acl.go b/cmd/tools/redis-acl.go index 7a83263..0f4496c 100644 --- a/cmd/tools/redis-acl.go +++ b/cmd/tools/redis-acl.go @@ -31,13 +31,12 @@ type User struct { Hash bool } -func NewUser(name, password string) User { +func NewUser(name, password string, pass_len uint) User { if len(password) < 1 { return User{ Name: name, - Password: randomString(32), + Password: randomString(pass_len), Generated: true, - Hash: false, } } return User{Name: name, Password: password} @@ -62,10 +61,10 @@ func (u User) PrintIfGeneratedPW() { } } -func randomString(length int) string { +func randomString(length uint) string { charset := "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUWXYZ0123456789" out := "" - for i := 0; i < length; i++ { + for i := 0; i < int(length); i++ { idx := rnd.Intn(len(charset)) out += string(charset[idx]) } @@ -115,10 +114,11 @@ var RedisACLCmd = &cobra.Command{ flagClient, _ := cmd.Flags().GetString("client") flagClientPw, _ := cmd.Flags().GetString("client-pw") flagPrefix, _ := cmd.Flags().GetString("prefix") + flagPassLen, _ := cmd.Flags().GetUint("pass-len") - defaultUser := NewUser("default", flagDefUserPw) - serverUser := NewUser(flagServer, flagServerPw) - clientUser := NewUser(flagClient, flagClientPw) + defaultUser := NewUser("default", flagDefUserPw, flagPassLen) + serverUser := NewUser(flagServer, flagServerPw, flagPassLen) + clientUser := NewUser(flagClient, flagClientPw, flagPassLen) atleastOneGeneratedPw := defaultUser.Generated || serverUser.Generated || clientUser.Generated From 8cbf350d5ee1cd4b5b928cef936068378ca7b0f8 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Thu, 14 Mar 2024 00:29:00 +0100 Subject: [PATCH 165/291] Adding docker image. --- docker/Dockerfile | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 docker/Dockerfile diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..78e5677 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,6 @@ +FROM alpine:latest +LABEL maintainer="Henrik Hautakoski " +ARG VERSION=1.1.0 +WORKDIR /thalos +ADD --chmod=755 https://github.com/eosswedenorg/thalos/releases/download/v$VERSION/thalos-server-${VERSION}-linux-amd64-musl thalos-server +ENTRYPOINT [ "./thalos-server" ] From fbbd7a16622a5ee11c48c135b4e5e69526d5dab1 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Thu, 14 Mar 2024 00:29:34 +0100 Subject: [PATCH 166/291] README.md: add a go report badge. --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 4a265dc..80f64b1 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ [![Test](https://github.com/eosswedenorg/thalos/actions/workflows/test.yml/badge.svg)](https://github.com/eosswedenorg/thalos/actions/workflows/test.yml) +[![Go Report Card](https://goreportcard.com/badge/github.com/eosswedenorg/thalos)](https://goreportcard.com/report/github.com/eosswedenorg/thalos) + Thalos is a application that makes it easy for users to stream blockchain data from an Antelope SHIP node. Consult the [documentation](https://thalos.waxsweden.org/docs) for more information. From e8f2e79e5b958c23730b9f5073d5c4aed63e292b Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Thu, 14 Mar 2024 00:31:16 +0100 Subject: [PATCH 167/291] internal/server/ship_processor.go: typo fix. --- internal/server/ship_processor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/server/ship_processor.go b/internal/server/ship_processor.go index 49470a6..eac7a1f 100644 --- a/internal/server/ship_processor.go +++ b/internal/server/ship_processor.go @@ -30,7 +30,7 @@ func logDecoratedEncoder(encoder message.Encoder) message.Encoder { } // A ShipProcessor will consume messages from a ship stream, convert the messages into -// thalos specfic ones, encode them and finally post them to an api.Writer +// thalos specific ones, encode them and finally post them to an api.Writer type ShipProcessor struct { // The ship stream to process. shipStream *shipclient.Stream From 8aa93dd0788dd66fd417c3468a8807d4f6ba18df Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Thu, 14 Mar 2024 23:24:31 +0100 Subject: [PATCH 168/291] internal/server/ship_processor.go: break processBlock into smaller functions. --- internal/server/helpers.go | 25 ++++ internal/server/ship_processor.go | 232 +++++++++++++++--------------- 2 files changed, 139 insertions(+), 118 deletions(-) create mode 100644 internal/server/helpers.go diff --git a/internal/server/helpers.go b/internal/server/helpers.go new file mode 100644 index 0000000..f7ce531 --- /dev/null +++ b/internal/server/helpers.go @@ -0,0 +1,25 @@ +package server + +import "github.com/eoscanada/eos-go/ship" + +// convert a ActionTrace to ActionTraceV1 +func toActionTraceV1(trace *ship.ActionTrace) *ship.ActionTraceV1 { + if trace_v0, ok := trace.Impl.(*ship.ActionTraceV0); ok { + // convert to v1 + return &ship.ActionTraceV1{ + ActionOrdinal: trace_v0.ActionOrdinal, + CreatorActionOrdinal: trace_v0.CreatorActionOrdinal, + Receipt: trace_v0.Receipt, + Receiver: trace_v0.Receiver, + Act: trace_v0.Act, + ContextFree: trace_v0.ContextFree, + Elapsed: trace_v0.Elapsed, + Console: trace_v0.Console, + AccountRamDeltas: trace_v0.AccountRamDeltas, + Except: trace_v0.Except, + ErrorCode: trace_v0.ErrorCode, + ReturnValue: []byte{}, + } + } + return trace.Impl.(*ship.ActionTraceV1) +} diff --git a/internal/server/ship_processor.go b/internal/server/ship_processor.go index eac7a1f..aa4017a 100644 --- a/internal/server/ship_processor.go +++ b/internal/server/ship_processor.go @@ -142,6 +142,119 @@ func (processor *ShipProcessor) GetCurrentBlock() uint32 { return processor.state.CurrentBlock } +func (processor *ShipProcessor) broadcastAction(act *message.ActionTrace) { + payload, err := processor.encode(*act) + if err != nil { + log.WithField("act", act).Warn("failed to encode action") + return + } + + channels := []api.Channel{ + api.ActionChannel{}.Channel(), + api.ActionChannel{Name: act.Name}.Channel(), + api.ActionChannel{Contract: act.Contract}.Channel(), + api.ActionChannel{Name: act.Name, Contract: act.Contract}.Channel(), + } + + for _, channel := range channels { + processor.queueMessage(channel, payload) + } +} + +func (processor *ShipProcessor) processTransactionTrace(log *log.Entry, block *ship.SignedBlockBytes, trace *ship.TransactionTraceV0) { + logger := log.WithField("type", "trace").WithField("tx_id", trace.ID.String()).Dup() + + transaction := message.TransactionTrace{ + ID: trace.ID.String(), + BlockNum: block.BlockNumber(), + Timestamp: block.Timestamp.UTC(), + Status: trace.Status.String(), + CPUUsageUS: trace.CPUUsageUS, + NetUsage: trace.NetUsage, + NetUsageWords: uint32(trace.NetUsageWords), + Elapsed: int64(trace.Elapsed), + Scheduled: trace.Scheduled, + Except: trace.Except, + Error: trace.ErrorCode, + } + + // Actions + for _, actionTraceVar := range trace.ActionTraces { + + actionTrace := toActionTraceV1(actionTraceVar) + actMsg := processor.proccessActionTrace(logger, actionTrace) + if actMsg != nil { + actMsg.TxID = trace.ID.String() + actMsg.BlockNum = block.BlockNumber() + actMsg.Timestamp = block.Timestamp.UTC() + + processor.broadcastAction(actMsg) + + transaction.ActionTraces = append(transaction.ActionTraces, *actMsg) + } + } + + processor.encodeQueue(api.TransactionChannel, transaction) +} + +func (processor *ShipProcessor) proccessActionTrace(logger *log.Entry, trace *ship.ActionTraceV1) *message.ActionTrace { + // Check if actions updates an abi. + if trace.Act.Account == processor.syscontract && trace.Act.Name == eos.ActionName("setabi") { + err := processor.updateAbiFromAction(trace.Act) + if err != nil { + logger.WithError(err).Warn("Failed to update abi") + } + } + + act := &message.ActionTrace{ + Name: trace.Act.Name.String(), + Contract: trace.Act.Account.String(), + Receiver: trace.Receiver.String(), + FirstReceiver: trace.Act.Account.String() == trace.Receiver.String(), + } + + if trace.Receipt != nil { + receipt := trace.Receipt.Impl.(*ship.ActionReceiptV0) + act.Receipt = &message.ActionReceipt{ + Receiver: receipt.Receiver.String(), + ActDigest: receipt.ActDigest.String(), + GlobalSequence: receipt.GlobalSequence, + RecvSequence: receipt.RecvSequence, + CodeSequence: uint32(receipt.CodeSequence), + ABISequence: uint32(receipt.ABISequence), + } + + for _, auth := range receipt.AuthSequence { + act.Receipt.AuthSequence = append(act.Receipt.AuthSequence, message.AccountAuthSequence{ + Account: auth.Account.String(), + Sequence: auth.Sequence, + }) + } + } + + for _, auth := range trace.Act.Authorization { + act.Authorization = append(act.Authorization, message.PermissionLevel{ + Actor: auth.Actor.String(), + Permission: auth.Permission.String(), + }) + } + + ABI, err := processor.abi.GetAbi(trace.Act.Account) + if err == nil { + if err = decode(ABI, trace.Act, &act.Data); err != nil { + logger.WithFields(log.Fields{ + "contract": trace.Act.Account, + "action": trace.Act.Name, + }).WithError(err).Warn("Failed to decode action") + } + } else { + logger.WithField("contract", trace.Act.Account). + WithError(err).Error("Failed to get abi for contract") + } + + return act +} + // Callback function called by shipclient.Stream when a new block arrives. func (processor *ShipProcessor) processBlock(block *ship.GetBlocksResultV0) { // Check to see if we have a microfork and post a message to @@ -178,124 +291,7 @@ func (processor *ShipProcessor) processBlock(block *ship.GetBlocksResultV0) { // Process traces if block.Traces != nil && len(block.Traces.Elem) > 0 { for _, trace := range block.Traces.AsTransactionTracesV0() { - - logger := mainLogger.WithField("type", "trace").WithField("tx_id", trace.ID.String()).Dup() - - transaction := message.TransactionTrace{ - ID: trace.ID.String(), - BlockNum: block.Block.BlockNumber(), - Timestamp: block.Block.Timestamp.UTC(), - Status: trace.Status.String(), - CPUUsageUS: trace.CPUUsageUS, - NetUsage: trace.NetUsage, - NetUsageWords: uint32(trace.NetUsageWords), - Elapsed: int64(trace.Elapsed), - Scheduled: trace.Scheduled, - Except: trace.Except, - Error: trace.ErrorCode, - } - - // Actions - for _, actionTraceVar := range trace.ActionTraces { - var act_trace *ship.ActionTraceV1 - - if trace_v0, ok := actionTraceVar.Impl.(*ship.ActionTraceV0); ok { - // convert to v1 - act_trace = &ship.ActionTraceV1{ - ActionOrdinal: trace_v0.ActionOrdinal, - CreatorActionOrdinal: trace_v0.CreatorActionOrdinal, - Receipt: trace_v0.Receipt, - Receiver: trace_v0.Receiver, - Act: trace_v0.Act, - ContextFree: trace_v0.ContextFree, - Elapsed: trace_v0.Elapsed, - Console: trace_v0.Console, - AccountRamDeltas: trace_v0.AccountRamDeltas, - Except: trace_v0.Except, - ErrorCode: trace_v0.ErrorCode, - ReturnValue: []byte{}, - } - } else { - act_trace = actionTraceVar.Impl.(*ship.ActionTraceV1) - } - - // Check if actions updates an abi. - if act_trace.Act.Account == processor.syscontract && act_trace.Act.Name == eos.ActionName("setabi") { - err := processor.updateAbiFromAction(act_trace.Act) - if err != nil { - logger.WithError(err).Warn("Failed to update abi") - } - } - - act := message.ActionTrace{ - TxID: trace.ID.String(), - BlockNum: block.Block.BlockNumber(), - Timestamp: block.Block.Timestamp.UTC(), - Name: act_trace.Act.Name.String(), - Contract: act_trace.Act.Account.String(), - Receiver: act_trace.Receiver.String(), - FirstReceiver: act_trace.Act.Account.String() == act_trace.Receiver.String(), - } - - if act_trace.Receipt != nil { - receipt := act_trace.Receipt.Impl.(*ship.ActionReceiptV0) - act.Receipt = &message.ActionReceipt{ - Receiver: receipt.Receiver.String(), - ActDigest: receipt.ActDigest.String(), - GlobalSequence: receipt.GlobalSequence, - RecvSequence: receipt.RecvSequence, - CodeSequence: uint32(receipt.CodeSequence), - ABISequence: uint32(receipt.ABISequence), - } - - for _, auth := range receipt.AuthSequence { - act.Receipt.AuthSequence = append(act.Receipt.AuthSequence, message.AccountAuthSequence{ - Account: auth.Account.String(), - Sequence: auth.Sequence, - }) - } - } - - for _, auth := range act_trace.Act.Authorization { - act.Authorization = append(act.Authorization, message.PermissionLevel{ - Actor: auth.Actor.String(), - Permission: auth.Permission.String(), - }) - } - - ABI, err := processor.abi.GetAbi(act_trace.Act.Account) - if err == nil { - if err = decode(ABI, act_trace.Act, &act.Data); err != nil { - logger.WithFields(log.Fields{ - "contract": act_trace.Act.Account, - "action": act_trace.Act.Name, - }).WithError(err).Warn("Failed to decode action") - } - } else { - logger.WithField("contract", act_trace.Act.Account). - WithError(err).Error("Failed to get abi for contract") - } - - payload, err := processor.encode(act) - if err != nil { - continue - } - - transaction.ActionTraces = append(transaction.ActionTraces, act) - - channels := []api.Channel{ - api.ActionChannel{}.Channel(), - api.ActionChannel{Name: act.Name}.Channel(), - api.ActionChannel{Contract: act.Contract}.Channel(), - api.ActionChannel{Name: act.Name, Contract: act.Contract}.Channel(), - } - - for _, channel := range channels { - processor.queueMessage(channel, payload) - } - } - - processor.encodeQueue(api.TransactionChannel, transaction) + processor.processTransactionTrace(mainLogger, block.Block, trace) } } From ef9a6e34a229ffb218fb8092806127ca38e854ec Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Tue, 9 Apr 2024 22:24:44 +0200 Subject: [PATCH 169/291] .gitignore: ignore vendor dir. --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 0406edd..0da5a1e 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ /config.yml /.pc /.vscode/ +/vendor From 108014f2cf27186cc2a3e28ca65b2c1ea0b1e450 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Tue, 9 Apr 2024 22:39:39 +0200 Subject: [PATCH 170/291] go: update dependencies --- go.mod | 24 ++++++++++++------------ go.sum | 47 +++++++++++++++++++++++++---------------------- 2 files changed, 37 insertions(+), 34 deletions(-) diff --git a/go.mod b/go.mod index 3645aa9..c61abcf 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/eosswedenorg/thalos go 1.20 require ( - github.com/cenkalti/backoff/v4 v4.2.1 + github.com/cenkalti/backoff/v4 v4.3.0 github.com/docker/go-units v0.5.0 github.com/eoscanada/eos-go v0.10.3-0.20231109144819-59afdfa3a37d github.com/eosswedenorg-go/antelope-ship-client v0.2.8 @@ -18,13 +18,13 @@ require ( github.com/spf13/cobra v1.8.0 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.18.2 - github.com/stretchr/testify v1.8.4 + github.com/stretchr/testify v1.9.0 gopkg.in/yaml.v3 v3.0.1 ) require ( github.com/blendle/zapdriver v1.3.1 // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/eosswedenorg-go/jsontime v0.0.0-20230509125027-08422d6236c7 // indirect @@ -34,14 +34,14 @@ require ( github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.17.7 // indirect + github.com/klauspost/compress v1.17.8 // indirect github.com/logrusorgru/aurora v2.0.3+incompatible // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mitchellh/go-testing-interface v1.14.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/onsi/gomega v1.31.1 // indirect - github.com/pelletier/go-toml/v2 v2.1.1 // indirect + github.com/onsi/gomega v1.32.0 // indirect + github.com/pelletier/go-toml/v2 v2.2.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect @@ -61,12 +61,12 @@ require ( github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect - golang.org/x/crypto v0.20.0 // indirect - golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 // indirect - golang.org/x/net v0.21.0 // indirect - golang.org/x/sync v0.6.0 // indirect - golang.org/x/sys v0.17.0 // indirect - golang.org/x/term v0.17.0 // indirect + golang.org/x/crypto v0.22.0 // indirect + golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 // indirect + golang.org/x/net v0.24.0 // indirect + golang.org/x/sync v0.7.0 // indirect + golang.org/x/sys v0.19.0 // indirect + golang.org/x/term v0.19.0 // indirect golang.org/x/text v0.14.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/ini.v1 v1.67.0 // indirect diff --git a/go.sum b/go.sum index 8809c73..f3c5bc3 100644 --- a/go.sum +++ b/go.sum @@ -5,11 +5,12 @@ github.com/blendle/zapdriver v1.3.1 h1:C3dydBOWYRiOk+B8X9IVZ5IOe+7cl+tGOexN4QqHf github.com/blendle/zapdriver v1.3.1/go.mod h1:mdXfREi6u5MArG4j9fewC+FGnXaBR+T4Ox4J2u4eHCc= github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= -github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= -github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -75,8 +76,8 @@ github.com/jordan-wright/email v4.0.1-0.20210109023952-943e75fe5223+incompatible github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg= -github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= +github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= @@ -125,10 +126,10 @@ github.com/onsi/gomega v1.22.1/go.mod h1:x6n7VNe4hw0vkyYUM4mjIXx3JbLiPaBPNgB7PRQ github.com/onsi/gomega v1.24.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg= github.com/onsi/gomega v1.24.1/go.mod h1:3AOiACssS3/MajrniINInwbfOOtfZvplPzuRSmvt1jM= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= -github.com/onsi/gomega v1.31.1 h1:KYppCUK+bUgAZwHOu7EXVBKyQA6ILvOESHkn/tgoqvo= -github.com/onsi/gomega v1.31.1/go.mod h1:y40C95dwAD1Nz36SsEnxvfFe8FFfNxzI5eJ0EYGyAy0= -github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI= -github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= +github.com/onsi/gomega v1.32.0 h1:JRYU78fJ1LPxlckP6Txi/EYqJvjtMrDC04/MM5XRHPk= +github.com/onsi/gomega v1.32.0/go.mod h1:a4x4gW6Pz2yK1MAmvluYme5lvYTn61afQ2ETw/8n4Lg= +github.com/pelletier/go-toml/v2 v2.2.0 h1:QLgLl2yMN7N+ruc31VynXs1vhMZa7CeHHejIeBAsoHo= +github.com/pelletier/go-toml/v2 v2.2.0/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -164,6 +165,7 @@ github.com/streamingfast/logging v0.0.0-20230608130331-f22c91403091/go.mod h1:Vl github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -171,8 +173,9 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/technoweenie/multipartstreamer v1.0.1 h1:XRztA5MXiR1TIRHxH2uNxXxaIkKQDeX7m2XsSOlQEnM= @@ -218,10 +221,10 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= -golang.org/x/crypto v0.20.0 h1:jmAMJJZXr5KiCw05dfYK9QnqaqKLYXijU23lsEdcQqg= -golang.org/x/crypto v0.20.0/go.mod h1:Xwo95rrVNIoSMx9wa1JroENMToLWn3RNVrTBpLHgZPQ= -golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 h1:LfspQV/FYTatPTr/3HzIcmiUFH7PGP+OQ6mgDYo3yuQ= -golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= +golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= +golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= +golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 h1:ESSUROHIBHg7USnszlcdmjBEwdMj9VUvU+OPk4yl2mc= +golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8/go.mod h1:/lliqkxwWAhPjf5oSOIJup2XcqJaw8RGS6k3TGEc7GI= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -247,16 +250,16 @@ golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= -golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= -golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= +golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= -golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -283,16 +286,16 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= -golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= +golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= -golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U= -golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q= +golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= From 7ff006637d7b76e38b73db2d927cc818e68b5d82 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Tue, 9 Apr 2024 22:41:00 +0200 Subject: [PATCH 171/291] Version 1.1.1 --- Makefile | 2 +- debian/changelog | 10 ++++++++++ docker/Dockerfile | 2 +- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 1637057..5c0704c 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ GO=go GOLDFLAGS=-v -s -w -X main.VersionString=$(PROGRAM_VERSION) GOBUILDFLAGS=-v -p $(shell nproc) -ldflags="$(GOLDFLAGS)" PROGRAM=thalos-server -PROGRAM_VERSION=1.1.0 +PROGRAM_VERSION=1.1.1 PREFIX=/usr/local BINDIR=$(PREFIX)/bin CFGDIR=$(PREFIX)/etc/thalos diff --git a/debian/changelog b/debian/changelog index 89e0c9a..8d0d57f 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,13 @@ +thalos (1.1.1) bionic focal jammy; urgency=medium + + * Build binaries linked with musl libc for alpine linux. + * Added docker image. + * redis-acl tool: added `--pass-len` flag. + * redis-acl tool: fix correct syntax for cleartext passwords. + * redis-acl tool: allow ping command for server user. + + -- Henrik Hautakoski Tue, 09 Apr 2024 22:40:20 +0200 + thalos (1.1.0) bionic focal jammy; urgency=medium * Adding `ActionTrace.FirstReceiver` flag, that is `true` diff --git a/docker/Dockerfile b/docker/Dockerfile index 78e5677..70c5fe5 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,6 +1,6 @@ FROM alpine:latest LABEL maintainer="Henrik Hautakoski " -ARG VERSION=1.1.0 +ARG VERSION=1.1.1 WORKDIR /thalos ADD --chmod=755 https://github.com/eosswedenorg/thalos/releases/download/v$VERSION/thalos-server-${VERSION}-linux-amd64-musl thalos-server ENTRYPOINT [ "./thalos-server" ] From 63da39f03b1042fb27c37e2b0e4a377c5c8b7ea1 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Mon, 29 Apr 2024 21:00:45 +0200 Subject: [PATCH 172/291] Update to use antelope-go instead of eos-go library --- cmd/thalos/server.go | 26 ++--- go.mod | 27 ++--- go.sum | 78 ++++--------- internal/abi/manager.go | 23 ++-- internal/abi/manager_test.go | 31 +++--- internal/server/helpers.go | 30 ++--- internal/server/ship_processor.go | 177 +++++++++++++++--------------- 7 files changed, 181 insertions(+), 211 deletions(-) diff --git a/cmd/thalos/server.go b/cmd/thalos/server.go index 712cc45..d4a764c 100644 --- a/cmd/thalos/server.go +++ b/cmd/thalos/server.go @@ -13,7 +13,6 @@ import ( "time" "github.com/cenkalti/backoff/v4" - eos "github.com/eoscanada/eos-go" shipclient "github.com/eosswedenorg-go/antelope-ship-client" shipws "github.com/eosswedenorg-go/antelope-ship-client/websocket" "github.com/eosswedenorg-go/pid" @@ -30,6 +29,7 @@ import ( redis_cache "github.com/go-redis/cache/v9" "github.com/nikoksr/notify" "github.com/nikoksr/notify/service/telegram" + antelopeapi "github.com/pnx/antelope-go/api" "github.com/redis/go-redis/v9" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" @@ -155,12 +155,12 @@ func LogLevels() []string { return list } -func initAbiManager(api *eos.API, store cache.Store, chain_id string) *abi.AbiManager { +func initAbiManager(api *antelopeapi.Client, store cache.Store, chain_id string) *abi.AbiManager { cache := cache.NewCache("thalos::cache::abi::"+chain_id, store) return abi.NewAbiManager(cache, api) } -func stateLoader(conf *config.Config, start_block_flag *pflag.Flag, chainInfo func() *eos.InfoResp, cache *cache.Cache, current_block_no_cache bool) StateLoader { +func stateLoader(conf *config.Config, start_block_flag *pflag.Flag, chainInfo func() *antelopeapi.Info, cache *cache.Cache, current_block_no_cache bool) StateLoader { return func(state *State) { var source string @@ -186,7 +186,7 @@ func stateLoader(conf *config.Config, start_block_flag *pflag.Flag, chainInfo fu // Otherwise, set from api. if conf.Ship.IrreversibleOnly { source = "api (LIB)" - state.CurrentBlock = uint32(chainInfo().LastIrreversibleBlockNum) + state.CurrentBlock = uint32(chainInfo().LastIrreversableBlockNum) } else { source = "api (HEAD)" state.CurrentBlock = uint32(chainInfo().HeadBlockNum) @@ -246,12 +246,12 @@ func GetConfig(flags *pflag.FlagSet) (*config.Config, error) { // that pointer will live as long as the closure lives. // and inside the closure we will reference the pointer and only // call the api if it is nil. -func chainInfoOnce(api *eos.API) func() *eos.InfoResp { - var info *eos.InfoResp - return func() *eos.InfoResp { +func chainInfoOnce(api *antelopeapi.Client) func() *antelopeapi.Info { + var info *antelopeapi.Info + return func() *antelopeapi.Info { if info == nil { - log.WithField("api", api.BaseURL).Info("Get chain info from api") + log.WithField("api", api.Url).Info("Get chain info from api") ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) defer cancel() @@ -262,7 +262,7 @@ func chainInfoOnce(api *eos.API) func() *eos.InfoResp { return nil } - info = result + info = &result } return info } @@ -381,7 +381,7 @@ func serverCmd(cmd *cobra.Command, args []string) { // Setup general cache cache := cache.NewCache("thalos::cache::instance::"+conf.Name, cacheStore) - eosClient := eos.New(conf.Api) + antelopeClient := antelopeapi.New(conf.Api) shClient := shipclient.NewStream(func(s *shipclient.Stream) { s.StartBlock = conf.Ship.StartBlockNum @@ -396,11 +396,11 @@ func serverCmd(cmd *cobra.Command, args []string) { return } - chainInfo := chainInfoOnce(eosClient) + chainInfo := chainInfoOnce(antelopeClient) chain_id := conf.Ship.Chain if len(chain_id) < 1 { - chain_id = chainInfo().ChainID.String() + chain_id = chainInfo().ChainID } processor := SpawnProccessor( @@ -411,7 +411,7 @@ func serverCmd(cmd *cobra.Command, args []string) { Prefix: conf.Redis.Prefix, ChainID: chain_id, }), - initAbiManager(eosClient, cacheStore, chain_id), + initAbiManager(antelopeClient, cacheStore, chain_id), codec, ) diff --git a/go.mod b/go.mod index 3645aa9..a3baa8e 100644 --- a/go.mod +++ b/go.mod @@ -5,39 +5,41 @@ go 1.20 require ( github.com/cenkalti/backoff/v4 v4.2.1 github.com/docker/go-units v0.5.0 - github.com/eoscanada/eos-go v0.10.3-0.20231109144819-59afdfa3a37d - github.com/eosswedenorg-go/antelope-ship-client v0.2.8 + github.com/eosswedenorg-go/antelope-ship-client v0.2.9 github.com/eosswedenorg-go/pid v1.0.1 github.com/eosswedenorg/thalos/api v1.0.0 github.com/go-redis/cache/v9 v9.0.0 github.com/go-redis/redismock/v9 v9.2.0 github.com/mitchellh/mapstructure v1.5.0 github.com/nikoksr/notify v0.41.0 + github.com/pnx/antelope-go v0.0.2-0.20240425200605-62ec2abfe722 github.com/redis/go-redis/v9 v9.5.1 github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.8.0 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.18.2 - github.com/stretchr/testify v1.8.4 + github.com/stretchr/testify v1.9.0 gopkg.in/yaml.v3 v3.0.1 ) require ( - github.com/blendle/zapdriver v1.3.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/eosswedenorg-go/jsontime v0.0.0-20230509125027-08422d6236c7 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-telegram-bot-api/telegram-bot-api v4.6.4+incompatible // indirect + github.com/google/go-cmp v0.6.0 // indirect github.com/gorilla/websocket v1.5.1 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/hcl v1.0.0 // indirect + github.com/imroc/req/v3 v3.7.6 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.17.7 // indirect - github.com/logrusorgru/aurora v2.0.3+incompatible // indirect + github.com/liamylian/jsontime/v2 v2.0.0 // indirect github.com/magiconair/properties v1.8.7 // indirect - github.com/mitchellh/go-testing-interface v1.14.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/onsi/gomega v1.31.1 // indirect @@ -49,27 +51,22 @@ require ( github.com/sourcegraph/conc v0.3.0 // indirect github.com/spf13/afero v1.11.0 // indirect github.com/spf13/cast v1.6.0 // indirect - github.com/streamingfast/logging v0.0.0-20230608130331-f22c91403091 // indirect github.com/subosito/gotenv v1.6.0 // indirect github.com/technoweenie/multipartstreamer v1.0.1 // indirect - github.com/tidwall/gjson v1.17.1 // indirect - github.com/tidwall/match v1.1.1 // indirect - github.com/tidwall/pretty v1.2.1 // indirect github.com/ugorji/go/codec v1.2.12 // indirect github.com/vmihailenco/go-tinylfu v0.2.2 // indirect github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect go.uber.org/multierr v1.11.0 // indirect - go.uber.org/zap v1.27.0 // indirect - golang.org/x/crypto v0.20.0 // indirect golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 // indirect - golang.org/x/net v0.21.0 // indirect + golang.org/x/net v0.22.0 // indirect golang.org/x/sync v0.6.0 // indirect - golang.org/x/sys v0.17.0 // indirect - golang.org/x/term v0.17.0 // indirect + golang.org/x/sys v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/ini.v1 v1.67.0 // indirect ) replace github.com/eosswedenorg/thalos/api => ./api + +replace github.com/pnx/antelope-go => ../../go/antelope-go diff --git a/go.sum b/go.sum index 8809c73..f040f95 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,5 @@ github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a h1:HbKu58rmZpUGpz5+4FfNmIU+FmZg2P3Xaj2v2bfNWmk= github.com/alicebob/miniredis/v2 v2.30.2 h1:lc1UAUT9ZA7h4srlfBmBt2aorm5Yftk9nBjxz7EyY9I= -github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= -github.com/blendle/zapdriver v1.3.1 h1:C3dydBOWYRiOk+B8X9IVZ5IOe+7cl+tGOexN4QqHfpE= -github.com/blendle/zapdriver v1.3.1/go.mod h1:mdXfREi6u5MArG4j9fewC+FGnXaBR+T4Ox4J2u4eHCc= github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= @@ -23,10 +20,8 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/r github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/eoscanada/eos-go v0.10.3-0.20231109144819-59afdfa3a37d h1:vK5PijzcJaUPOhgWvY9lL99H9t3lrRNHx2IDHqS0ILc= -github.com/eoscanada/eos-go v0.10.3-0.20231109144819-59afdfa3a37d/go.mod h1:L3avCf8OkDrjlUeNy9DdoV67TCmDNj2dSlc5Xp3DNNk= -github.com/eosswedenorg-go/antelope-ship-client v0.2.8 h1:xqtRrijqVcOgLeh5foXjNoqOXkU/w1l6T4unwLgCrP0= -github.com/eosswedenorg-go/antelope-ship-client v0.2.8/go.mod h1:YcOEgcsZs9a7MjFjRZiX6Qpgc+c0a09PPE4mg8I6IU4= +github.com/eosswedenorg-go/antelope-ship-client v0.2.9 h1:Isa90Tktdumj/P9XgQ8kBRqS+2MAdcPl7ev/n0KFvUQ= +github.com/eosswedenorg-go/antelope-ship-client v0.2.9/go.mod h1:a0Kp4BJID8DMD0pmHnK5/gojtpRhh2pqqgt4Hcy5Wrw= github.com/eosswedenorg-go/jsontime v0.0.0-20230509125027-08422d6236c7 h1:rLPu++RHaxg4WmUOXeWYioZuafWs0PVcYuvzOWbOJjk= github.com/eosswedenorg-go/jsontime v0.0.0-20230509125027-08422d6236c7/go.mod h1:eNUkVOymzgl0lViUhmm08PkutzqLnOQ6Dr+RUnf+Mq0= github.com/eosswedenorg-go/pid v1.0.1 h1:W4AEnnNwb041SpNR1uTZ/KbJ0OTA5eqiqIR1Q5Ah6A0= @@ -60,18 +55,26 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/imroc/req/v3 v3.7.6 h1:SUVWgFt/dJsSzpzpnc8pHdL79zoE6O8FSCfNvbTZXVU= +github.com/imroc/req/v3 v3.7.6/go.mod h1:3JIicOKEDHfCSYYNLb/ObZNpx64EV5y40VlHMwhUCzU= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/jarcoal/httpmock v1.2.0 h1:gSvTxxFR/MEMfsGrvRbdfpRUMBStovlSRLw0Ep1bwwc= github.com/jordan-wright/email v4.0.1-0.20210109023952-943e75fe5223+incompatible h1:jdpOPRN1zP63Td1hDQbZW73xKmzDvZHzVdNYxhnTMDA= +github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= @@ -85,17 +88,17 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8= -github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= +github.com/liamylian/jsontime/v2 v2.0.0 h1:3if2kDW/boymUdO+4Qj/m4uaXMBSF6np9KEgg90cwH0= +github.com/liamylian/jsontime/v2 v2.0.0/go.mod h1:UHp1oAPqCBfspokvGmaGe0IAl2IgOpgOgDaKPcvcGGY= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= -github.com/mitchellh/go-testing-interface v1.14.1 h1:jrgshOhYAUVNMAJiKbEu7EqAwgJJ2JqpQmpLJOu07cU= -github.com/mitchellh/go-testing-interface v1.14.1/go.mod h1:gfgS7OtZj6MA4U1UrDRp04twqAjfvlZyCfX3sDjEym8= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/nikoksr/notify v0.41.0 h1:4LGE41GpWdHX5M3Xo6DlWRwS2WLDbOq1Rk7IzY4vjmQ= @@ -129,7 +132,6 @@ github.com/onsi/gomega v1.31.1 h1:KYppCUK+bUgAZwHOu7EXVBKyQA6ILvOESHkn/tgoqvo= github.com/onsi/gomega v1.31.1/go.mod h1:y40C95dwAD1Nz36SsEnxvfFe8FFfNxzI5eJ0EYGyAy0= github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI= github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -159,8 +161,6 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.18.2 h1:LUXCnvUvSM6FXAsj6nnfc8Q2tp1dIgUfY9Kc8GsSOiQ= github.com/spf13/viper v1.18.2/go.mod h1:EKmWIqdnk5lOcmR72yw6hS+8OPYcwD0jteitLMVB+yk= -github.com/streamingfast/logging v0.0.0-20230608130331-f22c91403091 h1:RN5mrigyirb8anBEtdjtHFIufXdacyTi6i4KBfeNXeo= -github.com/streamingfast/logging v0.0.0-20230608130331-f22c91403091/go.mod h1:VlduQ80JcGJSargkRU4Sg9Xo63wZD/l8A5NC/Uo1/uU= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -171,21 +171,13 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/technoweenie/multipartstreamer v1.0.1 h1:XRztA5MXiR1TIRHxH2uNxXxaIkKQDeX7m2XsSOlQEnM= github.com/technoweenie/multipartstreamer v1.0.1/go.mod h1:jNVxdtShOxzAsukZwTSw6MDx5eUJoiEBsSvzDU9uzog= -github.com/test-go/testify v1.1.4 h1:Tf9lntrKUMHiXQ07qBScBTSA0dhYQlu83hswqelv1iE= -github.com/test-go/testify v1.1.4/go.mod h1:rH7cfJo/47vWGdi4GPj16x3/t1xGOj2YxzmNQzk2ghU= -github.com/tidwall/gjson v1.17.1 h1:wlYEnwqAHgzmhNUFfw7Xalt2JzQvsMx2Se4PcoFCT/U= -github.com/tidwall/gjson v1.17.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= -github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= -github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= -github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= -github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/vmihailenco/go-tinylfu v0.2.2 h1:H1eiG6HM36iniK6+21n9LLpzx1G9R3DJa2UjUjbynsI= @@ -196,50 +188,32 @@ github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21 github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/gopher-lua v1.1.0 h1:BojcDhfyDWgU2f2TOzYK/g5p2gxMrku8oupLDqlnSqE= -go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= -go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= -go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= -go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= -go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= -golang.org/x/crypto v0.20.0 h1:jmAMJJZXr5KiCw05dfYK9QnqaqKLYXijU23lsEdcQqg= -golang.org/x/crypto v0.20.0/go.mod h1:Xwo95rrVNIoSMx9wa1JroENMToLWn3RNVrTBpLHgZPQ= golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 h1:LfspQV/FYTatPTr/3HzIcmiUFH7PGP+OQ6mgDYo3yuQ= golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220111093109-d55c255bac03/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= @@ -247,8 +221,8 @@ golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= -golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= -golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= +golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -268,9 +242,7 @@ golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -283,16 +255,14 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= -golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= -golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U= -golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -303,10 +273,8 @@ golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= @@ -337,11 +305,9 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkep gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU= diff --git a/internal/abi/manager.go b/internal/abi/manager.go index 2441e01..c8bbe83 100644 --- a/internal/abi/manager.go +++ b/internal/abi/manager.go @@ -5,19 +5,20 @@ import ( "fmt" "time" - eos "github.com/eoscanada/eos-go" "github.com/eosswedenorg/thalos/internal/cache" + "github.com/pnx/antelope-go/api" + "github.com/pnx/antelope-go/chain" ) // AbiManager handles an ABI cache that fetches the ABI from an API on cache miss. type AbiManager struct { cache *cache.Cache - api *eos.API + api *api.Client ctx context.Context } // Create a new ABI Manager -func NewAbiManager(cache *cache.Cache, api *eos.API) *AbiManager { +func NewAbiManager(cache *cache.Cache, api *api.Client) *AbiManager { return &AbiManager{ cache: cache, api: api, @@ -26,24 +27,24 @@ func NewAbiManager(cache *cache.Cache, api *eos.API) *AbiManager { } // Set or update an ABI in the cache. -func (mgr *AbiManager) SetAbi(account eos.AccountName, abi *eos.ABI) error { +func (mgr *AbiManager) SetAbi(account chain.Name, abi *chain.Abi) error { ctx, cancel := context.WithTimeout(mgr.ctx, time.Millisecond*500) defer cancel() - return mgr.cache.Set(ctx, string(account), *abi, time.Hour) + return mgr.cache.Set(ctx, account.String(), *abi, time.Hour) } // Get an ABI from the cache, on cache miss it is fetched from the // API, gets cached and then returned to the user -func (mgr *AbiManager) GetAbi(account eos.AccountName) (*eos.ABI, error) { - var abi eos.ABI +func (mgr *AbiManager) GetAbi(account chain.Name) (*chain.Abi, error) { + var abi chain.Abi if err := mgr.cacheGet(account, &abi); err != nil { ctx, cancel := context.WithTimeout(mgr.ctx, time.Second) defer cancel() - resp, err := mgr.api.GetABI(ctx, account) + resp, err := mgr.api.GetAbi(ctx, account.String()) if err != nil { return nil, fmt.Errorf("api: %s", err) } - abi = resp.ABI + abi = resp.Abi err = mgr.SetAbi(account, &abi) if err != nil { @@ -53,8 +54,8 @@ func (mgr *AbiManager) GetAbi(account eos.AccountName) (*eos.ABI, error) { return &abi, nil } -func (mgr *AbiManager) cacheGet(account eos.AccountName, value any) error { +func (mgr *AbiManager) cacheGet(account chain.Name, value any) error { ctx, cancel := context.WithTimeout(mgr.ctx, time.Millisecond*500) defer cancel() - return mgr.cache.Get(ctx, string(account), value) + return mgr.cache.Get(ctx, account.String(), value) } diff --git a/internal/abi/manager_test.go b/internal/abi/manager_test.go index 7ecfc75..edd32f1 100644 --- a/internal/abi/manager_test.go +++ b/internal/abi/manager_test.go @@ -1,13 +1,14 @@ package abi import ( + "encoding/json" "fmt" "net/http" "net/http/httptest" - "strings" "testing" - eos "github.com/eoscanada/eos-go" + "github.com/pnx/antelope-go/api" + "github.com/pnx/antelope-go/chain" "github.com/eosswedenorg/thalos/internal/cache" "github.com/stretchr/testify/assert" @@ -73,7 +74,7 @@ var abiString = ` } ` -func assert_abi(t *testing.T, abi *eos.ABI) { +func assert_abi(t *testing.T, abi *chain.Abi) { assert.Equal(t, abi.Version, "eosio::abi/1.0") // Types @@ -110,12 +111,12 @@ func assert_abi(t *testing.T, abi *eos.ABI) { assert.Equal(t, abi.Structs[3].Fields[0].Type, "string") // Actions - assert.Equal(t, abi.Actions[0].Name, eos.ActN("action_name_1")) + assert.Equal(t, abi.Actions[0].Name, chain.N("action_name_1")) assert.Equal(t, abi.Actions[0].Type, "struct_name_1") assert.Equal(t, abi.Actions[0].RicardianContract, "") // Tables - assert.Equal(t, abi.Tables[0].Name, eos.TableName("table_name_1")) + assert.Equal(t, abi.Tables[0].Name, chain.N("table_name_1")) assert.Equal(t, abi.Tables[0].Type, "struct_name_1") assert.Equal(t, abi.Tables[0].IndexType, "i64") assert.Equal(t, abi.Tables[0].KeyNames[0], "key_name_1") @@ -124,15 +125,10 @@ func assert_abi(t *testing.T, abi *eos.ABI) { assert.Equal(t, abi.Tables[0].KeyTypes[1], "int") } -func mockAPI(handler http.HandlerFunc) (*eos.API, *httptest.Server) { +func mockAPI(handler http.HandlerFunc) (*api.Client, *httptest.Server) { server := httptest.NewServer(handler) - return &eos.API{ - HttpClient: server.Client(), - BaseURL: strings.TrimRight(server.URL, "/"), - Compress: eos.CompressionZlib, - Header: make(http.Header), - }, server + return api.New(server.URL), server } func TestManager_GetAbiFromCache(t *testing.T) { @@ -143,13 +139,14 @@ func TestManager_GetAbiFromCache(t *testing.T) { mgr := NewAbiManager(cache, api) - abi, err := eos.NewABI(strings.NewReader(abiString)) + abi := chain.Abi{} + err := json.Unmarshal([]byte(abiString), &abi) assert.NoError(t, err) - err = mgr.SetAbi("testaccount", abi) + err = mgr.SetAbi(chain.N("testaccount"), &abi) assert.NoError(t, err) - c_abi, err := mgr.GetAbi("testaccount") + c_abi, err := mgr.GetAbi(chain.N("testaccount")) assert.NoError(t, err) assert_abi(t, c_abi) } @@ -166,8 +163,10 @@ func TestManager_GetAbiFromAPI(t *testing.T) { mgr := NewAbiManager(cache, api) - c_abi, err := mgr.GetAbi("testaccount") + c_abi, err := mgr.GetAbi(chain.N("testaccount")) assert.NoError(t, err) + fmt.Println(c_abi) + assert_abi(t, c_abi) } diff --git a/internal/server/helpers.go b/internal/server/helpers.go index f7ce531..5b3a006 100644 --- a/internal/server/helpers.go +++ b/internal/server/helpers.go @@ -1,25 +1,27 @@ package server -import "github.com/eoscanada/eos-go/ship" +import ( + "github.com/pnx/antelope-go/ship" +) // convert a ActionTrace to ActionTraceV1 func toActionTraceV1(trace *ship.ActionTrace) *ship.ActionTraceV1 { - if trace_v0, ok := trace.Impl.(*ship.ActionTraceV0); ok { + if trace.V0 != nil { // convert to v1 return &ship.ActionTraceV1{ - ActionOrdinal: trace_v0.ActionOrdinal, - CreatorActionOrdinal: trace_v0.CreatorActionOrdinal, - Receipt: trace_v0.Receipt, - Receiver: trace_v0.Receiver, - Act: trace_v0.Act, - ContextFree: trace_v0.ContextFree, - Elapsed: trace_v0.Elapsed, - Console: trace_v0.Console, - AccountRamDeltas: trace_v0.AccountRamDeltas, - Except: trace_v0.Except, - ErrorCode: trace_v0.ErrorCode, + ActionOrdinal: trace.V0.ActionOrdinal, + CreatorActionOrdinal: trace.V0.CreatorActionOrdinal, + Receipt: trace.V0.Receipt, + Receiver: trace.V0.Receiver, + Act: trace.V0.Act, + ContextFree: trace.V0.ContextFree, + Elapsed: trace.V0.Elapsed, + Console: trace.V0.Console, + AccountRamDeltas: trace.V0.AccountRamDeltas, + Except: trace.V0.Except, + ErrorCode: trace.V0.ErrorCode, ReturnValue: []byte{}, } } - return trace.Impl.(*ship.ActionTraceV1) + return trace.V1 } diff --git a/internal/server/ship_processor.go b/internal/server/ship_processor.go index aa4017a..7377763 100644 --- a/internal/server/ship_processor.go +++ b/internal/server/ship_processor.go @@ -1,8 +1,8 @@ package server import ( + "bytes" "encoding/hex" - "encoding/json" "github.com/eosswedenorg/thalos/api" "github.com/eosswedenorg/thalos/api/message" @@ -11,9 +11,9 @@ import ( log "github.com/sirupsen/logrus" - "github.com/eoscanada/eos-go" - "github.com/eoscanada/eos-go/ship" shipclient "github.com/eosswedenorg-go/antelope-ship-client" + "github.com/pnx/antelope-go/chain" + "github.com/pnx/antelope-go/ship" ) // logDecoratedEncoder decorates a message.Encoder and logs any error. @@ -51,10 +51,10 @@ type ShipProcessor struct { state State // System contract ("eosio" per default) - syscontract eos.AccountName + syscontract chain.Name // ABI Returned from SHIP - shipABI *eos.ABI + shipABI *chain.Abi } // SpawnProcessor creates a new ShipProccessor that consumes the shipclient.Stream passed to it. @@ -65,7 +65,7 @@ func SpawnProccessor(shipStream *shipclient.Stream, loader StateLoader, saver St writer: writer, shipStream: shipStream, encode: logDecoratedEncoder(codec.Encoder), - syscontract: eos.AccountName("eosio"), + syscontract: chain.N("eosio"), } loader(&processor.state) @@ -75,13 +75,13 @@ func SpawnProccessor(shipStream *shipclient.Stream, loader StateLoader, saver St shipStream.InitHandler = processor.initHandler // Needed because if nil, traces/table deltas will not be included in the response from ship. - shipStream.TraceHandler = func([]*ship.TransactionTraceV0) {} - shipStream.TableDeltaHandler = func([]*ship.TableDeltaV0) {} + shipStream.TraceHandler = func(*ship.TransactionTraceArray) {} + shipStream.TableDeltaHandler = func(*ship.TableDeltaArray) {} return processor } -func (processor *ShipProcessor) initHandler(abi *eos.ABI) { +func (processor *ShipProcessor) initHandler(abi *chain.Abi) { processor.shipABI = abi } @@ -101,26 +101,14 @@ func (processor *ShipProcessor) encodeQueue(channel api.Channel, v interface{}) return false } -func decode(abi *eos.ABI, act *ship.Action, v any) error { - jsondata, err := abi.DecodeAction(act.Data, act.Name) - if err != nil { - return err - } - return json.Unmarshal(jsondata, v) -} - // updateAbiFromAction updates the contract abi based on the ship.Action passed. -func (processor *ShipProcessor) updateAbiFromAction(act *ship.Action) error { - ABI, err := processor.abi.GetAbi(processor.syscontract) - if err != nil { - return err - } - +func (processor *ShipProcessor) updateAbiFromAction(act *chain.Action) error { set_abi := struct { Abi string - Account eos.AccountName + Account chain.Name }{} - if err = decode(ABI, act, &set_abi); err != nil { + + if err := act.DecodeInto(&set_abi); err != nil { return err } @@ -129,11 +117,11 @@ func (processor *ShipProcessor) updateAbiFromAction(act *ship.Action) error { return err } - contract_abi := eos.ABI{} - if err = eos.UnmarshalBinary(binary_abi, &contract_abi); err != nil { + contract_abi := chain.Abi{} + err = chain.NewDecoder(bytes.NewReader(binary_abi)).Decode(&contract_abi) + if err != nil { return err } - return processor.abi.SetAbi(set_abi.Account, &contract_abi) } @@ -161,14 +149,16 @@ func (processor *ShipProcessor) broadcastAction(act *message.ActionTrace) { } } -func (processor *ShipProcessor) processTransactionTrace(log *log.Entry, block *ship.SignedBlockBytes, trace *ship.TransactionTraceV0) { +func (processor *ShipProcessor) processTransactionTrace(log *log.Entry, blockNumber uint32, block *ship.SignedBlock, trace *ship.TransactionTraceV0) { logger := log.WithField("type", "trace").WithField("tx_id", trace.ID.String()).Dup() + timestamp := block.BlockHeader.Timestamp.Time().UTC() + transaction := message.TransactionTrace{ - ID: trace.ID.String(), - BlockNum: block.BlockNumber(), - Timestamp: block.Timestamp.UTC(), - Status: trace.Status.String(), + ID: trace.ID.String(), + BlockNum: blockNumber, + Timestamp: timestamp, + // Status: trace.Status, CPUUsageUS: trace.CPUUsageUS, NetUsage: trace.NetUsage, NetUsageWords: uint32(trace.NetUsageWords), @@ -185,8 +175,8 @@ func (processor *ShipProcessor) processTransactionTrace(log *log.Entry, block *s actMsg := processor.proccessActionTrace(logger, actionTrace) if actMsg != nil { actMsg.TxID = trace.ID.String() - actMsg.BlockNum = block.BlockNumber() - actMsg.Timestamp = block.Timestamp.UTC() + actMsg.BlockNum = blockNumber + actMsg.Timestamp = timestamp processor.broadcastAction(actMsg) @@ -199,8 +189,8 @@ func (processor *ShipProcessor) processTransactionTrace(log *log.Entry, block *s func (processor *ShipProcessor) proccessActionTrace(logger *log.Entry, trace *ship.ActionTraceV1) *message.ActionTrace { // Check if actions updates an abi. - if trace.Act.Account == processor.syscontract && trace.Act.Name == eos.ActionName("setabi") { - err := processor.updateAbiFromAction(trace.Act) + if trace.Act.Account == processor.syscontract && trace.Act.Name == chain.N("setabi") { + err := processor.updateAbiFromAction(&trace.Act) if err != nil { logger.WithError(err).Warn("Failed to update abi") } @@ -214,7 +204,7 @@ func (processor *ShipProcessor) proccessActionTrace(logger *log.Entry, trace *sh } if trace.Receipt != nil { - receipt := trace.Receipt.Impl.(*ship.ActionReceiptV0) + receipt := trace.Receipt.V0 act.Receipt = &message.ActionReceipt{ Receiver: receipt.Receiver.String(), ActDigest: receipt.ActDigest.String(), @@ -241,7 +231,7 @@ func (processor *ShipProcessor) proccessActionTrace(logger *log.Entry, trace *sh ABI, err := processor.abi.GetAbi(trace.Act.Account) if err == nil { - if err = decode(ABI, trace.Act, &act.Data); err != nil { + if act.Data, err = trace.Act.Decode(ABI); err != nil { logger.WithFields(log.Fields{ "contract": trace.Act.Account, "action": trace.Act.Name, @@ -256,89 +246,104 @@ func (processor *ShipProcessor) proccessActionTrace(logger *log.Entry, trace *sh } // Callback function called by shipclient.Stream when a new block arrives. -func (processor *ShipProcessor) processBlock(block *ship.GetBlocksResultV0) { +func (processor *ShipProcessor) processBlock(blockResult *ship.GetBlocksResultV0) { + block := ship.SignedBlock{} + blockResult.Block.Unpack(&block) + timestamp := block.BlockHeader.Timestamp.Time().UTC() + blockNumber := blockResult.ThisBlock.BlockNum + // Check to see if we have a microfork and post a message to // the rollback channel in that case. - if processor.state.CurrentBlock > 0 && block.ThisBlock.BlockNum < processor.state.CurrentBlock { + if processor.state.CurrentBlock > 0 && blockNumber < processor.state.CurrentBlock { log.WithField("old_block", processor.state.CurrentBlock). - WithField("new_block", block.ThisBlock.BlockNum). + WithField("new_block", blockResult.ThisBlock.BlockNum). Warn("Fork detected, old_block is greater than new_block") processor.encodeQueue(api.RollbackChannel, message.RollbackMessage{ OldBlockNum: processor.state.CurrentBlock, - NewBlockNum: block.ThisBlock.BlockNum, + NewBlockNum: blockResult.ThisBlock.BlockNum, }) } - processor.state.CurrentBlock = block.ThisBlock.BlockNum + processor.state.CurrentBlock = blockNumber - if block.ThisBlock.BlockNum%100 == 0 { - log.Infof("Current: %d, Head: %d", processor.state.CurrentBlock, block.Head.BlockNum) + if blockResult.ThisBlock.BlockNum%100 == 0 { + log.Infof("Current: %d, Head: %d", processor.state.CurrentBlock, blockResult.Head.BlockNum) } - if block.ThisBlock.BlockNum%10 == 0 { + if blockResult.ThisBlock.BlockNum%10 == 0 { hb := message.HeartBeat{ - BlockNum: block.ThisBlock.BlockNum, - LastIrreversibleBlockNum: block.LastIrreversible.BlockNum, - HeadBlockNum: block.Head.BlockNum, + BlockNum: blockNumber, + LastIrreversibleBlockNum: blockResult.LastIrreversible.BlockNum, + HeadBlockNum: blockResult.Head.BlockNum, } processor.encodeQueue(api.HeartbeatChannel, hb) } - mainLogger := log.WithField("block", block.ThisBlock.BlockNum).Dup() + mainLogger := log.WithField("block", blockNumber).Dup() // Process traces - if block.Traces != nil && len(block.Traces.Elem) > 0 { - for _, trace := range block.Traces.AsTransactionTracesV0() { - processor.processTransactionTrace(mainLogger, block.Block, trace) + if blockResult.Traces != nil { + unpacked := []ship.TransactionTrace{} + if err := blockResult.Traces.Unpack(&unpacked); err != nil { + mainLogger.WithError(err).Error("Failed to unpack transaction traces") + } else { + for _, trace := range unpacked { + processor.processTransactionTrace(mainLogger, blockNumber, &block, trace.V0) + } } } // Process deltas - for _, delta := range block.Deltas.AsTableDeltasV0() { + deltas := []ship.TableDelta{} + if err := blockResult.Deltas.Unpack(&deltas); err != nil { + mainLogger.WithError(err).Error("Failed to unpack table deltas") + } else { + for _, delta := range deltas { - logger := mainLogger.WithField("type", "table_delta").WithField("table", delta.Name).Dup() + logger := mainLogger.WithField("type", "table_delta").Dup() - rows := []message.TableDeltaRow{} - for _, row := range delta.Rows { + rows := []message.TableDeltaRow{} + for _, row := range delta.V0.Rows { - msg := message.TableDeltaRow{ - Present: row.Present, - RawData: row.Data, - } + msg := message.TableDeltaRow{ + Present: row.Present, + RawData: row.Data, + } - if processor.shipABI != nil { - v, err := processor.shipABI.DecodeTableRowTyped(delta.Name, row.Data) - if err == nil { - err = json.Unmarshal(v, &msg.Data) - if err != nil { - logger.WithError(err).Error("Failed to decode json") + if processor.shipABI != nil { + + v, err := processor.shipABI.Decode(bytes.NewReader(row.Data), delta.V0.Name) + if err == nil { + var ok bool + if msg.Data, ok = v.(map[string]any); !ok { + // logger.Error("Failed to cast table data") + } + } else { + logger.Error("Failed to decode table delta") } } else { - logger.Error("Failed to decode table delta") + logger.Warn("No SHIP ABI present") } - } else { - logger.Warn("No SHIP ABI present") + rows = append(rows, msg) } - rows = append(rows, msg) - } + message := message.TableDelta{ + BlockNum: blockNumber, + Timestamp: timestamp, + Name: delta.V0.Name, + Rows: rows, + } - message := message.TableDelta{ - BlockNum: block.Block.BlockNumber(), - Timestamp: block.Block.Timestamp.UTC(), - Name: delta.Name, - Rows: rows, - } + channels := []api.Channel{ + api.TableDeltaChannel{}.Channel(), + api.TableDeltaChannel{Name: delta.V0.Name}.Channel(), + } - channels := []api.Channel{ - api.TableDeltaChannel{}.Channel(), - api.TableDeltaChannel{Name: delta.Name}.Channel(), - } - - for _, channel := range channels { - processor.encodeQueue(channel, message) + for _, channel := range channels { + processor.encodeQueue(channel, message) + } } } From bcf1b99bd870eb4efe73159aeb19f0e0145bb642 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Mon, 29 Apr 2024 22:44:12 +0200 Subject: [PATCH 173/291] go.mod: remove reference to local antelope-go and use 0.0.3 --- go.mod | 4 +--- go.sum | 2 ++ 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index a3baa8e..5d8e3a2 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/go-redis/redismock/v9 v9.2.0 github.com/mitchellh/mapstructure v1.5.0 github.com/nikoksr/notify v0.41.0 - github.com/pnx/antelope-go v0.0.2-0.20240425200605-62ec2abfe722 + github.com/pnx/antelope-go v0.0.3 github.com/redis/go-redis/v9 v9.5.1 github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.8.0 @@ -68,5 +68,3 @@ require ( ) replace github.com/eosswedenorg/thalos/api => ./api - -replace github.com/pnx/antelope-go => ../../go/antelope-go diff --git a/go.sum b/go.sum index f040f95..16c9574 100644 --- a/go.sum +++ b/go.sum @@ -137,6 +137,8 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pnx/antelope-go v0.0.3 h1:LUTQtCdErPqZqF+15ZinFXvQOHaPYuOHyO24TCaqrN8= +github.com/pnx/antelope-go v0.0.3/go.mod h1:NCCgAR1/Yzh6DigwtFFMcMhG6kGzfNx5sUorUMpBuNc= github.com/redis/go-redis/v9 v9.0.0-rc.4/go.mod h1:Vo3EsyWnicKnSKCA7HhgnvnyA74wOA69Cd2Meli5mmA= github.com/redis/go-redis/v9 v9.5.1 h1:H1X4D3yHPaYrkL5X06Wh6xNVM/pX0Ft4RV0vMGvLBh8= github.com/redis/go-redis/v9 v9.5.1/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= From 1c1f537850fe74b949a694c53e8e864d7d8140c2 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Mon, 29 Apr 2024 22:44:32 +0200 Subject: [PATCH 174/291] Version 1.1.2-rc1 --- Makefile | 2 +- debian/changelog | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 5c0704c..4595606 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ GO=go GOLDFLAGS=-v -s -w -X main.VersionString=$(PROGRAM_VERSION) GOBUILDFLAGS=-v -p $(shell nproc) -ldflags="$(GOLDFLAGS)" PROGRAM=thalos-server -PROGRAM_VERSION=1.1.1 +PROGRAM_VERSION=1.1.2-rc1 PREFIX=/usr/local BINDIR=$(PREFIX)/bin CFGDIR=$(PREFIX)/etc/thalos diff --git a/debian/changelog b/debian/changelog index 8d0d57f..8fa58cb 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +thalos (1.1.2~rc1) bionic focal jammy; urgency=medium + + * Moved from github.com/eoscanda/eos-go to github.com/pnx/antelope-go library + + -- Henrik Hautakoski Mon, 29 Apr 2024 21:14:34 +0200 + thalos (1.1.1) bionic focal jammy; urgency=medium * Build binaries linked with musl libc for alpine linux. From c7f83425bd46f8a8351a1b1c9de7dc96559bda59 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Tue, 7 May 2024 15:37:54 +0200 Subject: [PATCH 175/291] README.md: add telegram link --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 80f64b1..02c3dc3 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,8 @@ Thalos is a application that makes it easy for users to stream blockchain data f Consult the [documentation](https://thalos.waxsweden.org/docs) for more information. +Join the discussion on [telegram](https://t.me/antelopethalos) + ## Compiling You will need golang version `1.20` or later to compile the source. From ea52a3848e719e048327b90fb3db8c8e68e623cc Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Thu, 9 May 2024 11:38:09 +0200 Subject: [PATCH 176/291] makefile: adding docker-image and docker-publish rules --- Makefile | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 4595606..b42c686 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ PREFIX=/usr/local BINDIR=$(PREFIX)/bin CFGDIR=$(PREFIX)/etc/thalos -.PHONY: build build/$(PROGRAM) build/thalos-tools test +.PHONY: build build/$(PROGRAM) build/thalos-tools test docker-image docker-publish build: build/$(PROGRAM) @@ -20,6 +20,12 @@ tools : build/thalos-tools build/thalos-tools : $(GO) build $(GOBUILDFLAGS) -o $@ $(shell find cmd/tools -type f -name *.go) +docker-image: + docker image build --build-arg VERSION=$(PROGRAM_VERSION) -t ghcr.io/eosswedenorg/thalos:$(PROGRAM_VERSION) docker + +docker-publish: + docker publish ghcr.io/eosswedenorg/thalos:$(PROGRAM_VERSION) + install: build tools install -D build/$(PROGRAM) $(DESTDIR)$(BINDIR)/$(PROGRAM) install -D build/thalos-tools $(DESTDIR)$(BINDIR)/thalos-tools From 815fa13e948638a8efced2cb3c5b127db99cd036 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sat, 11 May 2024 14:56:53 +0200 Subject: [PATCH 177/291] go.mod: update pnx/antelope-go --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 5d8e3a2..093490c 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/go-redis/redismock/v9 v9.2.0 github.com/mitchellh/mapstructure v1.5.0 github.com/nikoksr/notify v0.41.0 - github.com/pnx/antelope-go v0.0.3 + github.com/pnx/antelope-go v0.0.4-0.20240511122657-905898ed2110 github.com/redis/go-redis/v9 v9.5.1 github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.8.0 diff --git a/go.sum b/go.sum index 16c9574..d0270c6 100644 --- a/go.sum +++ b/go.sum @@ -139,6 +139,8 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pnx/antelope-go v0.0.3 h1:LUTQtCdErPqZqF+15ZinFXvQOHaPYuOHyO24TCaqrN8= github.com/pnx/antelope-go v0.0.3/go.mod h1:NCCgAR1/Yzh6DigwtFFMcMhG6kGzfNx5sUorUMpBuNc= +github.com/pnx/antelope-go v0.0.4-0.20240511122657-905898ed2110 h1:ipRjWXy6HNkd3QdAdr2RJEHUxUETc9UbQB3dyU70Aq0= +github.com/pnx/antelope-go v0.0.4-0.20240511122657-905898ed2110/go.mod h1:NCCgAR1/Yzh6DigwtFFMcMhG6kGzfNx5sUorUMpBuNc= github.com/redis/go-redis/v9 v9.0.0-rc.4/go.mod h1:Vo3EsyWnicKnSKCA7HhgnvnyA74wOA69Cd2Meli5mmA= github.com/redis/go-redis/v9 v9.5.1 h1:H1X4D3yHPaYrkL5X06Wh6xNVM/pX0Ft4RV0vMGvLBh8= github.com/redis/go-redis/v9 v9.5.1/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= From 871f2ae04dddefadbb9b508c0ebc01ad9bfc6184 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sat, 11 May 2024 14:57:46 +0200 Subject: [PATCH 178/291] internal/server/ship_processor.go: set TransactionTrace.Status again. --- internal/server/ship_processor.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/server/ship_processor.go b/internal/server/ship_processor.go index 7377763..f348fa4 100644 --- a/internal/server/ship_processor.go +++ b/internal/server/ship_processor.go @@ -155,10 +155,10 @@ func (processor *ShipProcessor) processTransactionTrace(log *log.Entry, blockNum timestamp := block.BlockHeader.Timestamp.Time().UTC() transaction := message.TransactionTrace{ - ID: trace.ID.String(), - BlockNum: blockNumber, - Timestamp: timestamp, - // Status: trace.Status, + ID: trace.ID.String(), + BlockNum: blockNumber, + Timestamp: timestamp, + Status: trace.Status.String(), CPUUsageUS: trace.CPUUsageUS, NetUsage: trace.NetUsage, NetUsageWords: uint32(trace.NetUsageWords), From c876875a6ed888f5ba0be53f1c231b174e8dd0e7 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 12 May 2024 17:14:00 +0200 Subject: [PATCH 179/291] internal/server/message_queue.go: adding MessageQueue struct This struct will handle the routing to channels aswell as encoding of the messages. --- internal/server/message_queue.go | 74 ++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 internal/server/message_queue.go diff --git a/internal/server/message_queue.go b/internal/server/message_queue.go new file mode 100644 index 0000000..50386ec --- /dev/null +++ b/internal/server/message_queue.go @@ -0,0 +1,74 @@ +package server + +import ( + "errors" + + "github.com/eosswedenorg/thalos/api" + "github.com/eosswedenorg/thalos/api/message" + "github.com/eosswedenorg/thalos/internal/driver" +) + +// MessageQueue takes care of message routing and encoding +type MessageQueue struct { + // Writer to write messages to + writer driver.Writer + + // encoder to encode messages with + encode message.Encoder +} + +func NewMessageQueue(writer driver.Writer, encoder message.Encoder) MessageQueue { + return MessageQueue{ + writer: writer, + encode: encoder, + } +} + +func (mq MessageQueue) PostHeartbeat(hb message.HeartBeat) error { + return mq.post(hb, api.HeartbeatChannel) +} + +func (mq MessageQueue) PostRollback(rb message.RollbackMessage) error { + return mq.post(rb, api.RollbackChannel) +} + +func (mq MessageQueue) PostTransactionTrace(trace message.TransactionTrace) error { + return mq.post(trace, api.TransactionChannel) +} + +// Post a ActionTrace message to the queue +func (mq MessageQueue) PostAction(act message.ActionTrace) error { + return mq.post(act, + api.ActionChannel{}.Channel(), + api.ActionChannel{Name: act.Name}.Channel(), + api.ActionChannel{Contract: act.Contract}.Channel(), + api.ActionChannel{Name: act.Name, Contract: act.Contract}.Channel(), + ) +} + +func (mq MessageQueue) PostTableDelta(delta message.TableDelta) error { + return mq.post(delta, + api.TableDeltaChannel{}.Channel(), + api.TableDeltaChannel{Name: delta.Name}.Channel(), + ) +} + +func (mq MessageQueue) Flush() error { + return mq.writer.Flush() +} + +func (mq MessageQueue) Close() error { + return mq.writer.Close() +} + +func (mq MessageQueue) post(v interface{}, channels ...api.Channel) error { + payload, err := mq.encode(v) + if err == nil { + for _, channel := range channels { + if w_err := mq.writer.Write(channel, payload); err != nil { + err = errors.Join(w_err) + } + } + } + return err +} From 95365d0c267926040523e0cc3a439cd4e1318ba4 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 12 May 2024 17:15:03 +0200 Subject: [PATCH 180/291] internal/server/ship_processor.go: refactor to use MessageQueue struct. --- internal/server/ship_processor.go | 98 ++++++++----------------------- 1 file changed, 23 insertions(+), 75 deletions(-) diff --git a/internal/server/ship_processor.go b/internal/server/ship_processor.go index f348fa4..8724d22 100644 --- a/internal/server/ship_processor.go +++ b/internal/server/ship_processor.go @@ -4,7 +4,6 @@ import ( "bytes" "encoding/hex" - "github.com/eosswedenorg/thalos/api" "github.com/eosswedenorg/thalos/api/message" "github.com/eosswedenorg/thalos/internal/abi" "github.com/eosswedenorg/thalos/internal/driver" @@ -16,19 +15,6 @@ import ( "github.com/pnx/antelope-go/ship" ) -// logDecoratedEncoder decorates a message.Encoder and logs any error. -func logDecoratedEncoder(encoder message.Encoder) message.Encoder { - return func(v interface{}) ([]byte, error) { - payload, err := encoder(v) - if err != nil { - log.WithError(err). - WithField("v", v). - Warn("Failed to encode message") - } - return payload, err - } -} - // A ShipProcessor will consume messages from a ship stream, convert the messages into // thalos specific ones, encode them and finally post them to an api.Writer type ShipProcessor struct { @@ -38,11 +24,7 @@ type ShipProcessor struct { // Abi manager used for cacheing abi *abi.AbiManager - // Writer to send messages to. - writer driver.Writer - - // Encoder used to encode messages - encode message.Encoder + queue MessageQueue // Function for saving state. saver StateSaver @@ -62,10 +44,9 @@ func SpawnProccessor(shipStream *shipclient.Stream, loader StateLoader, saver St processor := &ShipProcessor{ saver: saver, abi: abi, - writer: writer, shipStream: shipStream, - encode: logDecoratedEncoder(codec.Encoder), syscontract: chain.N("eosio"), + queue: NewMessageQueue(writer, codec.Encoder), } loader(&processor.state) @@ -85,22 +66,6 @@ func (processor *ShipProcessor) initHandler(abi *chain.Abi) { processor.shipABI = abi } -func (processor *ShipProcessor) queueMessage(channel api.Channel, payload []byte) bool { - err := processor.writer.Write(channel, payload) - if err != nil { - log.WithError(err).Errorf("Failed to post to channel '%s'", channel) - return false - } - return true -} - -func (processor *ShipProcessor) encodeQueue(channel api.Channel, v interface{}) bool { - if payload, err := processor.encode(v); err == nil { - return processor.queueMessage(channel, payload) - } - return false -} - // updateAbiFromAction updates the contract abi based on the ship.Action passed. func (processor *ShipProcessor) updateAbiFromAction(act *chain.Action) error { set_abi := struct { @@ -130,25 +95,6 @@ func (processor *ShipProcessor) GetCurrentBlock() uint32 { return processor.state.CurrentBlock } -func (processor *ShipProcessor) broadcastAction(act *message.ActionTrace) { - payload, err := processor.encode(*act) - if err != nil { - log.WithField("act", act).Warn("failed to encode action") - return - } - - channels := []api.Channel{ - api.ActionChannel{}.Channel(), - api.ActionChannel{Name: act.Name}.Channel(), - api.ActionChannel{Contract: act.Contract}.Channel(), - api.ActionChannel{Name: act.Name, Contract: act.Contract}.Channel(), - } - - for _, channel := range channels { - processor.queueMessage(channel, payload) - } -} - func (processor *ShipProcessor) processTransactionTrace(log *log.Entry, blockNumber uint32, block *ship.SignedBlock, trace *ship.TransactionTraceV0) { logger := log.WithField("type", "trace").WithField("tx_id", trace.ID.String()).Dup() @@ -178,13 +124,15 @@ func (processor *ShipProcessor) processTransactionTrace(log *log.Entry, blockNum actMsg.BlockNum = blockNumber actMsg.Timestamp = timestamp - processor.broadcastAction(actMsg) + processor.queue.PostAction(*actMsg) transaction.ActionTraces = append(transaction.ActionTraces, *actMsg) } } - processor.encodeQueue(api.TransactionChannel, transaction) + if err := processor.queue.PostTransactionTrace(transaction); err != nil { + logger.WithError(err).Error("Failed to post transaction trace") + } } func (processor *ShipProcessor) proccessActionTrace(logger *log.Entry, trace *ship.ActionTraceV1) *message.ActionTrace { @@ -255,14 +203,18 @@ func (processor *ShipProcessor) processBlock(blockResult *ship.GetBlocksResultV0 // Check to see if we have a microfork and post a message to // the rollback channel in that case. if processor.state.CurrentBlock > 0 && blockNumber < processor.state.CurrentBlock { - log.WithField("old_block", processor.state.CurrentBlock). - WithField("new_block", blockResult.ThisBlock.BlockNum). - Warn("Fork detected, old_block is greater than new_block") - processor.encodeQueue(api.RollbackChannel, message.RollbackMessage{ + msg := message.RollbackMessage{ OldBlockNum: processor.state.CurrentBlock, NewBlockNum: blockResult.ThisBlock.BlockNum, - }) + } + log.WithField("old_block", msg.OldBlockNum). + WithField("new_block", msg.NewBlockNum). + Warn("Fork detected, old_block is greater than new_block") + + if err := processor.queue.PostRollback(msg); err != nil { + log.WithError(err).Error("Failed to write rollback message") + } } processor.state.CurrentBlock = blockNumber @@ -277,8 +229,9 @@ func (processor *ShipProcessor) processBlock(blockResult *ship.GetBlocksResultV0 LastIrreversibleBlockNum: blockResult.LastIrreversible.BlockNum, HeadBlockNum: blockResult.Head.BlockNum, } - - processor.encodeQueue(api.HeartbeatChannel, hb) + if err := processor.queue.PostHeartbeat(hb); err != nil { + log.WithError(err).Error("Failed to write heartbeat message") + } } mainLogger := log.WithField("block", blockNumber).Dup() @@ -329,25 +282,20 @@ func (processor *ShipProcessor) processBlock(blockResult *ship.GetBlocksResultV0 rows = append(rows, msg) } - message := message.TableDelta{ + msg := message.TableDelta{ BlockNum: blockNumber, Timestamp: timestamp, Name: delta.V0.Name, Rows: rows, } - channels := []api.Channel{ - api.TableDeltaChannel{}.Channel(), - api.TableDeltaChannel{Name: delta.V0.Name}.Channel(), - } - - for _, channel := range channels { - processor.encodeQueue(channel, message) + if err := processor.queue.PostTableDelta(msg); err != nil { + logger.WithError(err).Error("Failed to post table delta message") } } } - err := processor.writer.Flush() + err := processor.queue.Flush() if err != nil { log.WithError(err).Error("Failed to send messages") } @@ -360,5 +308,5 @@ func (processor *ShipProcessor) processBlock(blockResult *ship.GetBlocksResultV0 // Close closes the writer associated with the processor. func (processor *ShipProcessor) Close() error { - return processor.writer.Close() + return processor.queue.Close() } From 59bcc9f19f6093f988cc6691d5a752174b1d2982 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 12 May 2024 19:12:27 +0200 Subject: [PATCH 181/291] internal/server/helpers.go: adding parseTableDeltaData() --- internal/server/helpers.go | 123 ++++++++++++++++++++++++++++++++ internal/server/helpers_test.go | 69 ++++++++++++++++++ 2 files changed, 192 insertions(+) create mode 100644 internal/server/helpers_test.go diff --git a/internal/server/helpers.go b/internal/server/helpers.go index 5b3a006..13fb386 100644 --- a/internal/server/helpers.go +++ b/internal/server/helpers.go @@ -1,6 +1,9 @@ package server import ( + "fmt" + "reflect" + "github.com/pnx/antelope-go/ship" ) @@ -25,3 +28,123 @@ func toActionTraceV1(trace *ship.ActionTrace) *ship.ActionTraceV1 { } return trace.V1 } + +func isVariantName(name string) bool { + validVariants := []string{ + "get_status_request_v0", + "block_position", + "get_status_result_v0", + "get_blocks_request_v0", + "get_blocks_ack_request_v0", + "get_blocks_result_v0", + "row", + "table_delta_v0", + "action", + "account_auth_sequence", + "action_receipt_v0", + "account_delta", + "action_trace_v0", + "partial_transaction_v0", + "transaction_trace_v0", + "packed_transaction", + "transaction_receipt_header", + "transaction_receipt", + "extension", + "block_header", + "signed_block_header", + "signed_block", + "transaction_header", + "transaction", + "code_id", + "account_v0", + "account_metadata_v0", + "code_v0", + "contract_table_v0", + "contract_row_v0", + "contract_index64_v0", + "contract_index128_v0", + "contract_index256_v0", + "contract_index_double_v0", + "contract_index_long_double_v0", + "producer_key", + "producer_schedule", + "block_signing_authority_v0", + "producer_authority", + "producer_authority_schedule", + "chain_config_v0", + "global_property_v0", + "global_property_v1", + "generated_transaction_v0", + "activated_protocol_feature_v0", + "protocol_state_v0", + "key_weight", + "permission_level", + "permission_level_weight", + "wait_weight", + "authority", + "permission_v0", + "permission_link_v0", + "resource_limits_v0", + "usage_accumulator_v0", + "resource_usage_v0", + "resource_limits_state_v0", + "resource_limits_ratio_v0", + "elastic_limit_parameters_v0", + "resource_limits_config_v0", + } + + for _, v := range validVariants { + if v == name { + return true + } + } + return false +} + +// Check if a structure is a variant type. +// This is not 100% accurate. As variant types comes +// as a simple slice with the types name in the first index +// and the value as the second. +// So there could be some edge cases where this structure is actual data +// and not a variant type although should be super rare. +func isVariant(v reflect.Value) bool { + if v.Kind() != reflect.Slice || v.Len() != 2 { + return false + } + + if !isVariantName(v.Index(0).Elem().String()) { + return false + } + + return true +} + +func parseTableDeltaData(v any) (map[string]interface{}, error) { + iface := parseTableDeltaDataInner(reflect.ValueOf(v)).Interface() + if out, ok := iface.(map[string]interface{}); ok { + return out, nil + } + return nil, fmt.Errorf("data is not an map") +} + +func parseTableDeltaDataInner(v reflect.Value) reflect.Value { + if isVariant(v) { + v = v.Index(1) + } + + switch v.Kind() { + case reflect.Interface: + return parseTableDeltaDataInner(v.Elem()) + case reflect.Slice: + for i := 0; i < v.Len(); i++ { + v.Index(i).Set(parseTableDeltaDataInner(v.Index(i))) + } + case reflect.Map: + it := v.MapRange() + for it.Next() { + v.SetMapIndex(it.Key(), parseTableDeltaDataInner(it.Value())) + } + } + + return v +} diff --git a/internal/server/helpers_test.go b/internal/server/helpers_test.go new file mode 100644 index 0000000..5765146 --- /dev/null +++ b/internal/server/helpers_test.go @@ -0,0 +1,69 @@ +package server + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestParseTableDeltaData(t *testing.T) { + input := []interface{}{ + "resource_limits_state_v0", + map[string]interface{}{ + "average_block_cpu_usage": []interface{}{ + "usage_accumulator_v0", + map[string]interface{}{ + "consumed": 33679, + "last_ordinal": 308855607, + "value_ex": "18525321667", + }, + }, + "average_block_net_usage": []interface{}{ + "usage_accumulator_v0", + map[string]interface{}{ + "consumed": 8107, + "last_ordinal": 308855607, + "value_ex": 3854030492, + }, + }, + "slice": []interface{}{ + "generated_transaction_v0", + []interface{}{1, 2, "tree"}, + }, + "single_value": []interface{}{ + "generated_transaction_v0", + uint32(12933729), + }, + "total_cpu_weight": "44811223778385154", + "total_net_weight": "134285012330070718", + "total_ram_bytes": "172065109473", + "virtual_cpu_limit": 206081, + "virtual_net_limit": 1048576000, + }, + } + + expected := map[string]interface{}{ + "average_block_cpu_usage": map[string]interface{}{ + "consumed": 33679, + "last_ordinal": 308855607, + "value_ex": "18525321667", + }, + "average_block_net_usage": map[string]interface{}{ + "consumed": 8107, + "last_ordinal": 308855607, + "value_ex": 3854030492, + }, + "slice": []interface{}{1, 2, "tree"}, + "single_value": uint32(12933729), + "total_cpu_weight": "44811223778385154", + "total_net_weight": "134285012330070718", + "total_ram_bytes": "172065109473", + "virtual_cpu_limit": 206081, + "virtual_net_limit": 1048576000, + } + + actual, err := parseTableDeltaData(input) + + assert.NoError(t, err) + assert.Equal(t, expected, actual) +} From c02cff0e054ceb1269bcdb08a5c20005bbfc1da2 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Fri, 17 May 2024 16:54:57 +0200 Subject: [PATCH 182/291] internal/server/ship_processor.go: fix a bug where TableDeltaRow.Data was not set --- internal/server/ship_processor.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/internal/server/ship_processor.go b/internal/server/ship_processor.go index 8724d22..51b90e0 100644 --- a/internal/server/ship_processor.go +++ b/internal/server/ship_processor.go @@ -269,9 +269,11 @@ func (processor *ShipProcessor) processBlock(blockResult *ship.GetBlocksResultV0 v, err := processor.shipABI.Decode(bytes.NewReader(row.Data), delta.V0.Name) if err == nil { - var ok bool - if msg.Data, ok = v.(map[string]any); !ok { - // logger.Error("Failed to cast table data") + v, err := parseTableDeltaData(v) + if err == nil { + msg.Data = v + } else { + logger.WithError(err).Error("Failed to parse table delta data") } } else { logger.Error("Failed to decode table delta") From cc754ee976ef67de053e5ad64f43fa2ac7db528a Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Fri, 17 May 2024 16:57:13 +0200 Subject: [PATCH 183/291] internal/server/ship_processor.go: fix a bug where blockResult.Deltas whas not properly nil checked. Resulting in panic if accessed --- internal/server/ship_processor.go | 64 ++++++++++++++++--------------- 1 file changed, 33 insertions(+), 31 deletions(-) diff --git a/internal/server/ship_processor.go b/internal/server/ship_processor.go index 51b90e0..649828d 100644 --- a/internal/server/ship_processor.go +++ b/internal/server/ship_processor.go @@ -249,50 +249,52 @@ func (processor *ShipProcessor) processBlock(blockResult *ship.GetBlocksResultV0 } // Process deltas - deltas := []ship.TableDelta{} - if err := blockResult.Deltas.Unpack(&deltas); err != nil { - mainLogger.WithError(err).Error("Failed to unpack table deltas") - } else { - for _, delta := range deltas { + if blockResult.Deltas != nil { + deltas := []ship.TableDelta{} + if err := blockResult.Deltas.Unpack(&deltas); err != nil { + mainLogger.WithError(err).Error("Failed to unpack table deltas") + } else { + for _, delta := range deltas { - logger := mainLogger.WithField("type", "table_delta").Dup() + logger := mainLogger.WithField("type", "table_delta").Dup() - rows := []message.TableDeltaRow{} - for _, row := range delta.V0.Rows { + rows := []message.TableDeltaRow{} + for _, row := range delta.V0.Rows { - msg := message.TableDeltaRow{ - Present: row.Present, - RawData: row.Data, - } + msg := message.TableDeltaRow{ + Present: row.Present, + RawData: row.Data, + } - if processor.shipABI != nil { + if processor.shipABI != nil { - v, err := processor.shipABI.Decode(bytes.NewReader(row.Data), delta.V0.Name) - if err == nil { - v, err := parseTableDeltaData(v) + v, err := processor.shipABI.Decode(bytes.NewReader(row.Data), delta.V0.Name) if err == nil { - msg.Data = v + v, err := parseTableDeltaData(v) + if err == nil { + msg.Data = v + } else { + logger.WithError(err).Error("Failed to parse table delta data") + } } else { - logger.WithError(err).Error("Failed to parse table delta data") + logger.Error("Failed to decode table delta") } } else { - logger.Error("Failed to decode table delta") + logger.Warn("No SHIP ABI present") } - } else { - logger.Warn("No SHIP ABI present") + rows = append(rows, msg) } - rows = append(rows, msg) - } - msg := message.TableDelta{ - BlockNum: blockNumber, - Timestamp: timestamp, - Name: delta.V0.Name, - Rows: rows, - } + msg := message.TableDelta{ + BlockNum: blockNumber, + Timestamp: timestamp, + Name: delta.V0.Name, + Rows: rows, + } - if err := processor.queue.PostTableDelta(msg); err != nil { - logger.WithError(err).Error("Failed to post table delta message") + if err := processor.queue.PostTableDelta(msg); err != nil { + logger.WithError(err).Error("Failed to post table delta message") + } } } } From 8bd3736684bd891bd61f104cf990c3865c738d96 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Fri, 17 May 2024 17:08:29 +0200 Subject: [PATCH 184/291] internal/server/ship_processor.go: refactor out table delta row proccessing to its own function --- internal/server/ship_processor.go | 62 ++++++++++++++++--------------- 1 file changed, 32 insertions(+), 30 deletions(-) diff --git a/internal/server/ship_processor.go b/internal/server/ship_processor.go index 649828d..6b05262 100644 --- a/internal/server/ship_processor.go +++ b/internal/server/ship_processor.go @@ -193,6 +193,36 @@ func (processor *ShipProcessor) proccessActionTrace(logger *log.Entry, trace *sh return act } +func (processor *ShipProcessor) proccessDeltaRows(logger *log.Entry, table_name string, rows []ship.Row) []message.TableDeltaRow { + out := []message.TableDeltaRow{} + for _, row := range rows { + + msg := message.TableDeltaRow{ + Present: row.Present, + RawData: row.Data, + } + + if processor.shipABI != nil { + + v, err := processor.shipABI.Decode(bytes.NewReader(row.Data), table_name) + if err == nil { + v, err := parseTableDeltaData(v) + if err == nil { + msg.Data = v + } else { + logger.WithError(err).Error("Failed to parse table delta data") + } + } else { + logger.Error("Failed to decode table delta") + } + } else { + logger.Warn("No SHIP ABI present") + } + out = append(out, msg) + } + return out +} + // Callback function called by shipclient.Stream when a new block arrives. func (processor *ShipProcessor) processBlock(blockResult *ship.GetBlocksResultV0) { block := ship.SignedBlock{} @@ -254,42 +284,14 @@ func (processor *ShipProcessor) processBlock(blockResult *ship.GetBlocksResultV0 if err := blockResult.Deltas.Unpack(&deltas); err != nil { mainLogger.WithError(err).Error("Failed to unpack table deltas") } else { + logger := mainLogger.WithField("type", "table_delta").Dup() for _, delta := range deltas { - logger := mainLogger.WithField("type", "table_delta").Dup() - - rows := []message.TableDeltaRow{} - for _, row := range delta.V0.Rows { - - msg := message.TableDeltaRow{ - Present: row.Present, - RawData: row.Data, - } - - if processor.shipABI != nil { - - v, err := processor.shipABI.Decode(bytes.NewReader(row.Data), delta.V0.Name) - if err == nil { - v, err := parseTableDeltaData(v) - if err == nil { - msg.Data = v - } else { - logger.WithError(err).Error("Failed to parse table delta data") - } - } else { - logger.Error("Failed to decode table delta") - } - } else { - logger.Warn("No SHIP ABI present") - } - rows = append(rows, msg) - } - msg := message.TableDelta{ BlockNum: blockNumber, Timestamp: timestamp, Name: delta.V0.Name, - Rows: rows, + Rows: processor.proccessDeltaRows(logger, delta.V0.Name, delta.V0.Rows), } if err := processor.queue.PostTableDelta(msg); err != nil { From c414fad347fd6fc1188b6a444b2da9c805837197 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Fri, 17 May 2024 17:11:57 +0200 Subject: [PATCH 185/291] README.md: fix badges --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 80f64b1..d85f837 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,6 @@ # Thalos [![Test](https://github.com/eosswedenorg/thalos/actions/workflows/test.yml/badge.svg)](https://github.com/eosswedenorg/thalos/actions/workflows/test.yml) - [![Go Report Card](https://goreportcard.com/badge/github.com/eosswedenorg/thalos)](https://goreportcard.com/report/github.com/eosswedenorg/thalos) Thalos is a application that makes it easy for users to stream blockchain data from an Antelope SHIP node. From 3392533a91dfca184d61b20fa42842f9a623c99d Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Fri, 17 May 2024 18:20:14 +0200 Subject: [PATCH 186/291] Version 1.1.2-rc2 --- Makefile | 2 +- debian/changelog | 7 +++++++ docker/Dockerfile | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index b42c686..956ea03 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ GO=go GOLDFLAGS=-v -s -w -X main.VersionString=$(PROGRAM_VERSION) GOBUILDFLAGS=-v -p $(shell nproc) -ldflags="$(GOLDFLAGS)" PROGRAM=thalos-server -PROGRAM_VERSION=1.1.2-rc1 +PROGRAM_VERSION=1.1.2-rc2 PREFIX=/usr/local BINDIR=$(PREFIX)/bin CFGDIR=$(PREFIX)/etc/thalos diff --git a/debian/changelog b/debian/changelog index 8fa58cb..fe40add 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,10 @@ +thalos (1.1.2~rc2) bionic focal jammy; urgency=medium + + * fix a bug where TableDeltaRow.Data was not set + * fix a bug where blockResult.Deltas was not properly nil checked. Resulting in panic if accessed + + -- Henrik Hautakoski Fri, 17 May 2024 18:15:29 +0200 + thalos (1.1.2~rc1) bionic focal jammy; urgency=medium * Moved from github.com/eoscanda/eos-go to github.com/pnx/antelope-go library diff --git a/docker/Dockerfile b/docker/Dockerfile index 70c5fe5..d00fcb9 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,6 +1,6 @@ FROM alpine:latest LABEL maintainer="Henrik Hautakoski " -ARG VERSION=1.1.1 +ARG VERSION=1.1.2-rc2 WORKDIR /thalos ADD --chmod=755 https://github.com/eosswedenorg/thalos/releases/download/v$VERSION/thalos-server-${VERSION}-linux-amd64-musl thalos-server ENTRYPOINT [ "./thalos-server" ] From 634205a5464afb8f4e26d64ad0d6fe09a73baee8 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Fri, 17 May 2024 18:35:40 +0200 Subject: [PATCH 187/291] Makefile: should be "docker image push" instead of "docker publish" --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 956ea03..438f932 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,7 @@ docker-image: docker image build --build-arg VERSION=$(PROGRAM_VERSION) -t ghcr.io/eosswedenorg/thalos:$(PROGRAM_VERSION) docker docker-publish: - docker publish ghcr.io/eosswedenorg/thalos:$(PROGRAM_VERSION) + docker image push ghcr.io/eosswedenorg/thalos:$(PROGRAM_VERSION) install: build tools install -D build/$(PROGRAM) $(DESTDIR)$(BINDIR)/$(PROGRAM) From ea5b2b8fc2713d135f3a0de91647a6c2fea526de Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Tue, 28 May 2024 13:30:21 +0200 Subject: [PATCH 188/291] internal/server/helpers.go fix a bug in isVariant() where v.Elem() was called on non interface/pointer --- internal/server/helpers.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/internal/server/helpers.go b/internal/server/helpers.go index 13fb386..e35bb92 100644 --- a/internal/server/helpers.go +++ b/internal/server/helpers.go @@ -112,11 +112,10 @@ func isVariant(v reflect.Value) bool { return false } - if !isVariantName(v.Index(0).Elem().String()) { - return false + for v = v.Index(0); v.Kind() == reflect.Interface || v.Kind() == reflect.Pointer; v = v.Elem() { } - return true + return v.Kind() == reflect.String && isVariantName(v.String()) } func parseTableDeltaData(v any) (map[string]interface{}, error) { From 8b8867394f7bb4fac60a4ec0c820d1f5182c09fd Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Wed, 19 Jun 2024 21:45:45 +0200 Subject: [PATCH 189/291] tools: cleanup and refactor to make code more readable. --- cmd/tools/bench.go | 171 ++++++++++++++++++----------------- cmd/tools/flags.go | 14 +++ cmd/tools/main.go | 45 ++-------- cmd/tools/mock_publisher.go | 172 +++++++++++++++++++----------------- cmd/tools/redis-acl.go | 103 +++++++++++---------- cmd/tools/validate.go | 164 +++++++++++++++++----------------- 6 files changed, 341 insertions(+), 328 deletions(-) create mode 100644 cmd/tools/flags.go diff --git a/cmd/tools/bench.go b/cmd/tools/bench.go index cb96c8d..067ee90 100644 --- a/cmd/tools/bench.go +++ b/cmd/tools/bench.go @@ -18,101 +18,108 @@ import ( log "github.com/sirupsen/logrus" ) -var benchCmd = &cobra.Command{ - Use: "bench", - Short: "Run a benchmark against a thalos node", - Run: func(cmd *cobra.Command, args []string) { - counter := 0 - interval, _ := cmd.Flags().GetDuration("interval") +func CreateBenchCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "bench", + Short: "Run a benchmark against a thalos node", + Run: func(cmd *cobra.Command, args []string) { + counter := 0 + interval, _ := cmd.Flags().GetDuration("interval") - url, _ := cmd.Flags().GetString("redis-url") - user, _ := cmd.Flags().GetString("redis-user") - pw, _ := cmd.Flags().GetString("redis-pw") - prefix, _ := cmd.Flags().GetString("prefix") - chain_id, _ := cmd.Flags().GetString("chain_id") - db, _ := cmd.Flags().GetInt("redis-db") + url, _ := cmd.Flags().GetString("redis-url") + user, _ := cmd.Flags().GetString("redis-user") + pw, _ := cmd.Flags().GetString("redis-pw") + prefix, _ := cmd.Flags().GetString("prefix") + chain_id, _ := cmd.Flags().GetString("chain_id") + db, _ := cmd.Flags().GetInt("redis-db") - log.WithFields(log.Fields{ - "url": url, - "prefix": prefix, - "chain_id": chain_id, - "database": db, - }).Info("Connecting to redis") + log.WithFields(log.Fields{ + "url": url, + "prefix": prefix, + "chain_id": chain_id, + "database": db, + }).Info("Connecting to redis") - // Create redis client - rdb := redis.NewClient(&redis.Options{ - Addr: url, - Username: user, - Password: pw, - DB: db, - }) + // Create redis client + rdb := redis.NewClient(&redis.Options{ + Addr: url, + Username: user, + Password: pw, + DB: db, + }) - if err := rdb.Ping(context.Background()).Err(); err != nil { - log.WithError(err).Fatal("Failed to connect to redis") - return - } + if err := rdb.Ping(context.Background()).Err(); err != nil { + log.WithError(err).Fatal("Failed to connect to redis") + return + } - log.Println("Connected to redis") + log.Println("Connected to redis") - log.WithFields(log.Fields{ - "interval": interval, - }).Info("Starting benchmark") + log.WithFields(log.Fields{ + "interval": interval, + }).Info("Starting benchmark") - sub := api_redis.NewSubscriber(context.Background(), rdb, api_redis.Namespace{ - Prefix: prefix, - ChainID: chain_id, - }) + sub := api_redis.NewSubscriber(context.Background(), rdb, api_redis.Namespace{ + Prefix: prefix, + ChainID: chain_id, + }) - codec, err := message.GetCodec("json") - if err != nil { - log.WithError(err).Fatal("Failed to get codec") - return - } + codec, err := message.GetCodec("json") + if err != nil { + log.WithError(err).Fatal("Failed to get codec") + return + } - client := api.NewClient(sub, codec.Decoder) + client := api.NewClient(sub, codec.Decoder) - // Subscribe to all actions - if err = client.Subscribe(api.ActionChannel{}.Channel()); err != nil { - log.WithError(err).Fatal("Failed to subscribe to channels") - return - } + // Subscribe to all actions + if err = client.Subscribe(api.ActionChannel{}.Channel()); err != nil { + log.WithError(err).Fatal("Failed to subscribe to channels") + return + } - go func() { - for t := range client.Channel() { - switch err := t.(type) { - case message.ActionTrace: - counter++ - case error: - log.WithError(err).Error("Error when reading stream") + go func() { + for t := range client.Channel() { + switch err := t.(type) { + case message.ActionTrace: + counter++ + case error: + log.WithError(err).Error("Error when reading stream") + } + } + }() + + t := time.Now() + sig := make(chan os.Signal, 1) + signal.Notify(sig, os.Interrupt) + + // Read stuff. + for { + select { + case <-sig: + fmt.Println("Got interrupt") + client.Close() + return + case now := <-time.After(interval): + elapsed := now.Sub(t) + t = now + + log.WithFields(log.Fields{ + "num_messages": counter, + "elapsed": elapsed, + "msg_per_sec": float64(counter) / elapsed.Seconds(), + "msg_per_ms": float64(counter) / float64(elapsed.Milliseconds()), + "msg_per_min": float64(counter) / elapsed.Minutes(), + }).Info("Benchmark results") + + counter = 0 } } - }() + }, + } - t := time.Now() - sig := make(chan os.Signal, 1) - signal.Notify(sig, os.Interrupt) + cmd.Flags().AddFlagSet(RedisFlags()) + cmd.Flags().DurationP("interval", "i", time.Minute, "How often the benchmark results should be displayed.") - // Read stuff. - for { - select { - case <-sig: - fmt.Println("Got interrupt") - client.Close() - return - case now := <-time.After(interval): - elapsed := now.Sub(t) - t = now - - log.WithFields(log.Fields{ - "num_messages": counter, - "elapsed": elapsed, - "msg_per_sec": float64(counter) / elapsed.Seconds(), - "msg_per_ms": float64(counter) / float64(elapsed.Milliseconds()), - "msg_per_min": float64(counter) / elapsed.Minutes(), - }).Info("Benchmark results") - - counter = 0 - } - } - }, + return cmd } diff --git a/cmd/tools/flags.go b/cmd/tools/flags.go new file mode 100644 index 0000000..fd54332 --- /dev/null +++ b/cmd/tools/flags.go @@ -0,0 +1,14 @@ +package main + +import "github.com/spf13/pflag" + +func RedisFlags() *pflag.FlagSet { + set := pflag.FlagSet{} + set.String("redis-url", "127.0.0.1:6379", "host:port to the redis server") + set.String("redis-user", "", "User to use when authenticating to the server") + set.String("redis-pw", "", "Password to use when authenticating to the server") + set.Int("redis-db", 0, "What redis database we should connect to.") + set.String("prefix", "ship", "redis prefix") + set.String("chain_id", "1064487b3cd1a897ce03ae5b6a865651747e2e152090f99c1d19d44e01aea5a4", "chain id") + return &set +} diff --git a/cmd/tools/main.go b/cmd/tools/main.go index abb206d..3e131ac 100644 --- a/cmd/tools/main.go +++ b/cmd/tools/main.go @@ -1,28 +1,15 @@ package main import ( - "time" - _ "github.com/eosswedenorg/thalos/internal/log" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" - "github.com/spf13/pflag" ) var VersionString string = "dev" -var rootCmd *cobra.Command - -func init() { - redisFlags := pflag.FlagSet{} - redisFlags.String("redis-url", "127.0.0.1:6379", "host:port to the redis server") - redisFlags.String("redis-user", "", "User to use when authenticating to the server") - redisFlags.String("redis-pw", "", "Password to use when authenticating to the server") - redisFlags.Int("redis-db", 0, "What redis database we should connect to.") - redisFlags.String("prefix", "ship", "redis prefix") - redisFlags.String("chain_id", "1064487b3cd1a897ce03ae5b6a865651747e2e152090f99c1d19d44e01aea5a4", "chain id") - - rootCmd = &cobra.Command{ +func main() { + rootCmd := &cobra.Command{ Use: "thalos-tools", Short: "Collection of tools for dealing with the thalos application", FParseErrWhitelist: cobra.FParseErrWhitelist{ @@ -31,33 +18,13 @@ func init() { Version: VersionString, } - benchCmd.Flags().AddFlagSet(&redisFlags) - benchCmd.Flags().DurationP("interval", "i", time.Minute, "How often the benchmark results should be displayed.") - - validateCmd.Flags().AddFlagSet(&redisFlags) - - MockPublisherCmd.Flags().AddFlagSet(&redisFlags) - MockPublisherCmd.Flags().String("codec", "json", "codec to use") - - RedisACLCmd.Flags().String("default-pw", "", "Password to use for the default account, if not provided a random one will be generated") - RedisACLCmd.Flags().String("client", "thalos-client", "Thalos client account name") - RedisACLCmd.Flags().String("client-pw", "", "Password to use for the thalos client account, if not provided a random one will be generated") - RedisACLCmd.Flags().String("server", "thalos", "Thalos account name") - RedisACLCmd.Flags().String("server-pw", "", "Password to use for the thalos server account, if not provided a random one will be generated") - RedisACLCmd.Flags().String("prefix", "ship", "Redis key prefix") - RedisACLCmd.Flags().Bool("cleartext", false, "If passwords should be hashed or left in cleartext.") - RedisACLCmd.Flags().String("file", "", "Where the config should be written to (default: standard out)") - RedisACLCmd.Flags().Uint("pass-len", 32, "The length of generated passwords") - rootCmd.AddCommand( - validateCmd, - benchCmd, - RedisACLCmd, - MockPublisherCmd, + CreateValidateCmd(), + CreateBenchCmd(), + CreateRedisACLCmd(), + CreateMockPublisherCmd(), ) -} -func main() { if err := rootCmd.Execute(); err != nil { log.Fatal(err) } diff --git a/cmd/tools/mock_publisher.go b/cmd/tools/mock_publisher.go index ec38871..6a9e028 100644 --- a/cmd/tools/mock_publisher.go +++ b/cmd/tools/mock_publisher.go @@ -16,100 +16,106 @@ import ( log "github.com/sirupsen/logrus" ) -var MockPublisherCmd = &cobra.Command{ - Use: "mock_publisher", - Short: "Run a publisher that mocks messages to a redis server. tries to send as many messages as possible", - Run: func(cmd *cobra.Command, args []string) { - url, _ := cmd.Flags().GetString("redis-url") - user, _ := cmd.Flags().GetString("redis-user") - pw, _ := cmd.Flags().GetString("redis-pw") - prefix, _ := cmd.Flags().GetString("prefix") - chain_id, _ := cmd.Flags().GetString("chain_id") - db, _ := cmd.Flags().GetInt("redis-db") +func CreateMockPublisherCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "mock_publisher", + Short: "Run a publisher that mocks messages to a redis server. tries to send as many messages as possible", + Run: func(cmd *cobra.Command, args []string) { + url, _ := cmd.Flags().GetString("redis-url") + user, _ := cmd.Flags().GetString("redis-user") + pw, _ := cmd.Flags().GetString("redis-pw") + prefix, _ := cmd.Flags().GetString("prefix") + chain_id, _ := cmd.Flags().GetString("chain_id") + db, _ := cmd.Flags().GetInt("redis-db") - // Create redis client - rdb := redis.NewClient(&redis.Options{ - Addr: url, - Username: user, - Password: pw, - DB: db, - }) + // Create redis client + rdb := redis.NewClient(&redis.Options{ + Addr: url, + Username: user, + Password: pw, + DB: db, + }) - codecArg, _ := cmd.Flags().GetString("codec") + codecArg, _ := cmd.Flags().GetString("codec") - codec, err := message.GetCodec(codecArg) - if err != nil { - log.WithError(err).Fatal("Failed to get codec") - return - } + codec, err := message.GetCodec(codecArg) + if err != nil { + log.WithError(err).Fatal("Failed to get codec") + return + } - log.WithFields(log.Fields{ - "url": url, - "prefix": prefix, - "chain_id": chain_id, - "database": db, - }).Info("Starting mock publisher") + log.WithFields(log.Fields{ + "url": url, + "prefix": prefix, + "chain_id": chain_id, + "database": db, + }).Info("Starting mock publisher") - ns := api_redis.Namespace{ - Prefix: prefix, - ChainID: chain_id, - } - publisher := redis_driver.NewPublisher(context.Background(), rdb, ns) + ns := api_redis.Namespace{ + Prefix: prefix, + ChainID: chain_id, + } + publisher := redis_driver.NewPublisher(context.Background(), rdb, ns) - msg := message.ActionTrace{ - TxID: "401e8a7e5deb18a2a69fc6559f49509a155f4355c85efb69c1c1fab5b60ee532", - BlockNum: 18237917, - Timestamp: time.Date(2014, 3, 22, 11, 36, 43, 0, time.UTC), - Receipt: &message.ActionReceipt{ - Receiver: "acc1", - ActDigest: "4c5c08be612e937564fc526ebb5fadf34ae8c2a571fe9d7cdb3ffcdfc53b0e8d", - GlobalSequence: 12314, - RecvSequence: 237187239, - AuthSequence: []message.AccountAuthSequence{ + msg := message.ActionTrace{ + TxID: "401e8a7e5deb18a2a69fc6559f49509a155f4355c85efb69c1c1fab5b60ee532", + BlockNum: 18237917, + Timestamp: time.Date(2014, 3, 22, 11, 36, 43, 0, time.UTC), + Receipt: &message.ActionReceipt{ + Receiver: "acc1", + ActDigest: "4c5c08be612e937564fc526ebb5fadf34ae8c2a571fe9d7cdb3ffcdfc53b0e8d", + GlobalSequence: 12314, + RecvSequence: 237187239, + AuthSequence: []message.AccountAuthSequence{ + { + Account: "acc1", + Sequence: 2732863, + }, + { + Account: "acc2", + Sequence: 263762, + }, + }, + CodeSequence: 2327832, + ABISequence: 12376189, + }, + Name: "fake", + Contract: "fake", + Receiver: "acc1", + Data: map[string]interface{}{ + "one": 238771832, + "two": "str", + }, + Authorization: []message.PermissionLevel{ { - Account: "acc1", - Sequence: 2732863, + Actor: "acc1", + Permission: "active", }, { - Account: "acc2", - Sequence: 263762, + Actor: "acc2", + Permission: "owner", }, }, - CodeSequence: 2327832, - ABISequence: 12376189, - }, - Name: "fake", - Contract: "fake", - Receiver: "acc1", - Data: map[string]interface{}{ - "one": 238771832, - "two": "str", - }, - Authorization: []message.PermissionLevel{ - { - Actor: "acc1", - Permission: "active", - }, - { - Actor: "acc2", - Permission: "owner", - }, - }, - Except: "err", - Error: 2, - Return: []byte{0xbe, 0xef}, - } + Except: "err", + Error: 2, + Return: []byte{0xbe, 0xef}, + } - payload, err := codec.Encoder(msg) - if err != nil { - log.WithError(err).Fatal("Failed to encode message") - return - } - channel := api.ActionChannel{}.Channel() + payload, err := codec.Encoder(msg) + if err != nil { + log.WithError(err).Fatal("Failed to encode message") + return + } + channel := api.ActionChannel{}.Channel() - for { - _ = publisher.Write(channel, payload) - publisher.Flush() - } - }, + for { + _ = publisher.Write(channel, payload) + publisher.Flush() + } + }, + } + + cmd.Flags().AddFlagSet(RedisFlags()) + cmd.Flags().String("codec", "json", "codec to use") + return cmd } diff --git a/cmd/tools/redis-acl.go b/cmd/tools/redis-acl.go index 0f4496c..ad15036 100644 --- a/cmd/tools/redis-acl.go +++ b/cmd/tools/redis-acl.go @@ -43,7 +43,6 @@ func NewUser(name, password string, pass_len uint) User { } func (u *User) GetPassword() string { - if u.Hash { return "#" + hash(u.Password) } @@ -99,60 +98,74 @@ user {{.client}} on {{.clientpw}} resetchannels &{{.prefix}}::* -@all +subscribe }) } -var RedisACLCmd = &cobra.Command{ - Use: "redis-acl", - Short: "create a users.acl file", - Run: func(cmd *cobra.Command, args []string) { - var err error - out := os.Stdout +func CreateRedisACLCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "redis-acl", + Short: "create a users.acl file", + Run: func(cmd *cobra.Command, args []string) { + var err error + out := os.Stdout - rnd = rand.New(rand.NewSource(time.Now().UnixNano())) + rnd = rand.New(rand.NewSource(time.Now().UnixNano())) - flagDefUserPw, _ := cmd.Flags().GetString("default-pw") - flagServer, _ := cmd.Flags().GetString("server") - flagServerPw, _ := cmd.Flags().GetString("server-pw") - flagClient, _ := cmd.Flags().GetString("client") - flagClientPw, _ := cmd.Flags().GetString("client-pw") - flagPrefix, _ := cmd.Flags().GetString("prefix") - flagPassLen, _ := cmd.Flags().GetUint("pass-len") + flagDefUserPw, _ := cmd.Flags().GetString("default-pw") + flagServer, _ := cmd.Flags().GetString("server") + flagServerPw, _ := cmd.Flags().GetString("server-pw") + flagClient, _ := cmd.Flags().GetString("client") + flagClientPw, _ := cmd.Flags().GetString("client-pw") + flagPrefix, _ := cmd.Flags().GetString("prefix") + flagPassLen, _ := cmd.Flags().GetUint("pass-len") - defaultUser := NewUser("default", flagDefUserPw, flagPassLen) - serverUser := NewUser(flagServer, flagServerPw, flagPassLen) - clientUser := NewUser(flagClient, flagClientPw, flagPassLen) + defaultUser := NewUser("default", flagDefUserPw, flagPassLen) + serverUser := NewUser(flagServer, flagServerPw, flagPassLen) + clientUser := NewUser(flagClient, flagClientPw, flagPassLen) - atleastOneGeneratedPw := defaultUser.Generated || serverUser.Generated || clientUser.Generated + atleastOneGeneratedPw := defaultUser.Generated || serverUser.Generated || clientUser.Generated - cleartext, _ := cmd.Flags().GetBool("cleartext") - if !cleartext { - if atleastOneGeneratedPw { - println("Passwords") + cleartext, _ := cmd.Flags().GetBool("cleartext") + if !cleartext { + if atleastOneGeneratedPw { + println("Passwords") + } + + defaultUser.PrintIfGeneratedPW() + serverUser.PrintIfGeneratedPW() + clientUser.PrintIfGeneratedPW() + + defaultUser.Hash = true + serverUser.Hash = true + clientUser.Hash = true } - defaultUser.PrintIfGeneratedPW() - serverUser.PrintIfGeneratedPW() - clientUser.PrintIfGeneratedPW() + filename, _ := cmd.Flags().GetString("file") + if len(filename) > 0 { + out, err = os.Create(filename) + if err != nil { + log.WithError(err).Fatal("Failed to create output file") + return + } + defer out.Close() + } else if !cleartext && atleastOneGeneratedPw { + fmt.Println() + } - defaultUser.Hash = true - serverUser.Hash = true - clientUser.Hash = true - } - - filename, _ := cmd.Flags().GetString("file") - if len(filename) > 0 { - out, err = os.Create(filename) + err = writeTemplate(out, defaultUser, serverUser, clientUser, flagPrefix) if err != nil { - log.WithError(err).Fatal("Failed to create output file") + log.WithError(err).Fatal("Failed to writte config") return } - defer out.Close() - } else if !cleartext && atleastOneGeneratedPw { - fmt.Println() - } + }, + } - err = writeTemplate(out, defaultUser, serverUser, clientUser, flagPrefix) - if err != nil { - log.WithError(err).Fatal("Failed to writte config") - return - } - }, + cmd.Flags().String("default-pw", "", "Password to use for the default account, if not provided a random one will be generated") + cmd.Flags().String("client", "thalos-client", "Thalos client account name") + cmd.Flags().String("client-pw", "", "Password to use for the thalos client account, if not provided a random one will be generated") + cmd.Flags().String("server", "thalos", "Thalos account name") + cmd.Flags().String("server-pw", "", "Password to use for the thalos server account, if not provided a random one will be generated") + cmd.Flags().String("prefix", "ship", "Redis key prefix") + cmd.Flags().Bool("cleartext", false, "If passwords should be hashed or left in cleartext.") + cmd.Flags().String("file", "", "Where the config should be written to (default: standard out)") + cmd.Flags().Uint("pass-len", 32, "The length of generated passwords") + + return cmd } diff --git a/cmd/tools/validate.go b/cmd/tools/validate.go index 5c040b6..310c8c1 100644 --- a/cmd/tools/validate.go +++ b/cmd/tools/validate.go @@ -18,101 +18,107 @@ import ( log "github.com/sirupsen/logrus" ) -var validateCmd = &cobra.Command{ - Use: "validate", - Short: "Validate a thalos server by following action traces and makes sure that blocks arrive in order.", - Run: func(cmd *cobra.Command, args []string) { - status_duration := time.Second * 10 +func CreateValidateCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "validate", + Short: "Validate a thalos server by following action traces and makes sure that blocks arrive in order.", + Run: func(cmd *cobra.Command, args []string) { + status_duration := time.Second * 10 - url, _ := cmd.Flags().GetString("redis-url") - prefix, _ := cmd.Flags().GetString("prefix") - chain_id, _ := cmd.Flags().GetString("chain_id") - db, _ := cmd.Flags().GetInt("redis-db") + url, _ := cmd.Flags().GetString("redis-url") + prefix, _ := cmd.Flags().GetString("prefix") + chain_id, _ := cmd.Flags().GetString("chain_id") + db, _ := cmd.Flags().GetInt("redis-db") - log.WithFields(log.Fields{ - "url": url, - "prefix": prefix, - "chain_id": chain_id, - "database": db, - }).Info("Connecting to redis") + log.WithFields(log.Fields{ + "url": url, + "prefix": prefix, + "chain_id": chain_id, + "database": db, + }).Info("Connecting to redis") - // Create redis client - rdb := redis.NewClient(&redis.Options{ - Addr: url, - DB: db, - }) + // Create redis client + rdb := redis.NewClient(&redis.Options{ + Addr: url, + DB: db, + }) - if err := rdb.Ping(context.Background()).Err(); err != nil { - log.WithError(err).Fatal("Failed to connect to redis") - return - } + if err := rdb.Ping(context.Background()).Err(); err != nil { + log.WithError(err).Fatal("Failed to connect to redis") + return + } - log.Println("Connected to redis") + log.Println("Connected to redis") - log.Info("Starting validation, following the stream") + log.Info("Starting validation, following the stream") - sub := api_redis.NewSubscriber(context.Background(), rdb, api_redis.Namespace{ - Prefix: prefix, - ChainID: chain_id, - }) + sub := api_redis.NewSubscriber(context.Background(), rdb, api_redis.Namespace{ + Prefix: prefix, + ChainID: chain_id, + }) - codec, err := message.GetCodec("json") - if err != nil { - log.WithError(err).Fatal("Failed to get codec") - return - } + codec, err := message.GetCodec("json") + if err != nil { + log.WithError(err).Fatal("Failed to get codec") + return + } - client := api.NewClient(sub, codec.Decoder) + client := api.NewClient(sub, codec.Decoder) - // Subscribe to all actions - if err = client.Subscribe(api.ActionChannel{}.Channel()); err != nil { - log.WithError(err).Fatal("Failed to subscribe to channels") - return - } + // Subscribe to all actions + if err = client.Subscribe(api.ActionChannel{}.Channel()); err != nil { + log.WithError(err).Fatal("Failed to subscribe to channels") + return + } - block_num := uint32(0) - timeout := time.Second * 5 - timer := time.NewTicker(timeout) + block_num := uint32(0) + timeout := time.Second * 5 + timer := time.NewTicker(timeout) - go func() { - for t := range client.Channel() { - switch msg := t.(type) { - case error: - log.WithError(msg).Error("Error when reading stream") - case message.ActionTrace: - if block_num > 0 { - diff := int32(msg.BlockNum - block_num) - if diff < 0 || diff > 1 { - log.WithFields(log.Fields{ - "current_block": block_num, - "block": msg.BlockNum, - "diff": diff, - }).Warn("Invalid") + go func() { + for t := range client.Channel() { + switch msg := t.(type) { + case error: + log.WithError(msg).Error("Error when reading stream") + case message.ActionTrace: + if block_num > 0 { + diff := int32(msg.BlockNum - block_num) + if diff < 0 || diff > 1 { + log.WithFields(log.Fields{ + "current_block": block_num, + "block": msg.BlockNum, + "diff": diff, + }).Warn("Invalid") + } } + block_num = msg.BlockNum + timer.Reset(timeout) } - block_num = msg.BlockNum - timer.Reset(timeout) + } + }() + + sig := make(chan os.Signal, 1) + signal.Notify(sig, os.Interrupt) + + for { + select { + case <-sig: + fmt.Println("Got interrupt") + client.Close() + return + case <-timer.C: + log.WithField("duration", timeout). + Warn("Did not get any messages during the defined duration") + case <-time.After(status_duration): + log.WithFields(log.Fields{ + "current_block": block_num, + }).Info("Status") } } - }() + }, + } - sig := make(chan os.Signal, 1) - signal.Notify(sig, os.Interrupt) + cmd.Flags().AddFlagSet(RedisFlags()) - for { - select { - case <-sig: - fmt.Println("Got interrupt") - client.Close() - return - case <-timer.C: - log.WithField("duration", timeout). - Warn("Did not get any messages during the defined duration") - case <-time.After(status_duration): - log.WithFields(log.Fields{ - "current_block": block_num, - }).Info("Status") - } - } - }, + return cmd } From a2f8c3dc7c4ff45468fac70941c4f9c10122087f Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Wed, 19 Jun 2024 21:51:14 +0200 Subject: [PATCH 190/291] v1.1.2-rc3 --- Makefile | 2 +- debian/changelog | 7 +++++++ docker/Dockerfile | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 438f932..8af9320 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ GO=go GOLDFLAGS=-v -s -w -X main.VersionString=$(PROGRAM_VERSION) GOBUILDFLAGS=-v -p $(shell nproc) -ldflags="$(GOLDFLAGS)" PROGRAM=thalos-server -PROGRAM_VERSION=1.1.2-rc2 +PROGRAM_VERSION=1.1.2-rc3 PREFIX=/usr/local BINDIR=$(PREFIX)/bin CFGDIR=$(PREFIX)/etc/thalos diff --git a/debian/changelog b/debian/changelog index fe40add..ffcabbb 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,10 @@ +thalos (1.1.2~rc3) bionic focal jammy; urgency=medium + + * Fix a bug in isVariant() where v.Elem() was called on non interface/pointer + * Minor cleanups in tools + + -- Henrik Hautakoski Wed, 19 Jun 2024 21:50:15 +0200 + thalos (1.1.2~rc2) bionic focal jammy; urgency=medium * fix a bug where TableDeltaRow.Data was not set diff --git a/docker/Dockerfile b/docker/Dockerfile index d00fcb9..bc11854 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,6 +1,6 @@ FROM alpine:latest LABEL maintainer="Henrik Hautakoski " -ARG VERSION=1.1.2-rc2 +ARG VERSION=1.1.2-rc3 WORKDIR /thalos ADD --chmod=755 https://github.com/eosswedenorg/thalos/releases/download/v$VERSION/thalos-server-${VERSION}-linux-amd64-musl thalos-server ENTRYPOINT [ "./thalos-server" ] From 5b02dfa53f14e4345467cd6abcdd19c8e981b95c Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sat, 22 Jun 2024 16:19:48 +0200 Subject: [PATCH 191/291] jsontime: use struct tags instead of setting default format. Some other package (antelope-go in this case) also sets the default format and therefore it clashes with our code. --- api/message/json/codec.go | 3 ++- api/message/types.go | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/api/message/json/codec.go b/api/message/json/codec.go index d04e103..7024883 100644 --- a/api/message/json/codec.go +++ b/api/message/json/codec.go @@ -18,7 +18,8 @@ func createCodec() message.Codec { func init() { // Set timeformat used by SHIP api - jsontime.SetDefaultTimeFormat("2006-01-02T15:04:05.000", time.UTC) + jsontime.AddTimeFormatAlias("ship", "2006-01-02T15:04:05.000") + jsontime.AddLocaleAlias("ship", time.UTC) // Register the json codec. message.RegisterCodec("json", createCodec()) diff --git a/api/message/types.go b/api/message/types.go index b012b79..98bee44 100644 --- a/api/message/types.go +++ b/api/message/types.go @@ -24,7 +24,7 @@ type AccountAuthSequence struct { type TransactionTrace struct { ID string `json:"id" msgpack:"id"` BlockNum uint32 `json:"blocknum" msgpack:"blocknum"` - Timestamp time.Time `json:"blocktimestamp" msgpack:"blocktimestamp"` + Timestamp time.Time `json:"blocktimestamp" msgpack:"blocktimestamp" time_format:"ship" time_location:"ship"` Status string `json:"status" msgpack:"status"` CPUUsageUS uint32 `json:"cpu_usage_us" msgpack:"cpu_usage_us"` NetUsageWords uint32 `json:"net_usage_words" msgpack:"net_usage_words"` @@ -54,7 +54,7 @@ type ActionTrace struct { BlockNum uint32 `json:"blocknum" msgpack:"blocknum"` - Timestamp time.Time `json:"blocktimestamp" msgpack:"blocktimestamp"` + Timestamp time.Time `json:"blocktimestamp" msgpack:"blocktimestamp" time_format:"ship" time_location:"ship"` Receipt *ActionReceipt `json:"receipt,omitempty" msgpack:"receipt"` @@ -96,7 +96,7 @@ type TableDeltaRow struct { type TableDelta struct { BlockNum uint32 `json:"blocknum" msgpack:"blocknum"` - Timestamp time.Time `json:"blocktimestamp" msgpack:"blocktimestamp"` + Timestamp time.Time `json:"blocktimestamp" msgpack:"blocktimestamp" time_format:"ship" time_location:"ship"` Name string `json:"name" msgpack:"name"` Rows []TableDeltaRow `json:"rows" msgpack:"rows"` } From f19304bf55cd41b0a821de1d7276dbb717098d0b Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sat, 22 Jun 2024 16:47:00 +0200 Subject: [PATCH 192/291] use antelope-go v0.0.4 --- go.mod | 12 ++++++------ go.sum | 25 ++++++++++++------------- 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/go.mod b/go.mod index 093490c..0b2d150 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/go-redis/redismock/v9 v9.2.0 github.com/mitchellh/mapstructure v1.5.0 github.com/nikoksr/notify v0.41.0 - github.com/pnx/antelope-go v0.0.4-0.20240511122657-905898ed2110 + github.com/pnx/antelope-go v0.0.4 github.com/redis/go-redis/v9 v9.5.1 github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.8.0 @@ -23,7 +23,7 @@ require ( ) require ( - github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/eosswedenorg-go/jsontime v0.0.0-20230509125027-08422d6236c7 // indirect @@ -37,7 +37,7 @@ require ( github.com/imroc/req/v3 v3.7.6 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.17.7 // indirect + github.com/klauspost/compress v1.17.8 // indirect github.com/liamylian/jsontime/v2 v2.0.0 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect @@ -59,9 +59,9 @@ require ( github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 // indirect - golang.org/x/net v0.22.0 // indirect - golang.org/x/sync v0.6.0 // indirect - golang.org/x/sys v0.18.0 // indirect + golang.org/x/net v0.24.0 // indirect + golang.org/x/sync v0.7.0 // indirect + golang.org/x/sys v0.19.0 // indirect golang.org/x/text v0.14.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/ini.v1 v1.67.0 // indirect diff --git a/go.sum b/go.sum index d0270c6..0166289 100644 --- a/go.sum +++ b/go.sum @@ -5,8 +5,9 @@ github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -78,8 +79,8 @@ github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/u github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg= -github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= +github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= @@ -137,10 +138,8 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/pnx/antelope-go v0.0.3 h1:LUTQtCdErPqZqF+15ZinFXvQOHaPYuOHyO24TCaqrN8= -github.com/pnx/antelope-go v0.0.3/go.mod h1:NCCgAR1/Yzh6DigwtFFMcMhG6kGzfNx5sUorUMpBuNc= -github.com/pnx/antelope-go v0.0.4-0.20240511122657-905898ed2110 h1:ipRjWXy6HNkd3QdAdr2RJEHUxUETc9UbQB3dyU70Aq0= -github.com/pnx/antelope-go v0.0.4-0.20240511122657-905898ed2110/go.mod h1:NCCgAR1/Yzh6DigwtFFMcMhG6kGzfNx5sUorUMpBuNc= +github.com/pnx/antelope-go v0.0.4 h1:7qZUNfLSSAhlOI1qNwu+9X0lyttK2eGlmmYOj3u8uwI= +github.com/pnx/antelope-go v0.0.4/go.mod h1:NCCgAR1/Yzh6DigwtFFMcMhG6kGzfNx5sUorUMpBuNc= github.com/redis/go-redis/v9 v9.0.0-rc.4/go.mod h1:Vo3EsyWnicKnSKCA7HhgnvnyA74wOA69Cd2Meli5mmA= github.com/redis/go-redis/v9 v9.5.1 h1:H1X4D3yHPaYrkL5X06Wh6xNVM/pX0Ft4RV0vMGvLBh8= github.com/redis/go-redis/v9 v9.5.1/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= @@ -225,16 +224,16 @@ golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= -golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= -golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= +golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= -golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -259,8 +258,8 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= +golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= From 01054fb0bf17671aaa8dac11c19a685806dc71fd Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sat, 22 Jun 2024 17:02:19 +0200 Subject: [PATCH 193/291] internal/abi/manager_test.go: update because of antelope-go v0.0.4 --- internal/abi/manager_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/abi/manager_test.go b/internal/abi/manager_test.go index edd32f1..f44b9f7 100644 --- a/internal/abi/manager_test.go +++ b/internal/abi/manager_test.go @@ -111,12 +111,12 @@ func assert_abi(t *testing.T, abi *chain.Abi) { assert.Equal(t, abi.Structs[3].Fields[0].Type, "string") // Actions - assert.Equal(t, abi.Actions[0].Name, chain.N("action_name_1")) + assert.Equal(t, abi.Actions[0].Name, "action_name_1") assert.Equal(t, abi.Actions[0].Type, "struct_name_1") assert.Equal(t, abi.Actions[0].RicardianContract, "") // Tables - assert.Equal(t, abi.Tables[0].Name, chain.N("table_name_1")) + assert.Equal(t, abi.Tables[0].Name, "table_name_1") assert.Equal(t, abi.Tables[0].Type, "struct_name_1") assert.Equal(t, abi.Tables[0].IndexType, "i64") assert.Equal(t, abi.Tables[0].KeyNames[0], "key_name_1") From 2b3dc353936027f59cd03ea50857fcf5893689ed Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sat, 22 Jun 2024 17:39:10 +0200 Subject: [PATCH 194/291] jsontime: skip aliases and use actual format and location This is so weird. this works in tests but not when running the application. I guess there is some weird global state race condition going on because its used by antelope-go. Should rewrite the jsontime api because I have forked it anyway. --- api/message/json/codec.go | 6 ------ api/message/types.go | 6 +++--- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/api/message/json/codec.go b/api/message/json/codec.go index 7024883..b0fc25e 100644 --- a/api/message/json/codec.go +++ b/api/message/json/codec.go @@ -1,8 +1,6 @@ package json import ( - "time" - jsontime "github.com/eosswedenorg-go/jsontime/v2" "github.com/eosswedenorg/thalos/api/message" ) @@ -17,10 +15,6 @@ func createCodec() message.Codec { } func init() { - // Set timeformat used by SHIP api - jsontime.AddTimeFormatAlias("ship", "2006-01-02T15:04:05.000") - jsontime.AddLocaleAlias("ship", time.UTC) - // Register the json codec. message.RegisterCodec("json", createCodec()) } diff --git a/api/message/types.go b/api/message/types.go index 98bee44..b17b0ff 100644 --- a/api/message/types.go +++ b/api/message/types.go @@ -24,7 +24,7 @@ type AccountAuthSequence struct { type TransactionTrace struct { ID string `json:"id" msgpack:"id"` BlockNum uint32 `json:"blocknum" msgpack:"blocknum"` - Timestamp time.Time `json:"blocktimestamp" msgpack:"blocktimestamp" time_format:"ship" time_location:"ship"` + Timestamp time.Time `json:"blocktimestamp" msgpack:"blocktimestamp" time_format:"2006-01-02T15:04:05.000" time_location:"UTC"` Status string `json:"status" msgpack:"status"` CPUUsageUS uint32 `json:"cpu_usage_us" msgpack:"cpu_usage_us"` NetUsageWords uint32 `json:"net_usage_words" msgpack:"net_usage_words"` @@ -54,7 +54,7 @@ type ActionTrace struct { BlockNum uint32 `json:"blocknum" msgpack:"blocknum"` - Timestamp time.Time `json:"blocktimestamp" msgpack:"blocktimestamp" time_format:"ship" time_location:"ship"` + Timestamp time.Time `json:"blocktimestamp" msgpack:"blocktimestamp" time_format:"2006-01-02T15:04:05.000" time_location:"UTC"` Receipt *ActionReceipt `json:"receipt,omitempty" msgpack:"receipt"` @@ -96,7 +96,7 @@ type TableDeltaRow struct { type TableDelta struct { BlockNum uint32 `json:"blocknum" msgpack:"blocknum"` - Timestamp time.Time `json:"blocktimestamp" msgpack:"blocktimestamp" time_format:"ship" time_location:"ship"` + Timestamp time.Time `json:"blocktimestamp" msgpack:"blocktimestamp" time_format:"2006-01-02T15:04:05.000" time_location:"UTC"` Name string `json:"name" msgpack:"name"` Rows []TableDeltaRow `json:"rows" msgpack:"rows"` } From a466392b84f6ac56d78166722ae34aef164f7962 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 23 Jun 2024 11:33:35 +0200 Subject: [PATCH 195/291] use github.com/shufflingpixels/jsontime-go instead of github.com/eosswedenorg-go/jsontime --- api/go.mod | 2 +- api/go.sum | 4 ++-- api/message/json/codec.go | 6 ++++-- api/message/types.go | 6 +++--- go.mod | 2 +- go.sum | 4 ++-- 6 files changed, 13 insertions(+), 11 deletions(-) diff --git a/api/go.mod b/api/go.mod index fe492ef..f8d2a68 100644 --- a/api/go.mod +++ b/api/go.mod @@ -4,9 +4,9 @@ go 1.20 require ( github.com/alicebob/miniredis/v2 v2.30.2 - github.com/eosswedenorg-go/jsontime v0.0.0-20230509125027-08422d6236c7 github.com/go-redis/redismock/v9 v9.2.0 github.com/redis/go-redis/v9 v9.4.0 + github.com/shufflingpixels/jsontime-go v0.0.0-20240622163621-cf4b2804c92d github.com/stretchr/testify v1.8.4 github.com/ugorji/go/codec v1.2.12 ) diff --git a/api/go.sum b/api/go.sum index 7c06dcc..79690e8 100644 --- a/api/go.sum +++ b/api/go.sum @@ -14,8 +14,6 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= -github.com/eosswedenorg-go/jsontime v0.0.0-20230509125027-08422d6236c7 h1:rLPu++RHaxg4WmUOXeWYioZuafWs0PVcYuvzOWbOJjk= -github.com/eosswedenorg-go/jsontime v0.0.0-20230509125027-08422d6236c7/go.mod h1:eNUkVOymzgl0lViUhmm08PkutzqLnOQ6Dr+RUnf+Mq0= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/go-redis/redismock/v9 v9.2.0 h1:ZrMYQeKPECZPjOj5u9eyOjg8Nnb0BS9lkVIZ6IpsKLw= github.com/go-redis/redismock/v9 v9.2.0/go.mod h1:18KHfGDK4Y6c2R0H38EUGWAdc7ZQS9gfYxc94k7rWT0= @@ -35,6 +33,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/redis/go-redis/v9 v9.4.0 h1:Yzoz33UZw9I/mFhx4MNrB6Fk+XHO1VukNcCa1+lwyKk= github.com/redis/go-redis/v9 v9.4.0/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= +github.com/shufflingpixels/jsontime-go v0.0.0-20240622163621-cf4b2804c92d h1:nju7jR1Kf210tArPT6l//HlfLLFnvje1BWl5TSRsohQ= +github.com/shufflingpixels/jsontime-go v0.0.0-20240622163621-cf4b2804c92d/go.mod h1:W0TaKyg3kDqWmFUxlax3qAls/lRdo12EseM6f4f0dzE= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= diff --git a/api/message/json/codec.go b/api/message/json/codec.go index b0fc25e..d254b47 100644 --- a/api/message/json/codec.go +++ b/api/message/json/codec.go @@ -1,12 +1,14 @@ package json import ( - jsontime "github.com/eosswedenorg-go/jsontime/v2" + "time" + "github.com/eosswedenorg/thalos/api/message" + "github.com/shufflingpixels/jsontime-go" ) func createCodec() message.Codec { - json_codec := jsontime.ConfigWithCustomTimeFormat + json_codec := jsontime.New("2006-01-02T15:04:05.000", time.UTC) return message.Codec{ Encoder: json_codec.Marshal, diff --git a/api/message/types.go b/api/message/types.go index b17b0ff..b012b79 100644 --- a/api/message/types.go +++ b/api/message/types.go @@ -24,7 +24,7 @@ type AccountAuthSequence struct { type TransactionTrace struct { ID string `json:"id" msgpack:"id"` BlockNum uint32 `json:"blocknum" msgpack:"blocknum"` - Timestamp time.Time `json:"blocktimestamp" msgpack:"blocktimestamp" time_format:"2006-01-02T15:04:05.000" time_location:"UTC"` + Timestamp time.Time `json:"blocktimestamp" msgpack:"blocktimestamp"` Status string `json:"status" msgpack:"status"` CPUUsageUS uint32 `json:"cpu_usage_us" msgpack:"cpu_usage_us"` NetUsageWords uint32 `json:"net_usage_words" msgpack:"net_usage_words"` @@ -54,7 +54,7 @@ type ActionTrace struct { BlockNum uint32 `json:"blocknum" msgpack:"blocknum"` - Timestamp time.Time `json:"blocktimestamp" msgpack:"blocktimestamp" time_format:"2006-01-02T15:04:05.000" time_location:"UTC"` + Timestamp time.Time `json:"blocktimestamp" msgpack:"blocktimestamp"` Receipt *ActionReceipt `json:"receipt,omitempty" msgpack:"receipt"` @@ -96,7 +96,7 @@ type TableDeltaRow struct { type TableDelta struct { BlockNum uint32 `json:"blocknum" msgpack:"blocknum"` - Timestamp time.Time `json:"blocktimestamp" msgpack:"blocktimestamp" time_format:"2006-01-02T15:04:05.000" time_location:"UTC"` + Timestamp time.Time `json:"blocktimestamp" msgpack:"blocktimestamp"` Name string `json:"name" msgpack:"name"` Rows []TableDeltaRow `json:"rows" msgpack:"rows"` } diff --git a/go.mod b/go.mod index 0b2d150..136cacb 100644 --- a/go.mod +++ b/go.mod @@ -26,7 +26,6 @@ require ( github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect - github.com/eosswedenorg-go/jsontime v0.0.0-20230509125027-08422d6236c7 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-telegram-bot-api/telegram-bot-api v4.6.4+incompatible // indirect github.com/google/go-cmp v0.6.0 // indirect @@ -48,6 +47,7 @@ require ( github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect + github.com/shufflingpixels/jsontime-go v0.0.0-20240622163621-cf4b2804c92d // indirect github.com/sourcegraph/conc v0.3.0 // indirect github.com/spf13/afero v1.11.0 // indirect github.com/spf13/cast v1.6.0 // indirect diff --git a/go.sum b/go.sum index 0166289..b84d15d 100644 --- a/go.sum +++ b/go.sum @@ -23,8 +23,6 @@ github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4 github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/eosswedenorg-go/antelope-ship-client v0.2.9 h1:Isa90Tktdumj/P9XgQ8kBRqS+2MAdcPl7ev/n0KFvUQ= github.com/eosswedenorg-go/antelope-ship-client v0.2.9/go.mod h1:a0Kp4BJID8DMD0pmHnK5/gojtpRhh2pqqgt4Hcy5Wrw= -github.com/eosswedenorg-go/jsontime v0.0.0-20230509125027-08422d6236c7 h1:rLPu++RHaxg4WmUOXeWYioZuafWs0PVcYuvzOWbOJjk= -github.com/eosswedenorg-go/jsontime v0.0.0-20230509125027-08422d6236c7/go.mod h1:eNUkVOymzgl0lViUhmm08PkutzqLnOQ6Dr+RUnf+Mq0= github.com/eosswedenorg-go/pid v1.0.1 h1:W4AEnnNwb041SpNR1uTZ/KbJ0OTA5eqiqIR1Q5Ah6A0= github.com/eosswedenorg-go/pid v1.0.1/go.mod h1:wiOB/JXGt4YA3+T0j0xmCGSc3Jxzb7Ti/Ftli1fgWu4= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= @@ -150,6 +148,8 @@ github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6ke github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= +github.com/shufflingpixels/jsontime-go v0.0.0-20240622163621-cf4b2804c92d h1:nju7jR1Kf210tArPT6l//HlfLLFnvje1BWl5TSRsohQ= +github.com/shufflingpixels/jsontime-go v0.0.0-20240622163621-cf4b2804c92d/go.mod h1:W0TaKyg3kDqWmFUxlax3qAls/lRdo12EseM6f4f0dzE= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= From a79bf8f2f37ade4a09f69260b5af1ded8fa4174d Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 23 Jun 2024 11:42:26 +0200 Subject: [PATCH 196/291] Adding variabels for docker. --- Makefile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 8af9320..8123e73 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,8 @@ PROGRAM_VERSION=1.1.2-rc3 PREFIX=/usr/local BINDIR=$(PREFIX)/bin CFGDIR=$(PREFIX)/etc/thalos +DOCKER_IMAGE_REPO ?= ghcr.io/eosswedenorg/thalos +DOCKER_IMAGE_TAG ?= $(PROGRAM_VERSION) .PHONY: build build/$(PROGRAM) build/thalos-tools test docker-image docker-publish @@ -21,10 +23,10 @@ build/thalos-tools : $(GO) build $(GOBUILDFLAGS) -o $@ $(shell find cmd/tools -type f -name *.go) docker-image: - docker image build --build-arg VERSION=$(PROGRAM_VERSION) -t ghcr.io/eosswedenorg/thalos:$(PROGRAM_VERSION) docker + docker image build --build-arg VERSION=$(PROGRAM_VERSION) -t $(DOCKER_IMAGE_REPO):$(DOCKER_IMAGE_TAG) docker docker-publish: - docker image push ghcr.io/eosswedenorg/thalos:$(PROGRAM_VERSION) + docker image push $(DOCKER_IMAGE_REPO):$(DOCKER_IMAGE_TAG) install: build tools install -D build/$(PROGRAM) $(DESTDIR)$(BINDIR)/$(PROGRAM) From 93a816cb2cfbd5903f483976f9cb1a4a407daf0d Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 23 Jun 2024 14:46:38 +0200 Subject: [PATCH 197/291] Adding internal/types/blacklist.go --- internal/types/blacklist.go | 21 ++++++++++++++++ internal/types/blacklist_test.go | 43 ++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+) create mode 100644 internal/types/blacklist.go create mode 100644 internal/types/blacklist_test.go diff --git a/internal/types/blacklist.go b/internal/types/blacklist.go new file mode 100644 index 0000000..cbe28b5 --- /dev/null +++ b/internal/types/blacklist.go @@ -0,0 +1,21 @@ +package types + +type Blacklist map[string][]string + +func (bl Blacklist) Add(contract string, action string) { + if len(bl[contract]) < 1 { + bl[contract] = []string{} + } + bl[contract] = append(bl[contract], action) +} + +func (bl Blacklist) Lookup(contract string, action string) bool { + if v, ok := bl[contract]; ok { + for _, act := range v { + if act == action || act == "*" { + return true + } + } + } + return false +} diff --git a/internal/types/blacklist_test.go b/internal/types/blacklist_test.go new file mode 100644 index 0000000..a897802 --- /dev/null +++ b/internal/types/blacklist_test.go @@ -0,0 +1,43 @@ +package types + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestBlacklist_Add(t *testing.T) { + bl := Blacklist{} + bl.Add("contract", "action1") + bl.Add("contract", "action2") + bl.Add("contract2", "action1") + + expected := Blacklist{ + "contract": {"action1", "action2"}, + "contract2": {"action1"}, + } + + require.Equal(t, expected, bl) +} + +func TestBlacklist_Lookup(t *testing.T) { + bl := Blacklist{ + "mycontract": {"myaction", "noop"}, + } + + require.True(t, bl.Lookup("mycontract", "myaction")) + require.True(t, bl.Lookup("mycontract", "noop")) + require.False(t, bl.Lookup("mycontract", "xxx")) + require.False(t, bl.Lookup("xxx", "yyy")) +} + +func TestBlacklist_LookupWildcard(t *testing.T) { + bl := Blacklist{ + "mycontract": {"*"}, + } + + require.True(t, bl.Lookup("mycontract", "myaction")) + require.True(t, bl.Lookup("mycontract", "noop")) + require.True(t, bl.Lookup("mycontract", "xxx")) + require.False(t, bl.Lookup("xxx", "yyy")) +} From 6d14591f6bf521388993806d4d10d2c64c093232 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 23 Jun 2024 14:52:05 +0200 Subject: [PATCH 198/291] internal/config: adding blacklist field to Ship config --- internal/config/builder.go | 32 ++++++++++++++++++++++++++++++++ internal/config/builder_test.go | 33 +++++++++++++++++++++++++++++++++ internal/config/cli.go | 2 ++ internal/config/config.go | 14 ++++++++------ 4 files changed, 75 insertions(+), 6 deletions(-) diff --git a/internal/config/builder.go b/internal/config/builder.go index 3bcbbd3..026dd33 100644 --- a/internal/config/builder.go +++ b/internal/config/builder.go @@ -2,9 +2,13 @@ package config import ( "errors" + "fmt" "io" "os" + "reflect" + "strings" + "github.com/eosswedenorg/thalos/internal/types" "github.com/mitchellh/mapstructure" "github.com/spf13/pflag" "github.com/spf13/viper" @@ -48,6 +52,7 @@ func NewBuilder() *Builder { "ship.irreversible_only": "irreversible-only", "ship.max_messages_in_flight": "max-msg-in-flight", "ship.chain": "chain", + "ship.blacklist": "blacklist", }, } } @@ -110,6 +115,33 @@ func (b *Builder) Build() (*Config, error) { mapstructure.TextUnmarshallerHookFunc(), mapstructure.StringToTimeDurationHookFunc(), mapstructure.StringToSliceHookFunc(","), + func(f reflect.Type, t reflect.Type, in interface{}) (interface{}, error) { + if t == reflect.TypeOf(types.Blacklist{}) && f.Kind() == reflect.Slice { + if v, ok := in.([]string); ok { + list := types.Blacklist{} + for _, i := range v { + var action string + parts := strings.SplitN(i, ":", 2) + + if len(parts) < 2 { + action = "*" + } else { + action = parts[1] + } + + list.Add(parts[0], action) + } + + if len(list) < 1 { + list = nil + } + return list, nil + } + return nil, fmt.Errorf("Must be a string slice") + } + + return in, nil + }, ) err := v.Unmarshal(&conf, viper.DecodeHook(decoders)) diff --git a/internal/config/builder_test.go b/internal/config/builder_test.go index 24f7751..d226a30 100644 --- a/internal/config/builder_test.go +++ b/internal/config/builder_test.go @@ -6,6 +6,7 @@ import ( "time" "github.com/eosswedenorg/thalos/internal/log" + "github.com/eosswedenorg/thalos/internal/types" "github.com/stretchr/testify/require" ) @@ -27,6 +28,10 @@ func TestBuilder(t *testing.T) { EndBlockNum: 23872222, IrreversibleOnly: true, MaxMessagesInFlight: 1337, + Blacklist: types.Blacklist{ + "eosio": {"noop"}, + "contract": {"skip1", "skip2"}, + }, }, Telegram: TelegramConfig{ Id: "110201543:AAHdqTcvCH1vGWJxfSeofSAs0K5PALDsaw", @@ -58,6 +63,11 @@ ship: max_messages_in_flight: 1337 start_block_num: 23671836 end_block_num: 23872222 + blacklist: + eosio: ["noop"] + contract: + - skip1 + - skip2 telegram: id: "110201543:AAHdqTcvCH1vGWJxfSeofSAs0K5PALDsaw" channel: -123456789 @@ -175,6 +185,7 @@ func TestBuilder_Flags(t *testing.T) { require.NoError(t, flags.Set("irreversible-only", "true")) require.NoError(t, flags.Set("max-msg-in-flight", "98")) require.NoError(t, flags.Set("chain", "wax")) + require.NoError(t, flags.Set("blacklist", "contract:action1,contract:action2,contract2:action1")) cfg, err := NewBuilder(). SetSource(bytes.NewReader([]byte(``))). @@ -196,6 +207,10 @@ func TestBuilder_Flags(t *testing.T) { MaxMessagesInFlight: 98, IrreversibleOnly: true, Chain: "wax", + Blacklist: types.Blacklist{ + "contract": {"action1", "action2"}, + "contract2": {"action1"}, + }, }, Telegram: TelegramConfig{ Id: "72983126312982618", @@ -213,3 +228,21 @@ func TestBuilder_Flags(t *testing.T) { require.NoError(t, err) require.Equal(t, &expected, cfg) } + +func TestBuilder_BlacklistFlag(t *testing.T) { + flags := GetFlags() + + require.NoError(t, flags.Set("blacklist", "contract,contract:action2")) + + conf, err := NewBuilder(). + SetSource(bytes.NewReader([]byte(``))). + SetFlags(flags). + Build() + + expected := types.Blacklist{ + "contract": {"*", "action2"}, + } + + require.NoError(t, err) + require.Equal(t, expected, conf.Ship.Blacklist) +} diff --git a/internal/config/cli.go b/internal/config/cli.go index bace210..38439a5 100644 --- a/internal/config/cli.go +++ b/internal/config/cli.go @@ -43,5 +43,7 @@ func GetFlags() *pflag.FlagSet { flags.Int("max-msg-in-flight", 10, "Maximum messages that can be sent from SHIP without acknowledgement") flags.String("chain", "", "ChainID used in channel namespace, can be any string (default from api)") + flags.StringSlice("blacklist", []string{}, "Define a list of 'contract:action' pairs that will be blacklisted (thalos will not process those actions)") + return &flags } diff --git a/internal/config/config.go b/internal/config/config.go index 41777df..5e84072 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -2,6 +2,7 @@ package config import ( "github.com/eosswedenorg/thalos/internal/log" + "github.com/eosswedenorg/thalos/internal/types" ) type RedisConfig struct { @@ -18,12 +19,13 @@ type TelegramConfig struct { } type ShipConfig struct { - Url string `yaml:"url" mapstructure:"url"` - IrreversibleOnly bool `yaml:"irreversible_only" mapstructure:"irreversible_only"` - MaxMessagesInFlight uint32 `yaml:"max_messages_in_flight" mapstructure:"max_messages_in_flight"` - StartBlockNum uint32 `yaml:"start_block_num" mapstructure:"start_block_num"` - EndBlockNum uint32 `yaml:"end_block_num" mapstructure:"end_block_num"` - Chain string `yaml:"chain" mapstructure:"chain"` + Url string `yaml:"url" mapstructure:"url"` + IrreversibleOnly bool `yaml:"irreversible_only" mapstructure:"irreversible_only"` + MaxMessagesInFlight uint32 `yaml:"max_messages_in_flight" mapstructure:"max_messages_in_flight"` + StartBlockNum uint32 `yaml:"start_block_num" mapstructure:"start_block_num"` + EndBlockNum uint32 `yaml:"end_block_num" mapstructure:"end_block_num"` + Chain string `yaml:"chain" mapstructure:"chain"` + Blacklist types.Blacklist `yaml:"blacklist" mapstructure:"blacklist"` } type Config struct { From 0bc70f82d159b719874c65c48231938a7dd2fd8d Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 23 Jun 2024 14:52:17 +0200 Subject: [PATCH 199/291] internal/server/ship_processor.go: implement blacklist --- internal/server/ship_processor.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/internal/server/ship_processor.go b/internal/server/ship_processor.go index 6b05262..7670a50 100644 --- a/internal/server/ship_processor.go +++ b/internal/server/ship_processor.go @@ -7,6 +7,7 @@ import ( "github.com/eosswedenorg/thalos/api/message" "github.com/eosswedenorg/thalos/internal/abi" "github.com/eosswedenorg/thalos/internal/driver" + "github.com/eosswedenorg/thalos/internal/types" log "github.com/sirupsen/logrus" @@ -37,6 +38,9 @@ type ShipProcessor struct { // ABI Returned from SHIP shipABI *chain.Abi + + // Action blacklist + blacklist types.Blacklist } // SpawnProcessor creates a new ShipProccessor that consumes the shipclient.Stream passed to it. @@ -62,6 +66,10 @@ func SpawnProccessor(shipStream *shipclient.Stream, loader StateLoader, saver St return processor } +func (processor *ShipProcessor) SetBlacklist(list types.Blacklist) { + processor.blacklist = list +} + func (processor *ShipProcessor) initHandler(abi *chain.Abi) { processor.shipABI = abi } @@ -144,6 +152,15 @@ func (processor *ShipProcessor) proccessActionTrace(logger *log.Entry, trace *sh } } + // Check blacklist if we should skip this action + if processor.blacklist.Lookup(trace.Act.Account.String(), trace.Act.Name.String()) { + logger.WithFields(log.Fields{ + "contract": trace.Act.Account, + "action": trace.Act.Name, + }).Debug("Found in blacklist, skipping") + return nil + } + act := &message.ActionTrace{ Name: trace.Act.Name.String(), Contract: trace.Act.Account.String(), From 32f3f1dfb18e1e3d17d684b6339f4b6b0d1cc33a Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 23 Jun 2024 14:54:44 +0200 Subject: [PATCH 200/291] cmd/thalos/server.go: set blacklist in the processor --- cmd/thalos/server.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/thalos/server.go b/cmd/thalos/server.go index d4a764c..edfc960 100644 --- a/cmd/thalos/server.go +++ b/cmd/thalos/server.go @@ -415,6 +415,8 @@ func serverCmd(cmd *cobra.Command, args []string) { codec, ) + processor.SetBlacklist(conf.Ship.Blacklist) + // Run the application run(conf, shClient, processor) From be35936dc2aacdd89f0d846c52da9abd72b6eef5 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 23 Jun 2024 14:58:18 +0200 Subject: [PATCH 201/291] Version 1.1.2-rc4 --- Makefile | 2 +- debian/changelog | 9 +++++++++ docker/Dockerfile | 2 +- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 8123e73..79a0576 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ GO=go GOLDFLAGS=-v -s -w -X main.VersionString=$(PROGRAM_VERSION) GOBUILDFLAGS=-v -p $(shell nproc) -ldflags="$(GOLDFLAGS)" PROGRAM=thalos-server -PROGRAM_VERSION=1.1.2-rc3 +PROGRAM_VERSION=1.1.2-rc4 PREFIX=/usr/local BINDIR=$(PREFIX)/bin CFGDIR=$(PREFIX)/etc/thalos diff --git a/debian/changelog b/debian/changelog index ffcabbb..de87ee0 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,12 @@ +thalos (1.1.2~rc4) bionic focal jammy; urgency=medium + + * API: Fix a bug regarding json timestamp being encoded/decoded with wrong + format + * Implement action blacklist, it is not possible to configure a blacklist + that will be used to filter out processing of unwanted contracts/actions. + + -- Henrik Hautakoski Sun, 23 Jun 2024 14:55:03 +0200 + thalos (1.1.2~rc3) bionic focal jammy; urgency=medium * Fix a bug in isVariant() where v.Elem() was called on non interface/pointer diff --git a/docker/Dockerfile b/docker/Dockerfile index bc11854..ea38709 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,6 +1,6 @@ FROM alpine:latest LABEL maintainer="Henrik Hautakoski " -ARG VERSION=1.1.2-rc3 +ARG VERSION=1.1.2-rc4 WORKDIR /thalos ADD --chmod=755 https://github.com/eosswedenorg/thalos/releases/download/v$VERSION/thalos-server-${VERSION}-linux-amd64-musl thalos-server ENTRYPOINT [ "./thalos-server" ] From 690dabe6bd1df000d51a5b526fd78cfadf87c6c4 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 23 Jun 2024 15:21:56 +0200 Subject: [PATCH 202/291] config.example.yml: add blacklist --- config.example.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/config.example.yml b/config.example.yml index 0935d91..84990b1 100644 --- a/config.example.yml +++ b/config.example.yml @@ -43,6 +43,17 @@ ship: # Request ship to stop sending blocks when reaching this block. #end_block_num: 2000 + # Blacklist contract/actions + blacklist: + # this is a "useless" action that results in alot of warning messages. + # becase thalos does not know it's ABI. Its recommended to have this action blacklisted + # unless you have a reason to use it. + eosio.null: + - nonce + + # blacklist all action from a contract + # evilcontract: ["*"] + # Telegram notifications #telegram: # id: "123456789:GPdmGPBWvpgHPxlergJLavus-PoAURTjMWP" From aeeb049403dfe94744aa0e62e97217f2dfc42c61 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Thu, 27 Jun 2024 08:26:29 +0200 Subject: [PATCH 203/291] debian/patches/0001-fix-config-logpath.patch: update --- debian/patches/0001-fix-config-logpath.patch | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/debian/patches/0001-fix-config-logpath.patch b/debian/patches/0001-fix-config-logpath.patch index d78e5d9..eac48fa 100644 --- a/debian/patches/0001-fix-config-logpath.patch +++ b/debian/patches/0001-fix-config-logpath.patch @@ -1,12 +1,11 @@ diff --git a/config.example.yml b/config.example.yml -index 9f4272a..8daac0a 100644 --- a/config.example.yml +++ b/config.example.yml @@ -11,7 +11,7 @@ log: # Filename to use. - filename: thalos.log + filename: thalos # Directory to store the logfiles in. - directory: logs + directory: /var/log/thalos # Format to rename log files when rotating - time_format: 2006-01-02_150405 \ No newline at end of file + time_format: 2006-01-02_150405 From ec664ea20723d1e30a5f66e32325a013ae2a803c Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Thu, 27 Jun 2024 08:53:32 +0200 Subject: [PATCH 204/291] debian: adding sysconfig environment config --- debian/sysconfig/thalos-server | 6 ++++++ debian/thalos-server.service | 5 +++-- debian/thalos.install | 3 ++- 3 files changed, 11 insertions(+), 3 deletions(-) create mode 100644 debian/sysconfig/thalos-server diff --git a/debian/sysconfig/thalos-server b/debian/sysconfig/thalos-server new file mode 100644 index 0000000..6afed36 --- /dev/null +++ b/debian/sysconfig/thalos-server @@ -0,0 +1,6 @@ +# Environment variables for thalos-server +# +# This is file sourced by the systemd unit file + +# Append additional cli flags if needed. see: thalos-server -h +THALOS_SERVER_ARGS="-c /etc/thalos/config.yml" diff --git a/debian/thalos-server.service b/debian/thalos-server.service index 3265c1e..36bbb51 100644 --- a/debian/thalos-server.service +++ b/debian/thalos-server.service @@ -6,9 +6,10 @@ After=network.target User=thalos Group=thalos Type=simple -ExecStart=/usr/bin/thalos-server -c /etc/thalos/config.yml +EnvironmentFile=-/etc/sysconfig/thalos-server +ExecStart=/usr/bin/thalos-server $THALOS_SERVER_ARGS ExecReload=kill -HUP $MAINPID Restart=on-failure [Install] -WantedBy=multi-user.target \ No newline at end of file +WantedBy=multi-user.target diff --git a/debian/thalos.install b/debian/thalos.install index 510fa2e..6de9f98 100644 --- a/debian/thalos.install +++ b/debian/thalos.install @@ -1 +1,2 @@ -debian/thalos-server.service /lib/systemd/system \ No newline at end of file +debian/thalos-server.service /lib/systemd/system +debian/sysconfig/thalos-server /etc/sysconfig From 70eda8e3976eeb29820c666584eb38a2abeae752 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Thu, 27 Jun 2024 14:30:20 +0200 Subject: [PATCH 205/291] Version 1.1.2 --- Makefile | 2 +- debian/changelog | 14 ++++++++++++++ docker/Dockerfile | 2 +- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 79a0576..5c629de 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ GO=go GOLDFLAGS=-v -s -w -X main.VersionString=$(PROGRAM_VERSION) GOBUILDFLAGS=-v -p $(shell nproc) -ldflags="$(GOLDFLAGS)" PROGRAM=thalos-server -PROGRAM_VERSION=1.1.2-rc4 +PROGRAM_VERSION=1.1.2 PREFIX=/usr/local BINDIR=$(PREFIX)/bin CFGDIR=$(PREFIX)/etc/thalos diff --git a/debian/changelog b/debian/changelog index de87ee0..e1230f3 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,17 @@ +thalos (1.1.2) bionic focal jammy; urgency=medium + + * API: Fix a bug regarding json timestamp being encoded/decoded with wrong + format + * Implement action blacklist, it is not possible to configure a blacklist + that will be used to filter out processing of unwanted contracts/actions. + * Fix a bug in isVariant() where v.Elem() was called on non interface/pointer + * Minor cleanups in tools + * Fix a bug where TableDeltaRow.Data was not set + * Fix a bug where blockResult.Deltas was not properly nil checked. Resulting in panic if accessed + * Moved from github.com/eoscanda/eos-go to github.com/pnx/antelope-go library + + -- Henrik Hautakoski Thu, 27 Jun 2024 14:27:38 +0200 + thalos (1.1.2~rc4) bionic focal jammy; urgency=medium * API: Fix a bug regarding json timestamp being encoded/decoded with wrong diff --git a/docker/Dockerfile b/docker/Dockerfile index ea38709..5c5fd4d 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,6 +1,6 @@ FROM alpine:latest LABEL maintainer="Henrik Hautakoski " -ARG VERSION=1.1.2-rc4 +ARG VERSION=1.1.2 WORKDIR /thalos ADD --chmod=755 https://github.com/eosswedenorg/thalos/releases/download/v$VERSION/thalos-server-${VERSION}-linux-amd64-musl thalos-server ENTRYPOINT [ "./thalos-server" ] From dfa370a4621f54a1f7fff6c597be43f0e6487f56 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Tue, 2 Jul 2024 11:26:26 +0200 Subject: [PATCH 206/291] internal/server/ship_processor.go: adding some debug logging. --- internal/server/ship_processor.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/internal/server/ship_processor.go b/internal/server/ship_processor.go index 7670a50..edd33bd 100644 --- a/internal/server/ship_processor.go +++ b/internal/server/ship_processor.go @@ -146,6 +146,12 @@ func (processor *ShipProcessor) processTransactionTrace(log *log.Entry, blockNum func (processor *ShipProcessor) proccessActionTrace(logger *log.Entry, trace *ship.ActionTraceV1) *message.ActionTrace { // Check if actions updates an abi. if trace.Act.Account == processor.syscontract && trace.Act.Name == chain.N("setabi") { + + logger.WithFields(log.Fields{ + "contract": trace.Act.Account, + "action": trace.Act.Name, + }).Debug("Update contract ABI") + err := processor.updateAbiFromAction(&trace.Act) if err != nil { logger.WithError(err).Warn("Failed to update abi") @@ -194,6 +200,11 @@ func (processor *ShipProcessor) proccessActionTrace(logger *log.Entry, trace *sh }) } + logger.WithFields(log.Fields{ + "contract": trace.Act.Account, + "action": trace.Act.Name, + }).Debug("Reading contract ABI") + ABI, err := processor.abi.GetAbi(trace.Act.Account) if err == nil { if act.Data, err = trace.Act.Decode(ABI); err != nil { From a15cf5037740ebcefabbf878fc3b555ddf46f6ff Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Tue, 2 Jul 2024 16:51:39 +0200 Subject: [PATCH 207/291] gomod: update eosswedenorg-go/antelope-ship-client to v0.3.0 --- go.mod | 3 ++- go.sum | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 136cacb..ea97d8b 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.20 require ( github.com/cenkalti/backoff/v4 v4.2.1 github.com/docker/go-units v0.5.0 - github.com/eosswedenorg-go/antelope-ship-client v0.2.9 + github.com/eosswedenorg-go/antelope-ship-client v0.3.0 github.com/eosswedenorg-go/pid v1.0.1 github.com/eosswedenorg/thalos/api v1.0.0 github.com/go-redis/cache/v9 v9.0.0 @@ -47,6 +47,7 @@ require ( github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect + github.com/shufflingpixels/antelope-go v0.1.0 // indirect github.com/shufflingpixels/jsontime-go v0.0.0-20240622163621-cf4b2804c92d // indirect github.com/sourcegraph/conc v0.3.0 // indirect github.com/spf13/afero v1.11.0 // indirect diff --git a/go.sum b/go.sum index b84d15d..f939621 100644 --- a/go.sum +++ b/go.sum @@ -21,8 +21,8 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/r github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/eosswedenorg-go/antelope-ship-client v0.2.9 h1:Isa90Tktdumj/P9XgQ8kBRqS+2MAdcPl7ev/n0KFvUQ= -github.com/eosswedenorg-go/antelope-ship-client v0.2.9/go.mod h1:a0Kp4BJID8DMD0pmHnK5/gojtpRhh2pqqgt4Hcy5Wrw= +github.com/eosswedenorg-go/antelope-ship-client v0.3.0 h1:L1DTE2ZeOGSbehCAqVF5GhdaAb3ZYgvFFUhpDWmJ2vQ= +github.com/eosswedenorg-go/antelope-ship-client v0.3.0/go.mod h1:F3nAzlbcANY6zvZ+cTJH9WjJtvJU6hvduBAUAjX2/lA= github.com/eosswedenorg-go/pid v1.0.1 h1:W4AEnnNwb041SpNR1uTZ/KbJ0OTA5eqiqIR1Q5Ah6A0= github.com/eosswedenorg-go/pid v1.0.1/go.mod h1:wiOB/JXGt4YA3+T0j0xmCGSc3Jxzb7Ti/Ftli1fgWu4= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= @@ -148,6 +148,8 @@ github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6ke github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= +github.com/shufflingpixels/antelope-go v0.1.0 h1:nkW6G1TgYUSJdNKlJ9Pk86TvYGAbga+rBSpNnmQbcKY= +github.com/shufflingpixels/antelope-go v0.1.0/go.mod h1:Bi+zmFI5s7Qv2hx3v9FAcDyf4N4MUs5zaKNa2CetmSg= github.com/shufflingpixels/jsontime-go v0.0.0-20240622163621-cf4b2804c92d h1:nju7jR1Kf210tArPT6l//HlfLLFnvje1BWl5TSRsohQ= github.com/shufflingpixels/jsontime-go v0.0.0-20240622163621-cf4b2804c92d/go.mod h1:W0TaKyg3kDqWmFUxlax3qAls/lRdo12EseM6f4f0dzE= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= From 71f9d4d789cb7b8458ee64d7dd9969071699d649 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Tue, 2 Jul 2024 16:53:41 +0200 Subject: [PATCH 208/291] gomod: update pnx/antelope-go v0.0.4 to shufflingpixels/antelope-go v0.1.0 --- cmd/thalos/server.go | 2 +- go.mod | 3 +-- go.sum | 2 -- internal/abi/manager.go | 4 ++-- internal/abi/manager_test.go | 4 ++-- internal/server/helpers.go | 2 +- internal/server/ship_processor.go | 4 ++-- 7 files changed, 9 insertions(+), 12 deletions(-) diff --git a/cmd/thalos/server.go b/cmd/thalos/server.go index edfc960..2aed773 100644 --- a/cmd/thalos/server.go +++ b/cmd/thalos/server.go @@ -29,8 +29,8 @@ import ( redis_cache "github.com/go-redis/cache/v9" "github.com/nikoksr/notify" "github.com/nikoksr/notify/service/telegram" - antelopeapi "github.com/pnx/antelope-go/api" "github.com/redis/go-redis/v9" + antelopeapi "github.com/shufflingpixels/antelope-go/api" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/spf13/pflag" diff --git a/go.mod b/go.mod index ea97d8b..888bc90 100644 --- a/go.mod +++ b/go.mod @@ -12,8 +12,8 @@ require ( github.com/go-redis/redismock/v9 v9.2.0 github.com/mitchellh/mapstructure v1.5.0 github.com/nikoksr/notify v0.41.0 - github.com/pnx/antelope-go v0.0.4 github.com/redis/go-redis/v9 v9.5.1 + github.com/shufflingpixels/antelope-go v0.1.0 github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.8.0 github.com/spf13/pflag v1.0.5 @@ -47,7 +47,6 @@ require ( github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect - github.com/shufflingpixels/antelope-go v0.1.0 // indirect github.com/shufflingpixels/jsontime-go v0.0.0-20240622163621-cf4b2804c92d // indirect github.com/sourcegraph/conc v0.3.0 // indirect github.com/spf13/afero v1.11.0 // indirect diff --git a/go.sum b/go.sum index f939621..b6679e0 100644 --- a/go.sum +++ b/go.sum @@ -136,8 +136,6 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/pnx/antelope-go v0.0.4 h1:7qZUNfLSSAhlOI1qNwu+9X0lyttK2eGlmmYOj3u8uwI= -github.com/pnx/antelope-go v0.0.4/go.mod h1:NCCgAR1/Yzh6DigwtFFMcMhG6kGzfNx5sUorUMpBuNc= github.com/redis/go-redis/v9 v9.0.0-rc.4/go.mod h1:Vo3EsyWnicKnSKCA7HhgnvnyA74wOA69Cd2Meli5mmA= github.com/redis/go-redis/v9 v9.5.1 h1:H1X4D3yHPaYrkL5X06Wh6xNVM/pX0Ft4RV0vMGvLBh8= github.com/redis/go-redis/v9 v9.5.1/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= diff --git a/internal/abi/manager.go b/internal/abi/manager.go index c8bbe83..8d3acc3 100644 --- a/internal/abi/manager.go +++ b/internal/abi/manager.go @@ -6,8 +6,8 @@ import ( "time" "github.com/eosswedenorg/thalos/internal/cache" - "github.com/pnx/antelope-go/api" - "github.com/pnx/antelope-go/chain" + "github.com/shufflingpixels/antelope-go/api" + "github.com/shufflingpixels/antelope-go/chain" ) // AbiManager handles an ABI cache that fetches the ABI from an API on cache miss. diff --git a/internal/abi/manager_test.go b/internal/abi/manager_test.go index f44b9f7..cbf5893 100644 --- a/internal/abi/manager_test.go +++ b/internal/abi/manager_test.go @@ -7,8 +7,8 @@ import ( "net/http/httptest" "testing" - "github.com/pnx/antelope-go/api" - "github.com/pnx/antelope-go/chain" + "github.com/shufflingpixels/antelope-go/api" + "github.com/shufflingpixels/antelope-go/chain" "github.com/eosswedenorg/thalos/internal/cache" "github.com/stretchr/testify/assert" diff --git a/internal/server/helpers.go b/internal/server/helpers.go index e35bb92..05a2807 100644 --- a/internal/server/helpers.go +++ b/internal/server/helpers.go @@ -4,7 +4,7 @@ import ( "fmt" "reflect" - "github.com/pnx/antelope-go/ship" + "github.com/shufflingpixels/antelope-go/ship" ) // convert a ActionTrace to ActionTraceV1 diff --git a/internal/server/ship_processor.go b/internal/server/ship_processor.go index edd33bd..6b40114 100644 --- a/internal/server/ship_processor.go +++ b/internal/server/ship_processor.go @@ -12,8 +12,8 @@ import ( log "github.com/sirupsen/logrus" shipclient "github.com/eosswedenorg-go/antelope-ship-client" - "github.com/pnx/antelope-go/chain" - "github.com/pnx/antelope-go/ship" + "github.com/shufflingpixels/antelope-go/chain" + "github.com/shufflingpixels/antelope-go/ship" ) // A ShipProcessor will consume messages from a ship stream, convert the messages into From 0e124bb86606f8aedad6247a0e8d849d7459f6e2 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Tue, 2 Jul 2024 18:59:41 +0200 Subject: [PATCH 209/291] gomod: update shufflingpixels/antelope-go from v0.1.0 to v0.1.1 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 888bc90..aa2f28b 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/mitchellh/mapstructure v1.5.0 github.com/nikoksr/notify v0.41.0 github.com/redis/go-redis/v9 v9.5.1 - github.com/shufflingpixels/antelope-go v0.1.0 + github.com/shufflingpixels/antelope-go v0.1.1 github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.8.0 github.com/spf13/pflag v1.0.5 diff --git a/go.sum b/go.sum index b6679e0..8e7a473 100644 --- a/go.sum +++ b/go.sum @@ -146,8 +146,8 @@ github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6ke github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= -github.com/shufflingpixels/antelope-go v0.1.0 h1:nkW6G1TgYUSJdNKlJ9Pk86TvYGAbga+rBSpNnmQbcKY= -github.com/shufflingpixels/antelope-go v0.1.0/go.mod h1:Bi+zmFI5s7Qv2hx3v9FAcDyf4N4MUs5zaKNa2CetmSg= +github.com/shufflingpixels/antelope-go v0.1.1 h1:iNq5cjrApsRF4zJRg0IiDl9b1wx1dq4HIWN6GOZ8JC0= +github.com/shufflingpixels/antelope-go v0.1.1/go.mod h1:Bi+zmFI5s7Qv2hx3v9FAcDyf4N4MUs5zaKNa2CetmSg= github.com/shufflingpixels/jsontime-go v0.0.0-20240622163621-cf4b2804c92d h1:nju7jR1Kf210tArPT6l//HlfLLFnvje1BWl5TSRsohQ= github.com/shufflingpixels/jsontime-go v0.0.0-20240622163621-cf4b2804c92d/go.mod h1:W0TaKyg3kDqWmFUxlax3qAls/lRdo12EseM6f4f0dzE= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= From e0f598eba44259916a2dae5645d828d059c0a5f3 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Tue, 2 Jul 2024 19:46:24 +0200 Subject: [PATCH 210/291] internal/server/ship_processor.go: in updateAbiFromAction() fix field order in set_abi struct --- internal/server/ship_processor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/server/ship_processor.go b/internal/server/ship_processor.go index 6b40114..bc086cf 100644 --- a/internal/server/ship_processor.go +++ b/internal/server/ship_processor.go @@ -77,8 +77,8 @@ func (processor *ShipProcessor) initHandler(abi *chain.Abi) { // updateAbiFromAction updates the contract abi based on the ship.Action passed. func (processor *ShipProcessor) updateAbiFromAction(act *chain.Action) error { set_abi := struct { - Abi string Account chain.Name + Abi string }{} if err := act.DecodeInto(&set_abi); err != nil { From 1a782604d37837b45c7b772a077cc1560b02ec1d Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Wed, 3 Jul 2024 17:59:24 +0200 Subject: [PATCH 211/291] gomod: update shufflingpixels/antelope-go to v0.1.2 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index aa2f28b..ff4c34f 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/mitchellh/mapstructure v1.5.0 github.com/nikoksr/notify v0.41.0 github.com/redis/go-redis/v9 v9.5.1 - github.com/shufflingpixels/antelope-go v0.1.1 + github.com/shufflingpixels/antelope-go v0.1.2 github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.8.0 github.com/spf13/pflag v1.0.5 diff --git a/go.sum b/go.sum index 8e7a473..3e977f1 100644 --- a/go.sum +++ b/go.sum @@ -146,8 +146,8 @@ github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6ke github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= -github.com/shufflingpixels/antelope-go v0.1.1 h1:iNq5cjrApsRF4zJRg0IiDl9b1wx1dq4HIWN6GOZ8JC0= -github.com/shufflingpixels/antelope-go v0.1.1/go.mod h1:Bi+zmFI5s7Qv2hx3v9FAcDyf4N4MUs5zaKNa2CetmSg= +github.com/shufflingpixels/antelope-go v0.1.2 h1:XXrNZ7KWGRyaEE2igaEax/N7OxPzkKcyAepfun/BfMc= +github.com/shufflingpixels/antelope-go v0.1.2/go.mod h1:Bi+zmFI5s7Qv2hx3v9FAcDyf4N4MUs5zaKNa2CetmSg= github.com/shufflingpixels/jsontime-go v0.0.0-20240622163621-cf4b2804c92d h1:nju7jR1Kf210tArPT6l//HlfLLFnvje1BWl5TSRsohQ= github.com/shufflingpixels/jsontime-go v0.0.0-20240622163621-cf4b2804c92d/go.mod h1:W0TaKyg3kDqWmFUxlax3qAls/lRdo12EseM6f4f0dzE= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= From ef6329d1b7cb901ceeb234a87ba1a9c0c5578a11 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Wed, 3 Jul 2024 18:01:31 +0200 Subject: [PATCH 212/291] internal/server/ship_processor.go: updateAbiFromAction() should not hex decode abi as it is in binary format only when encoding the action to json it is in hex, but from ship = always binary --- internal/server/ship_processor.go | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/internal/server/ship_processor.go b/internal/server/ship_processor.go index bc086cf..38ef73f 100644 --- a/internal/server/ship_processor.go +++ b/internal/server/ship_processor.go @@ -2,7 +2,6 @@ package server import ( "bytes" - "encoding/hex" "github.com/eosswedenorg/thalos/api/message" "github.com/eosswedenorg/thalos/internal/abi" @@ -78,24 +77,19 @@ func (processor *ShipProcessor) initHandler(abi *chain.Abi) { func (processor *ShipProcessor) updateAbiFromAction(act *chain.Action) error { set_abi := struct { Account chain.Name - Abi string + Abi chain.Bytes }{} if err := act.DecodeInto(&set_abi); err != nil { return err } - binary_abi, err := hex.DecodeString(set_abi.Abi) - if err != nil { + abi := chain.Abi{} + decoder := chain.NewDecoder(bytes.NewReader(set_abi.Abi)) + if err := decoder.Decode(&abi); err != nil { return err } - - contract_abi := chain.Abi{} - err = chain.NewDecoder(bytes.NewReader(binary_abi)).Decode(&contract_abi) - if err != nil { - return err - } - return processor.abi.SetAbi(set_abi.Account, &contract_abi) + return processor.abi.SetAbi(set_abi.Account, &abi) } // Get the current block. From a9a129e418f14dd0613e20b908fc5fd359fcfc7f Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Wed, 3 Jul 2024 18:16:14 +0200 Subject: [PATCH 213/291] internal/abi/manager_test.go: update action and table names to chain.Name types --- internal/abi/manager_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/abi/manager_test.go b/internal/abi/manager_test.go index cbf5893..e7fdfca 100644 --- a/internal/abi/manager_test.go +++ b/internal/abi/manager_test.go @@ -111,12 +111,12 @@ func assert_abi(t *testing.T, abi *chain.Abi) { assert.Equal(t, abi.Structs[3].Fields[0].Type, "string") // Actions - assert.Equal(t, abi.Actions[0].Name, "action_name_1") + assert.Equal(t, abi.Actions[0].Name, chain.N("action_name_1")) assert.Equal(t, abi.Actions[0].Type, "struct_name_1") assert.Equal(t, abi.Actions[0].RicardianContract, "") // Tables - assert.Equal(t, abi.Tables[0].Name, "table_name_1") + assert.Equal(t, abi.Tables[0].Name, chain.N("table_name_1")) assert.Equal(t, abi.Tables[0].Type, "struct_name_1") assert.Equal(t, abi.Tables[0].IndexType, "i64") assert.Equal(t, abi.Tables[0].KeyNames[0], "key_name_1") From e2443dcd27611de1a616e53fcd23265e0e75c6f7 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Wed, 3 Jul 2024 18:17:27 +0200 Subject: [PATCH 214/291] Version 1.1.3 --- Makefile | 2 +- debian/changelog | 11 +++++++++++ docker/Dockerfile | 2 +- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 5c629de..4fce728 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ GO=go GOLDFLAGS=-v -s -w -X main.VersionString=$(PROGRAM_VERSION) GOBUILDFLAGS=-v -p $(shell nproc) -ldflags="$(GOLDFLAGS)" PROGRAM=thalos-server -PROGRAM_VERSION=1.1.2 +PROGRAM_VERSION=1.1.3 PREFIX=/usr/local BINDIR=$(PREFIX)/bin CFGDIR=$(PREFIX)/etc/thalos diff --git a/debian/changelog b/debian/changelog index e1230f3..595f2ca 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,14 @@ +thalos (1.1.3) bionic focal jammy; urgency=medium + + * Updated antelope-go library to v0.1.2 that fixes a bug in abi binary + decoder, it expects some fields to be strings while they are "names" + (strings encoded into a int64) + * Fix a bug with "set_abi" struct had the wrong order of fields in ShipProcessor.updateAbiFromAction() + * Fix a bug in ShipProcessor.updateAbiFromAction() that assumed the abi + was in hex format when in fact it is binary. + + -- Henrik Hautakoski Wed, 03 Jul 2024 18:05:33 +0200 + thalos (1.1.2) bionic focal jammy; urgency=medium * API: Fix a bug regarding json timestamp being encoded/decoded with wrong diff --git a/docker/Dockerfile b/docker/Dockerfile index 5c5fd4d..a4e7fde 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,6 +1,6 @@ FROM alpine:latest LABEL maintainer="Henrik Hautakoski " -ARG VERSION=1.1.2 +ARG VERSION=1.1.3 WORKDIR /thalos ADD --chmod=755 https://github.com/eosswedenorg/thalos/releases/download/v$VERSION/thalos-server-${VERSION}-linux-amd64-musl thalos-server ENTRYPOINT [ "./thalos-server" ] From cbd3196cf92139525c310056097ce0c71d8e1d44 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sat, 13 Jul 2024 15:54:35 +0200 Subject: [PATCH 215/291] internal/types/blacklist.go: change Lookup to IsAllowed and add IsDenied --- internal/server/ship_processor.go | 2 +- internal/types/blacklist.go | 10 +++++++--- internal/types/blacklist_test.go | 20 ++++++++++---------- 3 files changed, 18 insertions(+), 14 deletions(-) diff --git a/internal/server/ship_processor.go b/internal/server/ship_processor.go index 38ef73f..7106f4f 100644 --- a/internal/server/ship_processor.go +++ b/internal/server/ship_processor.go @@ -153,7 +153,7 @@ func (processor *ShipProcessor) proccessActionTrace(logger *log.Entry, trace *sh } // Check blacklist if we should skip this action - if processor.blacklist.Lookup(trace.Act.Account.String(), trace.Act.Name.String()) { + if !processor.blacklist.IsAllowed(trace.Act.Account.String(), trace.Act.Name.String()) { logger.WithFields(log.Fields{ "contract": trace.Act.Account, "action": trace.Act.Name, diff --git a/internal/types/blacklist.go b/internal/types/blacklist.go index cbe28b5..2b0812f 100644 --- a/internal/types/blacklist.go +++ b/internal/types/blacklist.go @@ -9,13 +9,17 @@ func (bl Blacklist) Add(contract string, action string) { bl[contract] = append(bl[contract], action) } -func (bl Blacklist) Lookup(contract string, action string) bool { +func (bl Blacklist) IsAllowed(contract string, action string) bool { if v, ok := bl[contract]; ok { for _, act := range v { if act == action || act == "*" { - return true + return false } } } - return false + return true +} + +func (bl Blacklist) IsDenied(contract string, action string) bool { + return bl.IsAllowed(contract, action) } diff --git a/internal/types/blacklist_test.go b/internal/types/blacklist_test.go index a897802..f156e78 100644 --- a/internal/types/blacklist_test.go +++ b/internal/types/blacklist_test.go @@ -20,24 +20,24 @@ func TestBlacklist_Add(t *testing.T) { require.Equal(t, expected, bl) } -func TestBlacklist_Lookup(t *testing.T) { +func TestBlacklist_IsAllowed(t *testing.T) { bl := Blacklist{ "mycontract": {"myaction", "noop"}, } - require.True(t, bl.Lookup("mycontract", "myaction")) - require.True(t, bl.Lookup("mycontract", "noop")) - require.False(t, bl.Lookup("mycontract", "xxx")) - require.False(t, bl.Lookup("xxx", "yyy")) + require.False(t, bl.IsAllowed("mycontract", "myaction")) + require.False(t, bl.IsAllowed("mycontract", "noop")) + require.True(t, bl.IsAllowed("mycontract", "xxx")) + require.True(t, bl.IsAllowed("xxx", "yyy")) } -func TestBlacklist_LookupWildcard(t *testing.T) { +func TestBlacklist_IsAllowedWildcard(t *testing.T) { bl := Blacklist{ "mycontract": {"*"}, } - require.True(t, bl.Lookup("mycontract", "myaction")) - require.True(t, bl.Lookup("mycontract", "noop")) - require.True(t, bl.Lookup("mycontract", "xxx")) - require.False(t, bl.Lookup("xxx", "yyy")) + require.False(t, bl.IsAllowed("mycontract", "myaction")) + require.False(t, bl.IsAllowed("mycontract", "noop")) + require.False(t, bl.IsAllowed("mycontract", "xxx")) + require.True(t, bl.IsAllowed("xxx", "yyy")) } From 0aee0a97aaddd2b055dcb312088e4cd6c2e03bc7 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Mon, 15 Jul 2024 22:57:05 +0200 Subject: [PATCH 216/291] internal/types/blacklist.go: adding Empty() --- internal/types/blacklist.go | 4 ++++ internal/types/blacklist_test.go | 10 ++++++++++ 2 files changed, 14 insertions(+) diff --git a/internal/types/blacklist.go b/internal/types/blacklist.go index 2b0812f..12ee19d 100644 --- a/internal/types/blacklist.go +++ b/internal/types/blacklist.go @@ -2,6 +2,10 @@ package types type Blacklist map[string][]string +func (bl Blacklist) Empty() bool { + return len(bl) < 1 +} + func (bl Blacklist) Add(contract string, action string) { if len(bl[contract]) < 1 { bl[contract] = []string{} diff --git a/internal/types/blacklist_test.go b/internal/types/blacklist_test.go index f156e78..1cbcbfe 100644 --- a/internal/types/blacklist_test.go +++ b/internal/types/blacklist_test.go @@ -6,6 +6,16 @@ import ( "github.com/stretchr/testify/require" ) +func TestBlacklist_Empty(t *testing.T) { + bl := Blacklist{} + + require.True(t, bl.Empty()) + + bl.Add("contract", "action1") + + require.False(t, bl.Empty()) +} + func TestBlacklist_Add(t *testing.T) { bl := Blacklist{} bl.Add("contract", "action1") From 4f27307c702b8af56c825e0f6cc27ef8946d0087 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Mon, 15 Jul 2024 23:02:29 +0200 Subject: [PATCH 217/291] internal/types/blacklist.go: add isWhitelist field --- internal/config/builder.go | 83 +++++++++++++++++++++++--------- internal/config/builder_test.go | 44 ++++++++++------- internal/types/blacklist.go | 40 ++++++++++----- internal/types/blacklist_test.go | 37 +++++++++++--- 4 files changed, 146 insertions(+), 58 deletions(-) diff --git a/internal/config/builder.go b/internal/config/builder.go index 026dd33..d273858 100644 --- a/internal/config/builder.go +++ b/internal/config/builder.go @@ -116,30 +116,9 @@ func (b *Builder) Build() (*Config, error) { mapstructure.StringToTimeDurationHookFunc(), mapstructure.StringToSliceHookFunc(","), func(f reflect.Type, t reflect.Type, in interface{}) (interface{}, error) { - if t == reflect.TypeOf(types.Blacklist{}) && f.Kind() == reflect.Slice { - if v, ok := in.([]string); ok { - list := types.Blacklist{} - for _, i := range v { - var action string - parts := strings.SplitN(i, ":", 2) - - if len(parts) < 2 { - action = "*" - } else { - action = parts[1] - } - - list.Add(parts[0], action) - } - - if len(list) < 1 { - list = nil - } - return list, nil - } - return nil, fmt.Errorf("Must be a string slice") + if t == reflect.TypeOf(types.Blacklist{}) { + return decodeIntoBlacklist(in) } - return in, nil }, ) @@ -151,3 +130,61 @@ func (b *Builder) Build() (*Config, error) { return &conf, nil } + +// Decode a generic structure into types.Blacklist +func decodeIntoBlacklist(in any) (*types.Blacklist, error) { + switch v := in.(type) { + // Standard map structure. + case map[string]any: + return blacklistParseMap(v) + + // slice of "contract:action" pairs. Usually from CLI + case []string: + return blacklistParseSlice(v) + + // Sometimes we have a slice of interfaces. + // Need to convert it to a slice of strings. + case []any: + sv := make([]string, len(v)) + for i, j := range v { + sv[i] = j.(string) + } + return blacklistParseSlice(sv) + } + + return nil, fmt.Errorf("Must be a string slice") +} + +// Blacklist map parser +func blacklistParseMap(in map[string]any) (*types.Blacklist, error) { + list := &types.Blacklist{} + for k, v := range in { + switch v := v.(type) { + case []any: + for _, v := range v { + list.Add(k, v.(string)) + } + case any: + list.Add(k, v.(string)) + } + } + return list, nil +} + +// Blacklist slice parser +func blacklistParseSlice(in []string) (*types.Blacklist, error) { + list := &types.Blacklist{} + for _, i := range in { + var action string + parts := strings.SplitN(i, ":", 2) + + if len(parts) < 2 { + action = "*" + } else { + action = parts[1] + } + + list.Add(parts[0], action) + } + return list, nil +} diff --git a/internal/config/builder_test.go b/internal/config/builder_test.go index d226a30..bec8178 100644 --- a/internal/config/builder_test.go +++ b/internal/config/builder_test.go @@ -28,10 +28,10 @@ func TestBuilder(t *testing.T) { EndBlockNum: 23872222, IrreversibleOnly: true, MaxMessagesInFlight: 1337, - Blacklist: types.Blacklist{ + Blacklist: *types.NewBlacklist(map[string][]string{ "eosio": {"noop"}, "contract": {"skip1", "skip2"}, - }, + }), }, Telegram: TelegramConfig{ Id: "110201543:AAHdqTcvCH1vGWJxfSeofSAs0K5PALDsaw", @@ -64,7 +64,7 @@ ship: start_block_num: 23671836 end_block_num: 23872222 blacklist: - eosio: ["noop"] + eosio: noop contract: - skip1 - skip2 @@ -207,10 +207,10 @@ func TestBuilder_Flags(t *testing.T) { MaxMessagesInFlight: 98, IrreversibleOnly: true, Chain: "wax", - Blacklist: types.Blacklist{ + Blacklist: *types.NewBlacklist(map[string][]string{ "contract": {"action1", "action2"}, "contract2": {"action1"}, - }, + }), }, Telegram: TelegramConfig{ Id: "72983126312982618", @@ -229,20 +229,28 @@ func TestBuilder_Flags(t *testing.T) { require.Equal(t, &expected, cfg) } -func TestBuilder_BlacklistFlag(t *testing.T) { - flags := GetFlags() - - require.NoError(t, flags.Set("blacklist", "contract,contract:action2")) - - conf, err := NewBuilder(). - SetSource(bytes.NewReader([]byte(``))). - SetFlags(flags). - Build() - - expected := types.Blacklist{ - "contract": {"*", "action2"}, +func TestBuilder_BlacklistSlice(t *testing.T) { + expected := Config{ + Ship: ShipConfig{ + Blacklist: *types.NewBlacklist(map[string][]string{ + "contract": {"action"}, + "contract2": {"action2"}, + "contract3": {"*"}, + }), + }, } + builder := NewBuilder() + builder.SetSource(bytes.NewBuffer([]byte(` +ship: + blacklist: + - "contract:action" + - "contract2:action2" + - contract3 +`))) + + cfg, err := builder.Build() + require.NoError(t, err) - require.Equal(t, expected, conf.Ship.Blacklist) + require.Equal(t, &expected, cfg) } diff --git a/internal/types/blacklist.go b/internal/types/blacklist.go index 12ee19d..24419c8 100644 --- a/internal/types/blacklist.go +++ b/internal/types/blacklist.go @@ -1,27 +1,45 @@ package types -type Blacklist map[string][]string - -func (bl Blacklist) Empty() bool { - return len(bl) < 1 +type Blacklist struct { + table map[string][]string + isWhitelist bool } -func (bl Blacklist) Add(contract string, action string) { - if len(bl[contract]) < 1 { - bl[contract] = []string{} +func NewBlacklist(entries map[string][]string) *Blacklist { + return &Blacklist{ + table: entries, } - bl[contract] = append(bl[contract], action) +} + +func (bl *Blacklist) SetWhitelist(value bool) *Blacklist { + bl.isWhitelist = value + return bl +} + +func (bl Blacklist) Empty() bool { + return len(bl.table) < 1 +} + +func (bl *Blacklist) Add(contract string, action string) { + if bl.table == nil { + bl.table = map[string][]string{} + } + + if len(bl.table[contract]) < 1 { + bl.table[contract] = []string{} + } + bl.table[contract] = append(bl.table[contract], action) } func (bl Blacklist) IsAllowed(contract string, action string) bool { - if v, ok := bl[contract]; ok { + if v, ok := bl.table[contract]; ok { for _, act := range v { if act == action || act == "*" { - return false + return bl.isWhitelist == true } } } - return true + return bl.isWhitelist == false } func (bl Blacklist) IsDenied(contract string, action string) bool { diff --git a/internal/types/blacklist_test.go b/internal/types/blacklist_test.go index 1cbcbfe..631d329 100644 --- a/internal/types/blacklist_test.go +++ b/internal/types/blacklist_test.go @@ -7,7 +7,9 @@ import ( ) func TestBlacklist_Empty(t *testing.T) { - bl := Blacklist{} + bl := Blacklist{ + table: map[string][]string{}, + } require.True(t, bl.Empty()) @@ -17,14 +19,18 @@ func TestBlacklist_Empty(t *testing.T) { } func TestBlacklist_Add(t *testing.T) { - bl := Blacklist{} + bl := Blacklist{ + table: map[string][]string{}, + } bl.Add("contract", "action1") bl.Add("contract", "action2") bl.Add("contract2", "action1") expected := Blacklist{ - "contract": {"action1", "action2"}, - "contract2": {"action1"}, + table: map[string][]string{ + "contract": {"action1", "action2"}, + "contract2": {"action1"}, + }, } require.Equal(t, expected, bl) @@ -32,7 +38,9 @@ func TestBlacklist_Add(t *testing.T) { func TestBlacklist_IsAllowed(t *testing.T) { bl := Blacklist{ - "mycontract": {"myaction", "noop"}, + table: map[string][]string{ + "mycontract": {"myaction", "noop"}, + }, } require.False(t, bl.IsAllowed("mycontract", "myaction")) @@ -43,7 +51,9 @@ func TestBlacklist_IsAllowed(t *testing.T) { func TestBlacklist_IsAllowedWildcard(t *testing.T) { bl := Blacklist{ - "mycontract": {"*"}, + table: map[string][]string{ + "mycontract": {"*"}, + }, } require.False(t, bl.IsAllowed("mycontract", "myaction")) @@ -51,3 +61,18 @@ func TestBlacklist_IsAllowedWildcard(t *testing.T) { require.False(t, bl.IsAllowed("mycontract", "xxx")) require.True(t, bl.IsAllowed("xxx", "yyy")) } + +func TestBlacklist_Whitelist(t *testing.T) { + bl := Blacklist{ + table: map[string][]string{ + "mycontract": {"myaction", "noop"}, + }, + } + + bl.SetWhitelist(true) + + require.True(t, bl.IsAllowed("mycontract", "myaction")) + require.True(t, bl.IsAllowed("mycontract", "noop")) + require.False(t, bl.IsAllowed("mycontract", "xxx")) + require.False(t, bl.IsAllowed("xxx", "yyy")) +} From 20168a9329fe4a4c24a7e1fa66a23233c8b4356b Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Tue, 16 Jul 2024 16:59:01 +0200 Subject: [PATCH 218/291] implement whitelist option in blacklist --- cmd/thalos/server.go | 2 ++ config.example.yml | 2 ++ internal/config/builder.go | 1 + internal/config/builder_test.go | 4 ++++ internal/config/cli.go | 1 + internal/config/config.go | 15 ++++++++------- 6 files changed, 18 insertions(+), 7 deletions(-) diff --git a/cmd/thalos/server.go b/cmd/thalos/server.go index 2aed773..0fa0c4a 100644 --- a/cmd/thalos/server.go +++ b/cmd/thalos/server.go @@ -238,6 +238,8 @@ func GetConfig(flags *pflag.FlagSet) (*config.Config, error) { } } + cfg.Ship.Blacklist.SetWhitelist(cfg.Ship.BlacklistIsWhitelist) + return cfg, nil } diff --git a/config.example.yml b/config.example.yml index 84990b1..9d6deef 100644 --- a/config.example.yml +++ b/config.example.yml @@ -54,6 +54,8 @@ ship: # blacklist all action from a contract # evilcontract: ["*"] + # blacklist_is_whitelist: true + # Telegram notifications #telegram: # id: "123456789:GPdmGPBWvpgHPxlergJLavus-PoAURTjMWP" diff --git a/internal/config/builder.go b/internal/config/builder.go index d273858..2967f63 100644 --- a/internal/config/builder.go +++ b/internal/config/builder.go @@ -53,6 +53,7 @@ func NewBuilder() *Builder { "ship.max_messages_in_flight": "max-msg-in-flight", "ship.chain": "chain", "ship.blacklist": "blacklist", + "ship.blacklist_is_whitelist": "blacklist-is-whitelist", }, } } diff --git a/internal/config/builder_test.go b/internal/config/builder_test.go index bec8178..df24552 100644 --- a/internal/config/builder_test.go +++ b/internal/config/builder_test.go @@ -32,6 +32,7 @@ func TestBuilder(t *testing.T) { "eosio": {"noop"}, "contract": {"skip1", "skip2"}, }), + BlacklistIsWhitelist: true, }, Telegram: TelegramConfig{ Id: "110201543:AAHdqTcvCH1vGWJxfSeofSAs0K5PALDsaw", @@ -68,6 +69,7 @@ ship: contract: - skip1 - skip2 + blacklist_is_whitelist: true telegram: id: "110201543:AAHdqTcvCH1vGWJxfSeofSAs0K5PALDsaw" channel: -123456789 @@ -186,6 +188,7 @@ func TestBuilder_Flags(t *testing.T) { require.NoError(t, flags.Set("max-msg-in-flight", "98")) require.NoError(t, flags.Set("chain", "wax")) require.NoError(t, flags.Set("blacklist", "contract:action1,contract:action2,contract2:action1")) + require.NoError(t, flags.Set("blacklist-is-whitelist", "true")) cfg, err := NewBuilder(). SetSource(bytes.NewReader([]byte(``))). @@ -211,6 +214,7 @@ func TestBuilder_Flags(t *testing.T) { "contract": {"action1", "action2"}, "contract2": {"action1"}, }), + BlacklistIsWhitelist: true, }, Telegram: TelegramConfig{ Id: "72983126312982618", diff --git a/internal/config/cli.go b/internal/config/cli.go index 38439a5..85d8f39 100644 --- a/internal/config/cli.go +++ b/internal/config/cli.go @@ -44,6 +44,7 @@ func GetFlags() *pflag.FlagSet { flags.String("chain", "", "ChainID used in channel namespace, can be any string (default from api)") flags.StringSlice("blacklist", []string{}, "Define a list of 'contract:action' pairs that will be blacklisted (thalos will not process those actions)") + flags.Bool("blacklist-is-whitelist", false, "Thalos will treat the blacklist as a whitelist") return &flags } diff --git a/internal/config/config.go b/internal/config/config.go index 5e84072..7bb8b4b 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -19,13 +19,14 @@ type TelegramConfig struct { } type ShipConfig struct { - Url string `yaml:"url" mapstructure:"url"` - IrreversibleOnly bool `yaml:"irreversible_only" mapstructure:"irreversible_only"` - MaxMessagesInFlight uint32 `yaml:"max_messages_in_flight" mapstructure:"max_messages_in_flight"` - StartBlockNum uint32 `yaml:"start_block_num" mapstructure:"start_block_num"` - EndBlockNum uint32 `yaml:"end_block_num" mapstructure:"end_block_num"` - Chain string `yaml:"chain" mapstructure:"chain"` - Blacklist types.Blacklist `yaml:"blacklist" mapstructure:"blacklist"` + Url string `yaml:"url" mapstructure:"url"` + IrreversibleOnly bool `yaml:"irreversible_only" mapstructure:"irreversible_only"` + MaxMessagesInFlight uint32 `yaml:"max_messages_in_flight" mapstructure:"max_messages_in_flight"` + StartBlockNum uint32 `yaml:"start_block_num" mapstructure:"start_block_num"` + EndBlockNum uint32 `yaml:"end_block_num" mapstructure:"end_block_num"` + Chain string `yaml:"chain" mapstructure:"chain"` + Blacklist types.Blacklist `yaml:"blacklist" mapstructure:"blacklist"` + BlacklistIsWhitelist bool `yaml:"blacklist_is_whitelist" mapstructure:"blacklist_is_whitelist"` } type Config struct { From d1a6d038a3739cd295237e8e90444b297da34107 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Tue, 16 Jul 2024 17:13:18 +0200 Subject: [PATCH 219/291] .github/workflows/test.yml: run tests on all supported os/arch pairs --- .github/workflows/test.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f33a3c4..2e847f8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -11,8 +11,10 @@ jobs: fail-fast: false matrix: go-version: ["1.20", "1.21"] + os: [ linux, freebsd ] + arch: [ 386, amd64, arm, arm64 ] runs-on: ubuntu-latest - name: Test (go v${{ matrix.go-version }}) + name: Test (${{matrix.os}} ${{matrix.arch}} go v${{ matrix.go-version }}) steps: - uses: actions/checkout@v4 @@ -22,10 +24,10 @@ jobs: go-version: ${{ matrix.go-version }} - name: Test - run: go test -v ./... + run: GOOS=${{matrix.os}} GOARCH=${{matrix.arch}} go test -v ./... - name: Test API - run: cd api; go test -v ./... + run: cd api; GOOS=${{matrix.os}} GOARCH=${{matrix.arch}} go test -v ./... test-alpine: strategy: From 5121098cfee46b66e8580a7be34e7edac0eccab2 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Tue, 16 Jul 2024 20:02:00 +0200 Subject: [PATCH 220/291] gomod: update shufflingpixels/antelope-go to v0.1.3 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index ff4c34f..33424c3 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/mitchellh/mapstructure v1.5.0 github.com/nikoksr/notify v0.41.0 github.com/redis/go-redis/v9 v9.5.1 - github.com/shufflingpixels/antelope-go v0.1.2 + github.com/shufflingpixels/antelope-go v0.1.3 github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.8.0 github.com/spf13/pflag v1.0.5 diff --git a/go.sum b/go.sum index 3e977f1..936dfc8 100644 --- a/go.sum +++ b/go.sum @@ -146,8 +146,8 @@ github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6ke github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= -github.com/shufflingpixels/antelope-go v0.1.2 h1:XXrNZ7KWGRyaEE2igaEax/N7OxPzkKcyAepfun/BfMc= -github.com/shufflingpixels/antelope-go v0.1.2/go.mod h1:Bi+zmFI5s7Qv2hx3v9FAcDyf4N4MUs5zaKNa2CetmSg= +github.com/shufflingpixels/antelope-go v0.1.3 h1:uZ0cCn6vitHZTz7UFy5n8AEv1k37MqzCTLIuAnX346Y= +github.com/shufflingpixels/antelope-go v0.1.3/go.mod h1:Bi+zmFI5s7Qv2hx3v9FAcDyf4N4MUs5zaKNa2CetmSg= github.com/shufflingpixels/jsontime-go v0.0.0-20240622163621-cf4b2804c92d h1:nju7jR1Kf210tArPT6l//HlfLLFnvje1BWl5TSRsohQ= github.com/shufflingpixels/jsontime-go v0.0.0-20240622163621-cf4b2804c92d/go.mod h1:W0TaKyg3kDqWmFUxlax3qAls/lRdo12EseM6f4f0dzE= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= From 621f9ceeeaca34d54a4feabc40f70e7a8064a17b Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Tue, 16 Jul 2024 20:44:36 +0200 Subject: [PATCH 221/291] internal/server/helpers_test.go: set large untyped int constants to int64. --- internal/server/helpers_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/server/helpers_test.go b/internal/server/helpers_test.go index 5765146..fdaf8ff 100644 --- a/internal/server/helpers_test.go +++ b/internal/server/helpers_test.go @@ -23,7 +23,7 @@ func TestParseTableDeltaData(t *testing.T) { map[string]interface{}{ "consumed": 8107, "last_ordinal": 308855607, - "value_ex": 3854030492, + "value_ex": int64(3854030492), }, }, "slice": []interface{}{ @@ -51,7 +51,7 @@ func TestParseTableDeltaData(t *testing.T) { "average_block_net_usage": map[string]interface{}{ "consumed": 8107, "last_ordinal": 308855607, - "value_ex": 3854030492, + "value_ex": int64(3854030492), }, "slice": []interface{}{1, 2, "tree"}, "single_value": uint32(12933729), From 8155d49ef4b005fe9517f0cfb04a6000d8841d9c Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Tue, 16 Jul 2024 20:56:03 +0200 Subject: [PATCH 222/291] .github/workflows/test.yml: can't run tests on different arch than host. So just do i386 and amd64 --- .github/workflows/test.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2e847f8..2df9f11 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -11,10 +11,9 @@ jobs: fail-fast: false matrix: go-version: ["1.20", "1.21"] - os: [ linux, freebsd ] - arch: [ 386, amd64, arm, arm64 ] + arch: [ 386, amd64 ] runs-on: ubuntu-latest - name: Test (${{matrix.os}} ${{matrix.arch}} go v${{ matrix.go-version }}) + name: Test (${{matrix.arch}} go v${{ matrix.go-version }}) steps: - uses: actions/checkout@v4 @@ -24,10 +23,10 @@ jobs: go-version: ${{ matrix.go-version }} - name: Test - run: GOOS=${{matrix.os}} GOARCH=${{matrix.arch}} go test -v ./... + run: GOARCH=${{matrix.arch}} go test -v ./... - name: Test API - run: cd api; GOOS=${{matrix.os}} GOARCH=${{matrix.arch}} go test -v ./... + run: cd api; GOARCH=${{matrix.arch}} go test -v ./... test-alpine: strategy: From 8c5815a2ce13d25dbb5fc45b93bea449b900cad0 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Wed, 17 Jul 2024 09:32:35 +0200 Subject: [PATCH 223/291] Version 1.1.4 --- Makefile | 2 +- debian/changelog | 7 +++++++ docker/Dockerfile | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 4fce728..8f312f2 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ GO=go GOLDFLAGS=-v -s -w -X main.VersionString=$(PROGRAM_VERSION) GOBUILDFLAGS=-v -p $(shell nproc) -ldflags="$(GOLDFLAGS)" PROGRAM=thalos-server -PROGRAM_VERSION=1.1.3 +PROGRAM_VERSION=1.1.4 PREFIX=/usr/local BINDIR=$(PREFIX)/bin CFGDIR=$(PREFIX)/etc/thalos diff --git a/debian/changelog b/debian/changelog index 595f2ca..1771e6e 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,10 @@ +thalos (1.1.4) bionic focal jammy; urgency=medium + + * Implement whitelist option for ship contract/action blacklist + * Fix bug with integer overflow on 32 bit CPUs. + + -- Henrik Hautakoski Tue, 16 Jul 2024 21:03:34 +0200 + thalos (1.1.3) bionic focal jammy; urgency=medium * Updated antelope-go library to v0.1.2 that fixes a bug in abi binary diff --git a/docker/Dockerfile b/docker/Dockerfile index a4e7fde..8f55319 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,6 +1,6 @@ FROM alpine:latest LABEL maintainer="Henrik Hautakoski " -ARG VERSION=1.1.3 +ARG VERSION=1.1.4 WORKDIR /thalos ADD --chmod=755 https://github.com/eosswedenorg/thalos/releases/download/v$VERSION/thalos-server-${VERSION}-linux-amd64-musl thalos-server ENTRYPOINT [ "./thalos-server" ] From 22896f8859f41a99449d40b78a0d4e8738031081 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Thu, 18 Jul 2024 21:35:01 +0200 Subject: [PATCH 224/291] internal/config/builder_test.go: rewrite ConfigWithFlags to WithDefaultConfig as we already have a test for flags --- internal/config/builder_test.go | 73 +++++++-------------------------- 1 file changed, 15 insertions(+), 58 deletions(-) diff --git a/internal/config/builder_test.go b/internal/config/builder_test.go index df24552..1de8eb8 100644 --- a/internal/config/builder_test.go +++ b/internal/config/builder_test.go @@ -87,74 +87,31 @@ redis: require.Equal(t, &expected, cfg) } -func TestBuilder_ConfigWithFlags(t *testing.T) { +func TestBuilder_WithDefaultConfig(t *testing.T) { expected := Config{ - Name: "ship-reader-1", - Api: "https://api.example.com", - MessageCodec: "msgpack", + MessageCodec: "json", Log: log.Config{ - Filename: "mylog.log", - Directory: "/var/log", - MaxFileSize: 200, - MaxTime: 30 * time.Minute, + MaxFileSize: 10 * 1000 * 1000, + MaxTime: time.Hour * 24, FileTimestampFormat: "2006-01-02_150405", }, - Ship: ShipConfig{ - Url: "127.0.0.1:8089", - StartBlockNum: 23671836, - EndBlockNum: 23872222, - IrreversibleOnly: true, - MaxMessagesInFlight: 1337, }, - Telegram: TelegramConfig{ - Id: "110201543:AAHdqTcvCH1vGWJxfSeofSAs0K5PALDsaw", - Channel: -123456789, + Ship: ShipConfig{ + Url: "ws://127.0.0.1:8080", + StartBlockNum: shipclient.NULL_BLOCK_NUMBER, + EndBlockNum: shipclient.NULL_BLOCK_NUMBER, + MaxMessagesInFlight: 10, }, Redis: RedisConfig{ - Addr: "localhost:6379", - User: "userfromcli", - Password: "passwd", - DB: 4, - Prefix: "some::ship", + Addr: "127.0.0.1:6379", + Prefix: "ship", }, } - builder := NewBuilder() - builder.SetSource(bytes.NewBuffer([]byte(` -name: "ship-reader-1" -api: "http://127.0.0.1:8080" -message_codec: "mojibake" -log: - filename: mylog.log - directory: /var/log - maxtime: 30m - maxfilesize: 200b -ship: - url: "127.0.0.1:8089" - irreversible_only: true - max_messages_in_flight: 1337 - start_block_num: 23671836 - end_block_num: 23872222 -telegram: - id: "110201543:AAHdqTcvCH1vGWJxfSeofSAs0K5PALDsaw" - channel: -123456789 -redis: - addr: "localhost:6379" - user: "myuser" - password: "passwd" - db: 4 - prefix: "some::ship" -`))) - - flags := GetFlags() - - require.NoError(t, flags.Set("url", "https://api.example.com")) - require.NoError(t, flags.Set("codec", "msgpack")) - require.NoError(t, flags.Set("redis-user", "userfromcli")) - - builder.SetFlags(flags) - - cfg, err := builder.Build() + cfg, err := NewBuilder(). + SetSource(bytes.NewReader([]byte(``))). + SetFlags(GetFlags()). + Build() require.NoError(t, err) require.Equal(t, &expected, cfg) From ad90966e246603d9c7cf9c4acb17c4ce17218bde Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Thu, 18 Jul 2024 21:42:35 +0200 Subject: [PATCH 225/291] internal/abi/manager_test.go: remove call to fmt.Println() --- internal/abi/manager_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/internal/abi/manager_test.go b/internal/abi/manager_test.go index e7fdfca..a523c6e 100644 --- a/internal/abi/manager_test.go +++ b/internal/abi/manager_test.go @@ -166,7 +166,5 @@ func TestManager_GetAbiFromAPI(t *testing.T) { c_abi, err := mgr.GetAbi(chain.N("testaccount")) assert.NoError(t, err) - fmt.Println(c_abi) - assert_abi(t, c_abi) } From c523f1c797345c36894cc6432ff5e7d9dcabcb03 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Fri, 19 Jul 2024 13:40:03 +0200 Subject: [PATCH 226/291] make: build all go files in cmd/thalos --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 8f312f2..63dd23a 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,7 @@ DOCKER_IMAGE_TAG ?= $(PROGRAM_VERSION) build: build/$(PROGRAM) build/$(PROGRAM) : - $(GO) build $(GOBUILDFLAGS) -o $@ cmd/thalos/main.go cmd/thalos/server.go + $(GO) build $(GOBUILDFLAGS) -o $@ ./cmd/thalos/ tools : build/thalos-tools From 1b1e6a1e3364a7c6d17eebdb15bfe339a83e1b75 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Fri, 19 Jul 2024 13:42:07 +0200 Subject: [PATCH 227/291] cmd/thalos/server.go: move chainInfoOnce to its own file --- cmd/thalos/antelope_api_helpers.go | 35 ++++++++++++++++++++++++++++++ cmd/thalos/server.go | 27 ----------------------- 2 files changed, 35 insertions(+), 27 deletions(-) create mode 100644 cmd/thalos/antelope_api_helpers.go diff --git a/cmd/thalos/antelope_api_helpers.go b/cmd/thalos/antelope_api_helpers.go new file mode 100644 index 0000000..0096143 --- /dev/null +++ b/cmd/thalos/antelope_api_helpers.go @@ -0,0 +1,35 @@ +package main + +import ( + "context" + "time" + + antelopeapi "github.com/shufflingpixels/antelope-go/api" + log "github.com/sirupsen/logrus" +) + +// "Clever" way to make sure we only call the api once. +// Store a info pointer outside the returned closure. +// that pointer will live as long as the closure lives. +// and inside the closure we will reference the pointer and only +// call the api if it is nil. +func chainInfoOnce(api *antelopeapi.Client) func() *antelopeapi.Info { + var info *antelopeapi.Info + return func() *antelopeapi.Info { + if info == nil { + log.WithField("api", api.Url).Info("Get chain info from api") + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + + result, err := api.GetInfo(ctx) + if err != nil { + log.WithError(err).Fatal("Failed to call eos api") + return nil + } + + info = &result + } + return info + } +} diff --git a/cmd/thalos/server.go b/cmd/thalos/server.go index 0fa0c4a..44d0b0d 100644 --- a/cmd/thalos/server.go +++ b/cmd/thalos/server.go @@ -243,33 +243,6 @@ func GetConfig(flags *pflag.FlagSet) (*config.Config, error) { return cfg, nil } -// "Clever" way to make sure we only call the api once. -// Store a info pointer outside the returned closure. -// that pointer will live as long as the closure lives. -// and inside the closure we will reference the pointer and only -// call the api if it is nil. -func chainInfoOnce(api *antelopeapi.Client) func() *antelopeapi.Info { - var info *antelopeapi.Info - return func() *antelopeapi.Info { - if info == nil { - - log.WithField("api", api.Url).Info("Get chain info from api") - - ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) - defer cancel() - - result, err := api.GetInfo(ctx) - if err != nil { - log.WithError(err).Fatal("Failed to call eos api") - return nil - } - - info = &result - } - return info - } -} - func ConnectRedis(conf *config.RedisConfig) (*redis.Client, error) { logEntry := log.WithFields(log.Fields{ "addr": conf.Addr, From dccd7c0520ff7e47a03232bfe8e29eb9c55911f4 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Fri, 19 Jul 2024 13:48:59 +0200 Subject: [PATCH 228/291] internal/config/builder_test.go: fix syntax error --- internal/config/builder_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/config/builder_test.go b/internal/config/builder_test.go index 1de8eb8..772f78b 100644 --- a/internal/config/builder_test.go +++ b/internal/config/builder_test.go @@ -5,6 +5,7 @@ import ( "testing" "time" + shipclient "github.com/eosswedenorg-go/antelope-ship-client" "github.com/eosswedenorg/thalos/internal/log" "github.com/eosswedenorg/thalos/internal/types" "github.com/stretchr/testify/require" @@ -95,7 +96,6 @@ func TestBuilder_WithDefaultConfig(t *testing.T) { MaxTime: time.Hour * 24, FileTimestampFormat: "2006-01-02_150405", }, - }, Ship: ShipConfig{ Url: "ws://127.0.0.1:8080", StartBlockNum: shipclient.NULL_BLOCK_NUMBER, From b60436c48a9006c9e52608e1e9fb4f6f77db4cb6 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 21 Jul 2024 12:50:41 +0200 Subject: [PATCH 229/291] config: adding AbiCache --- internal/config/builder.go | 3 +++ internal/config/builder_test.go | 12 ++++++++++++ internal/config/cli.go | 3 +++ internal/config/config.go | 8 ++++++++ 4 files changed, 26 insertions(+) diff --git a/internal/config/builder.go b/internal/config/builder.go index 2967f63..3941012 100644 --- a/internal/config/builder.go +++ b/internal/config/builder.go @@ -40,6 +40,9 @@ func NewBuilder() *Builder { "telegram.id": "telegram-id", "telegram.channel": "telegram-channel", + // AbiCache + "abi_cache.api_timeout": "abi-cache-api-timeout", + // Log "log.maxfilesize": "log-max-filesize", "log.maxtime": "log-max-time", diff --git a/internal/config/builder_test.go b/internal/config/builder_test.go index 772f78b..8a02880 100644 --- a/internal/config/builder_test.go +++ b/internal/config/builder_test.go @@ -23,6 +23,9 @@ func TestBuilder(t *testing.T) { MaxTime: 30 * time.Minute, FileTimestampFormat: "20060102@150405", }, + AbiCache: AbiCache{ + ApiTimeout: time.Minute * 300, + }, Ship: ShipConfig{ Url: "127.0.0.1:8089", StartBlockNum: 23671836, @@ -53,6 +56,8 @@ func TestBuilder(t *testing.T) { name: "ship-reader-1" api: "http://127.0.0.1:8080" message_codec: "mojibake" +abi_cache: + api_timeout: 300m log: filename: some_file.log directory: /path/to/whatever @@ -96,6 +101,9 @@ func TestBuilder_WithDefaultConfig(t *testing.T) { MaxTime: time.Hour * 24, FileTimestampFormat: "2006-01-02_150405", }, + AbiCache: AbiCache{ + ApiTimeout: time.Second, + }, Ship: ShipConfig{ Url: "ws://127.0.0.1:8080", StartBlockNum: shipclient.NULL_BLOCK_NUMBER, @@ -133,6 +141,7 @@ func TestBuilder_Flags(t *testing.T) { require.NoError(t, flags.Set("redis-password", "secret123")) require.NoError(t, flags.Set("redis-db", "3")) require.NoError(t, flags.Set("redis-prefix", "custom-prefix")) + require.NoError(t, flags.Set("abi-cache-api-timeout", "16h")) require.NoError(t, flags.Set("telegram-id", "72983126312982618")) require.NoError(t, flags.Set("telegram-channel", "-293492332")) require.NoError(t, flags.Set("log-max-filesize", "25mb")) @@ -160,6 +169,9 @@ func TestBuilder_Flags(t *testing.T) { MaxTime: time.Minute * 10, FileTimestampFormat: "0102-15:04:05", }, + AbiCache: AbiCache{ + ApiTimeout: time.Hour * 16, + }, Ship: ShipConfig{ Url: "ws://myship.com:7823", StartBlockNum: 7327833, diff --git a/internal/config/cli.go b/internal/config/cli.go index 85d8f39..b7c90ce 100644 --- a/internal/config/cli.go +++ b/internal/config/cli.go @@ -25,6 +25,9 @@ func GetFlags() *pflag.FlagSet { flags.String("telegram-id", "", "Id of telegram bot") flags.Int64("telegram-channel", 0, "Telegram channel to send notifications to") + // AbiCache + flags.Duration("abi-cache-api-timeout", time.Second, "") + // Log flags.StringP("log", "l", "", "Path to log file (default: print to stdout/stderr)") flags.String("log-max-filesize", "10mb", "Max filesize for logfile to rotate") diff --git a/internal/config/config.go b/internal/config/config.go index 7bb8b4b..6f60465 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1,6 +1,8 @@ package config import ( + "time" + "github.com/eosswedenorg/thalos/internal/log" "github.com/eosswedenorg/thalos/internal/types" ) @@ -18,6 +20,10 @@ type TelegramConfig struct { Channel int64 `yaml:"channel" mapstructure:"channel"` } +type AbiCache struct { + ApiTimeout time.Duration `yaml:"api_timeout" mapstructure:"api_timeout"` +} + type ShipConfig struct { Url string `yaml:"url" mapstructure:"url"` IrreversibleOnly bool `yaml:"irreversible_only" mapstructure:"irreversible_only"` @@ -39,5 +45,7 @@ type Config struct { Redis RedisConfig `yaml:"redis" mapstructure:"redis"` MessageCodec string `yaml:"message_codec" mapstructure:"message_codec"` + AbiCache AbiCache `yaml:"abi_cache" mapstructure:"abi_cache"` + Telegram TelegramConfig `yaml:"telegram" mapstructure:"telegram"` } From ec40e954f26d44a78ae5b206ce30e86f877ee931 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 21 Jul 2024 12:51:54 +0200 Subject: [PATCH 230/291] internal/abi/manager.go: Use config.AbiCache to configure the manager --- cmd/thalos/server.go | 6 +++--- internal/abi/manager.go | 7 +++++-- internal/abi/manager_test.go | 14 ++++++++++++-- 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/cmd/thalos/server.go b/cmd/thalos/server.go index 44d0b0d..b009e0a 100644 --- a/cmd/thalos/server.go +++ b/cmd/thalos/server.go @@ -155,9 +155,9 @@ func LogLevels() []string { return list } -func initAbiManager(api *antelopeapi.Client, store cache.Store, chain_id string) *abi.AbiManager { +func initAbiManager(cfg *config.AbiCache, api *antelopeapi.Client, store cache.Store, chain_id string) *abi.AbiManager { cache := cache.NewCache("thalos::cache::abi::"+chain_id, store) - return abi.NewAbiManager(cache, api) + return abi.NewAbiManager(cfg, cache, api) } func stateLoader(conf *config.Config, start_block_flag *pflag.Flag, chainInfo func() *antelopeapi.Info, cache *cache.Cache, current_block_no_cache bool) StateLoader { @@ -386,7 +386,7 @@ func serverCmd(cmd *cobra.Command, args []string) { Prefix: conf.Redis.Prefix, ChainID: chain_id, }), - initAbiManager(antelopeClient, cacheStore, chain_id), + initAbiManager(&conf.AbiCache, antelopeClient, cacheStore, chain_id), codec, ) diff --git a/internal/abi/manager.go b/internal/abi/manager.go index 8d3acc3..3df1cdb 100644 --- a/internal/abi/manager.go +++ b/internal/abi/manager.go @@ -6,22 +6,25 @@ import ( "time" "github.com/eosswedenorg/thalos/internal/cache" + "github.com/eosswedenorg/thalos/internal/config" "github.com/shufflingpixels/antelope-go/api" "github.com/shufflingpixels/antelope-go/chain" ) // AbiManager handles an ABI cache that fetches the ABI from an API on cache miss. type AbiManager struct { + cfg *config.AbiCache cache *cache.Cache api *api.Client ctx context.Context } // Create a new ABI Manager -func NewAbiManager(cache *cache.Cache, api *api.Client) *AbiManager { +func NewAbiManager(cfg *config.AbiCache, cache *cache.Cache, api *api.Client) *AbiManager { return &AbiManager{ cache: cache, api: api, + cfg: cfg, ctx: context.Background(), } } @@ -38,7 +41,7 @@ func (mgr *AbiManager) SetAbi(account chain.Name, abi *chain.Abi) error { func (mgr *AbiManager) GetAbi(account chain.Name) (*chain.Abi, error) { var abi chain.Abi if err := mgr.cacheGet(account, &abi); err != nil { - ctx, cancel := context.WithTimeout(mgr.ctx, time.Second) + ctx, cancel := context.WithTimeout(mgr.ctx, mgr.cfg.ApiTimeout) defer cancel() resp, err := mgr.api.GetAbi(ctx, account.String()) if err != nil { diff --git a/internal/abi/manager_test.go b/internal/abi/manager_test.go index a523c6e..2606ec1 100644 --- a/internal/abi/manager_test.go +++ b/internal/abi/manager_test.go @@ -6,11 +6,13 @@ import ( "net/http" "net/http/httptest" "testing" + "time" "github.com/shufflingpixels/antelope-go/api" "github.com/shufflingpixels/antelope-go/chain" "github.com/eosswedenorg/thalos/internal/cache" + "github.com/eosswedenorg/thalos/internal/config" "github.com/stretchr/testify/assert" ) @@ -132,12 +134,16 @@ func mockAPI(handler http.HandlerFunc) (*api.Client, *httptest.Server) { } func TestManager_GetAbiFromCache(t *testing.T) { + cfg := &config.AbiCache{ + ApiTimeout: time.Second, + } + cache := cache.NewCache("thalos::cache::abi::test", cache.NewMemoryStore()) api, _ := mockAPI(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { })) - mgr := NewAbiManager(cache, api) + mgr := NewAbiManager(cfg, cache, api) abi := chain.Abi{} err := json.Unmarshal([]byte(abiString), &abi) @@ -152,6 +158,10 @@ func TestManager_GetAbiFromCache(t *testing.T) { } func TestManager_GetAbiFromAPI(t *testing.T) { + cfg := &config.AbiCache{ + ApiTimeout: time.Second, + } + cache := cache.NewCache("thalos::cache::abi::test", cache.NewMemoryStore()) api, _ := mockAPI(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -161,7 +171,7 @@ func TestManager_GetAbiFromAPI(t *testing.T) { assert.NoError(t, err) })) - mgr := NewAbiManager(cache, api) + mgr := NewAbiManager(cfg, cache, api) c_abi, err := mgr.GetAbi(chain.N("testaccount")) assert.NoError(t, err) From 2f31eb47d4f89ac4f790371f5ff94c92f8da93d6 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Tue, 23 Jul 2024 16:31:18 +0200 Subject: [PATCH 231/291] cache: adding factory --- go.mod | 1 + go.sum | 2 ++ internal/cache/factory.go | 26 ++++++++++++++++++++++++++ internal/cache/factory_test.go | 20 ++++++++++++++++++++ 4 files changed, 49 insertions(+) create mode 100644 internal/cache/factory.go create mode 100644 internal/cache/factory_test.go diff --git a/go.mod b/go.mod index 33424c3..f58ad62 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/eosswedenorg/thalos/api v1.0.0 github.com/go-redis/cache/v9 v9.0.0 github.com/go-redis/redismock/v9 v9.2.0 + github.com/karlseguin/typed v1.1.8 github.com/mitchellh/mapstructure v1.5.0 github.com/nikoksr/notify v0.41.0 github.com/redis/go-redis/v9 v9.5.1 diff --git a/go.sum b/go.sum index 936dfc8..4797ab2 100644 --- a/go.sum +++ b/go.sum @@ -76,6 +76,8 @@ github.com/jordan-wright/email v4.0.1-0.20210109023952-943e75fe5223+incompatible github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/karlseguin/typed v1.1.8 h1:ND0eDpwiUFIrm/n1ehxUyh/XNGs9zkYrLxtGqENSalY= +github.com/karlseguin/typed v1.1.8/go.mod h1:pZlmYaWQ7MVpwfIOP88fASh3LopVxKeE+uNXW3hQ2D8= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= diff --git a/internal/cache/factory.go b/internal/cache/factory.go new file mode 100644 index 0000000..bdde3a9 --- /dev/null +++ b/internal/cache/factory.go @@ -0,0 +1,26 @@ +package cache + +import ( + "fmt" + + "github.com/karlseguin/typed" +) + +type Factory func(opts typed.Typed) (Store, error) + +var factories = map[string]Factory{ + "memory": func(opts typed.Typed) (Store, error) { + return NewMemoryStore(), nil + }, +} + +func RegisterFactory(driver string, factory Factory) { + factories[driver] = factory +} + +func Make(driver string, opts typed.Typed) (Store, error) { + if factory, ok := factories[driver]; ok { + return factory(opts) + } + return nil, fmt.Errorf("Invalid cache storage: %s", driver) +} diff --git a/internal/cache/factory_test.go b/internal/cache/factory_test.go new file mode 100644 index 0000000..af0d266 --- /dev/null +++ b/internal/cache/factory_test.go @@ -0,0 +1,20 @@ +package cache_test + +import ( + "testing" + + "github.com/eosswedenorg/thalos/internal/cache" + "github.com/stretchr/testify/require" +) + +func TestFactory_Make(t *testing.T) { + store, err := cache.Make("memory", map[string]any{}) + require.NoError(t, err) + require.Equal(t, cache.NewMemoryStore(), store) +} + +func TestFactory_MakeInvalidDriver(t *testing.T) { + store, err := cache.Make("87923yus", map[string]any{}) + require.Error(t, err) + require.Nil(t, store) +} From d1d4bf58b1f2b263fca44b0859bb57668ab5fc91 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Tue, 23 Jul 2024 16:31:40 +0200 Subject: [PATCH 232/291] cache: adding factory method for redis --- internal/cache/redis_store.go | 28 +++++++++++++++++++++++++ internal/cache/redis_store_test.go | 33 ++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+) diff --git a/internal/cache/redis_store.go b/internal/cache/redis_store.go index 54d7f4d..d8101ad 100644 --- a/internal/cache/redis_store.go +++ b/internal/cache/redis_store.go @@ -5,18 +5,46 @@ import ( "time" "github.com/go-redis/cache/v9" + "github.com/karlseguin/typed" + "github.com/redis/go-redis/v9" ) type RedisStore struct { c *cache.Cache } +type options struct { + Stats bool + Size int + TTL time.Duration +} + func NewRedisStore(options *cache.Options) *RedisStore { return &RedisStore{ c: cache.New(options), } } +func getOptions(opts typed.Typed) options { + return options{ + Stats: opts.Bool("stats"), + Size: opts.IntOr("size", 1000), + TTL: time.Duration(opts.IntOr("ttl", 10)) * time.Minute, + } +} + +func NewRedisFactory(client *redis.Client) Factory { + return func(opts typed.Typed) (Store, error) { + o := getOptions(opts) + + return NewRedisStore(&cache.Options{ + Redis: client, + StatsEnabled: o.Stats, + LocalCache: cache.NewTinyLFU(o.Size, o.TTL), + }), nil + } +} + func (s *RedisStore) Get(ctx context.Context, key string, value interface{}) error { return s.c.Get(ctx, key, value) } diff --git a/internal/cache/redis_store_test.go b/internal/cache/redis_store_test.go index f12cc33..7d4c7c3 100644 --- a/internal/cache/redis_store_test.go +++ b/internal/cache/redis_store_test.go @@ -6,6 +6,7 @@ import ( "time" "github.com/go-redis/redismock/v9" + "github.com/karlseguin/typed" redis_cache "github.com/go-redis/cache/v9" "github.com/stretchr/testify/assert" @@ -16,6 +17,38 @@ type testItem struct { Name string } +func TestRedisStore_getOptionsDefaults(t *testing.T) { + opts := typed.Typed{} + + expected := options{ + Stats: false, + Size: 1000, + TTL: 10 * time.Minute, + } + + actual := getOptions(opts) + + assert.Equal(t, expected, actual) +} + +func TestRedisStore_getOptions(t *testing.T) { + opts := typed.Typed{ + "stats": true, + "size": 123, + "ttl": 60, + } + + expected := options{ + Stats: true, + Size: 123, + TTL: 60 * time.Minute, + } + + actual := getOptions(opts) + + assert.Equal(t, expected, actual) +} + func TestRedisStore_Set(t *testing.T) { client, mock := redismock.NewClientMock() From ebbaf6e2c1df7fac687cce4fd4a7945021b1956c Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Tue, 23 Jul 2024 16:46:02 +0200 Subject: [PATCH 233/291] internal/config: Adding cache configuration --- internal/config/builder.go | 2 ++ internal/config/builder_test.go | 22 ++++++++++++++++++++++ internal/config/cli.go | 3 +++ internal/config/config.go | 8 ++++++++ 4 files changed, 35 insertions(+) diff --git a/internal/config/builder.go b/internal/config/builder.go index 3941012..6429f24 100644 --- a/internal/config/builder.go +++ b/internal/config/builder.go @@ -40,6 +40,8 @@ func NewBuilder() *Builder { "telegram.id": "telegram-id", "telegram.channel": "telegram-channel", + "cache.storage": "cache", + // AbiCache "abi_cache.api_timeout": "abi-cache-api-timeout", diff --git a/internal/config/builder_test.go b/internal/config/builder_test.go index 8a02880..234776a 100644 --- a/internal/config/builder_test.go +++ b/internal/config/builder_test.go @@ -8,6 +8,7 @@ import ( shipclient "github.com/eosswedenorg-go/antelope-ship-client" "github.com/eosswedenorg/thalos/internal/log" "github.com/eosswedenorg/thalos/internal/types" + "github.com/karlseguin/typed" "github.com/stretchr/testify/require" ) @@ -23,6 +24,14 @@ func TestBuilder(t *testing.T) { MaxTime: 30 * time.Minute, FileTimestampFormat: "20060102@150405", }, + Cache: Cache{ + Storage: "memcached", + Options: typed.Typed{ + "ttl": "300m", + "size": 400, + "super_fast_mode": true, + }, + }, AbiCache: AbiCache{ ApiTimeout: time.Minute * 300, }, @@ -56,6 +65,12 @@ func TestBuilder(t *testing.T) { name: "ship-reader-1" api: "http://127.0.0.1:8080" message_codec: "mojibake" +cache: + storage: memcached + options: + ttl: 300m + size: 400 + super_fast_mode: true abi_cache: api_timeout: 300m log: @@ -101,6 +116,9 @@ func TestBuilder_WithDefaultConfig(t *testing.T) { MaxTime: time.Hour * 24, FileTimestampFormat: "2006-01-02_150405", }, + Cache: Cache{ + Storage: "redis", + }, AbiCache: AbiCache{ ApiTimeout: time.Second, }, @@ -141,6 +159,7 @@ func TestBuilder_Flags(t *testing.T) { require.NoError(t, flags.Set("redis-password", "secret123")) require.NoError(t, flags.Set("redis-db", "3")) require.NoError(t, flags.Set("redis-prefix", "custom-prefix")) + require.NoError(t, flags.Set("cache", "memcached")) require.NoError(t, flags.Set("abi-cache-api-timeout", "16h")) require.NoError(t, flags.Set("telegram-id", "72983126312982618")) require.NoError(t, flags.Set("telegram-channel", "-293492332")) @@ -169,6 +188,9 @@ func TestBuilder_Flags(t *testing.T) { MaxTime: time.Minute * 10, FileTimestampFormat: "0102-15:04:05", }, + Cache: Cache{ + Storage: "memcached", + }, AbiCache: AbiCache{ ApiTimeout: time.Hour * 16, }, diff --git a/internal/config/cli.go b/internal/config/cli.go index b7c90ce..de35a92 100644 --- a/internal/config/cli.go +++ b/internal/config/cli.go @@ -25,6 +25,9 @@ func GetFlags() *pflag.FlagSet { flags.String("telegram-id", "", "Id of telegram bot") flags.Int64("telegram-channel", 0, "Telegram channel to send notifications to") + // Cache + flags.String("cache", "redis", "What cache driver to use") + // AbiCache flags.Duration("abi-cache-api-timeout", time.Second, "") diff --git a/internal/config/config.go b/internal/config/config.go index 6f60465..e1d815e 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -5,6 +5,7 @@ import ( "github.com/eosswedenorg/thalos/internal/log" "github.com/eosswedenorg/thalos/internal/types" + "github.com/karlseguin/typed" ) type RedisConfig struct { @@ -15,6 +16,11 @@ type RedisConfig struct { Prefix string `yaml:"prefix"` } +type Cache struct { + Storage string `yaml:"storage" mapstructure:"storage"` + Options typed.Typed `yaml:"options" mapstructure:"options"` +} + type TelegramConfig struct { Id string `yaml:"id" mapstructure:"id"` Channel int64 `yaml:"channel" mapstructure:"channel"` @@ -42,6 +48,8 @@ type Config struct { Log log.Config `yaml:"log" mapstructure:"log"` + Cache Cache `yaml:"cache" mapstructure:"cache"` + Redis RedisConfig `yaml:"redis" mapstructure:"redis"` MessageCodec string `yaml:"message_codec" mapstructure:"message_codec"` From 8a938c3f9e2450a2f197bff527fb02e680b5218b Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Tue, 23 Jul 2024 18:34:30 +0200 Subject: [PATCH 234/291] cmd/thalos/server.go: use factory and config to create cache store --- cmd/thalos/server.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/cmd/thalos/server.go b/cmd/thalos/server.go index b009e0a..5813aee 100644 --- a/cmd/thalos/server.go +++ b/cmd/thalos/server.go @@ -26,7 +26,6 @@ import ( driver "github.com/eosswedenorg/thalos/internal/driver/redis" . "github.com/eosswedenorg/thalos/internal/log" . "github.com/eosswedenorg/thalos/internal/server" - redis_cache "github.com/go-redis/cache/v9" "github.com/nikoksr/notify" "github.com/nikoksr/notify/service/telegram" "github.com/redis/go-redis/v9" @@ -346,12 +345,14 @@ func serverCmd(cmd *cobra.Command, args []string) { return } + cache.RegisterFactory("redis", cache.NewRedisFactory(rdb)) + // Setup cache storage - cacheStore := cache.NewRedisStore(&redis_cache.Options{ - Redis: rdb, - // Cache 10k keys for 10 minutes. - LocalCache: redis_cache.NewTinyLFU(10000, 10*time.Minute), - }) + cacheStore, err := cache.Make(conf.Cache.Storage, conf.Cache.Options) + if err != nil { + log.WithError(err).Fatal("Failed to setup cache") + return + } // Setup general cache cache := cache.NewCache("thalos::cache::instance::"+conf.Name, cacheStore) From b7e0cb7b56f54d87d1953248e4346cfe449ab152 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Tue, 23 Jul 2024 22:09:00 +0200 Subject: [PATCH 235/291] adding .github/workflows/devbuild.yaml --- .github/workflows/devbuild.yaml | 79 +++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 .github/workflows/devbuild.yaml diff --git a/.github/workflows/devbuild.yaml b/.github/workflows/devbuild.yaml new file mode 100644 index 0000000..6a04233 --- /dev/null +++ b/.github/workflows/devbuild.yaml @@ -0,0 +1,79 @@ +name: Development build + +on: + push: + branches: [ dev ] + +jobs: + cross-compile: + strategy: + fail-fast: false + matrix: + os: [ linux, freebsd ] + arch: [ 386, amd64, arm, arm64 ] + name: Crosscompile - ${{matrix.os}}-${{matrix.arch}} + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: 1.21 + + - name: compile + id: compile + run: | + VER=$(cat Makefile | grep 'PROGRAM_VERSION=' | sed 's/PROGRAM_VERSION=//g') + GOOS=${{matrix.os}} GOARCH=${{matrix.arch}} PROGRAM_VERSION="${VER}-${GITHUB_SHA}" make build tools + + - name: Upload thalos-server + uses: actions/upload-artifact@v4 + with: + name: thalos-server-${{github.sha}}-${{matrix.os}}-${{matrix.arch}} + path: build/thalos-server + retention-days: 7 + + - name: Upload thalos-tools + uses: actions/upload-artifact@v4 + with: + name: thalos-tools-${{github.sha}}-${{matrix.os}}-${{matrix.arch}} + path: build/thalos-tools + retention-days: 7 + + # Build thalos binaries that are linked with musl libc. + musl: + strategy: + fail-fast: false + matrix: + arch: [ 386, amd64, arm, arm64 ] + runs-on: ubuntu-latest + name: musl (${{ matrix.arch }}) + container: + image: golang:1.21-alpine3.19 + steps: + - uses: actions/checkout@v4 + + - name: install dependencies + run: apk add make + + - name: compile + id: compile + run: | + VER=$(cat Makefile | grep 'PROGRAM_VERSION=' | sed 's/PROGRAM_VERSION=//g') + GOOS=${{matrix.os}} GOARCH=${{matrix.arch}} PROGRAM_VERSION="${VER}-${GITHUB_SHA}" make build tools + + - name: Upload thalos-server + uses: actions/upload-artifact@v4 + with: + name: thalos-server-${{github.sha}}-linux-${{matrix.arch}}-musl + path: build/thalos-server + retention-days: 7 + + - name: Upload thalos-tools + uses: actions/upload-artifact@v4 + with: + name: thalos-tools-${{github.sha}}-linux-${{matrix.arch}}-musl + path: build/thalos-tools + retention-days: 7 + From e8b90dab7755ffba258e22a6249555f46cd51a95 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Wed, 24 Jul 2024 00:22:14 +0200 Subject: [PATCH 236/291] Makefile: only set PROGRAM_VERSION if its unset --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 63dd23a..73b8b87 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ GO=go GOLDFLAGS=-v -s -w -X main.VersionString=$(PROGRAM_VERSION) GOBUILDFLAGS=-v -p $(shell nproc) -ldflags="$(GOLDFLAGS)" PROGRAM=thalos-server -PROGRAM_VERSION=1.1.4 +PROGRAM_VERSION ?= 1.1.4 PREFIX=/usr/local BINDIR=$(PREFIX)/bin CFGDIR=$(PREFIX)/etc/thalos From 9778fa86108eb243724d6bef194150fcf9f06ad2 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Wed, 24 Jul 2024 00:45:16 +0200 Subject: [PATCH 237/291] .github/workflows/devbuild.yaml: fix version string bash stuff :) --- .github/workflows/devbuild.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/devbuild.yaml b/.github/workflows/devbuild.yaml index 6a04233..b8897d0 100644 --- a/.github/workflows/devbuild.yaml +++ b/.github/workflows/devbuild.yaml @@ -24,8 +24,8 @@ jobs: - name: compile id: compile run: | - VER=$(cat Makefile | grep 'PROGRAM_VERSION=' | sed 's/PROGRAM_VERSION=//g') - GOOS=${{matrix.os}} GOARCH=${{matrix.arch}} PROGRAM_VERSION="${VER}-${GITHUB_SHA}" make build tools + VER=$(cat Makefile | sed -n 's/^PROGRAM_VERSION\s*\(\?\)\s*=\s*//p') + GOOS=${{matrix.os}} GOARCH=${{matrix.arch}} PROGRAM_VERSION="${VER}-$(echo $GITHUB_SHA | cut -b -8)" make build tools - name: Upload thalos-server uses: actions/upload-artifact@v4 @@ -60,8 +60,8 @@ jobs: - name: compile id: compile run: | - VER=$(cat Makefile | grep 'PROGRAM_VERSION=' | sed 's/PROGRAM_VERSION=//g') - GOOS=${{matrix.os}} GOARCH=${{matrix.arch}} PROGRAM_VERSION="${VER}-${GITHUB_SHA}" make build tools + VER=$(cat Makefile | sed -n 's/^PROGRAM_VERSION\s*\(\?\)\s*=\s*//p') + GOOS=${{matrix.os}} GOARCH=${{matrix.arch}} PROGRAM_VERSION="${VER}-$(echo $GITHUB_SHA | cut -b -8)" make build tools - name: Upload thalos-server uses: actions/upload-artifact@v4 From ec874430c326d4a48b90c4e8138ddb046364664a Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Wed, 24 Jul 2024 00:53:24 +0200 Subject: [PATCH 238/291] .github/workflows/devbuild.yaml: improve the sed expression. --- .github/workflows/devbuild.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/devbuild.yaml b/.github/workflows/devbuild.yaml index b8897d0..3af1d58 100644 --- a/.github/workflows/devbuild.yaml +++ b/.github/workflows/devbuild.yaml @@ -24,7 +24,7 @@ jobs: - name: compile id: compile run: | - VER=$(cat Makefile | sed -n 's/^PROGRAM_VERSION\s*\(\?\)\s*=\s*//p') + VER=$(cat Makefile | sed -n 's/^PROGRAM_VERSION\s*\??=\s*//p') GOOS=${{matrix.os}} GOARCH=${{matrix.arch}} PROGRAM_VERSION="${VER}-$(echo $GITHUB_SHA | cut -b -8)" make build tools - name: Upload thalos-server @@ -60,7 +60,7 @@ jobs: - name: compile id: compile run: | - VER=$(cat Makefile | sed -n 's/^PROGRAM_VERSION\s*\(\?\)\s*=\s*//p') + VER=$(cat Makefile | sed -n 's/^PROGRAM_VERSION\s*\??=\s*//p') GOOS=${{matrix.os}} GOARCH=${{matrix.arch}} PROGRAM_VERSION="${VER}-$(echo $GITHUB_SHA | cut -b -8)" make build tools - name: Upload thalos-server From e038be5324aec6372d89b67fc3762474c942dddf Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Thu, 25 Jul 2024 12:21:53 +0200 Subject: [PATCH 239/291] .github/workflows/devbuild.yaml: update names --- .github/workflows/devbuild.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/devbuild.yaml b/.github/workflows/devbuild.yaml index 3af1d58..73ba184 100644 --- a/.github/workflows/devbuild.yaml +++ b/.github/workflows/devbuild.yaml @@ -11,7 +11,7 @@ jobs: matrix: os: [ linux, freebsd ] arch: [ 386, amd64, arm, arm64 ] - name: Crosscompile - ${{matrix.os}}-${{matrix.arch}} + name: ${{matrix.os}} (${{matrix.arch}}) runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v4 From 18a8feea9efd500b9b4ef7731a00d5cf6a1f0cca Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Thu, 25 Jul 2024 19:36:51 +0200 Subject: [PATCH 240/291] make: no need to use find, can specify the directory where the go files are --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 73b8b87..e0a62f0 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,7 @@ build/$(PROGRAM) : tools : build/thalos-tools build/thalos-tools : - $(GO) build $(GOBUILDFLAGS) -o $@ $(shell find cmd/tools -type f -name *.go) + $(GO) build $(GOBUILDFLAGS) -o $@ ./cmd/tools/ docker-image: docker image build --build-arg VERSION=$(PROGRAM_VERSION) -t $(DOCKER_IMAGE_REPO):$(DOCKER_IMAGE_TAG) docker From ecdde63f68ab80577749f501320c9965039a1668 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 28 Jul 2024 14:45:33 +0200 Subject: [PATCH 241/291] internal/config/cli.go: set description for abi-cache-api-timeout flag --- internal/config/cli.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/config/cli.go b/internal/config/cli.go index de35a92..a23794a 100644 --- a/internal/config/cli.go +++ b/internal/config/cli.go @@ -29,7 +29,7 @@ func GetFlags() *pflag.FlagSet { flags.String("cache", "redis", "What cache driver to use") // AbiCache - flags.Duration("abi-cache-api-timeout", time.Second, "") + flags.Duration("abi-cache-api-timeout", time.Second, "Duration before the api call times out when the ABI cache requests an abi.") // Log flags.StringP("log", "l", "", "Path to log file (default: print to stdout/stderr)") From 4bf589647c501586115fbfe23513d975731f12f4 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 11 Aug 2024 13:03:55 +0200 Subject: [PATCH 242/291] internal/server/helpers.go: add a comment in empty for loop (as i almost removed it because it was empty) --- internal/server/helpers.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/server/helpers.go b/internal/server/helpers.go index 05a2807..b34bbfc 100644 --- a/internal/server/helpers.go +++ b/internal/server/helpers.go @@ -113,6 +113,7 @@ func isVariant(v reflect.Value) bool { } for v = v.Index(0); v.Kind() == reflect.Interface || v.Kind() == reflect.Pointer; v = v.Elem() { + // Intentionally empty } return v.Kind() == reflect.String && isVariantName(v.String()) From 3d83fcc8690e46f106c2a1ea4cd89cbeb8effc0d Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 11 Aug 2024 15:41:02 +0200 Subject: [PATCH 243/291] go mod: update antelope-go to v0.1.4 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index f58ad62..ca4a378 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/mitchellh/mapstructure v1.5.0 github.com/nikoksr/notify v0.41.0 github.com/redis/go-redis/v9 v9.5.1 - github.com/shufflingpixels/antelope-go v0.1.3 + github.com/shufflingpixels/antelope-go v0.1.4 github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.8.0 github.com/spf13/pflag v1.0.5 diff --git a/go.sum b/go.sum index 4797ab2..41dd453 100644 --- a/go.sum +++ b/go.sum @@ -148,8 +148,8 @@ github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6ke github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= -github.com/shufflingpixels/antelope-go v0.1.3 h1:uZ0cCn6vitHZTz7UFy5n8AEv1k37MqzCTLIuAnX346Y= -github.com/shufflingpixels/antelope-go v0.1.3/go.mod h1:Bi+zmFI5s7Qv2hx3v9FAcDyf4N4MUs5zaKNa2CetmSg= +github.com/shufflingpixels/antelope-go v0.1.4 h1:MInk3tEo69OA/U6tZWXGrVzdQhrDFX8UfUkGq1Lq/2Y= +github.com/shufflingpixels/antelope-go v0.1.4/go.mod h1:Bi+zmFI5s7Qv2hx3v9FAcDyf4N4MUs5zaKNa2CetmSg= github.com/shufflingpixels/jsontime-go v0.0.0-20240622163621-cf4b2804c92d h1:nju7jR1Kf210tArPT6l//HlfLLFnvje1BWl5TSRsohQ= github.com/shufflingpixels/jsontime-go v0.0.0-20240622163621-cf4b2804c92d/go.mod h1:W0TaKyg3kDqWmFUxlax3qAls/lRdo12EseM6f4f0dzE= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= From 0e75fb869b55c57738bf68f245b812ffc729d11b Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 11 Aug 2024 16:14:15 +0200 Subject: [PATCH 244/291] adding internal/ship/contract_row.go --- internal/ship/contract_row.go | 21 ++++++++++++++++++++ internal/ship/contract_row_test.go | 32 ++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+) create mode 100644 internal/ship/contract_row.go create mode 100644 internal/ship/contract_row_test.go diff --git a/internal/ship/contract_row.go b/internal/ship/contract_row.go new file mode 100644 index 0000000..a5b785f --- /dev/null +++ b/internal/ship/contract_row.go @@ -0,0 +1,21 @@ +package ship + +import ( + "github.com/mitchellh/mapstructure" + "github.com/shufflingpixels/antelope-go/chain" +) + +type ContractRow struct { + Code chain.Name `mapstructure:"code"` + Scope chain.Name `mapstructure:"scope"` + Table chain.Name `mapstructure:"table"` + PrimaryKey string `mapstructure:"primary_key"` + Payer chain.Name `mapstructure:"payer"` + Value chain.Bytes `mapstructure:"value"` +} + +func DecodeContractRow(v map[string]interface{}) (*ContractRow, error) { + out := &ContractRow{} + err := mapstructure.WeakDecode(v, out) + return out, err +} diff --git a/internal/ship/contract_row_test.go b/internal/ship/contract_row_test.go new file mode 100644 index 0000000..6a46be6 --- /dev/null +++ b/internal/ship/contract_row_test.go @@ -0,0 +1,32 @@ +package ship_test + +import ( + "testing" + + "github.com/eosswedenorg/thalos/internal/ship" + "github.com/shufflingpixels/antelope-go/chain" + "github.com/stretchr/testify/assert" +) + +func TestContractRow_Decode(t *testing.T) { + expected := &ship.ContractRow{ + Code: chain.N("eosio"), + Scope: chain.N("scope"), + Table: chain.N("accounts"), + PrimaryKey: "1278127812", + Payer: chain.N("account1"), + Value: []byte{0x01, 0x01, 0x02, 0x03}, + } + + actual, err := ship.DecodeContractRow(map[string]any{ + "code": uint64(6138663577826885632), + "scope": uint64(13990807175891517440), + "table": uint64(3607749779137757184), + "primary_key": uint32(1278127812), + "payer": uint64(3607749778751881216), + "value": []byte{0x01, 0x01, 0x02, 0x03}, + }) + + assert.NoError(t, err) + assert.Equal(t, expected, actual) +} From 2ed95186358b20a296160b488c87b26ddfbacb36 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 11 Aug 2024 16:51:40 +0200 Subject: [PATCH 245/291] internal/server/helpers.go: refactor some functions to internal/ship --- internal/server/helpers.go | 123 ------------------ internal/server/ship_processor.go | 3 +- internal/ship/table_delta.go | 36 +++++ .../table_delta_test.go} | 5 +- internal/ship/variant.go | 93 +++++++++++++ 5 files changed, 134 insertions(+), 126 deletions(-) create mode 100644 internal/ship/table_delta.go rename internal/{server/helpers_test.go => ship/table_delta_test.go} (93%) create mode 100644 internal/ship/variant.go diff --git a/internal/server/helpers.go b/internal/server/helpers.go index b34bbfc..95ea78d 100644 --- a/internal/server/helpers.go +++ b/internal/server/helpers.go @@ -1,9 +1,6 @@ package server import ( - "fmt" - "reflect" - "github.com/shufflingpixels/antelope-go/ship" ) @@ -28,123 +25,3 @@ func toActionTraceV1(trace *ship.ActionTrace) *ship.ActionTraceV1 { } return trace.V1 } - -func isVariantName(name string) bool { - validVariants := []string{ - "get_status_request_v0", - "block_position", - "get_status_result_v0", - "get_blocks_request_v0", - "get_blocks_ack_request_v0", - "get_blocks_result_v0", - "row", - "table_delta_v0", - "action", - "account_auth_sequence", - "action_receipt_v0", - "account_delta", - "action_trace_v0", - "partial_transaction_v0", - "transaction_trace_v0", - "packed_transaction", - "transaction_receipt_header", - "transaction_receipt", - "extension", - "block_header", - "signed_block_header", - "signed_block", - "transaction_header", - "transaction", - "code_id", - "account_v0", - "account_metadata_v0", - "code_v0", - "contract_table_v0", - "contract_row_v0", - "contract_index64_v0", - "contract_index128_v0", - "contract_index256_v0", - "contract_index_double_v0", - "contract_index_long_double_v0", - "producer_key", - "producer_schedule", - "block_signing_authority_v0", - "producer_authority", - "producer_authority_schedule", - "chain_config_v0", - "global_property_v0", - "global_property_v1", - "generated_transaction_v0", - "activated_protocol_feature_v0", - "protocol_state_v0", - "key_weight", - "permission_level", - "permission_level_weight", - "wait_weight", - "authority", - "permission_v0", - "permission_link_v0", - "resource_limits_v0", - "usage_accumulator_v0", - "resource_usage_v0", - "resource_limits_state_v0", - "resource_limits_ratio_v0", - "elastic_limit_parameters_v0", - "resource_limits_config_v0", - } - - for _, v := range validVariants { - if v == name { - return true - } - } - return false -} - -// Check if a structure is a variant type. -// This is not 100% accurate. As variant types comes -// as a simple slice with the types name in the first index -// and the value as the second. -// So there could be some edge cases where this structure is actual data -// and not a variant type although should be super rare. -func isVariant(v reflect.Value) bool { - if v.Kind() != reflect.Slice || v.Len() != 2 { - return false - } - - for v = v.Index(0); v.Kind() == reflect.Interface || v.Kind() == reflect.Pointer; v = v.Elem() { - // Intentionally empty - } - - return v.Kind() == reflect.String && isVariantName(v.String()) -} - -func parseTableDeltaData(v any) (map[string]interface{}, error) { - iface := parseTableDeltaDataInner(reflect.ValueOf(v)).Interface() - if out, ok := iface.(map[string]interface{}); ok { - return out, nil - } - return nil, fmt.Errorf("data is not an map") -} - -func parseTableDeltaDataInner(v reflect.Value) reflect.Value { - if isVariant(v) { - v = v.Index(1) - } - - switch v.Kind() { - case reflect.Interface: - return parseTableDeltaDataInner(v.Elem()) - case reflect.Slice: - for i := 0; i < v.Len(); i++ { - v.Index(i).Set(parseTableDeltaDataInner(v.Index(i))) - } - case reflect.Map: - it := v.MapRange() - for it.Next() { - v.SetMapIndex(it.Key(), parseTableDeltaDataInner(it.Value())) - } - } - - return v -} diff --git a/internal/server/ship_processor.go b/internal/server/ship_processor.go index 7106f4f..ad56305 100644 --- a/internal/server/ship_processor.go +++ b/internal/server/ship_processor.go @@ -6,6 +6,7 @@ import ( "github.com/eosswedenorg/thalos/api/message" "github.com/eosswedenorg/thalos/internal/abi" "github.com/eosswedenorg/thalos/internal/driver" + ship_helper "github.com/eosswedenorg/thalos/internal/ship" "github.com/eosswedenorg/thalos/internal/types" log "github.com/sirupsen/logrus" @@ -228,7 +229,7 @@ func (processor *ShipProcessor) proccessDeltaRows(logger *log.Entry, table_name v, err := processor.shipABI.Decode(bytes.NewReader(row.Data), table_name) if err == nil { - v, err := parseTableDeltaData(v) + v, err := ship_helper.ParseTableDeltaData(v) if err == nil { msg.Data = v } else { diff --git a/internal/ship/table_delta.go b/internal/ship/table_delta.go new file mode 100644 index 0000000..9d011d3 --- /dev/null +++ b/internal/ship/table_delta.go @@ -0,0 +1,36 @@ +package ship + +import ( + "fmt" + "reflect" +) + +func parseTableDeltaDataInner(v reflect.Value) reflect.Value { + if IsVariant(v) { + v = v.Index(1) + } + + switch v.Kind() { + case reflect.Interface: + return parseTableDeltaDataInner(v.Elem()) + case reflect.Slice: + for i := 0; i < v.Len(); i++ { + v.Index(i).Set(parseTableDeltaDataInner(v.Index(i))) + } + case reflect.Map: + it := v.MapRange() + for it.Next() { + v.SetMapIndex(it.Key(), parseTableDeltaDataInner(it.Value())) + } + } + + return v +} + +func ParseTableDeltaData(v any) (map[string]interface{}, error) { + iface := parseTableDeltaDataInner(reflect.ValueOf(v)).Interface() + if out, ok := iface.(map[string]interface{}); ok { + return out, nil + } + return nil, fmt.Errorf("data is not an map") +} diff --git a/internal/server/helpers_test.go b/internal/ship/table_delta_test.go similarity index 93% rename from internal/server/helpers_test.go rename to internal/ship/table_delta_test.go index fdaf8ff..74b1d03 100644 --- a/internal/server/helpers_test.go +++ b/internal/ship/table_delta_test.go @@ -1,8 +1,9 @@ -package server +package ship_test import ( "testing" + "github.com/eosswedenorg/thalos/internal/ship" "github.com/stretchr/testify/assert" ) @@ -62,7 +63,7 @@ func TestParseTableDeltaData(t *testing.T) { "virtual_net_limit": 1048576000, } - actual, err := parseTableDeltaData(input) + actual, err := ship.ParseTableDeltaData(input) assert.NoError(t, err) assert.Equal(t, expected, actual) diff --git a/internal/ship/variant.go b/internal/ship/variant.go new file mode 100644 index 0000000..56ca7fb --- /dev/null +++ b/internal/ship/variant.go @@ -0,0 +1,93 @@ +package ship + +import "reflect" + +func IsVariantName(name string) bool { + validVariants := []string{ + "get_status_request_v0", + "block_position", + "get_status_result_v0", + "get_blocks_request_v0", + "get_blocks_ack_request_v0", + "get_blocks_result_v0", + "row", + "table_delta_v0", + "action", + "account_auth_sequence", + "action_receipt_v0", + "account_delta", + "action_trace_v0", + "partial_transaction_v0", + "transaction_trace_v0", + "packed_transaction", + "transaction_receipt_header", + "transaction_receipt", + "extension", + "block_header", + "signed_block_header", + "signed_block", + "transaction_header", + "transaction", + "code_id", + "account_v0", + "account_metadata_v0", + "code_v0", + "contract_table_v0", + "contract_row_v0", + "contract_index64_v0", + "contract_index128_v0", + "contract_index256_v0", + "contract_index_double_v0", + "contract_index_long_double_v0", + "producer_key", + "producer_schedule", + "block_signing_authority_v0", + "producer_authority", + "producer_authority_schedule", + "chain_config_v0", + "global_property_v0", + "global_property_v1", + "generated_transaction_v0", + "activated_protocol_feature_v0", + "protocol_state_v0", + "key_weight", + "permission_level", + "permission_level_weight", + "wait_weight", + "authority", + "permission_v0", + "permission_link_v0", + "resource_limits_v0", + "usage_accumulator_v0", + "resource_usage_v0", + "resource_limits_state_v0", + "resource_limits_ratio_v0", + "elastic_limit_parameters_v0", + "resource_limits_config_v0", + } + + for _, v := range validVariants { + if v == name { + return true + } + } + return false +} + +// Check if a structure is a variant type. +// This is not 100% accurate. As variant types comes +// as a simple slice with the types name in the first index +// and the value as the second. +// So there could be some edge cases where this structure is actual data +// and not a variant type although should be super rare. +func IsVariant(v reflect.Value) bool { + if v.Kind() != reflect.Slice || v.Len() != 2 { + return false + } + + for v = v.Index(0); v.Kind() == reflect.Interface || v.Kind() == reflect.Pointer; v = v.Elem() { + // Intentionally empty + } + + return v.Kind() == reflect.String && IsVariantName(v.String()) +} From 67d33a280ef287b999c3d74fa64224298a57018e Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 11 Aug 2024 16:56:55 +0200 Subject: [PATCH 246/291] move internal/server/helpers.go to internal/ship/action.go --- internal/server/ship_processor.go | 2 +- internal/{server/helpers.go => ship/action.go} | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) rename internal/{server/helpers.go => ship/action.go} (90%) diff --git a/internal/server/ship_processor.go b/internal/server/ship_processor.go index ad56305..654fd55 100644 --- a/internal/server/ship_processor.go +++ b/internal/server/ship_processor.go @@ -120,7 +120,7 @@ func (processor *ShipProcessor) processTransactionTrace(log *log.Entry, blockNum // Actions for _, actionTraceVar := range trace.ActionTraces { - actionTrace := toActionTraceV1(actionTraceVar) + actionTrace := ship_helper.ToActionTraceV1(actionTraceVar) actMsg := processor.proccessActionTrace(logger, actionTrace) if actMsg != nil { actMsg.TxID = trace.ID.String() diff --git a/internal/server/helpers.go b/internal/ship/action.go similarity index 90% rename from internal/server/helpers.go rename to internal/ship/action.go index 95ea78d..8a25a46 100644 --- a/internal/server/helpers.go +++ b/internal/ship/action.go @@ -1,11 +1,11 @@ -package server +package ship import ( "github.com/shufflingpixels/antelope-go/ship" ) // convert a ActionTrace to ActionTraceV1 -func toActionTraceV1(trace *ship.ActionTrace) *ship.ActionTraceV1 { +func ToActionTraceV1(trace *ship.ActionTrace) *ship.ActionTraceV1 { if trace.V0 != nil { // convert to v1 return &ship.ActionTraceV1{ From 14a498d7d6438df7075b8c90570f5b63d9ea1a56 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 11 Aug 2024 16:59:23 +0200 Subject: [PATCH 247/291] internal/server/ship_processor.go: add WithError() to a logging call --- internal/server/ship_processor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/server/ship_processor.go b/internal/server/ship_processor.go index 654fd55..6bbff10 100644 --- a/internal/server/ship_processor.go +++ b/internal/server/ship_processor.go @@ -236,7 +236,7 @@ func (processor *ShipProcessor) proccessDeltaRows(logger *log.Entry, table_name logger.WithError(err).Error("Failed to parse table delta data") } } else { - logger.Error("Failed to decode table delta") + logger.WithError(err).Error("Failed to decode table delta") } } else { logger.Warn("No SHIP ABI present") From a892b40cdeb720a381995277ce0b2433f3ae4556 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 11 Aug 2024 17:03:14 +0200 Subject: [PATCH 248/291] internal/ship/contract_row.go: rename DecodeContractRow to ParseContractRow --- internal/ship/contract_row.go | 2 +- internal/ship/contract_row_test.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/ship/contract_row.go b/internal/ship/contract_row.go index a5b785f..5449e58 100644 --- a/internal/ship/contract_row.go +++ b/internal/ship/contract_row.go @@ -14,7 +14,7 @@ type ContractRow struct { Value chain.Bytes `mapstructure:"value"` } -func DecodeContractRow(v map[string]interface{}) (*ContractRow, error) { +func ParseContractRow(v map[string]interface{}) (*ContractRow, error) { out := &ContractRow{} err := mapstructure.WeakDecode(v, out) return out, err diff --git a/internal/ship/contract_row_test.go b/internal/ship/contract_row_test.go index 6a46be6..d281732 100644 --- a/internal/ship/contract_row_test.go +++ b/internal/ship/contract_row_test.go @@ -8,7 +8,7 @@ import ( "github.com/stretchr/testify/assert" ) -func TestContractRow_Decode(t *testing.T) { +func TestContractRow_Parse(t *testing.T) { expected := &ship.ContractRow{ Code: chain.N("eosio"), Scope: chain.N("scope"), @@ -18,7 +18,7 @@ func TestContractRow_Decode(t *testing.T) { Value: []byte{0x01, 0x01, 0x02, 0x03}, } - actual, err := ship.DecodeContractRow(map[string]any{ + actual, err := ship.ParseContractRow(map[string]any{ "code": uint64(6138663577826885632), "scope": uint64(13990807175891517440), "table": uint64(3607749779137757184), From 6d7d004b2b16b41c8c5880a8bada2a9d349931c9 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 11 Aug 2024 17:03:42 +0200 Subject: [PATCH 249/291] internal/ship/contract_row.go: adding DecodeContractRow that actually decodes data from abi --- internal/ship/contract_row.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/internal/ship/contract_row.go b/internal/ship/contract_row.go index 5449e58..f980e90 100644 --- a/internal/ship/contract_row.go +++ b/internal/ship/contract_row.go @@ -1,6 +1,9 @@ package ship import ( + "bytes" + + "github.com/eosswedenorg/thalos/internal/abi" "github.com/mitchellh/mapstructure" "github.com/shufflingpixels/antelope-go/chain" ) @@ -19,3 +22,16 @@ func ParseContractRow(v map[string]interface{}) (*ContractRow, error) { err := mapstructure.WeakDecode(v, out) return out, err } + +func DecodeContractRow(manager *abi.AbiManager, data map[string]any) (any, error) { + row, err := ParseContractRow(data) + if err != nil { + return nil, err + } + + abi, err := manager.GetAbi(row.Code) + if err != nil { + return nil, err + } + return abi.DecodeTable(bytes.NewReader(row.Value), row.Table) +} From fa6508ba90042b83f76c5e2330481cb3ee7c80b5 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 11 Aug 2024 17:04:06 +0200 Subject: [PATCH 250/291] internal/server/ship_processor.go: try decoding table data for contract_row table deltas --- internal/server/ship_processor.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/internal/server/ship_processor.go b/internal/server/ship_processor.go index 6bbff10..e3ab11e 100644 --- a/internal/server/ship_processor.go +++ b/internal/server/ship_processor.go @@ -226,12 +226,20 @@ func (processor *ShipProcessor) proccessDeltaRows(logger *log.Entry, table_name } if processor.shipABI != nil { - v, err := processor.shipABI.Decode(bytes.NewReader(row.Data), table_name) if err == nil { - v, err := ship_helper.ParseTableDeltaData(v) + data, err := ship_helper.ParseTableDeltaData(v) if err == nil { - msg.Data = v + // Decode contract row data + if table_name == "contract_row" { + dec, err := ship_helper.DecodeContractRow(processor.abi, data) + if err != nil { + logger.WithError(err).Warn("Failed to decode contract row") + } else { + data["value"] = dec + } + } + msg.Data = data } else { logger.WithError(err).Error("Failed to parse table delta data") } From aaeb2739ab5308640b815d681fbc9933de13f7b2 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 11 Aug 2024 17:19:41 +0200 Subject: [PATCH 251/291] config.example.yml: add example for cache section --- config.example.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/config.example.yml b/config.example.yml index 9d6deef..85c289d 100644 --- a/config.example.yml +++ b/config.example.yml @@ -56,6 +56,16 @@ ship: # blacklist_is_whitelist: true +# Configure the cache. +# Default is to use redis. But if you need to +# you can set additional values or even change the driver +# to something else that Thalos supports. +# See the documentation for details + +# cache: +# storage: redis +# options: [] + # Telegram notifications #telegram: # id: "123456789:GPdmGPBWvpgHPxlergJLavus-PoAURTjMWP" From c0c062a1b5aef2338ceb98dc177351a426e7964a Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 11 Aug 2024 17:27:35 +0200 Subject: [PATCH 252/291] Version 1.1.5-rc1 --- Makefile | 2 +- debian/changelog | 11 +++++++++++ docker/Dockerfile | 2 +- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index e0a62f0..a139da8 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ GO=go GOLDFLAGS=-v -s -w -X main.VersionString=$(PROGRAM_VERSION) GOBUILDFLAGS=-v -p $(shell nproc) -ldflags="$(GOLDFLAGS)" PROGRAM=thalos-server -PROGRAM_VERSION ?= 1.1.4 +PROGRAM_VERSION ?= 1.1.5-rc1 PREFIX=/usr/local BINDIR=$(PREFIX)/bin CFGDIR=$(PREFIX)/etc/thalos diff --git a/debian/changelog b/debian/changelog index 1771e6e..f423520 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,14 @@ +thalos (1.1.5~rc1) bionic focal jammy; urgency=medium + + * New config section: `cache` + * New CLI flag: `cache` specify what cache driver to use + * New CLI flag: `abi-cache-api-timeout` configure the timeout for the HTTP + request made when Thalos wants to fetch a ABI from the api. + * API Table Deltas: abi decode the data in `value` field for contract_row deltas. + * golang: update github.com/shufflingpixels/antelope-go to v0.1.4 + + -- Henrik Hautakoski Sun, 11 Aug 2024 17:04:55 +0200 + thalos (1.1.4) bionic focal jammy; urgency=medium * Implement whitelist option for ship contract/action blacklist diff --git a/docker/Dockerfile b/docker/Dockerfile index 8f55319..066533c 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,6 +1,6 @@ FROM alpine:latest LABEL maintainer="Henrik Hautakoski " -ARG VERSION=1.1.4 +ARG VERSION=1.1.5-rc1 WORKDIR /thalos ADD --chmod=755 https://github.com/eosswedenorg/thalos/releases/download/v$VERSION/thalos-server-${VERSION}-linux-amd64-musl thalos-server ENTRYPOINT [ "./thalos-server" ] From 48ca5e4f92a4602637554103d11cd13383402381 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 11 Aug 2024 17:44:39 +0200 Subject: [PATCH 253/291] .github/workflows/release.yml: fix regex that parses the Makefile for version number --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5a93045..29dd27a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -27,7 +27,7 @@ jobs: mkdir -p build/bundle/{bin,logs} GOOS=${{matrix.os}} GOARCH=${{matrix.arch}} make -e DESTDIR=build/bundle PREFIX= CFGDIR= install install-scripts tar -C build/bundle -z -cf build/bundle.tar.gz . - echo "version=$(sed -n 's/.*PROGRAM_VERSION\s*=\s*//p' Makefile)" >> "$GITHUB_OUTPUT" + echo "version=$(sed -n 's/.*PROGRAM_VERSION.*=\s*//p' Makefile)" >> "$GITHUB_OUTPUT" - name: Upload thalos-server uses: actions/upload-release-asset@v1 From c769b6a700bff63e2282f3c388f9b46cc80d9b78 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Mon, 26 Aug 2024 13:08:58 +0200 Subject: [PATCH 254/291] typo fixes and added comments --- api/client.go | 9 +++++---- api/message/msgpack/codec.go | 2 +- api/reader.go | 2 +- api/redis/namespace.go | 2 +- api/redis/subscriber.go | 2 +- internal/config/cli.go | 1 + internal/server/message_queue.go | 2 +- 7 files changed, 11 insertions(+), 9 deletions(-) diff --git a/api/client.go b/api/client.go index 9157af0..ad01497 100644 --- a/api/client.go +++ b/api/client.go @@ -11,7 +11,7 @@ import ( type handler func([]byte) -// Client reads and decodes messages from a reader and posts thems to a go channel +// Client reads and decodes messages from a reader and posts them to a go channel type Client struct { reader Reader decoder message.Decoder @@ -35,6 +35,7 @@ func (c *Client) Channel() <-chan any { return c.channel } +// Helper method to post a message to a channel with timeout. func (c *Client) post(msg any) { select { case <-time.After(time.Second): @@ -46,7 +47,7 @@ func (c *Client) worker(channel Channel, h handler) { for { payload, err := c.reader.Read(channel) if err != nil { - // Dont report EOF as an error because it is used + // Don't report EOF as an error because it is used // by readers to signal an graceful end of input. if err != io.EOF { c.post(err) @@ -59,7 +60,7 @@ func (c *Client) worker(channel Channel, h handler) { } // Helper method to decode a message and post and error on the channel if it fails. -// Returns true if successfull. false otherwise +// Returns true if successful. False otherwise func (c *Client) decode(payload []byte, msg any) bool { if err := c.decoder(payload, msg); err != nil { c.post(err) @@ -152,7 +153,7 @@ func (c *Client) Run() { func (c *Client) Close() error { err := c.reader.Close() - // Wait for all goroutines before closing channel. + // Wait for all goroutines to finish before closing channel. c.wg.Wait() close(c.channel) return err diff --git a/api/message/msgpack/codec.go b/api/message/msgpack/codec.go index 2e684a6..ce16975 100644 --- a/api/message/msgpack/codec.go +++ b/api/message/msgpack/codec.go @@ -14,7 +14,7 @@ func createCodec() message.Codec { handle.MapType = reflect.TypeOf(map[string]any(nil)) handle.Canonical = true - // Wierd name but this is needed for the newest version of msgpack + // Weird name but this is needed for the newest version of msgpack // that has support for time and string datatypes etc. handle.WriteExt = true diff --git a/api/reader.go b/api/reader.go index 21b4be7..5dd719e 100644 --- a/api/reader.go +++ b/api/reader.go @@ -6,7 +6,7 @@ package api // This is a low-level interface typically implemented by backend drivers type Reader interface { // Read a message from a channel. - // Read may block until a message is ready or an error occured. + // Read may block until a message is ready or an error occurred. // // io.EOF is returned from a reader when there is no more data to be read. // If Read returns io.EOF all subsequent calls must also return io.EOF diff --git a/api/redis/namespace.go b/api/redis/namespace.go index 4e0167c..d02d579 100644 --- a/api/redis/namespace.go +++ b/api/redis/namespace.go @@ -19,7 +19,7 @@ const ( // // Contains a prefix and chain_id to guard keys against collision. // Prefix should be sufficient to not collide with other application using the same redis database. -// chain_id should be ok to not let multiple reader with different chains to write to the same channels. +// chain_id should be fine to not let multiple reader with different chains to write to the same channels. type Namespace struct { Prefix string diff --git a/api/redis/subscriber.go b/api/redis/subscriber.go index 07218ba..c8c7099 100644 --- a/api/redis/subscriber.go +++ b/api/redis/subscriber.go @@ -48,7 +48,7 @@ func NewSubscriber(ctx context.Context, client *redis.Client, ns Namespace, opti return sub } -// worker reads messages from redis pubsub and forwards them to +// worker reads messages from Redis pubsub and forwards them to // correct channels. func (s *Subscriber) worker() { for msg := range s.sub.Channel() { diff --git a/internal/config/cli.go b/internal/config/cli.go index a23794a..3b22223 100644 --- a/internal/config/cli.go +++ b/internal/config/cli.go @@ -7,6 +7,7 @@ import ( "github.com/spf13/pflag" ) +// Get a flag set with all flags mapping to a config value. func GetFlags() *pflag.FlagSet { flags := pflag.FlagSet{} diff --git a/internal/server/message_queue.go b/internal/server/message_queue.go index 50386ec..da46da5 100644 --- a/internal/server/message_queue.go +++ b/internal/server/message_queue.go @@ -13,7 +13,7 @@ type MessageQueue struct { // Writer to write messages to writer driver.Writer - // encoder to encode messages with + // Encoder to encode messages with encode message.Encoder } From 71977203508e0749b26ca7178ee1554bb2a87f43 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Mon, 26 Aug 2024 13:14:00 +0200 Subject: [PATCH 255/291] small fix to cli descriptions and flag help texts --- cmd/tools/main.go | 2 +- internal/config/cli.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/tools/main.go b/cmd/tools/main.go index 3e131ac..35b8684 100644 --- a/cmd/tools/main.go +++ b/cmd/tools/main.go @@ -11,7 +11,7 @@ var VersionString string = "dev" func main() { rootCmd := &cobra.Command{ Use: "thalos-tools", - Short: "Collection of tools for dealing with the thalos application", + Short: "Collection of tools for dealing with the Thalos application", FParseErrWhitelist: cobra.FParseErrWhitelist{ UnknownFlags: true, }, diff --git a/internal/config/cli.go b/internal/config/cli.go index 3b22223..e5e12db 100644 --- a/internal/config/cli.go +++ b/internal/config/cli.go @@ -43,14 +43,14 @@ func GetFlags() *pflag.FlagSet { flags.Uint32("start-block", shipclient.NULL_BLOCK_NUMBER, "Start to stream from this block") flags.Uint32("end-block", shipclient.NULL_BLOCK_NUMBER, "Stop streaming when this block is reached") - flags.Lookup("start-block").DefValue = "config value, cache, head from api" + flags.Lookup("start-block").DefValue = "Config value, cache, head from api" flags.Lookup("end-block").DefValue = "none" flags.Bool("irreversible-only", false, "Only stream irreversible blocks from ship") flags.Int("max-msg-in-flight", 10, "Maximum messages that can be sent from SHIP without acknowledgement") flags.String("chain", "", "ChainID used in channel namespace, can be any string (default from api)") - flags.StringSlice("blacklist", []string{}, "Define a list of 'contract:action' pairs that will be blacklisted (thalos will not process those actions)") + flags.StringSlice("blacklist", []string{}, "Define a list of 'contract:action' pairs that will be blacklisted (Thalos will not process those actions)") flags.Bool("blacklist-is-whitelist", false, "Thalos will treat the blacklist as a whitelist") return &flags From 0e94ca26889ef9c8cb57a1c65ac429ec1d85e17a Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Mon, 26 Aug 2024 14:55:43 +0200 Subject: [PATCH 256/291] debian/changelog: typo fix --- debian/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index f423520..69f3389 100644 --- a/debian/changelog +++ b/debian/changelog @@ -45,7 +45,7 @@ thalos (1.1.2~rc4) bionic focal jammy; urgency=medium * API: Fix a bug regarding json timestamp being encoded/decoded with wrong format - * Implement action blacklist, it is not possible to configure a blacklist + * Implement action blacklist, it is now possible to configure a blacklist that will be used to filter out processing of unwanted contracts/actions. -- Henrik Hautakoski Sun, 23 Jun 2024 14:55:03 +0200 From 7e42c27194b1cd1d06ee1aa194ccf4ce945169fb Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Thu, 29 Aug 2024 14:54:30 +0200 Subject: [PATCH 257/291] bump github.com/shufflingpixels/antelope-go to v0.1.5 --- go.mod | 18 +++++++++++++++--- go.sum | 49 ++++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 59 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index ca4a378..b8a0d91 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/eosswedenorg/thalos -go 1.20 +go 1.21 require ( github.com/cenkalti/backoff/v4 v4.2.1 @@ -14,7 +14,7 @@ require ( github.com/mitchellh/mapstructure v1.5.0 github.com/nikoksr/notify v0.41.0 github.com/redis/go-redis/v9 v9.5.1 - github.com/shufflingpixels/antelope-go v0.1.4 + github.com/shufflingpixels/antelope-go v0.1.5 github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.8.0 github.com/spf13/pflag v1.0.5 @@ -24,17 +24,21 @@ require ( ) require ( + github.com/andybalholm/brotli v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cloudflare/circl v1.3.7 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/go-telegram-bot-api/telegram-bot-api v4.6.4+incompatible // indirect github.com/google/go-cmp v0.6.0 // indirect + github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7 // indirect github.com/gorilla/websocket v1.5.1 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/hcl v1.0.0 // indirect - github.com/imroc/req/v3 v3.7.6 // indirect + github.com/imroc/req/v3 v3.43.4 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.17.8 // indirect @@ -42,10 +46,14 @@ require ( github.com/magiconair/properties v1.8.7 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/onsi/ginkgo/v2 v2.16.0 // indirect github.com/onsi/gomega v1.31.1 // indirect github.com/pelletier/go-toml/v2 v2.1.1 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/quic-go/qpack v0.4.0 // indirect + github.com/quic-go/quic-go v0.41.0 // indirect + github.com/refraction-networking/utls v1.6.3 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/shufflingpixels/jsontime-go v0.0.0-20240622163621-cf4b2804c92d // indirect @@ -58,12 +66,16 @@ require ( github.com/vmihailenco/go-tinylfu v0.2.2 // indirect github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect + go.uber.org/mock v0.4.0 // indirect go.uber.org/multierr v1.11.0 // indirect + golang.org/x/crypto v0.22.0 // indirect golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 // indirect + golang.org/x/mod v0.16.0 // indirect golang.org/x/net v0.24.0 // indirect golang.org/x/sync v0.7.0 // indirect golang.org/x/sys v0.19.0 // indirect golang.org/x/text v0.14.0 // indirect + golang.org/x/tools v0.19.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/ini.v1 v1.67.0 // indirect ) diff --git a/go.sum b/go.sum index 41dd453..e7dad3b 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,13 @@ github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a h1:HbKu58rmZpUGpz5+4FfNmIU+FmZg2P3Xaj2v2bfNWmk= +github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc= github.com/alicebob/miniredis/v2 v2.30.2 h1:lc1UAUT9ZA7h4srlfBmBt2aorm5Yftk9nBjxz7EyY9I= +github.com/alicebob/miniredis/v2 v2.30.2/go.mod h1:b25qWj4fCEsBeAAR2mlb0ufImGC6uH3VlUfb/HS5zKg= +github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= +github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -11,6 +17,8 @@ github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU= +github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -26,16 +34,21 @@ github.com/eosswedenorg-go/antelope-ship-client v0.3.0/go.mod h1:F3nAzlbcANY6zvZ github.com/eosswedenorg-go/pid v1.0.1 h1:W4AEnnNwb041SpNR1uTZ/KbJ0OTA5eqiqIR1Q5Ah6A0= github.com/eosswedenorg-go/pid v1.0.1/go.mod h1:wiOB/JXGt4YA3+T0j0xmCGSc3Jxzb7Ti/Ftli1fgWu4= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-redis/cache/v9 v9.0.0 h1:0thdtFo0xJi0/WXbRVu8B066z8OvVymXTJGaXrVWnN0= github.com/go-redis/cache/v9 v9.0.0/go.mod h1:cMwi1N8ASBOufbIvk7cdXe2PbPjK/WMRL95FFHWsSgI= github.com/go-redis/redismock/v9 v9.2.0 h1:ZrMYQeKPECZPjOj5u9eyOjg8Nnb0BS9lkVIZ6IpsKLw= github.com/go-redis/redismock/v9 v9.2.0/go.mod h1:18KHfGDK4Y6c2R0H38EUGWAdc7ZQS9gfYxc94k7rWT0= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/go-telegram-bot-api/telegram-bot-api v4.6.4+incompatible h1:2cauKuaELYAEARXRkq2LrJ0yDDv1rW7+wrTEdVL3uaU= github.com/go-telegram-bot-api/telegram-bot-api v4.6.4+incompatible/go.mod h1:qf9acutJ8cwBUhm1bqgz6Bei9/C/c93FPDljKWwsOgM= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -47,6 +60,8 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -57,6 +72,8 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7 h1:y3N7Bm7Y9/CtpiVkw/ZWj6lSlDF3F74SfKwfTCer72Q= +github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -68,11 +85,12 @@ github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/imroc/req/v3 v3.7.6 h1:SUVWgFt/dJsSzpzpnc8pHdL79zoE6O8FSCfNvbTZXVU= -github.com/imroc/req/v3 v3.7.6/go.mod h1:3JIicOKEDHfCSYYNLb/ObZNpx64EV5y40VlHMwhUCzU= +github.com/imroc/req/v3 v3.43.4 h1:NSXlB5dELZuxzGEFRWLWEQ9dQmh8d9pUMPa7MevK1K4= +github.com/imroc/req/v3 v3.43.4/go.mod h1:SQIz5iYop16MJxbo8ib+4LnostGCok8NQf8ToyQc2xA= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jordan-wright/email v4.0.1-0.20210109023952-943e75fe5223+incompatible h1:jdpOPRN1zP63Td1hDQbZW73xKmzDvZHzVdNYxhnTMDA= +github.com/jordan-wright/email v4.0.1-0.20210109023952-943e75fe5223+incompatible/go.mod h1:1c7szIrayyPPB/987hsnvNzLushdWf4o/79s3P08L8A= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= @@ -85,6 +103,7 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -119,6 +138,8 @@ github.com/onsi/ginkgo/v2 v2.3.0/go.mod h1:Eew0uilEqZmIEZr8JrvYlvOM7Rr6xzTmMV8Ay github.com/onsi/ginkgo/v2 v2.4.0/go.mod h1:iHkDK1fKGcBoEHT5W7YBq4RFWaQulw+caOMkAt4OrFo= github.com/onsi/ginkgo/v2 v2.5.0/go.mod h1:Luc4sArBICYCS8THh8v3i3i5CuSZO+RaQRaJoeNwomw= github.com/onsi/ginkgo/v2 v2.7.0/go.mod h1:yjiuMwPokqY1XauOgju45q3sJt6VzQ/Fict1LFVcsAo= +github.com/onsi/ginkgo/v2 v2.16.0 h1:7q1w9frJDzninhXxjZd+Y/x54XNjG/UlRLIYPZafsPM= +github.com/onsi/ginkgo/v2 v2.16.0/go.mod h1:llBI3WDLL9Z6taip6f33H76YcWtJv+7R3HigUjbIBOs= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= @@ -138,18 +159,25 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/quic-go/qpack v0.4.0 h1:Cr9BXA1sQS2SmDUWjSofMPNKmvF6IiIfDRmgU0w1ZCo= +github.com/quic-go/qpack v0.4.0/go.mod h1:UZVnYIfi5GRk+zI9UMaCPsmZ2xKJP7XBUvVyT1Knj9A= +github.com/quic-go/quic-go v0.41.0 h1:aD8MmHfgqTURWNJy48IYFg2OnxwHT3JL7ahGs73lb4k= +github.com/quic-go/quic-go v0.41.0/go.mod h1:qCkNjqczPEvgsOnxZ0eCD14lv+B2LHlFAB++CNOh9hA= github.com/redis/go-redis/v9 v9.0.0-rc.4/go.mod h1:Vo3EsyWnicKnSKCA7HhgnvnyA74wOA69Cd2Meli5mmA= github.com/redis/go-redis/v9 v9.5.1 h1:H1X4D3yHPaYrkL5X06Wh6xNVM/pX0Ft4RV0vMGvLBh8= github.com/redis/go-redis/v9 v9.5.1/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= +github.com/refraction-networking/utls v1.6.3 h1:MFOfRN35sSx6K5AZNIoESsBuBxS2LCgRilRIdHb6fDc= +github.com/refraction-networking/utls v1.6.3/go.mod h1:yil9+7qSl+gBwJqztoQseO6Pr3h62pQoY1lXiNR/FPs= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= -github.com/shufflingpixels/antelope-go v0.1.4 h1:MInk3tEo69OA/U6tZWXGrVzdQhrDFX8UfUkGq1Lq/2Y= -github.com/shufflingpixels/antelope-go v0.1.4/go.mod h1:Bi+zmFI5s7Qv2hx3v9FAcDyf4N4MUs5zaKNa2CetmSg= +github.com/shufflingpixels/antelope-go v0.1.5 h1:N0jCC5bya5shBb96Ff2cCpN+Yw99YldVjWvr0qoiW3E= +github.com/shufflingpixels/antelope-go v0.1.5/go.mod h1:VFULwUB/YfNZeZFzClTNrLyIKWChRhnPvxdzapeOcm0= github.com/shufflingpixels/jsontime-go v0.0.0-20240622163621-cf4b2804c92d h1:nju7jR1Kf210tArPT6l//HlfLLFnvje1BWl5TSRsohQ= github.com/shufflingpixels/jsontime-go v0.0.0-20240622163621-cf4b2804c92d/go.mod h1:W0TaKyg3kDqWmFUxlax3qAls/lRdo12EseM6f4f0dzE= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= @@ -196,6 +224,9 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/gopher-lua v1.1.0 h1:BojcDhfyDWgU2f2TOzYK/g5p2gxMrku8oupLDqlnSqE= +github.com/yuin/gopher-lua v1.1.0/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw= +go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU= +go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -203,6 +234,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= +golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= +golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 h1:LfspQV/FYTatPTr/3HzIcmiUFH7PGP+OQ6mgDYo3yuQ= golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -210,6 +243,8 @@ golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic= +golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -218,7 +253,6 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220111093109-d55c255bac03/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= @@ -284,6 +318,8 @@ golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= +golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw= +golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -297,6 +333,8 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -316,3 +354,4 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU= +gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= From 92ac8521ed83281211c376c400bc87c0b8b141c7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 29 Aug 2024 13:28:17 +0000 Subject: [PATCH 258/291] build(deps): bump github.com/quic-go/quic-go from 0.41.0 to 0.42.0 Bumps [github.com/quic-go/quic-go](https://github.com/quic-go/quic-go) from 0.41.0 to 0.42.0. - [Release notes](https://github.com/quic-go/quic-go/releases) - [Changelog](https://github.com/quic-go/quic-go/blob/master/Changelog.md) - [Commits](https://github.com/quic-go/quic-go/compare/v0.41.0...v0.42.0) --- updated-dependencies: - dependency-name: github.com/quic-go/quic-go dependency-type: indirect ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index b8a0d91..d11c158 100644 --- a/go.mod +++ b/go.mod @@ -52,7 +52,7 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/quic-go/qpack v0.4.0 // indirect - github.com/quic-go/quic-go v0.41.0 // indirect + github.com/quic-go/quic-go v0.42.0 // indirect github.com/refraction-networking/utls v1.6.3 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect diff --git a/go.sum b/go.sum index e7dad3b..36fcd3c 100644 --- a/go.sum +++ b/go.sum @@ -161,8 +161,8 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/quic-go/qpack v0.4.0 h1:Cr9BXA1sQS2SmDUWjSofMPNKmvF6IiIfDRmgU0w1ZCo= github.com/quic-go/qpack v0.4.0/go.mod h1:UZVnYIfi5GRk+zI9UMaCPsmZ2xKJP7XBUvVyT1Knj9A= -github.com/quic-go/quic-go v0.41.0 h1:aD8MmHfgqTURWNJy48IYFg2OnxwHT3JL7ahGs73lb4k= -github.com/quic-go/quic-go v0.41.0/go.mod h1:qCkNjqczPEvgsOnxZ0eCD14lv+B2LHlFAB++CNOh9hA= +github.com/quic-go/quic-go v0.42.0 h1:uSfdap0eveIl8KXnipv9K7nlwZ5IqLlYOpJ58u5utpM= +github.com/quic-go/quic-go v0.42.0/go.mod h1:132kz4kL3F9vxhW3CtQJLDVwcFe5wdWeJXXijhsO57M= github.com/redis/go-redis/v9 v9.0.0-rc.4/go.mod h1:Vo3EsyWnicKnSKCA7HhgnvnyA74wOA69Cd2Meli5mmA= github.com/redis/go-redis/v9 v9.5.1 h1:H1X4D3yHPaYrkL5X06Wh6xNVM/pX0Ft4RV0vMGvLBh8= github.com/redis/go-redis/v9 v9.5.1/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= @@ -311,6 +311,8 @@ golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= From 982b920576dbe5eb487c249d2e0fe665087d5bfa Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Thu, 29 Aug 2024 15:20:16 +0200 Subject: [PATCH 259/291] .github/workflows/test.yml: dont test on 1.20 --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2df9f11..af79d53 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,7 +10,7 @@ jobs: strategy: fail-fast: false matrix: - go-version: ["1.20", "1.21"] + go-version: ["1.21"] arch: [ 386, amd64 ] runs-on: ubuntu-latest name: Test (${{matrix.arch}} go v${{ matrix.go-version }}) @@ -32,7 +32,7 @@ jobs: strategy: fail-fast: false matrix: - tag: ["1.20-alpine3.19", "1.21-alpine3.19"] + tag: [ "1.21-alpine3.19"] runs-on: ubuntu-latest name: Test alpine (${{ matrix.tag }}) container: From a3c428ee5cb27ab1c37e71acfe89cab0e3201249 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Thu, 29 Aug 2024 15:33:45 +0200 Subject: [PATCH 260/291] Version 1.1.5 --- Makefile | 2 +- debian/changelog | 13 +++++++++++++ docker/Dockerfile | 2 +- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index a139da8..f4570d5 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ GO=go GOLDFLAGS=-v -s -w -X main.VersionString=$(PROGRAM_VERSION) GOBUILDFLAGS=-v -p $(shell nproc) -ldflags="$(GOLDFLAGS)" PROGRAM=thalos-server -PROGRAM_VERSION ?= 1.1.5-rc1 +PROGRAM_VERSION ?= 1.1.5 PREFIX=/usr/local BINDIR=$(PREFIX)/bin CFGDIR=$(PREFIX)/etc/thalos diff --git a/debian/changelog b/debian/changelog index 69f3389..102efc0 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,16 @@ +thalos (1.1.5) bionic focal jammy; urgency=medium + + * New config section: `cache` + * New CLI flag: `cache` specify what cache driver to use + * New CLI flag: `abi-cache-api-timeout` configure the timeout for the HTTP + request made when Thalos wants to fetch a ABI from the api. + * API Table Deltas: abi decode the data in `value` field for contract_row deltas. + * golang: update github.com/shufflingpixels/antelope-go to v0.1.5 + * golang: update github.com/quic-go/quic-go from 0.41.0 to 0.42.0 + * golang: version 1.20 can no longer be used to build the project. + + -- Henrik Hautakoski Thu, 29 Aug 2024 15:33:17 +0200 + thalos (1.1.5~rc1) bionic focal jammy; urgency=medium * New config section: `cache` diff --git a/docker/Dockerfile b/docker/Dockerfile index 066533c..c379f15 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,6 +1,6 @@ FROM alpine:latest LABEL maintainer="Henrik Hautakoski " -ARG VERSION=1.1.5-rc1 +ARG VERSION=1.1.5 WORKDIR /thalos ADD --chmod=755 https://github.com/eosswedenorg/thalos/releases/download/v$VERSION/thalos-server-${VERSION}-linux-amd64-musl thalos-server ENTRYPOINT [ "./thalos-server" ] From 08fcdb0590cc71fc06813e512f3691c2f4687402 Mon Sep 17 00:00:00 2001 From: Artem Maliuga Date: Wed, 9 Oct 2024 20:17:04 +0300 Subject: [PATCH 261/291] Fix typo in config.example.yml --- config.example.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.example.yml b/config.example.yml index 85c289d..212276f 100644 --- a/config.example.yml +++ b/config.example.yml @@ -80,7 +80,7 @@ redis: user: "" # Password to use when authenticating - pasword: "" + password: "" # database index db: 0 From ef0ac7a1b88905af21c2f2019d8d109eb5f0c28d Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Thu, 29 Aug 2024 15:41:18 +0200 Subject: [PATCH 262/291] .github/workflows/release.yml: need to update version regex for musl builds --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 29dd27a..4f8760d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -81,7 +81,7 @@ jobs: mkdir -p build/bundle/{bin,logs} GOARCH=${{matrix.arch}} make -e DESTDIR=build/bundle PREFIX= CFGDIR= install install-scripts tar -C build/bundle -z -cf build/bundle.tar.gz . - echo "version=$(sed -n 's/.*PROGRAM_VERSION\s*=\s*//p' Makefile)" >> "$GITHUB_OUTPUT" + echo "version=$(sed -n 's/.*PROGRAM_VERSION.*=\s*//p' Makefile)" >> "$GITHUB_OUTPUT" - name: Upload thalos-server uses: actions/upload-release-asset@v1 From 4f46188ed4a388ffa16b7e79f0642528cf25e68c Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Fri, 30 Aug 2024 13:10:55 +0200 Subject: [PATCH 263/291] README.md: Link to docker page --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index fdf2c0e..ea487ed 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,10 @@ Consult the [documentation](https://thalos.waxsweden.org/docs) for more informat Join the discussion on [telegram](https://t.me/antelopethalos) +## Docker images + +Docker images can be found [here](https://github.com/eosswedenorg/thalos/pkgs/container/thalos) + ## Compiling You will need golang version `1.20` or later to compile the source. From 020f81ed654b4d5356a1f2f75b58eda0c0c9691b Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Tue, 3 Sep 2024 21:51:52 +0200 Subject: [PATCH 264/291] README.md: Update minimum go version --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ea487ed..09ab23c 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ Docker images can be found [here](https://github.com/eosswedenorg/thalos/pkgs/co ## Compiling -You will need golang version `1.20` or later to compile the source. +You will need golang version `1.21` or later to compile the source. Compile using make: From 70aa6cd29550c29e19d57a0bff48ab3dad666051 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 8 Sep 2024 13:29:43 +0200 Subject: [PATCH 265/291] api/channel_test.go: rearange fields. --- api/channel_test.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/api/channel_test.go b/api/channel_test.go index caaf4e7..ae2edf1 100644 --- a/api/channel_test.go +++ b/api/channel_test.go @@ -55,13 +55,13 @@ func TestChannel_Is(t *testing.T) { func TestChannel_Format(t *testing.T) { tests := []struct { name string - c Channel delim string want string + c Channel }{ - {"Empty", Channel{}, ":", ""}, - {"Alot#1", Channel{"one", "two", "three"}, "-", "one-two-three"}, - {"Alot#2", Channel{"first", "second"}, ":", "first:second"}, + {"Empty", ":", "", Channel{}}, + {"Alot#1", "-", "one-two-three", Channel{"one", "two", "three"}}, + {"Alot#2", ":", "first:second", Channel{"first", "second"}}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -75,11 +75,11 @@ func TestChannel_Format(t *testing.T) { func TestChannel_String(t *testing.T) { tests := []struct { name string - c Channel want string + c Channel }{ - {"Empty", Channel{}, ""}, - {"Alot", Channel{"one", "two", "three"}, "one/two/three"}, + {"Empty", "", Channel{}}, + {"Alot", "one/two/three", Channel{"one", "two", "three"}}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -93,13 +93,13 @@ func TestChannel_String(t *testing.T) { func TestChannel_Type(t *testing.T) { tests := []struct { name string - c Channel want string + c Channel }{ - {"Empty", Channel{}, "unknown"}, - {"Heartbeat", HeartbeatChannel, "heartbeat"}, - {"Transaction", TransactionChannel, "transactions"}, - {"Actions", ActionChannel{}.Channel(), "actions"}, + {"Empty", "unknown", Channel{}}, + {"Heartbeat", "heartbeat", HeartbeatChannel}, + {"Transaction", "transactions", TransactionChannel}, + {"Actions", "actions", ActionChannel{}.Channel()}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { From 84547e5ad3f97f87ba786efd67a0b29eaa84255d Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Thu, 12 Sep 2024 08:20:16 +0200 Subject: [PATCH 266/291] minor style fixes. --- debian/thalos.postinst | 10 +++++----- debian/thalos.postrm | 10 ++++------ 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/debian/thalos.postinst b/debian/thalos.postinst index 9c31f4b..a643c01 100644 --- a/debian/thalos.postinst +++ b/debian/thalos.postinst @@ -2,14 +2,14 @@ set -e if [ "$1" = 'configure' ]; then - adduser --force-badname --system --home /nonexistent \ + adduser --force-badname --system --home /nonexistent \ --group --no-create-home --quiet thalos || true - # Create log directory - mkdir -p /var/log/thalos - chown thalos:adm /var/log/thalos + # Create log directory + mkdir -p /var/log/thalos + chown thalos:adm /var/log/thalos fi #DEBHELPER# -exit 0 \ No newline at end of file +exit 0 diff --git a/debian/thalos.postrm b/debian/thalos.postrm index 08a6475..dde4859 100644 --- a/debian/thalos.postrm +++ b/debian/thalos.postrm @@ -3,11 +3,9 @@ set -e #DEBHELPER# - -if [ "${1}" = "purge" ] -then - deluser --quiet thalos > /dev/null || true - rm -rf /var/log/thalos +if [ "${1}" = "purge" ]; then + deluser --quiet thalos >/dev/null || true + rm -rf /var/log/thalos fi -exit 0 \ No newline at end of file +exit 0 From 6231142e4a77315c0ada53ec4d92fe62e004682b Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Wed, 16 Oct 2024 16:22:32 +0200 Subject: [PATCH 267/291] makefile: make sure we apppend to GOBULDFLAGS if user wants to add their own. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index f4570d5..d4faff4 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ GO=go GOLDFLAGS=-v -s -w -X main.VersionString=$(PROGRAM_VERSION) -GOBUILDFLAGS=-v -p $(shell nproc) -ldflags="$(GOLDFLAGS)" +GOBUILDFLAGS+=-v -p $(shell nproc) -ldflags="$(GOLDFLAGS)" PROGRAM=thalos-server PROGRAM_VERSION ?= 1.1.5 PREFIX=/usr/local From ffd2504834fa9f3b72b06ec58214e25c721c73c5 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Wed, 16 Oct 2024 16:32:32 +0200 Subject: [PATCH 268/291] Version 1.1.6 --- Makefile | 2 +- debian/changelog | 17 +++++++++++++++++ docker/Dockerfile | 2 +- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index d4faff4..7969d11 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ GO=go GOLDFLAGS=-v -s -w -X main.VersionString=$(PROGRAM_VERSION) GOBUILDFLAGS+=-v -p $(shell nproc) -ldflags="$(GOLDFLAGS)" PROGRAM=thalos-server -PROGRAM_VERSION ?= 1.1.5 +PROGRAM_VERSION ?= 1.1.6 PREFIX=/usr/local BINDIR=$(PREFIX)/bin CFGDIR=$(PREFIX)/etc/thalos diff --git a/debian/changelog b/debian/changelog index 102efc0..36c1045 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,19 @@ +thalos (1.1.6) bionic focal jammy; urgency=medium + + [ Henrik Hautakoski ] + * makefile: make sure we apppend to GOBULDFLAGS if user wants to add their own. + * minor style fixes. + * api/channel_test.go: rearange fields. + * README.md: Update minimum go version + * README.md: Link to docker page + * .github/workflows/release.yml: need to update version regex for musl builds + + [ Avm07 ] + * Fix typo in config.example.yml + + +-- Henrik Hautakoski Wed, 16 Oct 2024 16:23:47 +0200 + thalos (1.1.5) bionic focal jammy; urgency=medium * New config section: `cache` @@ -218,3 +234,4 @@ thalos-server (0.1.0) bionic focal jammy; urgency=medium Initial release. -- Henrik Hautakoski Sun, 14 May 2023 18:17:35 +0200 + diff --git a/docker/Dockerfile b/docker/Dockerfile index c379f15..a6410a2 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,6 +1,6 @@ FROM alpine:latest LABEL maintainer="Henrik Hautakoski " -ARG VERSION=1.1.5 +ARG VERSION=1.1.6 WORKDIR /thalos ADD --chmod=755 https://github.com/eosswedenorg/thalos/releases/download/v$VERSION/thalos-server-${VERSION}-linux-amd64-musl thalos-server ENTRYPOINT [ "./thalos-server" ] From bedb8a92e8d0d09ad03957a785832db00964ab0e Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Mon, 21 Oct 2024 12:24:29 +0200 Subject: [PATCH 269/291] config: adding Ship.EnableTableDeltas and table_deltas cli flag --- internal/config/builder.go | 1 + internal/config/builder_test.go | 2 ++ internal/config/cli.go | 2 ++ internal/config/config.go | 1 + 4 files changed, 6 insertions(+) diff --git a/internal/config/builder.go b/internal/config/builder.go index 6429f24..e22bf45 100644 --- a/internal/config/builder.go +++ b/internal/config/builder.go @@ -59,6 +59,7 @@ func NewBuilder() *Builder { "ship.chain": "chain", "ship.blacklist": "blacklist", "ship.blacklist_is_whitelist": "blacklist-is-whitelist", + "ship.table_deltas": "table-deltas", }, } } diff --git a/internal/config/builder_test.go b/internal/config/builder_test.go index 234776a..b133353 100644 --- a/internal/config/builder_test.go +++ b/internal/config/builder_test.go @@ -127,6 +127,7 @@ func TestBuilder_WithDefaultConfig(t *testing.T) { StartBlockNum: shipclient.NULL_BLOCK_NUMBER, EndBlockNum: shipclient.NULL_BLOCK_NUMBER, MaxMessagesInFlight: 10, + EnableTableDeltas: true, }, Redis: RedisConfig{ Addr: "127.0.0.1:6379", @@ -174,6 +175,7 @@ func TestBuilder_Flags(t *testing.T) { require.NoError(t, flags.Set("chain", "wax")) require.NoError(t, flags.Set("blacklist", "contract:action1,contract:action2,contract2:action1")) require.NoError(t, flags.Set("blacklist-is-whitelist", "true")) + require.NoError(t, flags.Set("table-deltas", "false")) cfg, err := NewBuilder(). SetSource(bytes.NewReader([]byte(``))). diff --git a/internal/config/cli.go b/internal/config/cli.go index e5e12db..d4f26ac 100644 --- a/internal/config/cli.go +++ b/internal/config/cli.go @@ -46,6 +46,8 @@ func GetFlags() *pflag.FlagSet { flags.Lookup("start-block").DefValue = "Config value, cache, head from api" flags.Lookup("end-block").DefValue = "none" + flags.Bool("table-deltas", true, "True if thalos should receive and process table deltas from ship.") + flags.Bool("irreversible-only", false, "Only stream irreversible blocks from ship") flags.Int("max-msg-in-flight", 10, "Maximum messages that can be sent from SHIP without acknowledgement") flags.String("chain", "", "ChainID used in channel namespace, can be any string (default from api)") diff --git a/internal/config/config.go b/internal/config/config.go index e1d815e..feabb87 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -39,6 +39,7 @@ type ShipConfig struct { Chain string `yaml:"chain" mapstructure:"chain"` Blacklist types.Blacklist `yaml:"blacklist" mapstructure:"blacklist"` BlacklistIsWhitelist bool `yaml:"blacklist_is_whitelist" mapstructure:"blacklist_is_whitelist"` + EnableTableDeltas bool `yaml:"table_deltas" mapstructure:"table_deltas"` } type Config struct { From 33c983f6c5d091ca63d277e5373ea7292ef0855e Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Mon, 21 Oct 2024 12:27:29 +0200 Subject: [PATCH 270/291] internal/server/ship_processor.go: adding FetchDeltas() --- internal/server/ship_processor.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/internal/server/ship_processor.go b/internal/server/ship_processor.go index e3ab11e..a722722 100644 --- a/internal/server/ship_processor.go +++ b/internal/server/ship_processor.go @@ -66,6 +66,15 @@ func SpawnProccessor(shipStream *shipclient.Stream, loader StateLoader, saver St return processor } +func (processor *ShipProcessor) FetchDeltas(value bool) { + if value { + // empty callback will signal that traces should be included in the response from ship. + processor.shipStream.TableDeltaHandler = func(*ship.TableDeltaArray) {} + } else { + processor.shipStream.TableDeltaHandler = nil + } +} + func (processor *ShipProcessor) SetBlacklist(list types.Blacklist) { processor.blacklist = list } From 0cb86ff7719ba678a4479be12f53f445f64d48b1 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Mon, 21 Oct 2024 12:28:06 +0200 Subject: [PATCH 271/291] cmd/thalos/server.go: configure fetching of table deltas. --- cmd/thalos/server.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/thalos/server.go b/cmd/thalos/server.go index 5813aee..271452c 100644 --- a/cmd/thalos/server.go +++ b/cmd/thalos/server.go @@ -392,6 +392,7 @@ func serverCmd(cmd *cobra.Command, args []string) { ) processor.SetBlacklist(conf.Ship.Blacklist) + processor.FetchDeltas(conf.Ship.EnableTableDeltas) // Run the application run(conf, shClient, processor) From e2f7efe8b065c0ae716886269de42d40183faf2a Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Mon, 21 Oct 2024 12:28:55 +0200 Subject: [PATCH 272/291] config.example.yml: add ship.table_deltas --- config.example.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/config.example.yml b/config.example.yml index 212276f..efbcd44 100644 --- a/config.example.yml +++ b/config.example.yml @@ -42,6 +42,10 @@ ship: # Request ship to stop sending blocks when reaching this block. #end_block_num: 2000 + # + + # Disable processing of table deltas + # table_deltas: false # Blacklist contract/actions blacklist: From 4ae30f0cc6add676f6c5fc30d671ca0471bd8bab Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Mon, 21 Oct 2024 12:30:48 +0200 Subject: [PATCH 273/291] debian/changelog: format fix --- debian/changelog | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index 36c1045..621bd3b 100644 --- a/debian/changelog +++ b/debian/changelog @@ -11,8 +11,7 @@ thalos (1.1.6) bionic focal jammy; urgency=medium [ Avm07 ] * Fix typo in config.example.yml - --- Henrik Hautakoski Wed, 16 Oct 2024 16:23:47 +0200 + -- Henrik Hautakoski Wed, 16 Oct 2024 16:23:47 +0200 thalos (1.1.5) bionic focal jammy; urgency=medium From 213a31570dc6802b83a7f5005e2ec23f1881e769 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Mon, 21 Oct 2024 12:32:08 +0200 Subject: [PATCH 274/291] Version 1.1.7-rc1 --- Makefile | 2 +- debian/changelog | 6 ++++++ docker/Dockerfile | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 7969d11..c49f3d0 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ GO=go GOLDFLAGS=-v -s -w -X main.VersionString=$(PROGRAM_VERSION) GOBUILDFLAGS+=-v -p $(shell nproc) -ldflags="$(GOLDFLAGS)" PROGRAM=thalos-server -PROGRAM_VERSION ?= 1.1.6 +PROGRAM_VERSION ?= 1.1.7-rc1 PREFIX=/usr/local BINDIR=$(PREFIX)/bin CFGDIR=$(PREFIX)/etc/thalos diff --git a/debian/changelog b/debian/changelog index 621bd3b..d1c6ab8 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +thalos (1.1.7~rc1) bionic focal jammy; urgency=medium + + * Add support to disable processing of table deltas. + + -- Henrik Hautakoski Mon, 21 Oct 2024 12:31:21 +0200 + thalos (1.1.6) bionic focal jammy; urgency=medium [ Henrik Hautakoski ] diff --git a/docker/Dockerfile b/docker/Dockerfile index a6410a2..40e10fb 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,6 +1,6 @@ FROM alpine:latest LABEL maintainer="Henrik Hautakoski " -ARG VERSION=1.1.6 +ARG VERSION=1.1.7-rc1 WORKDIR /thalos ADD --chmod=755 https://github.com/eosswedenorg/thalos/releases/download/v$VERSION/thalos-server-${VERSION}-linux-amd64-musl thalos-server ENTRYPOINT [ "./thalos-server" ] From 50adfa002605c50e30a9f13b28c1790b60ac6e71 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Mon, 21 Oct 2024 13:52:11 +0200 Subject: [PATCH 275/291] config.example.yml: minor fixes. --- config.example.yml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/config.example.yml b/config.example.yml index efbcd44..2dc438e 100644 --- a/config.example.yml +++ b/config.example.yml @@ -38,13 +38,12 @@ ship: # Request ship to start sending blocks from this block. # If not set, the head block reported by the nodeos api is used. - #start_block_num: 1000 + # start_block_num: 1000 # Request ship to stop sending blocks when reaching this block. - #end_block_num: 2000 - # + # end_block_num: 2000 - # Disable processing of table deltas + # If true, Thalos will process table deltas # table_deltas: false # Blacklist contract/actions @@ -71,9 +70,9 @@ ship: # options: [] # Telegram notifications -#telegram: -# id: "123456789:GPdmGPBWvpgHPxlergJLavus-PoAURTjMWP" -# channel: -123456789 +# telegram: +# id: "123456789:GPdmGPBWvpgHPxlergJLavus-PoAURTjMWP" +# channel: -123456789 # Redis settings redis: From 9bdcbacaa0429fa656bb52bd4ad70e29b16a6503 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 3 Nov 2024 12:02:27 +0100 Subject: [PATCH 276/291] go.mod: update eosswedenrg-go/antelope-ship-client to v0.3.2 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index d11c158..4c07b37 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.21 require ( github.com/cenkalti/backoff/v4 v4.2.1 github.com/docker/go-units v0.5.0 - github.com/eosswedenorg-go/antelope-ship-client v0.3.0 + github.com/eosswedenorg-go/antelope-ship-client v0.3.2 github.com/eosswedenorg-go/pid v1.0.1 github.com/eosswedenorg/thalos/api v1.0.0 github.com/go-redis/cache/v9 v9.0.0 diff --git a/go.sum b/go.sum index 36fcd3c..6d81242 100644 --- a/go.sum +++ b/go.sum @@ -29,8 +29,8 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/r github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/eosswedenorg-go/antelope-ship-client v0.3.0 h1:L1DTE2ZeOGSbehCAqVF5GhdaAb3ZYgvFFUhpDWmJ2vQ= -github.com/eosswedenorg-go/antelope-ship-client v0.3.0/go.mod h1:F3nAzlbcANY6zvZ+cTJH9WjJtvJU6hvduBAUAjX2/lA= +github.com/eosswedenorg-go/antelope-ship-client v0.3.2 h1:mDXZkjQ0bTPJClkhoPEP5ltucxql6bR+QixhnQI/Og4= +github.com/eosswedenorg-go/antelope-ship-client v0.3.2/go.mod h1:DnUmaRGxz/V73CtVEJx/fReqhgGzhVyWpOrEKVYQSgE= github.com/eosswedenorg-go/pid v1.0.1 h1:W4AEnnNwb041SpNR1uTZ/KbJ0OTA5eqiqIR1Q5Ah6A0= github.com/eosswedenorg-go/pid v1.0.1/go.mod h1:wiOB/JXGt4YA3+T0j0xmCGSc3Jxzb7Ti/Ftli1fgWu4= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= From d959645836e664d0a75f57bec223e9167aad2376 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 3 Nov 2024 12:03:48 +0100 Subject: [PATCH 277/291] cmd/thalos/server.go: set MaxMessagesInFlight = 1 when creating ship client --- cmd/thalos/server.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/thalos/server.go b/cmd/thalos/server.go index 271452c..72ea05f 100644 --- a/cmd/thalos/server.go +++ b/cmd/thalos/server.go @@ -363,6 +363,7 @@ func serverCmd(cmd *cobra.Command, args []string) { s.StartBlock = conf.Ship.StartBlockNum s.EndBlock = conf.Ship.EndBlockNum s.IrreversibleOnly = conf.Ship.IrreversibleOnly + s.MaxMessagesInFlight = 1 }) // Get codec From 89011b8fc837be3af5901e417341a7f9c08bd5fb Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sun, 3 Nov 2024 12:11:09 +0100 Subject: [PATCH 278/291] Version 1.1.7-rc2 --- Makefile | 2 +- debian/changelog | 10 ++++++++++ docker/Dockerfile | 2 +- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index c49f3d0..627630a 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ GO=go GOLDFLAGS=-v -s -w -X main.VersionString=$(PROGRAM_VERSION) GOBUILDFLAGS+=-v -p $(shell nproc) -ldflags="$(GOLDFLAGS)" PROGRAM=thalos-server -PROGRAM_VERSION ?= 1.1.7-rc1 +PROGRAM_VERSION ?= 1.1.7-rc2 PREFIX=/usr/local BINDIR=$(PREFIX)/bin CFGDIR=$(PREFIX)/etc/thalos diff --git a/debian/changelog b/debian/changelog index d1c6ab8..c2fa6f8 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,13 @@ +thalos (1.1.7~rc2) bionic focal jammy; urgency=medium + + * ship: set MaxMessagesInFlight to 1. This forces the client/server to ack + every message and might be a workaround fix for issue #25 + according to this comment: + https://github.com/AntelopeIO/leap/issues/1358#issuecomment-2276294557 + * golang: update eosswedenrg-go/antelope-ship-client to v0.3.2 + + -- Henrik Hautakoski Sun, 03 Nov 2024 12:04:29 +0100 + thalos (1.1.7~rc1) bionic focal jammy; urgency=medium * Add support to disable processing of table deltas. diff --git a/docker/Dockerfile b/docker/Dockerfile index 40e10fb..73b42e2 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,6 +1,6 @@ FROM alpine:latest LABEL maintainer="Henrik Hautakoski " -ARG VERSION=1.1.7-rc1 +ARG VERSION=1.1.7-rc2 WORKDIR /thalos ADD --chmod=755 https://github.com/eosswedenorg/thalos/releases/download/v$VERSION/thalos-server-${VERSION}-linux-amd64-musl thalos-server ENTRYPOINT [ "./thalos-server" ] From d5bc23a63ed3cb30f43262887ff63614261cff47 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Mon, 11 Nov 2024 19:40:17 +0100 Subject: [PATCH 279/291] Version 1.1.7 --- Makefile | 2 +- debian/changelog | 11 +++++++++++ docker/Dockerfile | 2 +- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 627630a..4e543a7 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ GO=go GOLDFLAGS=-v -s -w -X main.VersionString=$(PROGRAM_VERSION) GOBUILDFLAGS+=-v -p $(shell nproc) -ldflags="$(GOLDFLAGS)" PROGRAM=thalos-server -PROGRAM_VERSION ?= 1.1.7-rc2 +PROGRAM_VERSION ?= 1.1.7 PREFIX=/usr/local BINDIR=$(PREFIX)/bin CFGDIR=$(PREFIX)/etc/thalos diff --git a/debian/changelog b/debian/changelog index c2fa6f8..a3ab8cc 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,14 @@ +thalos (1.1.7) bionic focal jammy; urgency=medium + + * ship: set MaxMessagesInFlight to 1. This forces the client/server to ack + every message and might be a workaround fix for issue #25 + according to this comment: + https://github.com/AntelopeIO/leap/issues/1358#issuecomment-2276294557 + * golang: update eosswedenrg-go/antelope-ship-client to v0.3.2 + * Add support to disable processing of table deltas. + + -- Henrik Hautakoski Mon, 11 Nov 2024 19:38:15 +0100 + thalos (1.1.7~rc2) bionic focal jammy; urgency=medium * ship: set MaxMessagesInFlight to 1. This forces the client/server to ack diff --git a/docker/Dockerfile b/docker/Dockerfile index 73b42e2..40308ee 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,6 +1,6 @@ FROM alpine:latest LABEL maintainer="Henrik Hautakoski " -ARG VERSION=1.1.7-rc2 +ARG VERSION=1.1.7 WORKDIR /thalos ADD --chmod=755 https://github.com/eosswedenorg/thalos/releases/download/v$VERSION/thalos-server-${VERSION}-linux-amd64-musl thalos-server ENTRYPOINT [ "./thalos-server" ] From 120c2acdc6a5b1a35f3c5ead863eedc0b3f63ec3 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Tue, 26 Nov 2024 06:52:14 +0100 Subject: [PATCH 280/291] internal/server/ship_processor.go: refactor processDeltaRows() to reduce nested blocks --- internal/server/ship_processor.go | 65 +++++++++++++++++-------------- 1 file changed, 36 insertions(+), 29 deletions(-) diff --git a/internal/server/ship_processor.go b/internal/server/ship_processor.go index a722722..1c1ee39 100644 --- a/internal/server/ship_processor.go +++ b/internal/server/ship_processor.go @@ -2,6 +2,7 @@ package server import ( "bytes" + "errors" "github.com/eosswedenorg/thalos/api/message" "github.com/eosswedenorg/thalos/internal/abi" @@ -228,41 +229,47 @@ func (processor *ShipProcessor) proccessActionTrace(logger *log.Entry, trace *sh func (processor *ShipProcessor) proccessDeltaRows(logger *log.Entry, table_name string, rows []ship.Row) []message.TableDeltaRow { out := []message.TableDeltaRow{} for _, row := range rows { - - msg := message.TableDeltaRow{ - Present: row.Present, - RawData: row.Data, - } - - if processor.shipABI != nil { - v, err := processor.shipABI.Decode(bytes.NewReader(row.Data), table_name) - if err == nil { - data, err := ship_helper.ParseTableDeltaData(v) - if err == nil { - // Decode contract row data - if table_name == "contract_row" { - dec, err := ship_helper.DecodeContractRow(processor.abi, data) - if err != nil { - logger.WithError(err).Warn("Failed to decode contract row") - } else { - data["value"] = dec - } - } - msg.Data = data - } else { - logger.WithError(err).Error("Failed to parse table delta data") - } - } else { - logger.WithError(err).Error("Failed to decode table delta") - } - } else { - logger.Warn("No SHIP ABI present") + msg, err := processor.proccessDeltaRow(row, table_name) + if err != nil { + logger.WithError(err).Warn("Failed to processs table delta row") } out = append(out, msg) } return out } +func (processor *ShipProcessor) proccessDeltaRow(row ship.Row, table_name string) (message.TableDeltaRow, error) { + msg := message.TableDeltaRow{ + Present: row.Present, + RawData: row.Data, + } + + if processor.shipABI == nil { + return msg, errors.New("No SHIP ABI present") + } + + v, err := processor.shipABI.Decode(bytes.NewReader(row.Data), table_name) + if err != nil { + return msg, errors.New("Failed to decode table delta") + } + data, err := ship_helper.ParseTableDeltaData(v) + if err != nil { + return msg, errors.New("Failed to parse table delta data") + } + + msg.Data = data + + // Decode contract row data + if table_name == "contract_row" { + dec, err := ship_helper.DecodeContractRow(processor.abi, data) + if err != nil { + return msg, errors.New("Failed to decode contract row") + } + msg.Data["value"] = dec + } + return msg, nil +} + // Callback function called by shipclient.Stream when a new block arrives. func (processor *ShipProcessor) processBlock(blockResult *ship.GetBlocksResultV0) { block := ship.SignedBlock{} From eb2032e233e3a4e3260aac35bb59fa09dfb18877 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Wed, 4 Dec 2024 15:14:10 +0100 Subject: [PATCH 281/291] internal/types/blacklist.go: implement wildcard for contracts --- internal/types/blacklist.go | 12 +++++++++++- internal/types/blacklist_test.go | 19 ++++++++++++++++++- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/internal/types/blacklist.go b/internal/types/blacklist.go index 24419c8..b123997 100644 --- a/internal/types/blacklist.go +++ b/internal/types/blacklist.go @@ -31,8 +31,18 @@ func (bl *Blacklist) Add(contract string, action string) { bl.table[contract] = append(bl.table[contract], action) } +func (bl Blacklist) list(contracts ...string) [][]string { + ret := [][]string{} + for _, contract := range contracts { + if v, ok := bl.table[contract]; ok { + ret = append(ret, v) + } + } + return ret +} + func (bl Blacklist) IsAllowed(contract string, action string) bool { - if v, ok := bl.table[contract]; ok { + for _, v := range bl.list(contract, "*") { for _, act := range v { if act == action || act == "*" { return bl.isWhitelist == true diff --git a/internal/types/blacklist_test.go b/internal/types/blacklist_test.go index 631d329..3c6a8a8 100644 --- a/internal/types/blacklist_test.go +++ b/internal/types/blacklist_test.go @@ -52,20 +52,32 @@ func TestBlacklist_IsAllowed(t *testing.T) { func TestBlacklist_IsAllowedWildcard(t *testing.T) { bl := Blacklist{ table: map[string][]string{ - "mycontract": {"*"}, + "mycontract": {"*"}, + "*": {"action1", "action2"}, + "evilcontract": {"evilaction"}, }, } require.False(t, bl.IsAllowed("mycontract", "myaction")) require.False(t, bl.IsAllowed("mycontract", "noop")) require.False(t, bl.IsAllowed("mycontract", "xxx")) + + // Wildcard contract + require.False(t, bl.IsAllowed("somecontract", "action1")) + require.False(t, bl.IsAllowed("someothercontract", "action1")) + require.False(t, bl.IsAllowed("randomcontract", "action2")) + require.False(t, bl.IsAllowed("evilcontract", "action2")) + require.False(t, bl.IsAllowed("evilcontract", "evilaction")) + require.True(t, bl.IsAllowed("xxx", "yyy")) + require.True(t, bl.IsAllowed("evilcontract", "alloweaction")) } func TestBlacklist_Whitelist(t *testing.T) { bl := Blacklist{ table: map[string][]string{ "mycontract": {"myaction", "noop"}, + "*": {"goodaction1", "goodaction2"}, }, } @@ -73,6 +85,11 @@ func TestBlacklist_Whitelist(t *testing.T) { require.True(t, bl.IsAllowed("mycontract", "myaction")) require.True(t, bl.IsAllowed("mycontract", "noop")) + + // Wildcard contract + require.True(t, bl.IsAllowed("mycontract", "goodaction1")) + require.True(t, bl.IsAllowed("someothercontract", "goodaction2")) + require.False(t, bl.IsAllowed("mycontract", "xxx")) require.False(t, bl.IsAllowed("xxx", "yyy")) } From 2006da6a19824bd5abdf5b7aa057121552c2409b Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Wed, 4 Dec 2024 15:15:22 +0100 Subject: [PATCH 282/291] internal/types/blacklist.go: add BlacklistWildcard constant --- internal/types/blacklist.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/internal/types/blacklist.go b/internal/types/blacklist.go index b123997..4606c5e 100644 --- a/internal/types/blacklist.go +++ b/internal/types/blacklist.go @@ -1,5 +1,7 @@ package types +const BlacklistWildcard = "*" + type Blacklist struct { table map[string][]string isWhitelist bool @@ -42,9 +44,9 @@ func (bl Blacklist) list(contracts ...string) [][]string { } func (bl Blacklist) IsAllowed(contract string, action string) bool { - for _, v := range bl.list(contract, "*") { + for _, v := range bl.list(contract, BlacklistWildcard) { for _, act := range v { - if act == action || act == "*" { + if act == action || act == BlacklistWildcard { return bl.isWhitelist == true } } From 96b6a79813d3c18acc9b2664ceb809d09e2d7bd1 Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Wed, 4 Dec 2024 15:21:50 +0100 Subject: [PATCH 283/291] Version v1.1.8-rc1 --- Makefile | 2 +- debian/changelog | 6 ++++++ docker/Dockerfile | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 4e543a7..d4f8331 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ GO=go GOLDFLAGS=-v -s -w -X main.VersionString=$(PROGRAM_VERSION) GOBUILDFLAGS+=-v -p $(shell nproc) -ldflags="$(GOLDFLAGS)" PROGRAM=thalos-server -PROGRAM_VERSION ?= 1.1.7 +PROGRAM_VERSION ?= 1.1.8-rc1 PREFIX=/usr/local BINDIR=$(PREFIX)/bin CFGDIR=$(PREFIX)/etc/thalos diff --git a/debian/changelog b/debian/changelog index a3ab8cc..6e5c94a 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +thalos (1.1.8~rc1) bionic focal jammy; urgency=medium + + * Support for wildcard contracts in Blacklist + + -- Henrik Hautakoski Wed, 04 Dec 2024 15:19:53 +0100 + thalos (1.1.7) bionic focal jammy; urgency=medium * ship: set MaxMessagesInFlight to 1. This forces the client/server to ack diff --git a/docker/Dockerfile b/docker/Dockerfile index 40308ee..8a27563 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,6 +1,6 @@ FROM alpine:latest LABEL maintainer="Henrik Hautakoski " -ARG VERSION=1.1.7 +ARG VERSION=1.1.8-rc1 WORKDIR /thalos ADD --chmod=755 https://github.com/eosswedenorg/thalos/releases/download/v$VERSION/thalos-server-${VERSION}-linux-amd64-musl thalos-server ENTRYPOINT [ "./thalos-server" ] From ece6ccfac19a25d9c1f8fbcabd482d019943be68 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 21 Dec 2024 17:58:53 +0000 Subject: [PATCH 284/291] build(deps): bump golang.org/x/crypto from 0.22.0 to 0.31.0 Bumps [golang.org/x/crypto](https://github.com/golang/crypto) from 0.22.0 to 0.31.0. - [Commits](https://github.com/golang/crypto/compare/v0.22.0...v0.31.0) --- updated-dependencies: - dependency-name: golang.org/x/crypto dependency-type: indirect ... Signed-off-by: dependabot[bot] --- go.mod | 14 +++++++------- go.sum | 28 ++++++++++++++-------------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/go.mod b/go.mod index 4c07b37..807c10d 100644 --- a/go.mod +++ b/go.mod @@ -68,14 +68,14 @@ require ( github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect go.uber.org/mock v0.4.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.22.0 // indirect + golang.org/x/crypto v0.31.0 // indirect golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 // indirect - golang.org/x/mod v0.16.0 // indirect - golang.org/x/net v0.24.0 // indirect - golang.org/x/sync v0.7.0 // indirect - golang.org/x/sys v0.19.0 // indirect - golang.org/x/text v0.14.0 // indirect - golang.org/x/tools v0.19.0 // indirect + golang.org/x/mod v0.17.0 // indirect + golang.org/x/net v0.25.0 // indirect + golang.org/x/sync v0.10.0 // indirect + golang.org/x/sys v0.28.0 // indirect + golang.org/x/text v0.21.0 // indirect + golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/ini.v1 v1.67.0 // indirect ) diff --git a/go.sum b/go.sum index 6d81242..883609a 100644 --- a/go.sum +++ b/go.sum @@ -234,8 +234,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= -golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= -golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= +golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 h1:LfspQV/FYTatPTr/3HzIcmiUFH7PGP+OQ6mgDYo3yuQ= golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -243,8 +243,8 @@ golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic= -golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -260,16 +260,16 @@ golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= -golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= -golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -294,8 +294,8 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= -golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -309,8 +309,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -320,8 +320,8 @@ golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= -golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw= -golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From ce80405254736ff1163ffbec840b4dae9cac67bf Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sat, 21 Dec 2024 19:14:26 +0100 Subject: [PATCH 285/291] github workflows: add permissions. --- .github/workflows/devbuild.yaml | 3 +++ .github/workflows/release.yml | 2 ++ .github/workflows/test.yml | 3 +++ 3 files changed, 8 insertions(+) diff --git a/.github/workflows/devbuild.yaml b/.github/workflows/devbuild.yaml index 73ba184..763d9a1 100644 --- a/.github/workflows/devbuild.yaml +++ b/.github/workflows/devbuild.yaml @@ -1,5 +1,8 @@ name: Development build +permissions: + contents: write + on: push: branches: [ dev ] diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4f8760d..c8f2b0d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,4 +1,6 @@ name: Package +permissions: + contents: write on: release: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index af79d53..936561a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,5 +1,8 @@ name: Test +permissions: + contents: read + on: - push - pull_request From 4bc6df8c83a45bc08dc7f3a9ffd30285ce9bf3b2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 4 Dec 2024 14:29:40 +0000 Subject: [PATCH 286/291] build(deps): bump github.com/quic-go/quic-go from 0.42.0 to 0.48.2 Bumps [github.com/quic-go/quic-go](https://github.com/quic-go/quic-go) from 0.42.0 to 0.48.2. - [Release notes](https://github.com/quic-go/quic-go/releases) - [Changelog](https://github.com/quic-go/quic-go/blob/master/Changelog.md) - [Commits](https://github.com/quic-go/quic-go/compare/v0.42.0...v0.48.2) --- updated-dependencies: - dependency-name: github.com/quic-go/quic-go dependency-type: indirect ... Signed-off-by: dependabot[bot] --- go.mod | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 807c10d..259d9d3 100644 --- a/go.mod +++ b/go.mod @@ -2,6 +2,8 @@ module github.com/eosswedenorg/thalos go 1.21 +toolchain go1.22.5 + require ( github.com/cenkalti/backoff/v4 v4.2.1 github.com/docker/go-units v0.5.0 @@ -51,8 +53,8 @@ require ( github.com/pelletier/go-toml/v2 v2.1.1 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/quic-go/qpack v0.4.0 // indirect - github.com/quic-go/quic-go v0.42.0 // indirect + github.com/quic-go/qpack v0.5.1 // indirect + github.com/quic-go/quic-go v0.48.2 // indirect github.com/refraction-networking/utls v1.6.3 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect From bb268aa9fa3dbb07cee7f6ca440a78718f54ca4e Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sat, 21 Dec 2024 19:47:51 +0100 Subject: [PATCH 287/291] update github.com/imroc/req to v3.49.0 --- go.mod | 31 +++++++++++------------- go.sum | 76 +++++++++++++++++++++++++++++----------------------------- 2 files changed, 52 insertions(+), 55 deletions(-) diff --git a/go.mod b/go.mod index 259d9d3..f8d1b28 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,6 @@ module github.com/eosswedenorg/thalos -go 1.21 - -toolchain go1.22.5 +go 1.22.0 require ( github.com/cenkalti/backoff/v4 v4.2.1 @@ -26,36 +24,35 @@ require ( ) require ( - github.com/andybalholm/brotli v1.1.0 // indirect + github.com/andybalholm/brotli v1.1.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/cloudflare/circl v1.3.7 // indirect + github.com/cloudflare/circl v1.5.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect - github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect + github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/go-telegram-bot-api/telegram-bot-api v4.6.4+incompatible // indirect github.com/google/go-cmp v0.6.0 // indirect - github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7 // indirect + github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad // indirect github.com/gorilla/websocket v1.5.1 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/hcl v1.0.0 // indirect - github.com/imroc/req/v3 v3.43.4 // indirect + github.com/imroc/req/v3 v3.49.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.17.8 // indirect + github.com/klauspost/compress v1.17.11 // indirect github.com/liamylian/jsontime/v2 v2.0.0 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/onsi/ginkgo/v2 v2.16.0 // indirect - github.com/onsi/gomega v1.31.1 // indirect + github.com/onsi/ginkgo/v2 v2.22.0 // indirect github.com/pelletier/go-toml/v2 v2.1.1 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/quic-go/qpack v0.5.1 // indirect github.com/quic-go/quic-go v0.48.2 // indirect - github.com/refraction-networking/utls v1.6.3 // indirect + github.com/refraction-networking/utls v1.6.7 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/shufflingpixels/jsontime-go v0.0.0-20240622163621-cf4b2804c92d // indirect @@ -68,16 +65,16 @@ require ( github.com/vmihailenco/go-tinylfu v0.2.2 // indirect github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect - go.uber.org/mock v0.4.0 // indirect + go.uber.org/mock v0.5.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.31.0 // indirect - golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 // indirect - golang.org/x/mod v0.17.0 // indirect - golang.org/x/net v0.25.0 // indirect + golang.org/x/exp v0.0.0-20241215155358-4a5509556b9e // indirect + golang.org/x/mod v0.22.0 // indirect + golang.org/x/net v0.32.0 // indirect golang.org/x/sync v0.10.0 // indirect golang.org/x/sys v0.28.0 // indirect golang.org/x/text v0.21.0 // indirect - golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect + golang.org/x/tools v0.28.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/ini.v1 v1.67.0 // indirect ) diff --git a/go.sum b/go.sum index 883609a..90931ae 100644 --- a/go.sum +++ b/go.sum @@ -2,8 +2,8 @@ github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a h1:HbKu58rmZp github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc= github.com/alicebob/miniredis/v2 v2.30.2 h1:lc1UAUT9ZA7h4srlfBmBt2aorm5Yftk9nBjxz7EyY9I= github.com/alicebob/miniredis/v2 v2.30.2/go.mod h1:b25qWj4fCEsBeAAR2mlb0ufImGC6uH3VlUfb/HS5zKg= -github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= -github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= +github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= +github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= @@ -17,8 +17,8 @@ github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU= -github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA= +github.com/cloudflare/circl v1.5.0 h1:hxIWksrX6XN5a1L2TI/h53AGPhNHoUBo+TD1ms9+pys= +github.com/cloudflare/circl v1.5.0/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -40,15 +40,15 @@ github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4 github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= -github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-redis/cache/v9 v9.0.0 h1:0thdtFo0xJi0/WXbRVu8B066z8OvVymXTJGaXrVWnN0= github.com/go-redis/cache/v9 v9.0.0/go.mod h1:cMwi1N8ASBOufbIvk7cdXe2PbPjK/WMRL95FFHWsSgI= github.com/go-redis/redismock/v9 v9.2.0 h1:ZrMYQeKPECZPjOj5u9eyOjg8Nnb0BS9lkVIZ6IpsKLw= github.com/go-redis/redismock/v9 v9.2.0/go.mod h1:18KHfGDK4Y6c2R0H38EUGWAdc7ZQS9gfYxc94k7rWT0= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= -github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= -github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/go-telegram-bot-api/telegram-bot-api v4.6.4+incompatible h1:2cauKuaELYAEARXRkq2LrJ0yDDv1rW7+wrTEdVL3uaU= github.com/go-telegram-bot-api/telegram-bot-api v4.6.4+incompatible/go.mod h1:qf9acutJ8cwBUhm1bqgz6Bei9/C/c93FPDljKWwsOgM= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -60,8 +60,6 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -72,8 +70,8 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7 h1:y3N7Bm7Y9/CtpiVkw/ZWj6lSlDF3F74SfKwfTCer72Q= -github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= +github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad h1:a6HEuzUHeKH6hwfN/ZoQgRgVIWFJljSWa/zetS2WTvg= +github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -85,8 +83,8 @@ github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/imroc/req/v3 v3.43.4 h1:NSXlB5dELZuxzGEFRWLWEQ9dQmh8d9pUMPa7MevK1K4= -github.com/imroc/req/v3 v3.43.4/go.mod h1:SQIz5iYop16MJxbo8ib+4LnostGCok8NQf8ToyQc2xA= +github.com/imroc/req/v3 v3.49.0 h1:5Rac2qvz7Dq0E3PeBo/c2szV3hagPQIGLoHtfBmYhu4= +github.com/imroc/req/v3 v3.49.0/go.mod h1:XZf4t94DNJzcA0UOBlA68hmSrWsAyvN407ADdH4mzCA= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jordan-wright/email v4.0.1-0.20210109023952-943e75fe5223+incompatible h1:jdpOPRN1zP63Td1hDQbZW73xKmzDvZHzVdNYxhnTMDA= @@ -97,8 +95,8 @@ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHm github.com/karlseguin/typed v1.1.8 h1:ND0eDpwiUFIrm/n1ehxUyh/XNGs9zkYrLxtGqENSalY= github.com/karlseguin/typed v1.1.8/go.mod h1:pZlmYaWQ7MVpwfIOP88fASh3LopVxKeE+uNXW3hQ2D8= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= -github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= +github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= @@ -138,8 +136,8 @@ github.com/onsi/ginkgo/v2 v2.3.0/go.mod h1:Eew0uilEqZmIEZr8JrvYlvOM7Rr6xzTmMV8Ay github.com/onsi/ginkgo/v2 v2.4.0/go.mod h1:iHkDK1fKGcBoEHT5W7YBq4RFWaQulw+caOMkAt4OrFo= github.com/onsi/ginkgo/v2 v2.5.0/go.mod h1:Luc4sArBICYCS8THh8v3i3i5CuSZO+RaQRaJoeNwomw= github.com/onsi/ginkgo/v2 v2.7.0/go.mod h1:yjiuMwPokqY1XauOgju45q3sJt6VzQ/Fict1LFVcsAo= -github.com/onsi/ginkgo/v2 v2.16.0 h1:7q1w9frJDzninhXxjZd+Y/x54XNjG/UlRLIYPZafsPM= -github.com/onsi/ginkgo/v2 v2.16.0/go.mod h1:llBI3WDLL9Z6taip6f33H76YcWtJv+7R3HigUjbIBOs= +github.com/onsi/ginkgo/v2 v2.22.0 h1:Yed107/8DjTr0lKCNt7Dn8yQ6ybuDRQoMGrNFKzMfHg= +github.com/onsi/ginkgo/v2 v2.22.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= @@ -150,8 +148,8 @@ github.com/onsi/gomega v1.22.1/go.mod h1:x6n7VNe4hw0vkyYUM4mjIXx3JbLiPaBPNgB7PRQ github.com/onsi/gomega v1.24.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg= github.com/onsi/gomega v1.24.1/go.mod h1:3AOiACssS3/MajrniINInwbfOOtfZvplPzuRSmvt1jM= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= -github.com/onsi/gomega v1.31.1 h1:KYppCUK+bUgAZwHOu7EXVBKyQA6ILvOESHkn/tgoqvo= -github.com/onsi/gomega v1.31.1/go.mod h1:y40C95dwAD1Nz36SsEnxvfFe8FFfNxzI5eJ0EYGyAy0= +github.com/onsi/gomega v1.34.2 h1:pNCwDkzrsv7MS9kpaQvVb1aVLahQXyJ/Tv5oAZMI3i8= +github.com/onsi/gomega v1.34.2/go.mod h1:v1xfxRgk0KIsG+QOdm7p8UosrOzPYRo60fd3B/1Dukc= github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI= github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -159,15 +157,15 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/quic-go/qpack v0.4.0 h1:Cr9BXA1sQS2SmDUWjSofMPNKmvF6IiIfDRmgU0w1ZCo= -github.com/quic-go/qpack v0.4.0/go.mod h1:UZVnYIfi5GRk+zI9UMaCPsmZ2xKJP7XBUvVyT1Knj9A= -github.com/quic-go/quic-go v0.42.0 h1:uSfdap0eveIl8KXnipv9K7nlwZ5IqLlYOpJ58u5utpM= -github.com/quic-go/quic-go v0.42.0/go.mod h1:132kz4kL3F9vxhW3CtQJLDVwcFe5wdWeJXXijhsO57M= +github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI= +github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg= +github.com/quic-go/quic-go v0.48.2 h1:wsKXZPeGWpMpCGSWqOcqpW2wZYic/8T3aqiOID0/KWE= +github.com/quic-go/quic-go v0.48.2/go.mod h1:yBgs3rWBOADpga7F+jJsb6Ybg1LSYiQvwWlLX+/6HMs= github.com/redis/go-redis/v9 v9.0.0-rc.4/go.mod h1:Vo3EsyWnicKnSKCA7HhgnvnyA74wOA69Cd2Meli5mmA= github.com/redis/go-redis/v9 v9.5.1 h1:H1X4D3yHPaYrkL5X06Wh6xNVM/pX0Ft4RV0vMGvLBh8= github.com/redis/go-redis/v9 v9.5.1/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= -github.com/refraction-networking/utls v1.6.3 h1:MFOfRN35sSx6K5AZNIoESsBuBxS2LCgRilRIdHb6fDc= -github.com/refraction-networking/utls v1.6.3/go.mod h1:yil9+7qSl+gBwJqztoQseO6Pr3h62pQoY1lXiNR/FPs= +github.com/refraction-networking/utls v1.6.7 h1:zVJ7sP1dJx/WtVuITug3qYUq034cDq9B2MR1K67ULZM= +github.com/refraction-networking/utls v1.6.7/go.mod h1:BC3O4vQzye5hqpmDTWUqi4P5DDhzJfkV1tdqtawQIH0= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= @@ -220,13 +218,15 @@ github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IU github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok= github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= +github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= +github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/gopher-lua v1.1.0 h1:BojcDhfyDWgU2f2TOzYK/g5p2gxMrku8oupLDqlnSqE= github.com/yuin/gopher-lua v1.1.0/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw= -go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU= -go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= +go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU= +go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -236,15 +236,15 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= -golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 h1:LfspQV/FYTatPTr/3HzIcmiUFH7PGP+OQ6mgDYo3yuQ= -golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= +golang.org/x/exp v0.0.0-20241215155358-4a5509556b9e h1:4qufH0hlUYs6AO6XmZC3GqfDPGSXHVXUFR6OND+iJX4= +golang.org/x/exp v0.0.0-20241215155358-4a5509556b9e/go.mod h1:qj5a5QZpwLU2NLQudwIN5koi3beDhSAlJwa67PuM98c= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= -golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4= +golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -260,8 +260,8 @@ golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= -golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= -golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= +golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -320,8 +320,8 @@ golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.28.0 h1:WuB6qZ4RPCQo5aP3WdKZS7i595EdWqWR8vqJTlwTVK8= +golang.org/x/tools v0.28.0/go.mod h1:dcIOrVd3mfQKTgrDVQHqCPMWy6lnhfhtX3hLXYVLfRw= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -335,8 +335,8 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= +google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From af5593b5f337b18748f5ce8342254c2d20d4888e Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Sat, 21 Dec 2024 19:50:01 +0100 Subject: [PATCH 288/291] github workflows: update to go version 1.22 --- .github/workflows/devbuild.yaml | 4 ++-- .github/workflows/release.yml | 6 +++--- .github/workflows/test.yml | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/devbuild.yaml b/.github/workflows/devbuild.yaml index 763d9a1..4576b25 100644 --- a/.github/workflows/devbuild.yaml +++ b/.github/workflows/devbuild.yaml @@ -22,7 +22,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v5 with: - go-version: 1.21 + go-version: 1.22 - name: compile id: compile @@ -53,7 +53,7 @@ jobs: runs-on: ubuntu-latest name: musl (${{ matrix.arch }}) container: - image: golang:1.21-alpine3.19 + image: golang:1.22-alpine3.19 steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c8f2b0d..9c900d7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -21,7 +21,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v5 with: - go-version: 1.21 + go-version: 1.22 - name: compile id: compile @@ -70,7 +70,7 @@ jobs: runs-on: ubuntu-latest name: Build musl (${{ matrix.arch }}) container: - image: golang:1.21-alpine3.19 + image: golang:1.22-alpine3.19 steps: - uses: actions/checkout@v4 @@ -129,7 +129,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v5 with: - go-version: 1.21 + go-version: 1.22 - name: Install build dependencies run: | diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 936561a..1b69220 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,7 @@ jobs: strategy: fail-fast: false matrix: - go-version: ["1.21"] + go-version: ["1.22"] arch: [ 386, amd64 ] runs-on: ubuntu-latest name: Test (${{matrix.arch}} go v${{ matrix.go-version }}) @@ -35,7 +35,7 @@ jobs: strategy: fail-fast: false matrix: - tag: [ "1.21-alpine3.19"] + tag: [ "1.22-alpine3.19"] runs-on: ubuntu-latest name: Test alpine (${{ matrix.tag }}) container: From 07ccab4628a6520a6d93b3aecf158811b33d144a Mon Sep 17 00:00:00 2001 From: Henrik Hautakoski Date: Mon, 23 Dec 2024 09:30:24 +0100 Subject: [PATCH 289/291] Version 1.1.8 --- Makefile | 2 +- debian/changelog | 8 ++++++++ docker/Dockerfile | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index d4f8331..b96ea39 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ GO=go GOLDFLAGS=-v -s -w -X main.VersionString=$(PROGRAM_VERSION) GOBUILDFLAGS+=-v -p $(shell nproc) -ldflags="$(GOLDFLAGS)" PROGRAM=thalos-server -PROGRAM_VERSION ?= 1.1.8-rc1 +PROGRAM_VERSION ?= 1.1.8 PREFIX=/usr/local BINDIR=$(PREFIX)/bin CFGDIR=$(PREFIX)/etc/thalos diff --git a/debian/changelog b/debian/changelog index 6e5c94a..50b934f 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,11 @@ +thalos (1.1.8) bionic focal jammy; urgency=medium + + * Support for wildcard contracts in Blacklist + * [Security CVE-2024-45337] Update golang.org/x/crypto to 0.31.0 + * [Security CVE-2024-53259] Update github.com/quic-go/quic-go to 0.48.2 + + -- Henrik Hautakoski Mon, 23 Dec 2024 09:25:44 +0100 + thalos (1.1.8~rc1) bionic focal jammy; urgency=medium * Support for wildcard contracts in Blacklist diff --git a/docker/Dockerfile b/docker/Dockerfile index 8a27563..ac810bf 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,6 +1,6 @@ FROM alpine:latest LABEL maintainer="Henrik Hautakoski " -ARG VERSION=1.1.8-rc1 +ARG VERSION=1.1.8 WORKDIR /thalos ADD --chmod=755 https://github.com/eosswedenorg/thalos/releases/download/v$VERSION/thalos-server-${VERSION}-linux-amd64-musl thalos-server ENTRYPOINT [ "./thalos-server" ]