diff --git a/cmd/thalos/main.go b/cmd/thalos/main.go index cdb13a1..3eb17e2 100644 --- a/cmd/thalos/main.go +++ b/cmd/thalos/main.go @@ -9,6 +9,7 @@ import ( "syscall" "time" + "github.com/cenkalti/backoff/v4" eos "github.com/eoscanada/eos-go" shipclient "github.com/eosswedenorg-go/antelope-ship-client" "github.com/eosswedenorg-go/pid" @@ -43,12 +44,39 @@ func readerLoop() { running = true recon_cnt := 0 - for running { - recon_cnt++ - log.Infof("Connecting to ship at: %s (Try %d)", conf.Ship.Url, recon_cnt) - if err := shClient.Connect(conf.Ship.Url); err != nil { - log.WithError(err).Error("Failed to connect") + exp := &backoff.ExponentialBackOff{ + InitialInterval: time.Second, + RandomizationFactor: 0.25, + Multiplier: 2, + MaxInterval: 10 * time.Minute, + MaxElapsedTime: 0, + Stop: -1, + Clock: backoff.SystemClock, + } + exp.Reset() + log.WithFields(log.Fields{ + "initial_interval": exp.InitialInterval, + "max_interval": exp.MaxInterval, + "randomization_factor": exp.RandomizationFactor, + "multiplier": exp.Multiplier, + }).Info("Connecting with Exponential Backoff") + + connectOp := func() error { + recon_cnt++ + + log.Infof("Connecting to ship at: %s (Try %d)", conf.Ship.Url, recon_cnt) + + if err := shClient.Connect(conf.Ship.Url); err != nil { + return err + } + + return shClient.SendBlocksRequest() + } + + for running { + + err := backoff.RetryNotify(connectOp, exp, func(err error, d time.Duration) { if recon_cnt >= 3 { msg := fmt.Sprintf("Failed to connect to ship at '%s'", conf.Ship.Url) if err := notify.Send(context.Background(), conf.Name, msg); err != nil { @@ -57,13 +85,16 @@ func readerLoop() { recon_cnt = 0 } - log.Info("Trying again in 5 seconds ....") - time.Sleep(5 * time.Second) - continue - } + log.WithError(err).Error("Failed to connect to SHIP") - if err := shClient.SendBlocksRequest(); err != nil { - log.WithError(err).Error("Failed to send block request") + log.WithFields(log.Fields{ + "reconn_at": time.Now().Add(d), + "reconn_in": d, + }).Info("Reconnecting in ", d) + }) + if err != nil { + log.WithError(err).Error("Failed to connect to SHIP") + running = false continue } diff --git a/go.mod b/go.mod index 870fa20..9557703 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/eosswedenorg/thalos go 1.18 require ( + github.com/cenkalti/backoff/v4 v4.1.3 github.com/docker/go-units v0.5.0 github.com/eoscanada/eos-go v0.10.3-0.20230413154640-bb75101dc2f6 github.com/eosswedenorg-go/antelope-ship-client v0.2.3 diff --git a/go.sum b/go.sum index f130b8e..44d5a48 100644 --- a/go.sum +++ b/go.sum @@ -4,6 +4,8 @@ github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLj github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/blendle/zapdriver v1.3.1 h1:C3dydBOWYRiOk+B8X9IVZ5IOe+7cl+tGOexN4QqHfpE= github.com/blendle/zapdriver v1.3.1/go.mod h1:mdXfREi6u5MArG4j9fewC+FGnXaBR+T4Ox4J2u4eHCc= +github.com/cenkalti/backoff/v4 v4.1.3 h1:cFAlzYUlVYDysBEH2T5hyJZMh3+5+WCBvSnK6Q8UtC4= +github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=