eosswedenorg/thalos
1
0
Fork 0
mirror of https://github.com/eosswedenorg/thalos synced 2026-06-16 04:24:56 +02:00
Code Releases Activity

cmd/thalos/main.go: implement exponential backoff algorithm in reconnect logic

Browse source
This commit is contained in:
Henrik Hautakoski 2023-05-15 15:17:20 +02:00
parent ae5dabff36
commit 5f9a54c0d8
3 changed files with 45 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

53
cmd/thalos/main.go
View file

@ -9,6 +9,7 @@ import (
"syscall"
"time"
"github.com/cenkalti/backoff/v4"
eos "github.com/eoscanada/eos-go"
shipclient "github.com/eosswedenorg-go/antelope-ship-client"
"github.com/eosswedenorg-go/pid"
@ -43,12 +44,39 @@ func readerLoop() {
running = true
recon_cnt := 0
for running {
recon_cnt++
log.Infof("Connecting to ship at: %s (Try %d)", conf.Ship.Url, recon_cnt)
if err := shClient.Connect(conf.Ship.Url); err != nil {
log.WithError(err).Error("Failed to connect")
exp := &backoff.ExponentialBackOff{
InitialInterval: time.Second,
RandomizationFactor: 0.25,
Multiplier: 2,
MaxInterval: 10 * time.Minute,
MaxElapsedTime: 0,
Stop: -1,
Clock: backoff.SystemClock,
}
exp.Reset()
log.WithFields(log.Fields{
"initial_interval": exp.InitialInterval,
"max_interval": exp.MaxInterval,
"randomization_factor": exp.RandomizationFactor,
"multiplier": exp.Multiplier,
}).Info("Connecting with Exponential Backoff")
connectOp := func() error {
recon_cnt++
log.Infof("Connecting to ship at: %s (Try %d)", conf.Ship.Url, recon_cnt)
if err := shClient.Connect(conf.Ship.Url); err != nil {
return err
}
return shClient.SendBlocksRequest()
}
for running {
err := backoff.RetryNotify(connectOp, exp, func(err error, d time.Duration) {
if recon_cnt >= 3 {
msg := fmt.Sprintf("Failed to connect to ship at '%s'", conf.Ship.Url)
if err := notify.Send(context.Background(), conf.Name, msg); err != nil {
@ -57,13 +85,16 @@ func readerLoop() {
recon_cnt = 0
}
log.Info("Trying again in 5 seconds ....")
time.Sleep(5 * time.Second)
continue
}
log.WithError(err).Error("Failed to connect to SHIP")
if err := shClient.SendBlocksRequest(); err != nil {
log.WithError(err).Error("Failed to send block request")
log.WithFields(log.Fields{
"reconn_at": time.Now().Add(d),
"reconn_in": d,
}).Info("Reconnecting in ", d)
})
if err != nil {
log.WithError(err).Error("Failed to connect to SHIP")
running = false
continue
}

1
go.mod
View file

@ -3,6 +3,7 @@ module github.com/eosswedenorg/thalos
go 1.18
require (
github.com/cenkalti/backoff/v4 v4.1.3
github.com/docker/go-units v0.5.0
github.com/eoscanada/eos-go v0.10.3-0.20230413154640-bb75101dc2f6
github.com/eosswedenorg-go/antelope-ship-client v0.2.3

2
go.sum
View file

@ -4,6 +4,8 @@ github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLj
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
github.com/blendle/zapdriver v1.3.1 h1:C3dydBOWYRiOk+B8X9IVZ5IOe+7cl+tGOexN4QqHfpE=
github.com/blendle/zapdriver v1.3.1/go.mod h1:mdXfREi6u5MArG4j9fewC+FGnXaBR+T4Ox4J2u4eHCc=
github.com/cenkalti/backoff/v4 v4.1.3 h1:cFAlzYUlVYDysBEH2T5hyJZMh3+5+WCBvSnK6Q8UtC4=
github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=