diff --git a/app/config/config.go b/app/config/config.go index dc32584..ec95169 100644 --- a/app/config/config.go +++ b/app/config/config.go @@ -2,6 +2,9 @@ package config import ( "io/ioutil" + "time" + + "github.com/eosswedenorg/thalos/app/log" "gopkg.in/yaml.v3" @@ -34,6 +37,8 @@ type Config struct { Ship ShipConfig `yaml:"ship"` Api string `yaml:"api"` + Log log.Config `yaml:"log"` + Redis RedisConfig `yaml:"redis"` MessageCodec string `yaml:"message_codec"` @@ -43,6 +48,10 @@ type Config struct { func Parse(data []byte) (*Config, error) { cfg := Config{ MessageCodec: "json", + Log: log.Config{ + MaxFileSize: 10 * 1000 * 1000, // 10 mb + MaxTime: time.Hour * 24, + }, Ship: ShipConfig{ StartBlockNum: shipclient.NULL_BLOCK_NUMBER, EndBlockNum: shipclient.NULL_BLOCK_NUMBER, diff --git a/app/config/config_test.go b/app/config/config_test.go index 9b61eb7..9d95755 100644 --- a/app/config/config_test.go +++ b/app/config/config_test.go @@ -2,7 +2,9 @@ package config import ( "testing" + "time" + "github.com/eosswedenorg/thalos/app/log" "github.com/stretchr/testify/require" shipclient "github.com/eosswedenorg-go/antelope-ship-client" @@ -12,6 +14,11 @@ func TestParse_Default(t *testing.T) { expected := Config{ MessageCodec: "json", + Log: log.Config{ + MaxFileSize: 10 * 1000 * 1000, // 10 mb + MaxTime: time.Hour * 24, + }, + Ship: ShipConfig{ StartBlockNum: shipclient.NULL_BLOCK_NUMBER, EndBlockNum: shipclient.NULL_BLOCK_NUMBER, @@ -37,6 +44,12 @@ func TestParse(t *testing.T) { Name: "ship-reader-1", Api: "http://127.0.0.1:8080", MessageCodec: "mojibake", + Log: log.Config{ + Filename: "some_file.log", + Directory: "/path/to/whatever", + MaxFileSize: 200, + MaxTime: 30 * time.Minute, + }, Ship: ShipConfig{ Url: "127.0.0.1:8089", StartBlockNum: 23671836, @@ -60,6 +73,11 @@ func TestParse(t *testing.T) { name: "ship-reader-1" api: "http://127.0.0.1:8080" message_codec: "mojibake" +log: + filename: some_file.log + directory: /path/to/whatever + maxtime: 30m + maxfilesize: 200b ship: url: "127.0.0.1:8089" irreversible_only: true @@ -85,6 +103,10 @@ func TestParseShorthandShipUrl(t *testing.T) { Name: "ship-reader-1", Api: "http://127.0.0.1:8080", MessageCodec: "json", + Log: log.Config{ + MaxFileSize: 10 * 1000 * 1000, // 10 mb + MaxTime: time.Hour * 24, + }, Ship: ShipConfig{ Url: "127.0.0.1:8089", StartBlockNum: shipclient.NULL_BLOCK_NUMBER, diff --git a/app/log/RotatingFile.go b/app/log/RotatingFile.go new file mode 100644 index 0000000..75fb813 --- /dev/null +++ b/app/log/RotatingFile.go @@ -0,0 +1,114 @@ +package log + +import ( + "fmt" + "io" + "os" + "path" + "time" +) + +type RotatingFile struct { + fd *os.File + size int64 + maxSize int64 + ts time.Time + maxAge time.Duration + format string +} + +func open(filename string) (*os.File, error) { + return os.OpenFile(filename, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0o666) +} + +func NewRotatingFile(filename string, maxSize int64, maxAge time.Duration) (*RotatingFile, error) { + if err := os.MkdirAll(path.Dir(filename), 0o766); err != nil && !os.IsExist(err) { + return nil, err + } + + fd, err := open(filename) + if err != nil { + return nil, err + } + + stat, err := fd.Stat() + if err != nil { + return nil, err + } + + return &RotatingFile{ + fd: fd, + size: stat.Size(), + maxSize: maxSize, + ts: time.Now(), + maxAge: maxAge, + format: "2006-01-02_150405", + }, nil +} + +func NewRotatingFileFromConfig(config Config) (*RotatingFile, error) { + return NewRotatingFile(config.GetFilePath(), int64(config.MaxFileSize), config.MaxTime) +} + +func (w *RotatingFile) newFilename(name string) string { + ext := path.Ext(name) + if len(ext) > 0 { + name = name[:len(name)-len(ext)] + } + return fmt.Sprintf("%s-%s%s", name, time.Now().Format(w.format), ext) +} + +// Rotate the file. +func (w *RotatingFile) Rotate() error { + dst, err := os.OpenFile(w.newFilename(w.fd.Name()), os.O_CREATE|os.O_WRONLY, 0o666) + if err != nil { + return err + } + defer dst.Close() + + // Seek to the beginning of file + if _, err = w.fd.Seek(0, io.SeekStart); err != nil { + return err + } + + // And copy the contents to the new file. + if _, err = io.Copy(dst, w.fd); err != nil { + return err + } + + // Then truncate the log. + if err = w.fd.Truncate(0); err != nil { + return err + } + + w.size = 0 + w.ts = time.Now() + + return nil +} + +// Implement io.Writer interface +func (w *RotatingFile) Write(p []byte) (int, error) { + n, err := w.fd.Write(p) + if err != nil { + return n, err + } + + w.size += int64(n) + + // Check if we should rotate + if w.size >= w.maxSize || time.Since(w.ts) >= w.maxAge { + if err := w.Rotate(); err != nil { + return n, err + } + } + + return n, nil +} + +// Implement io.Closer interface +func (w *RotatingFile) Close() error { + err := w.fd.Close() + w.fd = nil + return err +} diff --git a/app/log/config.go b/app/log/config.go new file mode 100644 index 0000000..54c3b55 --- /dev/null +++ b/app/log/config.go @@ -0,0 +1,27 @@ +package log + +import ( + "path" + "time" + + "github.com/eosswedenorg/thalos/app/types" +) + +type Config struct { + Filename string `yaml:"filename"` + Directory string `yaml:"directory"` + MaxFileSize types.Size `yaml:"maxfilesize"` + MaxTime time.Duration `yaml:"maxtime"` +} + +func (c Config) GetFilename() string { + return path.Base(c.Filename) +} + +func (c Config) GetDirectory() string { + return path.Clean(c.Directory) +} + +func (c Config) GetFilePath() string { + return path.Join(c.GetDirectory(), c.GetFilename()) +} diff --git a/app/log/config_test.go b/app/log/config_test.go new file mode 100644 index 0000000..e3ad7ac --- /dev/null +++ b/app/log/config_test.go @@ -0,0 +1,84 @@ +package log + +import ( + "testing" +) + +func TestConfig_GetDirectory(t *testing.T) { + tests := []struct { + name string + directory string + want string + }{ + {"empty", "", "."}, + {"root", "/", "/"}, + {"one", "dir", "dir"}, + {"path", "/path/to/some/directory", "/path/to/some/directory"}, + {"relative", "relative/directory", "relative/directory"}, + {"backtrace", "/path/./to/some/../directory", "/path/to/directory"}, + {"multislash", "//path/to///directory//", "/path/to/directory"}, + {"everything", "path/to/..//./from/directory//", "path/from/directory"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := Config{ + Directory: tt.directory, + } + if got := c.GetDirectory(); got != tt.want { + t.Errorf("Config.GetDirectory() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestConfig_GetFilename(t *testing.T) { + tests := []struct { + name string + filename string + want string + }{ + {"empty", "", "."}, + {"name", "some_file.txt", "some_file.txt"}, + {"path", "/path/to/my.log", "my.log"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := Config{ + Filename: tt.filename, + } + if got := c.GetFilename(); got != tt.want { + t.Errorf("Config.GetFilename() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestConfig_GetFilePath(t *testing.T) { + tests := []struct { + name string + filename string + directory string + want string + }{ + {"empty", "", "", "."}, + {"directory", "", "dir", "dir"}, + {"filename", "filename", "", "filename"}, + {"both", "filename", "dir", "dir/filename"}, + {"root", "filename", "/", "/filename"}, + {"abs", "filename", "/path/to/logs", "/path/to/logs/filename"}, + {"relative", "filename", "/srv/../log", "/log/filename"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := Config{ + Filename: tt.filename, + Directory: tt.directory, + } + if got := c.GetFilePath(); got != tt.want { + t.Errorf("Config.GetFilePath() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/app/types/size.go b/app/types/size.go new file mode 100644 index 0000000..a39b80d --- /dev/null +++ b/app/types/size.go @@ -0,0 +1,35 @@ +package types + +import ( + "github.com/docker/go-units" + "gopkg.in/yaml.v3" +) + +// Size is an alias of int64 that can handle sizes represented +// in human readable strings like "200mb", "20 GB" etc + +type Size int64 // Size in bytes. + +// Parse a string into number of bytes stored in a int64 +func (s *Size) Parse(value string) error { + // Empty strings are not an error, they represents zero bytes. + if len(value) < 1 { + *s = 0 + return nil + } + + v, err := units.FromHumanSize(value) + if err != nil { + return err + } + *s = Size(v) + return nil +} + +func (s Size) String() string { + return units.HumanSize(float64(s)) +} + +func (s *Size) UnmarshalYAML(value *yaml.Node) error { + return s.Parse(value.Value) +} diff --git a/app/types/size_test.go b/app/types/size_test.go new file mode 100644 index 0000000..b2cfbe2 --- /dev/null +++ b/app/types/size_test.go @@ -0,0 +1,34 @@ +package types + +import "testing" + +func TestSize_Parse(t *testing.T) { + tests := []struct { + name string + value string + expected int64 + wantErr bool + }{ + {"Empty", "", 0, false}, + {"NoDigit", "abcdefg", 0, true}, + {"Negative", "-10MB", 0, true}, + {"Invalid prefix", "100WAX", 0, true}, + {"Multiple spaces between prefix and value", "100 gb", 0, true}, + {"100kb", "100kb", 100 * 1000, false}, + {"10MB", "10 MB", 10 * 1000 * 1000, false}, + {"2gb", "2gb", 2 * 1000 * 1000 * 1000, false}, + {"4Tb", "4 Tb", 4 * 1000 * 1000 * 1000 * 1000, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Size(0) + if err := s.Parse(tt.value); (err != nil) != tt.wantErr { + t.Errorf("Size.Parse() error = %v, wantErr %v", err, tt.wantErr) + } + + if int64(s) != tt.expected { + t.Errorf("Size = %v, expected %v", s, tt.expected) + } + }) + } +} diff --git a/cmd/main/main.go b/cmd/main/main.go index cee7ef1..4e577a9 100644 --- a/cmd/main/main.go +++ b/cmd/main/main.go @@ -5,9 +5,13 @@ import ( "fmt" "os" "os/signal" + "path" "syscall" "time" + eos "github.com/eoscanada/eos-go" + shipclient "github.com/eosswedenorg-go/antelope-ship-client" + "github.com/eosswedenorg-go/pid" "github.com/eosswedenorg/thalos/api/message" _ "github.com/eosswedenorg/thalos/api/message/json" _ "github.com/eosswedenorg/thalos/api/message/msgpack" @@ -15,17 +19,12 @@ import ( "github.com/eosswedenorg/thalos/app" "github.com/eosswedenorg/thalos/app/abi" "github.com/eosswedenorg/thalos/app/config" - + . "github.com/eosswedenorg/thalos/app/log" "github.com/go-redis/redis/v8" - log "github.com/sirupsen/logrus" - "github.com/nikoksr/notify" "github.com/nikoksr/notify/service/telegram" - - eos "github.com/eoscanada/eos-go" - shipclient "github.com/eosswedenorg-go/antelope-ship-client" - "github.com/eosswedenorg-go/pid" "github.com/pborman/getopt/v2" + log "github.com/sirupsen/logrus" ) // --------------------------- @@ -113,6 +112,7 @@ func main() { showVersion := getopt.BoolLong("version", 'v', "display this help text") configFile := getopt.StringLong("config", 'c', "./config.yml", "Config file to read", "file") pidFile := getopt.StringLong("pid", 'p', "", "Where to write process id", "file") + logFile := getopt.StringLong("log", 'l', "", "Path to log file", "file") getopt.Parse() @@ -143,6 +143,27 @@ func main() { return } + // If log file is given on the commandline, override config values. + if len(*logFile) > 0 { + conf.Log.Directory = path.Dir(*logFile) + conf.Log.Filename = path.Base(*logFile) + } + + if len(conf.Log.Filename) > 0 { + writer, err := NewRotatingFileFromConfig(conf.Log) + if err != nil { + log.WithError(err).Fatal("Failed to open log") + return + } + log.WithFields(log.Fields{ + "maxfilesize": conf.Log.MaxFileSize, + "maxage": conf.Log.MaxTime, + "directory": conf.Log.GetDirectory(), + "filename": conf.Log.GetFilename(), + }).Info("Logging to file: ", conf.Log.GetFilePath()) + log.SetOutput(writer) + } + // Init telegram notification service telegram, err := telegram.New(conf.Telegram.Id) if err != nil { diff --git a/config.example.yml b/config.example.yml index 5ba32a7..15578f9 100644 --- a/config.example.yml +++ b/config.example.yml @@ -2,6 +2,13 @@ name: "ship-reader-1" api: "http://127.0.0.1:8080" message_codec: "json" +log: + filename: thalos.log + directory: logs + time_format: 2006-01-02_150405 + max_filesize: 200mb + max_time: 24h + ship: url: "ws://127.0.0.1:8089" irreversible_only: false diff --git a/go.mod b/go.mod index 04934f6..63a5aa4 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/eosswedenorg/thalos go 1.18 require ( + github.com/docker/go-units v0.5.0 github.com/eoscanada/eos-go v0.10.3-0.20230413154640-bb75101dc2f6 github.com/eosswedenorg-go/antelope-ship-client v0.2.3 github.com/eosswedenorg-go/pid v1.0.1 diff --git a/go.sum b/go.sum index 98e05d4..a7828ee 100644 --- a/go.sum +++ b/go.sum @@ -18,6 +18,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/eoscanada/eos-go v0.10.3-0.20230413154640-bb75101dc2f6 h1:93LUOgAmRkmz8DF2V62GBAFm+7JgWA15zI1uYukBeRk= github.com/eoscanada/eos-go v0.10.3-0.20230413154640-bb75101dc2f6/go.mod h1:L3avCf8OkDrjlUeNy9DdoV67TCmDNj2dSlc5Xp3DNNk= github.com/eosswedenorg-go/antelope-ship-client v0.2.3 h1:08HOQj3YtlEYVsm0RoNZ27JsZWikrUISKAUli6H1Qac=