Merge pull request #7 from eosswedenorg/log

app/config/config.go

@@ -2,6 +2,9 @@ package config
 
 import (
 	"io/ioutil"
+	"time"
+
+	"github.com/eosswedenorg/thalos/app/log"
 
 	"gopkg.in/yaml.v3"
 
@@ -34,6 +37,8 @@ type Config struct {
 	Ship ShipConfig `yaml:"ship"`
 	Api  string     `yaml:"api"`
 
+	Log log.Config `yaml:"log"`
+
 	Redis        RedisConfig `yaml:"redis"`
 	MessageCodec string      `yaml:"message_codec"`
 
@@ -43,6 +48,10 @@ type Config struct {
 func Parse(data []byte) (*Config, error) {
 	cfg := Config{
 		MessageCodec: "json",
+		Log: log.Config{
+			MaxFileSize: 10 * 1000 * 1000, // 10 mb
+			MaxTime:     time.Hour * 24,
+		},
 		Ship: ShipConfig{
 			StartBlockNum:       shipclient.NULL_BLOCK_NUMBER,
 			EndBlockNum:         shipclient.NULL_BLOCK_NUMBER,

app/config/config_test.go

@@ -2,7 +2,9 @@ package config
 
 import (
 	"testing"
+	"time"
 
+	"github.com/eosswedenorg/thalos/app/log"
 	"github.com/stretchr/testify/require"
 
 	shipclient "github.com/eosswedenorg-go/antelope-ship-client"
@@ -12,6 +14,11 @@ func TestParse_Default(t *testing.T) {
 	expected := Config{
 		MessageCodec: "json",
 
+		Log: log.Config{
+			MaxFileSize: 10 * 1000 * 1000, // 10 mb
+			MaxTime:     time.Hour * 24,
+		},
+
 		Ship: ShipConfig{
 			StartBlockNum:       shipclient.NULL_BLOCK_NUMBER,
 			EndBlockNum:         shipclient.NULL_BLOCK_NUMBER,
@@ -37,6 +44,12 @@ func TestParse(t *testing.T) {
 		Name:         "ship-reader-1",
 		Api:          "http://127.0.0.1:8080",
 		MessageCodec: "mojibake",
+		Log: log.Config{
+			Filename:    "some_file.log",
+			Directory:   "/path/to/whatever",
+			MaxFileSize: 200,
+			MaxTime:     30 * time.Minute,
+		},
 		Ship: ShipConfig{
 			Url:                 "127.0.0.1:8089",
 			StartBlockNum:       23671836,
@@ -60,6 +73,11 @@ func TestParse(t *testing.T) {
 name: "ship-reader-1"
 api: "http://127.0.0.1:8080"
 message_codec: "mojibake"
+log:
+  filename: some_file.log
+  directory: /path/to/whatever
+  maxtime: 30m
+  maxfilesize: 200b
 ship:
   url: "127.0.0.1:8089"
   irreversible_only: true
@@ -85,6 +103,10 @@ func TestParseShorthandShipUrl(t *testing.T) {
 		Name:         "ship-reader-1",
 		Api:          "http://127.0.0.1:8080",
 		MessageCodec: "json",
+		Log: log.Config{
+			MaxFileSize: 10 * 1000 * 1000, // 10 mb
+			MaxTime:     time.Hour * 24,
+		},
 		Ship: ShipConfig{
 			Url:                 "127.0.0.1:8089",
 			StartBlockNum:       shipclient.NULL_BLOCK_NUMBER,

app/log/RotatingFile.go

@@ -0,0 +1,114 @@
+package log
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path"
+	"time"
+)
+
+type RotatingFile struct {
+	fd      *os.File
+	size    int64
+	maxSize int64
+	ts      time.Time
+	maxAge  time.Duration
+	format  string
+}
+
+func open(filename string) (*os.File, error) {
+	return os.OpenFile(filename, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0o666)
+}
+
+func NewRotatingFile(filename string, maxSize int64, maxAge time.Duration) (*RotatingFile, error) {
+	if err := os.MkdirAll(path.Dir(filename), 0o766); err != nil && !os.IsExist(err) {
+		return nil, err
+	}
+
+	fd, err := open(filename)
+	if err != nil {
+		return nil, err
+	}
+
+	stat, err := fd.Stat()
+	if err != nil {
+		return nil, err
+	}
+
+	return &RotatingFile{
+		fd:      fd,
+		size:    stat.Size(),
+		maxSize: maxSize,
+		ts:      time.Now(),
+		maxAge:  maxAge,
+		format:  "2006-01-02_150405",
+	}, nil
+}
+
+func NewRotatingFileFromConfig(config Config) (*RotatingFile, error) {
+	return NewRotatingFile(config.GetFilePath(), int64(config.MaxFileSize), config.MaxTime)
+}
+
+func (w *RotatingFile) newFilename(name string) string {
+	ext := path.Ext(name)
+	if len(ext) > 0 {
+		name = name[:len(name)-len(ext)]
+	}
+	return fmt.Sprintf("%s-%s%s", name, time.Now().Format(w.format), ext)
+}
+
+// Rotate the file.
+func (w *RotatingFile) Rotate() error {
+	dst, err := os.OpenFile(w.newFilename(w.fd.Name()), os.O_CREATE|os.O_WRONLY, 0o666)
+	if err != nil {
+		return err
+	}
+	defer dst.Close()
+
+	// Seek to the beginning of file
+	if _, err = w.fd.Seek(0, io.SeekStart); err != nil {
+		return err
+	}
+
+	// And copy the contents to the new file.
+	if _, err = io.Copy(dst, w.fd); err != nil {
+		return err
+	}
+
+	// Then truncate the log.
+	if err = w.fd.Truncate(0); err != nil {
+		return err
+	}
+
+	w.size = 0
+	w.ts = time.Now()
+
+	return nil
+}
+
+// Implement io.Writer interface
+func (w *RotatingFile) Write(p []byte) (int, error) {
+	n, err := w.fd.Write(p)
+	if err != nil {
+		return n, err
+	}
+
+	w.size += int64(n)
+
+	// Check if we should rotate
+	if w.size >= w.maxSize || time.Since(w.ts) >= w.maxAge {
+		if err := w.Rotate(); err != nil {
+			return n, err
+		}
+	}
+
+	return n, nil
+}
+
+// Implement io.Closer interface
+func (w *RotatingFile) Close() error {
+	err := w.fd.Close()
+	w.fd = nil
+	return err
+}

app/log/config.go

@@ -0,0 +1,27 @@
+package log
+
+import (
+	"path"
+	"time"
+
+	"github.com/eosswedenorg/thalos/app/types"
+)
+
+type Config struct {
+	Filename    string        `yaml:"filename"`
+	Directory   string        `yaml:"directory"`
+	MaxFileSize types.Size    `yaml:"maxfilesize"`
+	MaxTime     time.Duration `yaml:"maxtime"`
+}
+
+func (c Config) GetFilename() string {
+	return path.Base(c.Filename)
+}
+
+func (c Config) GetDirectory() string {
+	return path.Clean(c.Directory)
+}
+
+func (c Config) GetFilePath() string {
+	return path.Join(c.GetDirectory(), c.GetFilename())
+}

app/log/config_test.go

@@ -0,0 +1,84 @@
+package log
+
+import (
+	"testing"
+)
+
+func TestConfig_GetDirectory(t *testing.T) {
+	tests := []struct {
+		name      string
+		directory string
+		want      string
+	}{
+		{"empty", "", "."},
+		{"root", "/", "/"},
+		{"one", "dir", "dir"},
+		{"path", "/path/to/some/directory", "/path/to/some/directory"},
+		{"relative", "relative/directory", "relative/directory"},
+		{"backtrace", "/path/./to/some/../directory", "/path/to/directory"},
+		{"multislash", "//path/to///directory//", "/path/to/directory"},
+		{"everything", "path/to/..//./from/directory//", "path/from/directory"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			c := Config{
+				Directory: tt.directory,
+			}
+			if got := c.GetDirectory(); got != tt.want {
+				t.Errorf("Config.GetDirectory() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestConfig_GetFilename(t *testing.T) {
+	tests := []struct {
+		name     string
+		filename string
+		want     string
+	}{
+		{"empty", "", "."},
+		{"name", "some_file.txt", "some_file.txt"},
+		{"path", "/path/to/my.log", "my.log"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			c := Config{
+				Filename: tt.filename,
+			}
+			if got := c.GetFilename(); got != tt.want {
+				t.Errorf("Config.GetFilename() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestConfig_GetFilePath(t *testing.T) {
+	tests := []struct {
+		name      string
+		filename  string
+		directory string
+		want      string
+	}{
+		{"empty", "", "", "."},
+		{"directory", "", "dir", "dir"},
+		{"filename", "filename", "", "filename"},
+		{"both", "filename", "dir", "dir/filename"},
+		{"root", "filename", "/", "/filename"},
+		{"abs", "filename", "/path/to/logs", "/path/to/logs/filename"},
+		{"relative", "filename", "/srv/../log", "/log/filename"},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			c := Config{
+				Filename:  tt.filename,
+				Directory: tt.directory,
+			}
+			if got := c.GetFilePath(); got != tt.want {
+				t.Errorf("Config.GetFilePath() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

app/types/size.go

@@ -0,0 +1,35 @@
+package types
+
+import (
+	"github.com/docker/go-units"
+	"gopkg.in/yaml.v3"
+)
+
+// Size is an alias of int64 that can handle sizes represented
+// in human readable strings like "200mb", "20 GB" etc
+
+type Size int64 // Size in bytes.
+
+// Parse a string into number of bytes stored in a int64
+func (s *Size) Parse(value string) error {
+	// Empty strings are not an error, they represents zero bytes.
+	if len(value) < 1 {
+		*s = 0
+		return nil
+	}
+
+	v, err := units.FromHumanSize(value)
+	if err != nil {
+		return err
+	}
+	*s = Size(v)
+	return nil
+}
+
+func (s Size) String() string {
+	return units.HumanSize(float64(s))
+}
+
+func (s *Size) UnmarshalYAML(value *yaml.Node) error {
+	return s.Parse(value.Value)
+}

app/types/size_test.go

@@ -0,0 +1,34 @@
+package types
+
+import "testing"
+
+func TestSize_Parse(t *testing.T) {
+	tests := []struct {
+		name     string
+		value    string
+		expected int64
+		wantErr  bool
+	}{
+		{"Empty", "", 0, false},
+		{"NoDigit", "abcdefg", 0, true},
+		{"Negative", "-10MB", 0, true},
+		{"Invalid prefix", "100WAX", 0, true},
+		{"Multiple spaces between prefix and value", "100  gb", 0, true},
+		{"100kb", "100kb", 100 * 1000, false},
+		{"10MB", "10 MB", 10 * 1000 * 1000, false},
+		{"2gb", "2gb", 2 * 1000 * 1000 * 1000, false},
+		{"4Tb", "4 Tb", 4 * 1000 * 1000 * 1000 * 1000, false},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := Size(0)
+			if err := s.Parse(tt.value); (err != nil) != tt.wantErr {
+				t.Errorf("Size.Parse() error = %v, wantErr %v", err, tt.wantErr)
+			}
+
+			if int64(s) != tt.expected {
+				t.Errorf("Size = %v, expected %v", s, tt.expected)
+			}
+		})
+	}
+}

cmd/main/main.go

@@ -5,9 +5,13 @@ import (
 	"fmt"
 	"os"
 	"os/signal"
+	"path"
 	"syscall"
 	"time"
 
+	eos "github.com/eoscanada/eos-go"
+	shipclient "github.com/eosswedenorg-go/antelope-ship-client"
+	"github.com/eosswedenorg-go/pid"
 	"github.com/eosswedenorg/thalos/api/message"
 	_ "github.com/eosswedenorg/thalos/api/message/json"
 	_ "github.com/eosswedenorg/thalos/api/message/msgpack"
@@ -15,17 +19,12 @@ import (
 	"github.com/eosswedenorg/thalos/app"
 	"github.com/eosswedenorg/thalos/app/abi"
 	"github.com/eosswedenorg/thalos/app/config"
-
+	. "github.com/eosswedenorg/thalos/app/log"
 	"github.com/go-redis/redis/v8"
-	log "github.com/sirupsen/logrus"
-
 	"github.com/nikoksr/notify"
 	"github.com/nikoksr/notify/service/telegram"
-
-	eos "github.com/eoscanada/eos-go"
-	shipclient "github.com/eosswedenorg-go/antelope-ship-client"
-	"github.com/eosswedenorg-go/pid"
 	"github.com/pborman/getopt/v2"
+	log "github.com/sirupsen/logrus"
 )
 
 // ---------------------------
@@ -113,6 +112,7 @@ func main() {
 	showVersion := getopt.BoolLong("version", 'v', "display this help text")
 	configFile := getopt.StringLong("config", 'c', "./config.yml", "Config file to read", "file")
 	pidFile := getopt.StringLong("pid", 'p', "", "Where to write process id", "file")
+	logFile := getopt.StringLong("log", 'l', "", "Path to log file", "file")
 
 	getopt.Parse()
 
@@ -143,6 +143,27 @@ func main() {
 		return
 	}
 
+	// If log file is given on the commandline, override config values.
+	if len(*logFile) > 0 {
+		conf.Log.Directory = path.Dir(*logFile)
+		conf.Log.Filename = path.Base(*logFile)
+	}
+
+	if len(conf.Log.Filename) > 0 {
+		writer, err := NewRotatingFileFromConfig(conf.Log)
+		if err != nil {
+			log.WithError(err).Fatal("Failed to open log")
+			return
+		}
+		log.WithFields(log.Fields{
+			"maxfilesize": conf.Log.MaxFileSize,
+			"maxage":      conf.Log.MaxTime,
+			"directory":   conf.Log.GetDirectory(),
+			"filename":    conf.Log.GetFilename(),
+		}).Info("Logging to file: ", conf.Log.GetFilePath())
+		log.SetOutput(writer)
+	}
+
 	// Init telegram notification service
 	telegram, err := telegram.New(conf.Telegram.Id)
 	if err != nil {

config.example.yml

@@ -2,6 +2,13 @@ name: "ship-reader-1"
 api: "http://127.0.0.1:8080"
 message_codec: "json"
 
+log:
+  filename: thalos.log
+  directory: logs
+  time_format: 2006-01-02_150405
+  max_filesize: 200mb
+  max_time: 24h
+
 ship:
   url: "ws://127.0.0.1:8089"
   irreversible_only: false

go.mod

@@ -3,6 +3,7 @@ module github.com/eosswedenorg/thalos
 go 1.18
 
 require (
+	github.com/docker/go-units v0.5.0
 	github.com/eoscanada/eos-go v0.10.3-0.20230413154640-bb75101dc2f6
 	github.com/eosswedenorg-go/antelope-ship-client v0.2.3
 	github.com/eosswedenorg-go/pid v1.0.1

go.sum

@@ -18,6 +18,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
+github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
+github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/eoscanada/eos-go v0.10.3-0.20230413154640-bb75101dc2f6 h1:93LUOgAmRkmz8DF2V62GBAFm+7JgWA15zI1uYukBeRk=
 github.com/eoscanada/eos-go v0.10.3-0.20230413154640-bb75101dc2f6/go.mod h1:L3avCf8OkDrjlUeNy9DdoV67TCmDNj2dSlc5Xp3DNNk=
 github.com/eosswedenorg-go/antelope-ship-client v0.2.3 h1:08HOQj3YtlEYVsm0RoNZ27JsZWikrUISKAUli6H1Qac=