diff --git a/.github/workflows/devbuild.yaml b/.github/workflows/devbuild.yaml new file mode 100644 index 0000000..4576b25 --- /dev/null +++ b/.github/workflows/devbuild.yaml @@ -0,0 +1,82 @@ +name: Development build + +permissions: + contents: write + +on: + push: + branches: [ dev ] + +jobs: + cross-compile: + strategy: + fail-fast: false + matrix: + os: [ linux, freebsd ] + arch: [ 386, amd64, arm, arm64 ] + name: ${{matrix.os}} (${{matrix.arch}}) + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: 1.22 + + - name: compile + id: compile + run: | + VER=$(cat Makefile | sed -n 's/^PROGRAM_VERSION\s*\??=\s*//p') + GOOS=${{matrix.os}} GOARCH=${{matrix.arch}} PROGRAM_VERSION="${VER}-$(echo $GITHUB_SHA | cut -b -8)" make build tools + + - name: Upload thalos-server + uses: actions/upload-artifact@v4 + with: + name: thalos-server-${{github.sha}}-${{matrix.os}}-${{matrix.arch}} + path: build/thalos-server + retention-days: 7 + + - name: Upload thalos-tools + uses: actions/upload-artifact@v4 + with: + name: thalos-tools-${{github.sha}}-${{matrix.os}}-${{matrix.arch}} + path: build/thalos-tools + retention-days: 7 + + # Build thalos binaries that are linked with musl libc. + musl: + strategy: + fail-fast: false + matrix: + arch: [ 386, amd64, arm, arm64 ] + runs-on: ubuntu-latest + name: musl (${{ matrix.arch }}) + container: + image: golang:1.22-alpine3.19 + steps: + - uses: actions/checkout@v4 + + - name: install dependencies + run: apk add make + + - name: compile + id: compile + run: | + VER=$(cat Makefile | sed -n 's/^PROGRAM_VERSION\s*\??=\s*//p') + GOOS=${{matrix.os}} GOARCH=${{matrix.arch}} PROGRAM_VERSION="${VER}-$(echo $GITHUB_SHA | cut -b -8)" make build tools + + - name: Upload thalos-server + uses: actions/upload-artifact@v4 + with: + name: thalos-server-${{github.sha}}-linux-${{matrix.arch}}-musl + path: build/thalos-server + retention-days: 7 + + - name: Upload thalos-tools + uses: actions/upload-artifact@v4 + with: + name: thalos-tools-${{github.sha}}-linux-${{matrix.arch}}-musl + path: build/thalos-tools + retention-days: 7 + diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e376c07..9c900d7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,4 +1,6 @@ name: Package +permissions: + contents: write on: release: @@ -14,12 +16,12 @@ jobs: name: Crosscompile - ${{matrix.os}}-${{matrix.arch}} runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v5 with: - go-version: 1.18 + go-version: 1.22 - name: compile id: compile @@ -27,7 +29,7 @@ jobs: mkdir -p build/bundle/{bin,logs} GOOS=${{matrix.os}} GOARCH=${{matrix.arch}} make -e DESTDIR=build/bundle PREFIX= CFGDIR= install install-scripts tar -C build/bundle -z -cf build/bundle.tar.gz . - echo "version=$(sed -n 's/.*PROGRAM_VERSION\s*=\s*//p' Makefile)" >> "$GITHUB_OUTPUT" + echo "version=$(sed -n 's/.*PROGRAM_VERSION.*=\s*//p' Makefile)" >> "$GITHUB_OUTPUT" - name: Upload thalos-server uses: actions/upload-release-asset@v1 @@ -59,6 +61,61 @@ jobs: asset_path: build/bundle.tar.gz asset_content_type: application/tar+gzip + # Build thalos binaries that are linked with musl libc. + musl: + strategy: + fail-fast: false + matrix: + arch: [ 386, amd64, arm, arm64 ] + runs-on: ubuntu-latest + name: Build musl (${{ matrix.arch }}) + container: + image: golang:1.22-alpine3.19 + steps: + - uses: actions/checkout@v4 + + - name: install dependencies + run: apk add make + + - name: compile + id: compile + run: | + mkdir -p build/bundle/{bin,logs} + GOARCH=${{matrix.arch}} make -e DESTDIR=build/bundle PREFIX= CFGDIR= install install-scripts + tar -C build/bundle -z -cf build/bundle.tar.gz . + echo "version=$(sed -n 's/.*PROGRAM_VERSION.*=\s*//p' Makefile)" >> "$GITHUB_OUTPUT" + + - name: Upload thalos-server + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ github.event.release.upload_url }} + asset_name: thalos-server-${{steps.compile.outputs.version}}-linux-${{matrix.arch}}-musl + asset_path: build/thalos-server + asset_content_type: application/octal-stream + + - name: Upload thalos-tools + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ github.event.release.upload_url }} + asset_name: thalos-tools-${{steps.compile.outputs.version}}-linux-${{matrix.arch}}-musl + asset_path: build/thalos-tools + asset_content_type: application/octal-stream + + - name: Upload bundle + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ github.event.release.upload_url }} + asset_name: thalos-${{steps.compile.outputs.version}}-linux-${{matrix.arch}}-musl.tar.gz + asset_path: build/bundle.tar.gz + asset_content_type: application/tar+gzip + + package-ubuntu: strategy: fail-fast: false @@ -67,12 +124,12 @@ jobs: name: Package - ${{matrix.os}} runs-on: ${{matrix.os}} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v5 with: - go-version: 1.18 + go-version: 1.22 - name: Install build dependencies run: | @@ -82,7 +139,7 @@ jobs: - name: Package id: package run: | - dpkg-buildpackage -b -us -uc + make build-deb DEB_FILE=$(ls ../*.deb | head -1) echo "deb_filename=$DEB_FILE" >> "$GITHUB_OUTPUT" echo "deb_name=$(basename $DEB_FILE)" >> "$GITHUB_OUTPUT" @@ -121,4 +178,4 @@ jobs: upload_url: ${{ github.event.release.upload_url }} asset_name: ${{ steps.package.outputs.info_name }} asset_path: ${{ steps.package.outputs.info_filename }} - asset_content_type: text/plain \ No newline at end of file + asset_content_type: text/plain diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e334239..1b69220 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,5 +1,8 @@ name: Test +permissions: + contents: read + on: - push - pull_request @@ -10,19 +13,39 @@ jobs: strategy: fail-fast: false matrix: - go-version: ["1.18", "1.19", "1.20"] + go-version: ["1.22"] + arch: [ 386, amd64 ] runs-on: ubuntu-latest - name: Test (go v${{ matrix.go-version }}) + name: Test (${{matrix.arch}} go v${{ matrix.go-version }}) steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v5 with: go-version: ${{ matrix.go-version }} - name: Test - run: go test -v ./... + run: GOARCH=${{matrix.arch}} go test -v ./... - name: Test API - run: cd api; go test -v ./... \ No newline at end of file + run: cd api; GOARCH=${{matrix.arch}} go test -v ./... + + test-alpine: + strategy: + fail-fast: false + matrix: + tag: [ "1.22-alpine3.19"] + runs-on: ubuntu-latest + name: Test alpine (${{ matrix.tag }}) + container: + image: golang:${{ matrix.tag }} + steps: + - uses: actions/checkout@v4 + + - name: Test + run: go test -v ./... + + - name: Test API + run: cd api; go test -v ./... + diff --git a/.gitignore b/.gitignore index 0406edd..0da5a1e 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ /config.yml /.pc /.vscode/ +/vendor diff --git a/LICENSE b/LICENSE index 05ea739..e3754ba 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2022-2023 Sw/eden +Copyright (c) 2022-2024 Sw/eden Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/Makefile b/Makefile index d4acb70..33fd99f 100644 --- a/Makefile +++ b/Makefile @@ -1,24 +1,32 @@ GO=go GOLDFLAGS=-v -s -w -X main.VersionString=$(PROGRAM_VERSION) -GOBUILDFLAGS=-v -p $(shell nproc) -ldflags="$(GOLDFLAGS)" +GOBUILDFLAGS+=-v -p $(shell nproc) -ldflags="$(GOLDFLAGS)" PROGRAM=thalos-server -PROGRAM_VERSION=0.2.0 +PROGRAM_VERSION ?= 1.1.9 PREFIX=/usr/local BINDIR=$(PREFIX)/bin CFGDIR=$(PREFIX)/etc/thalos +DOCKER_IMAGE_REPO ?= ghcr.io/eosswedenorg/thalos +DOCKER_IMAGE_TAG ?= $(PROGRAM_VERSION) -.PHONY: build build/$(PROGRAM) build/thalos-tools test +.PHONY: build build/$(PROGRAM) build/thalos-tools test docker-image docker-publish build: build/$(PROGRAM) build/$(PROGRAM) : - $(GO) build $(GOBUILDFLAGS) -o $@ cmd/thalos/main.go + $(GO) build $(GOBUILDFLAGS) -o $@ ./cmd/thalos/ tools : build/thalos-tools build/thalos-tools : - $(GO) build $(GOBUILDFLAGS) -o $@ cmd/tools/main.go cmd/tools/flags.go cmd/tools/bench.go cmd/tools/validate.go + $(GO) build $(GOBUILDFLAGS) -o $@ ./cmd/tools/ + +docker-image: + docker image build --build-arg VERSION=$(PROGRAM_VERSION) -t $(DOCKER_IMAGE_REPO):$(DOCKER_IMAGE_TAG) docker + +docker-publish: + docker image push $(DOCKER_IMAGE_REPO):$(DOCKER_IMAGE_TAG) install: build tools install -D build/$(PROGRAM) $(DESTDIR)$(BINDIR)/$(PROGRAM) @@ -28,8 +36,11 @@ install: build tools install-scripts: install -m 755 -t $(DESTDIR) scripts/start.sh scripts/stop.sh +build-deb: + dpkg-buildpackage -b -us -uc + test: $(GO) test -v ./... - + cd api; $(GO) test -v ./... clean : $(RM) -fr build diff --git a/README.md b/README.md index aa3ac84..09ab23c 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,21 @@ # Thalos +[![Test](https://github.com/eosswedenorg/thalos/actions/workflows/test.yml/badge.svg)](https://github.com/eosswedenorg/thalos/actions/workflows/test.yml) +[![Go Report Card](https://goreportcard.com/badge/github.com/eosswedenorg/thalos)](https://goreportcard.com/report/github.com/eosswedenorg/thalos) + Thalos is a application that makes it easy for users to stream blockchain data from an Antelope SHIP node. Consult the [documentation](https://thalos.waxsweden.org/docs) for more information. +Join the discussion on [telegram](https://t.me/antelopethalos) + +## Docker images + +Docker images can be found [here](https://github.com/eosswedenorg/thalos/pkgs/container/thalos) + ## Compiling -You will need golang version `1.18` or later to compile the source. +You will need golang version `1.21` or later to compile the source. Compile using make: @@ -17,8 +26,8 @@ $ make or using go directly if you dont have make installed. ```shell -$ go build -o build/thalos-server cmd/main/main.go +$ go build -o build/thalos-server cmd/thalos/*.go ``` ## Author -Henrik Hautakoski - [henrik@eossweden.org](mailto:henrik@eossweden.org) \ No newline at end of file +Henrik Hautakoski - [henrik@eossweden.org](mailto:henrik@eossweden.org) diff --git a/api/README.md b/api/README.md new file mode 100644 index 0000000..0748531 --- /dev/null +++ b/api/README.md @@ -0,0 +1,101 @@ +# Thalos Golang API + +## Usage + +The api is designed with go channels. + +## Example + +```go +package main + +import ( + "context" + "fmt" + "os" + "os/signal" + + "github.com/eosswedenorg/thalos/api" + "github.com/eosswedenorg/thalos/api/message" + _ "github.com/eosswedenorg/thalos/api/message/json" + api_redis "github.com/eosswedenorg/thalos/api/redis" + "github.com/redis/go-redis/v9" +) + +func main() { + // Create redis client + rdb := redis.NewClient(&redis.Options{}) + + sub := api_redis.NewSubscriber(context.Background(), rdb, api_redis.Namespace{ + Prefix: "ship", + ChainID: "1064487b3cd1a897ce03ae5b6a865651747e2e152090f99c1d19d44e01aea5a4", // Wax mainnet. + }) + + // Create thalos client + codec, err := message.GetCodec("json") + if err != nil { + fmt.Println("Failed to get json codec") + return + } + + client := api.NewClient(sub, codec.Decoder) + + // Subscribe to some channels. + err = client.Subscribe( + api.TransactionChannel, + api.ActionChannel{Contract: "eosio"}.Channel(), + api.ActionChannel{Name: "mine"}.Channel(), + api.HeartbeatChannel, + api.TableDeltaChannel{}.Channel(), + ) + + if err != nil { + fmt.Println(err) + return + } + + // Wait for interrupt in a go routine and close the client. + go func() { + sig := make(chan os.Signal) + signal.Notify(sig, os.Interrupt) + + <-sig + fmt.Println("Got interrupt") + + client.Close() + }() + + // Read messages + for t := range client.Channel() { + switch msg := t.(type) { + case error: + fmt.Println("Error:", msg) + case message.TransactionTrace: + fmt.Println("Transaction", msg.BlockNum, msg.ID) + fmt.Println(msg) + fmt.Println("---") + case message.HeartBeat : + fmt.Println("Heartbeat") + fmt.Println(msg) + fmt.Println("---") + } + } +} +``` + +## Message channels and types + +There are several types of channels to subscribe to aswell with their respectivly +message types. + +NOTE: this is not the same as an go channel. all messages will be posted to the same go channel that can +be accessed by `client.Channel()` + +| Channel type | Message type | Description | +| -------------------- | ------------------ | -------------------------------------------------------------------------------------------------------------------- | +| - | `error` | Posted if an error occured on the client. There is no channel to subscribe to. error messages will always be posted. | +| `HeartbeatChannel` | `HeartBeat` | Heartbeat message. Used to know if thalos is there or not if messages are not posted frequently on real channels. | +| `RollbackChannel` | `RollbackMessage` | This message is posted if the chain has experienced a microfork. | +| `TransactionChannel` | `TransactionTrace` | Information about an transaction | +| `ActionChannel` | `ActionTrace` | Information about an action | +| `TableDeltaChannel` | `TableDelta` | Information about an table change | diff --git a/api/channel.go b/api/channel.go index a843f54..baf33e8 100644 --- a/api/channel.go +++ b/api/channel.go @@ -45,6 +45,7 @@ func (c Channel) Is(other Channel) bool { var ( TransactionChannel = Channel{"transactions"} HeartbeatChannel = Channel{"heartbeat"} + RollbackChannel = Channel{"rollback"} ) // Action Channel @@ -66,3 +67,18 @@ func (a ActionChannel) Channel() Channel { return ch } + +// Table deltas +type TableDeltaChannel struct { + Name string +} + +func (td TableDeltaChannel) Channel() Channel { + ch := Channel{"tabledeltas"} + + if len(td.Name) > 0 { + ch.Append("name", td.Name) + } + + return ch +} diff --git a/api/channel_test.go b/api/channel_test.go index 58bc597..ae2edf1 100644 --- a/api/channel_test.go +++ b/api/channel_test.go @@ -55,13 +55,13 @@ func TestChannel_Is(t *testing.T) { func TestChannel_Format(t *testing.T) { tests := []struct { name string - c Channel delim string want string + c Channel }{ - {"Empty", Channel{}, ":", ""}, - {"Alot#1", Channel{"one", "two", "three"}, "-", "one-two-three"}, - {"Alot#2", Channel{"first", "second"}, ":", "first:second"}, + {"Empty", ":", "", Channel{}}, + {"Alot#1", "-", "one-two-three", Channel{"one", "two", "three"}}, + {"Alot#2", ":", "first:second", Channel{"first", "second"}}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -75,11 +75,11 @@ func TestChannel_Format(t *testing.T) { func TestChannel_String(t *testing.T) { tests := []struct { name string - c Channel want string + c Channel }{ - {"Empty", Channel{}, ""}, - {"Alot", Channel{"one", "two", "three"}, "one/two/three"}, + {"Empty", "", Channel{}}, + {"Alot", "one/two/three", Channel{"one", "two", "three"}}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -93,13 +93,13 @@ func TestChannel_String(t *testing.T) { func TestChannel_Type(t *testing.T) { tests := []struct { name string - c Channel want string + c Channel }{ - {"Empty", Channel{}, "unknown"}, - {"Heartbeat", HeartbeatChannel, "heartbeat"}, - {"Transaction", TransactionChannel, "transactions"}, - {"Actions", ActionChannel{}.Channel(), "actions"}, + {"Empty", "unknown", Channel{}}, + {"Heartbeat", "heartbeat", HeartbeatChannel}, + {"Transaction", "transactions", TransactionChannel}, + {"Actions", "actions", ActionChannel{}.Channel()}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -129,3 +129,21 @@ func TestAction_Channel(t *testing.T) { }) } } + +func TestTableDelta_Channel(t *testing.T) { + tests := []struct { + name string + action TableDeltaChannel + want Channel + }{ + {"Empty", TableDeltaChannel{}, Channel{"tabledeltas"}}, + {"Contract", TableDeltaChannel{Name: "delta_name"}, Channel{"tabledeltas", "name", "delta_name"}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.action.Channel(); !got.Is(tt.want) { + t.Errorf("TableDeltaChannel.String() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/api/client.go b/api/client.go index 023499f..ad01497 100644 --- a/api/client.go +++ b/api/client.go @@ -2,29 +2,44 @@ package api import ( "fmt" + "io" "sync" + "time" "github.com/eosswedenorg/thalos/api/message" ) type handler func([]byte) -// Client reads and decodes messages from a reader and provides callback functions. +// Client reads and decodes messages from a reader and posts them to a go channel type Client struct { reader Reader decoder message.Decoder + // waitgroup for worker threads. wg sync.WaitGroup - OnError func(error) - OnAction func(message.ActionTrace) - OnHeartbeat func(message.HeartBeat) + // Channel for messages and errors + channel chan any } func NewClient(reader Reader, decoder message.Decoder) *Client { return &Client{ reader: reader, decoder: decoder, + channel: make(chan any), + } +} + +func (c *Client) Channel() <-chan any { + return c.channel +} + +// Helper method to post a message to a channel with timeout. +func (c *Client) post(msg any) { + select { + case <-time.After(time.Second): + case c.channel <- msg: } } @@ -32,8 +47,10 @@ func (c *Client) worker(channel Channel, h handler) { for { payload, err := c.reader.Read(channel) if err != nil { - if c.OnError != nil { - c.OnError(err) + // Don't report EOF as an error because it is used + // by readers to signal an graceful end of input. + if err != io.EOF { + c.post(err) } return } @@ -42,54 +59,102 @@ func (c *Client) worker(channel Channel, h handler) { } } +// Helper method to decode a message and post and error on the channel if it fails. +// Returns true if successful. False otherwise +func (c *Client) decode(payload []byte, msg any) bool { + if err := c.decoder(payload, msg); err != nil { + c.post(err) + return false + } + return true +} + +// Rollback handler +func (c *Client) rollbackHandler(payload []byte) { + var rb message.RollbackMessage + if ok := c.decode(payload, &rb); ok { + c.post(rb) + } +} + +// Transaction handler +func (c *Client) transactionHandler(payload []byte) { + var trans message.TransactionTrace + if ok := c.decode(payload, &trans); ok { + c.post(trans) + } +} + +// Action handler func (c *Client) actHandler(payload []byte) { var act message.ActionTrace - if err := c.decoder(payload, &act); err != nil { - if c.OnError != nil { - c.OnError(err) - } - return + if ok := c.decode(payload, &act); ok { + c.post(act) } - c.OnAction(act) } +// TableDelta handler +func (c *Client) tableDeltaHandler(payload []byte) { + td := message.TableDelta{} + if ok := c.decode(payload, &td); ok { + c.post(td) + } +} + +// HeartBeat handler func (c *Client) hbHandler(payload []byte) { var hb message.HeartBeat - if err := c.decoder(payload, &hb); err != nil { - if c.OnError != nil { - c.OnError(err) - } - return + if ok := c.decode(payload, &hb); ok { + c.post(hb) } - c.OnHeartbeat(hb) } -func (c *Client) Subscribe(channel Channel) error { - var handler handler +func (c *Client) sub(channel Channel) error { + var h handler - switch t := channel.Type(); t { + switch channel.Type() { + case RollbackChannel.Type(): + h = c.rollbackHandler + case TransactionChannel.Type(): + h = c.transactionHandler case HeartbeatChannel.Type(): - handler = c.hbHandler + h = c.hbHandler case ActionChannel{}.Channel().Type(): - handler = c.actHandler + h = c.actHandler + case TableDeltaChannel{}.Channel().Type(): + h = c.tableDeltaHandler default: - return fmt.Errorf("invalid channel type. %s", t) + return fmt.Errorf("invalid channel type. %s", channel.Type()) } // Start a worker for this channel. c.wg.Add(1) go func() { defer c.wg.Done() - c.worker(channel, handler) + c.worker(channel, h) }() return nil } +func (c *Client) Subscribe(channels ...Channel) error { + for _, ch := range channels { + if err := c.sub(ch); err != nil { + return err + } + } + return nil +} + func (c *Client) Run() { + // Just wait for workers to complete. c.wg.Wait() } func (c *Client) Close() error { - return c.reader.Close() + err := c.reader.Close() + // Wait for all goroutines to finish before closing channel. + c.wg.Wait() + close(c.channel) + return err } diff --git a/api/client_test.go b/api/client_test.go index 8fae00e..784eee6 100644 --- a/api/client_test.go +++ b/api/client_test.go @@ -1,15 +1,28 @@ package api import ( + "bytes" + "io" "testing" "github.com/eosswedenorg/thalos/api/message" + _ "github.com/eosswedenorg/thalos/api/message/json" + "github.com/stretchr/testify/assert" ) -type mockReader struct{} +type mockReader struct { + r io.Reader +} func (m mockReader) Read(channel Channel) ([]byte, error) { - return []byte{}, nil + if m.r != nil { + b, err := io.ReadAll(m.r) + if err == nil && len(b) < 1 { + err = io.EOF + } + return b, err + } + return []byte{}, io.EOF } func (m mockReader) Close() error { @@ -20,12 +33,6 @@ func mockDecoder([]byte, any) error { return nil } -func mockHbHandler(message.HeartBeat) { -} - -func mockActionHandler(message.ActionTrace) { -} - func TestClient_Subscribe(t *testing.T) { tests := []struct { name string @@ -35,16 +42,36 @@ func TestClient_Subscribe(t *testing.T) { {"Channel", Channel{}, true}, {"ActionChannel", ActionChannel{}.Channel(), false}, {"HeartbeatChannel", HeartbeatChannel, false}, - {"TransactionChannel", TransactionChannel, true}, + {"TransactionChannel", TransactionChannel, false}, + {"InvalidChannel", Channel{"random_type"}, true}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { c := NewClient(&mockReader{}, mockDecoder) - c.OnHeartbeat = mockHbHandler - c.OnAction = mockActionHandler if err := c.Subscribe(tt.channel); (err != nil) != tt.wantErr { t.Errorf("Client.Subscribe() error = %v, wantErr %v", err, tt.wantErr) } }) } } + +func TestClient_ReadRollback(t *testing.T) { + expected := message.RollbackMessage{ + OldBlockNum: 1000, + NewBlockNum: 50, + } + + codec, err := message.GetCodec("json") + assert.NoError(t, err) + + payload, err := codec.Encoder(expected) + assert.NoError(t, err) + + client := NewClient(mockReader{bytes.NewReader(payload)}, codec.Decoder) + + err = client.Subscribe(RollbackChannel) + assert.NoError(t, err) + + actual := <-client.Channel() + assert.Equal(t, expected, actual) +} diff --git a/api/go.mod b/api/go.mod index 8090fbf..f8d2a68 100644 --- a/api/go.mod +++ b/api/go.mod @@ -1,14 +1,14 @@ module github.com/eosswedenorg/thalos/api -go 1.18 +go 1.20 require ( github.com/alicebob/miniredis/v2 v2.30.2 - github.com/eosswedenorg-go/jsontime v0.0.0-20230509125027-08422d6236c7 - github.com/go-redis/redismock/v9 v9.0.3 - github.com/redis/go-redis/v9 v9.0.5 - github.com/stretchr/testify v1.8.2 - github.com/ugorji/go/codec v1.2.11 + github.com/go-redis/redismock/v9 v9.2.0 + github.com/redis/go-redis/v9 v9.4.0 + github.com/shufflingpixels/jsontime-go v0.0.0-20240622163621-cf4b2804c92d + github.com/stretchr/testify v1.8.4 + github.com/ugorji/go/codec v1.2.12 ) require ( diff --git a/api/go.sum b/api/go.sum index 54abd34..79690e8 100644 --- a/api/go.sum +++ b/api/go.sum @@ -2,8 +2,8 @@ github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a h1:HbKu58rmZp github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc= github.com/alicebob/miniredis/v2 v2.30.2 h1:lc1UAUT9ZA7h4srlfBmBt2aorm5Yftk9nBjxz7EyY9I= github.com/alicebob/miniredis/v2 v2.30.2/go.mod h1:b25qWj4fCEsBeAAR2mlb0ufImGC6uH3VlUfb/HS5zKg= -github.com/bsm/ginkgo/v2 v2.7.0 h1:ItPMPH90RbmZJt5GtkcNvIRuGEdwlBItdNVoyzaNQao= -github.com/bsm/gomega v1.26.0 h1:LhQm+AFcgV2M0WyKroMASzAzCAJVpAxQXv4SaI9a69Y= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= @@ -14,11 +14,9 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= -github.com/eosswedenorg-go/jsontime v0.0.0-20230509125027-08422d6236c7 h1:rLPu++RHaxg4WmUOXeWYioZuafWs0PVcYuvzOWbOJjk= -github.com/eosswedenorg-go/jsontime v0.0.0-20230509125027-08422d6236c7/go.mod h1:eNUkVOymzgl0lViUhmm08PkutzqLnOQ6Dr+RUnf+Mq0= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= -github.com/go-redis/redismock/v9 v9.0.3 h1:mtHQi2l51lCmXIbTRTqb1EiHYe9tL5Yk5oorlSJJqR0= -github.com/go-redis/redismock/v9 v9.0.3/go.mod h1:F6tJRfnU8R/NZ0E+Gjvoluk14MqMC5ueSZX6vVQypc0= +github.com/go-redis/redismock/v9 v9.2.0 h1:ZrMYQeKPECZPjOj5u9eyOjg8Nnb0BS9lkVIZ6IpsKLw= +github.com/go-redis/redismock/v9 v9.2.0/go.mod h1:18KHfGDK4Y6c2R0H38EUGWAdc7ZQS9gfYxc94k7rWT0= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -33,18 +31,16 @@ github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/redis/go-redis/v9 v9.0.5 h1:CuQcn5HIEeK7BgElubPP8CGtE0KakrnbBSTLjathl5o= -github.com/redis/go-redis/v9 v9.0.5/go.mod h1:WqMKv5vnQbRuZstUwxQI195wHy+t4PuXDOjzMvcuQHk= +github.com/redis/go-redis/v9 v9.4.0 h1:Yzoz33UZw9I/mFhx4MNrB6Fk+XHO1VukNcCa1+lwyKk= +github.com/redis/go-redis/v9 v9.4.0/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= +github.com/shufflingpixels/jsontime-go v0.0.0-20240622163621-cf4b2804c92d h1:nju7jR1Kf210tArPT6l//HlfLLFnvje1BWl5TSRsohQ= +github.com/shufflingpixels/jsontime-go v0.0.0-20240622163621-cf4b2804c92d/go.mod h1:W0TaKyg3kDqWmFUxlax3qAls/lRdo12EseM6f4f0dzE= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= -github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= -github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= +github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/yuin/gopher-lua v1.1.0 h1:BojcDhfyDWgU2f2TOzYK/g5p2gxMrku8oupLDqlnSqE= github.com/yuin/gopher-lua v1.1.0/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw= golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw= @@ -54,6 +50,5 @@ golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/api/message/json/codec.go b/api/message/json/codec.go index fb6b0f7..d254b47 100644 --- a/api/message/json/codec.go +++ b/api/message/json/codec.go @@ -3,21 +3,20 @@ package json import ( "time" - jsontime "github.com/eosswedenorg-go/jsontime/v2" "github.com/eosswedenorg/thalos/api/message" + "github.com/shufflingpixels/jsontime-go" ) -var ( - json_codec = jsontime.ConfigWithCustomTimeFormat - encoder = json_codec.Marshal - decoder = json_codec.Unmarshal -) +func createCodec() message.Codec { + json_codec := jsontime.New("2006-01-02T15:04:05.000", time.UTC) + + return message.Codec{ + Encoder: json_codec.Marshal, + Decoder: json_codec.Unmarshal, + } +} func init() { - jsontime.SetDefaultTimeFormat("2006-01-02T15:04:05.000", time.UTC) - - message.RegisterCodec("json", message.Codec{ - Encoder: encoder, - Decoder: decoder, - }) + // Register the json codec. + message.RegisterCodec("json", createCodec()) } diff --git a/api/message/json/codec_test.go b/api/message/json/codec_test.go index 074d7d5..590720c 100644 --- a/api/message/json/codec_test.go +++ b/api/message/json/codec_test.go @@ -10,12 +10,13 @@ import ( func TestJson_EncodeActionTrace(t *testing.T) { msg := message.ActionTrace{ - TxID: "ed3b8e853647971cf8296f004c3a1aeac255f082b2cb3c12cc3222e2d7c174ab", - BlockNum: 267372365, - Timestamp: time.Unix(1048267389, int64(time.Millisecond)*500).UTC(), - Name: "transfer", - Contract: "eosio", - Receiver: "account2", + TxID: "ed3b8e853647971cf8296f004c3a1aeac255f082b2cb3c12cc3222e2d7c174ab", + BlockNum: 267372365, + Timestamp: time.Unix(1048267389, int64(time.Millisecond)*500).UTC(), + Name: "transfer", + Contract: "eosio", + Receiver: "account2", + FirstReceiver: true, Receipt: &message.ActionReceipt{ Receiver: "account2", ActDigest: "0a2c71dba327cf13a107d3a4f91c9c98f510a8fbbb483b233e222033f13a3e36", @@ -43,21 +44,22 @@ func TestJson_EncodeActionTrace(t *testing.T) { Return: []byte{0xde, 0xad, 0xbe, 0xef}, } - expected := `{"tx_id":"ed3b8e853647971cf8296f004c3a1aeac255f082b2cb3c12cc3222e2d7c174ab","blocknum":267372365,"blocktimestamp":"2003-03-21T17:23:09.500","receipt":{"receiver":"account2","act_digest":"0a2c71dba327cf13a107d3a4f91c9c98f510a8fbbb483b233e222033f13a3e36","global_sequence":2329381932,"recv_sequence":22,"auth_sequence":[{"account":"account1","sequence":1382772}],"code_sequence":1122,"abi_sequence":12352},"name":"transfer","contract":"eosio","receiver":"account2","data":{"from":"account1","quantity":"1000.0000 WAX","to":"account2"},"authorization":[{"actor":"account1","permission":"active"}],"except":"errstr","error":2,"return":"3q2+7w=="}` + expected := `{"tx_id":"ed3b8e853647971cf8296f004c3a1aeac255f082b2cb3c12cc3222e2d7c174ab","blocknum":267372365,"blocktimestamp":"2003-03-21T17:23:09.500","receipt":{"receiver":"account2","act_digest":"0a2c71dba327cf13a107d3a4f91c9c98f510a8fbbb483b233e222033f13a3e36","global_sequence":2329381932,"recv_sequence":22,"auth_sequence":[{"account":"account1","sequence":1382772}],"code_sequence":1122,"abi_sequence":12352},"name":"transfer","contract":"eosio","receiver":"account2","first_receiver":true,"data":{"from":"account1","quantity":"1000.0000 WAX","to":"account2"},"authorization":[{"actor":"account1","permission":"active"}],"except":"errstr","error":2,"return":"3q2+7w=="}` - data, err := encoder(msg) + data, err := createCodec().Encoder(msg) assert.NoError(t, err) assert.Equal(t, expected, string(data)) } func TestJson_DecodeActionTrace(t *testing.T) { expected := message.ActionTrace{ - TxID: "952989f7464237b6cf9926e533ecd331df6794ed07564bd052bc368cbd65b4bc", - BlockNum: 8723971, - Timestamp: time.Unix(1718957306, int64(time.Millisecond)*500).UTC(), - Name: "transfer", - Contract: "eosio", - Receiver: "account2", + TxID: "952989f7464237b6cf9926e533ecd331df6794ed07564bd052bc368cbd65b4bc", + BlockNum: 8723971, + Timestamp: time.Unix(1718957306, int64(time.Millisecond)*500).UTC(), + Name: "transfer", + Contract: "eosio", + Receiver: "account2", + FirstReceiver: true, Receipt: &message.ActionReceipt{ Receiver: "account2", ActDigest: "f2f682847fd5bf00fb315b075dc00b4ff0ce18776758077b86a233dea49dc047", @@ -85,10 +87,10 @@ func TestJson_DecodeActionTrace(t *testing.T) { Return: []byte{0xde, 0xad, 0xbe, 0xef}, } - input := `{"tx_id":"952989f7464237b6cf9926e533ecd331df6794ed07564bd052bc368cbd65b4bc","blocknum":8723971,"blocktimestamp":"2024-06-21T08:08:26.500","receipt":{"receiver":"account2","act_digest":"f2f682847fd5bf00fb315b075dc00b4ff0ce18776758077b86a233dea49dc047","global_sequence":287328,"recv_sequence":42,"auth_sequence":[{"account":"account1","sequence":271877283}],"code_sequence":237823,"abi_sequence":68323},"name":"transfer","contract":"eosio","receiver":"account2","data":{"from":"account1","quantity":"1000.0000 WAX","to":"account2"},"authorization":[{"actor":"account1","permission":"active"}],"except":"errstr","error":2,"return":"3q2+7w=="}` + input := `{"tx_id":"952989f7464237b6cf9926e533ecd331df6794ed07564bd052bc368cbd65b4bc","blocknum":8723971,"blocktimestamp":"2024-06-21T08:08:26.500","receipt":{"receiver":"account2","act_digest":"f2f682847fd5bf00fb315b075dc00b4ff0ce18776758077b86a233dea49dc047","global_sequence":287328,"recv_sequence":42,"auth_sequence":[{"account":"account1","sequence":271877283}],"code_sequence":237823,"abi_sequence":68323},"name":"transfer","contract":"eosio","receiver":"account2","first_receiver":true,"data":{"from":"account1","quantity":"1000.0000 WAX","to":"account2"},"authorization":[{"actor":"account1","permission":"active"}],"except":"errstr","error":2,"return":"3q2+7w=="}` msg := message.ActionTrace{} - err := decoder([]byte(input), &msg) + err := createCodec().Decoder([]byte(input), &msg) assert.NoError(t, err) assert.Equal(t, expected, msg) } @@ -129,12 +131,13 @@ func TestJson_EncodeTransactionTrace(t *testing.T) { Return: []byte{0x08, 0xf1}, }, { - TxID: tx_id, - BlockNum: block_num, - Timestamp: ts, - Receiver: "coolgame", - Contract: "coolgame", - Name: "addpoints", + TxID: tx_id, + BlockNum: block_num, + Timestamp: ts, + Receiver: "coolgame", + Contract: "coolgame", + FirstReceiver: true, + Name: "addpoints", Authorization: []message.PermissionLevel{ { Actor: "coolgame", @@ -153,9 +156,9 @@ func TestJson_EncodeTransactionTrace(t *testing.T) { Error: 2, } - expected := `{"id":"ed04516bdd1194aa5f0ab4c8c5445eec542c17f45a85bb3e9e4bc33e1a2486f8","blocknum":283781923,"blocktimestamp":"2029-02-08T15:10:05.500","status":"executed","cpu_usage_us":442,"net_usage_words":16,"elapsed":22,"net_usage":128,"scheduled":true,"action_traces":[{"tx_id":"ed04516bdd1194aa5f0ab4c8c5445eec542c17f45a85bb3e9e4bc33e1a2486f8","blocknum":283781923,"blocktimestamp":"2029-02-08T15:10:05.500","name":"mine","contract":"coolgame","receiver":"actor01","data":{"equipment_id":1234,"location_id":5445453},"authorization":[{"actor":"actor01","permission":"active"}],"except":"","error":0,"return":"CPE="},{"tx_id":"ed04516bdd1194aa5f0ab4c8c5445eec542c17f45a85bb3e9e4bc33e1a2486f8","blocknum":283781923,"blocktimestamp":"2029-02-08T15:10:05.500","name":"addpoints","contract":"coolgame","receiver":"coolgame","data":{"points":"1023.0423 SCAM"},"authorization":[{"actor":"coolgame","permission":"usrpoints"}],"except":"some error string","error":2,"return":"/wI="}],"except":"errstr","error":2}` + expected := `{"id":"ed04516bdd1194aa5f0ab4c8c5445eec542c17f45a85bb3e9e4bc33e1a2486f8","blocknum":283781923,"blocktimestamp":"2029-02-08T15:10:05.500","status":"executed","cpu_usage_us":442,"net_usage_words":16,"elapsed":22,"net_usage":128,"scheduled":true,"action_traces":[{"tx_id":"ed04516bdd1194aa5f0ab4c8c5445eec542c17f45a85bb3e9e4bc33e1a2486f8","blocknum":283781923,"blocktimestamp":"2029-02-08T15:10:05.500","name":"mine","contract":"coolgame","receiver":"actor01","first_receiver":false,"data":{"equipment_id":1234,"location_id":5445453},"authorization":[{"actor":"actor01","permission":"active"}],"except":"","error":0,"return":"CPE="},{"tx_id":"ed04516bdd1194aa5f0ab4c8c5445eec542c17f45a85bb3e9e4bc33e1a2486f8","blocknum":283781923,"blocktimestamp":"2029-02-08T15:10:05.500","name":"addpoints","contract":"coolgame","receiver":"coolgame","first_receiver":true,"data":{"points":"1023.0423 SCAM"},"authorization":[{"actor":"coolgame","permission":"usrpoints"}],"except":"some error string","error":2,"return":"/wI="}],"except":"errstr","error":2}` - data, err := encoder(msg) + data, err := createCodec().Encoder(msg) assert.NoError(t, err) assert.Equal(t, expected, string(data)) } @@ -165,7 +168,7 @@ func TestJson_DecodeTransactionTrace(t *testing.T) { ts := time.Unix(1730755743, int64(time.Millisecond)*500).UTC() block_num := uint32(2378197231) - input := `{"id":"f58bf8a0137fcea644dbc2b0cc5b6a017a848cd33b2e924703e7e3c6d1ca0c2e","blocknum":2378197231,"blocktimestamp":"2024-11-04T21:29:03.500","status":"executed","cpu_usage_us":442,"net_usage_words":16,"elapsed":22,"net_usage":128,"scheduled":true,"action_traces":[{"tx_id":"f58bf8a0137fcea644dbc2b0cc5b6a017a848cd33b2e924703e7e3c6d1ca0c2e","blocknum":2378197231,"blocktimestamp":"2024-11-04T21:29:03.500","receipt":{"receiver":"actor01","act_digest":"bf68a227a617aa9138215d31a2847005175f01151c6feeaeb6513fe2d090ff3a","global_sequence":47322,"recv_sequence":23289,"auth_sequence":[{"account":"actor01","sequence":56637232}],"code_sequence":98272,"abi_sequence":6823762},"name":"mine","contract":"","receiver":"actor01","data":{"equipment_id":1234,"location_id":5445453},"authorization":[{"actor":"actor01","permission":"active"}],"except":"","error":2,"return":"AQI="},{"tx_id":"f58bf8a0137fcea644dbc2b0cc5b6a017a848cd33b2e924703e7e3c6d1ca0c2e","blocknum":2378197231,"blocktimestamp":"2024-11-04T21:29:03.500","name":"addpoints","contract":"","receiver":"coolgame","data":{"points":"1023.0423 SCAM"},"authorization":[{"actor":"coolgame","permission":"usrpoints"}],"except":"","error":2,"return":"CPE="}],"except":"errstr","error":2}` + input := `{"id":"f58bf8a0137fcea644dbc2b0cc5b6a017a848cd33b2e924703e7e3c6d1ca0c2e","blocknum":2378197231,"blocktimestamp":"2024-11-04T21:29:03.500","status":"executed","cpu_usage_us":442,"net_usage_words":16,"elapsed":22,"net_usage":128,"scheduled":true,"action_traces":[{"tx_id":"f58bf8a0137fcea644dbc2b0cc5b6a017a848cd33b2e924703e7e3c6d1ca0c2e","blocknum":2378197231,"blocktimestamp":"2024-11-04T21:29:03.500","receipt":{"receiver":"actor01","act_digest":"bf68a227a617aa9138215d31a2847005175f01151c6feeaeb6513fe2d090ff3a","global_sequence":47322,"recv_sequence":23289,"auth_sequence":[{"account":"actor01","sequence":56637232}],"code_sequence":98272,"abi_sequence":6823762},"name":"mine","contract":"coolgame","receiver":"actor01","first_receiver":false,"data":{"equipment_id":1234,"location_id":5445453},"authorization":[{"actor":"actor01","permission":"active"}],"except":"","error":2,"return":"AQI="},{"tx_id":"f58bf8a0137fcea644dbc2b0cc5b6a017a848cd33b2e924703e7e3c6d1ca0c2e","blocknum":2378197231,"blocktimestamp":"2024-11-04T21:29:03.500","name":"addpoints","contract":"coolgame","receiver":"coolgame","first_receiver":true,"data":{"points":"1023.0423 SCAM"},"authorization":[{"actor":"coolgame","permission":"usrpoints"}],"except":"","error":2,"return":"CPE="}],"except":"errstr","error":2}` expected := message.TransactionTrace{ Timestamp: ts, @@ -183,6 +186,7 @@ func TestJson_DecodeTransactionTrace(t *testing.T) { BlockNum: block_num, Timestamp: ts, Receiver: "actor01", + Contract: "coolgame", Name: "mine", Receipt: &message.ActionReceipt{ Receiver: "actor01", @@ -212,11 +216,13 @@ func TestJson_DecodeTransactionTrace(t *testing.T) { Return: []byte{0x01, 0x02}, }, { - TxID: tx_id, - BlockNum: block_num, - Timestamp: ts, - Receiver: "coolgame", - Name: "addpoints", + TxID: tx_id, + BlockNum: block_num, + Timestamp: ts, + Receiver: "coolgame", + Contract: "coolgame", + FirstReceiver: true, + Name: "addpoints", Authorization: []message.PermissionLevel{ { Actor: "coolgame", @@ -235,7 +241,72 @@ func TestJson_DecodeTransactionTrace(t *testing.T) { } msg := message.TransactionTrace{} - err := decoder([]byte(input), &msg) + err := createCodec().Decoder([]byte(input), &msg) assert.NoError(t, err) assert.Equal(t, expected, msg) } + +func TestJson_EncodeTableDelta(t *testing.T) { + msg := message.TableDelta{ + BlockNum: 2736712863, + Timestamp: time.Unix(1724791632, 0).UTC(), + Name: "contract_row", + Rows: []message.TableDeltaRow{ + { + Present: true, + Data: map[string]any{ + "string": "value", + "int": 1234, + }, + RawData: []byte{0x23, 0x54, 0xF2, 0xab, 0xeb}, + }, + { + Present: false, + Data: map[string]any{ + "string": "value", + "int": 1234, + }, + RawData: []byte{0x23, 0xe2, 0x3c, 0xb9}, + }, + }, + } + + expected := `{"blocknum":2736712863,"blocktimestamp":"2024-08-27T20:47:12.000","name":"contract_row","rows":[{"present":true,"data":{"int":1234,"string":"value"},"raw_data":"I1Tyq+s="},{"present":false,"data":{"int":1234,"string":"value"},"raw_data":"I+I8uQ=="}]}` + + data, err := createCodec().Encoder(msg) + assert.NoError(t, err) + assert.Equal(t, expected, string(data)) +} + +func TestJson_DecodeTableDelta(t *testing.T) { + data := `{"blocknum":2787282732,"blocktimestamp":"2014-02-18T04:20:43.500","name":"contract_row","rows":[{"present":true,"data":{"id":2,"name":"some_name"},"raw_data":"XbosHA1WDbI="},{"present":false,"data":{"id":1234,"name":"some_other_name"},"raw_data":"KNQA1g=="}]}` + + expected := message.TableDelta{ + BlockNum: 2787282732, + Timestamp: time.Date(2014, time.February, 18, 4, 20, 43, 500000000, time.UTC), + Name: "contract_row", + Rows: []message.TableDeltaRow{ + { + Present: true, + Data: map[string]any{ + "name": "some_name", + "id": float64(2), + }, + RawData: []byte{0x5d, 0xba, 0x2c, 0x1c, 0x0d, 0x56, 0x0d, 0xb2}, + }, + { + Present: false, + Data: map[string]any{ + "name": "some_other_name", + "id": float64(1234), + }, + RawData: []byte{0x28, 0xd4, 0x00, 0xd6}, + }, + }, + } + + actual := message.TableDelta{} + err := createCodec().Decoder([]byte(data), &actual) + assert.NoError(t, err) + assert.Equal(t, expected, actual) +} diff --git a/api/message/msgpack/codec.go b/api/message/msgpack/codec.go index 783635c..ce16975 100644 --- a/api/message/msgpack/codec.go +++ b/api/message/msgpack/codec.go @@ -8,27 +8,28 @@ import ( "github.com/eosswedenorg/thalos/api/message" ) -var mh codec.MsgpackHandle +func createCodec() message.Codec { + // Create handler. + handle := codec.MsgpackHandle{} + handle.MapType = reflect.TypeOf(map[string]any(nil)) + handle.Canonical = true -func encode(a any) ([]byte, error) { - var b []byte - return b, codec.NewEncoderBytes(&b, &mh).Encode(a) -} + // Weird name but this is needed for the newest version of msgpack + // that has support for time and string datatypes etc. + handle.WriteExt = true -func decode(b []byte, a any) error { - return codec.NewDecoderBytes(b, &mh).Decode(a) + return message.Codec{ + Encoder: func(a any) ([]byte, error) { + var b []byte + return b, codec.NewEncoderBytes(&b, &handle).Encode(a) + }, + Decoder: func(b []byte, a any) error { + return codec.NewDecoderBytes(b, &handle).Decode(a) + }, + } } func init() { - mh.MapType = reflect.TypeOf(map[string]any(nil)) - mh.Canonical = true - - // Wierd name but this is needed for the newest version of msgpack - // that has support for time and string datatypes etc. - mh.WriteExt = true - - message.RegisterCodec("msgpack", message.Codec{ - Encoder: encode, - Decoder: decode, - }) + // Register codec. + message.RegisterCodec("msgpack", createCodec()) } diff --git a/api/message/msgpack/codec_test.go b/api/message/msgpack/codec_test.go index 2348257..0a11805 100644 --- a/api/message/msgpack/codec_test.go +++ b/api/message/msgpack/codec_test.go @@ -51,11 +51,11 @@ func TestMsgpack_EncodeActionTrace(t *testing.T) { Return: []byte{0xde, 0xad, 0xbe, 0xef}, } - data, err := encode(msg) + data, err := createCodec().Encoder(msg) assert.NoError(t, err) expected := []byte{ - 0x8c, 0xad, 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, + 0x8d, 0xad, 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x91, 0x82, 0xa5, 0x61, 0x63, 0x74, 0x6f, 0x72, 0xa6, 0x6d, 0x79, 0x67, 0x61, 0x6d, 0x65, 0xaa, 0x70, @@ -85,6 +85,8 @@ func TestMsgpack_EncodeActionTrace(t *testing.T) { 0x6e, 0x74, 0x32, 0xa5, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x02, 0xa6, 0x65, 0x78, 0x63, 0x65, 0x70, 0x74, 0xa6, 0x65, 0x72, 0x72, 0x73, 0x74, 0x72, + 0xae, 0x66, 0x69, 0x72, 0x73, 0x74, 0x5f, 0x72, + 0x65, 0x63, 0x65, 0x69, 0x76, 0x65, 0x72, 0xc2, 0xa4, 0x6e, 0x61, 0x6d, 0x65, 0xa8, 0x73, 0x65, 0x6c, 0x6c, 0x69, 0x74, 0x65, 0x6d, 0xa7, 0x72, 0x65, 0x63, 0x65, 0x69, 0x70, 0x74, 0x87, 0xac, @@ -177,7 +179,7 @@ func TestMsgpack_DecodeActionTrace(t *testing.T) { } res := message.ActionTrace{} - err := decode(data, &res) + err := createCodec().Decoder(data, &res) assert.NoError(t, err) assert.Equal(t, expected, res) @@ -190,14 +192,32 @@ func TestMsgpack_EncodeHeartbeat(t *testing.T) { LastIrreversibleBlockNum: 1236, } - data, err := encode(msg) + data, err := createCodec().Encoder(msg) assert.NoError(t, err) - assert.Equal(t, data, []byte{0x83, 0xa8, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x6e, 0x75, 0x6d, 0xcd, 0x4, 0xd2, 0xad, 0x68, 0x65, 0x61, 0x64, 0x5f, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x6e, 0x75, 0x6d, 0xcd, 0x4, 0xd3, 0xba, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x69, 0x72, 0x72, 0x65, 0x76, 0x65, 0x72, 0x73, 0x69, 0x62, 0x6c, 0x65, 0x5f, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x6e, 0x75, 0x6d, 0xcd, 0x4, 0xd4}) + assert.Equal(t, data, []byte{ + 0x83, 0xa8, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x6e, + 0x75, 0x6d, 0xcd, 0x04, 0xd2, 0xad, 0x68, 0x65, + 0x61, 0x64, 0x5f, 0x62, 0x6c, 0x6f, 0x63, 0x6b, + 0x6e, 0x75, 0x6d, 0xcd, 0x04, 0xd3, 0xba, 0x6c, + 0x61, 0x73, 0x74, 0x5f, 0x69, 0x72, 0x72, 0x65, + 0x76, 0x65, 0x72, 0x73, 0x69, 0x62, 0x6c, 0x65, + 0x5f, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x6e, 0x75, + 0x6d, 0xcd, 0x04, 0xd4, + }) } func TestMsgpack_DecodeHeartbeat(t *testing.T) { - data := []byte{0x83, 0xa8, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x6e, 0x75, 0x6d, 0xcd, 0x03, 0xe8, 0xad, 0x68, 0x65, 0x61, 0x64, 0x5f, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x6e, 0x75, 0x6d, 0xcd, 0x0b, 0xb8, 0xba, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x69, 0x72, 0x72, 0x65, 0x76, 0x65, 0x72, 0x73, 0x69, 0x62, 0x6c, 0x65, 0x5f, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x6e, 0x75, 0x6d, 0xcd, 0x04, 0x06} + data := []byte{ + 0x83, 0xa8, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x6e, + 0x75, 0x6d, 0xcd, 0x03, 0xe8, 0xad, 0x68, 0x65, + 0x61, 0x64, 0x5f, 0x62, 0x6c, 0x6f, 0x63, 0x6b, + 0x6e, 0x75, 0x6d, 0xcd, 0x0b, 0xb8, 0xba, 0x6c, + 0x61, 0x73, 0x74, 0x5f, 0x69, 0x72, 0x72, 0x65, + 0x76, 0x65, 0x72, 0x73, 0x69, 0x62, 0x6c, 0x65, + 0x5f, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x6e, 0x75, + 0x6d, 0xcd, 0x04, 0x06, + } expected := message.HeartBeat{ BlockNum: 1000, @@ -206,7 +226,7 @@ func TestMsgpack_DecodeHeartbeat(t *testing.T) { } msg := message.HeartBeat{} - err := decode(data, &msg) + err := createCodec().Decoder(data, &msg) assert.NoError(t, err) assert.Equal(t, expected, msg) } @@ -265,108 +285,127 @@ func TestMsgpack_EncodeTransactionTrace(t *testing.T) { Error: 2, } - data, err := encode(msg) + data, err := createCodec().Encoder(msg) assert.NoError(t, err) expected := []byte{ - 0x8c, 0xad, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x5f, 0x74, 0x72, 0x61, 0x63, 0x65, 0x73, 0x92, - 0x8c, 0xad, 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, - 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x91, - 0x82, 0xa5, 0x61, 0x63, 0x74, 0x6f, 0x72, 0xaa, - 0x61, 0x63, 0x74, 0x6f, 0x72, 0x31, 0x32, 0x31, - 0x32, 0x33, 0xaa, 0x70, 0x65, 0x72, 0x6d, 0x69, - 0x73, 0x73, 0x69, 0x6f, 0x6e, 0xa6, 0x61, 0x63, - 0x74, 0x69, 0x76, 0x65, 0xa8, 0x62, 0x6c, 0x6f, - 0x63, 0x6b, 0x6e, 0x75, 0x6d, 0x00, 0xae, 0x62, - 0x6c, 0x6f, 0x63, 0x6b, 0x74, 0x69, 0x6d, 0x65, - 0x73, 0x74, 0x61, 0x6d, 0x70, 0xc0, 0xa8, 0x63, - 0x6f, 0x6e, 0x74, 0x72, 0x61, 0x63, 0x74, 0xa7, - 0x73, 0x6b, 0x6a, 0x64, 0x68, 0x32, 0x33, 0xa4, - 0x64, 0x61, 0x74, 0x61, 0x81, 0xa3, 0x6b, 0x65, - 0x79, 0xa5, 0x76, 0x61, 0x6c, 0x75, 0x65, 0xa5, - 0x65, 0x72, 0x72, 0x6f, 0x72, 0x00, 0xa6, 0x65, - 0x78, 0x63, 0x65, 0x70, 0x74, 0xa0, 0xa4, 0x6e, - 0x61, 0x6d, 0x65, 0xa8, 0x73, 0x65, 0x6c, 0x6c, - 0x69, 0x74, 0x65, 0x6d, 0xa7, 0x72, 0x65, 0x63, - 0x65, 0x69, 0x70, 0x74, 0x87, 0xac, 0x61, 0x62, - 0x69, 0x5f, 0x73, 0x65, 0x71, 0x75, 0x65, 0x6e, - 0x63, 0x65, 0xce, 0x00, 0x48, 0xf8, 0x98, 0xaa, - 0x61, 0x63, 0x74, 0x5f, 0x64, 0x69, 0x67, 0x65, - 0x73, 0x74, 0xd9, 0x40, 0x36, 0x37, 0x36, 0x63, - 0x35, 0x33, 0x33, 0x36, 0x64, 0x65, 0x39, 0x64, - 0x35, 0x32, 0x38, 0x64, 0x34, 0x35, 0x36, 0x62, - 0x30, 0x31, 0x65, 0x39, 0x37, 0x35, 0x62, 0x31, - 0x30, 0x30, 0x36, 0x65, 0x32, 0x62, 0x64, 0x63, - 0x38, 0x36, 0x63, 0x38, 0x64, 0x35, 0x36, 0x36, - 0x33, 0x33, 0x30, 0x33, 0x32, 0x31, 0x65, 0x39, - 0x62, 0x33, 0x30, 0x39, 0x62, 0x36, 0x33, 0x34, - 0x66, 0x35, 0x32, 0x33, 0xad, 0x61, 0x75, 0x74, - 0x68, 0x5f, 0x73, 0x65, 0x71, 0x75, 0x65, 0x6e, - 0x63, 0x65, 0x91, 0x82, 0xa7, 0x61, 0x63, 0x63, - 0x6f, 0x75, 0x6e, 0x74, 0xaa, 0x61, 0x63, 0x74, - 0x6f, 0x72, 0x31, 0x32, 0x31, 0x32, 0x33, 0xa8, - 0x73, 0x65, 0x71, 0x75, 0x65, 0x6e, 0x63, 0x65, - 0xce, 0x00, 0x08, 0x5c, 0x39, 0xad, 0x63, 0x6f, - 0x64, 0x65, 0x5f, 0x73, 0x65, 0x71, 0x75, 0x65, - 0x6e, 0x63, 0x65, 0xce, 0x01, 0x61, 0xe8, 0x79, - 0xaf, 0x67, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x5f, - 0x73, 0x65, 0x71, 0x75, 0x65, 0x6e, 0x63, 0x65, - 0xce, 0x02, 0x12, 0xd7, 0x5d, 0xa8, 0x72, 0x65, - 0x63, 0x65, 0x69, 0x76, 0x65, 0x72, 0xa5, 0x65, - 0x6f, 0x73, 0x69, 0x6f, 0xad, 0x72, 0x65, 0x63, - 0x76, 0x5f, 0x73, 0x65, 0x71, 0x75, 0x65, 0x6e, - 0x63, 0x65, 0xce, 0x22, 0x81, 0x80, 0x7a, 0xa8, - 0x72, 0x65, 0x63, 0x65, 0x69, 0x76, 0x65, 0x72, - 0xa5, 0x65, 0x6f, 0x73, 0x69, 0x6f, 0xa6, 0x72, - 0x65, 0x74, 0x75, 0x72, 0x6e, 0xc0, 0xa5, 0x74, - 0x78, 0x5f, 0x69, 0x64, 0xa0, 0x8b, 0xad, 0x61, - 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x91, 0x82, 0xa5, 0x61, - 0x63, 0x74, 0x6f, 0x72, 0xa9, 0x61, 0x63, 0x74, - 0x6f, 0x72, 0x31, 0x34, 0x38, 0x32, 0xaa, 0x70, - 0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6f, - 0x6e, 0xa6, 0x61, 0x63, 0x74, 0x69, 0x76, 0x65, - 0xa8, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x6e, 0x75, - 0x6d, 0x00, 0xae, 0x62, 0x6c, 0x6f, 0x63, 0x6b, - 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, - 0x70, 0xc0, 0xa8, 0x63, 0x6f, 0x6e, 0x74, 0x72, - 0x61, 0x63, 0x74, 0xa7, 0x73, 0x6b, 0x6a, 0x64, - 0x68, 0x32, 0x34, 0xa4, 0x64, 0x61, 0x74, 0x61, - 0xc0, 0xa5, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x00, - 0xa6, 0x65, 0x78, 0x63, 0x65, 0x70, 0x74, 0xa0, - 0xa4, 0x6e, 0x61, 0x6d, 0x65, 0xa8, 0x73, 0x65, - 0x6c, 0x6c, 0x69, 0x74, 0x65, 0x6d, 0xa8, 0x72, - 0x65, 0x63, 0x65, 0x69, 0x76, 0x65, 0x72, 0xa5, - 0x65, 0x6f, 0x73, 0x69, 0x6f, 0xa6, 0x72, 0x65, - 0x74, 0x75, 0x72, 0x6e, 0xc0, 0xa5, 0x74, 0x78, - 0x5f, 0x69, 0x64, 0xa0, 0xa8, 0x62, 0x6c, 0x6f, - 0x63, 0x6b, 0x6e, 0x75, 0x6d, 0xce, 0xa3, 0x3d, - 0x9b, 0xc5, 0xae, 0x62, 0x6c, 0x6f, 0x63, 0x6b, - 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, - 0x70, 0xd7, 0xff, 0x77, 0x35, 0x94, 0x00, 0x65, - 0x4e, 0x19, 0xff, 0xac, 0x63, 0x70, 0x75, 0x5f, - 0x75, 0x73, 0x61, 0x67, 0x65, 0x5f, 0x75, 0x73, - 0x17, 0xa7, 0x65, 0x6c, 0x61, 0x70, 0x73, 0x65, - 0x64, 0x04, 0xa5, 0x65, 0x72, 0x72, 0x6f, 0x72, - 0x02, 0xa6, 0x65, 0x78, 0x63, 0x65, 0x70, 0x74, - 0xa9, 0x65, 0x78, 0x63, 0x65, 0x70, 0x74, 0x73, - 0x74, 0x72, 0xa2, 0x69, 0x64, 0xd9, 0x40, 0x65, - 0x64, 0x63, 0x30, 0x36, 0x64, 0x63, 0x65, 0x36, - 0x33, 0x32, 0x30, 0x34, 0x35, 0x39, 0x66, 0x64, - 0x36, 0x34, 0x34, 0x37, 0x35, 0x36, 0x39, 0x37, - 0x32, 0x30, 0x34, 0x38, 0x64, 0x61, 0x34, 0x35, - 0x33, 0x62, 0x32, 0x38, 0x31, 0x36, 0x62, 0x30, - 0x61, 0x34, 0x33, 0x34, 0x63, 0x33, 0x37, 0x64, - 0x64, 0x66, 0x66, 0x64, 0x65, 0x33, 0x36, 0x37, - 0x37, 0x38, 0x64, 0x63, 0x61, 0x62, 0x33, 0xa9, - 0x6e, 0x65, 0x74, 0x5f, 0x75, 0x73, 0x61, 0x67, - 0x65, 0xcc, 0x80, 0xaf, 0x6e, 0x65, 0x74, 0x5f, - 0x75, 0x73, 0x61, 0x67, 0x65, 0x5f, 0x77, 0x6f, - 0x72, 0x64, 0x73, 0x10, 0xa9, 0x73, 0x63, 0x68, - 0x65, 0x64, 0x75, 0x6c, 0x65, 0x64, 0xc3, 0xa6, - 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0xa9, 0x73, - 0x6f, 0x66, 0x74, 0x5f, 0x66, 0x61, 0x69, 0x6c, + 0x8c, 0xad, 0x61, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x5f, 0x74, 0x72, 0x61, 0x63, 0x65, + 0x73, 0x92, 0x8d, 0xad, 0x61, 0x75, 0x74, + 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x91, 0x82, 0xa5, 0x61, + 0x63, 0x74, 0x6f, 0x72, 0xaa, 0x61, 0x63, + 0x74, 0x6f, 0x72, 0x31, 0x32, 0x31, 0x32, + 0x33, 0xaa, 0x70, 0x65, 0x72, 0x6d, 0x69, + 0x73, 0x73, 0x69, 0x6f, 0x6e, 0xa6, 0x61, + 0x63, 0x74, 0x69, 0x76, 0x65, 0xa8, 0x62, + 0x6c, 0x6f, 0x63, 0x6b, 0x6e, 0x75, 0x6d, + 0x00, 0xae, 0x62, 0x6c, 0x6f, 0x63, 0x6b, + 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, + 0x6d, 0x70, 0xc0, 0xa8, 0x63, 0x6f, 0x6e, + 0x74, 0x72, 0x61, 0x63, 0x74, 0xa7, 0x73, + 0x6b, 0x6a, 0x64, 0x68, 0x32, 0x33, 0xa4, + 0x64, 0x61, 0x74, 0x61, 0x81, 0xa3, 0x6b, + 0x65, 0x79, 0xa5, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0xa5, 0x65, 0x72, 0x72, 0x6f, 0x72, + 0x00, 0xa6, 0x65, 0x78, 0x63, 0x65, 0x70, + 0x74, 0xa0, 0xae, 0x66, 0x69, 0x72, 0x73, + 0x74, 0x5f, 0x72, 0x65, 0x63, 0x65, 0x69, + 0x76, 0x65, 0x72, 0xc2, 0xa4, 0x6e, 0x61, + 0x6d, 0x65, 0xa8, 0x73, 0x65, 0x6c, 0x6c, + 0x69, 0x74, 0x65, 0x6d, 0xa7, 0x72, 0x65, + 0x63, 0x65, 0x69, 0x70, 0x74, 0x87, 0xac, + 0x61, 0x62, 0x69, 0x5f, 0x73, 0x65, 0x71, + 0x75, 0x65, 0x6e, 0x63, 0x65, 0xce, 0x00, + 0x48, 0xf8, 0x98, 0xaa, 0x61, 0x63, 0x74, + 0x5f, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, + 0xd9, 0x40, 0x36, 0x37, 0x36, 0x63, 0x35, + 0x33, 0x33, 0x36, 0x64, 0x65, 0x39, 0x64, + 0x35, 0x32, 0x38, 0x64, 0x34, 0x35, 0x36, + 0x62, 0x30, 0x31, 0x65, 0x39, 0x37, 0x35, + 0x62, 0x31, 0x30, 0x30, 0x36, 0x65, 0x32, + 0x62, 0x64, 0x63, 0x38, 0x36, 0x63, 0x38, + 0x64, 0x35, 0x36, 0x36, 0x33, 0x33, 0x30, + 0x33, 0x32, 0x31, 0x65, 0x39, 0x62, 0x33, + 0x30, 0x39, 0x62, 0x36, 0x33, 0x34, 0x66, + 0x35, 0x32, 0x33, 0xad, 0x61, 0x75, 0x74, + 0x68, 0x5f, 0x73, 0x65, 0x71, 0x75, 0x65, + 0x6e, 0x63, 0x65, 0x91, 0x82, 0xa7, 0x61, + 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0xaa, + 0x61, 0x63, 0x74, 0x6f, 0x72, 0x31, 0x32, + 0x31, 0x32, 0x33, 0xa8, 0x73, 0x65, 0x71, + 0x75, 0x65, 0x6e, 0x63, 0x65, 0xce, 0x00, + 0x08, 0x5c, 0x39, 0xad, 0x63, 0x6f, 0x64, + 0x65, 0x5f, 0x73, 0x65, 0x71, 0x75, 0x65, + 0x6e, 0x63, 0x65, 0xce, 0x01, 0x61, 0xe8, + 0x79, 0xaf, 0x67, 0x6c, 0x6f, 0x62, 0x61, + 0x6c, 0x5f, 0x73, 0x65, 0x71, 0x75, 0x65, + 0x6e, 0x63, 0x65, 0xce, 0x02, 0x12, 0xd7, + 0x5d, 0xa8, 0x72, 0x65, 0x63, 0x65, 0x69, + 0x76, 0x65, 0x72, 0xa5, 0x65, 0x6f, 0x73, + 0x69, 0x6f, 0xad, 0x72, 0x65, 0x63, 0x76, + 0x5f, 0x73, 0x65, 0x71, 0x75, 0x65, 0x6e, + 0x63, 0x65, 0xce, 0x22, 0x81, 0x80, 0x7a, + 0xa8, 0x72, 0x65, 0x63, 0x65, 0x69, 0x76, + 0x65, 0x72, 0xa5, 0x65, 0x6f, 0x73, 0x69, + 0x6f, 0xa6, 0x72, 0x65, 0x74, 0x75, 0x72, + 0x6e, 0xc0, 0xa5, 0x74, 0x78, 0x5f, 0x69, + 0x64, 0xa0, 0x8c, 0xad, 0x61, 0x75, 0x74, + 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x91, 0x82, 0xa5, 0x61, + 0x63, 0x74, 0x6f, 0x72, 0xa9, 0x61, 0x63, + 0x74, 0x6f, 0x72, 0x31, 0x34, 0x38, 0x32, + 0xaa, 0x70, 0x65, 0x72, 0x6d, 0x69, 0x73, + 0x73, 0x69, 0x6f, 0x6e, 0xa6, 0x61, 0x63, + 0x74, 0x69, 0x76, 0x65, 0xa8, 0x62, 0x6c, + 0x6f, 0x63, 0x6b, 0x6e, 0x75, 0x6d, 0x00, + 0xae, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x74, + 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, + 0x70, 0xc0, 0xa8, 0x63, 0x6f, 0x6e, 0x74, + 0x72, 0x61, 0x63, 0x74, 0xa7, 0x73, 0x6b, + 0x6a, 0x64, 0x68, 0x32, 0x34, 0xa4, 0x64, + 0x61, 0x74, 0x61, 0xc0, 0xa5, 0x65, 0x72, + 0x72, 0x6f, 0x72, 0x00, 0xa6, 0x65, 0x78, + 0x63, 0x65, 0x70, 0x74, 0xa0, 0xae, 0x66, + 0x69, 0x72, 0x73, 0x74, 0x5f, 0x72, 0x65, + 0x63, 0x65, 0x69, 0x76, 0x65, 0x72, 0xc2, + 0xa4, 0x6e, 0x61, 0x6d, 0x65, 0xa8, 0x73, + 0x65, 0x6c, 0x6c, 0x69, 0x74, 0x65, 0x6d, + 0xa8, 0x72, 0x65, 0x63, 0x65, 0x69, 0x76, + 0x65, 0x72, 0xa5, 0x65, 0x6f, 0x73, 0x69, + 0x6f, 0xa6, 0x72, 0x65, 0x74, 0x75, 0x72, + 0x6e, 0xc0, 0xa5, 0x74, 0x78, 0x5f, 0x69, + 0x64, 0xa0, 0xa8, 0x62, 0x6c, 0x6f, 0x63, + 0x6b, 0x6e, 0x75, 0x6d, 0xce, 0xa3, 0x3d, + 0x9b, 0xc5, 0xae, 0x62, 0x6c, 0x6f, 0x63, + 0x6b, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, + 0x61, 0x6d, 0x70, 0xd7, 0xff, 0x77, 0x35, + 0x94, 0x00, 0x65, 0x4e, 0x19, 0xff, 0xac, + 0x63, 0x70, 0x75, 0x5f, 0x75, 0x73, 0x61, + 0x67, 0x65, 0x5f, 0x75, 0x73, 0x17, 0xa7, + 0x65, 0x6c, 0x61, 0x70, 0x73, 0x65, 0x64, + 0x04, 0xa5, 0x65, 0x72, 0x72, 0x6f, 0x72, + 0x02, 0xa6, 0x65, 0x78, 0x63, 0x65, 0x70, + 0x74, 0xa9, 0x65, 0x78, 0x63, 0x65, 0x70, + 0x74, 0x73, 0x74, 0x72, 0xa2, 0x69, 0x64, + 0xd9, 0x40, 0x65, 0x64, 0x63, 0x30, 0x36, + 0x64, 0x63, 0x65, 0x36, 0x33, 0x32, 0x30, + 0x34, 0x35, 0x39, 0x66, 0x64, 0x36, 0x34, + 0x34, 0x37, 0x35, 0x36, 0x39, 0x37, 0x32, + 0x30, 0x34, 0x38, 0x64, 0x61, 0x34, 0x35, + 0x33, 0x62, 0x32, 0x38, 0x31, 0x36, 0x62, + 0x30, 0x61, 0x34, 0x33, 0x34, 0x63, 0x33, + 0x37, 0x64, 0x64, 0x66, 0x66, 0x64, 0x65, + 0x33, 0x36, 0x37, 0x37, 0x38, 0x64, 0x63, + 0x61, 0x62, 0x33, 0xa9, 0x6e, 0x65, 0x74, + 0x5f, 0x75, 0x73, 0x61, 0x67, 0x65, 0xcc, + 0x80, 0xaf, 0x6e, 0x65, 0x74, 0x5f, 0x75, + 0x73, 0x61, 0x67, 0x65, 0x5f, 0x77, 0x6f, + 0x72, 0x64, 0x73, 0x10, 0xa9, 0x73, 0x63, + 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x64, + 0xc3, 0xa6, 0x73, 0x74, 0x61, 0x74, 0x75, + 0x73, 0xa9, 0x73, 0x6f, 0x66, 0x74, 0x5f, + 0x66, 0x61, 0x69, 0x6c, } assert.Equal(t, expected, data) } @@ -406,8 +445,77 @@ func TestMsgpack_DecodeTransactionTrace(t *testing.T) { } res := message.TransactionTrace{} - err := decode(data, &res) + err := createCodec().Decoder(data, &res) assert.NoError(t, err) assert.Equal(t, expected, res) } + +func TestMsgpack_EncodeTableDelta(t *testing.T) { + msg := message.TableDelta{ + BlockNum: 6347293, + Timestamp: time.Date(1998, time.December, 4, 8, 54, 35, 500, time.UTC), + Name: "contract_row", + Rows: []message.TableDeltaRow{ + { + Present: true, + Data: map[string]any{ + "id": 2213, + "name": "Freddie Mercury", + "band": "Queen", + }, + RawData: []byte{0x23, 0x13, 0xe2}, + }, + { + Present: false, + Data: map[string]any{ + "id": 27182, + "name": "Eddie Van Halen", + "band": "Van Halen", + }, + RawData: []byte{0xfe, 0x4e, 0x52, 0x05}, + }, + }, + } + + expected := "\x84\xa8blocknum\xce\x00`\xda\x1d\xaeblocktimestamp\xd7\xff\x00\x00\a\xd06g\xa3K\xa4name\xaccontract_row\xa4rows\x92\x83\xa4data\x83\xa4band\xa5Queen\xa2id\xd1\b\xa5\xa4name\xafFreddie Mercury\xa7presentèraw_data\xc4\x03#\x13⃤data\x83\xa4band\xa9Van Halen\xa2id\xd1j.\xa4name\xafEddie Van Halen\xa7present¨raw_data\xc4\x04\xfeNR\x05" + + actual, err := createCodec().Encoder(msg) + assert.NoError(t, err) + assert.Equal(t, expected, string(actual)) +} + +func TestMsgpack_DecodeTableDelta(t *testing.T) { + data := []byte("\x84\xa8blocknum\xce\x00`\xda\x1d\xaeblocktimestamp\xd7\xff\x00\x00\a\xd06g\xa3K\xa4name\xaccontract_row\xa4rows\x92\x83\xa4data\x83\xa4band\xa5Queen\xa2id\xd1\b\xa5\xa4name\xafFreddie Mercury\xa7presentèraw_data\xc4\x03#\x13⃤data\x83\xa4band\xa9Van Halen\xa2id\xd1j.\xa4name\xafEddie Van Halen\xa7present¨raw_data\xc4\x04\xfeNR\x05") + + expected := message.TableDelta{ + BlockNum: 6347293, + Timestamp: time.Date(1998, time.December, 4, 8, 54, 35, 500, time.UTC), + Name: "contract_row", + Rows: []message.TableDeltaRow{ + { + Present: true, + Data: map[string]any{ + "id": int64(2213), + "name": "Freddie Mercury", + "band": "Queen", + }, + RawData: []byte{0x23, 0x13, 0xe2}, + }, + { + Present: false, + Data: map[string]any{ + "id": int64(27182), + "name": "Eddie Van Halen", + "band": "Van Halen", + }, + RawData: []byte{0xfe, 0x4e, 0x52, 0x05}, + }, + }, + } + actual := message.TableDelta{} + err := createCodec().Decoder(data, &actual) + assert.NoError(t, err) + + assert.Equal(t, expected, actual) +} diff --git a/api/message/types.go b/api/message/types.go index 5a2111e..b012b79 100644 --- a/api/message/types.go +++ b/api/message/types.go @@ -64,8 +64,10 @@ type ActionTrace struct { // Contract account. Contract string `json:"contract" msgpack:"contract"` - Receiver string `json:"receiver" msgpack:"receiver"` - Data interface{} `json:"data" msgpack:"data"` + Receiver string `json:"receiver" msgpack:"receiver"` + FirstReceiver bool `json:"first_receiver" msgpack:"first_receiver"` + + Data interface{} `json:"data" msgpack:"data"` Authorization []PermissionLevel `json:"authorization" msgpack:"authorization"` @@ -80,3 +82,21 @@ func (act ActionTrace) GetData() (map[string]any, error) { } return nil, errors.New("failed to convert data to map") } + +type RollbackMessage struct { + OldBlockNum uint32 `json:"old_block" msgpack:"old_block"` + NewBlockNum uint32 `json:"new_block" msgpack:"new_block"` +} + +type TableDeltaRow struct { + Present bool `json:"present" msgpack:"present"` + Data map[string]any `json:"data" msgpack:"data"` + RawData []byte `json:"raw_data" msgpack:"raw_data"` +} + +type TableDelta struct { + BlockNum uint32 `json:"blocknum" msgpack:"blocknum"` + Timestamp time.Time `json:"blocktimestamp" msgpack:"blocktimestamp"` + Name string `json:"name" msgpack:"name"` + Rows []TableDeltaRow `json:"rows" msgpack:"rows"` +} diff --git a/api/reader.go b/api/reader.go index 5310932..5dd719e 100644 --- a/api/reader.go +++ b/api/reader.go @@ -6,12 +6,15 @@ package api // This is a low-level interface typically implemented by backend drivers type Reader interface { // Read a message from a channel. - // Read may block until a message is ready or an error occured. + // Read may block until a message is ready or an error occurred. + // + // io.EOF is returned from a reader when there is no more data to be read. + // If Read returns io.EOF all subsequent calls must also return io.EOF // // This function should be designed to handle concurrent calls. eg. thread safe. Read(channel Channel) ([]byte, error) // Close closes the reader - // Any blocked Read operations will be unblocked. + // Any blocked Read operations will be unblocked and return io.EOF Close() error } diff --git a/api/redis/namespace.go b/api/redis/namespace.go index 4e0167c..d02d579 100644 --- a/api/redis/namespace.go +++ b/api/redis/namespace.go @@ -19,7 +19,7 @@ const ( // // Contains a prefix and chain_id to guard keys against collision. // Prefix should be sufficient to not collide with other application using the same redis database. -// chain_id should be ok to not let multiple reader with different chains to write to the same channels. +// chain_id should be fine to not let multiple reader with different chains to write to the same channels. type Namespace struct { Prefix string diff --git a/api/redis/subscriber.go b/api/redis/subscriber.go index 9a8e8fe..c8c7099 100644 --- a/api/redis/subscriber.go +++ b/api/redis/subscriber.go @@ -2,6 +2,7 @@ package redis import ( "context" + "io" "sync" "time" @@ -11,8 +12,10 @@ import ( ) type Subscriber struct { - sub *redis.PubSub - ctx context.Context + sub *redis.PubSub + ctx context.Context + + // Mutex for channels map. mu sync.RWMutex timeout time.Duration channels map[string]chan []byte @@ -45,24 +48,17 @@ func NewSubscriber(ctx context.Context, client *redis.Client, ns Namespace, opti return sub } -// forward forwards a message to the channel. -// as writes to a unbuffered channel will block until it's read. -// We run select on it and discard the message if no read happends during timeout -func forward(msg redis.Message, ch chan<- []byte, timeout time.Duration) { - select { - case <-time.After(timeout): - case ch <- []byte(msg.Payload): - } -} - -// worker reads messages from redis pubsub and forwards them to +// worker reads messages from Redis pubsub and forwards them to // correct channels. func (s *Subscriber) worker() { for msg := range s.sub.Channel() { // Route message to correct channel. s.mu.RLock() if ch, ok := s.channels[msg.Channel]; ok { - go forward(*msg, ch, s.timeout) + select { + case <-time.After(s.timeout): + case ch <- []byte(msg.Payload): + } } s.mu.RUnlock() } @@ -80,6 +76,10 @@ func (s *Subscriber) Read(channel api.Channel) ([]byte, error) { // Subscribe and insert it. err = s.sub.Subscribe(s.ctx, key) if err != nil { + // Closed redis client is considered an EOF. + if err == redis.ErrClosed { + err = io.EOF + } return nil, err } @@ -90,15 +90,24 @@ func (s *Subscriber) Read(channel api.Channel) ([]byte, error) { s.mu.Unlock() } - return <-ch, nil + data := <-ch + // Zero length data is considered an EOF + if len(data) == 0 { + return nil, io.EOF + } + return data, nil } func (s *Subscriber) Close() error { + // Close redis pubsub. err := s.sub.Close() + // Close all go channels, this will make Read() unblock. for _, ch := range s.channels { close(ch) } + + // Clear the channel map of old channels. s.mu.Lock() s.channels = make(map[string]chan []byte) s.mu.Unlock() diff --git a/app/abi/cache.go b/app/abi/cache.go deleted file mode 100644 index ac595f0..0000000 --- a/app/abi/cache.go +++ /dev/null @@ -1,42 +0,0 @@ -package abi - -import ( - "context" - "time" - - eos "github.com/eoscanada/eos-go" - redis_cache "github.com/go-redis/cache/v9" -) - -type Cache struct { - c *redis_cache.Cache - ctx context.Context - prefix string -} - -func NewCache(prefix string, options *redis_cache.Options) *Cache { - return &Cache{ - c: redis_cache.New(options), - ctx: context.Background(), - prefix: prefix, - } -} - -func (cache *Cache) Get(account string) (*eos.ABI, error) { - var v eos.ABI - err := cache.c.Get(cache.ctx, cache.key(account), &v) - return &v, err -} - -func (cache *Cache) Set(account string, abi *eos.ABI, ttl time.Duration) error { - return cache.c.Set(&redis_cache.Item{ - Ctx: cache.ctx, - Key: cache.key(account), - Value: *abi, - TTL: ttl, - }) -} - -func (cache *Cache) key(account string) string { - return cache.prefix + "::" + account -} diff --git a/app/abi/cache_test.go b/app/abi/cache_test.go deleted file mode 100644 index da74e74..0000000 --- a/app/abi/cache_test.go +++ /dev/null @@ -1,158 +0,0 @@ -package abi - -import ( - "strings" - "testing" - "time" - - eos "github.com/eoscanada/eos-go" - redis_cache "github.com/go-redis/cache/v9" - "github.com/go-redis/redismock/v9" - - "github.com/stretchr/testify/assert" -) - -var abiString = ` -{ - "version": "eosio::abi/1.0", - "types": [{ - "new_type_name": "new_type_name_1", - "type": "name" - }], - "structs": [ - { - "name": "struct_name_1", - "base": "struct_name_2", - "fields": [ - {"name":"struct_1_field_1", "type":"new_type_name_1"}, - {"name":"struct_1_field_2", "type":"struct_name_3"}, - {"name":"struct_1_field_3", "type":"string?"}, - {"name":"struct_1_field_4", "type":"string?"}, - {"name":"struct_1_field_5", "type":"struct_name_4[]"} - ] - },{ - "name": "struct_name_2", - "base": "", - "fields": [ - {"name":"struct_2_field_1", "type":"string"} - ] - },{ - "name": "struct_name_3", - "base": "", - "fields": [ - {"name":"struct_3_field_1", "type":"string"} - ] - },{ - "name": "struct_name_4", - "base": "", - "fields": [ - {"name":"struct_4_field_1", "type":"string"} - ] - } - ], - "actions": [{ - "name": "action_name_1", - "type": "struct_name_1", - "ricardian_contract": "" - }], - "tables": [{ - "name": "table_name_1", - "index_type": "i64", - "key_names": [ - "key_name_1", - "key_name_2" - ], - "key_types": [ - "string", - "int" - ], - "type": "struct_name_1" - } - ] -} -` - -func TestGetSet(t *testing.T) { - client, mock := redismock.NewClientMock() - - c := NewCache("thalos::cache::test", &redis_cache.Options{ - Redis: client, - // Cache 10k keys for 1 minute. - LocalCache: redis_cache.NewTinyLFU(10000, time.Minute), - }) - - abi, err := eos.NewABI(strings.NewReader(abiString)) - assert.NoError(t, err) - - bytes, _ := c.c.Marshal(*abi) - - mock.ExpectSet("thalos::cache::test::testaccount", bytes, time.Minute).SetVal("OK") - - err = c.Set("testaccount", abi, time.Minute) - assert.NoError(t, err) - - c_abi, err := c.Get("testaccount") - assert.NoError(t, err) - - assert.Equal(t, c_abi.Version, "eosio::abi/1.0") - - // Types - assert.Equal(t, c_abi.Types[0].NewTypeName, "new_type_name_1") - assert.Equal(t, c_abi.Types[0].Type, "name") - - // Structs - assert.Equal(t, c_abi.Structs[0].Name, "struct_name_1") - assert.Equal(t, c_abi.Structs[0].Base, "struct_name_2") - assert.Equal(t, c_abi.Structs[0].Fields[0].Name, "struct_1_field_1") - assert.Equal(t, c_abi.Structs[0].Fields[0].Type, "new_type_name_1") - assert.Equal(t, c_abi.Structs[0].Fields[1].Name, "struct_1_field_2") - assert.Equal(t, c_abi.Structs[0].Fields[1].Type, "struct_name_3") - assert.Equal(t, c_abi.Structs[0].Fields[2].Name, "struct_1_field_3") - assert.Equal(t, c_abi.Structs[0].Fields[2].Type, "string?") - assert.Equal(t, c_abi.Structs[0].Fields[3].Name, "struct_1_field_4") - assert.Equal(t, c_abi.Structs[0].Fields[3].Type, "string?") - assert.Equal(t, c_abi.Structs[0].Fields[4].Name, "struct_1_field_5") - assert.Equal(t, c_abi.Structs[0].Fields[4].Type, "struct_name_4[]") - - assert.Equal(t, c_abi.Structs[1].Name, "struct_name_2") - assert.Equal(t, c_abi.Structs[1].Base, "") - assert.Equal(t, c_abi.Structs[1].Fields[0].Name, "struct_2_field_1") - assert.Equal(t, c_abi.Structs[1].Fields[0].Type, "string") - - assert.Equal(t, c_abi.Structs[2].Name, "struct_name_3") - assert.Equal(t, c_abi.Structs[2].Base, "") - assert.Equal(t, c_abi.Structs[2].Fields[0].Name, "struct_3_field_1") - assert.Equal(t, c_abi.Structs[2].Fields[0].Type, "string") - - assert.Equal(t, c_abi.Structs[3].Name, "struct_name_4") - assert.Equal(t, c_abi.Structs[3].Base, "") - assert.Equal(t, c_abi.Structs[3].Fields[0].Name, "struct_4_field_1") - assert.Equal(t, c_abi.Structs[3].Fields[0].Type, "string") - - // Actions - assert.Equal(t, c_abi.Actions[0].Name, eos.ActN("action_name_1")) - assert.Equal(t, c_abi.Actions[0].Type, "struct_name_1") - assert.Equal(t, c_abi.Actions[0].RicardianContract, "") - - // Tables - assert.Equal(t, c_abi.Tables[0].Name, eos.TableName("table_name_1")) - assert.Equal(t, c_abi.Tables[0].Type, "struct_name_1") - assert.Equal(t, c_abi.Tables[0].IndexType, "i64") - assert.Equal(t, c_abi.Tables[0].KeyNames[0], "key_name_1") - assert.Equal(t, c_abi.Tables[0].KeyNames[1], "key_name_2") - assert.Equal(t, c_abi.Tables[0].KeyTypes[0], "string") - assert.Equal(t, c_abi.Tables[0].KeyTypes[1], "int") -} - -func TestCacheMiss(t *testing.T) { - client, _ := redismock.NewClientMock() - - c := NewCache("thalos::cache::test", &redis_cache.Options{ - Redis: client, - // Cache 10k keys for 1 minute. - LocalCache: redis_cache.NewTinyLFU(10000, time.Minute), - }) - - _, err := c.Get("nonexist") - assert.Error(t, err) -} diff --git a/app/abi/manager.go b/app/abi/manager.go deleted file mode 100644 index 2040a3a..0000000 --- a/app/abi/manager.go +++ /dev/null @@ -1,56 +0,0 @@ -package abi - -import ( - "context" - "fmt" - "time" - - eos "github.com/eoscanada/eos-go" - redis_cache "github.com/go-redis/cache/v9" - "github.com/redis/go-redis/v9" -) - -type AbiManager struct { - cache *Cache - api *eos.API - ctx context.Context -} - -func NewAbiManager(rdb *redis.Client, api *eos.API, id string) *AbiManager { - // Init abi cache - cache := NewCache("thalos::cache::"+id+"::abi", &redis_cache.Options{ - Redis: rdb, - // Cache 10k keys for 10 minutes. - LocalCache: redis_cache.NewTinyLFU(10000, 10*time.Minute), - }) - - return &AbiManager{ - cache: cache, - api: api, - ctx: context.Background(), - } -} - -// Set or update an ABI in the cache. -func (mgr *AbiManager) SetAbi(account eos.AccountName, abi *eos.ABI) error { - return mgr.cache.Set(string(account), abi, time.Hour) -} - -func (mgr *AbiManager) GetAbi(account eos.AccountName) (*eos.ABI, error) { - key := string(account) - - abi, err := mgr.cache.Get(key) - if err != nil { - resp, err := mgr.api.GetABI(mgr.ctx, account) - if err != nil { - return nil, fmt.Errorf("api: %s", err) - } - abi = &resp.ABI - - err = mgr.SetAbi(account, abi) - if err != nil { - return nil, fmt.Errorf("cache: %s", err) - } - } - return abi, nil -} diff --git a/app/config/config.go b/app/config/config.go deleted file mode 100644 index 18f9a00..0000000 --- a/app/config/config.go +++ /dev/null @@ -1,97 +0,0 @@ -package config - -import ( - "io/ioutil" - "time" - - "github.com/eosswedenorg/thalos/app/log" - - "gopkg.in/yaml.v3" - - shipclient "github.com/eosswedenorg-go/antelope-ship-client" -) - -type RedisConfig struct { - Addr string `yaml:"addr"` - User string `yaml:"user"` - Password string `yaml:"password"` - DB int `yaml:"db"` - Prefix string `yaml:"prefix"` -} - -type TelegramConfig struct { - Id string `yaml:"id"` - Channel int64 `yaml:"channel"` -} - -type ShipConfig struct { - Url string `yaml:"url"` - IrreversibleOnly bool `yaml:"irreversible_only"` - MaxMessagesInFlight uint32 `yaml:"max_messages_in_flight"` - StartBlockNum uint32 `yaml:"start_block_num"` - EndBlockNum uint32 `yaml:"end_block_num"` - Chain string `yaml:"chain"` -} - -type Config struct { - Name string `yaml:"name"` - Ship ShipConfig `yaml:"ship"` - Api string `yaml:"api"` - - Log log.Config `yaml:"log"` - - Redis RedisConfig `yaml:"redis"` - MessageCodec string `yaml:"message_codec"` - - Telegram TelegramConfig `yaml:"telegram"` -} - -func Parse(data []byte) (*Config, error) { - cfg := Config{ - MessageCodec: "json", - Log: log.Config{ - MaxFileSize: 10 * 1000 * 1000, // 10 mb - MaxTime: time.Hour * 24, - }, - Ship: ShipConfig{ - StartBlockNum: shipclient.NULL_BLOCK_NUMBER, - EndBlockNum: shipclient.NULL_BLOCK_NUMBER, - MaxMessagesInFlight: 10, - IrreversibleOnly: false, - }, - Redis: RedisConfig{ - Addr: "localhost:6379", - Password: "", - DB: 0, - Prefix: "ship", - }, - } - - err := yaml.Unmarshal(data, &cfg) - return &cfg, err -} - -func (ship *ShipConfig) UnmarshalYAML(value *yaml.Node) error { - var err error - - if value.Kind == yaml.ScalarNode { - ship.Url = value.Value - } else { - type ShipConfigRaw ShipConfig - raw := ShipConfigRaw(*ship) - if err = value.Decode(&raw); err == nil { - *ship = ShipConfig(raw) - } - } - - return err -} - -func Load(filename string) (*Config, error) { - bytes, err := ioutil.ReadFile(filename) - if err != nil { - return nil, err - } - - return Parse(bytes) -} diff --git a/app/config/config_test.go b/app/config/config_test.go deleted file mode 100644 index d9aad83..0000000 --- a/app/config/config_test.go +++ /dev/null @@ -1,147 +0,0 @@ -package config - -import ( - "testing" - "time" - - "github.com/eosswedenorg/thalos/app/log" - "github.com/stretchr/testify/require" - - shipclient "github.com/eosswedenorg-go/antelope-ship-client" -) - -func TestParse_Default(t *testing.T) { - expected := Config{ - MessageCodec: "json", - - Log: log.Config{ - MaxFileSize: 10 * 1000 * 1000, // 10 mb - MaxTime: time.Hour * 24, - }, - - Ship: ShipConfig{ - StartBlockNum: shipclient.NULL_BLOCK_NUMBER, - EndBlockNum: shipclient.NULL_BLOCK_NUMBER, - MaxMessagesInFlight: 10, - IrreversibleOnly: false, - }, - - Redis: RedisConfig{ - Addr: "localhost:6379", - Password: "", - DB: 0, - Prefix: "ship", - }, - } - - cfg, err := Parse([]byte(``)) - require.NoError(t, err) - require.Equal(t, cfg, &expected) -} - -func TestParse(t *testing.T) { - expected := Config{ - Name: "ship-reader-1", - Api: "http://127.0.0.1:8080", - MessageCodec: "mojibake", - Log: log.Config{ - Filename: "some_file.log", - Directory: "/path/to/whatever", - MaxFileSize: 200, - MaxTime: 30 * time.Minute, - }, - Ship: ShipConfig{ - Url: "127.0.0.1:8089", - StartBlockNum: 23671836, - EndBlockNum: 23872222, - IrreversibleOnly: true, - MaxMessagesInFlight: 1337, - }, - Telegram: TelegramConfig{ - Id: "110201543:AAHdqTcvCH1vGWJxfSeofSAs0K5PALDsaw", - Channel: -123456789, - }, - Redis: RedisConfig{ - Addr: "localhost:6379", - User: "myuser", - Password: "passwd", - DB: 4, - Prefix: "some::ship", - }, - } - - cfg, err := Parse([]byte(` -name: "ship-reader-1" -api: "http://127.0.0.1:8080" -message_codec: "mojibake" -log: - filename: some_file.log - directory: /path/to/whatever - maxtime: 30m - maxfilesize: 200b -ship: - url: "127.0.0.1:8089" - irreversible_only: true - max_messages_in_flight: 1337 - start_block_num: 23671836 - end_block_num: 23872222 -telegram: - id: "110201543:AAHdqTcvCH1vGWJxfSeofSAs0K5PALDsaw" - channel: -123456789 -redis: - addr: "localhost:6379" - user: "myuser" - password: "passwd" - db: 4 - prefix: "some::ship" -`)) - - require.NoError(t, err) - require.Equal(t, cfg, &expected) -} - -func TestParseShorthandShipUrl(t *testing.T) { - expected := Config{ - Name: "ship-reader-1", - Api: "http://127.0.0.1:8080", - MessageCodec: "json", - Log: log.Config{ - MaxFileSize: 10 * 1000 * 1000, // 10 mb - MaxTime: time.Hour * 24, - }, - Ship: ShipConfig{ - Url: "127.0.0.1:8089", - StartBlockNum: shipclient.NULL_BLOCK_NUMBER, - EndBlockNum: shipclient.NULL_BLOCK_NUMBER, - MaxMessagesInFlight: 10, - IrreversibleOnly: false, - }, - Telegram: TelegramConfig{ - Id: "110201543:AAHdqTcvCH1vGWJxfSeofSAs0K5PALDsaw", - Channel: -123456789, - }, - Redis: RedisConfig{ - Addr: "localhost:6379", - Password: "passwd", - DB: 4, - Prefix: "some::ship", - }, - } - - cfg, err := Parse([]byte(` -name: "ship-reader-1" -api: "http://127.0.0.1:8080" -ship: "127.0.0.1:8089" -telegram: - id: "110201543:AAHdqTcvCH1vGWJxfSeofSAs0K5PALDsaw" - channel: -123456789 -redis: - addr: "localhost:6379" - password: "passwd" - db: 4 - prefix: "some::ship" -`)) - - require.NoError(t, err) - require.Equal(t, cfg, &expected) -} diff --git a/app/log/config.go b/app/log/config.go deleted file mode 100644 index 54c3b55..0000000 --- a/app/log/config.go +++ /dev/null @@ -1,27 +0,0 @@ -package log - -import ( - "path" - "time" - - "github.com/eosswedenorg/thalos/app/types" -) - -type Config struct { - Filename string `yaml:"filename"` - Directory string `yaml:"directory"` - MaxFileSize types.Size `yaml:"maxfilesize"` - MaxTime time.Duration `yaml:"maxtime"` -} - -func (c Config) GetFilename() string { - return path.Base(c.Filename) -} - -func (c Config) GetDirectory() string { - return path.Clean(c.Directory) -} - -func (c Config) GetFilePath() string { - return path.Join(c.GetDirectory(), c.GetFilename()) -} diff --git a/app/ship_processor.go b/app/ship_processor.go deleted file mode 100644 index 7ce37f5..0000000 --- a/app/ship_processor.go +++ /dev/null @@ -1,266 +0,0 @@ -package app - -import ( - "encoding/hex" - "encoding/json" - - "github.com/eosswedenorg/thalos/api" - "github.com/eosswedenorg/thalos/api/message" - "github.com/eosswedenorg/thalos/app/abi" - - log "github.com/sirupsen/logrus" - - "github.com/eoscanada/eos-go" - "github.com/eoscanada/eos-go/ship" - shipclient "github.com/eosswedenorg-go/antelope-ship-client" -) - -// logDecoratedEncoder decorates a message.Encoder and logs any error. -func logDecoratedEncoder(encoder message.Encoder) message.Encoder { - return func(v interface{}) ([]byte, error) { - payload, err := encoder(v) - if err != nil { - log.WithError(err). - WithField("v", v). - Warn("Failed to encode message") - } - return payload, err - } -} - -type ShipProcessor struct { - abi *abi.AbiManager - writer api.Writer - shipStream *shipclient.Stream - encode message.Encoder - - // Keep track of the current block we have processed. - current_block uint32 - - // System contract ("eosio" per default) - syscontract eos.AccountName -} - -func SpawnProccessor(shipStream *shipclient.Stream, writer api.Writer, abi *abi.AbiManager, codec message.Codec) *ShipProcessor { - processor := &ShipProcessor{ - abi: abi, - writer: writer, - shipStream: shipStream, - encode: logDecoratedEncoder(codec.Encoder), - syscontract: eos.AccountName("eosio"), - current_block: shipStream.StartBlock, - } - - // Attach handlers - shipStream.BlockHandler = processor.processBlock - - // Needed because if nil, traces will not be included in the response from ship. - shipStream.TraceHandler = func([]*ship.TransactionTraceV0) {} - - return processor -} - -func (processor *ShipProcessor) queueMessage(channel api.Channel, payload []byte) bool { - err := processor.writer.Write(channel, payload) - if err != nil { - log.WithError(err).Errorf("Failed to post to channel '%s'", channel) - return false - } - return true -} - -func (processor *ShipProcessor) encodeQueue(channel api.Channel, v interface{}) bool { - if payload, err := processor.encode(v); err == nil { - return processor.queueMessage(channel, payload) - } - return false -} - -func decode(abi *eos.ABI, act *ship.Action, v any) error { - jsondata, err := abi.DecodeAction(act.Data, act.Name) - if err != nil { - return err - } - return json.Unmarshal(jsondata, v) -} - -func (processor *ShipProcessor) updateAbiFromAction(act *ship.Action) error { - ABI, err := processor.abi.GetAbi(processor.syscontract) - if err != nil { - return err - } - - set_abi := struct { - Abi string - Account eos.AccountName - }{} - if err := decode(ABI, act, &set_abi); err != nil { - return err - } - - binary_abi, err := hex.DecodeString(set_abi.Abi) - if err != nil { - return err - } - - contract_abi := eos.ABI{} - if err = eos.UnmarshalBinary(binary_abi, &contract_abi); err != nil { - return err - } - - return processor.abi.SetAbi(set_abi.Account, &contract_abi) -} - -func (processor *ShipProcessor) GetCurrentBlock() uint32 { - return processor.current_block -} - -func (processor *ShipProcessor) processBlock(block *ship.GetBlocksResultV0) { - processor.current_block = block.ThisBlock.BlockNum - - if block.ThisBlock.BlockNum%100 == 0 { - log.Infof("Current: %d, Head: %d", processor.current_block, block.Head.BlockNum) - } - - if block.ThisBlock.BlockNum%10 == 0 { - hb := message.HeartBeat{ - BlockNum: block.ThisBlock.BlockNum, - LastIrreversibleBlockNum: block.LastIrreversible.BlockNum, - HeadBlockNum: block.Head.BlockNum, - } - - processor.encodeQueue(api.HeartbeatChannel, hb) - } - - // Process traces - if block.Traces != nil && len(block.Traces.Elem) > 0 { - for _, trace := range block.Traces.AsTransactionTracesV0() { - - logger := log.WithField("tx_id", trace.ID.String()) - - transaction := message.TransactionTrace{ - ID: trace.ID.String(), - BlockNum: block.Block.BlockNumber(), - Timestamp: block.Block.Timestamp.Time.UTC(), - Status: trace.Status.String(), - CPUUsageUS: trace.CPUUsageUS, - NetUsage: trace.NetUsage, - NetUsageWords: uint32(trace.NetUsageWords), - Elapsed: int64(trace.Elapsed), - Scheduled: trace.Scheduled, - Except: trace.Except, - Error: trace.ErrorCode, - } - - // Actions - for _, actionTraceVar := range trace.ActionTraces { - var act_trace *ship.ActionTraceV1 - - if trace_v0, ok := actionTraceVar.Impl.(*ship.ActionTraceV0); ok { - // convert to v1 - act_trace = &ship.ActionTraceV1{ - ActionOrdinal: trace_v0.ActionOrdinal, - CreatorActionOrdinal: trace_v0.CreatorActionOrdinal, - Receipt: trace_v0.Receipt, - Receiver: trace_v0.Receiver, - Act: trace_v0.Act, - ContextFree: trace_v0.ContextFree, - Elapsed: trace_v0.Elapsed, - Console: trace_v0.Console, - AccountRamDeltas: trace_v0.AccountRamDeltas, - Except: trace_v0.Except, - ErrorCode: trace_v0.ErrorCode, - ReturnValue: []byte{}, - } - } else { - act_trace = actionTraceVar.Impl.(*ship.ActionTraceV1) - } - - // Check if actions updates an abi. - if act_trace.Act.Account == processor.syscontract && act_trace.Act.Name == eos.ActionName("setabi") { - err := processor.updateAbiFromAction(act_trace.Act) - if err != nil { - logger.WithError(err).Warn("Failed to update abi") - } - } - - act := message.ActionTrace{ - TxID: trace.ID.String(), - BlockNum: block.Block.BlockNumber(), - Timestamp: block.Block.Timestamp.Time.UTC(), - Name: act_trace.Act.Name.String(), - Contract: act_trace.Act.Account.String(), - Receiver: act_trace.Receiver.String(), - } - - if act_trace.Receipt != nil { - receipt := act_trace.Receipt.Impl.(*ship.ActionReceiptV0) - act.Receipt = &message.ActionReceipt{ - Receiver: receipt.Receiver.String(), - ActDigest: receipt.ActDigest.String(), - GlobalSequence: receipt.GlobalSequence, - RecvSequence: receipt.GlobalSequence, - CodeSequence: uint32(receipt.CodeSequence), - ABISequence: uint32(receipt.ABISequence), - } - - for _, auth := range receipt.AuthSequence { - act.Receipt.AuthSequence = append(act.Receipt.AuthSequence, message.AccountAuthSequence{ - Account: auth.Account.String(), - Sequence: auth.Sequence, - }) - } - } - - for _, auth := range act_trace.Act.Authorization { - act.Authorization = append(act.Authorization, message.PermissionLevel{ - Actor: auth.Actor.String(), - Permission: auth.Permission.String(), - }) - } - - ABI, err := processor.abi.GetAbi(act_trace.Act.Account) - if err == nil { - if err = decode(ABI, act_trace.Act, &act.Data); err != nil { - logger.WithFields(log.Fields{ - "contract": act_trace.Act.Account, - "action": act_trace.Act.Name, - }).WithError(err).Warn("Failed to decode action") - } - } else { - logger.WithField("contract", act_trace.Act.Account). - WithError(err).Errorf("Failed to get abi for contract %s", act_trace.Act.Account) - } - - payload, err := processor.encode(act) - if err != nil { - continue - } - - transaction.ActionTraces = append(transaction.ActionTraces, act) - - channels := []api.Channel{ - api.ActionChannel{}.Channel(), - api.ActionChannel{Name: act.Name}.Channel(), - api.ActionChannel{Contract: act.Contract}.Channel(), - api.ActionChannel{Name: act.Name, Contract: act.Contract}.Channel(), - } - - for _, channel := range channels { - processor.queueMessage(channel, payload) - } - } - - processor.encodeQueue(api.TransactionChannel, transaction) - } - } - - err := processor.writer.Flush() - if err != nil { - log.WithError(err).Error("Failed to send messages") - } -} - -func (processor *ShipProcessor) Close() error { - return processor.writer.Close() -} diff --git a/cmd/thalos/antelope_api_helpers.go b/cmd/thalos/antelope_api_helpers.go new file mode 100644 index 0000000..0096143 --- /dev/null +++ b/cmd/thalos/antelope_api_helpers.go @@ -0,0 +1,35 @@ +package main + +import ( + "context" + "time" + + antelopeapi "github.com/shufflingpixels/antelope-go/api" + log "github.com/sirupsen/logrus" +) + +// "Clever" way to make sure we only call the api once. +// Store a info pointer outside the returned closure. +// that pointer will live as long as the closure lives. +// and inside the closure we will reference the pointer and only +// call the api if it is nil. +func chainInfoOnce(api *antelopeapi.Client) func() *antelopeapi.Info { + var info *antelopeapi.Info + return func() *antelopeapi.Info { + if info == nil { + log.WithField("api", api.Url).Info("Get chain info from api") + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + + result, err := api.GetInfo(ctx) + if err != nil { + log.WithError(err).Fatal("Failed to call eos api") + return nil + } + + info = &result + } + return info + } +} diff --git a/cmd/thalos/main.go b/cmd/thalos/main.go index 087fef9..fc55eef 100644 --- a/cmd/thalos/main.go +++ b/cmd/thalos/main.go @@ -1,307 +1,46 @@ package main import ( - "context" - "errors" - "fmt" - "io" - "os" - "os/signal" - "path" - "syscall" - "time" - - "github.com/cenkalti/backoff/v4" - eos "github.com/eoscanada/eos-go" - shipclient "github.com/eosswedenorg-go/antelope-ship-client" - "github.com/eosswedenorg-go/pid" - "github.com/eosswedenorg/thalos/api/message" - _ "github.com/eosswedenorg/thalos/api/message/json" - _ "github.com/eosswedenorg/thalos/api/message/msgpack" - api_redis "github.com/eosswedenorg/thalos/api/redis" - "github.com/eosswedenorg/thalos/app" - "github.com/eosswedenorg/thalos/app/abi" - "github.com/eosswedenorg/thalos/app/config" - . "github.com/eosswedenorg/thalos/app/log" - "github.com/nikoksr/notify" - "github.com/nikoksr/notify/service/telegram" - "github.com/pborman/getopt/v2" - "github.com/redis/go-redis/v9" + "github.com/eosswedenorg/thalos/internal/config" log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + "github.com/spf13/pflag" ) -// --------------------------- -// Global variables -// --------------------------- - -var conf *config.Config - -var shClient *shipclient.Stream - -var running bool = false - var VersionString string = "dev" -var exit chan bool +var rootCmd *cobra.Command -func readerLoop(processor *app.ShipProcessor) { - running = true - recon_cnt := 0 - - exp := &backoff.ExponentialBackOff{ - InitialInterval: time.Second, - RandomizationFactor: 0.25, - Multiplier: 2, - MaxInterval: 10 * time.Minute, - MaxElapsedTime: 0, - Stop: -1, - Clock: backoff.SystemClock, - } - exp.Reset() - - log.WithFields(log.Fields{ - "initial_interval": exp.InitialInterval, - "max_interval": exp.MaxInterval, - "randomization_factor": exp.RandomizationFactor, - "multiplier": exp.Multiplier, - }).Info("Connecting with Exponential Backoff") - - connectOp := func() error { - recon_cnt++ - - log.Infof("Connecting to ship at: %s (Try %d)", conf.Ship.Url, recon_cnt) - - if err := shClient.Connect(conf.Ship.Url); err != nil { - return err - } - - // Set stream client start block to processors current block - // Both values should be the same on first connect, but when reconnecting - // We don't want to start from the beginning - shClient.StartBlock = processor.GetCurrentBlock() - - return shClient.SendBlocksRequest() +func init() { + rootCmd = &cobra.Command{ + Use: "thalos-server", + FParseErrWhitelist: cobra.FParseErrWhitelist{ + UnknownFlags: true, + }, + Version: VersionString, + Run: serverCmd, } - for running { + rootCmd.SetHelpTemplate( + `{{ .Use | trimTrailingWhitespaces}} v{{.Version}} - err := backoff.RetryNotify(connectOp, exp, func(err error, d time.Duration) { - if recon_cnt >= 3 { - msg := fmt.Sprintf("Failed to connect to ship at '%s'", conf.Ship.Url) - if err := notify.Send(context.Background(), conf.Name, msg); err != nil { - log.WithError(err).Error("Failed to send notification") - } - recon_cnt = 0 - } +{{if or .Runnable .HasSubCommands}}{{.UsageString}}{{end}} +`) + rootCmd.SetVersionTemplate(`{{with .Name}}{{printf "%s " .}}{{end}}{{printf "v%s" .Version}}` + "\n") - log.WithError(err).Error("Failed to connect to SHIP") + flags := pflag.FlagSet{} - log.WithFields(log.Fields{ - "reconn_at": time.Now().Add(d), - "reconn_in": d, - }).Info("Reconnecting in ", d) - }) - if err != nil { - log.WithError(err).Error("Failed to connect to SHIP") - running = false - continue - } + flags.StringP("config", "c", "./config.yml", "Config file to read") + flags.StringP("level", "L", "info", "Log level to use") + flags.StringP("pid", "p", "", "Where to write process id") + flags.BoolP("no-state-cache", "n", false, "Force the application to take start block from config/api") - recon_cnt = 0 - log.Infof("Connected, Start: %d, End: %d", shClient.StartBlock, shClient.EndBlock) - - if err := shClient.Run(); err != nil { - - if errors.Is(err, shipclient.ErrEndBlockReached) { - exit <- true - log.Info("Endblock reached.") - break - } - - log.WithError(err).Error("Failed to read from ship") - } - } -} - -func run(processor *app.ShipProcessor) { - // Spawn reader loop in another thread. - go readerLoop(processor) - - // Create interrupt channel. - signals := make(chan os.Signal, 1) - - // Register signal channel to receive signals from the os. - signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM) - - // Wait for interrupt - select { - case sig := <-signals: - log.WithField("signal", sig).Info("Signal received") - - // Cleanly close the connection by sending a close message. - err := shClient.Shutdown() - if err != nil { - log.WithError(err).Info("failed to send close message to ship server") - } - case <-exit: - // Do nothing, just exit. - } - - running = false -} - -func getChain(def string) string { - if len(conf.Ship.Chain) > 0 { - return conf.Ship.Chain - } - return def + rootCmd.PersistentFlags().AddFlagSet(&flags) + rootCmd.PersistentFlags().AddFlagSet(config.GetFlags()) } func main() { - var err error - var chainInfo *eos.InfoResp - - exit = make(chan bool) - - showHelp := getopt.BoolLong("help", 'h', "display this help text") - showVersion := getopt.BoolLong("version", 'v', "display this help text") - configFile := getopt.StringLong("config", 'c', "./config.yml", "Config file to read", "file") - pidFile := getopt.StringLong("pid", 'p', "", "Where to write process id", "file") - logFile := getopt.StringLong("log", 'l', "", "Path to log file", "file") - - getopt.Parse() - - if *showHelp { - getopt.Usage() - return + if err := rootCmd.Execute(); err != nil { + log.Fatal(err) } - - if *showVersion { - fmt.Println(VersionString) - return - } - - // Write PID file - if len(*pidFile) > 0 { - log.Infof("Writing pid to: %s", *pidFile) - err = pid.Save(*pidFile) - if err != nil { - log.WithError(err).Fatal("failed to write pid file") - return - } - } - - // Parse config - conf, err = config.Load(*configFile) - if err != nil { - log.WithError(err).Fatal("failed to read config file") - return - } - - // If log file is given on the commandline, override config values. - if len(*logFile) > 0 { - conf.Log.Directory = path.Dir(*logFile) - conf.Log.Filename = path.Base(*logFile) - } - - if len(conf.Log.Filename) > 0 { - stdWriter, err := NewRotatingFileFromConfig(conf.Log, "info") - if err != nil { - log.WithError(err).Fatal("Failed to open info log") - return - } - errWriter, err := NewRotatingFileFromConfig(conf.Log, "error") - if err != nil { - log.WithError(err).Fatal("Failed to open error log") - return - } - - log.WithFields(log.Fields{ - "maxfilesize": conf.Log.MaxFileSize, - "maxage": conf.Log.MaxTime, - "directory": conf.Log.GetDirectory(), - "info_filename": stdWriter.GetFilename(), - "error_filename": errWriter.GetFilename(), - }).Info("Logging to file") - - log.SetOutput(io.Discard) - log.AddHook(MakeStdHook(stdWriter)) - log.AddHook(MakeErrorHook(errWriter)) - } - - // Init telegram notification service - if len(conf.Telegram.Id) > 0 { - - telegram, err := telegram.New(conf.Telegram.Id) - if err != nil { - log.WithError(err).Fatal("Failed to initialize telegram") - return - } - - telegram.AddReceivers(conf.Telegram.Channel) - - // Register services in notification manager - notify.UseServices(telegram) - } - - // Connect to redis - rdb := redis.NewClient(&redis.Options{ - Addr: conf.Redis.Addr, - Username: conf.Redis.User, - Password: conf.Redis.Password, - DB: conf.Redis.DB, - }) - - err = rdb.Ping(context.Background()).Err() - if err != nil { - log.WithError(err).Fatal("Failed to connect to redis") - return - } - - log.Printf("Get chain info from api at: %s", conf.Api) - eosClient := eos.New(conf.Api) - chainInfo, err = eosClient.GetInfo(context.Background()) - if err != nil { - log.WithError(err).Fatal("Failed to get info") - return - } - - if conf.Ship.StartBlockNum == shipclient.NULL_BLOCK_NUMBER { - if conf.Ship.IrreversibleOnly { - conf.Ship.StartBlockNum = uint32(chainInfo.LastIrreversibleBlockNum) - } else { - conf.Ship.StartBlockNum = uint32(chainInfo.HeadBlockNum) - } - } - - shClient = shipclient.NewStream(func(s *shipclient.Stream) { - s.StartBlock = conf.Ship.StartBlockNum - s.EndBlock = conf.Ship.EndBlockNum - s.IrreversibleOnly = conf.Ship.IrreversibleOnly - }) - - // Get codec - codec, err := message.GetCodec(conf.MessageCodec) - if err != nil { - log.WithError(err).Fatal("Failed to load codec") - return - } - - chain_id := getChain(chainInfo.ChainID.String()) - - processor := app.SpawnProccessor( - shClient, - api_redis.NewPublisher(context.Background(), rdb, api_redis.Namespace{ - Prefix: conf.Redis.Prefix, - ChainID: chain_id, - }), - abi.NewAbiManager(rdb, eosClient, chain_id), - codec, - ) - - // Run the application - run(processor) - - // Close the processor properly - processor.Close() } diff --git a/cmd/thalos/server.go b/cmd/thalos/server.go new file mode 100644 index 0000000..72ea05f --- /dev/null +++ b/cmd/thalos/server.go @@ -0,0 +1,403 @@ +package main + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "os/signal" + "path" + "strings" + "syscall" + "time" + + "github.com/cenkalti/backoff/v4" + shipclient "github.com/eosswedenorg-go/antelope-ship-client" + shipws "github.com/eosswedenorg-go/antelope-ship-client/websocket" + "github.com/eosswedenorg-go/pid" + "github.com/eosswedenorg/thalos/api/message" + _ "github.com/eosswedenorg/thalos/api/message/json" + _ "github.com/eosswedenorg/thalos/api/message/msgpack" + api_redis "github.com/eosswedenorg/thalos/api/redis" + "github.com/eosswedenorg/thalos/internal/abi" + "github.com/eosswedenorg/thalos/internal/cache" + "github.com/eosswedenorg/thalos/internal/config" + driver "github.com/eosswedenorg/thalos/internal/driver/redis" + . "github.com/eosswedenorg/thalos/internal/log" + . "github.com/eosswedenorg/thalos/internal/server" + "github.com/nikoksr/notify" + "github.com/nikoksr/notify/service/telegram" + "github.com/redis/go-redis/v9" + antelopeapi "github.com/shufflingpixels/antelope-go/api" + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + "github.com/spf13/pflag" +) + +func readerLoop(conf *config.Config, running *bool, shClient *shipclient.Stream, processor *ShipProcessor) { + recon_cnt := 0 + + exp := &backoff.ExponentialBackOff{ + InitialInterval: time.Second, + RandomizationFactor: 0.25, + Multiplier: 2, + MaxInterval: 10 * time.Minute, + MaxElapsedTime: 0, + Stop: -1, + Clock: backoff.SystemClock, + } + exp.Reset() + + log.WithFields(log.Fields{ + "initial_interval": exp.InitialInterval, + "max_interval": exp.MaxInterval, + "randomization_factor": exp.RandomizationFactor, + "multiplier": exp.Multiplier, + }).Info("ship client: Connecting with Exponential Backoff") + + connectOp := func() error { + recon_cnt++ + + log.WithFields(log.Fields{ + "url": conf.Ship.Url, + "try": recon_cnt, + }).Info("ship client: Connecting") + + if err := shClient.Connect(conf.Ship.Url); err != nil { + return err + } + + // Set stream client start block to processors current block + // Both values should be the same on first connect, but when reconnecting + // We don't want to start from the beginning + shClient.StartBlock = processor.GetCurrentBlock() + + return shClient.SendBlocksRequest() + } + + for *running { + + err := backoff.RetryNotify(connectOp, exp, func(err error, d time.Duration) { + if recon_cnt >= 3 { + msg := fmt.Sprintf("Failed to connect to ship at '%s'", conf.Ship.Url) + if err := notify.Send(context.Background(), conf.Name, msg); err != nil { + log.WithError(err).Error("Failed to send notification") + } + recon_cnt = 0 + } + + log.WithError(err).Error("ship client: Failed to connect") + + log.WithFields(log.Fields{ + "reconn_at": time.Now().Add(d), + "reconn_in": d, + }).Info("ship client: Reconnecting in ", d) + }) + if err != nil { + log.WithError(err).Error("ship client:Failed to connect") + return + } + + recon_cnt = 0 + log.WithFields(log.Fields{ + "start": shClient.StartBlock, + "end": shClient.EndBlock, + }).Info("ship client: Connected to ship") + + if err := shClient.Run(); err != nil { + + if errors.Is(err, shipclient.ErrEndBlockReached) { + log.Info("ship client: Endblock reached.") + return + } + + if shipws.IsCloseError(err) { + log.Info("ship client: Connection closed normally") + return + } + + log.WithError(err).Error("ship client: Failed to read message") + } + } +} + +func run(conf *config.Config, shClient *shipclient.Stream, processor *ShipProcessor) { + running := true + + // Spawn reader loop in another thread. + go readerLoop(conf, &running, shClient, processor) + + // Create interrupt channel. + signals := make(chan os.Signal, 1) + + // Register signal channel to receive signals from the os. + signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM) + + // Wait for interrupt + sig := <-signals + log.WithField("signal", sig).Info("Signal received") + + // Cleanly close the connection by sending a close message. + running = false + err := shClient.Shutdown() + if err != nil { + log.WithError(err).Info("failed to send close message to ship server") + } +} + +func LogLevels() []string { + list := []string{} + for _, lvl := range log.AllLevels { + list = append(list, lvl.String()) + } + return list +} + +func initAbiManager(cfg *config.AbiCache, api *antelopeapi.Client, store cache.Store, chain_id string) *abi.AbiManager { + cache := cache.NewCache("thalos::cache::abi::"+chain_id, store) + return abi.NewAbiManager(cfg, cache, api) +} + +func stateLoader(conf *config.Config, start_block_flag *pflag.Flag, chainInfo func() *antelopeapi.Info, cache *cache.Cache, current_block_no_cache bool) StateLoader { + return func(state *State) { + var source string + + // Load state from cache. + ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*500) + err := cache.Get(ctx, "state", &state) + cancel() + + // on error (cache miss) or if current_block_no_cache is set. + // set current block from config/api + if current_block_no_cache || err != nil { + // Set from config if we have a sane value. + if conf.Ship.StartBlockNum != shipclient.NULL_BLOCK_NUMBER { + + if start_block_flag != nil && start_block_flag.Changed { + source = "cli" + } else { + source = "config" + } + + state.CurrentBlock = conf.Ship.StartBlockNum + } else { + // Otherwise, set from api. + if conf.Ship.IrreversibleOnly { + source = "api (LIB)" + state.CurrentBlock = uint32(chainInfo().LastIrreversableBlockNum) + } else { + source = "api (HEAD)" + state.CurrentBlock = uint32(chainInfo().HeadBlockNum) + } + } + } else { + source = "cache" + } + + log.WithFields(log.Fields{ + "block": state.CurrentBlock, + "source": source, + }).Info("Starting from block") + } +} + +func stateSaver(cache *cache.Cache) StateSaver { + return func(state State) error { + ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*500) + defer cancel() + return cache.Set(ctx, "state", state, 0) + } +} + +func GetConfig(flags *pflag.FlagSet) (*config.Config, error) { + filename, err := flags.GetString("config") + if err != nil { + return nil, err + } + + cfg, err := config.NewBuilder(). + SetConfigFile(filename). + SetFlags(flags). + Build() + if err != nil { + return nil, err + } + + logFile, _ := flags.GetString("log") + if len(logFile) > 0 { + cfg.Log.Directory = path.Dir(logFile) + cfg.Log.Filename = path.Base(logFile) + } + + // If start-block is provided, we should set no-state-cache to true. + if startBlock := flags.Lookup("start-block"); startBlock != nil && startBlock.Changed { + if err := flags.Set("no-state-cache", "true"); err != nil { + return cfg, nil + } + } + + cfg.Ship.Blacklist.SetWhitelist(cfg.Ship.BlacklistIsWhitelist) + + return cfg, nil +} + +func ConnectRedis(conf *config.RedisConfig) (*redis.Client, error) { + logEntry := log.WithFields(log.Fields{ + "addr": conf.Addr, + "db": conf.DB, + }) + + if len(conf.User) > 0 { + logEntry = logEntry.WithField("user", conf.User) + } + + if len(conf.Password) > 0 { + logEntry = logEntry.WithField("password", strings.Repeat("*", len(conf.Password))) + } + + logEntry.Info("Connecting to redis") + + rdb := redis.NewClient(&redis.Options{ + Addr: conf.Addr, + Username: conf.User, + Password: conf.Password, + DB: conf.DB, + }) + + return rdb, rdb.Ping(context.Background()).Err() +} + +func serverCmd(cmd *cobra.Command, args []string) { + var err error + + // Parse config + conf, err := GetConfig(cmd.Flags()) + if err != nil { + log.WithError(err).Fatal("Failed to read config") + return + } + + // Write PID file + pidFile, err := cmd.Flags().GetString("pid") + if err != nil { + log.WithField("file", pidFile).Info("Writing pid to file") + if err = pid.Save(pidFile); err != nil { + log.WithError(err).Fatal("Failed to write pid") + return + } + } + + skip_currentblock_cache, _ := cmd.Flags().GetBool("no-state-cache") + + flagLevel, _ := cmd.Flags().GetString("level") + lvl, err := log.ParseLevel(flagLevel) + if err == nil { + log.WithField("value", lvl).Info("Setting log level") + log.SetLevel(lvl) + } else { + log.WithError(err).Warn("Failed to parse level") + } + + if len(conf.Log.Filename) > 0 { + stdWriter, err := NewRotatingFileFromConfig(conf.Log, "info") + if err != nil { + log.WithError(err).Fatal("Failed to set standard log file") + return + } + errWriter, err := NewRotatingFileFromConfig(conf.Log, "error") + if err != nil { + log.WithError(err).Fatal("Failed to set error log file") + return + } + + log.WithFields(log.Fields{ + "maxfilesize": conf.Log.MaxFileSize, + "maxage": conf.Log.MaxTime, + "directory": conf.Log.GetDirectory(), + "info_filename": stdWriter.GetFilename(), + "error_filename": errWriter.GetFilename(), + }).Info("Logging to file") + + log.SetOutput(io.Discard) + log.AddHook(MakeStdHook(stdWriter)) + log.AddHook(MakeErrorHook(errWriter)) + } + + // Init telegram notification service + if len(conf.Telegram.Id) > 0 { + + telegram, err := telegram.New(conf.Telegram.Id) + if err != nil { + log.WithError(err).Fatal("Failed to initialize telegram notifier") + return + } + + telegram.AddReceivers(conf.Telegram.Channel) + + // Register services in notification manager + notify.UseServices(telegram) + } + + rdb, err := ConnectRedis(&conf.Redis) + if err != nil { + log.WithError(err).Fatal("Failed to connect to redis") + return + } + + cache.RegisterFactory("redis", cache.NewRedisFactory(rdb)) + + // Setup cache storage + cacheStore, err := cache.Make(conf.Cache.Storage, conf.Cache.Options) + if err != nil { + log.WithError(err).Fatal("Failed to setup cache") + return + } + + // Setup general cache + cache := cache.NewCache("thalos::cache::instance::"+conf.Name, cacheStore) + + antelopeClient := antelopeapi.New(conf.Api) + + shClient := shipclient.NewStream(func(s *shipclient.Stream) { + s.StartBlock = conf.Ship.StartBlockNum + s.EndBlock = conf.Ship.EndBlockNum + s.IrreversibleOnly = conf.Ship.IrreversibleOnly + s.MaxMessagesInFlight = 1 + }) + + // Get codec + codec, err := message.GetCodec(conf.MessageCodec) + if err != nil { + log.WithError(err).Fatal("Failed to initialze codec") + return + } + + chainInfo := chainInfoOnce(antelopeClient) + + chain_id := conf.Ship.Chain + if len(chain_id) < 1 { + chain_id = chainInfo().ChainID + } + + processor := SpawnProccessor( + shClient, + stateLoader(conf, cmd.Flags().Lookup("start-block"), chainInfo, cache, skip_currentblock_cache), + stateSaver(cache), + driver.NewPublisher(context.Background(), rdb, api_redis.Namespace{ + Prefix: conf.Redis.Prefix, + ChainID: chain_id, + }), + initAbiManager(&conf.AbiCache, antelopeClient, cacheStore, chain_id), + codec, + ) + + processor.SetBlacklist(conf.Ship.Blacklist) + processor.FetchDeltas(conf.Ship.EnableTableDeltas) + + // Run the application + run(conf, shClient, processor) + + // Close the processor properly + processor.Close() +} diff --git a/cmd/tools/bench.go b/cmd/tools/bench.go index 4657f85..067ee90 100644 --- a/cmd/tools/bench.go +++ b/cmd/tools/bench.go @@ -7,7 +7,7 @@ import ( "os/signal" "time" - "github.com/urfave/cli/v2" + "github.com/spf13/cobra" "github.com/eosswedenorg/thalos/api" "github.com/eosswedenorg/thalos/api/message" @@ -18,74 +18,82 @@ import ( log "github.com/sirupsen/logrus" ) -var benchCmd = &cli.Command{ - Name: "bench", - Usage: "Run a benchmark against a thalos node", - Flags: []cli.Flag{ - redisUrlFlag, - redisDbFlag, - redisPrefixFlag, - chainIdFlag, - &cli.DurationFlag{ - Name: "interval", - Aliases: []string{"i"}, - Value: time.Minute, - Usage: "How often the benchmark results should be displayed.", - }, - }, - Action: func(ctx *cli.Context) error { - var counter int = 0 - interval := ctx.Duration("interval") +func CreateBenchCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "bench", + Short: "Run a benchmark against a thalos node", + Run: func(cmd *cobra.Command, args []string) { + counter := 0 + interval, _ := cmd.Flags().GetDuration("interval") - log.WithFields(log.Fields{ - "url": ctx.String("redis-url"), - "prefix": ctx.String("redis-prefix"), - "chain_id": ctx.String("chain_id"), - "database": ctx.Int("redis-db"), - }).Info("Connecting to redis") + url, _ := cmd.Flags().GetString("redis-url") + user, _ := cmd.Flags().GetString("redis-user") + pw, _ := cmd.Flags().GetString("redis-pw") + prefix, _ := cmd.Flags().GetString("prefix") + chain_id, _ := cmd.Flags().GetString("chain_id") + db, _ := cmd.Flags().GetInt("redis-db") - // Create redis client - rdb := redis.NewClient(&redis.Options{ - Addr: ctx.String("redis-url"), - DB: ctx.Int("redis-db"), - }) + log.WithFields(log.Fields{ + "url": url, + "prefix": prefix, + "chain_id": chain_id, + "database": db, + }).Info("Connecting to redis") - if err := rdb.Ping(context.Background()).Err(); err != nil { - return err - } + // Create redis client + rdb := redis.NewClient(&redis.Options{ + Addr: url, + Username: user, + Password: pw, + DB: db, + }) - log.Println("Connected to redis") + if err := rdb.Ping(context.Background()).Err(); err != nil { + log.WithError(err).Fatal("Failed to connect to redis") + return + } - log.WithFields(log.Fields{ - "interval": interval, - }).Info("Starting benchmark") + log.Println("Connected to redis") - sub := api_redis.NewSubscriber(context.Background(), rdb, api_redis.Namespace{ - Prefix: ctx.String("redis-prefix"), - ChainID: ctx.String("chain_id"), - }) + log.WithFields(log.Fields{ + "interval": interval, + }).Info("Starting benchmark") - codec, err := message.GetCodec("json") - if err != nil { - return err - } + sub := api_redis.NewSubscriber(context.Background(), rdb, api_redis.Namespace{ + Prefix: prefix, + ChainID: chain_id, + }) - client := api.NewClient(sub, codec.Decoder) + codec, err := message.GetCodec("json") + if err != nil { + log.WithError(err).Fatal("Failed to get codec") + return + } - client.OnAction = func(act message.ActionTrace) { - counter++ - } + client := api.NewClient(sub, codec.Decoder) - // Subscribe to all actions - if err = client.Subscribe(api.ActionChannel{}.Channel()); err != nil { - return err - } + // Subscribe to all actions + if err = client.Subscribe(api.ActionChannel{}.Channel()); err != nil { + log.WithError(err).Fatal("Failed to subscribe to channels") + return + } + + go func() { + for t := range client.Channel() { + switch err := t.(type) { + case message.ActionTrace: + counter++ + case error: + log.WithError(err).Error("Error when reading stream") + } + } + }() - go func() { t := time.Now() sig := make(chan os.Signal, 1) signal.Notify(sig, os.Interrupt) + // Read stuff. for { select { case <-sig: @@ -107,11 +115,11 @@ var benchCmd = &cli.Command{ counter = 0 } } - }() + }, + } - // Read stuff. - client.Run() + cmd.Flags().AddFlagSet(RedisFlags()) + cmd.Flags().DurationP("interval", "i", time.Minute, "How often the benchmark results should be displayed.") - return nil - }, + return cmd } diff --git a/cmd/tools/flags.go b/cmd/tools/flags.go index ede7ede..fd54332 100644 --- a/cmd/tools/flags.go +++ b/cmd/tools/flags.go @@ -1,27 +1,14 @@ package main -import ( - "github.com/urfave/cli/v2" -) +import "github.com/spf13/pflag" -var redisPrefixFlag = &cli.StringFlag{ - Name: "prefix", - Value: "ship", -} - -var redisUrlFlag = &cli.StringFlag{ - Name: "redis-url", - Value: "127.0.0.1:6379", - Usage: "host:port to the redis server", -} - -var redisDbFlag = &cli.IntFlag{ - Name: "redis-db", - Value: 0, - Usage: "What redis database we should connect to.", -} - -var chainIdFlag = &cli.StringFlag{ - Name: "chain_id", - Value: "1064487b3cd1a897ce03ae5b6a865651747e2e152090f99c1d19d44e01aea5a4", +func RedisFlags() *pflag.FlagSet { + set := pflag.FlagSet{} + set.String("redis-url", "127.0.0.1:6379", "host:port to the redis server") + set.String("redis-user", "", "User to use when authenticating to the server") + set.String("redis-pw", "", "Password to use when authenticating to the server") + set.Int("redis-db", 0, "What redis database we should connect to.") + set.String("prefix", "ship", "redis prefix") + set.String("chain_id", "1064487b3cd1a897ce03ae5b6a865651747e2e152090f99c1d19d44e01aea5a4", "chain id") + return &set } diff --git a/cmd/tools/main.go b/cmd/tools/main.go index 0dab953..35b8684 100644 --- a/cmd/tools/main.go +++ b/cmd/tools/main.go @@ -1,27 +1,31 @@ package main import ( - "os" - - "github.com/urfave/cli/v2" - - _ "github.com/eosswedenorg/thalos/app/log" + _ "github.com/eosswedenorg/thalos/internal/log" log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" ) var VersionString string = "dev" func main() { - app := &cli.App{ - Usage: "Collection of tools for dealing with the thalos application", - Version: VersionString, - Commands: []*cli.Command{ - validateCmd, - benchCmd, + rootCmd := &cobra.Command{ + Use: "thalos-tools", + Short: "Collection of tools for dealing with the Thalos application", + FParseErrWhitelist: cobra.FParseErrWhitelist{ + UnknownFlags: true, }, + Version: VersionString, } - if err := app.Run(os.Args); err != nil { - log.WithError(err).Fatal("Application error") + rootCmd.AddCommand( + CreateValidateCmd(), + CreateBenchCmd(), + CreateRedisACLCmd(), + CreateMockPublisherCmd(), + ) + + if err := rootCmd.Execute(); err != nil { + log.Fatal(err) } } diff --git a/cmd/tools/mock_publisher.go b/cmd/tools/mock_publisher.go new file mode 100644 index 0000000..6a9e028 --- /dev/null +++ b/cmd/tools/mock_publisher.go @@ -0,0 +1,121 @@ +package main + +import ( + "context" + "time" + + "github.com/spf13/cobra" + + "github.com/eosswedenorg/thalos/api" + "github.com/eosswedenorg/thalos/api/message" + _ "github.com/eosswedenorg/thalos/api/message/json" + api_redis "github.com/eosswedenorg/thalos/api/redis" + redis_driver "github.com/eosswedenorg/thalos/internal/driver/redis" + + "github.com/redis/go-redis/v9" + log "github.com/sirupsen/logrus" +) + +func CreateMockPublisherCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "mock_publisher", + Short: "Run a publisher that mocks messages to a redis server. tries to send as many messages as possible", + Run: func(cmd *cobra.Command, args []string) { + url, _ := cmd.Flags().GetString("redis-url") + user, _ := cmd.Flags().GetString("redis-user") + pw, _ := cmd.Flags().GetString("redis-pw") + prefix, _ := cmd.Flags().GetString("prefix") + chain_id, _ := cmd.Flags().GetString("chain_id") + db, _ := cmd.Flags().GetInt("redis-db") + + // Create redis client + rdb := redis.NewClient(&redis.Options{ + Addr: url, + Username: user, + Password: pw, + DB: db, + }) + + codecArg, _ := cmd.Flags().GetString("codec") + + codec, err := message.GetCodec(codecArg) + if err != nil { + log.WithError(err).Fatal("Failed to get codec") + return + } + + log.WithFields(log.Fields{ + "url": url, + "prefix": prefix, + "chain_id": chain_id, + "database": db, + }).Info("Starting mock publisher") + + ns := api_redis.Namespace{ + Prefix: prefix, + ChainID: chain_id, + } + publisher := redis_driver.NewPublisher(context.Background(), rdb, ns) + + msg := message.ActionTrace{ + TxID: "401e8a7e5deb18a2a69fc6559f49509a155f4355c85efb69c1c1fab5b60ee532", + BlockNum: 18237917, + Timestamp: time.Date(2014, 3, 22, 11, 36, 43, 0, time.UTC), + Receipt: &message.ActionReceipt{ + Receiver: "acc1", + ActDigest: "4c5c08be612e937564fc526ebb5fadf34ae8c2a571fe9d7cdb3ffcdfc53b0e8d", + GlobalSequence: 12314, + RecvSequence: 237187239, + AuthSequence: []message.AccountAuthSequence{ + { + Account: "acc1", + Sequence: 2732863, + }, + { + Account: "acc2", + Sequence: 263762, + }, + }, + CodeSequence: 2327832, + ABISequence: 12376189, + }, + Name: "fake", + Contract: "fake", + Receiver: "acc1", + Data: map[string]interface{}{ + "one": 238771832, + "two": "str", + }, + Authorization: []message.PermissionLevel{ + { + Actor: "acc1", + Permission: "active", + }, + { + Actor: "acc2", + Permission: "owner", + }, + }, + Except: "err", + Error: 2, + Return: []byte{0xbe, 0xef}, + } + + payload, err := codec.Encoder(msg) + if err != nil { + log.WithError(err).Fatal("Failed to encode message") + return + } + channel := api.ActionChannel{}.Channel() + + for { + _ = publisher.Write(channel, payload) + publisher.Flush() + } + }, + } + + cmd.Flags().AddFlagSet(RedisFlags()) + cmd.Flags().String("codec", "json", "codec to use") + return cmd +} diff --git a/cmd/tools/redis-acl.go b/cmd/tools/redis-acl.go new file mode 100644 index 0000000..ad15036 --- /dev/null +++ b/cmd/tools/redis-acl.go @@ -0,0 +1,171 @@ +package main + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "math/rand" + "os" + "text/template" + "time" + + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" +) + +var rnd *rand.Rand + +// Helper struct representing a redis user. +type User struct { + // Username + Name string + + // Password + Password string + + // True if password was generated, false if not. + Generated bool + + // True if password should be hashed, false otherwise. + Hash bool +} + +func NewUser(name, password string, pass_len uint) User { + if len(password) < 1 { + return User{ + Name: name, + Password: randomString(pass_len), + Generated: true, + } + } + return User{Name: name, Password: password} +} + +func (u *User) GetPassword() string { + if u.Hash { + return "#" + hash(u.Password) + } + + return ">" + u.Password +} + +func (u User) Print() { + fmt.Println(u.Name+":", u.Password) +} + +func (u User) PrintIfGeneratedPW() { + if u.Generated { + u.Print() + } +} + +func randomString(length uint) string { + charset := "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUWXYZ0123456789" + out := "" + for i := 0; i < int(length); i++ { + idx := rnd.Intn(len(charset)) + out += string(charset[idx]) + } + return out +} + +func hash(str string) string { + data := sha256.Sum256([]byte(str)) + return hex.EncodeToString(data[:]) +} + +func writeTemplate(w io.Writer, defUser, serverUser, clientUser User, prefix string) error { + tmplStr := `# Created by thalos-tools on {{.timestamp}} +user default on {{.defaultpw}} ~* &* +@all +user {{.server}} on {{.serverpw}} resetchannels ~{{.prefix}}::* &{{.prefix}}::* -@all +ping +get +publish +set +user {{.client}} on {{.clientpw}} resetchannels &{{.prefix}}::* -@all +subscribe +` + + tmpl, err := template.New("acl").Parse(tmplStr) + if err != nil { + return err + } + + return tmpl.Execute(w, map[string]string{ + "defaultpw": defUser.GetPassword(), + "client": clientUser.Name, + "clientpw": clientUser.GetPassword(), + "server": serverUser.Name, + "serverpw": serverUser.GetPassword(), + "prefix": prefix, + "timestamp": time.Now().Format(time.UnixDate), + }) +} + +func CreateRedisACLCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "redis-acl", + Short: "create a users.acl file", + Run: func(cmd *cobra.Command, args []string) { + var err error + out := os.Stdout + + rnd = rand.New(rand.NewSource(time.Now().UnixNano())) + + flagDefUserPw, _ := cmd.Flags().GetString("default-pw") + flagServer, _ := cmd.Flags().GetString("server") + flagServerPw, _ := cmd.Flags().GetString("server-pw") + flagClient, _ := cmd.Flags().GetString("client") + flagClientPw, _ := cmd.Flags().GetString("client-pw") + flagPrefix, _ := cmd.Flags().GetString("prefix") + flagPassLen, _ := cmd.Flags().GetUint("pass-len") + + defaultUser := NewUser("default", flagDefUserPw, flagPassLen) + serverUser := NewUser(flagServer, flagServerPw, flagPassLen) + clientUser := NewUser(flagClient, flagClientPw, flagPassLen) + + atleastOneGeneratedPw := defaultUser.Generated || serverUser.Generated || clientUser.Generated + + cleartext, _ := cmd.Flags().GetBool("cleartext") + if !cleartext { + if atleastOneGeneratedPw { + println("Passwords") + } + + defaultUser.PrintIfGeneratedPW() + serverUser.PrintIfGeneratedPW() + clientUser.PrintIfGeneratedPW() + + defaultUser.Hash = true + serverUser.Hash = true + clientUser.Hash = true + } + + filename, _ := cmd.Flags().GetString("file") + if len(filename) > 0 { + out, err = os.Create(filename) + if err != nil { + log.WithError(err).Fatal("Failed to create output file") + return + } + defer out.Close() + } else if !cleartext && atleastOneGeneratedPw { + fmt.Println() + } + + err = writeTemplate(out, defaultUser, serverUser, clientUser, flagPrefix) + if err != nil { + log.WithError(err).Fatal("Failed to writte config") + return + } + }, + } + + cmd.Flags().String("default-pw", "", "Password to use for the default account, if not provided a random one will be generated") + cmd.Flags().String("client", "thalos-client", "Thalos client account name") + cmd.Flags().String("client-pw", "", "Password to use for the thalos client account, if not provided a random one will be generated") + cmd.Flags().String("server", "thalos", "Thalos account name") + cmd.Flags().String("server-pw", "", "Password to use for the thalos server account, if not provided a random one will be generated") + cmd.Flags().String("prefix", "ship", "Redis key prefix") + cmd.Flags().Bool("cleartext", false, "If passwords should be hashed or left in cleartext.") + cmd.Flags().String("file", "", "Where the config should be written to (default: standard out)") + cmd.Flags().Uint("pass-len", 32, "The length of generated passwords") + + return cmd +} diff --git a/cmd/tools/validate.go b/cmd/tools/validate.go index 1f2aa5b..310c8c1 100644 --- a/cmd/tools/validate.go +++ b/cmd/tools/validate.go @@ -7,7 +7,7 @@ import ( "os/signal" "time" - "github.com/urfave/cli/v2" + "github.com/spf13/cobra" "github.com/eosswedenorg/thalos/api" "github.com/eosswedenorg/thalos/api/message" @@ -18,90 +18,85 @@ import ( log "github.com/sirupsen/logrus" ) -type Tester struct { - block_num uint32 - timeout time.Duration - timer *time.Ticker -} +func CreateValidateCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "validate", + Short: "Validate a thalos server by following action traces and makes sure that blocks arrive in order.", + Run: func(cmd *cobra.Command, args []string) { + status_duration := time.Second * 10 -func NewTester(timeout time.Duration) *Tester { - return &Tester{ - block_num: 0, - timeout: timeout, - timer: time.NewTicker(timeout), - } -} + url, _ := cmd.Flags().GetString("redis-url") + prefix, _ := cmd.Flags().GetString("prefix") + chain_id, _ := cmd.Flags().GetString("chain_id") + db, _ := cmd.Flags().GetInt("redis-db") -func (t *Tester) OnAction(act message.ActionTrace) { - if t.block_num > 0 { - var diff int32 = int32(act.BlockNum - t.block_num) - if diff < 0 || diff > 1 { log.WithFields(log.Fields{ - "current_block": t.block_num, - "block": act.BlockNum, - "diff": diff, - }).Warn("Invalid") - } - } + "url": url, + "prefix": prefix, + "chain_id": chain_id, + "database": db, + }).Info("Connecting to redis") - t.block_num = act.BlockNum + // Create redis client + rdb := redis.NewClient(&redis.Options{ + Addr: url, + DB: db, + }) - t.timer.Reset(t.timeout) -} + if err := rdb.Ping(context.Background()).Err(); err != nil { + log.WithError(err).Fatal("Failed to connect to redis") + return + } -var validateCmd = &cli.Command{ - Name: "validate", - Usage: "Validate a thalos server by following action traces and makes sure that blocks arrive in order.", - Flags: []cli.Flag{ - redisUrlFlag, - redisDbFlag, - redisPrefixFlag, - chainIdFlag, - }, - Action: func(ctx *cli.Context) error { - tester := NewTester(time.Second * 5) - status_duration := time.Second * 10 + log.Println("Connected to redis") - log.WithFields(log.Fields{ - "url": ctx.String("redis-url"), - "prefix": ctx.String("redis-prefix"), - "chain_id": ctx.String("chain_id"), - "database": ctx.Int("redis-db"), - }).Info("Connecting to redis") + log.Info("Starting validation, following the stream") - // Create redis client - rdb := redis.NewClient(&redis.Options{ - Addr: ctx.String("redis-url"), - DB: ctx.Int("redis-db"), - }) + sub := api_redis.NewSubscriber(context.Background(), rdb, api_redis.Namespace{ + Prefix: prefix, + ChainID: chain_id, + }) - if err := rdb.Ping(context.Background()).Err(); err != nil { - return err - } + codec, err := message.GetCodec("json") + if err != nil { + log.WithError(err).Fatal("Failed to get codec") + return + } - log.Println("Connected to redis") + client := api.NewClient(sub, codec.Decoder) - log.Info("Starting validation, following the stream") + // Subscribe to all actions + if err = client.Subscribe(api.ActionChannel{}.Channel()); err != nil { + log.WithError(err).Fatal("Failed to subscribe to channels") + return + } - sub := api_redis.NewSubscriber(context.Background(), rdb, api_redis.Namespace{ - Prefix: ctx.String("redis-prefix"), - ChainID: ctx.String("chain_id"), - }) + block_num := uint32(0) + timeout := time.Second * 5 + timer := time.NewTicker(timeout) - codec, err := message.GetCodec("json") - if err != nil { - return err - } + go func() { + for t := range client.Channel() { + switch msg := t.(type) { + case error: + log.WithError(msg).Error("Error when reading stream") + case message.ActionTrace: + if block_num > 0 { + diff := int32(msg.BlockNum - block_num) + if diff < 0 || diff > 1 { + log.WithFields(log.Fields{ + "current_block": block_num, + "block": msg.BlockNum, + "diff": diff, + }).Warn("Invalid") + } + } + block_num = msg.BlockNum + timer.Reset(timeout) + } + } + }() - client := api.NewClient(sub, codec.Decoder) - client.OnAction = tester.OnAction - - // Subscribe to all actions - if err = client.Subscribe(api.ActionChannel{}.Channel()); err != nil { - return err - } - - go func() { sig := make(chan os.Signal, 1) signal.Notify(sig, os.Interrupt) @@ -111,20 +106,19 @@ var validateCmd = &cli.Command{ fmt.Println("Got interrupt") client.Close() return - case <-tester.timer.C: - log.WithField("duration", tester.timeout). + case <-timer.C: + log.WithField("duration", timeout). Warn("Did not get any messages during the defined duration") case <-time.After(status_duration): log.WithFields(log.Fields{ - "current_block": tester.block_num, + "current_block": block_num, }).Info("Status") } } - }() + }, + } - // Read stuff. - client.Run() + cmd.Flags().AddFlagSet(RedisFlags()) - return nil - }, + return cmd } diff --git a/config.example.yml b/config.example.yml index 0935d91..2dc438e 100644 --- a/config.example.yml +++ b/config.example.yml @@ -38,15 +38,41 @@ ship: # Request ship to start sending blocks from this block. # If not set, the head block reported by the nodeos api is used. - #start_block_num: 1000 + # start_block_num: 1000 # Request ship to stop sending blocks when reaching this block. - #end_block_num: 2000 + # end_block_num: 2000 + + # If true, Thalos will process table deltas + # table_deltas: false + + # Blacklist contract/actions + blacklist: + # this is a "useless" action that results in alot of warning messages. + # becase thalos does not know it's ABI. Its recommended to have this action blacklisted + # unless you have a reason to use it. + eosio.null: + - nonce + + # blacklist all action from a contract + # evilcontract: ["*"] + + # blacklist_is_whitelist: true + +# Configure the cache. +# Default is to use redis. But if you need to +# you can set additional values or even change the driver +# to something else that Thalos supports. +# See the documentation for details + +# cache: +# storage: redis +# options: [] # Telegram notifications -#telegram: -# id: "123456789:GPdmGPBWvpgHPxlergJLavus-PoAURTjMWP" -# channel: -123456789 +# telegram: +# id: "123456789:GPdmGPBWvpgHPxlergJLavus-PoAURTjMWP" +# channel: -123456789 # Redis settings redis: @@ -57,7 +83,7 @@ redis: user: "" # Password to use when authenticating - pasword: "" + password: "" # database index db: 0 diff --git a/debian/changelog b/debian/changelog index 646a2b6..ecc01aa 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,247 @@ +thalos (1.1.9) bionic focal jammy; urgency=medium + + * [Security CVE-2024-45338] Update golang.org/x/net to 0.33.0 + + -- Henrik Hautakoski Thu, 23 Jan 2025 19:30:31 +0100 + +thalos (1.1.8) bionic focal jammy; urgency=medium + + * Support for wildcard contracts in Blacklist + * [Security CVE-2024-45337] Update golang.org/x/crypto to 0.31.0 + * [Security CVE-2024-53259] Update github.com/quic-go/quic-go to 0.48.2 + + -- Henrik Hautakoski Mon, 23 Dec 2024 09:25:44 +0100 + +thalos (1.1.8~rc1) bionic focal jammy; urgency=medium + + * Support for wildcard contracts in Blacklist + + -- Henrik Hautakoski Wed, 04 Dec 2024 15:19:53 +0100 + +thalos (1.1.7) bionic focal jammy; urgency=medium + + * ship: set MaxMessagesInFlight to 1. This forces the client/server to ack + every message and might be a workaround fix for issue #25 + according to this comment: + https://github.com/AntelopeIO/leap/issues/1358#issuecomment-2276294557 + * golang: update eosswedenrg-go/antelope-ship-client to v0.3.2 + * Add support to disable processing of table deltas. + + -- Henrik Hautakoski Mon, 11 Nov 2024 19:38:15 +0100 + +thalos (1.1.7~rc2) bionic focal jammy; urgency=medium + + * ship: set MaxMessagesInFlight to 1. This forces the client/server to ack + every message and might be a workaround fix for issue #25 + according to this comment: + https://github.com/AntelopeIO/leap/issues/1358#issuecomment-2276294557 + * golang: update eosswedenrg-go/antelope-ship-client to v0.3.2 + + -- Henrik Hautakoski Sun, 03 Nov 2024 12:04:29 +0100 + +thalos (1.1.7~rc1) bionic focal jammy; urgency=medium + + * Add support to disable processing of table deltas. + + -- Henrik Hautakoski Mon, 21 Oct 2024 12:31:21 +0200 + +thalos (1.1.6) bionic focal jammy; urgency=medium + + [ Henrik Hautakoski ] + * makefile: make sure we apppend to GOBULDFLAGS if user wants to add their own. + * minor style fixes. + * api/channel_test.go: rearange fields. + * README.md: Update minimum go version + * README.md: Link to docker page + * .github/workflows/release.yml: need to update version regex for musl builds + + [ Avm07 ] + * Fix typo in config.example.yml + + -- Henrik Hautakoski Wed, 16 Oct 2024 16:23:47 +0200 + +thalos (1.1.5) bionic focal jammy; urgency=medium + + * New config section: `cache` + * New CLI flag: `cache` specify what cache driver to use + * New CLI flag: `abi-cache-api-timeout` configure the timeout for the HTTP + request made when Thalos wants to fetch a ABI from the api. + * API Table Deltas: abi decode the data in `value` field for contract_row deltas. + * golang: update github.com/shufflingpixels/antelope-go to v0.1.5 + * golang: update github.com/quic-go/quic-go from 0.41.0 to 0.42.0 + * golang: version 1.20 can no longer be used to build the project. + + -- Henrik Hautakoski Thu, 29 Aug 2024 15:33:17 +0200 + +thalos (1.1.5~rc1) bionic focal jammy; urgency=medium + + * New config section: `cache` + * New CLI flag: `cache` specify what cache driver to use + * New CLI flag: `abi-cache-api-timeout` configure the timeout for the HTTP + request made when Thalos wants to fetch a ABI from the api. + * API Table Deltas: abi decode the data in `value` field for contract_row deltas. + * golang: update github.com/shufflingpixels/antelope-go to v0.1.4 + + -- Henrik Hautakoski Sun, 11 Aug 2024 17:04:55 +0200 + +thalos (1.1.4) bionic focal jammy; urgency=medium + + * Implement whitelist option for ship contract/action blacklist + * Fix bug with integer overflow on 32 bit CPUs. + + -- Henrik Hautakoski Tue, 16 Jul 2024 21:03:34 +0200 + +thalos (1.1.3) bionic focal jammy; urgency=medium + + * Updated antelope-go library to v0.1.2 that fixes a bug in abi binary + decoder, it expects some fields to be strings while they are "names" + (strings encoded into a int64) + * Fix a bug with "set_abi" struct had the wrong order of fields in ShipProcessor.updateAbiFromAction() + * Fix a bug in ShipProcessor.updateAbiFromAction() that assumed the abi + was in hex format when in fact it is binary. + + -- Henrik Hautakoski Wed, 03 Jul 2024 18:05:33 +0200 + +thalos (1.1.2) bionic focal jammy; urgency=medium + + * API: Fix a bug regarding json timestamp being encoded/decoded with wrong + format + * Implement action blacklist, it is not possible to configure a blacklist + that will be used to filter out processing of unwanted contracts/actions. + * Fix a bug in isVariant() where v.Elem() was called on non interface/pointer + * Minor cleanups in tools + * Fix a bug where TableDeltaRow.Data was not set + * Fix a bug where blockResult.Deltas was not properly nil checked. Resulting in panic if accessed + * Moved from github.com/eoscanda/eos-go to github.com/pnx/antelope-go library + + -- Henrik Hautakoski Thu, 27 Jun 2024 14:27:38 +0200 + +thalos (1.1.2~rc4) bionic focal jammy; urgency=medium + + * API: Fix a bug regarding json timestamp being encoded/decoded with wrong + format + * Implement action blacklist, it is now possible to configure a blacklist + that will be used to filter out processing of unwanted contracts/actions. + + -- Henrik Hautakoski Sun, 23 Jun 2024 14:55:03 +0200 + +thalos (1.1.2~rc3) bionic focal jammy; urgency=medium + + * Fix a bug in isVariant() where v.Elem() was called on non interface/pointer + * Minor cleanups in tools + + -- Henrik Hautakoski Wed, 19 Jun 2024 21:50:15 +0200 + +thalos (1.1.2~rc2) bionic focal jammy; urgency=medium + + * fix a bug where TableDeltaRow.Data was not set + * fix a bug where blockResult.Deltas was not properly nil checked. Resulting in panic if accessed + + -- Henrik Hautakoski Fri, 17 May 2024 18:15:29 +0200 + +thalos (1.1.2~rc1) bionic focal jammy; urgency=medium + + * Moved from github.com/eoscanda/eos-go to github.com/pnx/antelope-go library + + -- Henrik Hautakoski Mon, 29 Apr 2024 21:14:34 +0200 + +thalos (1.1.1) bionic focal jammy; urgency=medium + + * Build binaries linked with musl libc for alpine linux. + * Added docker image. + * redis-acl tool: added `--pass-len` flag. + * redis-acl tool: fix correct syntax for cleartext passwords. + * redis-acl tool: allow ping command for server user. + + -- Henrik Hautakoski Tue, 09 Apr 2024 22:40:20 +0200 + +thalos (1.1.0) bionic focal jammy; urgency=medium + + * Adding `ActionTrace.FirstReceiver` flag, that is `true` + only if receiver is the same as contract name. + + -- Henrik Hautakoski Fri, 01 Mar 2024 16:41:25 +0100 + +thalos (1.1.0~rc2) bionic focal jammy; urgency=medium + + * Adding `log.file_timestamp_format` config field + * Added cli flag `--log-file-timestamp` + * Directory where log files are stored is created with correct permissions. + + -- Henrik Hautakoski Wed, 28 Feb 2024 23:16:44 +0100 + +thalos (1.1.0~rc1) bionic focal jammy; urgency=medium + + * Adding flags for almost all config values. + * Improved disconnect code for ship client. + - Application now waits for ship to reply with a close message before exiting + the application. + - The application now recognizes an close error and no longer reports is as + an actual error to the log. + * Application only calls "GetInfo" from the antelope API once and only if it actually needs the information. + * CI: update actions/checkout to version 4 + * CI: update actions/setup-go to version 5 + * golang: update github.com/eosswedenorg-go/antelope-ship-client to v0.2.7 + * golang: switched github.com/pborman/getopt for github.com/spf13/cobra + * golang: use github.com/spf13/viper to handle configuration. + + -- Henrik Hautakoski Mon, 19 Feb 2024 14:27:40 +0100 + +thalos (1.0.0) bionic focal jammy; urgency=medium + + * Improved code documentation. + * Updated dependancies. + * Small fixes for thalos tools. + + Go API: + + * Rewrite client api to provide a go channel instead of callback functions + * Support Transaction messages + * Support Rollback messages + * Support TableDelta messages + + -- Henrik Hautakoski Wed, 07 Feb 2024 19:32:10 +0100 + +thalos (0.3.1) bionic focal jammy; urgency=medium + + * Bug: fix incorrect log path in debian packages. + * Improved install.sh script. + + -- Henrik Hautakoski Mon, 22 Jan 2024 14:46:48 +0100 + +thalos (0.3.0) bionic focal jammy; urgency=medium + + * Feature: Table Delta message (issue https://github.com/eosswedenorg/thalos/issues/31) + * Feature: Rollback message (issue https://github.com/eosswedenorg/thalos/issues/30) + * Bug: Fix a bug where GlobalSequence was passed as ActionTrace.Receipt.RecvSequence + * Bug: fixed random password generator in redis-acl tool to not produce + the same password if executed during the same second. + + * Security: update golang.org/x/crypto from 0.14.0 to 0.17.0 + + -- Henrik Hautakoski Sun, 21 Jan 2024 14:29:23 +0100 + +thalos (0.2.2) bionic focal jammy; urgency=medium + + * Current processing state (block number) is now cached and loaded on start. + * cli: new `-n` flag that forces the application to take start block from config/api instead of cache. + * bug: the ABI Cache now has a timeout for the api call on cache miss fixing + a bug where the application would hang if api did not respond. + * Update to use antelope-ship-client v0.2.7 + + -- Henrik Hautakoski Sun, 17 Dec 2023 19:37:30 +0100 + +thalos (0.2.1) bionic focal jammy; urgency=medium + + * new tool: publisher mock (writes test data to redis channels.) + * golang: Drop support for 1.18, 1.19 + * new cli flag: "--level" to specify log level. + * new tool: redis ACL config generator. + * tools: adding flags for redis username/password + * code documentation + + -- Henrik Hautakoski Sat, 04 Nov 2023 13:35:06 +0100 + thalos (0.2.0) bionic focal jammy; urgency=medium * Update to use antelope-ship-client v0.2.5 @@ -35,4 +279,5 @@ thalos-server (0.1.0) bionic focal jammy; urgency=medium Initial release. - -- Henrik Hautakoski Sun, 14 May 2023 18:17:35 +0200 \ No newline at end of file + -- Henrik Hautakoski Sun, 14 May 2023 18:17:35 +0200 + diff --git a/debian/control b/debian/control index 7fa3acd..0db0f29 100644 --- a/debian/control +++ b/debian/control @@ -6,7 +6,7 @@ Standards-Version: 4.5.0 Vcs-Git: https://github.com/eosswedenorg/thalos.git Vcs-Browser: https://github.com/eosswedenorg/thalos Priority: optional -Maintainer: Henrik Hautakoski +Maintainer: Henrik Hautakoski Package: thalos Section: utils diff --git a/debian/patches/0001-fix-config-logpath.patch b/debian/patches/0001-fix-config-logpath.patch index 801fd2b..eac48fa 100644 --- a/debian/patches/0001-fix-config-logpath.patch +++ b/debian/patches/0001-fix-config-logpath.patch @@ -1,12 +1,11 @@ diff --git a/config.example.yml b/config.example.yml -index 9f4272a..8daac0a 100644 --- a/config.example.yml +++ b/config.example.yml @@ -11,7 +11,7 @@ log: # Filename to use. - filename: thalos.log + filename: thalos # Directory to store the logfiles in. - directory: logs -+ directory: /var/log ++ directory: /var/log/thalos # Format to rename log files when rotating - time_format: 2006-01-02_150405 \ No newline at end of file + time_format: 2006-01-02_150405 diff --git a/debian/sysconfig/thalos-server b/debian/sysconfig/thalos-server new file mode 100644 index 0000000..6afed36 --- /dev/null +++ b/debian/sysconfig/thalos-server @@ -0,0 +1,6 @@ +# Environment variables for thalos-server +# +# This is file sourced by the systemd unit file + +# Append additional cli flags if needed. see: thalos-server -h +THALOS_SERVER_ARGS="-c /etc/thalos/config.yml" diff --git a/debian/thalos-server.service b/debian/thalos-server.service index 3265c1e..36bbb51 100644 --- a/debian/thalos-server.service +++ b/debian/thalos-server.service @@ -6,9 +6,10 @@ After=network.target User=thalos Group=thalos Type=simple -ExecStart=/usr/bin/thalos-server -c /etc/thalos/config.yml +EnvironmentFile=-/etc/sysconfig/thalos-server +ExecStart=/usr/bin/thalos-server $THALOS_SERVER_ARGS ExecReload=kill -HUP $MAINPID Restart=on-failure [Install] -WantedBy=multi-user.target \ No newline at end of file +WantedBy=multi-user.target diff --git a/debian/thalos.install b/debian/thalos.install index 510fa2e..6de9f98 100644 --- a/debian/thalos.install +++ b/debian/thalos.install @@ -1 +1,2 @@ -debian/thalos-server.service /lib/systemd/system \ No newline at end of file +debian/thalos-server.service /lib/systemd/system +debian/sysconfig/thalos-server /etc/sysconfig diff --git a/debian/thalos.postinst b/debian/thalos.postinst index bad0440..a643c01 100644 --- a/debian/thalos.postinst +++ b/debian/thalos.postinst @@ -2,14 +2,14 @@ set -e if [ "$1" = 'configure' ]; then - adduser --force-badname --system --home /nonexistent \ - --group --no-create-home --quiet thalos || true + adduser --force-badname --system --home /nonexistent \ + --group --no-create-home --quiet thalos || true - # Create log directory - mkdir -p /var/log/thalos - chown thalos:adm /var/log/thalos + # Create log directory + mkdir -p /var/log/thalos + chown thalos:adm /var/log/thalos fi #DEBHELPER# -exit 0 \ No newline at end of file +exit 0 diff --git a/debian/thalos.postrm b/debian/thalos.postrm index 08a6475..dde4859 100644 --- a/debian/thalos.postrm +++ b/debian/thalos.postrm @@ -3,11 +3,9 @@ set -e #DEBHELPER# - -if [ "${1}" = "purge" ] -then - deluser --quiet thalos > /dev/null || true - rm -rf /var/log/thalos +if [ "${1}" = "purge" ]; then + deluser --quiet thalos >/dev/null || true + rm -rf /var/log/thalos fi -exit 0 \ No newline at end of file +exit 0 diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..b8fb57f --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,6 @@ +FROM alpine:latest +LABEL maintainer="Henrik Hautakoski " +ARG VERSION=1.1.9 +WORKDIR /thalos +ADD --chmod=755 https://github.com/eosswedenorg/thalos/releases/download/v$VERSION/thalos-server-${VERSION}-linux-amd64-musl thalos-server +ENTRYPOINT [ "./thalos-server" ] diff --git a/go.mod b/go.mod index 2def7f1..b7de7b0 100644 --- a/go.mod +++ b/go.mod @@ -1,65 +1,82 @@ module github.com/eosswedenorg/thalos -go 1.18 +go 1.22.0 require ( github.com/cenkalti/backoff/v4 v4.2.1 github.com/docker/go-units v0.5.0 - github.com/eoscanada/eos-go v0.10.3-0.20230413154640-bb75101dc2f6 - github.com/eosswedenorg-go/antelope-ship-client v0.2.6 + github.com/eosswedenorg-go/antelope-ship-client v0.3.2 github.com/eosswedenorg-go/pid v1.0.1 - github.com/eosswedenorg/thalos/api v0.0.0-00010101000000-000000000000 + github.com/eosswedenorg/thalos/api v1.0.0 github.com/go-redis/cache/v9 v9.0.0 - github.com/go-redis/redismock/v9 v9.0.3 + github.com/go-redis/redismock/v9 v9.2.0 + github.com/karlseguin/typed v1.1.8 + github.com/mitchellh/mapstructure v1.5.0 github.com/nikoksr/notify v0.41.0 - github.com/pborman/getopt/v2 v2.1.0 - github.com/redis/go-redis/v9 v9.0.5 + github.com/redis/go-redis/v9 v9.5.1 + github.com/shufflingpixels/antelope-go v0.1.5 github.com/sirupsen/logrus v1.9.3 - github.com/stretchr/testify v1.8.4 - github.com/urfave/cli/v2 v2.25.7 + github.com/spf13/cobra v1.8.0 + github.com/spf13/pflag v1.0.5 + github.com/spf13/viper v1.18.2 + github.com/stretchr/testify v1.9.0 gopkg.in/yaml.v3 v3.0.1 ) require ( - github.com/blendle/zapdriver v1.3.1 // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect + github.com/andybalholm/brotli v1.1.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cloudflare/circl v1.5.0 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect - github.com/eosswedenorg-go/jsontime v0.0.0-20230509125027-08422d6236c7 // indirect - github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/go-telegram-bot-api/telegram-bot-api v4.6.4+incompatible // indirect - github.com/gorilla/websocket v1.5.0 // indirect + github.com/google/go-cmp v0.6.0 // indirect + github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad // indirect + github.com/gorilla/websocket v1.5.1 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect + github.com/imroc/req/v3 v3.49.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.16.6 // indirect - github.com/kr/pretty v0.3.1 // indirect - github.com/logrusorgru/aurora v2.0.3+incompatible // indirect - github.com/mitchellh/go-testing-interface v1.14.1 // indirect + github.com/klauspost/compress v1.17.11 // indirect + github.com/liamylian/jsontime/v2 v2.0.0 // indirect + github.com/magiconair/properties v1.8.7 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/onsi/gomega v1.27.6 // indirect + github.com/onsi/ginkgo/v2 v2.22.0 // indirect + github.com/pelletier/go-toml/v2 v2.1.1 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/russross/blackfriday/v2 v2.1.0 // indirect - github.com/streamingfast/logging v0.0.0-20230608130331-f22c91403091 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/quic-go/qpack v0.5.1 // indirect + github.com/quic-go/quic-go v0.48.2 // indirect + github.com/refraction-networking/utls v1.6.7 // indirect + github.com/sagikazarmark/locafero v0.4.0 // indirect + github.com/sagikazarmark/slog-shim v0.1.0 // indirect + github.com/shufflingpixels/jsontime-go v0.0.0-20240622163621-cf4b2804c92d // indirect + github.com/sourcegraph/conc v0.3.0 // indirect + github.com/spf13/afero v1.11.0 // indirect + github.com/spf13/cast v1.6.0 // indirect + github.com/subosito/gotenv v1.6.0 // indirect github.com/technoweenie/multipartstreamer v1.0.1 // indirect - github.com/tidwall/gjson v1.14.4 // indirect - github.com/tidwall/match v1.1.1 // indirect - github.com/tidwall/pretty v1.2.1 // indirect - github.com/ugorji/go/codec v1.2.11 // indirect + github.com/ugorji/go/codec v1.2.12 // indirect github.com/vmihailenco/go-tinylfu v0.2.2 // indirect - github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect + github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect - github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect - go.uber.org/atomic v1.11.0 // indirect - go.uber.org/goleak v1.2.1 // indirect - go.uber.org/multierr v1.8.0 // indirect - go.uber.org/zap v1.24.0 // indirect - golang.org/x/crypto v0.10.0 // indirect - golang.org/x/sync v0.3.0 // indirect - golang.org/x/sys v0.9.0 // indirect - golang.org/x/term v0.9.0 // indirect + go.uber.org/mock v0.5.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + golang.org/x/crypto v0.31.0 // indirect + golang.org/x/exp v0.0.0-20241215155358-4a5509556b9e // indirect + golang.org/x/mod v0.22.0 // indirect + golang.org/x/net v0.33.0 // indirect + golang.org/x/sync v0.10.0 // indirect + golang.org/x/sys v0.28.0 // indirect + golang.org/x/text v0.21.0 // indirect + golang.org/x/tools v0.28.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect + gopkg.in/ini.v1 v1.67.0 // indirect ) replace github.com/eosswedenorg/thalos/api => ./api diff --git a/go.sum b/go.sum index 4619816..7a6c686 100644 --- a/go.sum +++ b/go.sum @@ -1,47 +1,54 @@ github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a h1:HbKu58rmZpUGpz5+4FfNmIU+FmZg2P3Xaj2v2bfNWmk= +github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc= github.com/alicebob/miniredis/v2 v2.30.2 h1:lc1UAUT9ZA7h4srlfBmBt2aorm5Yftk9nBjxz7EyY9I= -github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= -github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= -github.com/blendle/zapdriver v1.3.1 h1:C3dydBOWYRiOk+B8X9IVZ5IOe+7cl+tGOexN4QqHfpE= -github.com/blendle/zapdriver v1.3.1/go.mod h1:mdXfREi6u5MArG4j9fewC+FGnXaBR+T4Ox4J2u4eHCc= -github.com/bsm/ginkgo/v2 v2.7.0 h1:ItPMPH90RbmZJt5GtkcNvIRuGEdwlBItdNVoyzaNQao= -github.com/bsm/gomega v1.26.0 h1:LhQm+AFcgV2M0WyKroMASzAzCAJVpAxQXv4SaI9a69Y= +github.com/alicebob/miniredis/v2 v2.30.2/go.mod h1:b25qWj4fCEsBeAAR2mlb0ufImGC6uH3VlUfb/HS5zKg= +github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= +github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= -github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cloudflare/circl v1.5.0 h1:hxIWksrX6XN5a1L2TI/h53AGPhNHoUBo+TD1ms9+pys= +github.com/cloudflare/circl v1.5.0/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs= +github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/eoscanada/eos-go v0.10.3-0.20230413154640-bb75101dc2f6 h1:93LUOgAmRkmz8DF2V62GBAFm+7JgWA15zI1uYukBeRk= -github.com/eoscanada/eos-go v0.10.3-0.20230413154640-bb75101dc2f6/go.mod h1:L3avCf8OkDrjlUeNy9DdoV67TCmDNj2dSlc5Xp3DNNk= -github.com/eosswedenorg-go/antelope-ship-client v0.2.6 h1:0wPF9TC867eG/+rEzJd0L+TtSOrP09YQzcbDU050FoA= -github.com/eosswedenorg-go/antelope-ship-client v0.2.6/go.mod h1:kZ/4gkAIdAq4/WiZlVaSONpELcDCMJQJMmlikLUGCb8= -github.com/eosswedenorg-go/jsontime v0.0.0-20230509125027-08422d6236c7 h1:rLPu++RHaxg4WmUOXeWYioZuafWs0PVcYuvzOWbOJjk= -github.com/eosswedenorg-go/jsontime v0.0.0-20230509125027-08422d6236c7/go.mod h1:eNUkVOymzgl0lViUhmm08PkutzqLnOQ6Dr+RUnf+Mq0= +github.com/eosswedenorg-go/antelope-ship-client v0.3.2 h1:mDXZkjQ0bTPJClkhoPEP5ltucxql6bR+QixhnQI/Og4= +github.com/eosswedenorg-go/antelope-ship-client v0.3.2/go.mod h1:DnUmaRGxz/V73CtVEJx/fReqhgGzhVyWpOrEKVYQSgE= github.com/eosswedenorg-go/pid v1.0.1 h1:W4AEnnNwb041SpNR1uTZ/KbJ0OTA5eqiqIR1Q5Ah6A0= github.com/eosswedenorg-go/pid v1.0.1/go.mod h1:wiOB/JXGt4YA3+T0j0xmCGSc3Jxzb7Ti/Ftli1fgWu4= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= -github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-redis/cache/v9 v9.0.0 h1:0thdtFo0xJi0/WXbRVu8B066z8OvVymXTJGaXrVWnN0= github.com/go-redis/cache/v9 v9.0.0/go.mod h1:cMwi1N8ASBOufbIvk7cdXe2PbPjK/WMRL95FFHWsSgI= -github.com/go-redis/redismock/v9 v9.0.3 h1:mtHQi2l51lCmXIbTRTqb1EiHYe9tL5Yk5oorlSJJqR0= -github.com/go-redis/redismock/v9 v9.0.3/go.mod h1:F6tJRfnU8R/NZ0E+Gjvoluk14MqMC5ueSZX6vVQypc0= +github.com/go-redis/redismock/v9 v9.2.0 h1:ZrMYQeKPECZPjOj5u9eyOjg8Nnb0BS9lkVIZ6IpsKLw= +github.com/go-redis/redismock/v9 v9.2.0/go.mod h1:18KHfGDK4Y6c2R0H38EUGWAdc7ZQS9gfYxc94k7rWT0= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/go-telegram-bot-api/telegram-bot-api v4.6.4+incompatible h1:2cauKuaELYAEARXRkq2LrJ0yDDv1rW7+wrTEdVL3uaU= github.com/go-telegram-bot-api/telegram-bot-api v4.6.4+incompatible/go.mod h1:qf9acutJ8cwBUhm1bqgz6Bei9/C/c93FPDljKWwsOgM= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -58,21 +65,38 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= -github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad h1:a6HEuzUHeKH6hwfN/ZoQgRgVIWFJljSWa/zetS2WTvg= +github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= +github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/jarcoal/httpmock v1.2.0 h1:gSvTxxFR/MEMfsGrvRbdfpRUMBStovlSRLw0Ep1bwwc= +github.com/imroc/req/v3 v3.49.0 h1:5Rac2qvz7Dq0E3PeBo/c2szV3hagPQIGLoHtfBmYhu4= +github.com/imroc/req/v3 v3.49.0/go.mod h1:XZf4t94DNJzcA0UOBlA68hmSrWsAyvN407ADdH4mzCA= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jordan-wright/email v4.0.1-0.20210109023952-943e75fe5223+incompatible h1:jdpOPRN1zP63Td1hDQbZW73xKmzDvZHzVdNYxhnTMDA= +github.com/jordan-wright/email v4.0.1-0.20210109023952-943e75fe5223+incompatible/go.mod h1:1c7szIrayyPPB/987hsnvNzLushdWf4o/79s3P08L8A= +github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/karlseguin/typed v1.1.8 h1:ND0eDpwiUFIrm/n1ehxUyh/XNGs9zkYrLxtGqENSalY= +github.com/karlseguin/typed v1.1.8/go.mod h1:pZlmYaWQ7MVpwfIOP88fASh3LopVxKeE+uNXW3hQ2D8= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.16.6 h1:91SKEy4K37vkp255cJ8QesJhjyRO0hn9i9G0GoUwLsk= -github.com/klauspost/compress v1.16.6/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= +github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= @@ -82,13 +106,17 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8= -github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= -github.com/mitchellh/go-testing-interface v1.14.1 h1:jrgshOhYAUVNMAJiKbEu7EqAwgJJ2JqpQmpLJOu07cU= -github.com/mitchellh/go-testing-interface v1.14.1/go.mod h1:gfgS7OtZj6MA4U1UrDRp04twqAjfvlZyCfX3sDjEym8= +github.com/liamylian/jsontime/v2 v2.0.0 h1:3if2kDW/boymUdO+4Qj/m4uaXMBSF6np9KEgg90cwH0= +github.com/liamylian/jsontime/v2 v2.0.0/go.mod h1:UHp1oAPqCBfspokvGmaGe0IAl2IgOpgOgDaKPcvcGGY= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/nikoksr/notify v0.41.0 h1:4LGE41GpWdHX5M3Xo6DlWRwS2WLDbOq1Rk7IzY4vjmQ= @@ -108,6 +136,8 @@ github.com/onsi/ginkgo/v2 v2.3.0/go.mod h1:Eew0uilEqZmIEZr8JrvYlvOM7Rr6xzTmMV8Ay github.com/onsi/ginkgo/v2 v2.4.0/go.mod h1:iHkDK1fKGcBoEHT5W7YBq4RFWaQulw+caOMkAt4OrFo= github.com/onsi/ginkgo/v2 v2.5.0/go.mod h1:Luc4sArBICYCS8THh8v3i3i5CuSZO+RaQRaJoeNwomw= github.com/onsi/ginkgo/v2 v2.7.0/go.mod h1:yjiuMwPokqY1XauOgju45q3sJt6VzQ/Fict1LFVcsAo= +github.com/onsi/ginkgo/v2 v2.22.0 h1:Yed107/8DjTr0lKCNt7Dn8yQ6ybuDRQoMGrNFKzMfHg= +github.com/onsi/ginkgo/v2 v2.22.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= @@ -118,28 +148,50 @@ github.com/onsi/gomega v1.22.1/go.mod h1:x6n7VNe4hw0vkyYUM4mjIXx3JbLiPaBPNgB7PRQ github.com/onsi/gomega v1.24.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg= github.com/onsi/gomega v1.24.1/go.mod h1:3AOiACssS3/MajrniINInwbfOOtfZvplPzuRSmvt1jM= github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= -github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE= -github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg= -github.com/pborman/getopt/v2 v2.1.0 h1:eNfR+r+dWLdWmV8g5OlpyrTYHkhVNxHBdN2cCrJmOEA= -github.com/pborman/getopt/v2 v2.1.0/go.mod h1:4NtW75ny4eBw9fO1bhtNdYTlZKYX5/tBLtsOpwKIKd0= -github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/onsi/gomega v1.34.2 h1:pNCwDkzrsv7MS9kpaQvVb1aVLahQXyJ/Tv5oAZMI3i8= +github.com/onsi/gomega v1.34.2/go.mod h1:v1xfxRgk0KIsG+QOdm7p8UosrOzPYRo60fd3B/1Dukc= +github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI= +github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI= +github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg= +github.com/quic-go/quic-go v0.48.2 h1:wsKXZPeGWpMpCGSWqOcqpW2wZYic/8T3aqiOID0/KWE= +github.com/quic-go/quic-go v0.48.2/go.mod h1:yBgs3rWBOADpga7F+jJsb6Ybg1LSYiQvwWlLX+/6HMs= github.com/redis/go-redis/v9 v9.0.0-rc.4/go.mod h1:Vo3EsyWnicKnSKCA7HhgnvnyA74wOA69Cd2Meli5mmA= -github.com/redis/go-redis/v9 v9.0.5 h1:CuQcn5HIEeK7BgElubPP8CGtE0KakrnbBSTLjathl5o= -github.com/redis/go-redis/v9 v9.0.5/go.mod h1:WqMKv5vnQbRuZstUwxQI195wHy+t4PuXDOjzMvcuQHk= +github.com/redis/go-redis/v9 v9.5.1 h1:H1X4D3yHPaYrkL5X06Wh6xNVM/pX0Ft4RV0vMGvLBh8= +github.com/redis/go-redis/v9 v9.5.1/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= +github.com/refraction-networking/utls v1.6.7 h1:zVJ7sP1dJx/WtVuITug3qYUq034cDq9B2MR1K67ULZM= +github.com/refraction-networking/utls v1.6.7/go.mod h1:BC3O4vQzye5hqpmDTWUqi4P5DDhzJfkV1tdqtawQIH0= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= -github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= +github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= +github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= +github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= +github.com/shufflingpixels/antelope-go v0.1.5 h1:N0jCC5bya5shBb96Ff2cCpN+Yw99YldVjWvr0qoiW3E= +github.com/shufflingpixels/antelope-go v0.1.5/go.mod h1:VFULwUB/YfNZeZFzClTNrLyIKWChRhnPvxdzapeOcm0= +github.com/shufflingpixels/jsontime-go v0.0.0-20240622163621-cf4b2804c92d h1:nju7jR1Kf210tArPT6l//HlfLLFnvje1BWl5TSRsohQ= +github.com/shufflingpixels/jsontime-go v0.0.0-20240622163621-cf4b2804c92d/go.mod h1:W0TaKyg3kDqWmFUxlax3qAls/lRdo12EseM6f4f0dzE= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/streamingfast/logging v0.0.0-20230608130331-f22c91403091 h1:RN5mrigyirb8anBEtdjtHFIufXdacyTi6i4KBfeNXeo= -github.com/streamingfast/logging v0.0.0-20230608130331-f22c91403091/go.mod h1:VlduQ80JcGJSargkRU4Sg9Xo63wZD/l8A5NC/Uo1/uU= +github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= +github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= +github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= +github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= +github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= +github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= +github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.18.2 h1:LUXCnvUvSM6FXAsj6nnfc8Q2tp1dIgUfY9Kc8GsSOiQ= +github.com/spf13/viper v1.18.2/go.mod h1:EKmWIqdnk5lOcmR72yw6hS+8OPYcwD0jteitLMVB+yk= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -150,78 +202,57 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/technoweenie/multipartstreamer v1.0.1 h1:XRztA5MXiR1TIRHxH2uNxXxaIkKQDeX7m2XsSOlQEnM= github.com/technoweenie/multipartstreamer v1.0.1/go.mod h1:jNVxdtShOxzAsukZwTSw6MDx5eUJoiEBsSvzDU9uzog= -github.com/test-go/testify v1.1.4 h1:Tf9lntrKUMHiXQ07qBScBTSA0dhYQlu83hswqelv1iE= -github.com/test-go/testify v1.1.4/go.mod h1:rH7cfJo/47vWGdi4GPj16x3/t1xGOj2YxzmNQzk2ghU= -github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM= -github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= -github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= -github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= -github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= -github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= -github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= -github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= -github.com/urfave/cli/v2 v2.25.7 h1:VAzn5oq403l5pHjc4OhD54+XGO9cdKVL/7lDjF+iKUs= -github.com/urfave/cli/v2 v2.25.7/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= +github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= +github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/vmihailenco/go-tinylfu v0.2.2 h1:H1eiG6HM36iniK6+21n9LLpzx1G9R3DJa2UjUjbynsI= github.com/vmihailenco/go-tinylfu v0.2.2/go.mod h1:CutYi2Q9puTxfcolkliPq4npPuofg9N9t8JVrjzwa3Q= github.com/vmihailenco/msgpack/v5 v5.3.4/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= -github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU= -github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= +github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IUPn0Bjt8= +github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok= github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= -github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= -github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= +github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= +github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/gopher-lua v1.1.0 h1:BojcDhfyDWgU2f2TOzYK/g5p2gxMrku8oupLDqlnSqE= -go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= -go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= -go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= -go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= -go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= -go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8= -go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= -go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= -go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= -go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= +github.com/yuin/gopher-lua v1.1.0/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw= +go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU= +go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= -golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM= -golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/exp v0.0.0-20241215155358-4a5509556b9e h1:4qufH0hlUYs6AO6XmZC3GqfDPGSXHVXUFR6OND+iJX4= +golang.org/x/exp v0.0.0-20241215155358-4a5509556b9e/go.mod h1:qj5a5QZpwLU2NLQudwIN5koi3beDhSAlJwa67PuM98c= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4= +golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= @@ -229,15 +260,16 @@ golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= -golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= -golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -249,9 +281,7 @@ golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -260,21 +290,18 @@ golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= -golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= -golang.org/x/term v0.9.0 h1:GRRCnKYhdQrD8kfRAdQ6Zcw1P0OcELxGLKJvtjVMZ28= -golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -282,16 +309,19 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= +golang.org/x/tools v0.28.0 h1:WuB6qZ4RPCQo5aP3WdKZS7i595EdWqWR8vqJTlwTVK8= +golang.org/x/tools v0.28.0/go.mod h1:dcIOrVd3mfQKTgrDVQHqCPMWy6lnhfhtX3hLXYVLfRw= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -305,6 +335,8 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= +google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -312,15 +344,16 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o= +gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU= +gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= diff --git a/install.sh b/install.sh index a7126e1..a1d4c12 100755 --- a/install.sh +++ b/install.sh @@ -7,10 +7,29 @@ fi INSTALL_DIR=$1 -echo "Installing application in: $INSTALL_DIR" +echo -e "\033[34m-\033[0m Installing application in: $INSTALL_DIR" + +PROGRAMS=(make go) + +missing_prog=0 +for prog in ${PROGRAMS[@]}; do + CMD=$(which $prog) + if [ -z "$CMD" ]; then + echo -e "\033[31m!!\033[0m Failed to locate $prog, please install this program" + missing_prog=1 + fi +done + +if [ $missing_prog -ne 0 ]; then + exit 1 +fi mkdir -p "$INSTALL_DIR"/{bin,logs} - make -e DESTDIR=$INSTALL_DIR PREFIX= CFGDIR= install install-scripts +if [ $? -ne 0 ]; then + rm -fr "$INSTALL_DIR" + echo -e "\033[31m!!\033[0m Installation failed" + exit 1 +fi -echo "Done" +echo -e "\033[32m*\033[0m Done" diff --git a/internal/abi/manager.go b/internal/abi/manager.go new file mode 100644 index 0000000..3df1cdb --- /dev/null +++ b/internal/abi/manager.go @@ -0,0 +1,64 @@ +package abi + +import ( + "context" + "fmt" + "time" + + "github.com/eosswedenorg/thalos/internal/cache" + "github.com/eosswedenorg/thalos/internal/config" + "github.com/shufflingpixels/antelope-go/api" + "github.com/shufflingpixels/antelope-go/chain" +) + +// AbiManager handles an ABI cache that fetches the ABI from an API on cache miss. +type AbiManager struct { + cfg *config.AbiCache + cache *cache.Cache + api *api.Client + ctx context.Context +} + +// Create a new ABI Manager +func NewAbiManager(cfg *config.AbiCache, cache *cache.Cache, api *api.Client) *AbiManager { + return &AbiManager{ + cache: cache, + api: api, + cfg: cfg, + ctx: context.Background(), + } +} + +// Set or update an ABI in the cache. +func (mgr *AbiManager) SetAbi(account chain.Name, abi *chain.Abi) error { + ctx, cancel := context.WithTimeout(mgr.ctx, time.Millisecond*500) + defer cancel() + return mgr.cache.Set(ctx, account.String(), *abi, time.Hour) +} + +// Get an ABI from the cache, on cache miss it is fetched from the +// API, gets cached and then returned to the user +func (mgr *AbiManager) GetAbi(account chain.Name) (*chain.Abi, error) { + var abi chain.Abi + if err := mgr.cacheGet(account, &abi); err != nil { + ctx, cancel := context.WithTimeout(mgr.ctx, mgr.cfg.ApiTimeout) + defer cancel() + resp, err := mgr.api.GetAbi(ctx, account.String()) + if err != nil { + return nil, fmt.Errorf("api: %s", err) + } + abi = resp.Abi + + err = mgr.SetAbi(account, &abi) + if err != nil { + return nil, fmt.Errorf("cache: %s", err) + } + } + return &abi, nil +} + +func (mgr *AbiManager) cacheGet(account chain.Name, value any) error { + ctx, cancel := context.WithTimeout(mgr.ctx, time.Millisecond*500) + defer cancel() + return mgr.cache.Get(ctx, account.String(), value) +} diff --git a/internal/abi/manager_test.go b/internal/abi/manager_test.go new file mode 100644 index 0000000..2606ec1 --- /dev/null +++ b/internal/abi/manager_test.go @@ -0,0 +1,180 @@ +package abi + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/shufflingpixels/antelope-go/api" + "github.com/shufflingpixels/antelope-go/chain" + + "github.com/eosswedenorg/thalos/internal/cache" + "github.com/eosswedenorg/thalos/internal/config" + "github.com/stretchr/testify/assert" +) + +var abiString = ` +{ + "version": "eosio::abi/1.0", + "types": [{ + "new_type_name": "new_type_name_1", + "type": "name" + }], + "structs": [ + { + "name": "struct_name_1", + "base": "struct_name_2", + "fields": [ + {"name":"struct_1_field_1", "type":"new_type_name_1"}, + {"name":"struct_1_field_2", "type":"struct_name_3"}, + {"name":"struct_1_field_3", "type":"string?"}, + {"name":"struct_1_field_4", "type":"string?"}, + {"name":"struct_1_field_5", "type":"struct_name_4[]"} + ] + },{ + "name": "struct_name_2", + "base": "", + "fields": [ + {"name":"struct_2_field_1", "type":"string"} + ] + },{ + "name": "struct_name_3", + "base": "", + "fields": [ + {"name":"struct_3_field_1", "type":"string"} + ] + },{ + "name": "struct_name_4", + "base": "", + "fields": [ + {"name":"struct_4_field_1", "type":"string"} + ] + } + ], + "actions": [{ + "name": "action_name_1", + "type": "struct_name_1", + "ricardian_contract": "" + }], + "tables": [{ + "name": "table_name_1", + "index_type": "i64", + "key_names": [ + "key_name_1", + "key_name_2" + ], + "key_types": [ + "string", + "int" + ], + "type": "struct_name_1" + } + ] +} +` + +func assert_abi(t *testing.T, abi *chain.Abi) { + assert.Equal(t, abi.Version, "eosio::abi/1.0") + + // Types + assert.Equal(t, abi.Types[0].NewTypeName, "new_type_name_1") + assert.Equal(t, abi.Types[0].Type, "name") + + // Structs + assert.Equal(t, abi.Structs[0].Name, "struct_name_1") + assert.Equal(t, abi.Structs[0].Base, "struct_name_2") + assert.Equal(t, abi.Structs[0].Fields[0].Name, "struct_1_field_1") + assert.Equal(t, abi.Structs[0].Fields[0].Type, "new_type_name_1") + assert.Equal(t, abi.Structs[0].Fields[1].Name, "struct_1_field_2") + assert.Equal(t, abi.Structs[0].Fields[1].Type, "struct_name_3") + assert.Equal(t, abi.Structs[0].Fields[2].Name, "struct_1_field_3") + assert.Equal(t, abi.Structs[0].Fields[2].Type, "string?") + assert.Equal(t, abi.Structs[0].Fields[3].Name, "struct_1_field_4") + assert.Equal(t, abi.Structs[0].Fields[3].Type, "string?") + assert.Equal(t, abi.Structs[0].Fields[4].Name, "struct_1_field_5") + assert.Equal(t, abi.Structs[0].Fields[4].Type, "struct_name_4[]") + + assert.Equal(t, abi.Structs[1].Name, "struct_name_2") + assert.Equal(t, abi.Structs[1].Base, "") + assert.Equal(t, abi.Structs[1].Fields[0].Name, "struct_2_field_1") + assert.Equal(t, abi.Structs[1].Fields[0].Type, "string") + + assert.Equal(t, abi.Structs[2].Name, "struct_name_3") + assert.Equal(t, abi.Structs[2].Base, "") + assert.Equal(t, abi.Structs[2].Fields[0].Name, "struct_3_field_1") + assert.Equal(t, abi.Structs[2].Fields[0].Type, "string") + + assert.Equal(t, abi.Structs[3].Name, "struct_name_4") + assert.Equal(t, abi.Structs[3].Base, "") + assert.Equal(t, abi.Structs[3].Fields[0].Name, "struct_4_field_1") + assert.Equal(t, abi.Structs[3].Fields[0].Type, "string") + + // Actions + assert.Equal(t, abi.Actions[0].Name, chain.N("action_name_1")) + assert.Equal(t, abi.Actions[0].Type, "struct_name_1") + assert.Equal(t, abi.Actions[0].RicardianContract, "") + + // Tables + assert.Equal(t, abi.Tables[0].Name, chain.N("table_name_1")) + assert.Equal(t, abi.Tables[0].Type, "struct_name_1") + assert.Equal(t, abi.Tables[0].IndexType, "i64") + assert.Equal(t, abi.Tables[0].KeyNames[0], "key_name_1") + assert.Equal(t, abi.Tables[0].KeyNames[1], "key_name_2") + assert.Equal(t, abi.Tables[0].KeyTypes[0], "string") + assert.Equal(t, abi.Tables[0].KeyTypes[1], "int") +} + +func mockAPI(handler http.HandlerFunc) (*api.Client, *httptest.Server) { + server := httptest.NewServer(handler) + + return api.New(server.URL), server +} + +func TestManager_GetAbiFromCache(t *testing.T) { + cfg := &config.AbiCache{ + ApiTimeout: time.Second, + } + + cache := cache.NewCache("thalos::cache::abi::test", cache.NewMemoryStore()) + + api, _ := mockAPI(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + })) + + mgr := NewAbiManager(cfg, cache, api) + + abi := chain.Abi{} + err := json.Unmarshal([]byte(abiString), &abi) + assert.NoError(t, err) + + err = mgr.SetAbi(chain.N("testaccount"), &abi) + assert.NoError(t, err) + + c_abi, err := mgr.GetAbi(chain.N("testaccount")) + assert.NoError(t, err) + assert_abi(t, c_abi) +} + +func TestManager_GetAbiFromAPI(t *testing.T) { + cfg := &config.AbiCache{ + ApiTimeout: time.Second, + } + + cache := cache.NewCache("thalos::cache::abi::test", cache.NewMemoryStore()) + + api, _ := mockAPI(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body := fmt.Sprintf(`{"account_name": "testaccount", "abi": %s}`, abiString) + + _, err := w.Write([]byte(body)) + assert.NoError(t, err) + })) + + mgr := NewAbiManager(cfg, cache, api) + + c_abi, err := mgr.GetAbi(chain.N("testaccount")) + assert.NoError(t, err) + + assert_abi(t, c_abi) +} diff --git a/internal/cache/cache.go b/internal/cache/cache.go new file mode 100644 index 0000000..3cce5f5 --- /dev/null +++ b/internal/cache/cache.go @@ -0,0 +1,31 @@ +package cache + +import ( + "context" + "time" +) + +type Cache struct { + store Store + prefix string +} + +// Create a new cache +func NewCache(prefix string, store Store) *Cache { + return &Cache{ + store: store, + prefix: prefix, + } +} + +func (cache *Cache) Get(ctx context.Context, key string, value any) error { + return cache.store.Get(ctx, cache.key(key), value) +} + +func (cache *Cache) Set(ctx context.Context, key string, value any, ttl time.Duration) error { + return cache.store.Set(ctx, cache.key(key), value, ttl) +} + +func (cache *Cache) key(key string) string { + return cache.prefix + "::" + key +} diff --git a/internal/cache/factory.go b/internal/cache/factory.go new file mode 100644 index 0000000..bdde3a9 --- /dev/null +++ b/internal/cache/factory.go @@ -0,0 +1,26 @@ +package cache + +import ( + "fmt" + + "github.com/karlseguin/typed" +) + +type Factory func(opts typed.Typed) (Store, error) + +var factories = map[string]Factory{ + "memory": func(opts typed.Typed) (Store, error) { + return NewMemoryStore(), nil + }, +} + +func RegisterFactory(driver string, factory Factory) { + factories[driver] = factory +} + +func Make(driver string, opts typed.Typed) (Store, error) { + if factory, ok := factories[driver]; ok { + return factory(opts) + } + return nil, fmt.Errorf("Invalid cache storage: %s", driver) +} diff --git a/internal/cache/factory_test.go b/internal/cache/factory_test.go new file mode 100644 index 0000000..af0d266 --- /dev/null +++ b/internal/cache/factory_test.go @@ -0,0 +1,20 @@ +package cache_test + +import ( + "testing" + + "github.com/eosswedenorg/thalos/internal/cache" + "github.com/stretchr/testify/require" +) + +func TestFactory_Make(t *testing.T) { + store, err := cache.Make("memory", map[string]any{}) + require.NoError(t, err) + require.Equal(t, cache.NewMemoryStore(), store) +} + +func TestFactory_MakeInvalidDriver(t *testing.T) { + store, err := cache.Make("87923yus", map[string]any{}) + require.Error(t, err) + require.Nil(t, store) +} diff --git a/internal/cache/memory_store.go b/internal/cache/memory_store.go new file mode 100644 index 0000000..a198a70 --- /dev/null +++ b/internal/cache/memory_store.go @@ -0,0 +1,61 @@ +package cache + +import ( + "context" + "fmt" + "reflect" + "time" +) + +// Store time function in a variable. +// Makes it easy to travel in time when testing. +var now = time.Now + +type memoryStoreItem struct { + // Actual value stored. + value any + + // Cache expiration time. + expired time.Time +} + +type MemoryStore struct { + data map[string]memoryStoreItem +} + +func NewMemoryStore() *MemoryStore { + return &MemoryStore{make(map[string]memoryStoreItem)} +} + +func (s *MemoryStore) Get(ctx context.Context, key string, value any) error { + if item, ok := s.data[key]; ok { + + if item.expired.Before(now()) { + delete(s.data, key) + return fmt.Errorf("key: %s does not exist", key) + } + + v := reflect.ValueOf(value) + if v.Kind() != reflect.Pointer { + return fmt.Errorf("value must be of pointer type, '%s' passed", v.Kind().String()) + } + + v.Elem().Set(reflect.ValueOf(item.value)) + + return nil + } + return fmt.Errorf("key: %s does not exist", key) +} + +func (s *MemoryStore) Has(ctx context.Context, key string) bool { + _, hit := s.data[key] + return hit +} + +func (s *MemoryStore) Set(ctx context.Context, key string, value any, ttl time.Duration) error { + s.data[key] = memoryStoreItem{ + value: value, + expired: now().Add(ttl), + } + return nil +} diff --git a/internal/cache/memory_store_test.go b/internal/cache/memory_store_test.go new file mode 100644 index 0000000..9c949e3 --- /dev/null +++ b/internal/cache/memory_store_test.go @@ -0,0 +1,88 @@ +package cache + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +type memoryTestItem struct { + String string + Num uint32 + Float float32 +} + +func TestMemoryStore_Set(t *testing.T) { + now = func() time.Time { return time.Unix(1581315270, 0) } + + item := memoryTestItem{ + String: "MyString", + Num: 23, + Float: 3.14, + } + + expected := map[string]memoryStoreItem{ + "key1": { + value: item, + expired: now().Add(time.Hour), + }, + } + + store := NewMemoryStore() + err := store.Set(context.Background(), "key1", item, time.Hour) + assert.NoError(t, err) + + assert.Equal(t, expected, store.data) +} + +func TestMemoryStore_GetMiss(t *testing.T) { + store := NewMemoryStore() + + var v any + err := store.Get(context.Background(), "Key2", &v) + assert.Error(t, err) +} + +func TestMemoryStore_GetHit(t *testing.T) { + expected := memoryTestItem{ + String: "MyString", + Num: 23, + Float: 3.14, + } + + store := NewMemoryStore() + err := store.Set(context.Background(), "key1", expected, time.Hour) + assert.NoError(t, err) + + var actual memoryTestItem + err = store.Get(context.Background(), "key1", &actual) + assert.NoError(t, err) + assert.Equal(t, expected, actual) +} + +func TestMemoryStore_GetNonPointer(t *testing.T) { + expected := memoryTestItem{ + String: "MyString", + Num: 23, + Float: 3.14, + } + + store := NewMemoryStore() + err := store.Set(context.Background(), "key1", expected, time.Hour) + assert.NoError(t, err) + + var actual string + err = store.Get(context.Background(), "key1", actual) + assert.EqualError(t, err, "value must be of pointer type, 'string' passed") +} + +func TestMemoryStore_Has(t *testing.T) { + store := NewMemoryStore() + err := store.Set(context.Background(), "key1", "value", time.Hour) + assert.NoError(t, err) + + assert.True(t, store.Has(context.Background(), "key1")) + assert.False(t, store.Has(context.Background(), "key2")) +} diff --git a/internal/cache/redis_store.go b/internal/cache/redis_store.go new file mode 100644 index 0000000..d8101ad --- /dev/null +++ b/internal/cache/redis_store.go @@ -0,0 +1,63 @@ +package cache + +import ( + "context" + "time" + + "github.com/go-redis/cache/v9" + "github.com/karlseguin/typed" + "github.com/redis/go-redis/v9" +) + +type RedisStore struct { + c *cache.Cache +} + +type options struct { + Stats bool + Size int + TTL time.Duration +} + +func NewRedisStore(options *cache.Options) *RedisStore { + return &RedisStore{ + c: cache.New(options), + } +} + +func getOptions(opts typed.Typed) options { + return options{ + Stats: opts.Bool("stats"), + Size: opts.IntOr("size", 1000), + TTL: time.Duration(opts.IntOr("ttl", 10)) * time.Minute, + } +} + +func NewRedisFactory(client *redis.Client) Factory { + return func(opts typed.Typed) (Store, error) { + o := getOptions(opts) + + return NewRedisStore(&cache.Options{ + Redis: client, + StatsEnabled: o.Stats, + LocalCache: cache.NewTinyLFU(o.Size, o.TTL), + }), nil + } +} + +func (s *RedisStore) Get(ctx context.Context, key string, value interface{}) error { + return s.c.Get(ctx, key, value) +} + +func (s *RedisStore) Has(ctx context.Context, key string) bool { + return s.c.Exists(ctx, key) +} + +func (s *RedisStore) Set(ctx context.Context, key string, value any, ttl time.Duration) error { + return s.c.Set(&cache.Item{ + Ctx: ctx, + Key: key, + Value: value, + TTL: ttl, + }) +} diff --git a/internal/cache/redis_store_test.go b/internal/cache/redis_store_test.go new file mode 100644 index 0000000..7d4c7c3 --- /dev/null +++ b/internal/cache/redis_store_test.go @@ -0,0 +1,137 @@ +package cache + +import ( + "context" + "testing" + "time" + + "github.com/go-redis/redismock/v9" + "github.com/karlseguin/typed" + + redis_cache "github.com/go-redis/cache/v9" + "github.com/stretchr/testify/assert" +) + +type testItem struct { + Num uint32 + Name string +} + +func TestRedisStore_getOptionsDefaults(t *testing.T) { + opts := typed.Typed{} + + expected := options{ + Stats: false, + Size: 1000, + TTL: 10 * time.Minute, + } + + actual := getOptions(opts) + + assert.Equal(t, expected, actual) +} + +func TestRedisStore_getOptions(t *testing.T) { + opts := typed.Typed{ + "stats": true, + "size": 123, + "ttl": 60, + } + + expected := options{ + Stats: true, + Size: 123, + TTL: 60 * time.Minute, + } + + actual := getOptions(opts) + + assert.Equal(t, expected, actual) +} + +func TestRedisStore_Set(t *testing.T) { + client, mock := redismock.NewClientMock() + + store := NewRedisStore(&redis_cache.Options{ + Redis: client, + }) + + expected := testItem{ + Num: 24, + Name: "Some Name", + } + + bytes, err := store.c.Marshal(expected) + assert.NoError(t, err) + + mock.ExpectSet("mykey", bytes, time.Minute).SetVal("OK") + + err = store.Set(context.Background(), "mykey", expected, time.Minute) + assert.NoError(t, err) + + assert.NoError(t, mock.ExpectationsWereMet()) +} + +func TestRedisStore_GetMiss(t *testing.T) { + client, mock := redismock.NewClientMock() + + store := NewRedisStore(&redis_cache.Options{ + Redis: client, + }) + + mock.ExpectGet("mykey").SetErr(redis_cache.ErrCacheMiss) + + expected := testItem{} + err := store.Get(context.Background(), "mykey", &expected) + assert.ErrorIs(t, err, redis_cache.ErrCacheMiss) + assert.NoError(t, mock.ExpectationsWereMet()) +} + +func TestRedisStore_GetHit(t *testing.T) { + client, mock := redismock.NewClientMock() + + store := NewRedisStore(&redis_cache.Options{ + Redis: client, + }) + + expected := testItem{ + Num: 42, + Name: "MyName", + } + + bytes, err := store.c.Marshal(expected) + assert.NoError(t, err) + + mock.ExpectSet("mykey2", bytes, time.Second*20).SetVal("OK") + mock.ExpectGet("mykey2").SetVal(string(bytes)) + + err = store.Set(context.Background(), "mykey2", expected, time.Second*20) + assert.NoError(t, err) + + actual := testItem{} + err = store.Get(context.Background(), "mykey2", &actual) + assert.NoError(t, err) + assert.Equal(t, expected, actual) + assert.NoError(t, mock.ExpectationsWereMet()) +} + +func TestRedisStore_Has(t *testing.T) { + client, mock := redismock.NewClientMock() + + store := NewRedisStore(&redis_cache.Options{ + Redis: client, + }) + + bytes, err := store.c.Marshal("value") + assert.NoError(t, err) + + mock.ExpectSet("key1", bytes, time.Minute*15).SetVal("OK") + mock.ExpectGet("key1").SetVal(string(bytes)) + mock.ExpectGet("key2").RedisNil() + + err = store.Set(context.Background(), "key1", "value", time.Minute*15) + assert.NoError(t, err) + assert.True(t, store.Has(context.Background(), "key1")) + assert.False(t, store.Has(context.Background(), "key2")) + assert.NoError(t, mock.ExpectationsWereMet()) +} diff --git a/internal/cache/store.go b/internal/cache/store.go new file mode 100644 index 0000000..a9d85e8 --- /dev/null +++ b/internal/cache/store.go @@ -0,0 +1,19 @@ +package cache + +import ( + "context" + "time" +) + +// Interface to a cache storage. +type Store interface { + // Set an item in the store. + Set(ctx context.Context, key string, value any, TTL time.Duration) error + + // Get an item from the store. + // returns an error if key is not found or there is other problems. + Get(ctx context.Context, key string, value any) error + + // Check if a key exist in the store. + Has(ctx context.Context, key string) bool +} diff --git a/internal/config/builder.go b/internal/config/builder.go new file mode 100644 index 0000000..e22bf45 --- /dev/null +++ b/internal/config/builder.go @@ -0,0 +1,197 @@ +package config + +import ( + "errors" + "fmt" + "io" + "os" + "reflect" + "strings" + + "github.com/eosswedenorg/thalos/internal/types" + "github.com/mitchellh/mapstructure" + "github.com/spf13/pflag" + "github.com/spf13/viper" +) + +// This is a simple module that encapsulate the creation +// of a config object and can override values from cli flags. + +type Builder struct { + in io.Reader + flags *pflag.FlagSet + binds map[string]string +} + +func NewBuilder() *Builder { + return &Builder{ + binds: map[string]string{ + "api": "url", + "message_codec": "codec", + + // Redis + "redis.addr": "redis-addr", + "redis.user": "redis-user", + "redis.password": "redis-password", + "redis.db": "redis-db", + "redis.prefix": "redis-prefix", + + // Telegram + "telegram.id": "telegram-id", + "telegram.channel": "telegram-channel", + + "cache.storage": "cache", + + // AbiCache + "abi_cache.api_timeout": "abi-cache-api-timeout", + + // Log + "log.maxfilesize": "log-max-filesize", + "log.maxtime": "log-max-time", + "log.file_timestamp_format": "log-file-timestamp", + + // Ship + "ship.url": "ship-url", + "ship.start_block_num": "start-block", + "ship.end_block_num": "end-block", + "ship.irreversible_only": "irreversible-only", + "ship.max_messages_in_flight": "max-msg-in-flight", + "ship.chain": "chain", + "ship.blacklist": "blacklist", + "ship.blacklist_is_whitelist": "blacklist-is-whitelist", + "ship.table_deltas": "table-deltas", + }, + } +} + +// Set the config file to read +func (b *Builder) SetConfigFile(filename string) *Builder { + file, _ := os.Open(filename) + return b.SetSource(file) +} + +// Set the source to read +func (b *Builder) SetSource(in io.Reader) *Builder { + b.in = in + return b +} + +// Set all flags that the builder should use. +func (b *Builder) SetFlags(flags *pflag.FlagSet) *Builder { + b.flags = flags + return b +} + +// Add a flag to the builder. +func (b *Builder) AddFlag(flag *pflag.Flag) *Builder { + b.flags.AddFlag(flag) + return b +} + +// Build the config object from file, cli-flags +func (b *Builder) Build() (*Config, error) { + if b.in == nil { + return nil, errors.New("Config not set") + } + + conf := Config{} + + v := viper.New() + v.SetConfigType("yaml") + + if b.flags != nil { + // bind flags in viper. + for key, flagname := range b.binds { + flag := b.flags.Lookup(flagname) + if flag == nil { + continue + } + + if err := v.BindPFlag(key, flag); err != nil { + return nil, err + } + } + } + + // Read config and unmarshal + if err := v.ReadConfig(b.in); err != nil { + return nil, err + } + + decoders := mapstructure.ComposeDecodeHookFunc( + mapstructure.TextUnmarshallerHookFunc(), + mapstructure.StringToTimeDurationHookFunc(), + mapstructure.StringToSliceHookFunc(","), + func(f reflect.Type, t reflect.Type, in interface{}) (interface{}, error) { + if t == reflect.TypeOf(types.Blacklist{}) { + return decodeIntoBlacklist(in) + } + return in, nil + }, + ) + + err := v.Unmarshal(&conf, viper.DecodeHook(decoders)) + if err != nil { + return nil, err + } + + return &conf, nil +} + +// Decode a generic structure into types.Blacklist +func decodeIntoBlacklist(in any) (*types.Blacklist, error) { + switch v := in.(type) { + // Standard map structure. + case map[string]any: + return blacklistParseMap(v) + + // slice of "contract:action" pairs. Usually from CLI + case []string: + return blacklistParseSlice(v) + + // Sometimes we have a slice of interfaces. + // Need to convert it to a slice of strings. + case []any: + sv := make([]string, len(v)) + for i, j := range v { + sv[i] = j.(string) + } + return blacklistParseSlice(sv) + } + + return nil, fmt.Errorf("Must be a string slice") +} + +// Blacklist map parser +func blacklistParseMap(in map[string]any) (*types.Blacklist, error) { + list := &types.Blacklist{} + for k, v := range in { + switch v := v.(type) { + case []any: + for _, v := range v { + list.Add(k, v.(string)) + } + case any: + list.Add(k, v.(string)) + } + } + return list, nil +} + +// Blacklist slice parser +func blacklistParseSlice(in []string) (*types.Blacklist, error) { + list := &types.Blacklist{} + for _, i := range in { + var action string + parts := strings.SplitN(i, ":", 2) + + if len(parts) < 2 { + action = "*" + } else { + action = parts[1] + } + + list.Add(parts[0], action) + } + return list, nil +} diff --git a/internal/config/builder_test.go b/internal/config/builder_test.go new file mode 100644 index 0000000..b133353 --- /dev/null +++ b/internal/config/builder_test.go @@ -0,0 +1,253 @@ +package config + +import ( + "bytes" + "testing" + "time" + + shipclient "github.com/eosswedenorg-go/antelope-ship-client" + "github.com/eosswedenorg/thalos/internal/log" + "github.com/eosswedenorg/thalos/internal/types" + "github.com/karlseguin/typed" + "github.com/stretchr/testify/require" +) + +func TestBuilder(t *testing.T) { + expected := Config{ + Name: "ship-reader-1", + Api: "http://127.0.0.1:8080", + MessageCodec: "mojibake", + Log: log.Config{ + Filename: "some_file.log", + Directory: "/path/to/whatever", + MaxFileSize: 200, + MaxTime: 30 * time.Minute, + FileTimestampFormat: "20060102@150405", + }, + Cache: Cache{ + Storage: "memcached", + Options: typed.Typed{ + "ttl": "300m", + "size": 400, + "super_fast_mode": true, + }, + }, + AbiCache: AbiCache{ + ApiTimeout: time.Minute * 300, + }, + Ship: ShipConfig{ + Url: "127.0.0.1:8089", + StartBlockNum: 23671836, + EndBlockNum: 23872222, + IrreversibleOnly: true, + MaxMessagesInFlight: 1337, + Blacklist: *types.NewBlacklist(map[string][]string{ + "eosio": {"noop"}, + "contract": {"skip1", "skip2"}, + }), + BlacklistIsWhitelist: true, + }, + Telegram: TelegramConfig{ + Id: "110201543:AAHdqTcvCH1vGWJxfSeofSAs0K5PALDsaw", + Channel: -123456789, + }, + Redis: RedisConfig{ + Addr: "localhost:6379", + User: "myuser", + Password: "passwd", + DB: 4, + Prefix: "some::ship", + }, + } + + builder := NewBuilder() + builder.SetSource(bytes.NewBuffer([]byte(` +name: "ship-reader-1" +api: "http://127.0.0.1:8080" +message_codec: "mojibake" +cache: + storage: memcached + options: + ttl: 300m + size: 400 + super_fast_mode: true +abi_cache: + api_timeout: 300m +log: + filename: some_file.log + directory: /path/to/whatever + maxtime: 30m + maxfilesize: 200b + file_timestamp_format: 20060102@150405 +ship: + url: "127.0.0.1:8089" + irreversible_only: true + max_messages_in_flight: 1337 + start_block_num: 23671836 + end_block_num: 23872222 + blacklist: + eosio: noop + contract: + - skip1 + - skip2 + blacklist_is_whitelist: true +telegram: + id: "110201543:AAHdqTcvCH1vGWJxfSeofSAs0K5PALDsaw" + channel: -123456789 +redis: + addr: "localhost:6379" + user: "myuser" + password: "passwd" + db: 4 + prefix: "some::ship" +`))) + + cfg, err := builder.Build() + + require.NoError(t, err) + require.Equal(t, &expected, cfg) +} + +func TestBuilder_WithDefaultConfig(t *testing.T) { + expected := Config{ + MessageCodec: "json", + Log: log.Config{ + MaxFileSize: 10 * 1000 * 1000, + MaxTime: time.Hour * 24, + FileTimestampFormat: "2006-01-02_150405", + }, + Cache: Cache{ + Storage: "redis", + }, + AbiCache: AbiCache{ + ApiTimeout: time.Second, + }, + Ship: ShipConfig{ + Url: "ws://127.0.0.1:8080", + StartBlockNum: shipclient.NULL_BLOCK_NUMBER, + EndBlockNum: shipclient.NULL_BLOCK_NUMBER, + MaxMessagesInFlight: 10, + EnableTableDeltas: true, + }, + Redis: RedisConfig{ + Addr: "127.0.0.1:6379", + Prefix: "ship", + }, + } + + cfg, err := NewBuilder(). + SetSource(bytes.NewReader([]byte(``))). + SetFlags(GetFlags()). + Build() + + require.NoError(t, err) + require.Equal(t, &expected, cfg) +} + +func TestBuilder_NilSource(t *testing.T) { + cfg, err := NewBuilder().Build() + require.Nil(t, cfg) + require.EqualError(t, err, "Config not set") +} + +func TestBuilder_Flags(t *testing.T) { + flags := GetFlags() + + require.NoError(t, flags.Set("url", "https://myapi")) + require.NoError(t, flags.Set("codec", "binary")) + require.NoError(t, flags.Set("redis-addr", "154.223.38.15:6380")) + require.NoError(t, flags.Set("redis-user", "myuser")) + require.NoError(t, flags.Set("redis-password", "secret123")) + require.NoError(t, flags.Set("redis-db", "3")) + require.NoError(t, flags.Set("redis-prefix", "custom-prefix")) + require.NoError(t, flags.Set("cache", "memcached")) + require.NoError(t, flags.Set("abi-cache-api-timeout", "16h")) + require.NoError(t, flags.Set("telegram-id", "72983126312982618")) + require.NoError(t, flags.Set("telegram-channel", "-293492332")) + require.NoError(t, flags.Set("log-max-filesize", "25mb")) + require.NoError(t, flags.Set("log-max-time", "10m")) + require.NoError(t, flags.Set("log-file-timestamp", "0102-15:04:05")) + require.NoError(t, flags.Set("ship-url", "ws://myship.com:7823")) + require.NoError(t, flags.Set("start-block", "7327833")) + require.NoError(t, flags.Set("end-block", "329408392")) + require.NoError(t, flags.Set("irreversible-only", "true")) + require.NoError(t, flags.Set("max-msg-in-flight", "98")) + require.NoError(t, flags.Set("chain", "wax")) + require.NoError(t, flags.Set("blacklist", "contract:action1,contract:action2,contract2:action1")) + require.NoError(t, flags.Set("blacklist-is-whitelist", "true")) + require.NoError(t, flags.Set("table-deltas", "false")) + + cfg, err := NewBuilder(). + SetSource(bytes.NewReader([]byte(``))). + SetFlags(flags). + Build() + + expected := Config{ + Api: "https://myapi", + MessageCodec: "binary", + Log: log.Config{ + MaxFileSize: 25 * 1000 * 1000, // 25 mb + MaxTime: time.Minute * 10, + FileTimestampFormat: "0102-15:04:05", + }, + Cache: Cache{ + Storage: "memcached", + }, + AbiCache: AbiCache{ + ApiTimeout: time.Hour * 16, + }, + Ship: ShipConfig{ + Url: "ws://myship.com:7823", + StartBlockNum: 7327833, + EndBlockNum: 329408392, + MaxMessagesInFlight: 98, + IrreversibleOnly: true, + Chain: "wax", + Blacklist: *types.NewBlacklist(map[string][]string{ + "contract": {"action1", "action2"}, + "contract2": {"action1"}, + }), + BlacklistIsWhitelist: true, + }, + Telegram: TelegramConfig{ + Id: "72983126312982618", + Channel: -293492332, + }, + Redis: RedisConfig{ + Addr: "154.223.38.15:6380", + User: "myuser", + Password: "secret123", + DB: 3, + Prefix: "custom-prefix", + }, + } + + require.NoError(t, err) + require.Equal(t, &expected, cfg) +} + +func TestBuilder_BlacklistSlice(t *testing.T) { + expected := Config{ + Ship: ShipConfig{ + Blacklist: *types.NewBlacklist(map[string][]string{ + "contract": {"action"}, + "contract2": {"action2"}, + "contract3": {"*"}, + }), + }, + } + + builder := NewBuilder() + builder.SetSource(bytes.NewBuffer([]byte(` +ship: + blacklist: + - "contract:action" + - "contract2:action2" + - contract3 +`))) + + cfg, err := builder.Build() + + require.NoError(t, err) + require.Equal(t, &expected, cfg) +} diff --git a/internal/config/cli.go b/internal/config/cli.go new file mode 100644 index 0000000..d4f26ac --- /dev/null +++ b/internal/config/cli.go @@ -0,0 +1,59 @@ +package config + +import ( + "time" + + shipclient "github.com/eosswedenorg-go/antelope-ship-client" + "github.com/spf13/pflag" +) + +// Get a flag set with all flags mapping to a config value. +func GetFlags() *pflag.FlagSet { + flags := pflag.FlagSet{} + + // Generic + flags.StringP("url", "u", "", "Url to antelope api") + flags.String("codec", "json", "Codec used to send messages") + + // Redis + flags.String("redis-addr", "127.0.0.1:6379", "host:port to redis server") + flags.String("redis-user", "", "Redis username") + flags.String("redis-password", "", "Redis password") + flags.Int("redis-db", 0, "Redis database") + flags.String("redis-prefix", "ship", "Redis channel prefix") + + // Telegram + flags.String("telegram-id", "", "Id of telegram bot") + flags.Int64("telegram-channel", 0, "Telegram channel to send notifications to") + + // Cache + flags.String("cache", "redis", "What cache driver to use") + + // AbiCache + flags.Duration("abi-cache-api-timeout", time.Second, "Duration before the api call times out when the ABI cache requests an abi.") + + // Log + flags.StringP("log", "l", "", "Path to log file (default: print to stdout/stderr)") + flags.String("log-max-filesize", "10mb", "Max filesize for logfile to rotate") + flags.Duration("log-max-time", time.Hour*24, "Max time for logfile to rotate") + flags.String("log-file-timestamp", "2006-01-02_150405", "Timestamp format to use when rotating log files") + + // Ship + flags.String("ship-url", "ws://127.0.0.1:8080", "Url to ship node") + flags.Uint32("start-block", shipclient.NULL_BLOCK_NUMBER, "Start to stream from this block") + flags.Uint32("end-block", shipclient.NULL_BLOCK_NUMBER, "Stop streaming when this block is reached") + + flags.Lookup("start-block").DefValue = "Config value, cache, head from api" + flags.Lookup("end-block").DefValue = "none" + + flags.Bool("table-deltas", true, "True if thalos should receive and process table deltas from ship.") + + flags.Bool("irreversible-only", false, "Only stream irreversible blocks from ship") + flags.Int("max-msg-in-flight", 10, "Maximum messages that can be sent from SHIP without acknowledgement") + flags.String("chain", "", "ChainID used in channel namespace, can be any string (default from api)") + + flags.StringSlice("blacklist", []string{}, "Define a list of 'contract:action' pairs that will be blacklisted (Thalos will not process those actions)") + flags.Bool("blacklist-is-whitelist", false, "Thalos will treat the blacklist as a whitelist") + + return &flags +} diff --git a/internal/config/config.go b/internal/config/config.go new file mode 100644 index 0000000..feabb87 --- /dev/null +++ b/internal/config/config.go @@ -0,0 +1,60 @@ +package config + +import ( + "time" + + "github.com/eosswedenorg/thalos/internal/log" + "github.com/eosswedenorg/thalos/internal/types" + "github.com/karlseguin/typed" +) + +type RedisConfig struct { + Addr string `yaml:"addr"` + User string `yaml:"user"` + Password string `yaml:"password"` + DB int `yaml:"db"` + Prefix string `yaml:"prefix"` +} + +type Cache struct { + Storage string `yaml:"storage" mapstructure:"storage"` + Options typed.Typed `yaml:"options" mapstructure:"options"` +} + +type TelegramConfig struct { + Id string `yaml:"id" mapstructure:"id"` + Channel int64 `yaml:"channel" mapstructure:"channel"` +} + +type AbiCache struct { + ApiTimeout time.Duration `yaml:"api_timeout" mapstructure:"api_timeout"` +} + +type ShipConfig struct { + Url string `yaml:"url" mapstructure:"url"` + IrreversibleOnly bool `yaml:"irreversible_only" mapstructure:"irreversible_only"` + MaxMessagesInFlight uint32 `yaml:"max_messages_in_flight" mapstructure:"max_messages_in_flight"` + StartBlockNum uint32 `yaml:"start_block_num" mapstructure:"start_block_num"` + EndBlockNum uint32 `yaml:"end_block_num" mapstructure:"end_block_num"` + Chain string `yaml:"chain" mapstructure:"chain"` + Blacklist types.Blacklist `yaml:"blacklist" mapstructure:"blacklist"` + BlacklistIsWhitelist bool `yaml:"blacklist_is_whitelist" mapstructure:"blacklist_is_whitelist"` + EnableTableDeltas bool `yaml:"table_deltas" mapstructure:"table_deltas"` +} + +type Config struct { + Name string `yaml:"name" mapstructure:"name"` + Ship ShipConfig `yaml:"ship" mapstructure:"ship"` + Api string `yaml:"api" mapstructure:"api"` + + Log log.Config `yaml:"log" mapstructure:"log"` + + Cache Cache `yaml:"cache" mapstructure:"cache"` + + Redis RedisConfig `yaml:"redis" mapstructure:"redis"` + MessageCodec string `yaml:"message_codec" mapstructure:"message_codec"` + + AbiCache AbiCache `yaml:"abi_cache" mapstructure:"abi_cache"` + + Telegram TelegramConfig `yaml:"telegram" mapstructure:"telegram"` +} diff --git a/api/redis/publisher.go b/internal/driver/redis/publisher.go similarity index 93% rename from api/redis/publisher.go rename to internal/driver/redis/publisher.go index f37d02b..623d67b 100644 --- a/api/redis/publisher.go +++ b/internal/driver/redis/publisher.go @@ -4,6 +4,7 @@ import ( "context" "github.com/eosswedenorg/thalos/api" + . "github.com/eosswedenorg/thalos/api/redis" "github.com/redis/go-redis/v9" ) diff --git a/api/redis/publisher_test.go b/internal/driver/redis/publisher_test.go similarity index 94% rename from api/redis/publisher_test.go rename to internal/driver/redis/publisher_test.go index fa8e4ab..460daaf 100644 --- a/api/redis/publisher_test.go +++ b/internal/driver/redis/publisher_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/eosswedenorg/thalos/api" + . "github.com/eosswedenorg/thalos/api/redis" "github.com/go-redis/redismock/v9" "github.com/stretchr/testify/assert" diff --git a/api/writer.go b/internal/driver/writer.go similarity index 83% rename from api/writer.go rename to internal/driver/writer.go index 4cfc8b2..e9c2f73 100644 --- a/api/writer.go +++ b/internal/driver/writer.go @@ -1,4 +1,6 @@ -package api +package driver + +import "github.com/eosswedenorg/thalos/api" // Writer interface defines the required methods // to send messages over an channel. @@ -7,7 +9,7 @@ package api type Writer interface { // Write writes a message over a channel. // The message may or may not be buffered depending on the implementation. - Write(channel Channel, payload []byte) error + Write(channel api.Channel, payload []byte) error // Flush writes any buffered messages to the channel. // If the implementation does not support buffering. this is a noop. diff --git a/app/log/HookWriter.go b/internal/log/HookWriter.go similarity index 100% rename from app/log/HookWriter.go rename to internal/log/HookWriter.go diff --git a/app/log/RotatingFile.go b/internal/log/RotatingFile.go similarity index 71% rename from app/log/RotatingFile.go rename to internal/log/RotatingFile.go index 4654747..540a783 100644 --- a/app/log/RotatingFile.go +++ b/internal/log/RotatingFile.go @@ -8,6 +8,8 @@ import ( "time" ) +// Rotating file represents a file that can be rotated when either the file +// becomes to large or to old, whatever comes first type RotatingFile struct { fd *os.File size int64 @@ -21,8 +23,9 @@ func open(filename string) (*os.File, error) { return os.OpenFile(filename, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0o666) } -func NewRotatingFile(filename string, maxSize int64, maxAge time.Duration) (*RotatingFile, error) { - if err := os.MkdirAll(path.Dir(filename), 0o766); err != nil && !os.IsExist(err) { +// Open a new rotating file. +func NewRotatingFile(filename string, opts ...RotatingFileOption) (*RotatingFile, error) { + if err := os.MkdirAll(path.Dir(filename), 0o755); err != nil && !os.IsExist(err) { return nil, err } @@ -36,22 +39,33 @@ func NewRotatingFile(filename string, maxSize int64, maxAge time.Duration) (*Rot return nil, err } - return &RotatingFile{ - fd: fd, - size: stat.Size(), - maxSize: maxSize, - ts: time.Now(), - maxAge: maxAge, - format: "2006-01-02_150405", - }, nil + file := &RotatingFile{ + fd: fd, + size: stat.Size(), + ts: time.Now(), + format: "2006-01-02_150405", + } + + for _, opt := range opts { + opt(file) + } + + return file, nil } +// Open a new rotating file using a config struct. func NewRotatingFileFromConfig(config Config, suffix string) (*RotatingFile, error) { if len(suffix) > 0 { suffix = "_" + suffix } - return NewRotatingFile(config.GetFilePath()+suffix+".log", int64(config.MaxFileSize), config.MaxTime) + filename := config.GetFilePath() + suffix + ".log" + + return NewRotatingFile(filename, + WithMaxAge(config.MaxTime), + WithMaxSize(int64(config.MaxFileSize)), + WithTimestampFormat(config.FileTimestampFormat), + ) } func (w *RotatingFile) newFilename(name string) string { @@ -62,6 +76,7 @@ func (w *RotatingFile) newFilename(name string) string { return fmt.Sprintf("%s-%s%s", name, time.Now().Format(w.format), ext) } +// Get the filename func (w RotatingFile) GetFilename() string { return path.Base(w.fd.Name()) } diff --git a/internal/log/RotatingFileOptions.go b/internal/log/RotatingFileOptions.go new file mode 100644 index 0000000..be7b4c3 --- /dev/null +++ b/internal/log/RotatingFileOptions.go @@ -0,0 +1,25 @@ +package log + +import "time" + +type RotatingFileOption func(*RotatingFile) + +func WithTimestampFormat(value string) RotatingFileOption { + return func(f *RotatingFile) { + if len(value) > 0 { + f.format = value + } + } +} + +func WithMaxSize(value int64) RotatingFileOption { + return func(f *RotatingFile) { + f.maxSize = value + } +} + +func WithMaxAge(value time.Duration) RotatingFileOption { + return func(f *RotatingFile) { + f.maxAge = value + } +} diff --git a/internal/log/config.go b/internal/log/config.go new file mode 100644 index 0000000..2c50cc0 --- /dev/null +++ b/internal/log/config.go @@ -0,0 +1,38 @@ +package log + +import ( + "path" + "time" + + "github.com/eosswedenorg/thalos/internal/types" +) + +// Config represents configuration parameters for a log. +type Config struct { + // Filename where the log is stored. + Filename string `yaml:"filename" mapstructure:"filename"` + + // Directory where the log files are stored. + Directory string `yaml:"directory" mapstructure:"directory"` + + // Timestamp format when rotation files. + FileTimestampFormat string `yaml:"file_timestamp_format" mapstructure:"file_timestamp_format"` + + // Maximum filesize, the log is rotated when this size is exceeded. + MaxFileSize types.Size `yaml:"maxfilesize" mapstructure:"maxfilesize"` + + // Maximum lifetime of the file before it is rotated. + MaxTime time.Duration `yaml:"maxtime" mapstructure:"maxtime"` +} + +func (c Config) GetFilename() string { + return path.Base(c.Filename) +} + +func (c Config) GetDirectory() string { + return path.Clean(c.Directory) +} + +func (c Config) GetFilePath() string { + return path.Join(c.GetDirectory(), c.GetFilename()) +} diff --git a/app/log/config_test.go b/internal/log/config_test.go similarity index 100% rename from app/log/config_test.go rename to internal/log/config_test.go diff --git a/app/log/init.go b/internal/log/init.go similarity index 100% rename from app/log/init.go rename to internal/log/init.go diff --git a/internal/server/message_queue.go b/internal/server/message_queue.go new file mode 100644 index 0000000..da46da5 --- /dev/null +++ b/internal/server/message_queue.go @@ -0,0 +1,74 @@ +package server + +import ( + "errors" + + "github.com/eosswedenorg/thalos/api" + "github.com/eosswedenorg/thalos/api/message" + "github.com/eosswedenorg/thalos/internal/driver" +) + +// MessageQueue takes care of message routing and encoding +type MessageQueue struct { + // Writer to write messages to + writer driver.Writer + + // Encoder to encode messages with + encode message.Encoder +} + +func NewMessageQueue(writer driver.Writer, encoder message.Encoder) MessageQueue { + return MessageQueue{ + writer: writer, + encode: encoder, + } +} + +func (mq MessageQueue) PostHeartbeat(hb message.HeartBeat) error { + return mq.post(hb, api.HeartbeatChannel) +} + +func (mq MessageQueue) PostRollback(rb message.RollbackMessage) error { + return mq.post(rb, api.RollbackChannel) +} + +func (mq MessageQueue) PostTransactionTrace(trace message.TransactionTrace) error { + return mq.post(trace, api.TransactionChannel) +} + +// Post a ActionTrace message to the queue +func (mq MessageQueue) PostAction(act message.ActionTrace) error { + return mq.post(act, + api.ActionChannel{}.Channel(), + api.ActionChannel{Name: act.Name}.Channel(), + api.ActionChannel{Contract: act.Contract}.Channel(), + api.ActionChannel{Name: act.Name, Contract: act.Contract}.Channel(), + ) +} + +func (mq MessageQueue) PostTableDelta(delta message.TableDelta) error { + return mq.post(delta, + api.TableDeltaChannel{}.Channel(), + api.TableDeltaChannel{Name: delta.Name}.Channel(), + ) +} + +func (mq MessageQueue) Flush() error { + return mq.writer.Flush() +} + +func (mq MessageQueue) Close() error { + return mq.writer.Close() +} + +func (mq MessageQueue) post(v interface{}, channels ...api.Channel) error { + payload, err := mq.encode(v) + if err == nil { + for _, channel := range channels { + if w_err := mq.writer.Write(channel, payload); err != nil { + err = errors.Join(w_err) + } + } + } + return err +} diff --git a/internal/server/ship_processor.go b/internal/server/ship_processor.go new file mode 100644 index 0000000..1c1ee39 --- /dev/null +++ b/internal/server/ship_processor.go @@ -0,0 +1,365 @@ +package server + +import ( + "bytes" + "errors" + + "github.com/eosswedenorg/thalos/api/message" + "github.com/eosswedenorg/thalos/internal/abi" + "github.com/eosswedenorg/thalos/internal/driver" + ship_helper "github.com/eosswedenorg/thalos/internal/ship" + "github.com/eosswedenorg/thalos/internal/types" + + log "github.com/sirupsen/logrus" + + shipclient "github.com/eosswedenorg-go/antelope-ship-client" + "github.com/shufflingpixels/antelope-go/chain" + "github.com/shufflingpixels/antelope-go/ship" +) + +// A ShipProcessor will consume messages from a ship stream, convert the messages into +// thalos specific ones, encode them and finally post them to an api.Writer +type ShipProcessor struct { + // The ship stream to process. + shipStream *shipclient.Stream + + // Abi manager used for cacheing + abi *abi.AbiManager + + queue MessageQueue + + // Function for saving state. + saver StateSaver + + // Internal state + state State + + // System contract ("eosio" per default) + syscontract chain.Name + + // ABI Returned from SHIP + shipABI *chain.Abi + + // Action blacklist + blacklist types.Blacklist +} + +// SpawnProcessor creates a new ShipProccessor that consumes the shipclient.Stream passed to it. +func SpawnProccessor(shipStream *shipclient.Stream, loader StateLoader, saver StateSaver, writer driver.Writer, abi *abi.AbiManager, codec message.Codec) *ShipProcessor { + processor := &ShipProcessor{ + saver: saver, + abi: abi, + shipStream: shipStream, + syscontract: chain.N("eosio"), + queue: NewMessageQueue(writer, codec.Encoder), + } + + loader(&processor.state) + + // Attach handlers + shipStream.BlockHandler = processor.processBlock + shipStream.InitHandler = processor.initHandler + + // Needed because if nil, traces/table deltas will not be included in the response from ship. + shipStream.TraceHandler = func(*ship.TransactionTraceArray) {} + shipStream.TableDeltaHandler = func(*ship.TableDeltaArray) {} + + return processor +} + +func (processor *ShipProcessor) FetchDeltas(value bool) { + if value { + // empty callback will signal that traces should be included in the response from ship. + processor.shipStream.TableDeltaHandler = func(*ship.TableDeltaArray) {} + } else { + processor.shipStream.TableDeltaHandler = nil + } +} + +func (processor *ShipProcessor) SetBlacklist(list types.Blacklist) { + processor.blacklist = list +} + +func (processor *ShipProcessor) initHandler(abi *chain.Abi) { + processor.shipABI = abi +} + +// updateAbiFromAction updates the contract abi based on the ship.Action passed. +func (processor *ShipProcessor) updateAbiFromAction(act *chain.Action) error { + set_abi := struct { + Account chain.Name + Abi chain.Bytes + }{} + + if err := act.DecodeInto(&set_abi); err != nil { + return err + } + + abi := chain.Abi{} + decoder := chain.NewDecoder(bytes.NewReader(set_abi.Abi)) + if err := decoder.Decode(&abi); err != nil { + return err + } + return processor.abi.SetAbi(set_abi.Account, &abi) +} + +// Get the current block. +func (processor *ShipProcessor) GetCurrentBlock() uint32 { + return processor.state.CurrentBlock +} + +func (processor *ShipProcessor) processTransactionTrace(log *log.Entry, blockNumber uint32, block *ship.SignedBlock, trace *ship.TransactionTraceV0) { + logger := log.WithField("type", "trace").WithField("tx_id", trace.ID.String()).Dup() + + timestamp := block.BlockHeader.Timestamp.Time().UTC() + + transaction := message.TransactionTrace{ + ID: trace.ID.String(), + BlockNum: blockNumber, + Timestamp: timestamp, + Status: trace.Status.String(), + CPUUsageUS: trace.CPUUsageUS, + NetUsage: trace.NetUsage, + NetUsageWords: uint32(trace.NetUsageWords), + Elapsed: int64(trace.Elapsed), + Scheduled: trace.Scheduled, + Except: trace.Except, + Error: trace.ErrorCode, + } + + // Actions + for _, actionTraceVar := range trace.ActionTraces { + + actionTrace := ship_helper.ToActionTraceV1(actionTraceVar) + actMsg := processor.proccessActionTrace(logger, actionTrace) + if actMsg != nil { + actMsg.TxID = trace.ID.String() + actMsg.BlockNum = blockNumber + actMsg.Timestamp = timestamp + + processor.queue.PostAction(*actMsg) + + transaction.ActionTraces = append(transaction.ActionTraces, *actMsg) + } + } + + if err := processor.queue.PostTransactionTrace(transaction); err != nil { + logger.WithError(err).Error("Failed to post transaction trace") + } +} + +func (processor *ShipProcessor) proccessActionTrace(logger *log.Entry, trace *ship.ActionTraceV1) *message.ActionTrace { + // Check if actions updates an abi. + if trace.Act.Account == processor.syscontract && trace.Act.Name == chain.N("setabi") { + + logger.WithFields(log.Fields{ + "contract": trace.Act.Account, + "action": trace.Act.Name, + }).Debug("Update contract ABI") + + err := processor.updateAbiFromAction(&trace.Act) + if err != nil { + logger.WithError(err).Warn("Failed to update abi") + } + } + + // Check blacklist if we should skip this action + if !processor.blacklist.IsAllowed(trace.Act.Account.String(), trace.Act.Name.String()) { + logger.WithFields(log.Fields{ + "contract": trace.Act.Account, + "action": trace.Act.Name, + }).Debug("Found in blacklist, skipping") + return nil + } + + act := &message.ActionTrace{ + Name: trace.Act.Name.String(), + Contract: trace.Act.Account.String(), + Receiver: trace.Receiver.String(), + FirstReceiver: trace.Act.Account.String() == trace.Receiver.String(), + } + + if trace.Receipt != nil { + receipt := trace.Receipt.V0 + act.Receipt = &message.ActionReceipt{ + Receiver: receipt.Receiver.String(), + ActDigest: receipt.ActDigest.String(), + GlobalSequence: receipt.GlobalSequence, + RecvSequence: receipt.RecvSequence, + CodeSequence: uint32(receipt.CodeSequence), + ABISequence: uint32(receipt.ABISequence), + } + + for _, auth := range receipt.AuthSequence { + act.Receipt.AuthSequence = append(act.Receipt.AuthSequence, message.AccountAuthSequence{ + Account: auth.Account.String(), + Sequence: auth.Sequence, + }) + } + } + + for _, auth := range trace.Act.Authorization { + act.Authorization = append(act.Authorization, message.PermissionLevel{ + Actor: auth.Actor.String(), + Permission: auth.Permission.String(), + }) + } + + logger.WithFields(log.Fields{ + "contract": trace.Act.Account, + "action": trace.Act.Name, + }).Debug("Reading contract ABI") + + ABI, err := processor.abi.GetAbi(trace.Act.Account) + if err == nil { + if act.Data, err = trace.Act.Decode(ABI); err != nil { + logger.WithFields(log.Fields{ + "contract": trace.Act.Account, + "action": trace.Act.Name, + }).WithError(err).Warn("Failed to decode action") + } + } else { + logger.WithField("contract", trace.Act.Account). + WithError(err).Error("Failed to get abi for contract") + } + + return act +} + +func (processor *ShipProcessor) proccessDeltaRows(logger *log.Entry, table_name string, rows []ship.Row) []message.TableDeltaRow { + out := []message.TableDeltaRow{} + for _, row := range rows { + msg, err := processor.proccessDeltaRow(row, table_name) + if err != nil { + logger.WithError(err).Warn("Failed to processs table delta row") + } + out = append(out, msg) + } + return out +} + +func (processor *ShipProcessor) proccessDeltaRow(row ship.Row, table_name string) (message.TableDeltaRow, error) { + msg := message.TableDeltaRow{ + Present: row.Present, + RawData: row.Data, + } + + if processor.shipABI == nil { + return msg, errors.New("No SHIP ABI present") + } + + v, err := processor.shipABI.Decode(bytes.NewReader(row.Data), table_name) + if err != nil { + return msg, errors.New("Failed to decode table delta") + } + data, err := ship_helper.ParseTableDeltaData(v) + if err != nil { + return msg, errors.New("Failed to parse table delta data") + } + + msg.Data = data + + // Decode contract row data + if table_name == "contract_row" { + dec, err := ship_helper.DecodeContractRow(processor.abi, data) + if err != nil { + return msg, errors.New("Failed to decode contract row") + } + msg.Data["value"] = dec + } + return msg, nil +} + +// Callback function called by shipclient.Stream when a new block arrives. +func (processor *ShipProcessor) processBlock(blockResult *ship.GetBlocksResultV0) { + block := ship.SignedBlock{} + blockResult.Block.Unpack(&block) + timestamp := block.BlockHeader.Timestamp.Time().UTC() + blockNumber := blockResult.ThisBlock.BlockNum + + // Check to see if we have a microfork and post a message to + // the rollback channel in that case. + if processor.state.CurrentBlock > 0 && blockNumber < processor.state.CurrentBlock { + + msg := message.RollbackMessage{ + OldBlockNum: processor.state.CurrentBlock, + NewBlockNum: blockResult.ThisBlock.BlockNum, + } + log.WithField("old_block", msg.OldBlockNum). + WithField("new_block", msg.NewBlockNum). + Warn("Fork detected, old_block is greater than new_block") + + if err := processor.queue.PostRollback(msg); err != nil { + log.WithError(err).Error("Failed to write rollback message") + } + } + + processor.state.CurrentBlock = blockNumber + + if blockResult.ThisBlock.BlockNum%100 == 0 { + log.Infof("Current: %d, Head: %d", processor.state.CurrentBlock, blockResult.Head.BlockNum) + } + + if blockResult.ThisBlock.BlockNum%10 == 0 { + hb := message.HeartBeat{ + BlockNum: blockNumber, + LastIrreversibleBlockNum: blockResult.LastIrreversible.BlockNum, + HeadBlockNum: blockResult.Head.BlockNum, + } + if err := processor.queue.PostHeartbeat(hb); err != nil { + log.WithError(err).Error("Failed to write heartbeat message") + } + } + + mainLogger := log.WithField("block", blockNumber).Dup() + + // Process traces + if blockResult.Traces != nil { + unpacked := []ship.TransactionTrace{} + if err := blockResult.Traces.Unpack(&unpacked); err != nil { + mainLogger.WithError(err).Error("Failed to unpack transaction traces") + } else { + for _, trace := range unpacked { + processor.processTransactionTrace(mainLogger, blockNumber, &block, trace.V0) + } + } + } + + // Process deltas + if blockResult.Deltas != nil { + deltas := []ship.TableDelta{} + if err := blockResult.Deltas.Unpack(&deltas); err != nil { + mainLogger.WithError(err).Error("Failed to unpack table deltas") + } else { + logger := mainLogger.WithField("type", "table_delta").Dup() + for _, delta := range deltas { + + msg := message.TableDelta{ + BlockNum: blockNumber, + Timestamp: timestamp, + Name: delta.V0.Name, + Rows: processor.proccessDeltaRows(logger, delta.V0.Name, delta.V0.Rows), + } + + if err := processor.queue.PostTableDelta(msg); err != nil { + logger.WithError(err).Error("Failed to post table delta message") + } + } + } + } + + err := processor.queue.Flush() + if err != nil { + log.WithError(err).Error("Failed to send messages") + } + + err = processor.saver(processor.state) + if err != nil { + log.WithError(err).Error("Failed to save state") + } +} + +// Close closes the writer associated with the processor. +func (processor *ShipProcessor) Close() error { + return processor.queue.Close() +} diff --git a/internal/server/state.go b/internal/server/state.go new file mode 100644 index 0000000..d8312ef --- /dev/null +++ b/internal/server/state.go @@ -0,0 +1,15 @@ +package server + +// State represents thalos runtime state +type State struct { + // Last processed block + CurrentBlock uint32 +} + +type ( + // StateLoader is a function that loads a state. + StateLoader func(*State) + + // StateSaver is a function that saves a state. + StateSaver func(State) error +) diff --git a/internal/ship/action.go b/internal/ship/action.go new file mode 100644 index 0000000..8a25a46 --- /dev/null +++ b/internal/ship/action.go @@ -0,0 +1,27 @@ +package ship + +import ( + "github.com/shufflingpixels/antelope-go/ship" +) + +// convert a ActionTrace to ActionTraceV1 +func ToActionTraceV1(trace *ship.ActionTrace) *ship.ActionTraceV1 { + if trace.V0 != nil { + // convert to v1 + return &ship.ActionTraceV1{ + ActionOrdinal: trace.V0.ActionOrdinal, + CreatorActionOrdinal: trace.V0.CreatorActionOrdinal, + Receipt: trace.V0.Receipt, + Receiver: trace.V0.Receiver, + Act: trace.V0.Act, + ContextFree: trace.V0.ContextFree, + Elapsed: trace.V0.Elapsed, + Console: trace.V0.Console, + AccountRamDeltas: trace.V0.AccountRamDeltas, + Except: trace.V0.Except, + ErrorCode: trace.V0.ErrorCode, + ReturnValue: []byte{}, + } + } + return trace.V1 +} diff --git a/internal/ship/contract_row.go b/internal/ship/contract_row.go new file mode 100644 index 0000000..f980e90 --- /dev/null +++ b/internal/ship/contract_row.go @@ -0,0 +1,37 @@ +package ship + +import ( + "bytes" + + "github.com/eosswedenorg/thalos/internal/abi" + "github.com/mitchellh/mapstructure" + "github.com/shufflingpixels/antelope-go/chain" +) + +type ContractRow struct { + Code chain.Name `mapstructure:"code"` + Scope chain.Name `mapstructure:"scope"` + Table chain.Name `mapstructure:"table"` + PrimaryKey string `mapstructure:"primary_key"` + Payer chain.Name `mapstructure:"payer"` + Value chain.Bytes `mapstructure:"value"` +} + +func ParseContractRow(v map[string]interface{}) (*ContractRow, error) { + out := &ContractRow{} + err := mapstructure.WeakDecode(v, out) + return out, err +} + +func DecodeContractRow(manager *abi.AbiManager, data map[string]any) (any, error) { + row, err := ParseContractRow(data) + if err != nil { + return nil, err + } + + abi, err := manager.GetAbi(row.Code) + if err != nil { + return nil, err + } + return abi.DecodeTable(bytes.NewReader(row.Value), row.Table) +} diff --git a/internal/ship/contract_row_test.go b/internal/ship/contract_row_test.go new file mode 100644 index 0000000..d281732 --- /dev/null +++ b/internal/ship/contract_row_test.go @@ -0,0 +1,32 @@ +package ship_test + +import ( + "testing" + + "github.com/eosswedenorg/thalos/internal/ship" + "github.com/shufflingpixels/antelope-go/chain" + "github.com/stretchr/testify/assert" +) + +func TestContractRow_Parse(t *testing.T) { + expected := &ship.ContractRow{ + Code: chain.N("eosio"), + Scope: chain.N("scope"), + Table: chain.N("accounts"), + PrimaryKey: "1278127812", + Payer: chain.N("account1"), + Value: []byte{0x01, 0x01, 0x02, 0x03}, + } + + actual, err := ship.ParseContractRow(map[string]any{ + "code": uint64(6138663577826885632), + "scope": uint64(13990807175891517440), + "table": uint64(3607749779137757184), + "primary_key": uint32(1278127812), + "payer": uint64(3607749778751881216), + "value": []byte{0x01, 0x01, 0x02, 0x03}, + }) + + assert.NoError(t, err) + assert.Equal(t, expected, actual) +} diff --git a/internal/ship/table_delta.go b/internal/ship/table_delta.go new file mode 100644 index 0000000..9d011d3 --- /dev/null +++ b/internal/ship/table_delta.go @@ -0,0 +1,36 @@ +package ship + +import ( + "fmt" + "reflect" +) + +func parseTableDeltaDataInner(v reflect.Value) reflect.Value { + if IsVariant(v) { + v = v.Index(1) + } + + switch v.Kind() { + case reflect.Interface: + return parseTableDeltaDataInner(v.Elem()) + case reflect.Slice: + for i := 0; i < v.Len(); i++ { + v.Index(i).Set(parseTableDeltaDataInner(v.Index(i))) + } + case reflect.Map: + it := v.MapRange() + for it.Next() { + v.SetMapIndex(it.Key(), parseTableDeltaDataInner(it.Value())) + } + } + + return v +} + +func ParseTableDeltaData(v any) (map[string]interface{}, error) { + iface := parseTableDeltaDataInner(reflect.ValueOf(v)).Interface() + if out, ok := iface.(map[string]interface{}); ok { + return out, nil + } + return nil, fmt.Errorf("data is not an map") +} diff --git a/internal/ship/table_delta_test.go b/internal/ship/table_delta_test.go new file mode 100644 index 0000000..74b1d03 --- /dev/null +++ b/internal/ship/table_delta_test.go @@ -0,0 +1,70 @@ +package ship_test + +import ( + "testing" + + "github.com/eosswedenorg/thalos/internal/ship" + "github.com/stretchr/testify/assert" +) + +func TestParseTableDeltaData(t *testing.T) { + input := []interface{}{ + "resource_limits_state_v0", + map[string]interface{}{ + "average_block_cpu_usage": []interface{}{ + "usage_accumulator_v0", + map[string]interface{}{ + "consumed": 33679, + "last_ordinal": 308855607, + "value_ex": "18525321667", + }, + }, + "average_block_net_usage": []interface{}{ + "usage_accumulator_v0", + map[string]interface{}{ + "consumed": 8107, + "last_ordinal": 308855607, + "value_ex": int64(3854030492), + }, + }, + "slice": []interface{}{ + "generated_transaction_v0", + []interface{}{1, 2, "tree"}, + }, + "single_value": []interface{}{ + "generated_transaction_v0", + uint32(12933729), + }, + "total_cpu_weight": "44811223778385154", + "total_net_weight": "134285012330070718", + "total_ram_bytes": "172065109473", + "virtual_cpu_limit": 206081, + "virtual_net_limit": 1048576000, + }, + } + + expected := map[string]interface{}{ + "average_block_cpu_usage": map[string]interface{}{ + "consumed": 33679, + "last_ordinal": 308855607, + "value_ex": "18525321667", + }, + "average_block_net_usage": map[string]interface{}{ + "consumed": 8107, + "last_ordinal": 308855607, + "value_ex": int64(3854030492), + }, + "slice": []interface{}{1, 2, "tree"}, + "single_value": uint32(12933729), + "total_cpu_weight": "44811223778385154", + "total_net_weight": "134285012330070718", + "total_ram_bytes": "172065109473", + "virtual_cpu_limit": 206081, + "virtual_net_limit": 1048576000, + } + + actual, err := ship.ParseTableDeltaData(input) + + assert.NoError(t, err) + assert.Equal(t, expected, actual) +} diff --git a/internal/ship/variant.go b/internal/ship/variant.go new file mode 100644 index 0000000..56ca7fb --- /dev/null +++ b/internal/ship/variant.go @@ -0,0 +1,93 @@ +package ship + +import "reflect" + +func IsVariantName(name string) bool { + validVariants := []string{ + "get_status_request_v0", + "block_position", + "get_status_result_v0", + "get_blocks_request_v0", + "get_blocks_ack_request_v0", + "get_blocks_result_v0", + "row", + "table_delta_v0", + "action", + "account_auth_sequence", + "action_receipt_v0", + "account_delta", + "action_trace_v0", + "partial_transaction_v0", + "transaction_trace_v0", + "packed_transaction", + "transaction_receipt_header", + "transaction_receipt", + "extension", + "block_header", + "signed_block_header", + "signed_block", + "transaction_header", + "transaction", + "code_id", + "account_v0", + "account_metadata_v0", + "code_v0", + "contract_table_v0", + "contract_row_v0", + "contract_index64_v0", + "contract_index128_v0", + "contract_index256_v0", + "contract_index_double_v0", + "contract_index_long_double_v0", + "producer_key", + "producer_schedule", + "block_signing_authority_v0", + "producer_authority", + "producer_authority_schedule", + "chain_config_v0", + "global_property_v0", + "global_property_v1", + "generated_transaction_v0", + "activated_protocol_feature_v0", + "protocol_state_v0", + "key_weight", + "permission_level", + "permission_level_weight", + "wait_weight", + "authority", + "permission_v0", + "permission_link_v0", + "resource_limits_v0", + "usage_accumulator_v0", + "resource_usage_v0", + "resource_limits_state_v0", + "resource_limits_ratio_v0", + "elastic_limit_parameters_v0", + "resource_limits_config_v0", + } + + for _, v := range validVariants { + if v == name { + return true + } + } + return false +} + +// Check if a structure is a variant type. +// This is not 100% accurate. As variant types comes +// as a simple slice with the types name in the first index +// and the value as the second. +// So there could be some edge cases where this structure is actual data +// and not a variant type although should be super rare. +func IsVariant(v reflect.Value) bool { + if v.Kind() != reflect.Slice || v.Len() != 2 { + return false + } + + for v = v.Index(0); v.Kind() == reflect.Interface || v.Kind() == reflect.Pointer; v = v.Elem() { + // Intentionally empty + } + + return v.Kind() == reflect.String && IsVariantName(v.String()) +} diff --git a/internal/types/blacklist.go b/internal/types/blacklist.go new file mode 100644 index 0000000..4606c5e --- /dev/null +++ b/internal/types/blacklist.go @@ -0,0 +1,59 @@ +package types + +const BlacklistWildcard = "*" + +type Blacklist struct { + table map[string][]string + isWhitelist bool +} + +func NewBlacklist(entries map[string][]string) *Blacklist { + return &Blacklist{ + table: entries, + } +} + +func (bl *Blacklist) SetWhitelist(value bool) *Blacklist { + bl.isWhitelist = value + return bl +} + +func (bl Blacklist) Empty() bool { + return len(bl.table) < 1 +} + +func (bl *Blacklist) Add(contract string, action string) { + if bl.table == nil { + bl.table = map[string][]string{} + } + + if len(bl.table[contract]) < 1 { + bl.table[contract] = []string{} + } + bl.table[contract] = append(bl.table[contract], action) +} + +func (bl Blacklist) list(contracts ...string) [][]string { + ret := [][]string{} + for _, contract := range contracts { + if v, ok := bl.table[contract]; ok { + ret = append(ret, v) + } + } + return ret +} + +func (bl Blacklist) IsAllowed(contract string, action string) bool { + for _, v := range bl.list(contract, BlacklistWildcard) { + for _, act := range v { + if act == action || act == BlacklistWildcard { + return bl.isWhitelist == true + } + } + } + return bl.isWhitelist == false +} + +func (bl Blacklist) IsDenied(contract string, action string) bool { + return bl.IsAllowed(contract, action) +} diff --git a/i