1
0
Fork 0
mirror of https://github.com/eosswedenorg/thalos synced 2026-11-19 07:06:45 +01:00

refactor: migrate blacklist to internal/filter list

This commit is contained in:
Henrik Hautakoski 2026-08-28 10:08:01 +02:00
parent 09284231ed
commit b9ec306c7a
10 changed files with 260 additions and 223 deletions

View file

@ -24,6 +24,7 @@ import (
"github.com/eosswedenorg/thalos/internal/cache"
"github.com/eosswedenorg/thalos/internal/config"
driver "github.com/eosswedenorg/thalos/internal/driver/redis"
"github.com/eosswedenorg/thalos/internal/filter"
. "github.com/eosswedenorg/thalos/internal/log"
. "github.com/eosswedenorg/thalos/internal/server"
"github.com/nikoksr/notify"
@ -237,7 +238,10 @@ func GetConfig(flags *pflag.FlagSet) (*config.Config, error) {
}
}
cfg.Ship.Blacklist.SetWhitelist(cfg.Ship.BlacklistIsWhitelist)
cfg.Ship.Blacklist.SetMode(filter.Exclude)
if cfg.Ship.BlacklistIsWhitelist {
cfg.Ship.Blacklist.SetMode(filter.Include)
}
return cfg, nil
}

View file

@ -8,7 +8,7 @@ import (
"reflect"
"strings"
"github.com/eosswedenorg/thalos/internal/types"
"github.com/eosswedenorg/thalos/internal/filter"
"github.com/mitchellh/mapstructure"
"github.com/spf13/pflag"
"github.com/spf13/viper"
@ -124,8 +124,8 @@ func (b *Builder) Build() (*Config, error) {
mapstructure.StringToTimeDurationHookFunc(),
mapstructure.StringToSliceHookFunc(","),
func(f reflect.Type, t reflect.Type, in interface{}) (interface{}, error) {
if t == reflect.TypeOf(types.Blacklist{}) {
return decodeIntoBlacklist(in)
if t == reflect.TypeOf(filter.List{}) {
return decodeIntoFilter(in)
}
return in, nil
},
@ -139,16 +139,24 @@ func (b *Builder) Build() (*Config, error) {
return &conf, nil
}
// Decode a generic structure into types.Blacklist
func decodeIntoBlacklist(in any) (*types.Blacklist, error) {
// Decode a generic structure into filter.List.
func decodeIntoFilter(in any) (*filter.List, error) {
switch v := in.(type) {
// Standard map structure.
case map[string]any:
return blacklistParseMap(v)
list, err := filterParseMap(v)
if err != nil {
return nil, err
}
return list.SetMode(filter.Exclude), nil
// slice of "contract:action" pairs. Usually from CLI
case []string:
return blacklistParseSlice(v)
list, err := filterParseSlice(v)
if err != nil {
return nil, err
}
return list.SetMode(filter.Exclude), nil
// Sometimes we have a slice of interfaces.
// Need to convert it to a slice of strings.
@ -157,28 +165,32 @@ func decodeIntoBlacklist(in any) (*types.Blacklist, error) {
if err != nil {
return nil, err
}
return blacklistParseSlice(sv)
list, err := filterParseSlice(sv)
if err != nil {
return nil, err
}
return list.SetMode(filter.Exclude), nil
}
return nil, fmt.Errorf("Must be a string slice")
}
// Blacklist map parser
func blacklistParseMap(in map[string]any) (*types.Blacklist, error) {
list := &types.Blacklist{}
// Filter map parser
func filterParseMap(in map[string]any) (*filter.List, error) {
list := &filter.List{}
for contract, value := range in {
if err := blacklistParseValue(list, contract, value); err != nil {
if err := filterParseValue(list, contract, value); err != nil {
return nil, err
}
}
return list, nil
}
func blacklistParseValue(list *types.Blacklist, contract string, in any) error {
func filterParseValue(list *filter.List, contract string, in any) error {
switch value := in.(type) {
case map[string]any:
for nested, nestedValue := range value {
if err := blacklistParseValue(list, contract+"."+nested, nestedValue); err != nil {
if err := filterParseValue(list, contract+"."+nested, nestedValue); err != nil {
return err
}
}
@ -189,7 +201,7 @@ func blacklistParseValue(list *types.Blacklist, contract string, in any) error {
if !ok {
return fmt.Errorf("Must be a string slice")
}
if err := blacklistParseValue(list, contract+"."+nestedKey, nestedValue); err != nil {
if err := filterParseValue(list, contract+"."+nestedKey, nestedValue); err != nil {
return err
}
}
@ -208,9 +220,9 @@ func blacklistParseValue(list *types.Blacklist, contract string, in any) error {
return nil
}
// Blacklist slice parser
func blacklistParseSlice(in []string) (*types.Blacklist, error) {
list := &types.Blacklist{}
// Filter slice parser
func filterParseSlice(in []string) (*filter.List, error) {
list := &filter.List{}
for _, i := range in {
var action string
parts := strings.SplitN(i, ":", 2)

View file

@ -6,8 +6,8 @@ import (
"time"
shipclient "github.com/eosswedenorg-go/antelope-ship-client"
"github.com/eosswedenorg/thalos/internal/filter"
"github.com/eosswedenorg/thalos/internal/log"
"github.com/eosswedenorg/thalos/internal/types"
"github.com/karlseguin/typed"
"github.com/stretchr/testify/require"
)
@ -41,15 +41,15 @@ func TestBuilder(t *testing.T) {
EndBlockNum: 23872222,
IrreversibleOnly: true,
MaxMessagesInFlight: 1337,
Blacklist: *types.NewBlacklist(map[string][]string{
Blacklist: *filter.New(map[string][]string{
"eosio": {"noop"},
"contract": {"skip1", "skip2"},
}),
}).SetMode(filter.Exclude),
BlacklistIsWhitelist: true,
TableDeltaWhitelist: *types.NewBlacklist(map[string][]string{
TableDeltaWhitelist: *filter.New(map[string][]string{
"eosio.token": {"accounts"},
"mycontract": {"*"},
}),
}).SetMode(filter.Exclude),
},
Telegram: TelegramConfig{
Id: "110201543:AAHdqTcvCH1vGWJxfSeofSAs0K5PALDsaw",
@ -137,6 +137,8 @@ func TestBuilder_WithDefaultConfig(t *testing.T) {
EndBlockNum: shipclient.NULL_BLOCK_NUMBER,
MaxMessagesInFlight: 10,
EnableTableDeltas: true,
Blacklist: *filter.New(nil).SetMode(filter.Exclude),
TableDeltaWhitelist: *filter.New(nil).SetMode(filter.Exclude),
},
Redis: RedisConfig{
Addr: "127.0.0.1:6379",
@ -213,15 +215,15 @@ func TestBuilder_Flags(t *testing.T) {
MaxMessagesInFlight: 98,
IrreversibleOnly: true,
Chain: "wax",
Blacklist: *types.NewBlacklist(map[string][]string{
Blacklist: *filter.New(map[string][]string{
"contract": {"action1", "action2"},
"contract2": {"action1"},
}),
}).SetMode(filter.Exclude),
BlacklistIsWhitelist: true,
TableDeltaWhitelist: *types.NewBlacklist(map[string][]string{
TableDeltaWhitelist: *filter.New(map[string][]string{
"eosio.token": {"accounts"},
"mycontract": {"*"},
}),
}).SetMode(filter.Exclude),
},
Telegram: TelegramConfig{
Id: "72983126312982618",
@ -243,11 +245,11 @@ func TestBuilder_Flags(t *testing.T) {
func TestBuilder_BlacklistSlice(t *testing.T) {
expected := Config{
Ship: ShipConfig{
Blacklist: *types.NewBlacklist(map[string][]string{
Blacklist: *filter.New(map[string][]string{
"contract": {"action"},
"contract2": {"action2"},
"contract3": {"*"},
}),
}).SetMode(filter.Exclude),
},
}

View file

@ -3,8 +3,8 @@ package config
import (
"time"
"github.com/eosswedenorg/thalos/internal/filter"
"github.com/eosswedenorg/thalos/internal/log"
"github.com/eosswedenorg/thalos/internal/types"
"github.com/karlseguin/typed"
)
@ -31,16 +31,16 @@ type AbiCache struct {
}
type ShipConfig struct {
Url string `yaml:"url" mapstructure:"url"`
IrreversibleOnly bool `yaml:"irreversible_only" mapstructure:"irreversible_only"`
MaxMessagesInFlight uint32 `yaml:"max_messages_in_flight" mapstructure:"max_messages_in_flight"`
StartBlockNum uint32 `yaml:"start_block_num" mapstructure:"start_block_num"`
EndBlockNum uint32 `yaml:"end_block_num" mapstructure:"end_block_num"`
Chain string `yaml:"chain" mapstructure:"chain"`
Blacklist types.Blacklist `yaml:"blacklist" mapstructure:"blacklist"`
BlacklistIsWhitelist bool `yaml:"blacklist_is_whitelist" mapstructure:"blacklist_is_whitelist"`
EnableTableDeltas bool `yaml:"table_deltas" mapstructure:"table_deltas"`
TableDeltaWhitelist types.Blacklist `yaml:"table_delta_whitelist" mapstructure:"table_delta_whitelist"`
Url string `yaml:"url" mapstructure:"url"`
IrreversibleOnly bool `yaml:"irreversible_only" mapstructure:"irreversible_only"`
MaxMessagesInFlight uint32 `yaml:"max_messages_in_flight" mapstructure:"max_messages_in_flight"`
StartBlockNum uint32 `yaml:"start_block_num" mapstructure:"start_block_num"`
EndBlockNum uint32 `yaml:"end_block_num" mapstructure:"end_block_num"`
Chain string `yaml:"chain" mapstructure:"chain"`
Blacklist filter.List `yaml:"blacklist" mapstructure:"blacklist"`
BlacklistIsWhitelist bool `yaml:"blacklist_is_whitelist" mapstructure:"blacklist_is_whitelist"`
EnableTableDeltas bool `yaml:"table_deltas" mapstructure:"table_deltas"`
TableDeltaWhitelist filter.List `yaml:"table_delta_whitelist" mapstructure:"table_delta_whitelist"`
}
type Config struct {

76
internal/filter/filter.go Normal file
View file

@ -0,0 +1,76 @@
package filter
const wildcard = "*"
type Mode uint8
const (
Include Mode = iota // whitelist: match = included, default = excluded
Exclude // blacklist: match = excluded, default = included
)
type List struct {
table map[string][]string
mode Mode
}
func New(entries map[string][]string) *List {
return &List{
table: entries,
mode: Include,
}
}
func (l *List) SetMode(mode Mode) *List {
l.mode = mode
return l
}
func (l List) Mode() Mode {
return l.mode
}
func (l List) Empty() bool {
return len(l.table) < 1
}
func (l *List) Add(contract string, action string) *List {
if l.table == nil {
l.table = map[string][]string{}
}
l.table[contract] = append(l.table[contract], action)
return l
}
func (l List) matches(contracts ...string) [][]string {
ret := [][]string{}
for _, contract := range contracts {
if v, ok := l.table[contract]; ok {
ret = append(ret, v)
}
}
return ret
}
func (l List) In(contract string, action string) bool {
for _, v := range l.matches(contract, wildcard) {
for _, act := range v {
if act == action || act == wildcard {
return true
}
}
}
return false
}
func (l List) IsIncluded(contract string, action string) bool {
res := l.In(contract, action)
if l.mode == Exclude {
res = !res
}
return res
}
func (l List) IsExcluded(contract string, action string) bool {
return !l.IsIncluded(contract, action)
}

View file

@ -0,0 +1,113 @@
package filter
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestList_Empty(t *testing.T) {
list := List{
table: map[string][]string{},
}
require.True(t, list.Empty())
list.Add("contract", "action1")
require.False(t, list.Empty())
}
func TestList_Add(t *testing.T) {
list := List{
table: map[string][]string{},
}
list.Add("contract", "action1")
list.Add("contract", "action2")
list.Add("contract2", "action1")
expected := List{
table: map[string][]string{
"contract": {"action1", "action2"},
"contract2": {"action1"},
},
}
require.Equal(t, expected, list)
}
func TestList_IsIncluded_ExcludeMode(t *testing.T) {
list := List{
table: map[string][]string{
"mycontract": {"myaction", "noop"},
},
mode: Exclude,
}
require.False(t, list.IsIncluded("mycontract", "myaction"))
require.False(t, list.IsIncluded("mycontract", "noop"))
require.True(t, list.IsIncluded("mycontract", "xxx"))
require.True(t, list.IsIncluded("xxx", "yyy"))
}
func TestList_IsIncludedWildcard_ExcludeMode(t *testing.T) {
list := List{
table: map[string][]string{
"mycontract": {"*"},
"*": {"action1", "action2"},
"evilcontract": {"evilaction"},
},
mode: Exclude,
}
require.False(t, list.IsIncluded("mycontract", "myaction"))
require.False(t, list.IsIncluded("mycontract", "noop"))
require.False(t, list.IsIncluded("mycontract", "xxx"))
// Wildcard contract
require.False(t, list.IsIncluded("somecontract", "action1"))
require.False(t, list.IsIncluded("someothercontract", "action1"))
require.False(t, list.IsIncluded("randomcontract", "action2"))
require.False(t, list.IsIncluded("evilcontract", "action2"))
require.False(t, list.IsIncluded("evilcontract", "evilaction"))
require.True(t, list.IsIncluded("xxx", "yyy"))
require.True(t, list.IsIncluded("evilcontract", "alloweaction"))
}
func TestList_IncludeMode(t *testing.T) {
list := List{
table: map[string][]string{
"mycontract": {"myaction", "noop"},
"*": {"goodaction1", "goodaction2"},
},
mode: Include,
}
require.True(t, list.IsIncluded("mycontract", "myaction"))
require.True(t, list.IsIncluded("mycontract", "noop"))
// Wildcard contract
require.True(t, list.IsIncluded("mycontract", "goodaction1"))
require.True(t, list.IsIncluded("someothercontract", "goodaction2"))
require.False(t, list.IsIncluded("mycontract", "xxx"))
require.False(t, list.IsIncluded("xxx", "yyy"))
}
func TestList_IsExcluded(t *testing.T) {
list := List{
table: map[string][]string{
"mycontract": {"myaction", "noop"},
},
mode: Exclude,
}
require.True(t, list.IsExcluded("mycontract", "myaction"))
require.False(t, list.IsExcluded("mycontract", "randomaction"))
list.SetMode(Include)
require.False(t, list.IsExcluded("mycontract", "myaction"))
require.True(t, list.IsExcluded("mycontract", "randomaction"))
}

View file

@ -7,8 +7,8 @@ import (
"github.com/eosswedenorg/thalos/api/message"
"github.com/eosswedenorg/thalos/internal/abi"
"github.com/eosswedenorg/thalos/internal/driver"
"github.com/eosswedenorg/thalos/internal/filter"
ship_helper "github.com/eosswedenorg/thalos/internal/ship"
"github.com/eosswedenorg/thalos/internal/types"
log "github.com/sirupsen/logrus"
@ -41,10 +41,10 @@ type ShipProcessor struct {
shipABI *chain.Abi
// Action blacklist
blacklist types.Blacklist
blacklist filter.List
// Optional whitelist for contract_row table delta contract+table names.
tableDeltaWhitelist types.Blacklist
tableDeltaWhitelist filter.List
}
// SpawnProcessor creates a new ShipProccessor that consumes the shipclient.Stream passed to it.
@ -79,12 +79,12 @@ func (processor *ShipProcessor) FetchDeltas(value bool) {
}
}
func (processor *ShipProcessor) SetBlacklist(list types.Blacklist) {
func (processor *ShipProcessor) SetBlacklist(list filter.List) {
processor.blacklist = list
}
func (processor *ShipProcessor) SetTableDeltaWhitelist(list types.Blacklist) {
list.SetWhitelist(true)
func (processor *ShipProcessor) SetTableDeltaWhitelist(list filter.List) {
list.SetMode(filter.Include)
processor.tableDeltaWhitelist = list
}
@ -138,7 +138,7 @@ func (processor *ShipProcessor) filterWhitelistedContractRows(rows []message.Tab
continue
}
if processor.tableDeltaWhitelist.IsAllowed(contract, table) {
if processor.tableDeltaWhitelist.IsIncluded(contract, table) {
out = append(out, row)
}
}
@ -230,7 +230,7 @@ func (processor *ShipProcessor) proccessActionTrace(logger *log.Entry, trace *sh
}
// Check blacklist if we should skip this action
if !processor.blacklist.IsAllowed(trace.Act.Account.String(), trace.Act.Name.String()) {
if !processor.blacklist.IsIncluded(trace.Act.Account.String(), trace.Act.Name.String()) {
logger.WithFields(log.Fields{
"contract": trace.Act.Account,
"action": trace.Act.Name,

View file

@ -4,7 +4,7 @@ import (
"testing"
"github.com/eosswedenorg/thalos/api/message"
"github.com/eosswedenorg/thalos/internal/types"
"github.com/eosswedenorg/thalos/internal/filter"
"github.com/shufflingpixels/antelope-go/chain"
"github.com/stretchr/testify/require"
)
@ -15,7 +15,7 @@ func TestShipProcessor_ShouldProcessTableDeltaName(t *testing.T) {
require.True(t, processor.shouldProcessTableDeltaName("contract_row"))
require.True(t, processor.shouldProcessTableDeltaName("resource_limits"))
processor.SetTableDeltaWhitelist(*types.NewBlacklist(map[string][]string{
processor.SetTableDeltaWhitelist(*filter.New(map[string][]string{
"eosio.token": {"accounts"},
}))
@ -25,7 +25,7 @@ func TestShipProcessor_ShouldProcessTableDeltaName(t *testing.T) {
func TestShipProcessor_FilterWhitelistedContractRows(t *testing.T) {
processor := &ShipProcessor{}
processor.SetTableDeltaWhitelist(*types.NewBlacklist(map[string][]string{
processor.SetTableDeltaWhitelist(*filter.New(map[string][]string{
"eosio.token": {"accounts"},
"eosio": {"*"},
}))
@ -50,7 +50,7 @@ func TestShipProcessor_FilterWhitelistedContractRows(t *testing.T) {
func TestShipProcessor_FilterWhitelistedContractRows_StringerCode(t *testing.T) {
processor := &ShipProcessor{}
processor.SetTableDeltaWhitelist(*types.NewBlacklist(map[string][]string{
processor.SetTableDeltaWhitelist(*filter.New(map[string][]string{
"eosio.token": {"accounts"},
}))

View file

@ -1,59 +0,0 @@
package types
const BlacklistWildcard = "*"
type Blacklist struct {
table map[string][]string
isWhitelist bool
}
func NewBlacklist(entries map[string][]string) *Blacklist {
return &Blacklist{
table: entries,
}
}
func (bl *Blacklist) SetWhitelist(value bool) *Blacklist {
bl.isWhitelist = value
return bl
}
func (bl Blacklist) Empty() bool {
return len(bl.table) < 1
}
func (bl *Blacklist) Add(contract string, action string) {
if bl.table == nil {
bl.table = map[string][]string{}
}
if len(bl.table[contract]) < 1 {
bl.table[contract] = []string{}
}
bl.table[contract] = append(bl.table[contract], action)
}
func (bl Blacklist) list(contracts ...string) [][]string {
ret := [][]string{}
for _, contract := range contracts {
if v, ok := bl.table[contract]; ok {
ret = append(ret, v)
}
}
return ret
}
func (bl Blacklist) IsAllowed(contract string, action string) bool {
for _, v := range bl.list(contract, BlacklistWildcard) {
for _, act := range v {
if act == action || act == BlacklistWildcard {
return bl.isWhitelist == true
}
}
}
return bl.isWhitelist == false
}
func (bl Blacklist) IsDenied(contract string, action string) bool {
return !bl.IsAllowed(contract, action)
}

View file

@ -1,111 +0,0 @@
package types
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestBlacklist_Empty(t *testing.T) {
bl := Blacklist{
table: map[string][]string{},
}
require.True(t, bl.Empty())
bl.Add("contract", "action1")
require.False(t, bl.Empty())
}
func TestBlacklist_Add(t *testing.T) {
bl := Blacklist{
table: map[string][]string{},
}
bl.Add("contract", "action1")
bl.Add("contract", "action2")
bl.Add("contract2", "action1")
expected := Blacklist{
table: map[string][]string{
"contract": {"action1", "action2"},
"contract2": {"action1"},
},
}
require.Equal(t, expected, bl)
}
func TestBlacklist_IsAllowed(t *testing.T) {
bl := Blacklist{
table: map[string][]string{
"mycontract": {"myaction", "noop"},
},
}
require.False(t, bl.IsAllowed("mycontract", "myaction"))
require.False(t, bl.IsAllowed("mycontract", "noop"))
require.True(t, bl.IsAllowed("mycontract", "xxx"))
require.True(t, bl.IsAllowed("xxx", "yyy"))
}
func TestBlacklist_IsAllowedWildcard(t *testing.T) {
bl := Blacklist{
table: map[string][]string{
"mycontract": {"*"},
"*": {"action1", "action2"},
"evilcontract": {"evilaction"},
},
}
require.False(t, bl.IsAllowed("mycontract", "myaction"))
require.False(t, bl.IsAllowed("mycontract", "noop"))
require.False(t, bl.IsAllowed("mycontract", "xxx"))
// Wildcard contract
require.False(t, bl.IsAllowed("somecontract", "action1"))
require.False(t, bl.IsAllowed("someothercontract", "action1"))
require.False(t, bl.IsAllowed("randomcontract", "action2"))
require.False(t, bl.IsAllowed("evilcontract", "action2"))
require.False(t, bl.IsAllowed("evilcontract", "evilaction"))
require.True(t, bl.IsAllowed("xxx", "yyy"))
require.True(t, bl.IsAllowed("evilcontract", "alloweaction"))
}
func TestBlacklist_Whitelist(t *testing.T) {
bl := Blacklist{
table: map[string][]string{
"mycontract": {"myaction", "noop"},
"*": {"goodaction1", "goodaction2"},
},
}
bl.SetWhitelist(true)
require.True(t, bl.IsAllowed("mycontract", "myaction"))
require.True(t, bl.IsAllowed("mycontract", "noop"))
// Wildcard contract
require.True(t, bl.IsAllowed("mycontract", "goodaction1"))
require.True(t, bl.IsAllowed("someothercontract", "goodaction2"))
require.False(t, bl.IsAllowed("mycontract", "xxx"))
require.False(t, bl.IsAllowed("xxx", "yyy"))
}
func TestBlacklist_IsDenied(t *testing.T) {
bl := Blacklist{
table: map[string][]string{
"mycontract": {"myaction", "noop"},
},
}
require.True(t, bl.IsDenied("mycontract", "myaction"))
require.False(t, bl.IsDenied("mycontract", "randomaction"))
bl.SetWhitelist(true)
require.False(t, bl.IsDenied("mycontract", "myaction"))
require.True(t, bl.IsDenied("mycontract", "randomaction"))
}