rename transport to api

api/channel.go

@@ -0,0 +1,61 @@
+package api
+
+import (
+	"strings"
+)
+
+// Channel is just a wrapper around string slice
+type Channel []string
+
+func (c *Channel) Append(name ...string) {
+	*c = append(*c, name...)
+}
+
+func (c Channel) Format(delimiter string) string {
+	return strings.Join(c, delimiter)
+}
+
+func (c Channel) String() string {
+	return c.Format("/")
+}
+
+// Check if two channels are equal
+func (c Channel) Is(other Channel) bool {
+	if len(c) != len(other) {
+		return false
+	}
+
+	for i, item := range c {
+		if item != other[i] {
+			return false
+		}
+	}
+
+	return true
+}
+
+// Define channels without any variables.
+var (
+	TransactionChannel = Channel{"transaction"}
+	HeartbeatChannel   = Channel{"heartbeat"}
+)
+
+// Action Channel
+type Action struct {
+	Name     string
+	Contract string
+}
+
+func (a Action) Channel() Channel {
+	ch := Channel{"actions"}
+
+	if len(a.Contract) > 0 {
+		ch.Append("contract", a.Contract)
+	}
+
+	if len(a.Name) > 0 {
+		ch.Append("name", a.Name)
+	}
+
+	return ch
+}

api/channel_test.go

@@ -0,0 +1,111 @@
+package api
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestChannel_Append(t *testing.T) {
+	tests := []struct {
+		name     string
+		arg      string
+		obj      Channel
+		expected Channel
+	}{
+		{"One", "one", Channel{}, Channel{"one"}},
+		{"More", "more", Channel{"one", "two"}, Channel{"one", "two", "more"}},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			tt.obj.Append(tt.arg)
+			if reflect.DeepEqual(tt.obj, tt.expected) == false {
+				t.Errorf("Channel.Append() expected %v, got %v", tt.expected, tt.obj)
+			}
+		})
+	}
+}
+
+func TestChannel_Is(t *testing.T) {
+	tests := []struct {
+		name string
+		a    Channel
+		b    Channel
+		want bool
+	}{
+		{"Empty valid", Channel{}, Channel{}, true},
+		{"Valid #1", Channel{"one"}, Channel{"one"}, true},
+		{"Valid #2", Channel{"one", "two"}, Channel{"one", "two"}, true},
+		{"Invalid #1", Channel{"one"}, Channel{"one", "two"}, false},
+		{"Invalid #2", Channel{"one", "three"}, Channel{"one", "two"}, false},
+		{"Invalid #3", Channel{"two", "one"}, Channel{"one", "two"}, false},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := tt.a.Is(tt.b); got != tt.want {
+				t.Errorf("a.Is(b) = %v, want %v", got, tt.want)
+			}
+
+			if got := tt.b.Is(tt.a); got != tt.want {
+				t.Errorf("b.Is(a) = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestChannel_Format(t *testing.T) {
+	tests := []struct {
+		name  string
+		c     Channel
+		delim string
+		want  string
+	}{
+		{"Empty", Channel{}, ":", ""},
+		{"Alot#1", Channel{"one", "two", "three"}, "-", "one-two-three"},
+		{"Alot#2", Channel{"first", "second"}, ":", "first:second"},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := tt.c.Format(tt.delim); got != tt.want {
+				t.Errorf("Channel.Format() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestChannel_String(t *testing.T) {
+	tests := []struct {
+		name string
+		c    Channel
+		want string
+	}{
+		{"Empty", Channel{}, ""},
+		{"Alot", Channel{"one", "two", "three"}, "one/two/three"},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := tt.c.String(); got != tt.want {
+				t.Errorf("Channel.String() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestAction_Channel(t *testing.T) {
+	tests := []struct {
+		name   string
+		action Action
+		want   Channel
+	}{
+		{"Empty", Action{}, Channel{"actions"}},
+		{"Contract", Action{Contract: "mycontract"}, Channel{"actions", "contract", "mycontract"}},
+		{"Action", Action{Name: "myaction"}, Channel{"actions", "name", "myaction"}},
+		{"ContractAndName", Action{Contract: "mycontract", Name: "myaction"}, Channel{"actions", "contract", "mycontract", "name", "myaction"}},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := tt.action.Channel(); !got.Is(tt.want) {
+				t.Errorf("ActionChannel.String() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

api/message/encoding.go

@@ -0,0 +1,13 @@
+package message
+
+// Encoder is a function that can encode a object to the encoded format.
+type Encoder func(any) ([]byte, error)
+
+// Decoder is a function that can decode a format into an object
+type Decoder func([]byte, any) error
+
+// Codec is a type that can has a matching Encoder and Decoder function.
+type Codec struct {
+	Encoder Encoder
+	Decoder Decoder
+}

api/message/types.go

@@ -0,0 +1,21 @@
+package message
+
+type HearthBeat struct {
+	BlockNum                 uint32 `json:"blocknum"`
+	HeadBlockNum             uint32 `json:"head_blocknum"`
+	LastIrreversibleBlockNum uint32 `json:"last_irreversible_blocknum"`
+}
+
+type ActionTrace struct {
+	TxID string `json:"tx_id"`
+
+	// Action name
+	Name string `json:"name"`
+
+	// Contract account.
+	Contract string `json:"contract"`
+
+	Receiver string      `json:"receiver"`
+	Data     interface{} `json:"data"`
+	HexData  string      `json:"hex_data"`
+}

api/reader.go

@@ -0,0 +1,17 @@
+package api
+
+// Reader interface defines the required method
+// to read a message from an channel.
+//
+// This is a low-level interface typically implemented by transport drivers.
+type Reader interface {
+	// Read a message from a channel.
+	// Read may block until a message is ready or an error occured.
+	//
+	// This function should be designed to handle concurrent calls. eg. thread safe.
+	Read(channel Channel) ([]byte, error)
+
+	// Close closes the reader
+	// Any blocked Read operations will be unblocked.
+	Close() error
+}

api/redis_common/key.go

@@ -0,0 +1,19 @@
+package redis_common
+
+import (
+	"fmt"
+
+	"thalos/api"
+)
+
+// Key consists of a namespace and a channel.
+// And is encoded to a string in this format: `<namespace>::<channel>`
+
+type Key struct {
+	NS      Namespace
+	Channel api.Channel
+}
+
+func (k Key) String() string {
+	return fmt.Sprintf("%s::%s", k.NS, k.Channel.Format("/"))
+}

api/redis_common/key_test.go

@@ -0,0 +1,35 @@
+package redis_common
+
+import (
+	"testing"
+
+	"thalos/api"
+)
+
+func TestKey_String(t *testing.T) {
+	type fields struct {
+		NS      Namespace
+		Channel api.Channel
+	}
+	tests := []struct {
+		name   string
+		fields fields
+		want   string
+	}{
+		{"Empty", fields{NS: Namespace{}, Channel: api.Channel{}}, "ship::0000000000000000000000000000000000000000000000000000000000000000::"},
+		{"Transactions", fields{NS: Namespace{ChainID: "id"}, Channel: api.Channel{"transactions"}}, "ship::id::transactions"},
+		{"Nested", fields{NS: Namespace{ChainID: "id"}, Channel: api.Channel{"one.two"}}, "ship::id::one.two"},
+		{"Action", fields{NS: Namespace{ChainID: "id"}, Channel: api.Action{Contract: "mycontract"}.Channel()}, "ship::id::actions/contract/mycontract"},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			k := Key{
+				NS:      tt.fields.NS,
+				Channel: tt.fields.Channel,
+			}
+			if got := k.String(); got != tt.want {
+				t.Errorf("Key.String() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

api/redis_common/namespace.go

@@ -0,0 +1,47 @@
+package redis_common
+
+import (
+	"strings"
+
+	"thalos/api"
+)
+
+const (
+	// Default prefix to use when none is set.
+	defaultPrefix = "ship"
+
+	// We need to have some chain_id, so if no one is specified.
+	// we use a "null" id that is all zeros.
+	nullChain = "0000000000000000000000000000000000000000000000000000000000000000"
+)
+
+// Namespace type.
+//
+// Contains a prefix and chain_id to guard keys against collision.
+// Prefix should be sufficient to not collide with other application using the same redis database.
+// chain_id should be ok to not let multiple reader with different chains to write to the same channels.
+
+type Namespace struct {
+	Prefix  string
+	ChainID string
+}
+
+// Create a new key with this namespace.
+func (ns Namespace) NewKey(ch api.Channel) Key {
+	return Key{NS: ns, Channel: ch}
+}
+
+func (ns Namespace) String() string {
+	// No Chain id, set to "nullChain"
+	if len(ns.ChainID) < 1 {
+		ns.ChainID = nullChain
+	}
+
+	// Set default prefix if empty.
+	if len(ns.Prefix) < 1 {
+		ns.Prefix = defaultPrefix
+	}
+
+	// Otherwise. return both.
+	return strings.Join([]string{ns.Prefix, ns.ChainID}, "::")
+}

api/redis_common/namespace_test.go

@@ -0,0 +1,23 @@
+package redis_common
+
+import "testing"
+
+func TestNamespace_String(t *testing.T) {
+	tests := []struct {
+		name string
+		ns   Namespace
+		want string
+	}{
+		{"Empty", Namespace{}, "ship::0000000000000000000000000000000000000000000000000000000000000000"},
+		{"Prefix Only", Namespace{Prefix: "some.prefix"}, "some.prefix::0000000000000000000000000000000000000000000000000000000000000000"},
+		{"ChainID Only", Namespace{ChainID: "1234"}, "ship::1234"},
+		{"Both", Namespace{Prefix: "my.prefix", ChainID: "1234"}, "my.prefix::1234"},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := tt.ns.String(); got != tt.want {
+				t.Errorf("Namespace.String() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

api/redis_pubsub/publisher.go

@@ -0,0 +1,37 @@
+package redis_pubsub
+
+import (
+	"context"
+
+	"thalos/api"
+	. "thalos/api/redis_common"
+
+	"github.com/go-redis/redis/v8"
+)
+
+type Publisher struct {
+	pipeline redis.Pipeliner
+	ctx      context.Context
+	ns       Namespace
+}
+
+func NewPublisher(client *redis.Client, ns Namespace) *Publisher {
+	return &Publisher{
+		pipeline: client.Pipeline(),
+		ctx:      client.Context(),
+		ns:       ns,
+	}
+}
+
+func (r *Publisher) Write(channel api.Channel, payload []byte) error {
+	return r.pipeline.Publish(r.ctx, r.ns.NewKey(channel).String(), payload).Err()
+}
+
+func (r *Publisher) Flush() error {
+	_, err := r.pipeline.Exec(r.ctx)
+	return err
+}
+
+func (r *Publisher) Close() error {
+	return r.pipeline.Close()
+}

api/redis_pubsub/publisher_test.go

@@ -0,0 +1,27 @@
+package redis_pubsub
+
+import (
+	"testing"
+
+	"thalos/api"
+	. "thalos/api/redis_common"
+
+	"github.com/go-redis/redismock/v8"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestPublisher_Write(t *testing.T) {
+	client, mock := redismock.NewClientMock()
+
+	pub := NewPublisher(client, Namespace{ChainID: "id"})
+
+	mock.MatchExpectationsInOrder(true)
+	mock.ExpectPublish("ship::id::test", []byte("some string")).SetVal(0)
+	mock.ExpectPublish("ship::id::test2", []byte("some other string")).SetVal(0)
+
+	assert.NoError(t, pub.Write(api.Channel{"test"}, []byte("some string")))
+	assert.NoError(t, pub.Write(api.Channel{"test2"}, []byte("some other string")))
+	assert.NoError(t, pub.Flush())
+
+	assert.NoError(t, mock.ExpectationsWereMet())
+}

api/redis_pubsub/subscriber.go

@@ -0,0 +1,110 @@
+package redis_pubsub
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	"thalos/api"
+	. "thalos/api/redis_common"
+
+	"github.com/go-redis/redis/v8"
+)
+
+type Subscriber struct {
+	client   *redis.Client
+	sub      *redis.PubSub
+	ctx      context.Context
+	mu       sync.RWMutex
+	timeout  time.Duration
+	channels map[string]chan []byte
+	ns       Namespace
+}
+
+type SubscriberOption func(*Subscriber)
+
+func WithTimeout(value time.Duration) SubscriberOption {
+	return func(s *Subscriber) {
+		s.timeout = value
+	}
+}
+
+func NewSubscriber(client *redis.Client, ns Namespace, options ...SubscriberOption) *Subscriber {
+	sub := &Subscriber{
+		client:   client,
+		ctx:      client.Context(),
+		sub:      client.PSubscribe(client.Context()),
+		channels: make(map[string]chan []byte),
+		timeout:  time.Millisecond * 200,
+		ns:       ns,
+	}
+
+	for _, opt := range options {
+		opt(sub)
+	}
+
+	go sub.worker()
+
+	return sub
+}
+
+// forward forwards a message to the channel.
+// as writes to a unbuffered channel will block until it's read.
+// We run select on it and discard the message if no read happends during timeout
+func forward(msg redis.Message, ch chan<- []byte, timeout time.Duration) {
+	select {
+	case <-time.After(timeout):
+	case ch <- []byte(msg.Payload):
+	}
+}
+
+// worker reads messages from redis pubsub and forwards them to
+// correct channels.
+func (s *Subscriber) worker() {
+	for msg := range s.sub.Channel() {
+		// Route message to correct channel.
+		s.mu.RLock()
+		if ch, ok := s.channels[msg.Channel]; ok {
+			go forward(*msg, ch, s.timeout)
+		}
+		s.mu.RUnlock()
+	}
+}
+
+func (s *Subscriber) Read(channel api.Channel) ([]byte, error) {
+	var err error
+
+	key := s.ns.NewKey(channel).String()
+	s.mu.RLock()
+	ch, ok := s.channels[key]
+	s.mu.RUnlock()
+	if !ok {
+		// Channel does not exist in the map.
+		// Subscribe and insert it.
+		err = s.sub.Subscribe(s.ctx, key)
+		if err != nil {
+			return nil, err
+		}
+
+		// Guard race condition to map with mutex.
+		s.mu.Lock()
+		ch = make(chan []byte)
+		s.channels[key] = ch
+		s.mu.Unlock()
+	}
+
+	return <-ch, nil
+}
+
+func (s *Subscriber) Close() error {
+	err := s.sub.Close()
+
+	for _, ch := range s.channels {
+		close(ch)
+	}
+	s.mu.Lock()
+	s.channels = make(map[string]chan []byte)
+	s.mu.Unlock()
+
+	return err
+}

api/redis_pubsub/subscriber_test.go

@@ -0,0 +1,58 @@
+package redis_pubsub
+
+import (
+	"testing"
+	"time"
+
+	"thalos/api"
+	. "thalos/api/redis_common"
+
+	"github.com/alicebob/miniredis/v2"
+	"github.com/go-redis/redis/v8"
+	"github.com/go-redis/redismock/v8"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestSubscriber_Construct(t *testing.T) {
+	client, _ := redismock.NewClientMock()
+	ns := Namespace{Prefix: "prefix", ChainID: "8f2f6ec19400d372c9b3340b1438e9c805cf9e69be962fa81d055bc037ceed8d"}
+
+	s := NewSubscriber(client, ns)
+
+	assert.Equal(t, s.client, client)
+	assert.Equal(t, s.ctx, client.Context())
+	assert.NotNil(t, s.sub)
+	assert.Equal(t, s.ns, ns)
+	assert.Equal(t, s.timeout, 200*time.Millisecond)
+
+	s = NewSubscriber(client, ns, WithTimeout(4*time.Second))
+	assert.Equal(t, s.timeout, 4*time.Second)
+}
+
+func TestSubscriber_Read(t *testing.T) {
+	expectedMessages := []string{"payload", "payload2", "payload3"}
+
+	server := miniredis.RunT(t)
+
+	client := redis.NewClient(&redis.Options{
+		Addr: server.Addr(),
+	})
+
+	s := NewSubscriber(client, Namespace{Prefix: "prefix", ChainID: "d41dbd2921d5a377325661427090c6c508904d60920d6b7ea771c58da5299754"})
+
+	go func() {
+		time.Sleep(time.Millisecond * 10)
+
+		for _, msg := range expectedMessages {
+			server.Publish("prefix::d41dbd2921d5a377325661427090c6c508904d60920d6b7ea771c58da5299754::test", msg)
+		}
+	}()
+
+	// Redis pubsub does not guarentee that messages are sent in the correct order.
+	for range expectedMessages {
+		msg, err := s.Read(api.Channel{"test"})
+		assert.NoError(t, err)
+
+		assert.Contains(t, expectedMessages, string(msg))
+	}
+}

api/writer.go

@@ -0,0 +1,17 @@
+package api
+
+// Writer interface defines the required methods
+// to send messages over an channel.
+type Writer interface {
+	// Write writes a message over a channel.
+	// The message may or may not be buffered depending on the implementation.
+	Write(channel Channel, payload []byte) error
+
+	// Flush writes any buffered messages to the channel.
+	// If the implementation does not support buffering. this is a noop.
+	Flush() error
+
+	// Close closes the writer
+	// Any blocked Flush or Write operations will be unblocked.
+	Close() error
+}