Skip to content

Commit d7fb8f1

Browse files
RobertIndieRobertIndie
authored
feat: support pluggable state store (#173)
1 parent 4618660 commit d7fb8f1

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

44 files changed

+219
-132
lines changed

‎Makefile

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,9 @@ build_example:
2020
lint:
2121
golangci-lint run
2222

23+
lint-fix:
24+
golangci-lint run --fix
25+
2326
build_all: build build_example
2427

2528
test:

‎admin/utils/utils.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,8 @@ package utils
1818

1919
import (
2020
"fmt"
21-
"github.com/functionstream/function-stream/admin/client"
21+
22+
adminclient "github.com/functionstream/function-stream/admin/client"
2223
)
2324

2425
func MakeQueueSourceTubeConfig(subName string, topics ...string) []adminclient.ModelTubeConfig {

‎benchmark/bench_test.go

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -18,21 +18,22 @@ package benchmark
1818

1919
import (
2020
"context"
21+
"math/rand"
22+
"os"
23+
"runtime/pprof"
24+
"strconv"
25+
"testing"
26+
"time"
27+
2128
"github.com/apache/pulsar-client-go/pulsaradmin"
2229
"github.com/apache/pulsar-client-go/pulsaradmin/pkg/utils"
23-
"github.com/functionstream/function-stream/admin/client"
30+
adminclient "github.com/functionstream/function-stream/admin/client"
2431
adminutils "github.com/functionstream/function-stream/admin/utils"
2532
"github.com/functionstream/function-stream/common"
2633
"github.com/functionstream/function-stream/fs"
2734
"github.com/functionstream/function-stream/fs/contube"
2835
"github.com/functionstream/function-stream/perf"
2936
"github.com/functionstream/function-stream/server"
30-
"math/rand"
31-
"os"
32-
"runtime/pprof"
33-
"strconv"
34-
"testing"
35-
"time"
3637
)
3738

3839
func BenchmarkStressForBasicFunc(b *testing.B) {

‎cmd/client/cmd.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,8 @@ var (
3535
)
3636

3737
func init() {
38-
Cmd.PersistentFlags().StringVarP(&c.Config.ServiceAddr, "service-address", "s", "http://localhost:7300", "Service address")
38+
Cmd.PersistentFlags().StringVarP(&c.Config.ServiceAddr, "service-address", "s",
39+
"http://localhost:7300", "Service address")
3940

4041
Cmd.AddCommand(create.Cmd)
4142
Cmd.AddCommand(list.Cmd)

‎cmd/client/consume/cmd.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,12 @@ package consume
1818

1919
import (
2020
"fmt"
21-
"github.com/functionstream/function-stream/admin/client"
21+
"os"
22+
23+
adminclient "github.com/functionstream/function-stream/admin/client"
2224
"github.com/functionstream/function-stream/cmd/client/common"
2325
"github.com/spf13/cobra"
2426
"golang.org/x/net/context"
25-
"os"
2627
)
2728

2829
var (

‎cmd/client/create/cmd.go

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,13 +19,14 @@ package create
1919
import (
2020
"context"
2121
"fmt"
22-
"github.com/functionstream/function-stream/admin/client"
22+
"io"
23+
"os"
24+
25+
adminclient "github.com/functionstream/function-stream/admin/client"
2326
"github.com/functionstream/function-stream/admin/utils"
2427
"github.com/functionstream/function-stream/cmd/client/common"
2528
fs_cmmon "github.com/functionstream/function-stream/common"
2629
"github.com/spf13/cobra"
27-
"io"
28-
"os"
2930
)
3031

3132
var (

‎cmd/client/delete/cmd.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717
package del
1818

1919
import (
20-
"github.com/functionstream/function-stream/admin/client"
20+
adminclient "github.com/functionstream/function-stream/admin/client"
2121
"github.com/functionstream/function-stream/cmd/client/common"
2222
"github.com/spf13/cobra"
2323
)

‎cmd/client/list/cmd.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,11 @@ package list
1919
import (
2020
"context"
2121
"fmt"
22-
"github.com/functionstream/function-stream/admin/client"
22+
"os"
23+
24+
adminclient "github.com/functionstream/function-stream/admin/client"
2325
"github.com/functionstream/function-stream/cmd/client/common"
2426
"github.com/spf13/cobra"
25-
"os"
2627
)
2728

2829
var Cmd = &cobra.Command{

‎cmd/client/produce/cmd.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,12 @@ package produce
1818

1919
import (
2020
"fmt"
21-
"github.com/functionstream/function-stream/admin/client"
21+
"os"
22+
23+
adminclient "github.com/functionstream/function-stream/admin/client"
2224
"github.com/functionstream/function-stream/cmd/client/common"
2325
"github.com/spf13/cobra"
2426
"golang.org/x/net/context"
25-
"os"
2627
)
2728

2829
var (

‎cmd/main.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,12 +18,13 @@ package main
1818

1919
import (
2020
"fmt"
21+
"os"
22+
2123
"github.com/functionstream/function-stream/cmd/client"
2224
"github.com/functionstream/function-stream/cmd/perf"
2325
"github.com/functionstream/function-stream/cmd/server"
2426
"github.com/functionstream/function-stream/cmd/standalone"
2527
"github.com/spf13/cobra"
26-
"os"
2728
)
2829

2930
var (

‎cmd/perf/cmd.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,11 @@ package perf
1818

1919
import (
2020
"context"
21+
"io"
22+
2123
"github.com/functionstream/function-stream/common"
2224
"github.com/functionstream/function-stream/perf"
2325
"github.com/spf13/cobra"
24-
"io"
2526
)
2627

2728
var (

‎cmd/server/cmd.go

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,11 @@ package server
1818

1919
import (
2020
"context"
21+
"io"
22+
2123
"github.com/functionstream/function-stream/common"
2224
"github.com/functionstream/function-stream/server"
2325
"github.com/spf13/cobra"
24-
"io"
2526
)
2627

2728
var (
@@ -45,7 +46,8 @@ var (
4546
func init() {
4647
Cmd.Flags().StringVarP(&config.configFile, "config-file", "c", "conf/function-stream.yaml",
4748
"path to the config file (default is conf/function-stream.yaml)")
48-
Cmd.Flags().BoolVarP(&config.loadConfigFromEnv, "load-config-from-env", "e", false, "load config from env (default is false)")
49+
Cmd.Flags().BoolVarP(&config.loadConfigFromEnv, "load-config-from-env", "e", false,
50+
"load config from env (default is false)")
4951
}
5052

5153
func exec(*cobra.Command, []string) {

‎cmd/standalone/cmd.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,11 @@ package standalone
1818

1919
import (
2020
"context"
21+
"io"
22+
2123
"github.com/functionstream/function-stream/common"
2224
"github.com/functionstream/function-stream/server"
2325
"github.com/spf13/cobra"
24-
"io"
2526
)
2627

2728
var (

‎common/constants.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,9 +21,9 @@ const (
2121
MemoryTubeType = "memory"
2222
HttpTubeType = "http"
2323

24-
DefaultAddr = "localhost:7300"
25-
DefaultPulsarURL = "pulsar://localhost:6650"
26-
DefaultTubeType = PulsarTubeType
24+
WASMRuntime = "wasm"
2725

2826
RuntimeArchiveConfigKey = "archive"
27+
28+
StateStorePebble = "pebble"
2929
)

‎common/model/function.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,10 @@
1717
package model
1818

1919
import (
20+
"strings"
21+
2022
"github.com/functionstream/function-stream/fs/contube"
2123
"github.com/pkg/errors"
22-
"strings"
2324
)
2425

2526
type TubeConfig struct {

‎common/model/function_serde_test.go

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,11 @@ package model
1919
import (
2020
"encoding/json"
2121
"fmt"
22-
"github.com/functionstream/function-stream/common"
23-
"gopkg.in/yaml.v3"
2422
"reflect"
2523
"testing"
24+
25+
"github.com/functionstream/function-stream/common"
26+
"gopkg.in/yaml.v3"
2627
)
2728

2829
func TestFunctionSerde(t *testing.T) {
@@ -110,7 +111,8 @@ func TestFunctionSerdeWithNil(t *testing.T) {
110111

111112
fmt.Println(string(data))
112113

113-
f.Sources = []*TubeConfig{} // The nil would be expected to be converted to a zero-length array for the YAML serialization
114+
f.Sources = []*TubeConfig{} // The nil would be expected to be converted to a zero-length array for the YAML
115+
// serialization
114116

115117
// YAML Deserialization
116118
err = yaml.Unmarshal(data, &f2)

‎fs/api/instance.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,11 @@
1717
package api
1818

1919
import (
20+
"log/slog"
21+
2022
"github.com/functionstream/function-stream/common/model"
2123
"github.com/functionstream/function-stream/fs/contube"
2224
"golang.org/x/net/context"
23-
"log/slog"
2425
)
2526

2627
type FunctionInstance interface {

‎fs/contube/contube.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ package contube
1919
import (
2020
"context"
2121
"fmt"
22+
2223
"github.com/pkg/errors"
2324
)
2425

‎fs/contube/http.go

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,13 +17,14 @@
1717
package contube
1818

1919
import (
20-
"github.com/pkg/errors"
21-
"golang.org/x/net/context"
2220
"io"
2321
"log/slog"
2422
"net/http"
2523
"sync"
2624
"sync/atomic"
25+
26+
"github.com/pkg/errors"
27+
"golang.org/x/net/context"
2728
)
2829

2930
type state int
@@ -133,7 +134,8 @@ func (f *HttpTubeFactory) NewSinkTube(_ context.Context, _ ConfigMap) (chan<- Re
133134
return nil, ErrSinkTubeNotImplemented
134135
}
135136

136-
func (f *HttpTubeFactory) GetHandleFunc(getEndpoint func(r *http.Request) (string, error), logger *slog.Logger) func(http.ResponseWriter, *http.Request) {
137+
func (f *HttpTubeFactory) GetHandleFunc(getEndpoint func(r *http.Request) (string, error),
138+
logger *slog.Logger) func(http.ResponseWriter, *http.Request) {
137139
return func(w http.ResponseWriter, r *http.Request) {
138140
endpoint, err := getEndpoint(r)
139141
if err != nil {

‎fs/contube/http_test.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,10 @@
1717
package contube
1818

1919
import (
20+
"testing"
21+
2022
"github.com/stretchr/testify/assert"
2123
"golang.org/x/net/context"
22-
"testing"
2324
)
2425

2526
func TestHttpTubeHandleRecord(t *testing.T) {

‎fs/contube/memory_test.go

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,8 @@ func TestMemoryTube(t *testing.T) {
3535
var events []Record
3636

3737
topics := []string{"topic1", "topic2", "topic3"}
38-
source, err := memoryQueueFactory.NewSourceTube(ctx, (&SourceQueueConfig{Topics: topics, SubName: "consume-" + strconv.Itoa(rand.Int())}).ToConfigMap())
38+
source, err := memoryQueueFactory.NewSourceTube(ctx, (&SourceQueueConfig{Topics: topics,
39+
SubName: "consume-" + strconv.Itoa(rand.Int())}).ToConfigMap())
3940
if err != nil {
4041
t.Fatal(err)
4142
}
@@ -72,7 +73,8 @@ func TestMemoryTube(t *testing.T) {
7273
wg.Wait()
7374
cancel()
7475

75-
// Give enough time to ensure that the goroutine execution within NewSource Tube and NewSinkTube is complete and the released queue is successful.
76+
// Give enough time to ensure that the goroutine execution within NewSource Tube and NewSinkTube is complete and
77+
// the released queue is successful.
7678
time.Sleep(100 * time.Millisecond)
7779

7880
// assert the memoryQueueFactory.queues is empty.

‎fs/contube/pulsar.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,11 @@ package contube
1818

1919
import (
2020
"context"
21-
"github.com/apache/pulsar-client-go/pulsar"
22-
"github.com/pkg/errors"
2321
"log/slog"
2422
"sync/atomic"
23+
24+
"github.com/apache/pulsar-client-go/pulsar"
25+
"github.com/pkg/errors"
2526
)
2627

2728
const (

‎fs/func_ctx_impl_test.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,9 @@
1717
package fs
1818

1919
import (
20-
"github.com/stretchr/testify/assert"
2120
"testing"
21+
22+
"github.com/stretchr/testify/assert"
2223
)
2324

2425
func TestFuncCtx_NilStore(t *testing.T) {

‎fs/instance_impl.go

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,13 +18,14 @@ package fs
1818

1919
import (
2020
"context"
21+
"log/slog"
22+
"reflect"
23+
2124
"github.com/functionstream/function-stream/common"
2225
"github.com/functionstream/function-stream/common/model"
2326
"github.com/functionstream/function-stream/fs/api"
2427
"github.com/functionstream/function-stream/fs/contube"
2528
"github.com/pkg/errors"
26-
"log/slog"
27-
"reflect"
2829
)
2930

3031
type FunctionInstanceImpl struct {
@@ -53,7 +54,8 @@ func NewDefaultInstanceFactory() api.FunctionInstanceFactory {
5354
return &DefaultInstanceFactory{}
5455
}
5556

56-
func (f *DefaultInstanceFactory) NewFunctionInstance(definition *model.Function, funcCtx api.FunctionContext, index int32, logger *slog.Logger) api.FunctionInstance {
57+
func (f *DefaultInstanceFactory) NewFunctionInstance(definition *model.Function, funcCtx api.FunctionContext,
58+
index int32, logger *slog.Logger) api.FunctionInstance {
5759
ctx, cancelFunc := context.WithCancel(context.Background())
5860
ctx = context.WithValue(ctx, CtxKeyFunctionName, definition.Name)
5961
ctx = context.WithValue(ctx, CtxKeyInstanceIndex, index)
@@ -69,7 +71,8 @@ func (f *DefaultInstanceFactory) NewFunctionInstance(definition *model.Function,
6971
}
7072
}
7173

72-
func (instance *FunctionInstanceImpl) Run(runtimeFactory api.FunctionRuntimeFactory, sources []<-chan contube.Record, sink chan<- contube.Record) {
74+
func (instance *FunctionInstanceImpl) Run(runtimeFactory api.FunctionRuntimeFactory, sources []<-chan contube.Record,
75+
sink chan<- contube.Record) {
7376
runtime, err := runtimeFactory.NewFunctionRuntime(instance)
7477
if err != nil {
7578
instance.readyCh <- errors.Wrap(err, "Error creating runtime")

0 commit comments

Comments
 (0)