diff --git a/apps/api-gateway/package.json b/apps/api-gateway/package.json
index a516588..1b4658c 100755
--- a/apps/api-gateway/package.json
+++ b/apps/api-gateway/package.json
@@ -23,7 +23,6 @@
"@repo/logger": "workspace:*",
"@repo/prisma": "workspace:*",
"@repo/proto-defs": "workspace:*",
- "@repo/utils": "workspace:*",
"dotenv": "^17.2.2",
"envalid": "^8.1.0",
"express": "^5.1.0",
diff --git a/apps/api-gateway/src/app.ts b/apps/api-gateway/src/app.ts
index 4253053..21f9b2e 100755
--- a/apps/api-gateway/src/app.ts
+++ b/apps/api-gateway/src/app.ts
@@ -8,9 +8,9 @@ import orderRouter from "@/routers/order.route";
const app: Express = express();
app.use(express.json());
-app.use("/api/v1/trade", tradeRouter);
-app.use("/api/v1/user", userRouter);
-app.use("/api/v1/order", orderRouter);
+app.use("/api/v1/trades", tradeRouter);
+app.use("/api/v1/users", userRouter);
+app.use("/api/v1/orders", orderRouter);
app.get("/", (_, res) => {
res.json({ message: "Hello World from Nerve trade platform's backend" });
diff --git a/apps/candle-service/.air.toml b/apps/candle-service/.air.toml
new file mode 100644
index 0000000..cfe1ab3
--- /dev/null
+++ b/apps/candle-service/.air.toml
@@ -0,0 +1,27 @@
+root = "."
+tmp_dir = "tmp"
+
+[build]
+cmd = "go build -o ./tmp/candle-service ./cmd/candle-service"
+bin = "tmp/candle-service"
+
+include_ext = ["go"]
+
+include_dir = [
+ ".",
+ "../../packages/proto-defs/go/generated"
+]
+
+exclude_dir = [
+ "tmp",
+ "vendor",
+ "node_modules"
+]
+
+delay = 200
+
+[run]
+cmd = "./tmp/candle-service"
+
+[log]
+time = true
diff --git a/apps/candle-service/cmd/candle-service/main.go b/apps/candle-service/cmd/candle-service/main.go
new file mode 100644
index 0000000..486bd23
--- /dev/null
+++ b/apps/candle-service/cmd/candle-service/main.go
@@ -0,0 +1,87 @@
+package main
+
+import (
+ "context"
+ "log/slog"
+ "net"
+ "os"
+ "os/signal"
+ "syscall"
+
+ "github.com/sameerkrdev/nerve/apps/candle-service/internal"
+ "github.com/sameerkrdev/nerve/apps/candle-service/internal/engine"
+ "github.com/sameerkrdev/nerve/apps/candle-service/internal/kafka"
+ memorystore "github.com/sameerkrdev/nerve/apps/candle-service/internal/memoryStore"
+)
+
+//* func: define grpc server and start consumer and workers
+//* func: start the kafka consumer
+//* func: start the workers
+//* - each worker recieve gets single symbol trade data via channel
+//* - calculate the candlestick data for multiple timeframe
+//* - L1: In-memory (last 1000 candles)
+//* - L2: Redis Memory (last 5000 candles)
+// - L3: store the trades into clickhouse which will eventually generate the candles data
+//* - Fanout:
+//* - publish to kafka for other services
+//* - redis pub/sub for websockets servers
+// func: to get the historical data of candles
+// func: graceful shutdown
+
+// in main or in server, initialize router workers, then initialize kafka consumer handler then initialize kafka client with consume func call
+
+func main() {
+ if err := memorystore.InitRedis(); err != nil {
+ slog.Error("redis init failed", "error", err)
+ os.Exit(1)
+ }
+
+ brokerAddresses := []string{"localhost:19092", "localhost:19093", "localhost:19094"}
+
+ if err := kafka.InitKafkaProducer(brokerAddresses); err != nil {
+ slog.Error("kafka producer init failed", "error", err)
+ os.Exit(1)
+ }
+
+ workerRouter := engine.NewWorkerRouter(10, kafka.PublishCandleEventToKafka)
+ // mux := internal.NewServer(workerRouter)
+
+ kafkaConsumerClient, err := kafka.NewKafkaConsumerClient(brokerAddresses)
+ if err != nil {
+ slog.Error("kafka consumer connection failed", "error", err)
+ os.Exit(1)
+ }
+
+ kafkaConsumerHandler := kafka.NewConsumerHandler(workerRouter)
+
+ topics := []string{
+ "trades",
+ }
+
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ go kafkaConsumerClient.Consume(ctx, topics, kafkaConsumerHandler)
+
+ PORT := "50054"
+
+ listener, err := net.Listen("tcp", ":"+PORT)
+ if err != nil {
+ slog.Error("net server failed", "error", err)
+ os.Exit(1)
+ }
+
+ slog.Info("Net server listening", "port", PORT)
+
+ grpcServer := internal.NewGrpcServer(workerRouter, listener)
+
+ // graceful shutdown
+ quit := make(chan os.Signal, 1)
+ signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM)
+
+ <-quit
+
+ slog.Info("shutting down...")
+
+ cancel()
+ grpcServer.GracefulStop()
+}
diff --git a/apps/candle-service/go.mod b/apps/candle-service/go.mod
new file mode 100644
index 0000000..60bee1d
--- /dev/null
+++ b/apps/candle-service/go.mod
@@ -0,0 +1,49 @@
+module github.com/sameerkrdev/nerve/apps/candle-service
+
+go 1.25.4
+
+require (
+ github.com/IBM/sarama v1.46.3
+ github.com/redis/go-redis/v9 v9.18.0
+ google.golang.org/protobuf v1.36.11
+)
+
+require (
+ github.com/ClickHouse/ch-go v0.71.0 // indirect
+ github.com/ClickHouse/clickhouse-go/v2 v2.45.0 // indirect
+ github.com/andybalholm/brotli v1.2.0 // indirect
+ github.com/cespare/xxhash/v2 v2.3.0 // indirect
+ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
+ github.com/eapache/go-resiliency v1.7.0 // indirect
+ github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect
+ github.com/eapache/queue v1.1.0 // indirect
+ github.com/go-faster/city v1.0.1 // indirect
+ github.com/go-faster/errors v0.7.1 // indirect
+ github.com/golang/snappy v0.0.4 // indirect
+ github.com/google/uuid v1.6.0 // indirect
+ github.com/hashicorp/go-uuid v1.0.3 // indirect
+ github.com/jcmturner/aescts/v2 v2.0.0 // indirect
+ github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
+ github.com/jcmturner/gofork v1.7.6 // indirect
+ github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect
+ github.com/jcmturner/rpc/v2 v2.0.3 // indirect
+ github.com/klauspost/compress v1.18.3 // indirect
+ github.com/paulmach/orb v0.12.0 // indirect
+ github.com/pierrec/lz4/v4 v4.1.25 // indirect
+ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
+ github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9 // indirect
+ github.com/segmentio/asm v1.2.1 // indirect
+ github.com/shopspring/decimal v1.4.0 // indirect
+ go.opentelemetry.io/otel v1.41.0 // indirect
+ go.opentelemetry.io/otel/trace v1.41.0 // indirect
+ go.uber.org/atomic v1.11.0 // indirect
+ go.yaml.in/yaml/v3 v3.0.4 // indirect
+ golang.org/x/crypto v0.50.0 // indirect
+ golang.org/x/net v0.53.0 // indirect
+ golang.org/x/sync v0.20.0 // indirect
+ golang.org/x/sys v0.43.0 // indirect
+ golang.org/x/text v0.36.0 // indirect
+ google.golang.org/genproto/googleapis/rpc v0.0.0-20260427160629-7cedc36a6bc4 // indirect
+ google.golang.org/grpc v1.80.0 // indirect
+)
diff --git a/apps/candle-service/go.sum b/apps/candle-service/go.sum
new file mode 100644
index 0000000..1535690
--- /dev/null
+++ b/apps/candle-service/go.sum
@@ -0,0 +1,191 @@
+github.com/ClickHouse/ch-go v0.71.0 h1:bUdZ/EZj/LcVHsMqaRUP2holqygrPWQKeMjc6nZoyRM=
+github.com/ClickHouse/ch-go v0.71.0/go.mod h1:NwbNc+7jaqfY58dmdDUbG4Jl22vThgx1cYjBw0vtgXw=
+github.com/ClickHouse/clickhouse-go/v2 v2.45.0 h1:iHt15nA4iYhfde5bDQAcLAat9BAh7B5ksPRNRa4UI7s=
+github.com/ClickHouse/clickhouse-go/v2 v2.45.0/go.mod h1:giJfUVlMkcfUEPVfRpt51zZaGEx9i17gCos8gBl392c=
+github.com/IBM/sarama v1.46.3 h1:njRsX6jNlnR+ClJ8XmkO+CM4unbrNr/2vB5KK6UA+IE=
+github.com/IBM/sarama v1.46.3/go.mod h1:GTUYiF9DMOZVe3FwyGT+dtSPceGFIgA+sPc5u6CBwko=
+github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ=
+github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
+github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
+github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
+github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
+github.com/eapache/go-resiliency v1.7.0 h1:n3NRTnBn5N0Cbi/IeOHuQn9s2UwVUH7Ga0ZWcP+9JTA=
+github.com/eapache/go-resiliency v1.7.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho=
+github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 h1:Oy0F4ALJ04o5Qqpdz8XLIpNA3WM/iSIXqxtqo7UGVws=
+github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3/go.mod h1:YvSRo5mw33fLEx1+DlK6L2VV43tJt5Eyel9n9XBcR+0=
+github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc=
+github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
+github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
+github.com/go-faster/city v1.0.1 h1:4WAxSZ3V2Ws4QRDrscLEDcibJY8uf41H6AhXDrNDcGw=
+github.com/go-faster/city v1.0.1/go.mod h1:jKcUJId49qdW3L1qKHH/3wPeUstCVpVSXTM6vO3VcTw=
+github.com/go-faster/errors v0.7.1 h1:MkJTnDoEdi9pDabt1dpWf7AA8/BaSYZqibYyhZ20AYg=
+github.com/go-faster/errors v0.7.1/go.mod h1:5ySTjWFiphBs07IKuiL69nxdfd5+fzh1u7FPGZP2quo=
+github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
+github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
+github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM=
+github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8=
+github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8=
+github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs=
+github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo=
+github.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7G4t3HTlQ184QM=
+github.com/jcmturner/gofork v1.7.6 h1:QH0l3hzAU1tfT3rZCnW5zXl+orbkNMMRGJfdJjHVETg=
+github.com/jcmturner/gofork v1.7.6/go.mod h1:1622LH6i/EZqLloHfE7IeZ0uEJwMSUyQ/nDd82IeqRo=
+github.com/jcmturner/goidentity/v6 v6.0.1 h1:VKnZd2oEIMorCTsFBnJWbExfNN7yZr3EhJAxwOkZg6o=
+github.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg=
+github.com/jcmturner/gokrb5/v8 v8.4.4 h1:x1Sv4HaTpepFkXbt2IkL29DXRf8sOfZXo8eRKh687T8=
+github.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP+F6aCACiMrs=
+github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY=
+github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc=
+github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
+github.com/klauspost/compress v1.18.1 h1:bcSGx7UbpBqMChDtsF28Lw6v/G94LPrrbMbdC3JH2co=
+github.com/klauspost/compress v1.18.1/go.mod h1:ZQFFVG+MdnR0P+l6wpXgIL4NTtwiKIdBnrBd8Nrxr+0=
+github.com/klauspost/compress v1.18.3 h1:9PJRvfbmTabkOX8moIpXPbMMbYN60bWImDDU7L+/6zw=
+github.com/klauspost/compress v1.18.3/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
+github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
+github.com/paulmach/orb v0.12.0 h1:z+zOwjmG3MyEEqzv92UN49Lg1JFYx0L9GpGKNVDKk1s=
+github.com/paulmach/orb v0.12.0/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU=
+github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY=
+github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU=
+github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
+github.com/pierrec/lz4/v4 v4.1.25 h1:kocOqRffaIbU5djlIBr7Wh+cx82C0vtFb0fOurZHqD0=
+github.com/pierrec/lz4/v4 v4.1.25/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9 h1:bsUq1dX0N8AOIL7EB/X911+m4EHsnWEHeJ0c+3TTBrg=
+github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
+github.com/redis/go-redis/v9 v9.18.0 h1:pMkxYPkEbMPwRdenAzUNyFNrDgHx9U+DrBabWNfSRQs=
+github.com/redis/go-redis/v9 v9.18.0/go.mod h1:k3ufPphLU5YXwNTUcCRXGxUoF1fqxnhFQmscfkCoDA0=
+github.com/segmentio/asm v1.2.1 h1:DTNbBqs57ioxAD4PrArqftgypG4/qNpXoJx8TVXxPR0=
+github.com/segmentio/asm v1.2.1/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs=
+github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k=
+github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
+github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
+github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
+github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
+github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=
+go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g=
+go.opentelemetry.io/otel v1.41.0 h1:YlEwVsGAlCvczDILpUXpIpPSL/VPugt7zHThEMLce1c=
+go.opentelemetry.io/otel v1.41.0/go.mod h1:Yt4UwgEKeT05QbLwbyHXEwhnjxNO6D8L5PQP51/46dE=
+go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa9TIN0=
+go.opentelemetry.io/otel/trace v1.41.0/go.mod h1:U1NU4ULCoxeDKc09yCWdWe+3QoyweJcISEVa1RBzOis=
+go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
+go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
+go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
+go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
+golang.org/x/crypto v0.47.0 h1:V6e3FRj+n4dbpw86FJ8Fv7XVOql7TEwpHapKoMJ/GO8=
+golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI=
+golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o=
+golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA=
+golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
+golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
+golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI=
+golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg=
+golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260427160629-7cedc36a6bc4 h1:tEkOQcXgF6dH1G+MVKZrfpYvozGrzb91k6ha7jireSM=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260427160629-7cedc36a6bc4/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
+google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM=
+google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
+google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
diff --git a/apps/candle-service/internal/clickhouse/candleRepository.go b/apps/candle-service/internal/clickhouse/candleRepository.go
new file mode 100644
index 0000000..0c86348
--- /dev/null
+++ b/apps/candle-service/internal/clickhouse/candleRepository.go
@@ -0,0 +1,5 @@
+package clickhouse
+
+func FetchCandles() {
+
+}
diff --git a/apps/candle-service/internal/clickhouse/clickhouse.go b/apps/candle-service/internal/clickhouse/clickhouse.go
new file mode 100644
index 0000000..ba3750d
--- /dev/null
+++ b/apps/candle-service/internal/clickhouse/clickhouse.go
@@ -0,0 +1,54 @@
+package clickhouse
+
+import (
+ "context"
+ "crypto/tls"
+ "fmt"
+ "sync"
+
+ "github.com/ClickHouse/clickhouse-go/v2"
+ "github.com/ClickHouse/clickhouse-go/v2/lib/driver"
+)
+
+var (
+ ClickhouseClient driver.Conn
+ once sync.Once
+ initErr error
+)
+
+func NewClickhouseClient(ctx context.Context) (driver.Conn, error) {
+ once.Do(func() {
+ var conn driver.Conn
+
+ conn, initErr = clickhouse.Open(&clickhouse.Options{
+ Addr: []string{"zjc5ukn020.eu-west-2.aws.clickhouse.cloud:9440"}, // 9440 is a secure native TCP port
+ Protocol: clickhouse.Native,
+ TLS: &tls.Config{
+ InsecureSkipVerify: true,
+ }, // enable secure TLS
+ Auth: clickhouse.Auth{
+ Username: "default",
+ Password: "9FzzlTeRu~V5V",
+ },
+ })
+
+ if initErr != nil {
+ return
+ }
+
+ if err := conn.Ping(ctx); err != nil {
+ if exception, ok := err.(*clickhouse.Exception); ok {
+ fmt.Printf("Exception [%d] %s\n%s\n", exception.Code, exception.Message, exception.StackTrace)
+ }
+ initErr = err
+ return
+ }
+
+ ClickhouseClient = conn
+ })
+ return ClickhouseClient, initErr
+}
+
+func InsertRawTrade() {
+
+}
diff --git a/apps/candle-service/internal/engine/router.go b/apps/candle-service/internal/engine/router.go
new file mode 100644
index 0000000..acc358e
--- /dev/null
+++ b/apps/candle-service/internal/engine/router.go
@@ -0,0 +1,44 @@
+package engine
+
+import (
+ memorystore "github.com/sameerkrdev/nerve/apps/candle-service/internal/memoryStore"
+ "github.com/sameerkrdev/nerve/apps/candle-service/internal/utils"
+ pbAggeration "github.com/sameerkrdev/nerve/packages/proto-defs/go/generated/aggeration/v1"
+ pb "github.com/sameerkrdev/nerve/packages/proto-defs/go/generated/engine"
+)
+
+type WorkerRouter struct {
+ workers []*Worker
+ count int
+}
+
+func NewWorkerRouter(workerCount int, onCandleClosed memorystore.OnCandleClosedFn) *WorkerRouter {
+ var workers []*Worker
+
+ for i := range workerCount {
+ candleCache := memorystore.NewCandleStore(onCandleClosed)
+ worker := NewWorker(i, candleCache)
+
+ go worker.Process()
+
+ workers = append(workers, worker)
+ }
+
+ return &WorkerRouter{
+ workers: workers,
+ count: workerCount,
+ }
+}
+
+func (wr *WorkerRouter) Route(event *pb.EngineEvent) {
+ workerIndex := int(utils.Hash(event.Symbol) % uint32(wr.count))
+
+ worker := wr.workers[workerIndex]
+
+ worker.eventQueue <- event
+}
+
+func (wr *WorkerRouter) GetCandles(symbol string, timeframe pbAggeration.Timeframe) ([]*pbAggeration.Candle, error) {
+ workerIndex := int(utils.Hash(symbol) % uint32(wr.count))
+ return wr.workers[workerIndex].candleCache.GetCandles(symbol, timeframe)
+}
diff --git a/apps/candle-service/internal/engine/worker.go b/apps/candle-service/internal/engine/worker.go
new file mode 100644
index 0000000..c4851ee
--- /dev/null
+++ b/apps/candle-service/internal/engine/worker.go
@@ -0,0 +1,33 @@
+package engine
+
+import (
+ memorystore "github.com/sameerkrdev/nerve/apps/candle-service/internal/memoryStore"
+ pb "github.com/sameerkrdev/nerve/packages/proto-defs/go/generated/engine"
+ "google.golang.org/protobuf/proto"
+)
+
+type Worker struct {
+ eventQueue chan *pb.EngineEvent
+ id int
+ candleCache *memorystore.CandleStore
+}
+
+func NewWorker(id int, candleCache *memorystore.CandleStore) *Worker {
+ eventQueue := make(chan *pb.EngineEvent, 100000)
+
+ return &Worker{
+ eventQueue: eventQueue,
+ id: id,
+ candleCache: candleCache,
+ }
+}
+
+func (w *Worker) Process() {
+ for event := range w.eventQueue {
+ tradeEvent := &pb.TradeEvent{}
+
+ proto.Unmarshal(event.Data, tradeEvent)
+
+ w.candleCache.AddNewCandle(tradeEvent.Symbol, tradeEvent)
+ }
+}
diff --git a/apps/candle-service/internal/kafka/consumerClient.go b/apps/candle-service/internal/kafka/consumerClient.go
new file mode 100644
index 0000000..ea95cd3
--- /dev/null
+++ b/apps/candle-service/internal/kafka/consumerClient.go
@@ -0,0 +1,67 @@
+package kafka
+
+import (
+ "context"
+ "log"
+ "time"
+
+ "github.com/IBM/sarama"
+)
+
+type KafkaConsumerClient struct {
+ group sarama.ConsumerGroup
+ brokers []string
+}
+
+func NewKafkaConsumerClient(brokers []string) (*KafkaConsumerClient, error) {
+ config := sarama.NewConfig()
+
+ config.Consumer.Fetch.Default = 5 * 1024 * 1024
+ config.Consumer.MaxProcessingTime = 3 * time.Second
+ config.Consumer.Group.Rebalance.Strategy = sarama.NewBalanceStrategySticky()
+ config.Consumer.Offsets.AutoCommit.Enable = true
+ config.Consumer.Offsets.AutoCommit.Interval = 2 * time.Second
+ config.Consumer.MaxWaitTime = 500 * time.Millisecond
+
+ consumerGroup, err := sarama.NewConsumerGroup(brokers, "candle_service_group", config)
+
+ if err != nil {
+ return nil, err
+ }
+
+ return &KafkaConsumerClient{
+ group: consumerGroup,
+ brokers: brokers,
+ }, nil
+}
+
+func (k *KafkaConsumerClient) Close() error {
+ return k.group.Close()
+}
+
+func (k *KafkaConsumerClient) Consume(
+ ctx context.Context,
+ topics []string,
+ handler sarama.ConsumerGroupHandler,
+) {
+ defer k.Close()
+
+ // log errors
+ go func() {
+ for err := range k.group.Errors() {
+ log.Println("kafka error:", err)
+ }
+ }()
+
+ for {
+ if err := k.group.Consume(ctx, topics, handler); err != nil {
+ log.Println("consumer error:", err)
+ }
+
+ // exit cleanly when context is cancelled
+ if ctx.Err() != nil {
+ log.Println("kafka consumer stopped")
+ return
+ }
+ }
+}
diff --git a/apps/candle-service/internal/kafka/consumerHandler.go b/apps/candle-service/internal/kafka/consumerHandler.go
new file mode 100644
index 0000000..f9a902f
--- /dev/null
+++ b/apps/candle-service/internal/kafka/consumerHandler.go
@@ -0,0 +1,47 @@
+package kafka
+
+import (
+ "log"
+
+ "github.com/IBM/sarama"
+ "github.com/sameerkrdev/nerve/apps/candle-service/internal/engine"
+ "github.com/sameerkrdev/nerve/packages/proto-defs/go/generated/common"
+ pb "github.com/sameerkrdev/nerve/packages/proto-defs/go/generated/engine"
+ "google.golang.org/protobuf/proto"
+)
+
+type ConsumerHandler struct {
+ router *engine.WorkerRouter
+}
+
+func NewConsumerHandler(router *engine.WorkerRouter) *ConsumerHandler {
+ return &ConsumerHandler{router: router}
+}
+
+func (ch *ConsumerHandler) Setup(session sarama.ConsumerGroupSession) error {
+ log.Println("consumer group session started")
+
+ return nil
+}
+
+func (ch *ConsumerHandler) Cleanup(session sarama.ConsumerGroupSession) error {
+ log.Println("consumer group session ended")
+
+ return nil
+}
+
+func (ch *ConsumerHandler) ConsumeClaim(session sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error {
+ for msg := range claim.Messages() {
+
+ event := &pb.EngineEvent{}
+ proto.Unmarshal(msg.Value, event)
+
+ if event.EventType == common.EventType_TRADE_EXECUTED {
+ ch.router.Route(event)
+ }
+
+ // mark message as processed
+ session.MarkMessage(msg, "")
+ }
+ return nil
+}
diff --git a/apps/candle-service/internal/kafka/producerClient.go b/apps/candle-service/internal/kafka/producerClient.go
new file mode 100644
index 0000000..91f55e3
--- /dev/null
+++ b/apps/candle-service/internal/kafka/producerClient.go
@@ -0,0 +1,71 @@
+package kafka
+
+import (
+ "fmt"
+ "log"
+ "strings"
+ "sync"
+ "time"
+
+ "github.com/IBM/sarama"
+ pbAggegration "github.com/sameerkrdev/nerve/packages/proto-defs/go/generated/aggeration/v1"
+ "google.golang.org/protobuf/proto"
+)
+
+var (
+ KafkaProducerClient sarama.AsyncProducer
+ once sync.Once
+)
+
+func InitKafkaProducer(brokers []string) error {
+ var err error
+ once.Do(func() {
+
+ config := sarama.NewConfig()
+ config.Producer.RequiredAcks = sarama.WaitForAll
+ config.Producer.Retry.Max = 5
+ config.Producer.Return.Successes = true
+ config.Producer.Idempotent = true
+ config.Net.MaxOpenRequests = 1
+
+ config.Producer.Flush.Frequency = 10 * time.Millisecond
+ config.Producer.Flush.Bytes = 1024 * 1024
+
+ KafkaProducerClient, err = sarama.NewAsyncProducer(brokers, config)
+ if err != nil {
+ return
+ }
+
+ go func() {
+ for msg := range KafkaProducerClient.Successes() {
+ log.Printf("sent: topic=%s partition=%d offset=%d\n",
+ msg.Topic, msg.Partition, msg.Offset)
+ }
+ }()
+
+ go func() {
+ for err := range KafkaProducerClient.Errors() {
+ log.Println("kafka error:", err.Err)
+ }
+ }()
+ })
+
+ return err
+}
+
+func PublishCandleEventToKafka(symbol string, timeframe string, candle *pbAggegration.Candle) {
+
+ value, err := proto.Marshal(candle)
+ if err != nil {
+ log.Println("marshal error:", err)
+ return
+ }
+
+ msg := &sarama.ProducerMessage{
+ Topic: "candles",
+ Key: sarama.StringEncoder(fmt.Sprintf("%s:%s", strings.ToUpper(symbol), strings.ToLower(timeframe))),
+ Value: sarama.ByteEncoder(value),
+ }
+
+ KafkaProducerClient.Input() <- msg
+}
diff --git a/apps/candle-service/internal/memoryStore/candleStore.go b/apps/candle-service/internal/memoryStore/candleStore.go
new file mode 100644
index 0000000..4f00683
--- /dev/null
+++ b/apps/candle-service/internal/memoryStore/candleStore.go
@@ -0,0 +1,155 @@
+package memorystore
+
+import (
+ "fmt"
+
+ pb "github.com/sameerkrdev/nerve/packages/proto-defs/go/generated/aggeration/v1"
+ matchingEnigne "github.com/sameerkrdev/nerve/packages/proto-defs/go/generated/engine"
+)
+
+type OnCandleClosedFn func(symbol, timeframe string, candle *pb.Candle)
+
+/*
+
+{
+ "BTCUSD": {
+ "current":{
+ "1m": *pb.Candle,
+ },
+ history:{
+ "1m":[
+ *pb.Candle,
+ *pb.Candle,
+ *pb.Candle,
+ .....
+ ],
+ }
+ },
+ .....
+}
+
+*/
+
+type SymbolStore struct {
+ current map[string]*pb.Candle
+ history map[string][]*pb.Candle
+}
+
+type CandleStore struct {
+ store map[string]*SymbolStore
+ onCandleClosed OnCandleClosedFn
+}
+
+func NewCandleStore(onCandleClosed OnCandleClosedFn) *CandleStore {
+ return &CandleStore{
+ store: make(map[string]*SymbolStore),
+ onCandleClosed: onCandleClosed,
+ }
+}
+
+func (cache *CandleStore) getOrCreateSymbol(symbol string) *SymbolStore {
+ if store, exists := cache.store[symbol]; exists {
+ return store
+ }
+
+ store := &SymbolStore{
+ current: make(map[string]*pb.Candle),
+ history: make(map[string][]*pb.Candle),
+ }
+
+ cache.store[symbol] = store
+
+ return store
+}
+
+func (cache *CandleStore) AddNewCandle(
+ symbol string,
+ tradeData *matchingEnigne.TradeEvent,
+) {
+ store := cache.getOrCreateSymbol(symbol)
+
+ timestamp := tradeData.Timestamp
+
+ for tfName, tfSeconds := range pb.Timeframe_value {
+ activeCandle, exists := store.current[tfName]
+
+ openTimeBucket := (timestamp.Seconds / int64(tfSeconds)) * int64(tfSeconds)
+
+ if !exists {
+ store.current[tfName] = cache.newCandle(tradeData, tfName, tfSeconds, openTimeBucket)
+
+ continue
+ }
+
+ if activeCandle.OpenTime == openTimeBucket {
+ cache.updateCandle(activeCandle, tradeData)
+ PublishCandleEventToRedis(symbol, tfName, store.current[tfName])
+ continue
+ }
+
+ //close the current candle if current trade passes the bucket time
+ activeCandle.IsClosed = true
+ store.history[tfName] = append(store.history[tfName], activeCandle)
+
+ store.current[tfName] = cache.newCandle(tradeData, tfName, tfSeconds, openTimeBucket)
+ store.history[tfName] = trim(store.history[tfName])
+
+ if cache.onCandleClosed != nil {
+ cache.onCandleClosed(symbol, tfName, activeCandle)
+ }
+ }
+}
+
+func (cache *CandleStore) updateCandle(candle *pb.Candle, trade *matchingEnigne.TradeEvent) {
+ price := float64(trade.Price / 100)
+
+ if candle.H < price {
+ candle.H = price
+ }
+
+ if candle.L > price {
+ candle.L = price
+ }
+
+ candle.C = price
+ candle.V += trade.Quantity
+}
+
+func (cache *CandleStore) newCandle(trade *matchingEnigne.TradeEvent, _ string, _ int32, openTimeBucket int64) *pb.Candle {
+ candle := &pb.Candle{
+ O: float64(trade.Price / 100),
+ H: float64(trade.Price / 100),
+ L: float64(trade.Price / 100),
+ C: float64(trade.Price / 100),
+ V: trade.Quantity,
+ OpenTime: openTimeBucket,
+ }
+
+ return candle
+}
+
+func trim(candles []*pb.Candle) []*pb.Candle {
+
+ if len(candles) > 1000 {
+ return candles[1:]
+ }
+
+ return candles
+}
+
+func (cache *CandleStore) GetCandles(
+ symbol string,
+ timeframe pb.Timeframe,
+) ([]*pb.Candle, error) {
+
+ store := cache.getOrCreateSymbol(symbol)
+
+ tfName, ok := pb.Timeframe_name[int32(timeframe)]
+ if !ok {
+ return nil, fmt.Errorf("invalid interval")
+ }
+
+ candles := store.history[tfName]
+
+ return candles, nil
+}
diff --git a/apps/candle-service/internal/memoryStore/redis.go b/apps/candle-service/internal/memoryStore/redis.go
new file mode 100644
index 0000000..1de1d08
--- /dev/null
+++ b/apps/candle-service/internal/memoryStore/redis.go
@@ -0,0 +1,95 @@
+package memorystore
+
+import (
+ "context"
+ "fmt"
+ "log/slog"
+ "strings"
+ "sync"
+
+ "github.com/redis/go-redis/v9"
+ pb "github.com/sameerkrdev/nerve/packages/proto-defs/go/generated/aggeration/v1"
+ "google.golang.org/protobuf/proto"
+)
+
+var (
+ RedisClient *redis.Client
+ once sync.Once
+ ctx = context.Background()
+ initErr error
+)
+
+func InitRedis() error {
+ once.Do(func() {
+ opt, err := redis.ParseURL("rediss://default:gQAAAAAAATZNAAIgcDEyNDllZDYyYWM5ZTk0NTgo:6379")
+ if err != nil {
+ initErr = fmt.Errorf("parse error %w", err)
+ return
+ }
+
+ client := redis.NewClient(opt)
+
+ if e := client.Ping(ctx).Err(); e != nil {
+ initErr = fmt.Errorf("connection error: %w", e)
+ return
+ }
+
+ fmt.Println("Redis connected")
+
+ RedisClient = client
+ })
+ return initErr
+}
+
+func candleKey(symbol, timeframe string) string {
+ return fmt.Sprintf("candles:%s:%s",
+ strings.ToUpper(symbol),
+ strings.ToLower(timeframe),
+ )
+}
+
+func PushCandle(symbol, timeframe string, candle *pb.Candle) error {
+ data, err := proto.Marshal(candle)
+ if err != nil {
+ return fmt.Errorf("marshal candle: %w", err)
+ }
+
+ key := candleKey(symbol, timeframe)
+ // err = RedisClient.LPush(ctx, key, data).Err()
+ // if err != nil {
+ // return err
+ // }
+
+ // return RedisClient.LTrim(ctx, key, 0, 4999).Err()
+
+ pipe := RedisClient.TxPipeline()
+
+ pipe.LPush(ctx, key, data)
+ pipe.LTrim(ctx, key, 0, 4999)
+
+ _, err = pipe.Exec(ctx)
+ return err
+}
+
+func GetCandlesFromRedis(symbol, timeframe string, count int64) ([]*pb.Candle, error) {
+ results, err := RedisClient.LRange(ctx, candleKey(symbol, timeframe), 0, count-1).Result()
+ if err != nil {
+ return nil, fmt.Errorf("redis LRange: %w", err)
+ }
+
+ candles := make([]*pb.Candle, 0, len(results))
+ for _, r := range results {
+ c := &pb.Candle{}
+ if err := proto.Unmarshal([]byte(r), c); err != nil {
+ return nil, fmt.Errorf("unmarshal candle: %w", err)
+ }
+ candles = append(candles, c)
+ }
+ return candles, nil
+}
+
+func PublishCandleEventToRedis(symbol string, timeframe string, candle *pb.Candle) {
+ if err := RedisClient.Publish(ctx, candleKey(symbol, timeframe), candle).Err(); err != nil {
+ slog.Error("failed to pulish candle event", "error", err)
+ }
+}
diff --git a/apps/candle-service/internal/server.go b/apps/candle-service/internal/server.go
new file mode 100644
index 0000000..c6bd318
--- /dev/null
+++ b/apps/candle-service/internal/server.go
@@ -0,0 +1,115 @@
+package internal
+
+import (
+ "context"
+ "log"
+ "net"
+
+ "google.golang.org/grpc"
+ "google.golang.org/grpc/codes"
+ "google.golang.org/grpc/reflection"
+ "google.golang.org/grpc/status"
+
+ "github.com/sameerkrdev/nerve/apps/candle-service/internal/engine"
+ pbAggeration "github.com/sameerkrdev/nerve/packages/proto-defs/go/generated/aggeration/v1"
+)
+
+type Server struct {
+ router *engine.WorkerRouter
+ pbAggeration.UnimplementedCandleServiceServer
+}
+
+func NewGrpcServer(router *engine.WorkerRouter, netListener net.Listener) *grpc.Server {
+ s := &Server{
+ router: router,
+ }
+
+ srv := grpc.NewServer()
+ pbAggeration.RegisterCandleServiceServer(srv, s)
+ reflection.Register(srv)
+
+ go func() {
+ if err := srv.Serve(netListener); err != nil {
+ log.Fatalf("failed to serve: %v", err)
+ }
+ }()
+
+ return srv
+}
+
+// TODO: Implement the from, to feature :- inmemory, redis, clickhouse
+func (s *Server) GetCandles(ctx context.Context, req *pbAggeration.GetCandlesRequest) (*pbAggeration.GetCandlesResponse, error) {
+ tf := req.GetTimeframe()
+ symbol := req.GetSymbol()
+
+ if symbol == "" {
+ return nil, status.Error(codes.InvalidArgument, "symbol is required")
+ }
+
+ if tf == pbAggeration.Timeframe_TIMEFRAME_UNSPECIFIED {
+ return nil, status.Error(codes.InvalidArgument, "invalid timeframe")
+ }
+
+ candles, err := s.router.GetCandles(symbol, tf)
+ if err != nil {
+ return nil, status.Error(codes.Internal, "something went wrong. try again later")
+ }
+
+ candleBatch := &pbAggeration.CandleBatch{
+ Symbol: symbol,
+ Timeframe: tf,
+ Candles: candles,
+ }
+ response := &pbAggeration.GetCandlesResponse{
+ Result: &pbAggeration.GetCandlesResponse_Data{
+ Data: candleBatch,
+ },
+ }
+
+ return response, nil
+}
+
+// func HealthCheck(w http.ResponseWriter, r *http.Request) {
+// resp := utils.APIReponse{
+// Success: true,
+// Data: map[string]string{
+// "message": "Yooo!, I am healthy",
+// },
+// Error: "",
+// }
+
+// w.WriteHeader(http.StatusOK)
+// w.Header().Set("Content-Type", "application/json")
+// json.NewEncoder(w).Encode(resp)
+// }
+
+// func (s *Server) GetCandles(w http.ResponseWriter, r *http.Request) {
+// symbol := r.URL.Query().Get("symbol")
+// interval := r.URL.Query().Get("timeframe")
+
+// if symbol == "" || interval == "" {
+// w.WriteHeader(http.StatusBadRequest)
+// w.Header().Set("Content-Type", "application/json")
+// json.NewEncoder(w).Encode(utils.APIReponse{Success: false, Error: "symbol and timeframe are required"})
+// return
+// }
+
+// timeframeEnumVal, ok := pbAggeration.Timeframe_value[interval]
+// if !ok {
+// w.WriteHeader(http.StatusBadRequest)
+// w.Header().Set("Content-Type", "application/json")
+// json.NewEncoder(w).Encode(utils.APIReponse{Success: false, Error: "invalid timeframe"})
+// return
+// }
+
+// candles, err := s.router.GetCandles(symbol, pbAggeration.Timeframe(timeframeEnumVal))
+// if err != nil {
+// w.WriteHeader(http.StatusInternalServerError)
+// w.Header().Set("Content-Type", "application/json")
+// json.NewEncoder(w).Encode((utils.APIReponse{Success: false, Error: "something went wrong"}))
+// }
+
+// w.WriteHeader(http.StatusOK)
+// w.Header().Set("Content-Type", "application/json")
+// json.NewEncoder(w).Encode(utils.APIReponse{Success: true, Data: candles})
+// }
diff --git a/apps/candle-service/internal/utils/utils.go b/apps/candle-service/internal/utils/utils.go
new file mode 100644
index 0000000..fb17bd8
--- /dev/null
+++ b/apps/candle-service/internal/utils/utils.go
@@ -0,0 +1,18 @@
+package utils
+
+import (
+ "hash/fnv"
+)
+
+type APIReponse struct {
+ Success bool `json:"success"`
+ Data interface{} `json:"data"`
+ Error string `json:"error,omitempty"`
+}
+
+func Hash(str string) uint32 {
+ h := fnv.New32a()
+ h.Write([]byte(str))
+
+ return h.Sum32()
+}
diff --git a/apps/candle-service/makefile b/apps/candle-service/makefile
new file mode 100644
index 0000000..bcee480
--- /dev/null
+++ b/apps/candle-service/makefile
@@ -0,0 +1,21 @@
+APP_NAME := candle-service
+CMD_PATH := ./cmd/candle-service
+BUILD_DIR := bin
+ROOT_DIR := ../..
+
+post-install:
+ @echo "➡ Generating go.mod for proto-generated code..."
+ @mkdir -p $(ROOT_DIR)/packages/proto-defs/go/generated
+ @printf "module github.com/sameerkrdev/nerve/packages/proto-defs/go/generated\n\ngo 1.21\n" > \
+ $(ROOT_DIR)/packages/proto-defs/go/generated/go.mod
+
+dev:
+ air
+
+build:
+ @echo "➡ Building $(APP_NAME)..."
+ go build -o $(BUILD_DIR)/$(APP_NAME) $(CMD_PATH)
+
+tidy:
+ @echo "➡ Tidying modules..."
+ go mod tidy
diff --git a/apps/candle-service/package.json b/apps/candle-service/package.json
new file mode 100644
index 0000000..f81cb4b
--- /dev/null
+++ b/apps/candle-service/package.json
@@ -0,0 +1,14 @@
+{
+ "name": "@repo/candle-service",
+ "version": "1.0.0",
+ "description": "",
+ "main": "",
+ "scripts": {
+ "dev": "make post-install && make dev",
+ "build": "make post-install && make build",
+ "clean": "rm -rf dist"
+ },
+ "keywords": [],
+ "author": "",
+ "license": "ISC"
+}
diff --git a/apps/matching-engine/SYSTEM_DESIGN.md b/apps/matching-engine/SYSTEM_DESIGN.md
new file mode 100644
index 0000000..8957a02
--- /dev/null
+++ b/apps/matching-engine/SYSTEM_DESIGN.md
@@ -0,0 +1,1208 @@
+# Matching Engine — Full System Design Document
+
+## Table of Contents
+
+1. [System Overview](#1-system-overview)
+2. [Architecture](#2-architecture)
+3. [Directory Structure](#3-directory-structure)
+4. [Data Structures](#4-data-structures)
+5. [Core Algorithms](#5-core-algorithms)
+6. [Event System](#6-event-system)
+7. [gRPC API](#7-grpc-api)
+8. [Actor Model & Concurrency](#8-actor-model--concurrency)
+9. [Write-Ahead Log (WAL)](#9-write-ahead-log-wal)
+10. [Kafka Event Pipeline](#10-kafka-event-pipeline)
+11. [Order Lifecycle (End-to-End)](#11-order-lifecycle-end-to-end)
+12. [WAL Replay & Recovery](#12-wal-replay--recovery)
+13. [Error Handling](#13-error-handling)
+14. [Miro Diagram Guide](#14-miro-diagram-guide)
+
+---
+
+## 1. System Overview
+
+The matching engine is a **high-performance, event-driven, exchange-grade order matching system** written in Go. It processes orders for multiple trading symbols (BTCUSD, ETHUSD, SOLUSD) with strict **price-time priority** using an actor model that guarantees per-symbol sequential consistency.
+
+### Key Properties
+
+| Property | Value |
+| ------------------ | ----------------------------------- |
+| Language | Go |
+| Transport | gRPC (protobuf) |
+| Persistence | Write-Ahead Log (WAL) |
+| Event broadcast | Kafka (`engine-events` topic) |
+| Matching algorithm | Price-Time Priority (FIFO) |
+| Order types | LIMIT, MARKET |
+| Supported symbols | BTCUSD, ETHUSD, SOLUSD |
+| gRPC port | `localhost:50052` |
+| Kafka brokers | `localhost:19092`, `19093`, `19094` |
+
+---
+
+## 2. Architecture
+
+```
+┌─────────────────────────────────────────────────────────────────────┐
+│ CLIENTS │
+│ PlaceOrder / CancelOrder / ModifyOrder / SubscribeSymbol (gRPC) │
+└──────────────────────────┬──────────────────────────────────────────┘
+ │
+ ▼
+┌─────────────────────────────────────────────────────────────────────┐
+│ gRPC SERVER (:50052) │
+│ server.go — PlaceOrder, CancelOrder, ModifyOrder, Subscribe │
+└───┬──────────────────┬──────────────────┬───────────────────────────┘
+ │ │ │
+ ▼ ▼ ▼
+┌─────────┐ ┌─────────┐ ┌─────────┐
+│ ACTOR │ │ ACTOR │ │ ACTOR │
+│ BTCUSD │ │ ETHUSD │ │ SOLUSD │
+│inbox:8k │ │inbox:8k │ │inbox:8k │
+└────┬────┘ └────┬────┘ └────┬────┘
+ │ │ │
+ ▼ ▼ ▼
+┌─────────┐ ┌─────────┐ ┌─────────┐
+│MATCHING │ │MATCHING │ │MATCHING │
+│ ENGINE │ │ ENGINE │ │ ENGINE │
+│ BTCUSD │ │ ETHUSD │ │ SOLUSD │
+└────┬────┘ └────┬────┘ └────┬────┘
+ │ │ │
+ ├──────────────────────────────── ┤
+ │ DUAL OUTPUT │
+ ▼ ▼
+┌──────────────────┐ ┌────────────────────────┐
+│ WAL (per sym) │ │ gRPC Streams (push) │
+│ wal/BTCUSD/ │ │ All subscribed clients │
+│ wal/ETHUSD/ │ └────────────────────────┘
+│ wal/SOLUSD/ │
+└────────┬─────────┘
+ │ (batch read every 2s)
+ ▼
+┌────────────────────────────────────────┐
+│ Kafka Producer → engine-events │
+│ checkpoint.meta tracks last offset │
+└────────────────────────────────────────┘
+```
+
+---
+
+## 3. Directory Structure
+
+```
+matching-engine/
+├── cmd/
+│ └── matching-engine/
+│ └── main.go # Entry point: gRPC server init, actor setup
+├── internal/
+│ ├── engine.go # Core matching logic, price levels, actors (1372 lines)
+│ ├── server.go # gRPC handler implementations (126 lines)
+│ ├── actor_registry.go # Actor dispatch, message routing (162 lines)
+│ ├── kafka.go # Kafka producer wrapper (186 lines)
+│ ├── wal.go # Write-ahead log (469 lines)
+│ └── utils.go # Protobuf encoding helpers (112 lines)
+├── wal/
+│ ├── BTCUSD/
+│ │ ├── 0.log # Segment 0 (up to 64 MB)
+│ │ ├── 1.log # Segment 1 (auto-rotated)
+│ │ └── checkpoint.meta # Last Kafka-emitted offset
+│ ├── ETHUSD/
+│ └── SOLUSD/
+├── go.mod
+├── makefile
+└── .air.toml # Hot-reload config
+```
+
+---
+
+## 4. Data Structures
+
+### 4.1 Order
+
+The fundamental unit of the system. Stored in price levels as a doubly-linked list node.
+
+```
+Order
+├── Symbol int64 trading symbol code
+├── Price int64 limit price (0 for MARKET)
+├── AveragePrice int64 weighted avg fill price
+├── ExecutedValue int64 total fill value
+├── Quantity int64 original order quantity
+├── FilledQuantity int64 how much has been matched
+├── RemainingQuantity int64 quantity still to fill
+├── CancelledQuantity int64 quantity cancelled
+├── Side enum BUY | SELL
+├── Type enum LIMIT | MARKET
+├── Status enum PENDING | ACCEPTED | FILLED | PARTIAL_FILLED | CANCELLED | REJECTED
+├── UserID string owner of this order
+├── ClientOrderID string unique ID supplied by client (used as map key)
+├── StatusMessage string human-readable status reason
+├── ClientTimestamp *Timestamp when client sent
+├── GatewayTimestamp *Timestamp when gateway received
+├── EngineTimestamp *Timestamp when engine processed
+├── Prev *Order ← linked list pointer (within price level)
+├── Next *Order → linked list pointer (within price level)
+└── PriceLevel *PriceLevel pointer back to parent level
+```
+
+### 4.2 PriceLevel
+
+Represents all orders at a single price. Maintains FIFO order queue and links to adjacent price levels.
+
+```
+PriceLevel
+├── Price int64 the price point
+├── TotalVolume uint64 sum of RemainingQuantity of all orders
+├── OrderCount uint64 number of live orders
+├── HeadOrder *Order first (oldest) order — matched first
+├── TailOrder *Order last (newest) order — added to tail
+├── PrevPrice *PriceLevel next better price (toward book top)
+└── NextPrice *PriceLevel next worse price (toward book bottom)
+```
+
+**Price Level Sorted Order:**
+
+- **BID side**: descending (highest price = BestPriceLevel, i.e. 100 → 99 → 98...)
+- **ASK side**: ascending (lowest price = BestPriceLevel, i.e. 90 → 91 → 92...)
+
+### 4.3 OrderBookSide
+
+One half of the order book (either all bids or all asks).
+
+```
+OrderBookSide
+├── Side enum BUY | SELL
+├── PriceLevels map[int64]*PriceLevel O(1) lookup by price
+└── BestPriceLevel *PriceLevel pointer to top of book
+```
+
+### 4.4 MatchingEngine
+
+The state machine for one symbol. Always lives inside exactly one SymbolActor.
+
+```
+MatchingEngine
+├── Symbol string
+├── Bids *OrderBookSide all buy orders
+├── Asks *OrderBookSide all sell orders
+├── AllOrders map[string]*Order all active orders by ClientOrderID
+├── TotalMatches uint64
+├── TotalVolume uint64
+├── TradeSequence uint64 monotonic trade counter
+├── OrderSequence uint64 monotonic order counter
+└── wal *SymbolWAL reference for logging
+```
+
+### 4.5 SymbolActor
+
+Owns one matching engine. All operations are serialized through the inbox channel.
+
+```
+SymbolActor
+├── symbol string
+├── inbox chan EngineMsg buffered, capacity 8192
+├── engine *MatchingEngine
+├── wal *SymbolWAL
+├── kafkaEmitter *KafkaProducerWorker
+├── grpcStreams []SubscribeServer active subscriber streams
+└── mu sync.RWMutex guards grpcStreams
+```
+
+### 4.6 SymbolWAL
+
+Per-symbol segmented write-ahead log.
+
+```
+SymbolWAL
+├── dirPath string e.g. "wal/BTCUSD"
+├── symbol string
+├── bufferWriter *bufio.Writer in-memory write buffer
+├── nextOffset uint64 next sequence number to assign
+├── currentSegmentFile *os.File file handle
+├── maxFileSize int64 67,108,864 bytes (64 MB)
+├── currentSegmentIndex int 0 → 1 → 2 ... (0.log, 1.log, ...)
+├── shouldFsync bool fsync after flush
+├── syncIntervalMM int 400 ms periodic flush
+├── syncTimer *time.Timer
+├── ctx / cancel context for goroutine lifecycle
+└── mu sync.Mutex guards writes and rotation
+```
+
+### 4.7 KafkaProducerWorker
+
+Reads WAL in batches and publishes to Kafka.
+
+```
+KafkaProducerWorker
+├── producer sarama.SyncProducer
+├── Symbol string
+├── dirPath string
+├── batchSize int 300 events per batch
+├── emitTimeMM int 2000 ms between emit ticks
+├── wal *SymbolWAL
+├── checkpointFile *os.File checkpoint.meta
+└── ctx context.Context
+```
+
+### 4.8 Trade
+
+Immutable record of a matched trade.
+
+```
+Trade
+├── TradeID string "{Symbol}-T{timestamp}-{sequence}"
+├── Symbol string
+├── TradeSequence uint64
+├── Price int64 price at which trade executed (resting order's price)
+├── Quantity uint64 matched quantity
+├── Timeline *Timestamp
+├── BuyerID string
+├── SellerID string
+├── BuyOrderID string
+├── SellOrderID string
+└── IsBuyerMaker bool true if the resting order was the BUY side
+```
+
+### 4.9 Message Types (Actor Inbox)
+
+```
+PlaceOrderMsg
+├── Order *Order
+├── replay chan *AddOrderInternalResponse
+└── Err chan error
+
+CancelOrderMsg
+├── OrderID string
+├── UserID string
+├── Symbol string
+├── replay chan *CancelOrderInternalResponse
+└── Err chan error
+
+ModifyOrderMsg
+├── OrderID string
+├── UserID string
+├── Symbol string
+├── NewPrice *int64
+├── NewQuantity *int64
+├── replay chan *ModifyOrderInternalResponse
+└── Err chan error
+```
+
+---
+
+## 5. Core Algorithms
+
+### 5.1 Matching Algorithm (Price-Time Priority)
+
+**Entry point:** `MatchingEngine.AddOrderInternal(order)`
+
+```
+1. Check duplicate ClientOrderID → error if found
+
+2. Determine opposite book:
+ BUY order → check Asks
+ SELL order → check Bids
+
+3. MARKET order special case:
+ If opposite book empty → REJECT (no liquidity)
+
+4. Matching loop:
+ while incoming.RemainingQuantity > 0 AND CanMatch(incoming, oppositeBook):
+
+ a. Get BestPriceLevel from opposite book
+ b. Get HeadOrder (oldest order at best price) — FIFO
+ c. matchQty = min(incoming.RemainingQuantity, resting.RemainingQuantity)
+ d. Execute trade at RESTING order's price
+ e. incoming.RemainingQuantity -= matchQty
+ incoming.FilledQuantity += matchQty
+ resting.RemainingQuantity -= matchQty
+ resting.FilledQuantity += matchQty
+ f. Update average prices for both sides
+ g. If resting.RemainingQuantity == 0:
+ Remove resting from PriceLevel
+ Add resting to filledRestingOrders[]
+ If PriceLevel.OrderCount == 0: RemovePriceLevel()
+ h. Record Trade
+
+5. After loop:
+ Set incoming status:
+ incoming.RemainingQuantity == 0 → FILLED
+ incoming.FilledQuantity > 0 → PARTIAL_FILLED (resting)
+ else → stays PENDING (goes to book)
+
+6. MARKET with remaining after loop → CANCEL remainder
+
+7. LIMIT with remaining → add to own book:
+ GetOrCreatePriceLevel(incoming.Price)
+ PriceLevel.Push(incoming)
+ AllOrders[incoming.ClientOrderID] = incoming
+```
+
+### 5.2 CanMatch (Price Validation)
+
+```
+MARKET order: always true (liquidity check done before loop)
+
+LIMIT BUY: best ASK price <= incoming BUY price → match
+LIMIT SELL: best BID price >= incoming SELL price → match
+```
+
+### 5.3 Average Price Calculation
+
+```
+New FilledQty = old FilledQty + matchQty
+New AvgPrice = ((old AvgPrice × old FilledQty) + (matchPrice × matchQty)) / New FilledQty
+ExecutedValue = AvgPrice × FilledQuantity
+```
+
+### 5.4 Price Level Linked List Insertion (LinkPriceLevel)
+
+```
+BID side (descending insertion):
+ Start at BestPriceLevel
+ Traverse NextPrice until:
+ current == nil (new worst price) OR
+ current.Price < newLevel.Price (insert before current)
+ If new level beats BestPriceLevel → becomes new best
+
+ASK side (ascending insertion):
+ Start at BestPriceLevel
+ Traverse NextPrice until:
+ current == nil (new worst price) OR
+ current.Price > newLevel.Price (insert before current)
+ If new level beats BestPriceLevel → becomes new best
+```
+
+### 5.5 Modify Order Logic
+
+Three possible outcomes:
+
+| Condition | Action |
+| ------------------------------------------------- | --------------------------------------------------------- |
+| Price changed OR new quantity > original quantity | **Replace**: cancel existing order + place new order |
+| New quantity < current remaining quantity | **Reduce**: update quantities in-place (no priority loss) |
+| No meaningful change | **No-op** |
+
+Validation:
+
+- Order must exist and belong to the requesting user
+- Order must still be active (RemainingQuantity > 0)
+- New quantity must be >= already-executed quantity
+
+---
+
+## 6. Event System
+
+### 6.1 Event Types
+
+| EventType | Trigger | Data Payload | Written to WAL | Sent to gRPC Streams |
+| ---------------------- | ----------------------------------------- | ---------------------------------- | -------------- | -------------------- |
+| `ORDER_ACCEPTED` | Order passes validation | `OrderStatusEvent` | Yes | Yes |
+| `ORDER_REJECTED` | MARKET with no liquidity, or duplicate ID | `OrderStatusEvent` | Yes | Yes |
+| `TRADE_EXECUTED` | Two orders match | `TradeEvent` | Yes | Yes |
+| `ORDER_FILLED` | Order fully matched | `OrderStatusEvent` | Yes | Yes |
+| `ORDER_PARTIAL_FILLED` | Order partially matched, resting | `OrderStatusEvent` | Yes | Yes |
+| `ORDER_CANCELLED` | User cancel or replace during modify | `OrderStatusEvent` | Yes | Yes |
+| `ORDER_REDUCED` | Quantity reduced in-place | `OrderReducedEvent` | Yes | Yes |
+| `DEPTH` | Any book change | `DepthEvent` (top 100 levels) | **No** | Yes |
+| `TICKER` | Trade executes | `TickerEvent` (last/bid/ask price) | **No** | Yes |
+
+> DEPTH and TICKER are ephemeral market-data events — they are NOT persisted to WAL and NOT replayed during recovery.
+
+### 6.2 Event Sequence for a Matched Order
+
+```
+Incoming LIMIT BUY order partially fills 2 resting SELL orders:
+
+1. ORDER_ACCEPTED (incoming order acknowledged)
+2. TRADE_EXECUTED (match #1 with resting order A)
+3. DEPTH (book updated after match #1)
+4. TICKER (price update after match #1)
+5. ORDER_FILLED (resting order A fully filled)
+6. TRADE_EXECUTED (match #2 with resting order B)
+7. DEPTH (book updated after match #2)
+8. TICKER (price update after match #2)
+9. ORDER_FILLED (resting order B fully filled)
+10. ORDER_PARTIAL_FILLED (incoming still has remaining qty → rests in book)
+11. DEPTH (final book state)
+```
+
+### 6.3 buildEvents Logic
+
+```
+func buildEvents(order, trades, filledResting):
+
+ if order.REJECTED:
+ return [ORDER_REJECTED]
+
+ events = [ORDER_ACCEPTED]
+
+ for each trade:
+ events += [TRADE_EXECUTED]
+ events += [DEPTH]
+ events += [TICKER]
+
+ for each filledRestingOrder:
+ events += [ORDER_FILLED]
+
+ switch order.Status:
+ FILLED → events += [ORDER_FILLED]
+ PARTIAL_FILLED → events += [ORDER_PARTIAL_FILLED]
+ CANCELLED → events += [ORDER_CANCELLED]
+
+ events += [DEPTH] // always emit final depth
+
+ return events
+```
+
+### 6.4 Depth Event (Market Depth)
+
+Collects top 100 price levels from each side:
+
+- **Bids**: descending from BestBidPrice (100, 99, 98...)
+- **Asks**: ascending from BestAskPrice (90, 91, 92...)
+
+Each level includes: price, total volume, order count.
+
+### 6.5 Ticker Event
+
+```
+TickerEvent {
+ last_price: price of most recent trade
+ bid_price: current best bid (BestPriceLevel.Price from Bids)
+ ask_price: current best ask (BestPriceLevel.Price from Asks)
+}
+```
+
+---
+
+## 7. gRPC API
+
+**Service: `MatchingEngine`** (port 50052)
+
+### PlaceOrder
+
+```
+Request:
+ PlaceOrderRequest {
+ symbol, price, quantity
+ side (BUY|SELL), type (LIMIT|MARKET)
+ user_id, client_order_id
+ client_timestamp, gateway_timestamp
+ }
+
+Response:
+ PlaceOrderResponse {
+ order { all Order fields after processing }
+ }
+
+Behavior:
+ - Synchronous: blocks until matching complete
+ - Returns final order state (FILLED, PARTIAL_FILLED, REJECTED, or PENDING)
+ - All events generated and persisted before response
+```
+
+### CancelOrder
+
+```
+Request:
+ CancelOrderRequest {
+ order_id, user_id, symbol
+ }
+
+Response:
+ CancelOrderResponse {
+ order { cancelled order state }
+ }
+
+Errors:
+ - "order not found"
+ - "unauthorized cancel" (wrong user_id)
+ - "order already completed" (remaining qty = 0)
+```
+
+### ModifyOrder
+
+```
+Request:
+ ModifyOrderRequest {
+ order_id, user_id, symbol
+ new_price (optional)
+ new_quantity (optional)
+ }
+
+Response:
+ ModifyOrderResponse {
+ order { final order state }
+ }
+
+Errors:
+ - "order not found"
+ - "order does not belong to this user"
+ - "order symbol mismatch"
+ - "order is not modifiable"
+ - "new quantity < executed quantity"
+```
+
+### SubscribeSymbol
+
+```
+Request:
+ SubscribeRequest { symbol }
+
+Stream:
+ → EngineEvent (continuously pushed)
+
+Behavior:
+ - Server pushes events to client as they occur
+ - Client added to actor's grpcStreams list
+ - Removed automatically when context cancelled
+ - Receives ALL event types for that symbol
+```
+
+---
+
+## 8. Actor Model & Concurrency
+
+### 8.1 Goroutine Topology
+
+```
+main goroutine
+│
+├── gRPC server (managed by grpc framework)
+│
+├── SymbolActor.Run() [BTCUSD] ← single goroutine per symbol
+│ ├── wal.keepSyncing() ← periodic WAL flush (400ms)
+│ └── kafkaEmitter.Run() ← periodic Kafka batch (2000ms)
+│
+├── SymbolActor.Run() [ETHUSD]
+│ ├── wal.keepSyncing()
+│ └── kafkaEmitter.Run()
+│
+└── SymbolActor.Run() [SOLUSD]
+ ├── wal.keepSyncing()
+ └── kafkaEmitter.Run()
+
+Total goroutines: 1 (main) + 1 (gRPC) + 3×3 (per symbol) = ~11 goroutines
+```
+
+### 8.2 Message Flow (Request/Reply via Channels)
+
+```
+gRPC handler (goroutine N)
+│
+│ Create reply + error channels (buffered 1)
+│ Build PlaceOrderMsg{Order, replay, Err}
+│ actor.inbox <- msg
+│
+│ block on:
+│ select {
+│ case res := <-replay → return res
+│ case err := <-Err → return error
+│ }
+│
+└──────────────────────────────────────────┐
+ ▼
+ actor.Run() loop
+ │
+ │ msg := <-inbox
+ │ switch msg.(type):
+ │ PlaceOrderMsg:
+ │ resp, events, err = engine.AddOrderInternal()
+ │ handle events (WAL, streams)
+ │ msg.replay <- resp
+ │ continue
+ │
+ └──────────────────
+```
+
+### 8.3 Locking Summary
+
+| Lock | Owner | Protects | Pattern |
+| -------------------------- | ------------- | ------------------------------------- | --------------------------------------------- |
+| `SymbolActor.mu` (RWMutex) | SymbolActor | `grpcStreams` slice | Write: subscribe/unsubscribe. Read: broadcast |
+| `SymbolWAL.mu` (Mutex) | SymbolWAL | All file operations, sequence counter | Every WAL write, rotation, sync |
+| `kafkaOnce` (sync.Once) | package-level | Kafka producer initialization | One-time singleton |
+
+**No locks on MatchingEngine** — all access is serialized through actor inbox (single goroutine processes all messages).
+
+---
+
+## 9. Write-Ahead Log (WAL)
+
+### 9.1 File Layout
+
+```
+wal/
+└── {SYMBOL}/
+ ├── 0.log first segment
+ ├── 1.log second segment (after rotation)
+ └── checkpoint.meta last Kafka-emitted WAL offset (uint64 as string)
+```
+
+### 9.2 Entry Wire Format
+
+```
+┌──────────────────────────────────────────────────────┐
+│ [4 bytes] entry length (little-endian int32) │
+├──────────────────────────────────────────────────────┤
+│ WAL_Entry (protobuf): │
+│ sequence_number uint64 │
+│ data bytes (serialized EngineEvent) │
+│ CRC uint32 (CRC32 of data+seq_bytes) │
+└──────────────────────────────────────────────────────┘
+```
+
+### 9.3 Write Path
+
+```
+WriteEntry(data []byte):
+ 1. Lock mutex
+ 2. rotateFile() if (file_size + buffered) >= 64 MB
+ 3. Encode sequence as [8]byte little-endian
+ 4. CRC = CRC32(data + seqBytes)
+ 5. Build WAL_Entry{sequence, data, CRC}
+ 6. Marshal to protobuf bytes
+ 7. Write [4-byte length][marshaled entry] to bufio.Writer
+ 8. Increment nextOffset
+ 9. Unlock mutex
+```
+
+### 9.4 File Rotation
+
+```
+rotateFile():
+ if currentFile.Size() + bufferWriter.Buffered() >= maxFileSize:
+ Sync() // flush + fsync
+ currentFile.Close()
+ currentSegmentIndex++
+ Create new file: "{index}.log"
+ Reset bufio.Writer on new file
+```
+
+### 9.5 Periodic Sync
+
+```
+keepSyncing() goroutine:
+ for {
+ select {
+ case <-syncTimer.C:
+ Lock()
+ bufferWriter.Flush()
+ file.Sync() if shouldFsync
+ Unlock()
+ reset timer (400ms)
+ }
+ }
+```
+
+**Durability guarantee**: Events buffered in memory for at most 400ms before reaching disk. On crash within 400ms window, last events may be lost.
+
+### 9.6 Sequence Numbering
+
+- Starts at 0 from empty WAL
+- On restart: reads last entry in most recent `.log` file, sets `nextOffset = lastSeq + 1`
+- Monotonically increasing, never resets across file rotations
+- Used as stable offset for Kafka checkpoint
+
+### 9.7 Corruption Detection
+
+Every read verifies:
+
+```
+actual = CRC32(entry.data + encode(entry.sequence_number))
+if actual != entry.CRC → error "CRC mismatch: data may be corrupted"
+```
+
+---
+
+## 10. Kafka Event Pipeline
+
+### 10.1 Producer Configuration
+
+```
+RequiredAcks = WaitForAll // all ISR replicas must ACK
+MaxRetries = 5
+Idempotent = true // exactly-once per session
+MaxOpenRequests = 1 // required for idempotent mode
+Topic = "engine-events"
+MessageKey = Symbol // ensures all symbol events → same partition
+```
+
+### 10.2 Batch Emit Loop
+
+```
+kafkaEmitter.Run():
+ every 2000ms:
+ processBatch()
+
+processBatch():
+ 1. checkpoint = loadCheckpoint() // read checkpoint.meta
+ 2. entries = wal.ReadFromTo( // read WAL batch
+ from: checkpoint + 1,
+ to: checkpoint + batchSize // 300 events
+ )
+ 3. if len(entries) == 0: return
+ 4. messages = build Kafka messages from entries
+ 5. err = producer.SendMessages(messages)
+ 6. if err != nil:
+ log error
+ return // ← do NOT save checkpoint → batch will retry
+ 7. saveCheckpoint(entries.last.sequence_number)
+```
+
+### 10.3 At-Least-Once Delivery Guarantee
+
+```
+Timeline:
+ t=0 WAL entry written (seq=100)
+ t=2s batch sent to Kafka (seq 1-100)
+ t=2s Kafka ACK received
+ t=2s checkpoint.meta = "100"
+
+ If crash at t=2s before checkpoint saved:
+ t=restart checkpoint still = "50"
+ t=restart batch resent for seq 51-100 (duplicates for 51-100)
+ Consumers must handle idempotency by seq number or event ID
+```
+
+### 10.4 Kafka Topic Partitioning
+
+```
+engine-events topic
+├── Partition 0 — BTCUSD events (key="BTCUSD")
+├── Partition 1 — ETHUSD events (key="ETHUSD")
+└── Partition 2 — SOLUSD events (key="SOLUSD")
+```
+
+---
+
+## 11. Order Lifecycle (End-to-End)
+
+### 11.1 LIMIT BUY with Partial Fill
+
+```
+Client → PlaceOrder(BUY, LIMIT, price=100, qty=10)
+
+[gRPC handler]
+ Build Order struct
+ PlaceOrderMsg → actor.inbox
+
+[Actor.Run()]
+ AddOrderInternal(order)
+ MatchOrder():
+ BestASK = 98 (≤ 100, can match)
+ HeadOrder at 98 = resting SELL qty=6
+ matchQty = min(10, 6) = 6
+ Trade { price=98, qty=6 }
+ resting SELL → FILLED, removed from book
+ BestASK = 99 (≤ 100, can match)
+ HeadOrder at 99 = resting SELL qty=7
+ matchQty = min(4, 7) = 4
+ Trade { price=99, qty=4 }
+ incoming BUY → FILLED (remaining=0)
+ resting SELL qty → 3 remaining
+
+ AvgPrice = (98×6 + 99×4) / 10 = 98.4
+
+ buildEvents():
+ ORDER_ACCEPTED
+ TRADE_EXECUTED (qty=6, price=98)
+ DEPTH
+ TICKER
+ ORDER_FILLED (resting SELL at 98)
+ TRADE_EXECUTED (qty=4, price=99)
+ DEPTH
+ TICKER
+ ORDER_FILLED (incoming BUY)
+ DEPTH
+
+ For each event:
+ → Write to WAL (except DEPTH/TICKER)
+ → Send to all gRPC subscribers
+
+ Reply → actor sends on replay channel
+
+[gRPC handler]
+ recv from replay channel
+ Return PlaceOrderResponse
+```
+
+### 11.2 MARKET Order Rejection
+
+```
+Client → PlaceOrder(BUY, MARKET, qty=5)
+
+MatchOrder():
+ Asks.IsEmpty() == true
+ incoming.Status = REJECTED
+ incoming.StatusMessage = "Market order rejected: no liquidity..."
+
+buildEvents():
+ return [ORDER_REJECTED]
+
+WAL.WriteEntry(ORDER_REJECTED)
+Stream: send ORDER_REJECTED to subscribers
+Reply: PlaceOrderResponse{Status: REJECTED}
+```
+
+### 11.3 Cancel Order
+
+```
+Client → CancelOrder(order_id="X", user_id="U", symbol="BTCUSD")
+
+CancelOrderInternal():
+ order = AllOrders["X"]
+ if order.UserID != "U" → error "unauthorized cancel"
+ if order.RemainingQuantity == 0 → error "order already completed"
+ Remove from PriceLevel
+ if PriceLevel.OrderCount == 0: RemovePriceLevel()
+ order.CancelledQuantity = order.RemainingQuantity
+ order.RemainingQuantity = 0
+ order.Status = CANCELLED
+ buildCancelEvent()
+
+WAL.WriteEntry(ORDER_CANCELLED)
+Stream: send ORDER_CANCELLED
+Reply: CancelOrderResponse
+```
+
+---
+
+## 12. WAL Replay & Recovery
+
+### 12.1 Startup Sequence
+
+```
+main():
+ for each symbol:
+ 1. OpenWAL(symbol)
+ - find last .log file
+ - read last entry → set nextOffset = lastSeq + 1
+ 2. NewMatchingEngine(symbol, wal)
+ 3. NewKafkaProducerWorker(symbol, wal)
+ 4. actor = NewSymbolActor(symbol, engine, wal, kafka)
+ 5. actor.replayWAL(from=0) ← reconstruct order book
+ 6. go actor.Run()
+ 7. go wal.keepSyncing()
+ 8. go kafkaEmitter.Run()
+```
+
+### 12.2 replayWAL Event Handling
+
+```
+replayWAL(from uint64):
+ entries = wal.ReadFromToLast(from)
+ for each entry:
+ event = unmarshal(entry.data)
+ switch event.Type:
+
+ ORDER_ACCEPTED:
+ order = reconstruct from OrderStatusEvent
+ GetOrCreatePriceLevel(order.Price)
+ PriceLevel.Push(order)
+ AllOrders[order.ClientOrderID] = order
+
+ TRADE_EXECUTED:
+ update buy and sell orders: qty, avg price, status
+
+ ORDER_CANCELLED:
+ remove order from PriceLevel
+ delete from AllOrders
+
+ ORDER_REDUCED:
+ update remaining + cancelled qty on order
+
+ ORDER_REJECTED:
+ delete from AllOrders (if exists)
+
+ ORDER_FILLED:
+ remove from PriceLevel
+ delete from AllOrders
+```
+
+**Result**: After replay, `MatchingEngine.Bids`, `MatchingEngine.Asks`, and `AllOrders` are identical to their state at the moment of the crash.
+
+---
+
+## 13. Error Handling
+
+### 13.1 Validation Errors (Returned to Client)
+
+| Scenario | Error Message |
+| --------------------------------- | -------------------------------------------------------- |
+| Duplicate ClientOrderID | `"Duplicate Order ID: {id}"` |
+| MARKET with no opposite liquidity | `"Market order rejected: no liquidity on opposite side"` |
+| Cancel: order not found | `"order not found"` |
+| Cancel: wrong user | `"unauthorized cancel"` |
+| Cancel: already completed | `"order already completed"` |
+| Modify: new qty < executed | `"new quantity < executed quantity"` |
+| Modify: order not modifiable | `"order is not modifiable"` |
+
+### 13.2 I/O Error Strategy
+
+| Error | Strategy |
+| ----------------------- | --------------------------------------------------------------- |
+| WAL write failure | Skip remaining events, return error to caller via `Err` channel |
+| Kafka emit failure | Log error, **do not checkpoint**, retry on next 2s tick |
+| Checkpoint save failure | Log error, continue (next batch will retry from same offset) |
+| WAL CRC mismatch | Return error from read function, propagates to replay |
+
+### 13.3 Stream Error Handling
+
+```
+On stream.Send(event) failure:
+ error logged
+ stream removed from grpcStreams list
+ client must reconnect via SubscribeSymbol
+```
+
+---
+
+## 14. Miro Diagram Guide
+
+> Miro MCP is not connected to this session. Use the layout below to build the diagram manually in Miro. Create one board with 5 frames.
+
+---
+
+### Frame 1 — High-Level Architecture
+
+**Shapes to create (left to right, top to bottom):**
+
+```
+[Clients]
+ → (arrow) → [gRPC Server :50052]
+ → (arrow) → [Actor: BTCUSD | inbox:8192]
+ → (arrow) → [Actor: ETHUSD | inbox:8192]
+ → (arrow) → [Actor: SOLUSD | inbox:8192]
+
+Each Actor:
+ → (arrow down) → [Matching Engine]
+ → (arrow right) → [WAL wal/{SYMBOL}/]
+ → (arrow right, dashed) → [gRPC Streams → Subscribers]
+
+WAL:
+ → (arrow down, timer icon) → [Kafka Producer Worker]
+ → (arrow right) → [Kafka: engine-events topic]
+ ↓ Partition 0: BTCUSD
+ ↓ Partition 1: ETHUSD
+ ↓ Partition 2: SOLUSD
+
+checkpoint.meta
+ → (arrow) → [Kafka Producer Worker] (shows last emitted offset)
+```
+
+**Color coding:**
+
+- Clients: blue
+- gRPC layer: purple
+- Actor + Engine: green
+- WAL: orange
+- Kafka: red
+
+---
+
+### Frame 2 — Order Book Structure
+
+**Shapes to create:**
+
+```
+OrderBookSide (BID)
+ BestPriceLevel → PriceLevel[100]
+ TotalVolume: 50
+ OrderCount: 3
+ HeadOrder → [Order A] ↔ [Order B] ↔ [Order C]
+ ↓
+ PriceLevel[99]
+ HeadOrder → [Order D] ↔ [Order E]
+ ↓
+ PriceLevel[98]
+ HeadOrder → [Order F]
+
+OrderBookSide (ASK)
+ BestPriceLevel → PriceLevel[101]
+ HeadOrder → [Order G] ↔ [Order H]
+ ↓
+ PriceLevel[102]
+ HeadOrder → [Order I]
+```
+
+**Notes to add:**
+
+- "BID side sorted descending (best = highest)"
+- "ASK side sorted ascending (best = lowest)"
+- "Orders at each level follow FIFO — Head matched first"
+- "PrevPrice / NextPrice = doubly linked list"
+- "Prev / Next on Order = FIFO queue within level"
+
+---
+
+### Frame 3 — Order Matching Flow
+
+**Flowchart (top to bottom):**
+
+```
+START: PlaceOrder gRPC call
+ ↓
+Build Order struct
+ ↓
+Send PlaceOrderMsg to actor.inbox (block on reply channel)
+ ↓
+[Actor.Run() receives from inbox]
+ ↓
+AddOrderInternal()
+ ↓
+Duplicate ClientOrderID? ──YES──→ Error to caller
+ ↓ NO
+Order type = MARKET? ──YES──→ Opposite book empty? ──YES──→ ORDER_REJECTED
+ ↓ ↓ NO
+ ↓ continue to match
+ ↓
+[Matching Loop]
+ Can match? (CanMatch()) ──NO──→ [Exit loop]
+ ↓ YES
+ Get BestPriceLevel from opposite book
+ ↓
+ Get HeadOrder (FIFO)
+ ↓
+ matchQty = min(incoming.remaining, resting.remaining)
+ ↓
+ ExecuteTrade() → create Trade record
+ ↓
+ Update both orders: qty, avgPrice, status
+ ↓
+ resting.remaining == 0? ──YES──→ Remove from PriceLevel → filledResting[]
+ ↓ NO (partial fill of resting)
+ ↓
+ incoming.remaining == 0? ──YES──→ Exit loop (FILLED)
+ ↓ NO
+ [Loop back to CanMatch]
+ ↓
+[After loop]
+ MARKET with remaining? ──YES──→ Cancel remainder
+ ↓ NO
+ LIMIT with remaining? ──YES──→ Add to own order book (GetOrCreatePriceLevel → Push)
+ ↓
+buildEvents()
+ ↓
+For each event: WAL.WriteEntry() + stream.Send() to subscribers
+ ↓
+Send reply on replay channel
+ ↓
+gRPC returns PlaceOrderResponse
+ ↓
+END
+```
+
+---
+
+### Frame 4 — Event Flow & WAL Pipeline
+
+**Swimlane diagram (3 lanes):**
+
+```
+Lane 1: Matching Engine
+ → Generates events []EngineEvent
+
+Lane 2: Per-Event Processing
+ For each event:
+ Is DEPTH or TICKER?
+ YES → only → gRPC stream.Send()
+ NO → WAL.WriteEntry(event) + gRPC stream.Send()
+
+Lane 3: WAL → Kafka (async)
+ keepSyncing() goroutine:
+ Every 400ms → flush bufio.Writer → fsync
+
+ kafkaEmitter.Run() goroutine:
+ Every 2000ms:
+ read checkpoint.meta
+ read WAL batch [offset+1 .. offset+300]
+ send batch to Kafka "engine-events" (key=symbol)
+ if ACK: save new checkpoint
+ if FAIL: do not checkpoint (retry next tick)
+```
+
+**Add notes:**
+
+- "WAL seq numbers are monotonic across file rotations"
+- "Each .log file max 64 MB then rotated"
+- "CRC32 checksum on every WAL entry"
+- "Kafka key=Symbol ensures partition affinity"
+- "At-least-once: checkpoint saved ONLY after Kafka ACK"
+
+**Event sequence box (right side):**
+
+```
+ORDER_ACCEPTED
+TRADE_EXECUTED × N
+DEPTH × N ← not WAL
+TICKER × N ← not WAL
+ORDER_FILLED × N (resting)
+ORDER_FILLED / PARTIAL_FILLED / CANCELLED (incoming)
+DEPTH ← not WAL
+```
+
+---
+
+### Frame 5 — Startup & WAL Recovery
+
+**Timeline diagram (left to right):**
+
+```
+CRASH ──────────────────────────────────────────────────→ RESTART
+ ↓
+ OpenWAL()
+ Find last .log
+ Read last entry
+ nextOffset = lastSeq+1
+ ↓
+ replayWAL(from=0)
+ ↓
+ For each WAL entry:
+ ┌─────────────────────┐
+ │ ORDER_ACCEPTED │
+ │ → add to book │
+ │ ORDER_FILLED │
+ │ → remove from book │
+ │ ORDER_CANCELLED │
+ │ → remove from book │
+ │ TRADE_EXECUTED │
+ │ → update quantities │
+ │ ORDER_REDUCED │
+ │ → update quantities │
+ └─────────────────────┘
+ ↓
+ Order book fully reconstructed
+ ↓
+ Actor.Run() starts
+ kafkaEmitter.Run() starts
+ (resumes from checkpoint.meta)
+```
+
+**Add note:**
+
+- "DEPTH and TICKER events are NOT in WAL — they are reconstructed live"
+- "Kafka checkpoint is independent of WAL — Kafka resumes from checkpoint.meta"
+
+---
+
+### Miro Setup Steps
+
+1. Open Miro → New Board → name it **"Matching Engine Architecture"**
+2. Create 5 frames (use F key): name them as above
+3. In each frame, add shapes using the toolbar:
+ - Rectangles for systems/components
+ - Diamonds for decisions
+ - Swimlanes for event flows
+ - Arrows for data flow (solid = sync, dashed = async)
+4. Color code:
+ - **Blue**: external clients
+ - **Purple**: gRPC layer
+ - **Green**: actor + matching engine
+ - **Orange**: WAL
+ - **Red**: Kafka
+
+---
+
+_Document generated: 2026-04-16_
+_Codebase: ~2,475 lines of Go across 7 files_
diff --git a/apps/matching-engine/go.mod b/apps/matching-engine/go.mod
index 7f88deb..45c8dbd 100644
--- a/apps/matching-engine/go.mod
+++ b/apps/matching-engine/go.mod
@@ -3,12 +3,12 @@ module github.com/sameerkrdev/nerve/apps/matching-engine
go 1.25.4
require (
- google.golang.org/grpc v1.78.0
+ google.golang.org/grpc v1.80.0
google.golang.org/protobuf v1.36.11
)
require (
- github.com/davecgh/go-spew v1.1.1 // indirect
+ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/eapache/go-resiliency v1.7.0 // indirect
github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect
github.com/eapache/queue v1.1.0 // indirect
@@ -19,16 +19,18 @@ require (
github.com/jcmturner/gofork v1.7.6 // indirect
github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect
github.com/jcmturner/rpc/v2 v2.0.3 // indirect
- github.com/klauspost/compress v1.18.1 // indirect
- github.com/pierrec/lz4/v4 v4.1.22 // indirect
+ github.com/klauspost/compress v1.18.3 // indirect
+ github.com/pierrec/lz4/v4 v4.1.25 // indirect
+ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9 // indirect
- golang.org/x/crypto v0.44.0 // indirect
+ go.opentelemetry.io/otel v1.41.0 // indirect
+ golang.org/x/crypto v0.50.0 // indirect
)
require (
github.com/IBM/sarama v1.46.3
- golang.org/x/net v0.47.0 // indirect
- golang.org/x/sys v0.38.0 // indirect
- golang.org/x/text v0.31.0 // indirect
- google.golang.org/genproto/googleapis/rpc v0.0.0-20251029180050-ab9386a59fda // indirect
+ golang.org/x/net v0.53.0 // indirect
+ golang.org/x/sys v0.43.0 // indirect
+ golang.org/x/text v0.36.0 // indirect
+ google.golang.org/genproto/googleapis/rpc v0.0.0-20260427160629-7cedc36a6bc4 // indirect
)
diff --git a/apps/matching-engine/go.sum b/apps/matching-engine/go.sum
index f05d0e3..655d370 100644
--- a/apps/matching-engine/go.sum
+++ b/apps/matching-engine/go.sum
@@ -1,8 +1,9 @@
github.com/IBM/sarama v1.46.3 h1:njRsX6jNlnR+ClJ8XmkO+CM4unbrNr/2vB5KK6UA+IE=
github.com/IBM/sarama v1.46.3/go.mod h1:GTUYiF9DMOZVe3FwyGT+dtSPceGFIgA+sPc5u6CBwko=
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
github.com/eapache/go-resiliency v1.7.0 h1:n3NRTnBn5N0Cbi/IeOHuQn9s2UwVUH7Ga0ZWcP+9JTA=
github.com/eapache/go-resiliency v1.7.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho=
github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 h1:Oy0F4ALJ04o5Qqpdz8XLIpNA3WM/iSIXqxtqo7UGVws=
@@ -34,12 +35,10 @@ github.com/jcmturner/gokrb5/v8 v8.4.4 h1:x1Sv4HaTpepFkXbt2IkL29DXRf8sOfZXo8eRKh6
github.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP+F6aCACiMrs=
github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY=
github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc=
-github.com/klauspost/compress v1.18.1 h1:bcSGx7UbpBqMChDtsF28Lw6v/G94LPrrbMbdC3JH2co=
-github.com/klauspost/compress v1.18.1/go.mod h1:ZQFFVG+MdnR0P+l6wpXgIL4NTtwiKIdBnrBd8Nrxr+0=
-github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU=
-github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
-github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/klauspost/compress v1.18.3 h1:9PJRvfbmTabkOX8moIpXPbMMbYN60bWImDDU7L+/6zw=
+github.com/pierrec/lz4/v4 v4.1.25 h1:kocOqRffaIbU5djlIBr7Wh+cx82C0vtFb0fOurZHqD0=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9 h1:bsUq1dX0N8AOIL7EB/X911+m4EHsnWEHeJ0c+3TTBrg=
github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -52,16 +51,15 @@ github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
-go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8=
-go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA=
-go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E=
-go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM=
-go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE=
+go.opentelemetry.io/otel v1.41.0 h1:YlEwVsGAlCvczDILpUXpIpPSL/VPugt7zHThEMLce1c=
+go.opentelemetry.io/otel/metric v1.41.0 h1:rFnDcs4gRzBcsO9tS8LCpgR0dxg4aaxWlJxCno7JlTQ=
+go.opentelemetry.io/otel/sdk v1.39.0 h1:nMLYcjVsvdui1B/4FRkwjzoRVsMK8uL/cj0OyhKzt18=
+go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2WKg+sEJTtB8=
+go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa9TIN0=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
-golang.org/x/crypto v0.44.0 h1:A97SsFvM3AIwEEmTBiaxPPTYpDC47w720rdiiUvgoAU=
-golang.org/x/crypto v0.44.0/go.mod h1:013i+Nw79BMiQiMsOPcVCB5ZIJbYkerPrGnOa00tvmc=
+golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -69,19 +67,17 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
-golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY=
-golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU=
+golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I=
+golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
-golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
@@ -89,17 +85,14 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM=
-golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM=
+golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20251029180050-ab9386a59fda h1:i/Q+bfisr7gq6feoJnS/DlpdwEL4ihp41fvRiM3Ork0=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20251029180050-ab9386a59fda/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk=
-google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc=
-google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U=
+gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260427160629-7cedc36a6bc4 h1:tEkOQcXgF6dH1G+MVKZrfpYvozGrzb91k6ha7jireSM=
+google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM=
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
diff --git a/apps/matching-engine/wal/BTCUSD/0.log b/apps/matching-engine/wal/BTCUSD/0.log
index e69de29..c4295e8 100644
Binary files a/apps/matching-engine/wal/BTCUSD/0.log and b/apps/matching-engine/wal/BTCUSD/0.log differ
diff --git a/apps/matching-engine/wal/BTCUSD/checkpoint.meta b/apps/matching-engine/wal/BTCUSD/checkpoint.meta
index e69de29..56a6051 100644
--- a/apps/matching-engine/wal/BTCUSD/checkpoint.meta
+++ b/apps/matching-engine/wal/BTCUSD/checkpoint.meta
@@ -0,0 +1 @@
+1
\ No newline at end of file
diff --git a/apps/matching-engine/wal/SOLUSD/0.log b/apps/matching-engine/wal/SOLUSD/0.log
index 0e63dbb..fc25c6d 100644
Binary files a/apps/matching-engine/wal/SOLUSD/0.log and b/apps/matching-engine/wal/SOLUSD/0.log differ
diff --git a/apps/matching-engine/wal/SOLUSD/checkpoint.meta b/apps/matching-engine/wal/SOLUSD/checkpoint.meta
index ac4213d..d2e1cef 100644
--- a/apps/matching-engine/wal/SOLUSD/checkpoint.meta
+++ b/apps/matching-engine/wal/SOLUSD/checkpoint.meta
@@ -1 +1 @@
-43
\ No newline at end of file
+44
\ No newline at end of file
diff --git a/apps/order-service/src/config/dotenv.ts b/apps/order-service/src/config/dotenv.ts
index 514ef20..7b226fd 100755
--- a/apps/order-service/src/config/dotenv.ts
+++ b/apps/order-service/src/config/dotenv.ts
@@ -11,8 +11,6 @@ export const env = cleanEnv(process.env, {
MATCHING_SERVICE_GRPC_URL: str(),
NODE_ENV: str({ choices: ["development", "test", "production"] }),
-
- KAFKA_BROKERS: str(),
});
export default env;
diff --git a/apps/order-trade-worker-service/.env.example b/apps/order-trade-store-service/.env.example
similarity index 100%
rename from apps/order-trade-worker-service/.env.example
rename to apps/order-trade-store-service/.env.example
diff --git a/apps/order-trade-worker-service/README.md b/apps/order-trade-store-service/README.md
similarity index 100%
rename from apps/order-trade-worker-service/README.md
rename to apps/order-trade-store-service/README.md
diff --git a/apps/order-trade-worker-service/eslint.config.mjs b/apps/order-trade-store-service/eslint.config.mjs
similarity index 100%
rename from apps/order-trade-worker-service/eslint.config.mjs
rename to apps/order-trade-store-service/eslint.config.mjs
diff --git a/apps/order-trade-worker-service/nodemon.json b/apps/order-trade-store-service/nodemon.json
similarity index 100%
rename from apps/order-trade-worker-service/nodemon.json
rename to apps/order-trade-store-service/nodemon.json
diff --git a/apps/order-trade-worker-service/package.json b/apps/order-trade-store-service/package.json
similarity index 95%
rename from apps/order-trade-worker-service/package.json
rename to apps/order-trade-store-service/package.json
index 3a9d1bb..4ed7d7d 100644
--- a/apps/order-trade-worker-service/package.json
+++ b/apps/order-trade-store-service/package.json
@@ -1,5 +1,5 @@
{
- "name": "@repo/order-worker-service",
+ "name": "@repo/order-trade-store-service",
"version": "1.0.0",
"description": "",
"main": "src/index.ts",
diff --git a/apps/order-trade-worker-service/src/config/dotenv.ts b/apps/order-trade-store-service/src/config/dotenv.ts
similarity index 100%
rename from apps/order-trade-worker-service/src/config/dotenv.ts
rename to apps/order-trade-store-service/src/config/dotenv.ts
diff --git a/apps/order-trade-worker-service/src/constants.ts b/apps/order-trade-store-service/src/constants.ts
similarity index 100%
rename from apps/order-trade-worker-service/src/constants.ts
rename to apps/order-trade-store-service/src/constants.ts
diff --git a/apps/order-trade-worker-service/src/controllers/order.controller.ts b/apps/order-trade-store-service/src/controllers/order.controller.ts
similarity index 100%
rename from apps/order-trade-worker-service/src/controllers/order.controller.ts
rename to apps/order-trade-store-service/src/controllers/order.controller.ts
diff --git a/apps/order-trade-worker-service/src/controllers/trade.controller.ts b/apps/order-trade-store-service/src/controllers/trade.controller.ts
similarity index 100%
rename from apps/order-trade-worker-service/src/controllers/trade.controller.ts
rename to apps/order-trade-store-service/src/controllers/trade.controller.ts
diff --git a/apps/order-trade-worker-service/src/index.ts b/apps/order-trade-store-service/src/index.ts
similarity index 100%
rename from apps/order-trade-worker-service/src/index.ts
rename to apps/order-trade-store-service/src/index.ts
diff --git a/apps/order-trade-worker-service/src/kafka.consumer.ts b/apps/order-trade-store-service/src/kafka.consumer.ts
similarity index 100%
rename from apps/order-trade-worker-service/src/kafka.consumer.ts
rename to apps/order-trade-store-service/src/kafka.consumer.ts
diff --git a/apps/order-trade-worker-service/tsconfig.json b/apps/order-trade-store-service/tsconfig.json
similarity index 100%
rename from apps/order-trade-worker-service/tsconfig.json
rename to apps/order-trade-store-service/tsconfig.json
diff --git a/apps/trade-ingestor-service/.air.toml b/apps/trade-ingestor-service/.air.toml
new file mode 100644
index 0000000..f9a30d6
--- /dev/null
+++ b/apps/trade-ingestor-service/.air.toml
@@ -0,0 +1,27 @@
+root = "."
+tmp_dir = "tmp"
+
+[build]
+cmd = "go build -o ./tmp/trader-ingestor-service ./cmd/trader-ingestor-service"
+bin = "tmp/trader-ingestor-service"
+
+include_ext = ["go"]
+
+include_dir = [
+ ".",
+ "../../packages/proto-defs/go/generated"
+]
+
+exclude_dir = [
+ "tmp",
+ "vendor",
+ "node_modules"
+]
+
+delay = 200
+
+[run]
+cmd = "./tmp/trader-ingestor-service"
+
+[log]
+time = true
diff --git a/apps/trade-ingestor-service/.env.example b/apps/trade-ingestor-service/.env.example
new file mode 100644
index 0000000..6bef26f
--- /dev/null
+++ b/apps/trade-ingestor-service/.env.example
@@ -0,0 +1,3 @@
+CLICKHOUSE_ADDR=
+CLICKHOUSE_USER=
+CLICKHOUSE_PASSWORD=
\ No newline at end of file
diff --git a/apps/trade-ingestor-service/cmd/trade-ingestor-service/main.go b/apps/trade-ingestor-service/cmd/trade-ingestor-service/main.go
new file mode 100644
index 0000000..f275968
--- /dev/null
+++ b/apps/trade-ingestor-service/cmd/trade-ingestor-service/main.go
@@ -0,0 +1,48 @@
+package main
+
+import (
+ "context"
+ "log/slog"
+ "os"
+ "os/signal"
+ "syscall"
+
+ "github.com/sameerkrdev/nerve/apps/trade-ingestor-service/internal/clickhouse"
+ "github.com/sameerkrdev/nerve/apps/trade-ingestor-service/internal/kafka"
+)
+
+// Kafka and clickhouse connection is now completed
+// TODO: make a fucntion to insert the trade batch to clichouse --> InsertTrades([]*Trade)
+// TODO: make a trade batching system which flush the buffer data to clichouse after some interval 50ms and mark the kafka mark msg
+func main() {
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ _, err := clickhouse.NewClickhouseClient(ctx)
+ if err != nil {
+ slog.Error("Clickhouse init failed", "error", err)
+ os.Exit(1)
+ }
+
+ brokerAddresses := []string{"localhost:19092", "localhost:19093", "localhost:19094"}
+ _, err = kafka.InitKafkaConsumerClient(brokerAddresses)
+ if err != nil {
+ slog.Error("Kafka init failed", "error", err)
+ os.Exit(1)
+ }
+
+ consumerHandler := kafka.NewConsumerHandler()
+
+ topics := []string{
+ "trades",
+ }
+
+ go kafka.Consume(ctx, topics, consumerHandler)
+
+ quit := make(chan os.Signal, 1)
+ signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM)
+
+ <-quit
+
+ slog.Info("shutting down...")
+}
diff --git a/apps/trade-ingestor-service/go.mod b/apps/trade-ingestor-service/go.mod
new file mode 100644
index 0000000..ec9b79e
--- /dev/null
+++ b/apps/trade-ingestor-service/go.mod
@@ -0,0 +1,3 @@
+module github.com/sameerkrdev/nerve/apps/trade-ingestor-service
+
+go 1.25.4
diff --git a/apps/trade-ingestor-service/go.sum b/apps/trade-ingestor-service/go.sum
new file mode 100644
index 0000000..e69de29
diff --git a/apps/trade-ingestor-service/internal/clickhouse/clickhouse.go b/apps/trade-ingestor-service/internal/clickhouse/clickhouse.go
new file mode 100644
index 0000000..ecda6da
--- /dev/null
+++ b/apps/trade-ingestor-service/internal/clickhouse/clickhouse.go
@@ -0,0 +1,53 @@
+package clickhouse
+
+import (
+ "context"
+ "crypto/tls"
+ "fmt"
+ "os"
+ "sync"
+
+ "github.com/ClickHouse/clickhouse-go/v2"
+ "github.com/ClickHouse/clickhouse-go/v2/lib/driver"
+)
+
+var (
+ ClickhouseClient driver.Conn
+ once sync.Once
+ initErr error
+)
+
+func NewClickhouseClient(ctx context.Context) (driver.Conn, error) {
+ once.Do(func() {
+ var conn driver.Conn
+
+ conn, initErr = clickhouse.Open(&clickhouse.Options{
+ Addr: []string{os.Getenv("CLICKHOUSE_ADDR")},
+ Protocol: clickhouse.Native,
+ TLS: &tls.Config{},
+ Auth: clickhouse.Auth{
+ Username: os.Getenv("CLICKHOUSE_USER"),
+ Password: os.Getenv("CLICKHOUSE_PASSWORD"),
+ },
+ })
+
+ if initErr != nil {
+ return
+ }
+
+ if err := conn.Ping(ctx); err != nil {
+ if exception, ok := err.(*clickhouse.Exception); ok {
+ fmt.Printf("Exception [%d] %s\n%s\n", exception.Code, exception.Message, exception.StackTrace)
+ }
+ initErr = err
+ return
+ }
+
+ ClickhouseClient = conn
+ })
+ return ClickhouseClient, initErr
+}
+
+func InsertRawTrade() {
+
+}
diff --git a/apps/trade-ingestor-service/internal/clickhouse/tradeRepository.go b/apps/trade-ingestor-service/internal/clickhouse/tradeRepository.go
new file mode 100644
index 0000000..c337539
--- /dev/null
+++ b/apps/trade-ingestor-service/internal/clickhouse/tradeRepository.go
@@ -0,0 +1,29 @@
+package clickhouse
+
+import (
+ "time"
+
+ "github.com/ClickHouse/clickhouse-go/v2/lib/driver"
+ "github.com/IBM/sarama"
+ pbEngine "github.com/sameerkrdev/nerve/packages/proto-defs/go/generated/engine"
+)
+
+type BatchItem struct {
+ msg *sarama.ConsumerMessage
+ trade *pbEngine.TradeEvent
+}
+
+type TradeBatcher struct {
+ ch chan BatchItem
+ buffer []BatchItem
+
+ maxSize int
+ flushTime time.Duration
+
+ kafkaClient *sarama.ConsumerGroupSession
+ clickhouseClient *driver.Conn
+}
+
+func Insert() {
+
+}
diff --git a/apps/trade-ingestor-service/internal/kafka/consumerHandler.go b/apps/trade-ingestor-service/internal/kafka/consumerHandler.go
new file mode 100644
index 0000000..8c798d5
--- /dev/null
+++ b/apps/trade-ingestor-service/internal/kafka/consumerHandler.go
@@ -0,0 +1,33 @@
+package kafka
+
+import (
+ "log/slog"
+
+ "github.com/IBM/sarama"
+)
+
+type ConsumerHandler struct{}
+
+func NewConsumerHandler() *ConsumerHandler {
+ return &ConsumerHandler{}
+}
+
+func (h *ConsumerHandler) Setup(session sarama.ConsumerGroupSession) error {
+ slog.Info("Consumer group session started")
+ return nil
+}
+
+func (h *ConsumerHandler) Cleanup(session sarama.ConsumerGroupSession) error {
+ slog.Info("Consumer group session ended")
+ return nil
+}
+
+func (h *ConsumerHandler) ConsumeClaim(session sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error {
+ for msg := range claim.Messages() {
+ slog.Info("message consumed", "topic", msg.Topic, "partition", msg.Partition, "offset", msg.Offset, "value", string(msg.Value))
+
+ // mark message as processed
+ session.MarkMessage(msg, "")
+ }
+ return nil
+}
diff --git a/apps/trade-ingestor-service/internal/kafka/kafka.go b/apps/trade-ingestor-service/internal/kafka/kafka.go
new file mode 100644
index 0000000..cd15254
--- /dev/null
+++ b/apps/trade-ingestor-service/internal/kafka/kafka.go
@@ -0,0 +1,65 @@
+package kafka
+
+import (
+ "context"
+ "log/slog"
+ "sync"
+ "time"
+
+ "github.com/IBM/sarama"
+)
+
+var (
+ KafkaConsumerClient sarama.ConsumerGroup
+ initErr error
+ once sync.Once
+)
+
+func InitKafkaConsumerClient(brokers []string) (*sarama.ConsumerGroup, error) {
+ once.Do(func() {
+ config := sarama.NewConfig()
+
+ config.Consumer.Fetch.Default = 5 * 1024 * 1024
+ config.Consumer.MaxProcessingTime = 3 * time.Second
+ config.Consumer.Group.Rebalance.Strategy = sarama.NewBalanceStrategySticky()
+ config.Consumer.Offsets.AutoCommit.Enable = true
+ config.Consumer.Offsets.AutoCommit.Interval = 2 * time.Second
+ config.Consumer.MaxWaitTime = 500 * time.Millisecond
+
+ conn, err := sarama.NewConsumerGroup(brokers, "trade-ingestor-service", config)
+
+ if err != nil {
+ initErr = err
+ return
+ }
+
+ KafkaConsumerClient = conn
+ })
+ return &KafkaConsumerClient, initErr
+}
+
+func Close() error {
+ return KafkaConsumerClient.Close()
+}
+
+func Consume(ctx context.Context, topics []string, handler sarama.ConsumerGroupHandler) {
+ defer Close()
+
+ go func() {
+ for err := range KafkaConsumerClient.Errors() {
+ slog.Error("kafka consume error", "error", err)
+ }
+ }()
+
+ for {
+ if err := KafkaConsumerClient.Consume(ctx, topics, handler); err != nil {
+ slog.Error("consumer error:", "error", err)
+ }
+
+ // exit cleanly when context is cancelled
+ if ctx.Err() != nil {
+ slog.Info("kafka consumer stopped")
+ return
+ }
+ }
+}
diff --git a/apps/trade-ingestor-service/makefile b/apps/trade-ingestor-service/makefile
new file mode 100644
index 0000000..b4e6f16
--- /dev/null
+++ b/apps/trade-ingestor-service/makefile
@@ -0,0 +1,21 @@
+APP_NAME := trade-ingestor-service
+CMD_PATH := ./cmd/trade-ingestor-service
+BUILD_DIR := bin
+ROOT_DIR := ../..
+
+post-install:
+ @echo "➡ Generating go.mod for proto-generated code..."
+ @mkdir -p $(ROOT_DIR)/packages/proto-defs/go/generated
+ @printf "module github.com/sameerkrdev/nerve/packages/proto-defs/go/generated\n\ngo 1.21\n" > \
+ $(ROOT_DIR)/packages/proto-defs/go/generated/go.mod
+
+dev:
+ air
+
+build:
+ @echo "➡ Building $(APP_NAME)..."
+ go build -o $(BUILD_DIR)/$(APP_NAME) $(CMD_PATH)
+
+tidy:
+ @echo "➡ Tidying modules..."
+ go mod tidy
diff --git a/apps/trade-ingestor-service/package.json b/apps/trade-ingestor-service/package.json
new file mode 100644
index 0000000..71ec768
--- /dev/null
+++ b/apps/trade-ingestor-service/package.json
@@ -0,0 +1,14 @@
+{
+ "name": "@repo/trade-ingestor-service",
+ "version": "1.0.0",
+ "description": "",
+ "main": "",
+ "scripts": {
+ "dev": "make post-install && make dev",
+ "build": "make post-install && make build",
+ "clean": "rm -rf dist"
+ },
+ "keywords": [],
+ "author": "",
+ "license": "ISC"
+}
diff --git a/apps/websocket-server/go.mod b/apps/websocket-server/go.mod
index 369caaa..f7c7a4d 100644
--- a/apps/websocket-server/go.mod
+++ b/apps/websocket-server/go.mod
@@ -4,13 +4,14 @@ go 1.25.4
require (
github.com/gorilla/websocket v1.5.3
- google.golang.org/grpc v1.78.0
+ google.golang.org/grpc v1.80.0
google.golang.org/protobuf v1.36.11
)
require (
- golang.org/x/net v0.47.0 // indirect
- golang.org/x/sys v0.38.0 // indirect
- golang.org/x/text v0.31.0 // indirect
- google.golang.org/genproto/googleapis/rpc v0.0.0-20251029180050-ab9386a59fda // indirect
+ go.opentelemetry.io/otel v1.41.0 // indirect
+ golang.org/x/net v0.53.0 // indirect
+ golang.org/x/sys v0.43.0 // indirect
+ golang.org/x/text v0.36.0 // indirect
+ google.golang.org/genproto/googleapis/rpc v0.0.0-20260427160629-7cedc36a6bc4 // indirect
)
diff --git a/apps/websocket-server/go.sum b/apps/websocket-server/go.sum
index 1f4d300..a85bda4 100644
--- a/apps/websocket-server/go.sum
+++ b/apps/websocket-server/go.sum
@@ -1,3 +1,4 @@
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
@@ -6,15 +7,15 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
-go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8=
-go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA=
-go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E=
-go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM=
-go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE=
-golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY=
-golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
-golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM=
-gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20251029180050-ab9386a59fda h1:i/Q+bfisr7gq6feoJnS/DlpdwEL4ihp41fvRiM3Ork0=
-google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc=
+go.opentelemetry.io/otel v1.41.0 h1:YlEwVsGAlCvczDILpUXpIpPSL/VPugt7zHThEMLce1c=
+go.opentelemetry.io/otel/metric v1.41.0 h1:rFnDcs4gRzBcsO9tS8LCpgR0dxg4aaxWlJxCno7JlTQ=
+go.opentelemetry.io/otel/sdk v1.39.0 h1:nMLYcjVsvdui1B/4FRkwjzoRVsMK8uL/cj0OyhKzt18=
+go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2WKg+sEJTtB8=
+go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa9TIN0=
+golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA=
+golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI=
+golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg=
+gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20260427160629-7cedc36a6bc4 h1:tEkOQcXgF6dH1G+MVKZrfpYvozGrzb91k6ha7jireSM=
+google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM=
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
diff --git a/docs/data-service.html b/docs/data-service.html
new file mode 100644
index 0000000..a48b67f
--- /dev/null
+++ b/docs/data-service.html
@@ -0,0 +1,894 @@
+
+
+
+data-service.md
+
+
+
+
+
+
+
+
+
+
+
+
+
+- Complete System Overview
+
+graph TB
+subgraph "External"
+EX[Exchange APIs
Binance, Coinbase]
+end
+
+ subgraph "Data Ingestion Layer"
+ KT[Kafka Topics
trades, candles-1m, candles-5m]
+ EX -->|Trade Events| KT
+ end
+
+ subgraph "Stream Processing Layer"
+ CB[Candle Builder
Stateful Stream]
+ PIC[Popular Indicator Computer
EMA, RSI, MACD]
+ DIC[Dynamic Indicator Computer
User Custom]
+
+ KT -->|trades| CB
+ CB -->|candles-1m| KT
+ KT -->|candles-1m| PIC
+ KT -->|candles-1m| DIC
+ end
+
+ subgraph "Storage Layer"
+ CH[(ClickHouse
OLAP Database)]
+ RD[(Redis
Cache + Pub/Sub)]
+
+ CB -->|Persist Candles| CH
+ PIC -->|Cache Popular| RD
+ DIC -->|Cache Custom| RD
+ end
+
+ subgraph "Data Service"
+ DS[Data Service
gRPC Server]
+
+ DS <-->|Query/Write| CH
+ DS <-->|Cache Get/Set| RD
+ end
+
+ subgraph "WebSocket Gateway Layer"
+ WS1[WebSocket Server 1]
+ WS2[WebSocket Server 2]
+ WS3[WebSocket Server N]
+ LB[Load Balancer]
+
+ LB --> WS1
+ LB --> WS2
+ LB --> WS3
+ end
+
+ subgraph "Clients"
+ C1[Web Browser]
+ C2[Mobile App]
+ C3[Desktop App]
+
+ C1 --> LB
+ C2 --> LB
+ C3 --> LB
+ end
+
+ RD -->|Pub/Sub| WS1
+ RD -->|Pub/Sub| WS2
+ RD -->|Pub/Sub| WS3
+
+ WS1 <-->|gRPC| DS
+ WS2 <-->|gRPC| DS
+ WS3 <-->|gRPC| DS
+
+ PIC -->|Publish Updates| RD
+ DIC -->|Publish Updates| RD
+
+ style KT fill:#ff9999
+ style CH fill:#99ccff
+ style RD fill:#ffcc99
+ style DS fill:#99ff99
+ style LB fill:#cc99ff
+
+
+
+- Real-Time Data Flow (Trade → Client)
+
+sequenceDiagram
+ participant Exchange
+ participant Kafka
+ participant CandleBuilder
+ participant Redis
+ participant WebSocket
+ participant Client
+
+ Exchange->>Kafka: Trade Event
{BTCUSDT, price:50000, vol:0.5}
+
+ Kafka->>CandleBuilder: Consume Trade
+
+ Note over CandleBuilder: Update In-Memory
OHLCV Candle
+
+ alt Candle Still Open
+ CandleBuilder->>CandleBuilder: Update High/Low/Close/Volume
+ end
+
+ alt Candle Closed (1 min elapsed)
+ CandleBuilder->>Kafka: Publish Candle
Topic: candles-1m
+ CandleBuilder->>ClickHouse: Persist Candle
+ CandleBuilder->>Redis: Publish Update
Channel: BTCUSDT:1m
+
+ Redis->>WebSocket: Fan Out Update
+
+ WebSocket->>Client: Binary Protobuf Message
MarketDataUpdate
+ end
+
+ Note over Client: Update Chart
in Real-Time
+
+
+
+- Indicator Subscription Flow
+
+sequenceDiagram
+ participant Client
+ participant WebSocket
+ participant SubscriptionMgr
+ participant DataService
+ participant Redis
+ participant ClickHouse
+
+ Client->>WebSocket: Subscribe Request
BTCUSDT:1m + RSI(14)
+
+ WebSocket->>SubscriptionMgr: Register Subscription
+
+ SubscriptionMgr->>Redis: Subscribe to Channel
"BTCUSDT:1m:rsi-14"
+
+ WebSocket->>DataService: GetMarketData
(Historical Data)
+
+ DataService->>Redis: Check Cache
+
+ alt Cache Hit
+ Redis-->>DataService: Cached Data
+ else Cache Miss
+ DataService->>ClickHouse: Query Candles
+ ClickHouse-->>DataService: OHLCV Data
+ DataService->>DataService: Compute RSI(14)
+ DataService->>Redis: Cache Result
+ end
+
+ DataService-->>WebSocket: Historical Candles + RSI
+
+ WebSocket->>Client: Send Historical Data
(Replay 100 bars)
+
+ WebSocket->>Client: Subscription Confirmed
+
+ Note over Client,Redis: Real-time updates begin
+
+ loop Every New Candle
+ Redis->>WebSocket: Update via Pub/Sub
+ WebSocket->>Client: MarketDataUpdate
+ end
+
+
+
+
+- WebSocket Server Internal Architecture
+
+graph TB
+ subgraph "WebSocket Server Process"
+ HTTP[HTTP Handler
:8080/v1/stream]
+
+ subgraph "Connection Management"
+ UP[Upgrader
HTTP → WebSocket]
+ CP[Client Pool
sync.Map]
+ end
+
+ subgraph "Client Goroutines"
+ direction LR
+ C1[Client 1]
+ C2[Client 2]
+ CN[Client N]
+
+ subgraph "Per Client"
+ RP[Read Pump
Goroutine]
+ WP[Write Pump
Goroutine]
+ SC[Send Channel
buffered 256]
+ end
+ end
+
+ subgraph "Subscription Management"
+ SM[Subscription Manager]
+ SB[Subscriptions Map
symbol:tf → []clients]
+ end
+
+ subgraph "External Communication"
+ GRPC[gRPC Client
→ Data Service]
+ RPUB[Redis Pub/Sub
Receiver]
+ end
+
+ HTTP --> UP
+ UP --> CP
+ CP --> C1
+ CP --> C2
+ CP --> CN
+
+ C1 --> RP
+ C1 --> WP
+ RP --> SC
+ SC --> WP
+
+ RP -->|Subscribe Request| SM
+ SM --> SB
+ SM -->|Subscribe Channel| RPUB
+
+ RP -->|Get Data Request| GRPC
+ GRPC -->|Response| SC
+
+ RPUB -->|Update| SM
+ SM -->|Fan Out| SC
+ end
+
+ EXT_DS[Data Service
gRPC Server]
+ EXT_RD[Redis
Pub/Sub]
+
+ GRPC <-->|gRPC| EXT_DS
+ RPUB <-->|Pub/Sub| EXT_RD
+
+ style HTTP fill:#ffcccc
+ style CP fill:#ccffcc
+ style SM fill:#ccccff
+ style GRPC fill:#ffffcc
+ style RPUB fill:#ffccff
+
+
+
+- Data Service Request Flow
+
+graph LR
+ subgraph "WebSocket Server"
+ WS[Client Connection]
+ end
+
+ subgraph "Data Service"
+ GRPC[gRPC Handler]
+
+ subgraph "Cache Layer"
+ RC[Redis Cache]
+ CHK{Cache Hit?}
+ end
+
+ subgraph "Computation Layer"
+ IC[Indicator Computer]
+
+ subgraph "Indicator Types"
+ PRSI[RSI Calculator]
+ PMACD[MACD Calculator]
+ PBB[Bollinger Calculator]
+ PCUST[Pine Script Engine]
+ end
+ end
+
+ subgraph "Storage Layer"
+ CH[(ClickHouse)]
+ end
+ end
+
+ WS -->|GetMarketData
BTCUSDT:1m
RSI(14)| GRPC
+
+ GRPC --> CHK
+
+ CHK -->|Hit| RC
+ RC -->|Cached Data| GRPC
+
+ CHK -->|Miss| CH
+ CH -->|OHLCV Candles| IC
+
+ IC --> PRSI
+ PRSI -->|RSI Values| IC
+
+ IC -->|Cache Write| RC
+ IC -->|Result| GRPC
+
+ GRPC -->|Response| WS
+
+ style CHK fill:#ffff99
+ style RC fill:#99ff99
+ style IC fill:#ff99ff
+ style CH fill:#9999ff
+
+
+
+- Subscription Manager Architecture
+
+graph TB
+ subgraph "Subscription Manager"
+ SM[Manager Instance]
+
+ subgraph "Subscription Storage"
+ SMAP["subscriptions map
key: 'BTCUSDT:1m'
value: []*Subscription"]
+ end
+
+ subgraph "Redis Integration"
+ direction LR
+ RSB[Redis Subscribe]
+ RUS[Redis Unsubscribe]
+ RHD[Redis Handler]
+ end
+
+ subgraph "Fan-Out Engine"
+ FO[Fan Out to Clients]
+
+ subgraph "Client Channels"
+ CH1[Client 1 Send Chan]
+ CH2[Client 2 Send Chan]
+ CHN[Client N Send Chan]
+ end
+ end
+
+ subgraph "Lifecycle Management"
+ GC[Garbage Collector
Every 1 min]
+ INACT[Remove Inactive
>5 min idle]
+ end
+ end
+
+ CLI1[Client 1
Subscribe]
+ CLI2[Client 2
Subscribe]
+ CLI3[Client 3
Unsubscribe]
+
+ RD[(Redis Pub/Sub)]
+
+ CLI1 -->|Subscribe| SM
+ CLI2 -->|Subscribe| SM
+ CLI3 -->|Unsubscribe| SM
+
+ SM --> SMAP
+
+ SM -->|First Subscriber| RSB
+ RSB -->|Subscribe| RD
+
+ SM -->|Last Subscriber| RUS
+ RUS -->|Unsubscribe| RD
+
+ RD -->|Update Message| RHD
+ RHD --> FO
+
+ FO --> CH1
+ FO --> CH2
+ FO --> CHN
+
+ GC -->|Check Activity| SMAP
+ INACT -->|Cleanup| SMAP
+
+ style SMAP fill:#ffcccc
+ style RHD fill:#ccffcc
+ style FO fill:#ccccff
+ style GC fill:#ffffcc
+
+
+
+- Stream Processor State Management
+
+graph TB
+ subgraph "Stream Processor"
+ KCS[Kafka Consumer]
+
+ subgraph "Per-Symbol State"
+ direction TB
+
+ STATE["State Store
(RocksDB/Flink)"]
+
+ subgraph "Symbol: BTCUSDT"
+ S1["Current Candle
{open, high, low, close, vol}"]
+
+ subgraph "Indicator States"
+ I1["RSI(14)
{avg_gain, avg_loss, prices[14]}"]
+ I2["MACD(12,26,9)
{ema12, ema26, signal}"]
+ I3["BB(20,2)
{sma, std_dev}"]
+ end
+ end
+
+ subgraph "Symbol: ETHUSDT"
+ S2["Current Candle"]
+ I4["RSI(14)"]
+ end
+ end
+
+ subgraph "Output"
+ KP[Kafka Producer]
+ RDP[Redis Publisher]
+ CHW[ClickHouse Writer]
+ end
+
+ subgraph "Checkpointing"
+ CP[Checkpoint
Every 60s]
+ SNAP[State Snapshot]
+ end
+ end
+
+ KAFKA[(Kafka
trades topic)]
+ REDIS[(Redis)]
+ CH[(ClickHouse)]
+
+ KAFKA -->|Trade Events| KCS
+
+ KCS -->|Update State| STATE
+
+ STATE --> S1
+ S1 --> I1
+ S1 --> I2
+ S1 --> I3
+
+ STATE --> S2
+ S2 --> I4
+
+ S1 -->|Candle Closed| KP
+ I1 -->|New Value| RDP
+ I2 -->|New Value| RDP
+
+ KP -->|candles-1m| KAFKA
+ RDP -->|Pub/Sub| REDIS
+ S1 -->|Persist| CHW
+ CHW --> CH
+
+ STATE -->|Periodic| CP
+ CP --> SNAP
+ SNAP -.->|Recovery| STATE
+
+ style STATE fill:#ffcccc
+ style S1 fill:#ccffcc
+ style I1 fill:#ccccff
+ style CP fill:#ffffcc
+
+
+- Horizontal Scaling Architecture
+
+graph TB
+ subgraph "Load Balancers"
+ LB1[Load Balancer
WebSocket]
+ LB2[Load Balancer
gRPC]
+ end
+
+ subgraph "WebSocket Tier - Stateless"
+ WS1[WS Server 1
40K connections]
+ WS2[WS Server 2
40K connections]
+ WS3[WS Server 3
20K connections]
+ end
+
+ subgraph "Data Service Tier - Stateless"
+ DS1[Data Service 1]
+ DS2[Data Service 2]
+ DS3[Data Service 3]
+ end
+
+ subgraph "Stream Processing Tier - Stateful"
+ direction LR
+
+ subgraph "Candle Builders"
+ CB1[Builder 1
Partition 0-5]
+ CB2[Builder 2
Partition 6-11]
+ CB3[Builder 3
Partition 12-15]
+ end
+
+ subgraph "Indicator Computers"
+ IC1[Computer 1
Partition 0-5]
+ IC2[Computer 2
Partition 6-11]
+ IC3[Computer 3
Partition 12-15]
+ end
+ end
+
+ subgraph "Shared State"
+ KAFKA[(Kafka
16 Partitions)]
+ REDIS[(Redis Cluster
3 Master + 3 Replica)]
+ CH[(ClickHouse
Distributed)]
+ end
+
+ CLIENTS[100K Clients]
+
+ CLIENTS --> LB1
+
+ LB1 --> WS1
+ LB1 --> WS2
+ LB1 --> WS3
+
+ WS1 --> LB2
+ WS2 --> LB2
+ WS3 --> LB2
+
+ LB2 --> DS1
+ LB2 --> DS2
+ LB2 --> DS3
+
+ DS1 --> REDIS
+ DS1 --> CH
+ DS2 --> REDIS
+ DS2 --> CH
+ DS3 --> REDIS
+ DS3 --> CH
+
+ KAFKA --> CB1
+ KAFKA --> CB2
+ KAFKA --> CB3
+
+ KAFKA --> IC1
+ KAFKA --> IC2
+ KAFKA --> IC3
+
+ CB1 --> KAFKA
+ CB2 --> KAFKA
+ CB3 --> KAFKA
+
+ CB1 --> REDIS
+ CB2 --> REDIS
+ CB3 --> REDIS
+
+ IC1 --> REDIS
+ IC2 --> REDIS
+ IC3 --> REDIS
+
+ WS1 -.->|Pub/Sub| REDIS
+ WS2 -.->|Pub/Sub| REDIS
+ WS3 -.->|Pub/Sub| REDIS
+
+ style LB1 fill:#ff9999
+ style WS1 fill:#99ccff
+ style DS1 fill:#99ff99
+ style CB1 fill:#ffcc99
+ style IC1 fill:#cc99ff
+ style KAFKA fill:#ffcccc
+ style REDIS fill:#ccffff
+
+
+
+
diff --git a/docs/data-service.md b/docs/data-service.md
new file mode 100644
index 0000000..6c58752
--- /dev/null
+++ b/docs/data-service.md
@@ -0,0 +1,537 @@
+1. Complete System Overview
+
+```mermaid
+graph TB
+subgraph "External"
+EX[Exchange APIs
Binance, Coinbase]
+end
+
+ subgraph "Data Ingestion Layer"
+ KT[Kafka Topics
trades, candles-1m, candles-5m]
+ EX -->|Trade Events| KT
+ end
+
+ subgraph "Stream Processing Layer"
+ CB[Candle Builder
Stateful Stream]
+ PIC[Popular Indicator Computer
EMA, RSI, MACD]
+ DIC[Dynamic Indicator Computer
User Custom]
+
+ KT -->|trades| CB
+ CB -->|candles-1m| KT
+ KT -->|candles-1m| PIC
+ KT -->|candles-1m| DIC
+ end
+
+ subgraph "Storage Layer"
+ CH[(ClickHouse
OLAP Database)]
+ RD[(Redis
Cache + Pub/Sub)]
+
+ CB -->|Persist Candles| CH
+ PIC -->|Cache Popular| RD
+ DIC -->|Cache Custom| RD
+ end
+
+ subgraph "Data Service"
+ DS[Data Service
gRPC Server]
+
+ DS <-->|Query/Write| CH
+ DS <-->|Cache Get/Set| RD
+ end
+
+ subgraph "WebSocket Gateway Layer"
+ WS1[WebSocket Server 1]
+ WS2[WebSocket Server 2]
+ WS3[WebSocket Server N]
+ LB[Load Balancer]
+
+ LB --> WS1
+ LB --> WS2
+ LB --> WS3
+ end
+
+ subgraph "Clients"
+ C1[Web Browser]
+ C2[Mobile App]
+ C3[Desktop App]
+
+ C1 --> LB
+ C2 --> LB
+ C3 --> LB
+ end
+
+ RD -->|Pub/Sub| WS1
+ RD -->|Pub/Sub| WS2
+ RD -->|Pub/Sub| WS3
+
+ WS1 <-->|gRPC| DS
+ WS2 <-->|gRPC| DS
+ WS3 <-->|gRPC| DS
+
+ PIC -->|Publish Updates| RD
+ DIC -->|Publish Updates| RD
+
+ style KT fill:#ff9999
+ style CH fill:#99ccff
+ style RD fill:#ffcc99
+ style DS fill:#99ff99
+ style LB fill:#cc99ff
+```
+
+---
+
+2. Real-Time Data Flow (Trade → Client)
+
+```mermaid
+sequenceDiagram
+ participant Exchange
+ participant Kafka
+ participant CandleBuilder
+ participant Redis
+ participant WebSocket
+ participant Client
+
+ Exchange->>Kafka: Trade Event
{BTCUSDT, price:50000, vol:0.5}
+
+ Kafka->>CandleBuilder: Consume Trade
+
+ Note over CandleBuilder: Update In-Memory
OHLCV Candle
+
+ alt Candle Still Open
+ CandleBuilder->>CandleBuilder: Update High/Low/Close/Volume
+ end
+
+ alt Candle Closed (1 min elapsed)
+ CandleBuilder->>Kafka: Publish Candle
Topic: candles-1m
+ CandleBuilder->>ClickHouse: Persist Candle
+ CandleBuilder->>Redis: Publish Update
Channel: BTCUSDT:1m
+
+ Redis->>WebSocket: Fan Out Update
+
+ WebSocket->>Client: Binary Protobuf Message
MarketDataUpdate
+ end
+
+ Note over Client: Update Chart
in Real-Time
+```
+
+---
+
+3. Indicator Subscription Flow
+
+```mermaid
+sequenceDiagram
+ participant Client
+ participant WebSocket
+ participant SubscriptionMgr
+ participant DataService
+ participant Redis
+ participant ClickHouse
+
+ Client->>WebSocket: Subscribe Request
BTCUSDT:1m + RSI(14)
+
+ WebSocket->>SubscriptionMgr: Register Subscription
+
+ SubscriptionMgr->>Redis: Subscribe to Channel
"BTCUSDT:1m:rsi-14"
+
+ WebSocket->>DataService: GetMarketData
(Historical Data)
+
+ DataService->>Redis: Check Cache
+
+ alt Cache Hit
+ Redis-->>DataService: Cached Data
+ else Cache Miss
+ DataService->>ClickHouse: Query Candles
+ ClickHouse-->>DataService: OHLCV Data
+ DataService->>DataService: Compute RSI(14)
+ DataService->>Redis: Cache Result
+ end
+
+ DataService-->>WebSocket: Historical Candles + RSI
+
+ WebSocket->>Client: Send Historical Data
(Replay 100 bars)
+
+ WebSocket->>Client: Subscription Confirmed
+
+ Note over Client,Redis: Real-time updates begin
+
+ loop Every New Candle
+ Redis->>WebSocket: Update via Pub/Sub
+ WebSocket->>Client: MarketDataUpdate
+ end
+
+```
+
+---
+
+4. WebSocket Server Internal Architecture
+
+```mermaid
+graph TB
+ subgraph "WebSocket Server Process"
+ HTTP[HTTP Handler
:8080/v1/stream]
+
+ subgraph "Connection Management"
+ UP[Upgrader
HTTP → WebSocket]
+ CP[Client Pool
sync.Map]
+ end
+
+ subgraph "Client Goroutines"
+ direction LR
+ C1[Client 1]
+ C2[Client 2]
+ CN[Client N]
+
+ subgraph "Per Client"
+ RP[Read Pump
Goroutine]
+ WP[Write Pump
Goroutine]
+ SC[Send Channel
buffered 256]
+ end
+ end
+
+ subgraph "Subscription Management"
+ SM[Subscription Manager]
+ SB[Subscriptions Map
symbol:tf → []clients]
+ end
+
+ subgraph "External Communication"
+ GRPC[gRPC Client
→ Data Service]
+ RPUB[Redis Pub/Sub
Receiver]
+ end
+
+ HTTP --> UP
+ UP --> CP
+ CP --> C1
+ CP --> C2
+ CP --> CN
+
+ C1 --> RP
+ C1 --> WP
+ RP --> SC
+ SC --> WP
+
+ RP -->|Subscribe Request| SM
+ SM --> SB
+ SM -->|Subscribe Channel| RPUB
+
+ RP -->|Get Data Request| GRPC
+ GRPC -->|Response| SC
+
+ RPUB -->|Update| SM
+ SM -->|Fan Out| SC
+ end
+
+ EXT_DS[Data Service
gRPC Server]
+ EXT_RD[Redis
Pub/Sub]
+
+ GRPC <-->|gRPC| EXT_DS
+ RPUB <-->|Pub/Sub| EXT_RD
+
+ style HTTP fill:#ffcccc
+ style CP fill:#ccffcc
+ style SM fill:#ccccff
+ style GRPC fill:#ffffcc
+ style RPUB fill:#ffccff
+```
+
+---
+
+6. Data Service Request Flow
+
+```mermaid
+graph LR
+ subgraph "WebSocket Server"
+ WS[Client Connection]
+ end
+
+ subgraph "Data Service"
+ GRPC[gRPC Handler]
+
+ subgraph "Cache Layer"
+ RC[Redis Cache]
+ CHK{Cache Hit?}
+ end
+
+ subgraph "Computation Layer"
+ IC[Indicator Computer]
+
+ subgraph "Indicator Types"
+ PRSI[RSI Calculator]
+ PMACD[MACD Calculator]
+ PBB[Bollinger Calculator]
+ PCUST[Pine Script Engine]
+ end
+ end
+
+ subgraph "Storage Layer"
+ CH[(ClickHouse)]
+ end
+ end
+
+ WS -->|GetMarketData
BTCUSDT:1m
RSI(14)| GRPC
+
+ GRPC --> CHK
+
+ CHK -->|Hit| RC
+ RC -->|Cached Data| GRPC
+
+ CHK -->|Miss| CH
+ CH -->|OHLCV Candles| IC
+
+ IC --> PRSI
+ PRSI -->|RSI Values| IC
+
+ IC -->|Cache Write| RC
+ IC -->|Result| GRPC
+
+ GRPC -->|Response| WS
+
+ style CHK fill:#ffff99
+ style RC fill:#99ff99
+ style IC fill:#ff99ff
+ style CH fill:#9999ff
+```
+
+---
+
+7. Subscription Manager Architecture
+
+```mermaid
+graph TB
+ subgraph "Subscription Manager"
+ SM[Manager Instance]
+
+ subgraph "Subscription Storage"
+ SMAP["subscriptions map
key: 'BTCUSDT:1m'
value: []*Subscription"]
+ end
+
+ subgraph "Redis Integration"
+ direction LR
+ RSB[Redis Subscribe]
+ RUS[Redis Unsubscribe]
+ RHD[Redis Handler]
+ end
+
+ subgraph "Fan-Out Engine"
+ FO[Fan Out to Clients]
+
+ subgraph "Client Channels"
+ CH1[Client 1 Send Chan]
+ CH2[Client 2 Send Chan]
+ CHN[Client N Send Chan]
+ end
+ end
+
+ subgraph "Lifecycle Management"
+ GC[Garbage Collector
Every 1 min]
+ INACT[Remove Inactive
>5 min idle]
+ end
+ end
+
+ CLI1[Client 1
Subscribe]
+ CLI2[Client 2
Subscribe]
+ CLI3[Client 3
Unsubscribe]
+
+ RD[(Redis Pub/Sub)]
+
+ CLI1 -->|Subscribe| SM
+ CLI2 -->|Subscribe| SM
+ CLI3 -->|Unsubscribe| SM
+
+ SM --> SMAP
+
+ SM -->|First Subscriber| RSB
+ RSB -->|Subscribe| RD
+
+ SM -->|Last Subscriber| RUS
+ RUS -->|Unsubscribe| RD
+
+ RD -->|Update Message| RHD
+ RHD --> FO
+
+ FO --> CH1
+ FO --> CH2
+ FO --> CHN
+
+ GC -->|Check Activity| SMAP
+ INACT -->|Cleanup| SMAP
+
+ style SMAP fill:#ffcccc
+ style RHD fill:#ccffcc
+ style FO fill:#ccccff
+ style GC fill:#ffffcc
+```
+
+---
+
+8. Stream Processor State Management
+
+```mermaid
+graph TB
+ subgraph "Stream Processor"
+ KCS[Kafka Consumer]
+
+ subgraph "Per-Symbol State"
+ direction TB
+
+ STATE["State Store
(RocksDB/Flink)"]
+
+ subgraph "Symbol: BTCUSDT"
+ S1["Current Candle
{open, high, low, close, vol}"]
+
+ subgraph "Indicator States"
+ I1["RSI(14)
{avg_gain, avg_loss, prices[14]}"]
+ I2["MACD(12,26,9)
{ema12, ema26, signal}"]
+ I3["BB(20,2)
{sma, std_dev}"]
+ end
+ end
+
+ subgraph "Symbol: ETHUSDT"
+ S2["Current Candle"]
+ I4["RSI(14)"]
+ end
+ end
+
+ subgraph "Output"
+ KP[Kafka Producer]
+ RDP[Redis Publisher]
+ CHW[ClickHouse Writer]
+ end
+
+ subgraph "Checkpointing"
+ CP[Checkpoint
Every 60s]
+ SNAP[State Snapshot]
+ end
+ end
+
+ KAFKA[(Kafka
trades topic)]
+ REDIS[(Redis)]
+ CH[(ClickHouse)]
+
+ KAFKA -->|Trade Events| KCS
+
+ KCS -->|Update State| STATE
+
+ STATE --> S1
+ S1 --> I1
+ S1 --> I2
+ S1 --> I3
+
+ STATE --> S2
+ S2 --> I4
+
+ S1 -->|Candle Closed| KP
+ I1 -->|New Value| RDP
+ I2 -->|New Value| RDP
+
+ KP -->|candles-1m| KAFKA
+ RDP -->|Pub/Sub| REDIS
+ S1 -->|Persist| CHW
+ CHW --> CH
+
+ STATE -->|Periodic| CP
+ CP --> SNAP
+ SNAP -.->|Recovery| STATE
+
+ style STATE fill:#ffcccc
+ style S1 fill:#ccffcc
+ style I1 fill:#ccccff
+ style CP fill:#ffffcc
+```
+
+9. Horizontal Scaling Architecture
+
+```mermaid
+graph TB
+ subgraph "Load Balancers"
+ LB1[Load Balancer
WebSocket]
+ LB2[Load Balancer
gRPC]
+ end
+
+ subgraph "WebSocket Tier - Stateless"
+ WS1[WS Server 1
40K connections]
+ WS2[WS Server 2
40K connections]
+ WS3[WS Server 3
20K connections]
+ end
+
+ subgraph "Data Service Tier - Stateless"
+ DS1[Data Service 1]
+ DS2[Data Service 2]
+ DS3[Data Service 3]
+ end
+
+ subgraph "Stream Processing Tier - Stateful"
+ direction LR
+
+ subgraph "Candle Builders"
+ CB1[Builder 1
Partition 0-5]
+ CB2[Builder 2
Partition 6-11]
+ CB3[Builder 3
Partition 12-15]
+ end
+
+ subgraph "Indicator Computers"
+ IC1[Computer 1
Partition 0-5]
+ IC2[Computer 2
Partition 6-11]
+ IC3[Computer 3
Partition 12-15]
+ end
+ end
+
+ subgraph "Shared State"
+ KAFKA[(Kafka
16 Partitions)]
+ REDIS[(Redis Cluster
3 Master + 3 Replica)]
+ CH[(ClickHouse
Distributed)]
+ end
+
+ CLIENTS[100K Clients]
+
+ CLIENTS --> LB1
+
+ LB1 --> WS1
+ LB1 --> WS2
+ LB1 --> WS3
+
+ WS1 --> LB2
+ WS2 --> LB2
+ WS3 --> LB2
+
+ LB2 --> DS1
+ LB2 --> DS2
+ LB2 --> DS3
+
+ DS1 --> REDIS
+ DS1 --> CH
+ DS2 --> REDIS
+ DS2 --> CH
+ DS3 --> REDIS
+ DS3 --> CH
+
+ KAFKA --> CB1
+ KAFKA --> CB2
+ KAFKA --> CB3
+
+ KAFKA --> IC1
+ KAFKA --> IC2
+ KAFKA --> IC3
+
+ CB1 --> KAFKA
+ CB2 --> KAFKA
+ CB3 --> KAFKA
+
+ CB1 --> REDIS
+ CB2 --> REDIS
+ CB3 --> REDIS
+
+ IC1 --> REDIS
+ IC2 --> REDIS
+ IC3 --> REDIS
+
+ WS1 -.->|Pub/Sub| REDIS
+ WS2 -.->|Pub/Sub| REDIS
+ WS3 -.->|Pub/Sub| REDIS
+
+ style LB1 fill:#ff9999
+ style WS1 fill:#99ccff
+ style DS1 fill:#99ff99
+ style CB1 fill:#ffcc99
+ style IC1 fill:#cc99ff
+ style KAFKA fill:#ffcccc
+ style REDIS fill:#ccffff
+```
diff --git a/go.work b/go.work
index c2291c1..f4987e8 100644
--- a/go.work
+++ b/go.work
@@ -1,7 +1,9 @@
-go 1.25.4
+go 1.26.1
use (
+ ./apps/candle-service
./apps/matching-engine
+ ./apps/trade-ingestor-service
./apps/websocket-server
./packages/proto-defs/go/generated
)
diff --git a/go.work.sum b/go.work.sum
index 404be8c..7374d9c 100644
--- a/go.work.sum
+++ b/go.work.sum
@@ -1,29 +1,128 @@
cel.dev/expr v0.24.0/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw=
+cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4=
cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10=
+codeberg.org/go-fonts/liberation v0.5.0/go.mod h1:zS/2e1354/mJ4pGzIIaEtm/59VFCFnYC7YV6YdGl5GU=
+codeberg.org/go-latex/latex v0.1.0/go.mod h1:LA0q/AyWIYrqVd+A9Upkgsb+IqPcmSTKc9Dny04MHMw=
+codeberg.org/go-pdf/fpdf v0.10.0/go.mod h1:Y0DGRAdZ0OmnZPvjbMp/1bYxmIPxm0ws4tfoPOc4LjU=
+dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA=
+git.sr.ht/~sbinet/gg v0.6.0/go.mod h1:uucygbfC9wVPQIfrmwM2et0imr8L7KQWywX0xpFMm94=
+github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0/go.mod h1:P4WPRUkOhJC13W//jWpyfJNDAIpvRbAUIYLX/4jtlE0=
-github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0/go.mod h1:P4WPRUkOhJC13W//jWpyfJNDAIpvRbAUIYLX/4jtlE0=
+github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
+github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b/go.mod h1:1KcenG0jGWcpt8ov532z81sp/kMMUG485J2InIOyADM=
+github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
+github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
+github.com/campoy/embedmd v1.0.0/go.mod h1:oxyr9RCiSXg0M3VJ3ks0UGfp98BpSSGr0kpiX3MzVl8=
+github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f/go.mod h1:HlzOvOjVBOfTGSRXRyY0OiCS/3J1akRGQQpRO/7zyF4=
+github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5/go.mod h1:KdCmV+x/BuvyMxRnYBlmVaq4OLiKW6iRQfvC62cvdkI=
+github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M=
+github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk=
+github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
+github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw=
+github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc=
+github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
+github.com/dmarkham/enumer v1.6.3/go.mod h1:DyjXaqCglj4GhELF73oWiparNkYkXvmOBLza/o4kO74=
+github.com/docker/docker v28.5.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE=
+github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
+github.com/ebitengine/purego v0.8.4/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
github.com/envoyproxy/go-control-plane v0.13.5-0.20251024222203-75eaa193e329/go.mod h1:Alz8LEClvR7xKsrq3qzoc4N0guvVNSS8KmSChGYr9hs=
+github.com/envoyproxy/go-control-plane v0.14.0/go.mod h1:NcS5X47pLl/hfqxU70yPwL9ZMkUlwlKxtAohpi2wBEU=
github.com/envoyproxy/go-control-plane/envoy v1.35.0/go.mod h1:09qwbGVuSWWAyN5t/b3iyVfz5+z8QWGrzkoqm/8SbEs=
+github.com/envoyproxy/go-control-plane/envoy v1.36.0/go.mod h1:ty89S1YCCVruQAm9OtKeEkQLTb+Lkz0k8v9W0Oxsv98=
github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4=
github.com/envoyproxy/protoc-gen-validate v1.2.1/go.mod h1:d/C80l/jxXLdfEIhX1W2TmLfsJ31lvEjwamM4DxlWXU=
+github.com/envoyproxy/protoc-gen-validate v1.3.0/go.mod h1:HvYl7zwPa5mffgyeTUHA9zHIH36nmrm7oCbo4YKoSWA=
+github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
+github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
+github.com/goccmack/gocc v0.0.0-20230228185258-2292f9e40198/go.mod h1:DTh/Y2+NbnOVVoypCCQrovMPDKUGp4yZpSbWg5D0XIM=
+github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
github.com/golang/glog v1.2.5/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w=
-github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
-github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/hashicorp/go-version v1.8.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
+github.com/iancoleman/strcase v0.3.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho=
+github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
+github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
+github.com/lyft/protoc-gen-star/v2 v2.0.4-0.20230330145011-496ad1ac90a4/go.mod h1:amey7yeodaJhXSbf/TlLvWiqQfLOSpEk//mLlc+axEk=
+github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
+github.com/mkevac/debugcharts v0.0.0-20191222103121-ae1c48aa8615/go.mod h1:Ad7oeElCZqA1Ufj0U9/liOF4BtVepxRcTvr2ey7zTvM=
+github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
+github.com/moby/go-archive v0.1.0/go.mod h1:G9B+YoujNohJmrIYFBpSd54GTUB4lt9S+xVQvsJyFuo=
+github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc=
+github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko=
+github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs=
+github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28=
+github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y=
+github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
+github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
+github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M=
+github.com/pascaldekloe/name v1.0.1/go.mod h1:Z//MfYJnH4jVpQ9wkclwu2I2MkHmXTlT9wR5UZScttM=
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8=
+github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
+github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
+github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
+github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
+github.com/shirou/gopsutil/v4 v4.25.6/go.mod h1:PfybzyydfZcN+JMMjkF6Zb8Mq1A/VcogFFg7hj50W9c=
+github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
+github.com/spf13/afero v1.10.0/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ=
github.com/spiffe/go-spiffe/v2 v2.6.0/go.mod h1:gm2SeUoMZEtpnzPNs2Csc0D/gX33k1xIx7lEzqblHEs=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
+github.com/testcontainers/testcontainers-go v0.40.0/go.mod h1:FSXV5KQtX2HAMlm7U3APNyLkkap35zNLxukw9oBi/MY=
+github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI=
+github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY=
+github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
+github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
+github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
go.opentelemetry.io/contrib/detectors/gcp v1.38.0/go.mod h1:SU+iU7nu5ud4oCb3LQOhIZ3nRLj6FNVrKgtflbaf2ts=
+go.opentelemetry.io/contrib/detectors/gcp v1.39.0/go.mod h1:t/OGqzHBa5v6RHZwrDBJ2OirWc+4q/w2fTbLZwAKjTk=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw=
+go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0/go.mod h1:IPtUMKL4O3tH5y+iXVyAXqpAwMuzC1IrxVS81rummfE=
+go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI=
+go.opentelemetry.io/otel/metric v1.41.0/go.mod h1:xPvCwd9pU0VN8tPZYzDZV/BMj9CM9vs00GuBjeKhJps=
+go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg=
+go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA=
+go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs=
+go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM=
+go.opentelemetry.io/proto/otlp v1.7.1/go.mod h1:b2rVh6rfI/s2pHWNlB7ILJcRALpcNDzKhACevjI+ZnE=
+go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
+go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
+go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
+golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU=
+golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0=
+golang.org/x/crypto v0.47.0/go.mod h1:ff3Y9VzzKbwSSEzWqJsJVBnWmRwRSHt/6Op5n9bQc4A=
+golang.org/x/image v0.25.0/go.mod h1:tCAmOEGthTtkalusGp1g3xa2gke8J6c2N565dTyl9Rs=
golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w=
+golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc=
+golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY=
+golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw=
golang.org/x/oauth2 v0.32.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA=
+golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA=
golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
+golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
+golang.org/x/telemetry v0.0.0-20251111182119-bc8e575c7b54/go.mod h1:hKdjCMrbv9skySur+Nek8Hd0uJ0GuxJIoIX2payrIdQ=
+golang.org/x/telemetry v0.0.0-20260311193753-579e4da9a98c/go.mod h1:TpUTTEp9frx7rTdLpC9gFG9kdI7zVLFTFFlqaH2Cncw=
golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254=
+golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg=
+golang.org/x/term v0.42.0/go.mod h1:Dq/D+snpsbazcBG5+F9Q1n2rXV8Ma+71xEjTRufARgY=
golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs=
-golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ=
+golang.org/x/tools v0.43.0/go.mod h1:uHkMso649BX2cZK6+RpuIPXS3ho2hZo4FVwfoy1vIk0=
+gonum.org/v1/plot v0.15.2/go.mod h1:DX+x+DWso3LTha+AdkJEv5Txvi+Tql3KAGkehP0/Ubg=
google.golang.org/genproto/googleapis/api v0.0.0-20251029180050-ab9386a59fda/go.mod h1:fDMmzKV90WSg1NbozdqrE64fkuTv6mlq2zxo9ad+3yo=
-google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:+rXWjjaukWZun3mLfjmVnQi18E1AsFbDN9QdJ5YXLto=
+google.golang.org/genproto/googleapis/api v0.0.0-20260120221211-b8f7ae30c516/go.mod h1:p3MLuOwURrGBRoEyFHBT3GjUwaCQVKeNqqWxlcISGdw=
+google.golang.org/grpc/examples v0.0.0-20250407062114-b368379ef8f6/go.mod h1:6ytKWczdvnpnO+m+JiG9NjEDzR1FJfsnmJdG7B8QVZ8=
google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
+google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
diff --git a/packages/proto-defs/proto/aggeration/v1/candles.proto b/packages/proto-defs/proto/aggeration/v1/candles.proto
new file mode 100644
index 0000000..1fc7ef3
--- /dev/null
+++ b/packages/proto-defs/proto/aggeration/v1/candles.proto
@@ -0,0 +1,46 @@
+syntax = "proto3";
+
+package aggeration.v1;
+
+import "aggeration/v1/common.proto";
+
+option go_package = "github.com/sameerkrdev/nerve/packages/proto-defs/proto/aggeration/v1";
+
+message Candle {
+ double o = 1;
+ double h = 2;
+ double l = 3;
+ double c = 4;
+ int64 v = 5;
+
+ bool is_closed = 6;
+ optional int64 close_time = 7; // seconds
+ int64 open_time = 8; // seconds
+}
+
+message CandleBatch {
+ string symbol = 1;
+ Timeframe timeframe = 2;
+ repeated Candle candles = 3;
+}
+
+message GetCandlesRequest {
+ string symbol = 1;
+ // string currency = 2;
+ Timeframe timeframe = 3;
+ oneof method {
+ int64 number_of_candlesticks = 4;
+ DateRange date_range = 5;
+ }
+}
+
+message GetCandlesResponse {
+ oneof result {
+ CandleBatch data = 1;
+ Error error = 2;
+ }
+}
+
+service CandleService {
+ rpc GetCandles(GetCandlesRequest) returns (GetCandlesResponse);
+}
diff --git a/packages/proto-defs/proto/aggeration/v1/common.proto b/packages/proto-defs/proto/aggeration/v1/common.proto
new file mode 100644
index 0000000..0b21506
--- /dev/null
+++ b/packages/proto-defs/proto/aggeration/v1/common.proto
@@ -0,0 +1,53 @@
+syntax = "proto3";
+
+package aggeration.v1;
+
+option go_package = "github.com/sameerkrdev/nerve/packages/proto-defs/proto/aggeration/v1";
+
+enum Timeframe {
+ TIMEFRAME_UNSPECIFIED = 0;
+ TIMEFRAME_1M = 60; // 1 minute
+ TIMEFRAME_3M = 180; // 3 minutes
+ TIMEFRAME_5M = 300; // 5 minutes
+ TIMEFRAME_15M = 900; // 15 minutes
+ TIMEFRAME_30M = 1800; // 30 minutes
+ TIMEFRAME_1H = 3600; // 1 hour
+ TIMEFRAME_2H = 7200; // 2 hours
+ TIMEFRAME_4H = 14400; // 4 hours
+ TIMEFRAME_6H = 21600; // 6 hours
+ TIMEFRAME_12H = 43200; // 12 hours
+ TIMEFRAME_1D = 86400; // 1 day
+ TIMEFRAME_1W = 604800; // 1 week
+ TIMEFRAME_1MON = 2592000; // 1 month (30 days)
+}
+
+enum Source {
+ SOURCE_UNSPECIFIED = 0;
+ SOURCE_CLOSE = 1;
+ SOURCE_OPEN = 2;
+ SOURCE_HIGH = 3;
+ SOURCE_LOW = 4;
+ SOURCE_MIDDLE = 5;
+}
+
+enum ErrorCode {
+ ERROR_CODE_UNSPECIFIED = 0;
+ ERROR_CODE_INVALID_SYMBOL = 1;
+ ERROR_CODE_INVALID_TIMEFRAME = 2;
+ ERROR_CODE_RATE_LIMIT_EXCEEDED = 3;
+ ERROR_CODE_UNAUTHORIZED = 4;
+ ERROR_CODE_INTERNAL_ERROR = 5;
+ ERROR_CODE_DATA_NOT_AVAILABLE = 6;
+ ERROR_CODE_INVALID_PARAMETERS = 7;
+}
+
+message Error {
+ ErrorCode code = 1;
+ string message = 2;
+ map details = 3;
+}
+
+message DateRange {
+ int64 from = 1;
+ int64 to = 2;
+}
diff --git a/packages/proto-defs/proto/aggeration/v1/indicators.proto b/packages/proto-defs/proto/aggeration/v1/indicators.proto
new file mode 100644
index 0000000..6460a7e
--- /dev/null
+++ b/packages/proto-defs/proto/aggeration/v1/indicators.proto
@@ -0,0 +1,101 @@
+syntax = "proto3";
+
+package aggeration.v1;
+
+import "aggeration/v1/common.proto";
+import "google/protobuf/empty.proto";
+
+option go_package = "github.com/sameerkrdev/nerve/packages/proto-defs/proto/aggeration/v1";
+
+enum IndicatorCategory {
+ INDICATOR_CATEGORY_UNSPECIFIED = 0;
+ INDICATOR_CATEGORY_OVERLAY = 1; // plotted on price (EMA, VWAP)
+ INDICATOR_CATEGORY_OSCILLATOR = 2; // separate pane (RSI, MACD)
+ INDICATOR_CATEGORY_VOLUME = 3; // volume-based
+}
+
+message IndicatorParam {
+ string key = 1;
+
+ oneof value {
+ int64 int_value = 2;
+ double double_value = 3;
+ string string_value = 4; // source = "close", "hlc3"
+ bool bool_value = 5;
+ }
+
+ // SMA/EMA/WMA/VWMA: {source: Source, period:""}
+ // RSI: {source: Source, period:""}
+ // MACD: {source: Source, fast: "", slow:"", signal: "", }
+}
+
+message IndicatorRegistryItem {
+ string indicator_id = 1; // "ema", "rsi", "macd"
+ string name = 2;
+ string description = 3;
+
+ IndicatorCategory category = 4;
+ bool multi_output = 5; // true for MACD, BB, etc.
+}
+
+message GetIndicatorRegistryResponse {
+ repeated IndicatorRegistryItem indicators = 1;
+}
+
+message IndicatorInstance {
+ string instance_id = 1; // unique per user/chart
+
+ string indicator_id = 2;
+
+ string symbol = 3;
+ string timeframe = 4;
+
+ repeated IndicatorParam params = 5;
+}
+
+message AddIndicatorRequest {
+ IndicatorInstance instance = 1;
+}
+
+message UpdateIndicatorRequest {
+ string instance_id = 1;
+ repeated IndicatorParam params = 2;
+}
+
+message RemoveIndicatorRequest {
+ string instance_id = 1;
+}
+
+message IndicatorOutput {
+ string key = 1; // "value", "macd", "signal", "histogram"
+ double value = 2;
+}
+
+message IndicatorValue {
+ string instance_id = 1;
+
+ string symbol = 2;
+ string timeframe = 3;
+
+ int64 timestamp = 4;
+
+ repeated IndicatorOutput outputs = 5;
+}
+
+service IndicatorService {
+ // Registry
+ rpc GetIndicatorRegistry(google.protobuf.Empty) returns (GetIndicatorRegistryResponse);
+
+ // Basic indicator lifecycle
+ rpc AddIndicator(AddIndicatorRequest) returns (google.protobuf.Empty);
+
+ rpc UpdateIndicator(UpdateIndicatorRequest) returns (google.protobuf.Empty);
+
+ rpc RemoveIndicator(RemoveIndicatorRequest) returns (google.protobuf.Empty);
+
+ // Custom graph indicator
+ // rpc AddCustomIndicator(AddCustomIndicatorRequest) returns (google.protobuf.Empty);
+
+ // Realtime stream
+ // rpc SubscribeIndicatorUpdates(SubscribeIndicatorUpdatesRequest) returns (stream IndicatorValue);
+}
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 6f00e6a..40e2795 100755
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -23,8 +23,6 @@ importers:
specifier: 5.9.2
version: 5.9.2
- apps/aggeration-service: {}
-
apps/api-gateway:
dependencies:
"@grpc/grpc-js":
@@ -45,9 +43,6 @@ importers:
"@repo/proto-defs":
specifier: workspace:*
version: link:../../packages/proto-defs
- "@repo/utils":
- specifier: workspace:*
- version: link:../../packages/utils
dotenv:
specifier: ^17.2.2
version: 17.2.2
@@ -95,6 +90,8 @@ importers:
specifier: ^5.9.2
version: 5.9.2
+ apps/candle-service: {}
+
apps/express-base-template:
dependencies:
"@repo/logger":
@@ -543,28 +540,6 @@ importers:
specifier: 5.9.2
version: 5.9.2
- packages/utils:
- dependencies:
- "@repo/proto-defs":
- specifier: workspace:*
- version: link:../proto-defs
- devDependencies:
- "@repo/eslint-config":
- specifier: workspace:*
- version: link:../eslint-config
- "@repo/types":
- specifier: workspace:*
- version: link:../types
- "@repo/typescript-config":
- specifier: workspace:*
- version: link:../typescript-config
- "@types/node":
- specifier: ^22.15.3
- version: 22.15.3
- typescript:
- specifier: ^5.9.2
- version: 5.9.3
-
packages/validator:
dependencies:
zod: