chore: centralize proto files

.github/workflows/ci.yaml

@@ -7,26 +7,20 @@ on:
     branches: [ main ]
 
 jobs:
-  build:
-    name: Unit Tests
+  proto:
+    name: Proto Generation
     runs-on: ubuntu-latest
     steps:
-
     - name: Set up Go 1.x
       uses: actions/setup-go@v2
       with:
-        go-version: ^1.17
-
+        go-version: ^1.20
     - name: Check out code into the Go module directory
       uses: actions/checkout@v2
-
-    - name: Get dependencies
-      run: go mod download
-
     - name: Install protoc
       run: |
         set -eux -o pipefail
-        PROTOC_VERSION=3.19.4
+        PROTOC_VERSION=24.3
         PROTOC_ZIP=protoc-$PROTOC_VERSION-linux-x86_64.zip
         curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOC_VERSION/$PROTOC_ZIP
         sudo unzip -o $PROTOC_ZIP -d /usr/local bin/protoc
@@ -36,10 +30,22 @@ jobs:
         sudo find /usr/local/include -type d | xargs sudo chmod a+rx
         rm -f $PROTOC_ZIP
         ls /usr/local/include/google/protobuf/
-
+    - name: Proto Gen
+      run: make proto
+  build:
+    name: Unit Tests
+    runs-on: ubuntu-latest
+    steps:
+    - name: Set up Go 1.x
+      uses: actions/setup-go@v2
+      with:
+        go-version: ^1.20
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v2
+    - name: Get dependencies
+      run: go mod download
     - name: Install mockgen
       run: |
         go install github.com/golang/mock/[email protected]
-
     - name: Test
       run: make all

Makefile

@@ -1,19 +1,29 @@
+SHELL:=/bin/bash
+
+CURRENT_DIR=$(shell pwd)
+
+ORG?=numaproj
+PROJECT?=numaflow
+BRANCH?=main
+
 .PHONY: all
-all: proto generate test
+all: generate test
+
+clean:
+	-rm -rf ${CURRENT_DIR}/dist
 
 .PHONY: test
 test:
 	go test -race -v ./...
 
 .PHONY: proto
-proto:
+proto: clean
 	go mod vendor
-	./hack/protogen.sh
+	ORG=$(ORG) PROJECT=$(PROJECT) BRANCH=$(BRANCH) ./hack/protogen.sh
 	rm -rf ./vendor
 	go mod tidy
 
 .PHONY: generate
 generate:
 	go generate ./...
 
-

README.md

@@ -2,10 +2,41 @@
 
 This SDK provides the interfaces to implement [Numaflow](https://github.com/numaproj/numaflow) User Defined Sources,
 Source Transformer, Functions, Sinks or SideInputs in Golang.
+
 - Implement [User Defined Sources](https://pkg.go.dev/github.com/numaproj/numaflow-go/pkg/sourcer)
 - Implement [User Defined Source Transformers](https://pkg.go.dev/github.com/numaproj/numaflow-go/pkg/sourcetransformer)
 - Implement User Defined Functions
   - [Map](https://pkg.go.dev/github.com/numaproj/numaflow-go/pkg/mapper)
   - [Reduce](https://pkg.go.dev/github.com/numaproj/numaflow-go/pkg/reducer)
 - Implement [User Defined Sinks](https://pkg.go.dev/github.com/numaproj/numaflow-go/pkg/sinker)
-- Implement [User Defined SideInputs](https://pkg.go.dev/github.com/numaproj/numaflow-go/pkg/sideinput)
+- Implement [User Defined SideInputs](https://pkg.go.dev/github.com/numaproj/numaflow-go/pkg/sideinput)
+
+## Development
+
+### Useful Commands
+
+`make test` - Run the tests.
+`make proto`- Regenerate the protobuf files from the [proto files](https://github.com/numaproj/numaflow/tree/main/pkg/apis/proto) defined in [numaproj/numaflow](https://github.com/numaproj/numaflow) repository.
+`make proto ORG=xxx PROJECT=xxx BRANCH=xxx` - Regenerate the protobuf files from specified github repository. Default values: `ORG=numaproj PROJECT=numaflow BRANCH=main`
+
+### Proto File Changes
+
+The source proto files are located at [numaflow](https://github.com/numaproj/numaflow/tree/main/pkg/apis/proto) repository, so any proto file change should start from numaflow repository change, and Go should always be the 1st SDK to update.
+
+This means, usually there will be two PRs to update gRPC contracts, one in `numaflow` for proto file changes, the other one in `numaflow-go` to update the detailed implementation. There is one trick to unblock the code change in `numaflow-go` to run codegen for proto files, before the proto file change PR is merged in `numaflow` repository:
+
+```shell
+make proto ORG=xxx BRANCH=xxx
+```
+
+This will run proto gen with the source files defined in your own repository and branch.
+
+For example:
+
+```shell
+# Try to fetch proto files defined in branch "proto-chg" of https://github.com/anyone/numaflow
+make proto ORG=anyone BRANCH=proto-chg
+
+# Try to fetch proto files defined in branch "my-change" of https://github.com/numaproj/numaflow
+make proto BRANCH=my-change
+```

hack/protogen.sh

@@ -4,6 +4,8 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
+readonly REPO_ROOT="$(git rev-parse --show-toplevel)"
+
 if [ "`command -v protoc`" = "" ]; then
   echo "Please install protobuf with - brew install protobuf"
   exit 1
@@ -19,4 +21,22 @@ if [ "`command -v protoc-gen-go-grpc`" = "" ]; then
   go install -mod=vendor ./vendor/google.golang.org/grpc/cmd/protoc-gen-go-grpc
 fi
 
-protoc --go_out=paths=source_relative:. --go-grpc_out=paths=source_relative:. -I. $(find pkg/apis/proto -name '*.proto')
+mkdir -p ${REPO_ROOT}/dist
+
+ORG=${ORG:-numaproj}
+PROJECT=${PROJECT:-numaflow}
+BRANCH=${BRANCH:-main}
+
+REMOTE_URL_PRE="https://raw.githubusercontent.com/${ORG}/${PROJECT}/${BRANCH}/pkg/apis/proto"
+echo "Remote URL prefix: $REMOTE_URL_PRE"
+
+curl -Ls -o ${REPO_ROOT}/dist/map.proto ${REMOTE_URL_PRE}/map/v1/map.proto
+curl -Ls -o ${REPO_ROOT}/dist/reduce.proto ${REMOTE_URL_PRE}/reduce/v1/reduce.proto
+curl -Ls -o ${REPO_ROOT}/dist/mapstream.proto ${REMOTE_URL_PRE}/mapstream/v1/mapstream.proto
+curl -Ls -o ${REPO_ROOT}/dist/sideinput.proto ${REMOTE_URL_PRE}/sideinput/v1/sideinput.proto
+curl -Ls -o ${REPO_ROOT}/dist/sink.proto ${REMOTE_URL_PRE}/sink/v1/sink.proto
+curl -Ls -o ${REPO_ROOT}/dist/source.proto ${REMOTE_URL_PRE}/source/v1/source.proto
+curl -Ls -o ${REPO_ROOT}/dist/transform.proto ${REMOTE_URL_PRE}/sourcetransform/v1/transform.proto
+curl -Ls -o ${REPO_ROOT}/dist/sessionreduce.proto ${REMOTE_URL_PRE}/sessionreduce/v1/sessionreduce.proto
+
+protoc --go_out=module=github.com/numaproj/numaflow-go:. --go-grpc_out=module=github.com/numaproj/numaflow-go:. -I ${REPO_ROOT}/dist $(find ${REPO_ROOT}/dist -name '*.proto')