diff --git a/.github/workflows/mac.yaml b/.github/workflows/mac.yaml index ccb21ebaab5af..ccc36c364f4f2 100644 --- a/.github/workflows/mac.yaml +++ b/.github/workflows/mac.yaml @@ -56,7 +56,7 @@ jobs: - name: Setup Go environment uses: actions/setup-go@v2.2.0 with: - go-version: '~1.21.10' + go-version: '~1.21.11' - name: Mac Cache Go Mod Volumes uses: actions/cache@v3 with: diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 99bfc0f1546ae..838033c0b38bb 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -245,8 +245,8 @@ pip3 install conan==1.61.0 #### Install GO 1.80 ```bash -wget https://go.dev/dl/go1.21.10.linux-arm64.tar.gz -tar zxf go1.21.10.linux-arm64.tar.gz +wget https://go.dev/dl/go1.21.11.linux-arm64.tar.gz +tar zxf go1.21.11.linux-arm64.tar.gz mv ./go /usr/local vi /etc/profile export PATH=$PATH:/usr/local/go/bin diff --git a/build/docker/builder/cpu/amazonlinux2023/Dockerfile b/build/docker/builder/cpu/amazonlinux2023/Dockerfile index d052c37755b73..18e12115d06ce 100644 --- a/build/docker/builder/cpu/amazonlinux2023/Dockerfile +++ b/build/docker/builder/cpu/amazonlinux2023/Dockerfile @@ -22,7 +22,7 @@ ENV GOPATH /go ENV GOROOT /usr/local/go ENV GO111MODULE on ENV PATH $GOPATH/bin:$GOROOT/bin:$PATH -RUN mkdir -p /usr/local/go && wget -qO- "https://go.dev/dl/go1.21.10.linux-$TARGETARCH.tar.gz" | tar --strip-components=1 -xz -C /usr/local/go && \ +RUN mkdir -p /usr/local/go && wget -qO- "https://go.dev/dl/go1.21.11.linux-$TARGETARCH.tar.gz" | tar --strip-components=1 -xz -C /usr/local/go && \ mkdir -p "$GOPATH/src" "$GOPATH/bin" && \ go clean --modcache && \ chmod -R 777 "$GOPATH" && chmod -R a+w $(go env GOTOOLDIR) @@ -52,7 +52,7 @@ RUN mkdir /tmp/ccache && cd /tmp/ccache &&\ # refer: https://code.visualstudio.com/docs/remote/containers-advanced#_avoiding-extension-reinstalls-on-container-rebuild RUN mkdir -p /home/milvus/.vscode-server/extensions \ - /home/milvus/.vscode-server-insiders/extensions \ + /home/milvus/.vscode-server-insiders/extensions \ && chmod -R 777 /home/milvus COPY --chown=0:0 build/docker/builder/entrypoint.sh / diff --git a/build/docker/builder/cpu/rockylinux8/Dockerfile b/build/docker/builder/cpu/rockylinux8/Dockerfile index ec1ea089035c3..0cd0aa117080f 100644 --- a/build/docker/builder/cpu/rockylinux8/Dockerfile +++ b/build/docker/builder/cpu/rockylinux8/Dockerfile @@ -1,8 +1,8 @@ FROM rockylinux/rockylinux:8 as vcpkg-installer RUN dnf -y install curl wget tar zip unzip git \ - gcc gcc-c++ make cmake \ - perl-IPC-Cmd perl-Digest-SHA + gcc gcc-c++ make cmake \ + perl-IPC-Cmd perl-Digest-SHA # install ninjia RUN dnf -y update && \ @@ -32,8 +32,8 @@ FROM rockylinux/rockylinux:8 ARG TARGETARCH RUN dnf install -y make cmake automake gcc gcc-c++ curl zip unzip tar git which \ - libaio libuuid-devel wget python3 python3-pip \ - pkg-config perl-IPC-Cmd perl-Digest-SHA libatomic libtool + libaio libuuid-devel wget python3 python3-pip \ + pkg-config perl-IPC-Cmd perl-Digest-SHA libatomic libtool # install openblas-devel texinfo ninja RUN dnf -y update && \ @@ -43,7 +43,7 @@ RUN dnf -y update && \ RUN pip3 install conan==1.61.0 -RUN mkdir -p /usr/local/go && wget -qO- "https://go.dev/dl/go1.21.10.linux-$TARGETARCH.tar.gz" | tar --strip-components=1 -xz -C /usr/local/go +RUN mkdir -p /usr/local/go && wget -qO- "https://go.dev/dl/go1.21.11.linux-$TARGETARCH.tar.gz" | tar --strip-components=1 -xz -C /usr/local/go RUN curl https://sh.rustup.rs -sSf | \ sh -s -- --default-toolchain=1.73 -y diff --git a/build/docker/builder/cpu/ubuntu20.04/Dockerfile b/build/docker/builder/cpu/ubuntu20.04/Dockerfile index 8e7f98ee89cf1..a76f8d5cb8384 100644 --- a/build/docker/builder/cpu/ubuntu20.04/Dockerfile +++ b/build/docker/builder/cpu/ubuntu20.04/Dockerfile @@ -40,14 +40,14 @@ ENV GOPATH /go ENV GOROOT /usr/local/go ENV GO111MODULE on ENV PATH $GOPATH/bin:$GOROOT/bin:$PATH -RUN mkdir -p /usr/local/go && wget -qO- "https://go.dev/dl/go1.21.10.linux-$TARGETARCH.tar.gz" | tar --strip-components=1 -xz -C /usr/local/go && \ +RUN mkdir -p /usr/local/go && wget -qO- "https://go.dev/dl/go1.21.11.linux-$TARGETARCH.tar.gz" | tar --strip-components=1 -xz -C /usr/local/go && \ mkdir -p "$GOPATH/src" "$GOPATH/bin" && \ go clean --modcache && \ chmod -R 777 "$GOPATH" && chmod -R a+w $(go env GOTOOLDIR) # refer: https://code.visualstudio.com/docs/remote/containers-advanced#_avoiding-extension-reinstalls-on-container-rebuild RUN mkdir -p /home/milvus/.vscode-server/extensions \ - /home/milvus/.vscode-server-insiders/extensions \ + /home/milvus/.vscode-server-insiders/extensions \ && chmod -R 777 /home/milvus COPY --chown=0:0 build/docker/builder/entrypoint.sh / diff --git a/build/docker/builder/cpu/ubuntu22.04/Dockerfile b/build/docker/builder/cpu/ubuntu22.04/Dockerfile index be108908caf1b..f99bd90bc6911 100644 --- a/build/docker/builder/cpu/ubuntu22.04/Dockerfile +++ b/build/docker/builder/cpu/ubuntu22.04/Dockerfile @@ -46,14 +46,14 @@ ENV GOPATH /go ENV GOROOT /usr/local/go ENV GO111MODULE on ENV PATH $GOPATH/bin:$GOROOT/bin:$PATH -RUN mkdir -p /usr/local/go && wget -qO- "https://go.dev/dl/go1.21.10.linux-$TARGETARCH.tar.gz" | tar --strip-components=1 -xz -C /usr/local/go && \ +RUN mkdir -p /usr/local/go && wget -qO- "https://go.dev/dl/go1.21.11.linux-$TARGETARCH.tar.gz" | tar --strip-components=1 -xz -C /usr/local/go && \ mkdir -p "$GOPATH/src" "$GOPATH/bin" && \ go clean --modcache && \ chmod -R 777 "$GOPATH" && chmod -R a+w $(go env GOTOOLDIR) # refer: https://code.visualstudio.com/docs/remote/containers-advanced#_avoiding-extension-reinstalls-on-container-rebuild RUN mkdir -p /home/milvus/.vscode-server/extensions \ - /home/milvus/.vscode-server-insiders/extensions \ + /home/milvus/.vscode-server-insiders/extensions \ && chmod -R 777 /home/milvus COPY --chown=0:0 build/docker/builder/entrypoint.sh / diff --git a/build/docker/builder/gpu/ubuntu20.04/Dockerfile b/build/docker/builder/gpu/ubuntu20.04/Dockerfile index 9378b3fd861b1..749197eeef0bf 100644 --- a/build/docker/builder/gpu/ubuntu20.04/Dockerfile +++ b/build/docker/builder/gpu/ubuntu20.04/Dockerfile @@ -51,7 +51,7 @@ ENV GOPATH /go ENV GOROOT /usr/local/go ENV GO111MODULE on ENV PATH $GOPATH/bin:$GOROOT/bin:$PATH -RUN mkdir -p /usr/local/go && wget -qO- "https://go.dev/dl/go1.21.10.linux-$TARGETARCH.tar.gz" | tar --strip-components=1 -xz -C /usr/local/go && \ +RUN mkdir -p /usr/local/go && wget -qO- "https://go.dev/dl/go1.21.11.linux-$TARGETARCH.tar.gz" | tar --strip-components=1 -xz -C /usr/local/go && \ mkdir -p "$GOPATH/src" "$GOPATH/bin" && \ curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b ${GOROOT}/bin v1.46.2 && \ # export GO111MODULE=on && go get github.com/quasilyte/go-ruleguard/cmd/ruleguard@v0.2.1 && \ @@ -74,7 +74,7 @@ RUN echo 'root:root' | chpasswd # refer: https://code.visualstudio.com/docs/remote/containers-advanced#_avoiding-extension-reinstalls-on-container-rebuild RUN mkdir -p /home/milvus/.vscode-server/extensions \ - /home/milvus/.vscode-server-insiders/extensions \ + /home/milvus/.vscode-server-insiders/extensions \ && chmod -R 777 /home/milvus COPY --chown=0:0 build/docker/builder/entrypoint.sh / diff --git a/build/docker/builder/gpu/ubuntu22.04/Dockerfile b/build/docker/builder/gpu/ubuntu22.04/Dockerfile index b27a917d584e3..35d3edfe6b16d 100644 --- a/build/docker/builder/gpu/ubuntu22.04/Dockerfile +++ b/build/docker/builder/gpu/ubuntu22.04/Dockerfile @@ -13,7 +13,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends wget curl ca-ce # Install go -RUN mkdir -p /usr/local/go && wget -qO- "https://go.dev/dl/go1.21.10.linux-$TARGETARCH.tar.gz" | tar --strip-components=1 -xz -C /usr/local/go +RUN mkdir -p /usr/local/go && wget -qO- "https://go.dev/dl/go1.21.11.linux-$TARGETARCH.tar.gz" | tar --strip-components=1 -xz -C /usr/local/go # Install conan RUN pip3 install conan==1.61.0 # Install rust @@ -31,7 +31,7 @@ RUN vcpkg install azure-identity-cpp azure-storage-blobs-cpp gtest # refer: https://code.visualstudio.com/docs/remote/containers-advanced#_avoiding-extension-reinstalls-on-container-rebuild RUN mkdir -p /home/milvus/.vscode-server/extensions \ - /home/milvus/.vscode-server-insiders/extensions \ + /home/milvus/.vscode-server-insiders/extensions \ && chmod -R 777 /home/milvus diff --git a/build/docker/meta-migration/builder/Dockerfile b/build/docker/meta-migration/builder/Dockerfile index f102266fcfc44..f299854cb60c9 100644 --- a/build/docker/meta-migration/builder/Dockerfile +++ b/build/docker/meta-migration/builder/Dockerfile @@ -1,2 +1,2 @@ -FROM golang:1.21.10-alpine3.19 +FROM golang:1.21.11-alpine3.19 RUN apk add --no-cache make bash \ No newline at end of file diff --git a/internal/storage/serde_events_test.go b/internal/storage/serde_events_test.go index 4361bbcc3f539..83953de999453 100644 --- a/internal/storage/serde_events_test.go +++ b/internal/storage/serde_events_test.go @@ -19,9 +19,7 @@ package storage import ( "bytes" "context" - "fmt" "io" - "math" "strconv" "testing" @@ -495,116 +493,3 @@ func readDeltaLog(size int, blob *Blob) error { } return nil } - -func TestReaderIO(t *testing.T) { - totalMemory := 64 // MB - wFileSize := 1000 // MB - nFileSize := 32 // MB - totalRows := 400000 - - wRowsPerRowGroup := totalRows / wFileSize - nRowsPerRowGroup := totalRows / nFileSize - - wRowGroupSize := 1 // MB - nRowGroupSize := 1 // MB - - maxAlignment := func(wRowsInMemory, nRowsInMemory int, memorySize int) (int, int, int) { - dp := make([][]int, memorySize+1) - for i := range dp { - dp[i] = make([]int, memorySize+1) - } - for i := 0; i <= memorySize; i++ { - for j := 0; j <= i; j++ { - a := j / wRowGroupSize // - b := (i - j) / nRowGroupSize - wRows := wRowsInMemory + a*wRowsPerRowGroup - nRows := nRowsInMemory + b*nRowsPerRowGroup - dp[i][j] = min(wRows, nRows) - } - } - maxMinRows := 0 - wRowGroupToRead := 0 - nRowGroupToRead := 0 - for i := 0; i <= memorySize; i++ { - for j := 0; j <= i; j++ { - if dp[i][j] > maxMinRows { - maxMinRows = dp[i][j] - wRowGroupToRead = j - nRowGroupToRead = i - j - } - } - } - return wRowGroupToRead, nRowGroupToRead, maxMinRows - } - - wio := 0 - nio := 0 - - wRowsInMemory := 0 - nRowsInMemory := 0 - - memory := totalMemory - - for totalRows > 0 { - // memory allocator - wRowGroupToRead, nRowGroupToRead, maxMinRows := maxAlignment(wRowsInMemory, nRowsInMemory, memory) - fmt.Printf("read wide %d row groups , read narrow %d row groups, max rows %d \n", wRowGroupToRead, nRowGroupToRead, maxMinRows) - - // IO Merge - if wRowGroupToRead > 0 { - wio += 1 - } - if nRowGroupToRead > 0 { - nio += 1 - } - - // deserailize maxMinRows - totalRows -= maxMinRows - wRowsInMemory += wRowGroupToRead*wRowsPerRowGroup - maxMinRows - nRowsInMemory += nRowGroupToRead*nRowsPerRowGroup - maxMinRows - - ceilFloatToInt := func(n float64) int { - return int(math.Ceil(n)) - } - memory -= (ceilFloatToInt(float64(wRowGroupSize/wRowsPerRowGroup)) + ceilFloatToInt(float64(nRowGroupSize/nRowsPerRowGroup))) * maxMinRows - - fmt.Printf("%d wide rows in memory, %d narrow rows in memory, memory left: %d\n", wRowsInMemory, nRowsInMemory, memory) - } - - fmt.Printf("wide column read IO: %d, narrow columns read IO: %d, total read IO: %d under reading memory %d MB", wio, nio, wio+nio, totalMemory) -} - -func TestWriterIO(t *testing.T) { - records := 400000 - wide_size := 2.5 - narrow_size := 0.08 - memory := float64(64 * 1024) - - wm := float64(0) - nm := float64(0) - wio := 0 - nio := 0 - for i := 0; i < records; i++ { - wm += wide_size - nm += narrow_size - // 扩展：use max heap to get max memory use of column group - if wm+nm >= memory { - if wm >= nm { - wm = 0 - wio += 1 - } else { - nm = 0 - nio += 1 - } - } - } - if wm > 0 { - wio += 1 - } - if nm > 0 { - nio += 1 - } - fmt.Println(wio) - fmt.Println(nio) - fmt.Println(wio + nio) -} diff --git a/tests/go_client/go.mod b/tests/go_client/go.mod index bf0e7de0b959d..043ffab743b4c 100644 --- a/tests/go_client/go.mod +++ b/tests/go_client/go.mod @@ -2,7 +2,7 @@ module github.com/milvus-io/milvus/tests/go_client go 1.21 -toolchain go1.21.10 +toolchain go1.21.11 require ( github.com/milvus-io/milvus/client/v2 v2.0.0-20240704083609-fcafdb6d5f68