Skip to content

Commit

Permalink
upgrade go version to 1.21.11
Browse files Browse the repository at this point in the history 
Signed-off-by: shaoting-huang <[email protected]>
  • Loading branch information
@shaoting-huang
shaoting-huang committed Aug 5, 2024
1 parent c9b40ce commit d46d024
Show file tree
Hide file tree
Showing 11 changed files with 20 additions and 135 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/mac.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ jobs:
- name: Setup Go environment
uses: actions/[email protected]
with:
go-version: '~1.21.10'
go-version: '~1.21.11'
- name: Mac Cache Go Mod Volumes
uses: actions/cache@v3
with:
Expand Down
4 changes: 2 additions & 2 deletions DEVELOPMENT.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -245,8 +245,8 @@ pip3 install conan==1.61.0
#### Install GO 1.80

```bash
wget https://go.dev/dl/go1.21.10.linux-arm64.tar.gz
tar zxf go1.21.10.linux-arm64.tar.gz
wget https://go.dev/dl/go1.21.11.linux-arm64.tar.gz
tar zxf go1.21.11.linux-arm64.tar.gz
mv ./go /usr/local
vi /etc/profile
export PATH=$PATH:/usr/local/go/bin
Expand Down
4 changes: 2 additions & 2 deletions build/docker/builder/cpu/amazonlinux2023/Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ ENV GOPATH /go
ENV GOROOT /usr/local/go
ENV GO111MODULE on
ENV PATH $GOPATH/bin:$GOROOT/bin:$PATH
RUN mkdir -p /usr/local/go && wget -qO- "https://go.dev/dl/go1.21.10.linux-$TARGETARCH.tar.gz" | tar --strip-components=1 -xz -C /usr/local/go && \
RUN mkdir -p /usr/local/go && wget -qO- "https://go.dev/dl/go1.21.11.linux-$TARGETARCH.tar.gz" | tar --strip-components=1 -xz -C /usr/local/go && \
mkdir -p "$GOPATH/src" "$GOPATH/bin" && \
go clean --modcache && \
chmod -R 777 "$GOPATH" && chmod -R a+w $(go env GOTOOLDIR)
Expand Down Expand Up @@ -52,7 +52,7 @@ RUN mkdir /tmp/ccache && cd /tmp/ccache &&\

# refer: https://code.visualstudio.com/docs/remote/containers-advanced#_avoiding-extension-reinstalls-on-container-rebuild
RUN mkdir -p /home/milvus/.vscode-server/extensions \
/home/milvus/.vscode-server-insiders/extensions \
/home/milvus/.vscode-server-insiders/extensions \
&& chmod -R 777 /home/milvus

COPY --chown=0:0 build/docker/builder/entrypoint.sh /
Expand Down
10 changes: 5 additions & 5 deletions build/docker/builder/cpu/rockylinux8/Dockerfile
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
FROM rockylinux/rockylinux:8 as vcpkg-installer

RUN dnf -y install curl wget tar zip unzip git \
gcc gcc-c++ make cmake \
perl-IPC-Cmd perl-Digest-SHA
gcc gcc-c++ make cmake \
perl-IPC-Cmd perl-Digest-SHA

# install ninjia
RUN dnf -y update && \
Expand Down Expand Up @@ -32,8 +32,8 @@ FROM rockylinux/rockylinux:8
ARG TARGETARCH

RUN dnf install -y make cmake automake gcc gcc-c++ curl zip unzip tar git which \
libaio libuuid-devel wget python3 python3-pip \
pkg-config perl-IPC-Cmd perl-Digest-SHA libatomic libtool
libaio libuuid-devel wget python3 python3-pip \
pkg-config perl-IPC-Cmd perl-Digest-SHA libatomic libtool

# install openblas-devel texinfo ninja
RUN dnf -y update && \
Expand All @@ -43,7 +43,7 @@ RUN dnf -y update && \


RUN pip3 install conan==1.61.0
RUN mkdir -p /usr/local/go && wget -qO- "https://go.dev/dl/go1.21.10.linux-$TARGETARCH.tar.gz" | tar --strip-components=1 -xz -C /usr/local/go
RUN mkdir -p /usr/local/go && wget -qO- "https://go.dev/dl/go1.21.11.linux-$TARGETARCH.tar.gz" | tar --strip-components=1 -xz -C /usr/local/go
RUN curl https://sh.rustup.rs -sSf | \
sh -s -- --default-toolchain=1.73 -y

Expand Down
4 changes: 2 additions & 2 deletions build/docker/builder/cpu/ubuntu20.04/Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,14 +40,14 @@ ENV GOPATH /go
ENV GOROOT /usr/local/go
ENV GO111MODULE on
ENV PATH $GOPATH/bin:$GOROOT/bin:$PATH
RUN mkdir -p /usr/local/go && wget -qO- "https://go.dev/dl/go1.21.10.linux-$TARGETARCH.tar.gz" | tar --strip-components=1 -xz -C /usr/local/go && \
RUN mkdir -p /usr/local/go && wget -qO- "https://go.dev/dl/go1.21.11.linux-$TARGETARCH.tar.gz" | tar --strip-components=1 -xz -C /usr/local/go && \
mkdir -p "$GOPATH/src" "$GOPATH/bin" && \
go clean --modcache && \
chmod -R 777 "$GOPATH" && chmod -R a+w $(go env GOTOOLDIR)

# refer: https://code.visualstudio.com/docs/remote/containers-advanced#_avoiding-extension-reinstalls-on-container-rebuild
RUN mkdir -p /home/milvus/.vscode-server/extensions \
/home/milvus/.vscode-server-insiders/extensions \
/home/milvus/.vscode-server-insiders/extensions \
&& chmod -R 777 /home/milvus

COPY --chown=0:0 build/docker/builder/entrypoint.sh /
Expand Down
4 changes: 2 additions & 2 deletions build/docker/builder/cpu/ubuntu22.04/Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,14 +46,14 @@ ENV GOPATH /go
ENV GOROOT /usr/local/go
ENV GO111MODULE on
ENV PATH $GOPATH/bin:$GOROOT/bin:$PATH
RUN mkdir -p /usr/local/go && wget -qO- "https://go.dev/dl/go1.21.10.linux-$TARGETARCH.tar.gz" | tar --strip-components=1 -xz -C /usr/local/go && \
RUN mkdir -p /usr/local/go && wget -qO- "https://go.dev/dl/go1.21.11.linux-$TARGETARCH.tar.gz" | tar --strip-components=1 -xz -C /usr/local/go && \
mkdir -p "$GOPATH/src" "$GOPATH/bin" && \
go clean --modcache && \
chmod -R 777 "$GOPATH" && chmod -R a+w $(go env GOTOOLDIR)

# refer: https://code.visualstudio.com/docs/remote/containers-advanced#_avoiding-extension-reinstalls-on-container-rebuild
RUN mkdir -p /home/milvus/.vscode-server/extensions \
/home/milvus/.vscode-server-insiders/extensions \
/home/milvus/.vscode-server-insiders/extensions \
&& chmod -R 777 /home/milvus

COPY --chown=0:0 build/docker/builder/entrypoint.sh /
Expand Down
4 changes: 2 additions & 2 deletions build/docker/builder/gpu/ubuntu20.04/Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ ENV GOPATH /go
ENV GOROOT /usr/local/go
ENV GO111MODULE on
ENV PATH $GOPATH/bin:$GOROOT/bin:$PATH
RUN mkdir -p /usr/local/go && wget -qO- "https://go.dev/dl/go1.21.10.linux-$TARGETARCH.tar.gz" | tar --strip-components=1 -xz -C /usr/local/go && \
RUN mkdir -p /usr/local/go && wget -qO- "https://go.dev/dl/go1.21.11.linux-$TARGETARCH.tar.gz" | tar --strip-components=1 -xz -C /usr/local/go && \
mkdir -p "$GOPATH/src" "$GOPATH/bin" && \
curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b ${GOROOT}/bin v1.46.2 && \
# export GO111MODULE=on && go get github.com/quasilyte/go-ruleguard/cmd/[email protected] && \
Expand All @@ -74,7 +74,7 @@ RUN echo 'root:root' | chpasswd

# refer: https://code.visualstudio.com/docs/remote/containers-advanced#_avoiding-extension-reinstalls-on-container-rebuild
RUN mkdir -p /home/milvus/.vscode-server/extensions \
/home/milvus/.vscode-server-insiders/extensions \
/home/milvus/.vscode-server-insiders/extensions \
&& chmod -R 777 /home/milvus

COPY --chown=0:0 build/docker/builder/entrypoint.sh /
Expand Down
4 changes: 2 additions & 2 deletions build/docker/builder/gpu/ubuntu22.04/Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends wget curl ca-ce


# Install go
RUN mkdir -p /usr/local/go && wget -qO- "https://go.dev/dl/go1.21.10.linux-$TARGETARCH.tar.gz" | tar --strip-components=1 -xz -C /usr/local/go
RUN mkdir -p /usr/local/go && wget -qO- "https://go.dev/dl/go1.21.11.linux-$TARGETARCH.tar.gz" | tar --strip-components=1 -xz -C /usr/local/go
# Install conan
RUN pip3 install conan==1.61.0
# Install rust
Expand All @@ -31,7 +31,7 @@ RUN vcpkg install azure-identity-cpp azure-storage-blobs-cpp gtest

# refer: https://code.visualstudio.com/docs/remote/containers-advanced#_avoiding-extension-reinstalls-on-container-rebuild
RUN mkdir -p /home/milvus/.vscode-server/extensions \
/home/milvus/.vscode-server-insiders/extensions \
/home/milvus/.vscode-server-insiders/extensions \
&& chmod -R 777 /home/milvus


Expand Down
2 changes: 1 addition & 1 deletion build/docker/meta-migration/builder/Dockerfile
View file
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
FROM golang:1.21.10-alpine3.19
FROM golang:1.21.11-alpine3.19
RUN apk add --no-cache make bash
115 changes: 0 additions & 115 deletions internal/storage/serde_events_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,7 @@ package storage
import (
"bytes"
"context"
"fmt"
"io"
"math"
"strconv"
"testing"

Expand Down Expand Up @@ -495,116 +493,3 @@ func readDeltaLog(size int, blob *Blob) error {
}
return nil
}

func TestReaderIO(t *testing.T) {
totalMemory := 64 // MB
wFileSize := 1000 // MB
nFileSize := 32 // MB
totalRows := 400000

wRowsPerRowGroup := totalRows / wFileSize
nRowsPerRowGroup := totalRows / nFileSize

wRowGroupSize := 1 // MB
nRowGroupSize := 1 // MB

maxAlignment := func(wRowsInMemory, nRowsInMemory int, memorySize int) (int, int, int) {
dp := make([][]int, memorySize+1)
for i := range dp {
dp[i] = make([]int, memorySize+1)
}
for i := 0; i <= memorySize; i++ {
for j := 0; j <= i; j++ {
a := j / wRowGroupSize //
b := (i - j) / nRowGroupSize
wRows := wRowsInMemory + a*wRowsPerRowGroup
nRows := nRowsInMemory + b*nRowsPerRowGroup
dp[i][j] = min(wRows, nRows)
}
}
maxMinRows := 0
wRowGroupToRead := 0
nRowGroupToRead := 0
for i := 0; i <= memorySize; i++ {
for j := 0; j <= i; j++ {
if dp[i][j] > maxMinRows {
maxMinRows = dp[i][j]
wRowGroupToRead = j
nRowGroupToRead = i - j
}
}
}
return wRowGroupToRead, nRowGroupToRead, maxMinRows
}

wio := 0
nio := 0

wRowsInMemory := 0
nRowsInMemory := 0

memory := totalMemory

for totalRows > 0 {
// memory allocator
wRowGroupToRead, nRowGroupToRead, maxMinRows := maxAlignment(wRowsInMemory, nRowsInMemory, memory)
fmt.Printf("read wide %d row groups , read narrow %d row groups, max rows %d \n", wRowGroupToRead, nRowGroupToRead, maxMinRows)

// IO Merge
if wRowGroupToRead > 0 {
wio += 1
}
if nRowGroupToRead > 0 {
nio += 1
}

// deserailize maxMinRows
totalRows -= maxMinRows
wRowsInMemory += wRowGroupToRead*wRowsPerRowGroup - maxMinRows
nRowsInMemory += nRowGroupToRead*nRowsPerRowGroup - maxMinRows

ceilFloatToInt := func(n float64) int {
return int(math.Ceil(n))
}
memory -= (ceilFloatToInt(float64(wRowGroupSize/wRowsPerRowGroup)) + ceilFloatToInt(float64(nRowGroupSize/nRowsPerRowGroup))) * maxMinRows

fmt.Printf("%d wide rows in memory, %d narrow rows in memory, memory left: %d\n", wRowsInMemory, nRowsInMemory, memory)
}

fmt.Printf("wide column read IO: %d, narrow columns read IO: %d, total read IO: %d under reading memory %d MB", wio, nio, wio+nio, totalMemory)
}

func TestWriterIO(t *testing.T) {
records := 400000
wide_size := 2.5
narrow_size := 0.08
memory := float64(64 * 1024)

wm := float64(0)
nm := float64(0)
wio := 0
nio := 0
for i := 0; i < records; i++ {
wm += wide_size
nm += narrow_size
// 扩展：use max heap to get max memory use of column group
if wm+nm >= memory {
if wm >= nm {
wm = 0
wio += 1
} else {
nm = 0
nio += 1
}
}
}
if wm > 0 {
wio += 1
}
if nm > 0 {
nio += 1
}
fmt.Println(wio)
fmt.Println(nio)
fmt.Println(wio + nio)
}
2 changes: 1 addition & 1 deletion tests/go_client/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ module github.com/milvus-io/milvus/tests/go_client

go 1.21

toolchain go1.21.10
toolchain go1.21.11

require (
github.com/milvus-io/milvus/client/v2 v2.0.0-20240704083609-fcafdb6d5f68
Expand Down

0 comments on commit d46d024

Please sign in to comment.