diff --git a/.circleci/config.yml b/.circleci/config.yml index e9c43ab4f..b2f9dbbf1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -46,6 +46,35 @@ commands: steps: - run: sudo apt-get update - run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev + - when: + condition: <> + steps: + - run: + name: Install Go + command: | + curl https://dl.google.com/go/go1.17.9.darwin-amd64.pkg -o /tmp/go.pkg && \ + sudo installer -pkg /tmp/go.pkg -target / + - run: + name: Export Go + command: | + echo 'export GOPATH="${HOME}/go"' >> $BASH_ENV + - run: go version + - run: + name: Install pkg-config, goreleaser, and sha512sum + command: HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config goreleaser/tap/goreleaser coreutils + - run: + name: Install Rust + command: | + curl https://sh.rustup.rs -sSf | sh -s -- -y + - run: + name: Install hwloc + command: | + mkdir ~/hwloc + curl --location https://download.open-mpi.org/release/hwloc/v2.4/hwloc-2.4.1.tar.gz --output ~/hwloc/hwloc-2.4.1.tar.gz + cd ~/hwloc + tar -xvzpf hwloc-2.4.1.tar.gz + cd hwloc-2.4.1 + ./configure && make && sudo make install - run: git submodule sync - run: git submodule update --init download-params: @@ -77,6 +106,16 @@ commands: tar -xf go-ipfs_v0.12.2_linux-amd64.tar.gz mv go-ipfs/ipfs /usr/local/bin/ipfs chmod +x /usr/local/bin/ipfs + install_ipfs_macos: + steps: + - run: | + curl -O https://dist.ipfs.io/kubo/v0.14.0/kubo_v0.14.0_darwin-amd64.tar.gz + tar -xvzf kubo_v0.14.0_darwin-amd64.tar.gz + pushd kubo + sudo bash install.sh + popd + rm -rf kubo/ + rm kubo_v0.14.0_darwin-amd64.tar.gz git_fetch_all_tags: steps: - run: @@ -335,9 +374,13 @@ jobs: - run: name: "trigger payment channel stress testplan on taas" command: ~/testground-cli run composition -f $HOME/testground/plans/lotus-soup/_compositions/paych-stress-k8s.toml --metadata-commit=$CIRCLE_SHA1 --metadata-repo=filecoin-project/lotus --metadata-branch=$CIRCLE_BRANCH - build-macos: description: build darwin lotus binary + parameters: + publish: + default: false + description: publish github release and homebrew? + type: boolean macos: xcode: "12.5.0" working_directory: ~/go/src/github.com/filecoin-project/lotus @@ -345,48 +388,28 @@ jobs: - prepare: linux: false darwin: true - - run: - name: Install go - command: | - curl -O https://dl.google.com/go/go1.17.9.darwin-amd64.pkg && \ - sudo installer -pkg go1.17.9.darwin-amd64.pkg -target / - - run: - name: Install pkg-config - command: HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config - - run: go version - - run: - name: Install Rust - command: | - curl https://sh.rustup.rs -sSf | sh -s -- -y - - run: - name: Install hwloc - command: | - mkdir ~/hwloc - curl --location https://download.open-mpi.org/release/hwloc/v2.4/hwloc-2.4.1.tar.gz --output ~/hwloc/hwloc-2.4.1.tar.gz - cd ~/hwloc - tar -xvzpf hwloc-2.4.1.tar.gz - cd hwloc-2.4.1 - ./configure && make && sudo make install + - install_ipfs_macos - restore_cache: name: restore cargo cache key: v3-go-deps-{{ arch }}-{{ checksum "~/go/src/github.com/filecoin-project/lotus/go.sum" }} - - run: - command: make build - no_output_timeout: 30m - - run: - name: check tag and version output match - command: ./scripts/version-check.sh ./lotus - - store_artifacts: - path: lotus - - store_artifacts: - path: lotus-miner + - when: + condition: << parameters.publish >> + steps: + - run: goreleaser release --rm-dist + - run: ./scripts/generate-checksums.sh + - run: ./scripts/publish-checksums.sh + - when: + condition: + not: << parameters.publish >> + steps: + - run: goreleaser release --rm-dist --snapshot + - run: ./scripts/generate-checksums.sh - store_artifacts: - path: lotus-worker - - run: mkdir darwin && mv lotus lotus-miner lotus-worker darwin/ + path: dist - persist_to_workspace: root: "." paths: - - darwin + - dist - save_cache: name: save cargo cache key: v3-go-deps-{{ arch }}-{{ checksum "~/go/src/github.com/filecoin-project/lotus/go.sum" }} @@ -523,10 +546,6 @@ jobs: default: false description: publish linux binaries? type: boolean - darwin: - default: false - description: publish darwin binaries? - type: boolean appimage: default: false description: publish appimage binaries? @@ -546,11 +565,6 @@ jobs: steps: - run: ./scripts/build-arch-bundle.sh linux - run: ./scripts/publish-arch-release.sh linux - - when: - condition: << parameters.darwin>> - steps: - - run: ./scripts/build-arch-bundle.sh darwin - - run: ./scripts/publish-arch-release.sh darwin - when: condition: << parameters.appimage >> steps: @@ -976,9 +990,9 @@ workflows: target: "./itests/verifreg_test.go" - test: - name: test-itest-wdpost_check_config - suite: itest-wdpost_check_config - target: "./itests/wdpost_check_config_test.go" + name: test-itest-wdpost_config + suite: itest-wdpost_config + target: "./itests/wdpost_config_test.go" - test: name: test-itest-wdpost_dispute @@ -1057,27 +1071,25 @@ workflows: - /^v\d+\.\d+\.\d+(-rc\d+)?$/ - build-lotus-soup - build-macos: + name: publish-macos + publish: true filters: + branches: + ignore: + - /.*/ tags: only: - /^v\d+\.\d+\.\d+(-rc\d+)?$/ - - build-appimage: + - build-macos: filters: branches: only: - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/ - tags: - only: - - /^v\d+\.\d+\.\d+(-rc\d+)?$/ - - publish: - name: publish-macos - darwin: true - requires: - - build-macos + - build-appimage: filters: branches: - ignore: - - /.*/ + only: + - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/ tags: only: - /^v\d+\.\d+\.\d+(-rc\d+)?$/ @@ -1106,12 +1118,14 @@ workflows: only: - /^v\d+\.\d+\.\d+(-rc\d+)?$/ - build-and-push-image: + name: build-and-push/lotus-all-in-one dockerfile: Dockerfile.lotus path: . repo: lotus-dev tag: '${CIRCLE_SHA1:0:8}' target: lotus-all-in-one - build-and-push-image: + name: build-and-push/lotus-test dockerfile: Dockerfile.lotus path: . repo: lotus-test diff --git a/.circleci/template.yml b/.circleci/template.yml index a7f6590e7..c401f655c 100644 --- a/.circleci/template.yml +++ b/.circleci/template.yml @@ -46,6 +46,35 @@ commands: steps: - run: sudo apt-get update - run: sudo apt-get install ocl-icd-opencl-dev libhwloc-dev + - when: + condition: <> + steps: + - run: + name: Install Go + command: | + curl https://dl.google.com/go/go1.17.9.darwin-amd64.pkg -o /tmp/go.pkg && \ + sudo installer -pkg /tmp/go.pkg -target / + - run: + name: Export Go + command: | + echo 'export GOPATH="${HOME}/go"' >> $BASH_ENV + - run: go version + - run: + name: Install pkg-config, goreleaser, and sha512sum + command: HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config goreleaser/tap/goreleaser coreutils + - run: + name: Install Rust + command: | + curl https://sh.rustup.rs -sSf | sh -s -- -y + - run: + name: Install hwloc + command: | + mkdir ~/hwloc + curl --location https://download.open-mpi.org/release/hwloc/v2.4/hwloc-2.4.1.tar.gz --output ~/hwloc/hwloc-2.4.1.tar.gz + cd ~/hwloc + tar -xvzpf hwloc-2.4.1.tar.gz + cd hwloc-2.4.1 + ./configure && make && sudo make install - run: git submodule sync - run: git submodule update --init download-params: @@ -77,6 +106,16 @@ commands: tar -xf go-ipfs_v0.12.2_linux-amd64.tar.gz mv go-ipfs/ipfs /usr/local/bin/ipfs chmod +x /usr/local/bin/ipfs + install_ipfs_macos: + steps: + - run: | + curl -O https://dist.ipfs.io/kubo/v0.14.0/kubo_v0.14.0_darwin-amd64.tar.gz + tar -xvzf kubo_v0.14.0_darwin-amd64.tar.gz + pushd kubo + sudo bash install.sh + popd + rm -rf kubo/ + rm kubo_v0.14.0_darwin-amd64.tar.gz git_fetch_all_tags: steps: - run: @@ -335,9 +374,13 @@ jobs: - run: name: "trigger payment channel stress testplan on taas" command: ~/testground-cli run composition -f $HOME/testground/plans/lotus-soup/_compositions/paych-stress-k8s.toml --metadata-commit=$CIRCLE_SHA1 --metadata-repo=filecoin-project/lotus --metadata-branch=$CIRCLE_BRANCH - build-macos: description: build darwin lotus binary + parameters: + publish: + default: false + description: publish github release and homebrew? + type: boolean macos: xcode: "12.5.0" working_directory: ~/go/src/github.com/filecoin-project/lotus @@ -345,48 +388,28 @@ jobs: - prepare: linux: false darwin: true - - run: - name: Install go - command: | - curl -O https://dl.google.com/go/go1.17.9.darwin-amd64.pkg && \ - sudo installer -pkg go1.17.9.darwin-amd64.pkg -target / - - run: - name: Install pkg-config - command: HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config - - run: go version - - run: - name: Install Rust - command: | - curl https://sh.rustup.rs -sSf | sh -s -- -y - - run: - name: Install hwloc - command: | - mkdir ~/hwloc - curl --location https://download.open-mpi.org/release/hwloc/v2.4/hwloc-2.4.1.tar.gz --output ~/hwloc/hwloc-2.4.1.tar.gz - cd ~/hwloc - tar -xvzpf hwloc-2.4.1.tar.gz - cd hwloc-2.4.1 - ./configure && make && sudo make install + - install_ipfs_macos - restore_cache: name: restore cargo cache key: v3-go-deps-{{ arch }}-{{ checksum "~/go/src/github.com/filecoin-project/lotus/go.sum" }} - - run: - command: make build - no_output_timeout: 30m - - run: - name: check tag and version output match - command: ./scripts/version-check.sh ./lotus - - store_artifacts: - path: lotus - - store_artifacts: - path: lotus-miner + - when: + condition: << parameters.publish >> + steps: + - run: goreleaser release --rm-dist + - run: ./scripts/generate-checksums.sh + - run: ./scripts/publish-checksums.sh + - when: + condition: + not: << parameters.publish >> + steps: + - run: goreleaser release --rm-dist --snapshot + - run: ./scripts/generate-checksums.sh - store_artifacts: - path: lotus-worker - - run: mkdir darwin && mv lotus lotus-miner lotus-worker darwin/ + path: dist - persist_to_workspace: root: "." paths: - - darwin + - dist - save_cache: name: save cargo cache key: v3-go-deps-{{ arch }}-{{ checksum "~/go/src/github.com/filecoin-project/lotus/go.sum" }} @@ -523,10 +546,6 @@ jobs: default: false description: publish linux binaries? type: boolean - darwin: - default: false - description: publish darwin binaries? - type: boolean appimage: default: false description: publish appimage binaries? @@ -546,11 +565,6 @@ jobs: steps: - run: ./scripts/build-arch-bundle.sh linux - run: ./scripts/publish-arch-release.sh linux - - when: - condition: << parameters.darwin>> - steps: - - run: ./scripts/build-arch-bundle.sh darwin - - run: ./scripts/publish-arch-release.sh darwin - when: condition: << parameters.appimage >> steps: @@ -817,27 +831,25 @@ workflows: - /^v\d+\.\d+\.\d+(-rc\d+)?$/ - build-lotus-soup - build-macos: + name: publish-macos + publish: true filters: + branches: + ignore: + - /.*/ tags: only: - /^v\d+\.\d+\.\d+(-rc\d+)?$/ - - build-appimage: + - build-macos: filters: branches: only: - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/ - tags: - only: - - /^v\d+\.\d+\.\d+(-rc\d+)?$/ - - publish: - name: publish-macos - darwin: true - requires: - - build-macos + - build-appimage: filters: branches: - ignore: - - /.*/ + only: + - /^release\/v\d+\.\d+\.\d+(-rc\d+)?$/ tags: only: - /^v\d+\.\d+\.\d+(-rc\d+)?$/ @@ -866,12 +878,14 @@ workflows: only: - /^v\d+\.\d+\.\d+(-rc\d+)?$/ - build-and-push-image: + name: build-and-push/lotus-all-in-one dockerfile: Dockerfile.lotus path: . repo: lotus-dev tag: '${CIRCLE_SHA1:0:8}' target: lotus-all-in-one - build-and-push-image: + name: build-and-push/lotus-test dockerfile: Dockerfile.lotus path: . repo: lotus-test diff --git a/.gitignore b/.gitignore index 1db5e2858..b111195cf 100644 --- a/.gitignore +++ b/.gitignore @@ -51,3 +51,5 @@ scratchpad build/builtin-actors/v* build/builtin-actors/*.car + +dist/ diff --git a/.goreleaser.yaml b/.goreleaser.yaml new file mode 100644 index 000000000..e027d57b0 --- /dev/null +++ b/.goreleaser.yaml @@ -0,0 +1,166 @@ +project_name: lotus +before: + hooks: + - go mod tidy + - make deps + +universal_binaries: + - id: lotus + replace: true + name_template: lotus + ids: + - lotus_darwin_amd64 + - lotus_darwin_arm64 + - id: lotus-miner + replace: true + name_template: lotus-miner + ids: + - lotus-miner_darwin_amd64 + - lotus-miner_darwin_arm64 + - id: lotus-worker + replace: true + name_template: lotus-worker + ids: + - lotus-worker_darwin_amd64 + - lotus-worker_darwin_arm64 + +builds: + - id: lotus_darwin_amd64 + main: ./cmd/lotus + binary: lotus + goos: + - darwin + goarch: + - amd64 + env: + - CGO_ENABLED=1 + - FFI_BUILD_FROM_SOURCE=1 + ldflags: + - -X=github.com/filecoin-project/lotus/build.CurrentCommit=+git.{{.ShortCommit}} + - id: lotus-miner_darwin_amd64 + main: ./cmd/lotus-miner + binary: lotus-miner + goos: + - darwin + goarch: + - amd64 + env: + - CGO_ENABLED=1 + - FFI_BUILD_FROM_SOURCE=1 + ldflags: + - -X=github.com/filecoin-project/lotus/build.CurrentCommit=+git.{{.ShortCommit}} + - id: lotus-worker_darwin_amd64 + main: ./cmd/lotus-worker + binary: lotus-worker + goos: + - darwin + goarch: + - amd64 + env: + - CGO_ENABLED=1 + - FFI_BUILD_FROM_SOURCE=1 + ldflags: + - -X=github.com/filecoin-project/lotus/build.CurrentCommit=+git.{{.ShortCommit}} + - id: lotus_darwin_arm64 + main: ./cmd/lotus + binary: lotus + goos: + - darwin + goarch: + - arm64 + env: + - CGO_ENABLED=1 + - FFI_BUILD_FROM_SOURCE=1 + - CPATH=/opt/homebrew/include + - LIBRARY_PATH=/opt/homebrew/lib + ldflags: + - -X=github.com/filecoin-project/lotus/build.CurrentCommit=+git.{{.ShortCommit}} + - id: lotus-miner_darwin_arm64 + main: ./cmd/lotus-miner + binary: lotus-miner + goos: + - darwin + goarch: + - arm64 + env: + - CGO_ENABLED=1 + - FFI_BUILD_FROM_SOURCE=1 + - CPATH=/opt/homebrew/include + - LIBRARY_PATH=/opt/homebrew/lib + ldflags: + - -X=github.com/filecoin-project/lotus/build.CurrentCommit=+git.{{.ShortCommit}} + - id: lotus-worker_darwin_arm64 + main: ./cmd/lotus-worker + binary: lotus-worker + goos: + - darwin + goarch: + - arm64 + env: + - CGO_ENABLED=1 + - FFI_BUILD_FROM_SOURCE=1 + - CPATH=/opt/homebrew/include + - LIBRARY_PATH=/opt/homebrew/lib + ldflags: + - -X=github.com/filecoin-project/lotus/build.CurrentCommit=+git.{{.ShortCommit}} +# - id: linux +# main: ./cmd/lotus +# binary: lotus +# goos: +# - linux +# goarch: +# - amd64 +# env: +# - CGO_ENABLED=1 +# ldflags: +# - -X=github.com/filecoin-project/lotus/build.CurrentCommit=+git.{{.ShortCommit}} + +archives: + - id: primary + format: tar.gz + wrap_in_directory: true + files: + # this is a dumb but required hack so it doesn't include the default files + # https://github.com/goreleaser/goreleaser/issues/602 + - _n_o_n_e_* + +release: + github: + owner: filecoin-project + name: lotus + prerelease: auto + mode: append + name_template: "Release v{{.Version}}" + + +brews: + - tap: + owner: filecoin-project + name: homebrew-lotus + branch: master + ids: + - primary + install: | + bin.install "lotus" + bin.install "lotus-miner" + bin.install "lotus-worker" + test: | + system "#{bin}/lotus --version" + system "#{bin}/lotus-miner --version" + system "#{bin}/lotus-worker --version" + folder: Formula + homepage: "https://filecoin.io" + description: "A homebrew cask for installing filecoin-project/lotus on MacOS" + license: MIT + dependencies: + - name: pkg-config + - name: jq + - name: bzr + - name: hwloc + +# produced manually so we can include cid checksums +checksum: + disable: true + +snapshot: + name_template: "{{ .Tag }}" diff --git a/Makefile b/Makefile index 21a278c80..c3788d5a9 100644 --- a/Makefile +++ b/Makefile @@ -302,11 +302,14 @@ type-gen: api-gen method-gen: api-gen (cd ./lotuspond/front/src/chain && $(GOCC) run ./methodgen.go) -actors-gen: +actors-code-gen: $(GOCC) run ./gen/inline-gen . gen/inlinegen-data.json $(GOCC) run ./chain/actors/agen $(GOCC) fmt ./... +actors-gen: actors-code-gen fiximports +.PHONY: actors-gen + bundle-gen: $(GOCC) run ./gen/bundle $(GOCC) fmt ./build/... @@ -364,7 +367,7 @@ docsgen-openrpc-gateway: docsgen-openrpc-bin fiximports: ./scripts/fiximports -gen: actors-gen type-gen method-gen cfgdoc-gen docsgen api-gen circleci bundle-gen fiximports +gen: actors-code-gen type-gen method-gen cfgdoc-gen docsgen api-gen circleci bundle-gen fiximports @echo ">>> IF YOU'VE MODIFIED THE CLI OR CONFIG, REMEMBER TO ALSO MAKE docsgen-cli" .PHONY: gen diff --git a/api/api_storage.go b/api/api_storage.go index e9135e30d..4379031b0 100644 --- a/api/api_storage.go +++ b/api/api_storage.go @@ -48,6 +48,11 @@ type StorageMiner interface { ActorSectorSize(context.Context, address.Address) (abi.SectorSize, error) //perm:read ActorAddressConfig(ctx context.Context) (AddressConfig, error) //perm:read + // WithdrawBalance allows to withdraw balance from miner actor to owner address + // Specify amount as "0" to withdraw full balance. This method returns a message CID + // and does not wait for message execution + ActorWithdrawBalance(ctx context.Context, amount abi.TokenAmount) (cid.Cid, error) //perm:admin + MiningBase(context.Context) (*types.TipSet, error) //perm:read ComputeWindowPoSt(ctx context.Context, dlIdx uint64, tsk types.TipSetKey) ([]miner.SubmitWindowedPoStParams, error) //perm:admin diff --git a/api/proxy_gen.go b/api/proxy_gen.go index 2c80a6315..f1e520b7e 100644 --- a/api/proxy_gen.go +++ b/api/proxy_gen.go @@ -656,6 +656,8 @@ type StorageMinerStruct struct { ActorSectorSize func(p0 context.Context, p1 address.Address) (abi.SectorSize, error) `perm:"read"` + ActorWithdrawBalance func(p0 context.Context, p1 abi.TokenAmount) (cid.Cid, error) `perm:"admin"` + CheckProvable func(p0 context.Context, p1 abi.RegisteredPoStProof, p2 []storiface.SectorRef, p3 bool) (map[abi.SectorNumber]string, error) `perm:"admin"` ComputeDataCid func(p0 context.Context, p1 abi.UnpaddedPieceSize, p2 storiface.Data) (abi.PieceInfo, error) `perm:"admin"` @@ -3956,6 +3958,17 @@ func (s *StorageMinerStub) ActorSectorSize(p0 context.Context, p1 address.Addres return *new(abi.SectorSize), ErrNotSupported } +func (s *StorageMinerStruct) ActorWithdrawBalance(p0 context.Context, p1 abi.TokenAmount) (cid.Cid, error) { + if s.Internal.ActorWithdrawBalance == nil { + return *new(cid.Cid), ErrNotSupported + } + return s.Internal.ActorWithdrawBalance(p0, p1) +} + +func (s *StorageMinerStub) ActorWithdrawBalance(p0 context.Context, p1 abi.TokenAmount) (cid.Cid, error) { + return *new(cid.Cid), ErrNotSupported +} + func (s *StorageMinerStruct) CheckProvable(p0 context.Context, p1 abi.RegisteredPoStProof, p2 []storiface.SectorRef, p3 bool) (map[abi.SectorNumber]string, error) { if s.Internal.CheckProvable == nil { return *new(map[abi.SectorNumber]string), ErrNotSupported diff --git a/build/openrpc/full.json.gz b/build/openrpc/full.json.gz index 397c8a4dd..8554e0a01 100644 Binary files a/build/openrpc/full.json.gz and b/build/openrpc/full.json.gz differ diff --git a/build/openrpc/gateway.json.gz b/build/openrpc/gateway.json.gz index 3e1b629d9..c8489039b 100644 Binary files a/build/openrpc/gateway.json.gz and b/build/openrpc/gateway.json.gz differ diff --git a/build/openrpc/miner.json.gz b/build/openrpc/miner.json.gz index ecdae43d9..f1f95854c 100644 Binary files a/build/openrpc/miner.json.gz and b/build/openrpc/miner.json.gz differ diff --git a/build/openrpc/worker.json.gz b/build/openrpc/worker.json.gz index 87f6da985..c2b5bdedd 100644 Binary files a/build/openrpc/worker.json.gz and b/build/openrpc/worker.json.gz differ diff --git a/chain/types/message.go b/chain/types/message.go index 5b2aad47e..547d4c353 100644 --- a/chain/types/message.go +++ b/chain/types/message.go @@ -196,7 +196,7 @@ func (m *Message) ValidForBlockInclusion(minGas int64, version network.Version) } if m.GasLimit > build.BlockGasLimit { - return xerrors.New("'GasLimit' field cannot be greater than a block's gas limit") + return xerrors.Errorf("'GasLimit' field cannot be greater than a block's gas limit (%d > %d)", m.GasLimit, build.BlockGasLimit) } // since prices might vary with time, this is technically semantic validation diff --git a/cmd/lotus-miner/actor.go b/cmd/lotus-miner/actor.go index 945f6e185..3707f3f5b 100644 --- a/cmd/lotus-miner/actor.go +++ b/cmd/lotus-miner/actor.go @@ -218,36 +218,8 @@ var actorWithdrawCmd = &cli.Command{ }, }, Action: func(cctx *cli.Context) error { - nodeApi, closer, err := lcli.GetStorageMinerAPI(cctx) - if err != nil { - return err - } - defer closer() - - api, acloser, err := lcli.GetFullNodeAPI(cctx) - if err != nil { - return err - } - defer acloser() - - ctx := lcli.ReqContext(cctx) - - maddr, err := nodeApi.ActorAddress(ctx) - if err != nil { - return err - } - - mi, err := api.StateMinerInfo(ctx, maddr, types.EmptyTSK) - if err != nil { - return err - } + amount := abi.NewTokenAmount(0) - available, err := api.StateMinerAvailableBalance(ctx, maddr, types.EmptyTSK) - if err != nil { - return err - } - - amount := available if cctx.Args().Present() { f, err := types.ParseFIL(cctx.Args().First()) if err != nil { @@ -255,44 +227,35 @@ var actorWithdrawCmd = &cli.Command{ } amount = abi.TokenAmount(f) - - if amount.GreaterThan(available) { - return xerrors.Errorf("can't withdraw more funds than available; requested: %s; available: %s", types.FIL(amount), types.FIL(available)) - } } - params, err := actors.SerializeParams(&miner.WithdrawBalanceParams{ - AmountRequested: amount, // Default to attempting to withdraw all the extra funds in the miner actor - }) + nodeApi, closer, err := lcli.GetStorageMinerAPI(cctx) if err != nil { return err } + defer closer() - smsg, err := api.MpoolPushMessage(ctx, &types.Message{ - To: maddr, - From: mi.Owner, - Value: types.NewInt(0), - Method: builtin.MethodsMiner.WithdrawBalance, - Params: params, - }, nil) + api, acloser, err := lcli.GetFullNodeAPI(cctx) if err != nil { return err } + defer acloser() - fmt.Printf("Requested rewards withdrawal in message %s\n", smsg.Cid()) - - // wait for it to get mined into a block - fmt.Printf("waiting for %d epochs for confirmation..\n", uint64(cctx.Int("confidence"))) + ctx := lcli.ReqContext(cctx) - wait, err := api.StateWaitMsg(ctx, smsg.Cid(), uint64(cctx.Int("confidence"))) + res, err := nodeApi.ActorWithdrawBalance(ctx, amount) if err != nil { return err } - // check it executed successfully + // wait for it to get mined into a block + wait, err := api.StateWaitMsg(ctx, res, uint64(cctx.Int("confidence"))) + if err != nil { + return xerrors.Errorf("Timeout waiting for withdrawal message %s", wait.Message) + } + if wait.Receipt.ExitCode != 0 { - fmt.Println(cctx.App.Writer, "withdrawal failed!") - return err + return xerrors.Errorf("Failed to execute withdrawal message %s: %w", wait.Message, wait.Receipt.ExitCode.Error()) } nv, err := api.StateNetworkVersion(ctx, wait.TipSet) diff --git a/documentation/en/api-v0-methods-miner.md b/documentation/en/api-v0-methods-miner.md index 2c781f5b0..e24520b20 100644 --- a/documentation/en/api-v0-methods-miner.md +++ b/documentation/en/api-v0-methods-miner.md @@ -9,6 +9,7 @@ * [ActorAddress](#ActorAddress) * [ActorAddressConfig](#ActorAddressConfig) * [ActorSectorSize](#ActorSectorSize) + * [ActorWithdrawBalance](#ActorWithdrawBalance) * [Auth](#Auth) * [AuthNew](#AuthNew) * [AuthVerify](#AuthVerify) @@ -292,6 +293,28 @@ Inputs: Response: `34359738368` +### ActorWithdrawBalance +WithdrawBalance allows to withdraw balance from miner actor to owner address +Specify amount as "0" to withdraw full balance. This method returns a message CID +and does not wait for message execution + + +Perms: admin + +Inputs: +```json +[ + "0" +] +``` + +Response: +```json +{ + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" +} +``` + ## Auth diff --git a/itests/wdpost_check_config_test.go b/itests/wdpost_config_test.go similarity index 61% rename from itests/wdpost_check_config_test.go rename to itests/wdpost_config_test.go index cc10af04f..61b64387d 100644 --- a/itests/wdpost_check_config_test.go +++ b/itests/wdpost_config_test.go @@ -186,3 +186,117 @@ func TestWindowPostNoPreChecks(t *testing.T) { sectors = p.MinerPower.RawBytePower.Uint64() / uint64(ssz) require.Equal(t, nSectors+kit.DefaultPresealsPerBootstrapMiner-2+1, int(sectors)) // -2 not recovered sectors + 1 just pledged } + +func TestWindowPostMaxSectorsRecoveryConfig(t *testing.T) { + oldVal := wdpost.RecoveringSectorLimit + defer func() { + wdpost.RecoveringSectorLimit = oldVal + }() + wdpost.RecoveringSectorLimit = 1 + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + client, miner, ens := kit.EnsembleMinimal(t, + kit.LatestActorsAt(-1), + kit.MockProofs()) + ens.InterconnectAll().BeginMining(2 * time.Millisecond) + + nSectors := 10 + + miner.PledgeSectors(ctx, nSectors, 0, nil) + + maddr, err := miner.ActorAddress(ctx) + require.NoError(t, err) + di, err := client.StateMinerProvingDeadline(ctx, maddr, types.EmptyTSK) + require.NoError(t, err) + + mid, err := address.IDFromAddress(maddr) + require.NoError(t, err) + + t.Log("Running one proving period") + waitUntil := di.Open + di.WPoStProvingPeriod + t.Logf("End for head.Height > %d", waitUntil) + + ts := client.WaitTillChain(ctx, kit.HeightAtLeast(waitUntil)) + t.Logf("Now head.Height = %d", ts.Height()) + + p, err := client.StateMinerPower(ctx, maddr, types.EmptyTSK) + require.NoError(t, err) + + ssz, err := miner.ActorSectorSize(ctx, maddr) + require.NoError(t, err) + + require.Equal(t, p.MinerPower, p.TotalPower) + require.Equal(t, p.MinerPower.RawBytePower, types.NewInt(uint64(ssz)*uint64(nSectors+kit.DefaultPresealsPerBootstrapMiner))) + + t.Log("Drop some sectors") + + // Drop 2 sectors from deadline 2 partition 0 (full partition / deadline) + parts, err := client.StateMinerPartitions(ctx, maddr, 2, types.EmptyTSK) + require.NoError(t, err) + require.Greater(t, len(parts), 0) + + secs := parts[0].AllSectors + n, err := secs.Count() + require.NoError(t, err) + require.Equal(t, uint64(2), n) + + // Drop the partition + err = secs.ForEach(func(sid uint64) error { + return miner.StorageMiner.(*impl.StorageMinerAPI).IStorageMgr.(*mock.SectorMgr).MarkFailed(storiface.SectorRef{ + ID: abi.SectorID{ + Miner: abi.ActorID(mid), + Number: abi.SectorNumber(sid), + }, + }, true) + }) + require.NoError(t, err) + + di, err = client.StateMinerProvingDeadline(ctx, maddr, types.EmptyTSK) + require.NoError(t, err) + + t.Log("Go through another PP, wait for sectors to become faulty") + waitUntil = di.Open + di.WPoStProvingPeriod + t.Logf("End for head.Height > %d", waitUntil) + + ts = client.WaitTillChain(ctx, kit.HeightAtLeast(waitUntil)) + t.Logf("Now head.Height = %d", ts.Height()) + + p, err = client.StateMinerPower(ctx, maddr, types.EmptyTSK) + require.NoError(t, err) + + require.Equal(t, p.MinerPower, p.TotalPower) + + sectors := p.MinerPower.RawBytePower.Uint64() / uint64(ssz) + require.Equal(t, nSectors+kit.DefaultPresealsPerBootstrapMiner-2, int(sectors)) // -2 just removed sectors + + t.Log("Make the sectors recoverable") + + err = secs.ForEach(func(sid uint64) error { + return miner.StorageMiner.(*impl.StorageMinerAPI).IStorageMgr.(*mock.SectorMgr).MarkFailed(storiface.SectorRef{ + ID: abi.SectorID{ + Miner: abi.ActorID(mid), + Number: abi.SectorNumber(sid), + }, + }, false) + }) + require.NoError(t, err) + + di, err = client.StateMinerProvingDeadline(ctx, maddr, types.EmptyTSK) + require.NoError(t, err) + + waitUntil = di.Open + di.WPoStProvingPeriod + 200 + t.Logf("End for head.Height > %d", waitUntil) + + ts = client.WaitTillChain(ctx, kit.HeightAtLeast(waitUntil)) + t.Logf("Now head.Height = %d", ts.Height()) + + p, err = client.StateMinerPower(ctx, maddr, types.EmptyTSK) + require.NoError(t, err) + + require.Equal(t, p.MinerPower, p.TotalPower) + + sectors = p.MinerPower.RawBytePower.Uint64() / uint64(ssz) + require.Equal(t, nSectors+kit.DefaultPresealsPerBootstrapMiner-1, int(sectors)) // -1 not recovered sector +} diff --git a/node/impl/storminer.go b/node/impl/storminer.go index 3b4e1e23e..74b1dc776 100644 --- a/node/impl/storminer.go +++ b/node/impl/storminer.go @@ -32,12 +32,15 @@ import ( filmktsstore "github.com/filecoin-project/go-fil-markets/stores" "github.com/filecoin-project/go-jsonrpc/auth" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/big" + builtintypes "github.com/filecoin-project/go-state-types/builtin" minertypes "github.com/filecoin-project/go-state-types/builtin/v8/miner" "github.com/filecoin-project/go-state-types/network" "github.com/filecoin-project/lotus/api" apitypes "github.com/filecoin-project/lotus/api/types" "github.com/filecoin-project/lotus/build" + "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/actors/builtin" "github.com/filecoin-project/lotus/chain/gen" "github.com/filecoin-project/lotus/chain/types" @@ -1279,3 +1282,43 @@ func (sm *StorageMinerAPI) ComputeProof(ctx context.Context, ssi []builtin.Exten func (sm *StorageMinerAPI) RuntimeSubsystems(context.Context) (res api.MinerSubsystems, err error) { return sm.EnabledSubsystems, nil } + +func (sm *StorageMinerAPI) ActorWithdrawBalance(ctx context.Context, amount abi.TokenAmount) (cid.Cid, error) { + available, err := sm.Full.StateMinerAvailableBalance(ctx, sm.Miner.Address(), types.EmptyTSK) + if err != nil { + return cid.Undef, xerrors.Errorf("Error getting miner balance: %w", err) + } + + if amount.GreaterThan(available) { + return cid.Undef, xerrors.Errorf("can't withdraw more funds than available; requested: %s; available: %s", types.FIL(amount), types.FIL(available)) + } + + if amount.Equals(big.Zero()) { + amount = available + } + + params, err := actors.SerializeParams(&minertypes.WithdrawBalanceParams{ + AmountRequested: amount, + }) + if err != nil { + return cid.Undef, err + } + + mi, err := sm.Full.StateMinerInfo(ctx, sm.Miner.Address(), types.EmptyTSK) + if err != nil { + return cid.Undef, xerrors.Errorf("Error getting miner's owner address: %w", err) + } + + smsg, err := sm.Full.MpoolPushMessage(ctx, &types.Message{ + To: sm.Miner.Address(), + From: mi.Owner, + Value: types.NewInt(0), + Method: builtintypes.MethodsMiner.WithdrawBalance, + Params: params, + }, nil) + if err != nil { + return cid.Undef, err + } + + return smsg.Cid(), nil +} diff --git a/scripts/generate-checksums.sh b/scripts/generate-checksums.sh new file mode 100755 index 000000000..ae0920138 --- /dev/null +++ b/scripts/generate-checksums.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +set -exo + +REQUIRED=( + "sha512sum" + "ipfs" +) +for REQUIRE in "${REQUIRED[@]}" +do + command -v "${REQUIRE}" >/dev/null 2>&1 || echo >&2 "'${REQUIRE}' must be installed" +done + +# start ipfs +export IPFS_PATH=`mktemp -d` +ipfs init +ipfs daemon & +PID="$!" +trap "kill -9 ${PID}" EXIT + +# generate checksums +for FILE in dist/*.tar.gz +do + sha512sum "${FILE}" > "${FILE}.sha512" + until ipfs add -q "${FILE}" > "${FILE}.cid" + do + echo "Waiting for ipfs daemon to start..." + sleep 2 + done +done diff --git a/scripts/publish-checksums.sh b/scripts/publish-checksums.sh new file mode 100755 index 000000000..346ca9b4c --- /dev/null +++ b/scripts/publish-checksums.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +set -exo + +pushd dist + +# make sure we have a token set, api requests won't work otherwise +if [ -z "${GITHUB_TOKEN}" ]; then + echo "\${GITHUB_TOKEN} not set, publish failed" + exit 1 +fi + +REQUIRED=( + "jq" + "curl" +) +for REQUIRE in "${REQUIRED[@]}" +do + command -v "${REQUIRE}" >/dev/null 2>&1 || echo >&2 "'${REQUIRE}' must be installed" +done + +#see if the release already exists by tag +RELEASE_RESPONSE=` + curl \ + --fail \ + --header "Authorization: token ${GITHUB_TOKEN}" \ + "https://api.github.com/repos/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}/releases/tags/${CIRCLE_TAG}" +` +RELEASE_ID=`echo "${RELEASE_RESPONSE}" | jq '.id'` + +if [ "${RELEASE_ID}" = "null" ]; then + echo "creating release" + + COND_CREATE_DISCUSSION="" + PRERELEASE=true + if [[ ${CIRCLE_TAG} =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + COND_CREATE_DISCUSSION="\"discussion_category_name\": \"announcement\"," + PRERELEASE=false + fi + + RELEASE_DATA="{ + \"tag_name\": \"${CIRCLE_TAG}\", + \"target_commitish\": \"${CIRCLE_SHA1}\", + ${COND_CREATE_DISCUSSION} + \"name\": \"${CIRCLE_TAG}\", + \"body\": \"\", + \"prerelease\": ${PRERELEASE} + }" + + # create it if it doesn't exist yet + RELEASE_RESPONSE=` + curl \ + --fail \ + --request POST \ + --header "Authorization: token ${GITHUB_TOKEN}" \ + --header "Content-Type: application/json" \ + --data "${RELEASE_DATA}" \ + "https://api.github.com/repos/$CIRCLE_PROJECT_USERNAME/${CIRCLE_PROJECT_REPONAME}/releases" + ` +else + echo "release already exists" +fi + +RELEASE_UPLOAD_URL=`echo "${RELEASE_RESPONSE}" | jq -r '.upload_url' | cut -d'{' -f1` +echo "Preparing to send artifacts to ${RELEASE_UPLOAD_URL}" + +for CHECKSUM_FILE in *.{cid,sha512} +do + echo "Uploading ${CHECKSUM_FILE}..." + curl \ + --fail \ + --request POST \ + --header "Authorization: token ${GITHUB_TOKEN}" \ + --header "Content-Type: application/octet-stream" \ + --data-binary "@${CHECKSUM_FILE}" \ + "$RELEASE_UPLOAD_URL?name=$(basename "${CHECKSUM_FILE}")" + + echo "Uploaded ${CHECKSUM_FILE}" +done + +popd diff --git a/storage/wdpost/wdpost_run_faults.go b/storage/wdpost/wdpost_run_faults.go index 60352e8b8..22186b551 100644 --- a/storage/wdpost/wdpost_run_faults.go +++ b/storage/wdpost/wdpost_run_faults.go @@ -2,6 +2,9 @@ package wdpost import ( "context" + "math" + "os" + "strconv" "github.com/ipfs/go-cid" "go.opencensus.io/trace" @@ -19,6 +22,18 @@ import ( "github.com/filecoin-project/lotus/chain/types" ) +var RecoveringSectorLimit uint64 = 0 + +func init() { + if rcl := os.Getenv("LOTUS_RECOVERING_SECTOR_LIMIT"); rcl != "" { + var err error + RecoveringSectorLimit, err = strconv.ParseUint(rcl, 10, 64) + if err != nil { + log.Errorw("parsing LOTUS_RECOVERING_SECTOR_LIMIT", "error", err) + } + } +} + // declareRecoveries identifies sectors that were previously marked as faulty // for our miner, but are now recovered (i.e. are now provable again) and // still not reported as such. @@ -44,7 +59,7 @@ func (s *WindowPoStScheduler) declareRecoveries(ctx context.Context, dlIdx uint6 var batchedRecoveryDecls [][]miner.RecoveryDeclaration batchedRecoveryDecls = append(batchedRecoveryDecls, []miner.RecoveryDeclaration{}) - totalRecoveries := 0 + totalSectorsToRecover := uint64(0) for partIdx, partition := range partitions { unrecovered, err := bitfield.SubtractBitField(partition.FaultySectors, partition.RecoveringSectors) @@ -78,6 +93,31 @@ func (s *WindowPoStScheduler) declareRecoveries(ctx context.Context, dlIdx uint6 continue } + // rules to follow if we have indicated that we don't want to recover more than X sectors in a deadline + if RecoveringSectorLimit > 0 { + // something weird happened, break because we can't recover any more + if RecoveringSectorLimit < totalSectorsToRecover { + log.Warnf("accepted more recoveries (%d) than RecoveringSectorLimit (%d)", totalSectorsToRecover, RecoveringSectorLimit) + break + } + + maxNewRecoverable := RecoveringSectorLimit - totalSectorsToRecover + + // we need to trim the recover bitfield + if recoveredCount > maxNewRecoverable { + recoverySlice, err := recovered.All(math.MaxUint64) + if err != nil { + log.Errorw("failed to slice recovery bitfield, breaking out of recovery loop", err) + break + } + + log.Warnf("only adding %d sectors to respect RecoveringSectorLimit %d", maxNewRecoverable, RecoveringSectorLimit) + + recovered = bitfield.NewFromSet(recoverySlice[:maxNewRecoverable]) + recoveredCount = maxNewRecoverable + } + } + // respect user config if set if s.maxPartitionsPerRecoveryMessage > 0 && len(batchedRecoveryDecls[len(batchedRecoveryDecls)-1]) >= s.maxPartitionsPerRecoveryMessage { @@ -90,10 +130,17 @@ func (s *WindowPoStScheduler) declareRecoveries(ctx context.Context, dlIdx uint6 Sectors: recovered, }) - totalRecoveries++ + totalSectorsToRecover += recoveredCount + + if RecoveringSectorLimit > 0 && totalSectorsToRecover >= RecoveringSectorLimit { + log.Errorf("reached recovering sector limit %d, only marking %d sectors for recovery now", + RecoveringSectorLimit, + totalSectorsToRecover) + break + } } - if totalRecoveries == 0 { + if totalSectorsToRecover == 0 { if faulty != 0 { log.Warnw("No recoveries to declare", "deadline", dlIdx, "faulty", faulty) } @@ -101,6 +148,7 @@ func (s *WindowPoStScheduler) declareRecoveries(ctx context.Context, dlIdx uint6 return nil, nil, nil } + log.Infof("attempting recovery declarations for %d sectors", totalSectorsToRecover) var msgs []*types.SignedMessage for _, recovery := range batchedRecoveryDecls { params := &miner.DeclareFaultsRecoveredParams{ @@ -122,7 +170,6 @@ func (s *WindowPoStScheduler) declareRecoveries(ctx context.Context, dlIdx uint6 if err := s.prepareMessage(ctx, msg, spec); err != nil { return nil, nil, err } - sm, err := s.api.MpoolPushMessage(ctx, msg, &api.MessageSendSpec{MaxFee: abi.TokenAmount(s.feeCfg.MaxWindowPoStGasFee)}) if err != nil { return nil, nil, xerrors.Errorf("pushing message to mpool: %w", err)