diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..6ada5fe8e --- /dev/null +++ b/.dockerignore @@ -0,0 +1,10 @@ +docker/devnet/data +react/node_modules +# filecoin-ffi installation +build/.filecoin-install +extern/filecoin-ffi/.install-filcrypto +extern/filecoin-ffi/filcrypto.* +extern/filecoin-ffi/libfilcrypto.a +# local binaries +boost* +devnet diff --git a/README.md b/README.md index 06e6460a3..1897ecd14 100644 --- a/README.md +++ b/README.md @@ -4,79 +4,290 @@ Boost is a tool for Filecoin storage providers to manage data storage and retrie See the docs at [https://boost.filecoin.io](https://boost.filecoin.io/getting-started) to get started. -## For development: +## Table of Contents +- [Building and Installing Boost](#building-and-installing-boost) +- [Running Boost for development](#running-boost-for-development) +- [Running Boost devnet in Docker](#running-boost-devnet-in-docker) +- [License](#license) -1. Install using instructions in the building and installation section in [the docs](https://boost.filecoin.io/getting-started#building-and-installing). +## Building and Installing Boost -2. Make sure you have a local Lotus fullnode and miner running and listening to `localhost:1234` and `localhost:2345` respectively, for example with a devnet: +Compile and install using the instructions at the `Building and installing` section in [the docs](https://boost.filecoin.io/getting-started#building-and-installing). +## Running Boost for development + +To run Boost on your development machine, you will need to set up a devnet: + +1. Remove any existing Lotus and Boost repositories ``` -devnet +rm -rf ~/.lotusmarkets ~/.lotus ~/.lotusminer ~/.genesis_sectors +rm -rf ~/.boost ``` -Note that currently `devnet` is using the default paths that `lotus` and `lotus-miner` use for their repositories, and you should make sure these directories are empty: +2. Build Lotus in debug mode +The version of lotus needs to match the version in Boost's go.mod ``` -LOTUS_PATH=~/.lotus -LOTUS_MINER_PATH=~/.lotusminer +cd lotus +git checkout +make debug +``` + +3. Install Lotus -rm -rf ~/.lotus ~/.lotusminer +The devnet script uses the installed `lotus` and `lotus-miner` binaries to run the miner and daemon. +``` +make install +install -C ./lotus-seed /usr/local/bin/lotus-seed ``` +4. Build Boost in debug mode -3. Create Boost repository +Double check if environment variables are set: +``` +export LIBRARY_PATH=/opt/homebrew/lib +export PATH="$(brew --prefix coreutils)/libexec/gnubin:/usr/local/bin:$PATH" +``` +Build and install ``` -export $(lotus auth api-info --perm=admin) -export $(lotus-miner auth api-info --perm=admin) +cd boost +make debug +make install +``` + +5. Start the devnet -boostd --vv init \ - --api-sealer=`lotus-miner auth api-info --perm=admin` \ - --api-sector-index=`lotus-miner auth api-info --perm=admin` \ - --wallet-publish-storage-deals=`lotus wallet new bls` \ - --wallet-deal-collateral=`lotus wallet new bls` \ - --max-staging-deals-bytes=50000000000 +The following command will use the binaries that you built and installed above, and will run `lotus`, `lotus-miner` and `lotus-seed`. The `lotus` version must match the version in Boost's go.mod. +``` +cd boost +./devnet ``` -4. Run the Boost daemon service +The first time you run it, it will download the Filecoin proof parameters. It will take at least 10 minutes depending on your connection speed. You may need to restart the command multiple times as your terminal will probably timeout before it finishes downloading everything. +The devnet isn't designed to be restartable. After it has been successfully run once, you'll have to clear out the previous data before re-running `./devnet`: +``` +rm -rf ~/.lotusmarkets && rm -rf ~/.lotus && rm -rf ~/.lotusminer && rm -rf ~/.genesis_sectors ``` -export $(lotus auth api-info --perm=admin) -boostd --vv run +6. Wait for `lotus-miner` to come up (through the command above) + +Unset these variables as they interfere with the `lotus-miner` command. +``` +unset MINER_API_INFO +unset FULLNODE_API_INFO ``` -5. Interact with Boost +Then repeatedly run this command until it succeeds: +``` +lotus-miner auth api-info --perm=admin +``` -Pass the client address (wallet) and the provider address to the `dummydeal` command. -Note that -- the client address is the address of a wallet with funds in `lotus wallet list` -- you can find the provider address in `~/.boost/config.toml` under the config key `Wallets.Miner` +7. Get the authentication tokens to connect to the lotus daemon and miner: ``` -boostd dummydeal +export ENV_MINER_API_INFO=`lotus-miner auth api-info --perm=admin` +export ENV_FULLNODE_API_INFO=`lotus auth api-info --perm=admin` + +export MINER_API_INFO=`echo $ENV_MINER_API_INFO | awk '{split($0,a,"="); print a[2]}'` +export FULLNODE_API_INFO=`echo $ENV_FULLNODE_API_INFO | awk '{split($0,a,"="); print a[2]}'` + +echo MINER_API_INFO=$MINER_API_INFO +echo FULLNODE_API_INFO=$FULLNODE_API_INFO ``` -## Running the UI in Development Mode: +8. Create the wallets needed for Boost -1. Run the server +``` +export DEFAULT_WALLET=`lotus wallet list | tail -1 | awk '{print $1}'` +export COLLAT_WALLET=`lotus wallet new bls` +export PUBMSG_WALLET=`lotus wallet new bls` +export CLIENT_WALLET=`lotus wallet new bls` +``` + +9. Add funds to the wallets ``` -cd react -npm install -npm start +lotus send --from $DEFAULT_WALLET $COLLAT_WALLET 10 +lotus send --from $DEFAULT_WALLET $PUBMSG_WALLET 10 +lotus send --from $DEFAULT_WALLET $CLIENT_WALLET 10 ``` -2. Open UI +Run this command repeatedly until each wallet you created has 10 FIL: +``` +lotus wallet list +``` + +This should take about 10 seconds. + +10. Set the Publish Message Wallet as a control address on the miner ``` -http://localhost:3000 +lotus-miner actor control set --really-do-it $PUBMSG_WALLET ``` -## Running a devnet: +11. Add funds into the Market Actor escrow for the client and Collateral wallets + +``` +lotus wallet market add --from $DEFAULT_WALLET --address $CLIENT_WALLET 5 +lotus wallet market add --address $COLLAT_WALLET 5 +``` -Follow the instructions in the [devnet guide](./documentation/devnet.md) +12. Initialize Boost / Create Boost repository + +``` +boostd -vv init \ + --api-sealer=$MINER_API_INFO \ + --api-sector-index=$MINER_API_INFO \ + --wallet-publish-storage-deals=$PUBMSG_WALLET \ + --wallet-deal-collateral=$COLLAT_WALLET \ + --max-staging-deals-bytes=2000000000 +``` + +13. Build the Web UI +``` +make react +``` + +14. Edit config to set a fixed listen address + +Edit `~/.boost/config.toml` + +Set the port in the `ListenAddresses` key to `50000` +``` +[Libp2p] + ListenAddresses = ["/ip4/0.0.0.0/tcp/50000", "/ip6/::/tcp/0"] +``` + +15. Run Boost +``` +boostd -vv run +``` + +Note the peer ID of the boost instance: +``` +2022-06-10T09:32:28.819Z INFO boostd boostd/run.go:114 Boost libp2p node listening {"maddr": "{12D3KooWQNNWNiJ1mieEk9EHjDVF2qBc1FSjJGEzwjnMJzteApaW: [/ip4/172.17.0.2/tcp/50000 /ip4/127.0.0.1/tcp/50000]}"} +``` +In this example: `12D3KooWQNNWNiJ1mieEk9EHjDVF2qBc1FSjJGEzwjnMJzteApaW` + +14. Set the peer ID and multi-address of the miner on chain +``` +lotus-miner actor set-peer-id +lotus-miner actor set-addrs /ip4/127.0.0.1/tcp/50000 +``` + +16. Open the Web UI + +Open http://localhost:8080 to see the Boost UI + +### Make a deal with Boost + +1. Initialize the Boost client +``` +boost init +``` + +This will output the address of the wallet (it's safe to run the init command repeatedly). + +2. Send funds to the client wallet +``` +lotus send --from=$DEFAULT_WALLET 10 +``` + +3. Follow the guide at https://boost.filecoin.io/tutorials/how-to-store-files-with-boost-on-filecoin + +Note that above you already ran a command to export FULLNODE_API (and point it to your local devnet lotus daemon). + +Note also that the provider address is `t01000` and you will need to supply an appropriate `--storage-price` when using `boost deal` since the devnet has a minimum price. Alternatively, using "Settings" in the Boost web UI to set the deal price to zero. + +## Running Boost devnet in Docker + +### Building Docker images + +1. Select Lotus version, for example: `lotus_version=1.17.1-rc2`. It must be the tag name of [the Lotus git repo](https://github.com/filecoin-project/lotus/tags) without `v` prefix. + +2. Select Boost version, for example: `boost_version=1.3.0-rc1`. + +3. Build images + +``` +cd docker/devnet +make build/all +``` + +If you need to build a different version, edit the `.env` file. + +### Start devnet docker stack + +1. Run + +``` +cd docker/devnet +docker compose up -d +``` + +It will spin up `lotus`, `lotus-miner`, `boost`, `booster-http` and `demo-http-server` containers. All temporary data will be saved in `./data` folder. + +The initial setup could take up to 20 min or more as it needs to download Filecoin proof parameters. During the initial setup, it is normal to see error messages in the log. Containers are waiting for the lotus to be ready. It may timeout several times. Restart is expected to be managed by `docker`. + +2. Try opening the Boost GUI http://localhost:8080 . Devnet is ready to operate when the URL opens and indicates no errors on the startup page. + +You can inspect the status using `docker compose logs -f`. + +### Start monitoring docker stack + +``` +docker plugin install grafana/loki-docker-driver:latest --alias loki --grant-all-permissions + +cd docker/monitoring +docker compose up -d +``` + +### Connect monitoring stack to devnet stack + +``` +docker network connect devnet tempo +``` + +### Explore Grafana / Tempo and search for traces + +http://localhost:3333 + +### Making a deal + +The `boost` container is packed with `boost` and `lotus` clients. You can connect to the container with the command `docker compose exec boost /bin/bash` and follow instructions for [storing files with Boost guide](https://boost.filecoin.io/tutorials/how-to-store-files-with-boost-on-filecoin). But the recommended startup is to follow the semi-interactive demo first: + +``` +# Attach to a running boost container +docker compose exec boost /bin/bash + +# Execute the demo script /app/sample/make-a-deal.sh +root@83260455bbd2:/app# ./sample/make-a-deal.sh +``` + +### Accessing Lotus from localhost + +By default the [docker-compose.yaml](./docker-compose.yaml) does not expose any port of the `lotus` container. To access the `lotus` from a local machine: +1. You can either expose `1234` in [docker-compose.yaml](./docker-compose.yaml) or find the IP of the `lotus` container using `docker inspect lotus | grep IPAddress` command. +2. Get the `FULLNODE_API_INFO` +``` +docker exec -it lotus lotus auth api-info --perm=admin +FULLNODE_API_INFO=eyJ...ms4:/dns/lotus/tcp/1234/http +``` +3. Change the `dns/lotus/tcp/1234/http` to `ip4/<127.0.0.1 or container's IP>/tcp/1234/http` for the use in `FULLNODE_API_INFO`. + +### Cleaning up + +To stop containers and drop everything: +``` +docker compose down --rmi local + +rm -rf ./data + +rm -rf /var/tmp/filecoin-proof-parameters +``` ## License diff --git a/build/openrpc/boost.json.gz b/build/openrpc/boost.json.gz index 2124f961d..45408a8f2 100644 Binary files a/build/openrpc/boost.json.gz and b/build/openrpc/boost.json.gz differ diff --git a/build/version.go b/build/version.go index e11be0ee5..4283f6756 100644 --- a/build/version.go +++ b/build/version.go @@ -2,7 +2,7 @@ package build var CurrentCommit string -const BuildVersion = "1.4.0-rc1" +const BuildVersion = "1.4.0-rc2" func UserVersion() string { return BuildVersion + CurrentCommit diff --git a/cmd/boostd/dagstore.go b/cmd/boostd/dagstore.go index 45bdbf349..57ec2265e 100644 --- a/cmd/boostd/dagstore.go +++ b/cmd/boostd/dagstore.go @@ -6,6 +6,7 @@ import ( "strings" "github.com/filecoin-project/lotus/lib/tablewriter" + "github.com/ipfs/go-cid" "github.com/fatih/color" bapi "github.com/filecoin-project/boost/api" @@ -25,6 +26,7 @@ var dagstoreCmd = &cli.Command{ dagstoreListShardsCmd, dagstoreGcCmd, dagstoreDestroyShardCmd, + dagstoreLookupCmd, }, } @@ -309,3 +311,47 @@ var dagstoreDestroyShardCmd = &cli.Command{ return nil }, } + +var dagstoreLookupCmd = &cli.Command{ + Name: "lookup-piece-cid", + ArgsUsage: "[key]", + Usage: "Performs a reverse lookup with payload CID to get Piece CID", + Aliases: []string{"lpc"}, + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "color", + Usage: "use color in display output", + DefaultText: "depends on output being a TTY", + }, + }, + Action: func(cctx *cli.Context) error { + if cctx.IsSet("color") { + color.NoColor = !cctx.Bool("color") + } + + if cctx.NArg() != 1 { + return fmt.Errorf("must provide a single payload CID") + } + + napi, closer, err := bcli.GetBoostAPI(cctx) + if err != nil { + return err + } + defer closer() + + ctx := lcli.ReqContext(cctx) + + shardKey := cctx.Args().First() + payloadCid, err := cid.Parse(shardKey) + if err != nil { + return fmt.Errorf("Unable to parse the provided CID: %s", shardKey) + } + pieceCid, err := napi.BoostDagstorePiecesContainingMultihash(ctx, payloadCid.Hash()) + if err != nil { + return err + } + + fmt.Printf("Given CID was found in the following pieces: %s", pieceCid) + return nil + }, +} diff --git a/cmd/boostd/init.go b/cmd/boostd/init.go index dd173b740..ac5dbddec 100644 --- a/cmd/boostd/init.go +++ b/cmd/boostd/init.go @@ -540,10 +540,6 @@ func initBoost(ctx context.Context, cctx *cli.Context, marketsRepo lotus_repo.Lo return nil, fmt.Errorf("failed to parse wallet-deal-collateral: %s; err: %w", cctx.String("wallet-deal-collateral"), err) } - if walletPSD.String() == walletCP.String() { - return nil, fmt.Errorf("wallets for PublishStorageDeals and deal collateral must be different") - } - if cctx.Int64("max-staging-deals-bytes") <= 0 { return nil, fmt.Errorf("max size for staging deals area must be > 0 bytes") } @@ -700,8 +696,6 @@ func setBoostDealMakingCfg(bdm *config.DealmakingConfig, mktsCfg *lotus_config.S bdm.MaxDealStartDelay = config.Duration(ldm.MaxDealStartDelay) bdm.MaxProviderCollateralMultiplier = ldm.MaxProviderCollateralMultiplier bdm.MaxStagingDealsBytes = ldm.MaxStagingDealsBytes - bdm.SimultaneousTransfersForStorage = ldm.SimultaneousTransfersForStorage - bdm.SimultaneousTransfersForRetrieval = ldm.SimultaneousTransfersForRetrieval bdm.StartEpochSealingBuffer = ldm.StartEpochSealingBuffer bdm.Filter = ldm.Filter bdm.RetrievalFilter = ldm.RetrievalFilter diff --git a/cmd/booster-http/run.go b/cmd/booster-http/run.go index f2ed270c9..f9f336cd9 100644 --- a/cmd/booster-http/run.go +++ b/cmd/booster-http/run.go @@ -11,6 +11,7 @@ import ( "github.com/filecoin-project/boost/api" bclient "github.com/filecoin-project/boost/api/client" cliutil "github.com/filecoin-project/boost/cli/util" + "github.com/filecoin-project/boost/tracing" "github.com/filecoin-project/dagstore/mount" "github.com/filecoin-project/go-fil-markets/piecestore" "github.com/filecoin-project/go-jsonrpc" @@ -67,10 +68,20 @@ var runCmd = &cli.Command{ Required: true, }, &cli.StringFlag{ - Name: "api-sealer", - Usage: "the endpoint for the sealer API", + Name: "api-storage", + Usage: "the endpoint for the storage node API", Required: true, }, + &cli.BoolFlag{ + Name: "tracing", + Usage: "enables tracing of booster-http calls", + Value: false, + }, + &cli.StringFlag{ + Name: "tracing-endpoint", + Usage: "the endpoint for the tracing exporter", + Value: "http://tempo:14268/api/traces", + }, }, Action: func(cctx *cli.Context) error { if cctx.Bool("pprof") { @@ -99,19 +110,30 @@ var runCmd = &cli.Command{ } defer ncloser() - // Connect to the sealing API - sealingApiInfo := cctx.String("api-sealer") - sauth, err := storageAuthWithURL(sealingApiInfo) + // Connect to the storage API + storageApiInfo := cctx.String("api-storage") + sauth, err := storageAuthWithURL(storageApiInfo) if err != nil { - return fmt.Errorf("parsing sealing API endpoint: %w", err) + return fmt.Errorf("parsing storage API endpoint: %w", err) } - sealingService, sealerCloser, err := getMinerApi(ctx, sealingApiInfo) + storageService, storageCloser, err := getMinerApi(ctx, storageApiInfo) if err != nil { return fmt.Errorf("getting miner API: %w", err) } - defer sealerCloser() + defer storageCloser() + + // Instantiate the tracer and exporter + enableTracing := cctx.Bool("tracing") + var tracingStopper func(context.Context) error + if enableTracing { + tracingStopper, err = tracing.New("booster-http", cctx.String("tracing-endpoint")) + if err != nil { + return fmt.Errorf("failed to instantiate tracer: %w", err) + } + log.Info("Tracing exporter enabled") + } - maddr, err := sealingService.ActorAddress(ctx) + maddr, err := storageService.ActorAddress(ctx) if err != nil { return fmt.Errorf("getting miner actor address: %w", err) } @@ -129,17 +151,17 @@ var runCmd = &cli.Command{ // Create the store interface var urls []string - lstor, err := paths.NewLocal(ctx, lr, sealingService, urls) + lstor, err := paths.NewLocal(ctx, lr, storageService, urls) if err != nil { return fmt.Errorf("creating new local store: %w", err) } - storage := lotus_modules.RemoteStorage(lstor, sealingService, sauth, sealer.Config{ + storage := lotus_modules.RemoteStorage(lstor, storageService, sauth, sealer.Config{ // TODO: Not sure if I need this, or any of the other fields in this struct ParallelFetchLimit: 1, }) // Create the piece provider and sector accessors - pp := sealer.NewPieceProvider(storage, sealingService, sealingService) - sa := sectoraccessor.NewSectorAccessor(dtypes.MinerAddress(maddr), sealingService, pp, fullnodeApi) + pp := sealer.NewPieceProvider(storage, storageService, storageService) + sa := sectoraccessor.NewSectorAccessor(dtypes.MinerAddress(maddr), storageService, pp, fullnodeApi) allowIndexing := cctx.Bool("allow-indexing") // Create the server API sapi := serverApi{ctx: ctx, bapi: bapi, sa: sa} @@ -176,6 +198,13 @@ var runCmd = &cli.Command{ // Sync all loggers. _ = log.Sync() //nolint:errcheck + if enableTracing { + err = tracingStopper(ctx) + if err != nil { + return err + } + } + return nil }, } @@ -198,8 +227,8 @@ type serverApi struct { var _ HttpServerApi = (*serverApi)(nil) -func (s serverApi) PiecesContainingMultihash(mh multihash.Multihash) ([]cid.Cid, error) { - return s.bapi.BoostDagstorePiecesContainingMultihash(s.ctx, mh) +func (s serverApi) PiecesContainingMultihash(ctx context.Context, mh multihash.Multihash) ([]cid.Cid, error) { + return s.bapi.BoostDagstorePiecesContainingMultihash(ctx, mh) } func (s serverApi) GetMaxPieceOffset(pieceCid cid.Cid) (uint64, error) { @@ -269,7 +298,7 @@ func getMinerApi(ctx context.Context, ai string) (v0api.StorageMiner, jsonrpc.Cl return nil, nil, fmt.Errorf("could not get DialArgs: %w", err) } - log.Infof("Using sealing API at %s", addr) + log.Infof("Using storage API at %s", addr) api, closer, err := client.NewStorageMinerRPCV0(ctx, addr, info.AuthHeader()) if err != nil { return nil, nil, fmt.Errorf("creating miner service API: %w", err) diff --git a/cmd/booster-http/server.go b/cmd/booster-http/server.go index 1c49b25e4..457d78a44 100644 --- a/cmd/booster-http/server.go +++ b/cmd/booster-http/server.go @@ -12,6 +12,7 @@ import ( "time" "github.com/fatih/color" + "github.com/filecoin-project/boost/tracing" "github.com/filecoin-project/dagstore/mount" "github.com/filecoin-project/go-fil-markets/piecestore" "github.com/filecoin-project/go-fil-markets/retrievalmarket" @@ -46,7 +47,7 @@ type HttpServer struct { } type HttpServerApi interface { - PiecesContainingMultihash(mh multihash.Multihash) ([]cid.Cid, error) + PiecesContainingMultihash(ctx context.Context, mh multihash.Multihash) ([]cid.Cid, error) GetMaxPieceOffset(pieceCid cid.Cid) (uint64, error) GetPieceInfo(pieceCID cid.Cid) (*piecestore.PieceInfo, error) IsUnsealed(ctx context.Context, sectorID abi.SectorNumber, offset abi.UnpaddedPieceSize, length abi.UnpaddedPieceSize) (bool, error) @@ -147,6 +148,9 @@ func (s *HttpServer) handleIndex(w http.ResponseWriter, r *http.Request) { } func (s *HttpServer) handleByPayloadCid(w http.ResponseWriter, r *http.Request) { + ctx, span := tracing.Tracer.Start(r.Context(), "http.payload_cid") + defer span.End() + // Remove the path up to the payload cid prefixLen := len(s.payloadBasePath()) if len(r.URL.Path) <= prefixLen { @@ -166,7 +170,7 @@ func (s *HttpServer) handleByPayloadCid(w http.ResponseWriter, r *http.Request) } // Find all the pieces that contain the payload cid - pieces, err := s.api.PiecesContainingMultihash(payloadCid.Hash()) + pieces, err := s.api.PiecesContainingMultihash(ctx, payloadCid.Hash()) if err != nil { if isNotFoundError(err) { msg := fmt.Sprintf("getting piece that contains payload CID '%s': %s", payloadCid, err.Error()) @@ -182,11 +186,8 @@ func (s *HttpServer) handleByPayloadCid(w http.ResponseWriter, r *http.Request) // Just get the content of the first piece returned (if the client wants a // different piece they can just call the /piece endpoint) pieceCid := pieces[0] - ctx := r.Context() content, err := s.getPieceContent(ctx, pieceCid) - if err == nil && isCar && r.Method != "HEAD" { - // Note: Getting the CAR content out of the piece is non-trivial, so - // we don't do it for HEAD requests + if err == nil && isCar { content, err = s.getCarContent(pieceCid, content) } if err != nil { @@ -233,9 +234,7 @@ func (s *HttpServer) handleByPieceCid(w http.ResponseWriter, r *http.Request) { // Get a reader over the the piece ctx := r.Context() content, err := s.getPieceContent(ctx, pieceCid) - if err == nil && isCar && r.Method != "HEAD" { - // Note: Getting the CAR content out of the piece is non-trivial, so - // we don't do it for HEAD requests + if err == nil && isCar { content, err = s.getCarContent(pieceCid, content) } if err != nil { @@ -272,7 +271,7 @@ func serveContent(w http.ResponseWriter, r *http.Request, content io.ReadSeeker, w.Header().Set("Content-Type", contentType) if r.Method == "HEAD" { - // For an HTTP HEAD request we don't send any data (just headers) + // For an HTTP HEAD request ServeContent doesn't send any data (just headers) http.ServeContent(w, r, "", time.Time{}, content) alog("%s\tHEAD %s", color.New(color.FgGreen).Sprintf("%d", http.StatusOK), r.URL) return diff --git a/db/deals_test.go b/db/deals_test.go index 85455759e..f2f4303f9 100644 --- a/db/deals_test.go +++ b/db/deals_test.go @@ -103,7 +103,7 @@ func TestDealsDBSearch(t *testing.T) { start := time.Now() db := NewDealsDB(sqldb) - deals, err := generateDeals(5) + deals, err := GenerateNDeals(5) req.NoError(err) t.Logf("generated %d deals in %s", len(deals), time.Since(start)) diff --git a/db/fixtures.go b/db/fixtures.go index 15db3b1bf..6b6b0f1e1 100644 --- a/db/fixtures.go +++ b/db/fixtures.go @@ -20,10 +20,10 @@ import ( var clientAddrs = []uint64{01312, 42134, 01322, 43242, 04212} func GenerateDeals() ([]types.ProviderDealState, error) { - return generateDeals(len(clientAddrs)) + return GenerateNDeals(len(clientAddrs)) } -func generateDeals(count int) ([]types.ProviderDealState, error) { +func GenerateNDeals(count int) ([]types.ProviderDealState, error) { provAddr, err := address.NewActorAddress([]byte("f1523")) if err != nil { return nil, err @@ -81,7 +81,7 @@ func generateDeals(count int) ([]types.ProviderDealState, error) { InboundFilePath: fmt.Sprintf("/data/staging/inbound/file-%d.car", rand.Intn(10000)), Transfer: types.Transfer{ Type: "http", - Params: []byte(fmt.Sprintf("{url:'http://files.org/file%d.car'}", rand.Intn(1000))), + Params: []byte(fmt.Sprintf(`{"url":"http://files.org/file%d.car"}`, rand.Intn(1000))), Size: uint64(rand.Intn(10000)), }, ChainDealID: abi.DealID(rand.Intn(10000)), diff --git a/db/migrations/20220908122510_storagetagged_add_host.sql b/db/migrations/20220908122510_storagetagged_add_host.sql new file mode 100644 index 000000000..1cc937324 --- /dev/null +++ b/db/migrations/20220908122510_storagetagged_add_host.sql @@ -0,0 +1,10 @@ +-- +goose Up +-- +goose StatementBegin +ALTER TABLE StorageTagged + ADD TransferHost TEXT; +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +SELECT 'down SQL query'; +-- +goose StatementEnd diff --git a/db/migrations/20220908122516_storagetagged_set_host.go b/db/migrations/20220908122516_storagetagged_set_host.go new file mode 100644 index 000000000..7766ccd27 --- /dev/null +++ b/db/migrations/20220908122516_storagetagged_set_host.go @@ -0,0 +1,58 @@ +package migrations + +import ( + "database/sql" + "fmt" + "github.com/filecoin-project/boost/storagemarket/types" + "github.com/pressly/goose/v3" +) + +func init() { + goose.AddMigration(UpSetStorageTaggedTransferHost, DownSetStorageTaggedTransferHost) +} + +func UpSetStorageTaggedTransferHost(tx *sql.Tx) error { + errPrefix := "setting StorageTagged.TransferHost: " + qry := "SELECT Deals.ID, Deals.TransferType, Deals.TransferParams " + + "FROM Deals INNER JOIN StorageTagged " + + "ON Deals.ID = StorageTagged.DealUUID" + rows, err := tx.Query(qry) + if err != nil { + return fmt.Errorf(errPrefix+"getting deals from DB: %w", err) + } + defer rows.Close() + + for rows.Next() { + var id string + var xferType string + var params []byte + err := rows.Scan(&id, &xferType, ¶ms) + if err != nil { + return fmt.Errorf(errPrefix+"scanning Deals row: %w", err) + } + + dealErrPrefix := fmt.Sprintf(errPrefix+"deal %s: ", id) + + xfer := types.Transfer{ + Type: xferType, + Params: params, + } + host, err := xfer.Host() + if err != nil { + log.Warnw(dealErrPrefix+"ignoring - couldn't parse transfer params %s: '%s': %s", xferType, params, err) + continue + } + + _, err = tx.Exec("UPDATE StorageTagged SET TransferHost = ? WHERE DealUUID = ?", host, id) + if err != nil { + return fmt.Errorf(dealErrPrefix+"saving TransferHost to DB: %w", err) + } + } + return rows.Err() +} + +func DownSetStorageTaggedTransferHost(tx *sql.Tx) error { + // This code is executed when the migration is rolled back. + // Do nothing because sqlite doesn't support removing a column. + return nil +} diff --git a/db/migrations_tests/storagetagged_set_host_test.go b/db/migrations_tests/storagetagged_set_host_test.go new file mode 100644 index 000000000..7777309c1 --- /dev/null +++ b/db/migrations_tests/storagetagged_set_host_test.go @@ -0,0 +1,70 @@ +package migrations_tests + +import ( + "context" + "fmt" + "github.com/filecoin-project/boost/db" + "github.com/filecoin-project/boost/db/migrations" + "github.com/filecoin-project/boost/storagemarket/types" + "github.com/pressly/goose/v3" + "github.com/stretchr/testify/require" + "testing" +) + +func TestStorageTaggedSetHost(t *testing.T) { + req := require.New(t) + ctx := context.Background() + + sqldb := db.CreateTestTmpDB(t) + req.NoError(db.CreateAllBoostTables(ctx, sqldb, sqldb)) + + // Run migrations up to the one that adds the TransferHost field to StorageTagged + goose.SetBaseFS(migrations.EmbedMigrations) + req.NoError(goose.SetDialect("sqlite3")) + req.NoError(goose.UpTo(sqldb, ".", 20220908122510)) + + // Generate 2 deals + dealsDB := db.NewDealsDB(sqldb) + deals, err := db.GenerateNDeals(2) + req.NoError(err) + + // Set the transfer params such that each deal has a different host + getHost := func(i int) string { + return fmt.Sprintf("files.org:%d", 1000+i) + } + for i, deal := range deals { + deal.Transfer = types.Transfer{ + Type: "http", + Params: []byte(fmt.Sprintf(`{"url":"http://%s/file.car"}`, getHost(i))), + Size: uint64(1024), + } + err = dealsDB.Insert(ctx, &deal) + req.NoError(err) + } + + // Simulate tagging a deal + taggedStorageDB := db.NewStorageDB(sqldb) + err = taggedStorageDB.Tag(ctx, deals[0].DealUuid, 1024, "") + req.NoError(err) + + // Run the migration that reads the deal transfer params and sets + // StorageTagged.TransferHost + req.NoError(goose.UpByOne(sqldb, ".")) + + // Check that after migrating up, the host is set correctly + rows, err := sqldb.QueryContext(ctx, "SELECT TransferHost FROM StorageTagged") + req.NoError(err) + defer rows.Close() //nolint:errcheck + + rowIdx := 0 + for ; rows.Next(); rowIdx++ { + var host string + err := rows.Scan(&host) + req.NoError(err) + req.Equal(getHost(0), host) + } + + // Even though there are two deals in DB, there is only one deal that is + // tagged, so there should only be one row + req.Equal(1, rowIdx) +} diff --git a/db/storage.go b/db/storage.go index 240b64dcc..88fef71e8 100644 --- a/db/storage.go +++ b/db/storage.go @@ -26,10 +26,10 @@ func NewStorageDB(db *sql.DB) *StorageDB { return &StorageDB{db: db} } -func (s *StorageDB) Tag(ctx context.Context, dealUuid uuid.UUID, size uint64) error { - qry := "INSERT INTO StorageTagged (DealUUID, CreatedAt, TransferSize) " - qry += "VALUES (?, ?, ?)" - values := []interface{}{dealUuid, time.Now(), fmt.Sprintf("%d", size)} +func (s *StorageDB) Tag(ctx context.Context, dealUuid uuid.UUID, size uint64, host string) error { + qry := "INSERT INTO StorageTagged (DealUUID, CreatedAt, TransferSize, TransferHost) " + qry += "VALUES (?, ?, ?, ?)" + values := []interface{}{dealUuid, time.Now(), fmt.Sprintf("%d", size), host} _, err := s.db.ExecContext(ctx, qry, values...) return err } @@ -112,8 +112,22 @@ func (s *StorageDB) Logs(ctx context.Context) ([]StorageLog, error) { return storageLogs, nil } +func (s *StorageDB) TotalTaggedForHost(ctx context.Context, host string) (uint64, error) { + return s.totalTagged(ctx, host) +} + func (s *StorageDB) TotalTagged(ctx context.Context) (uint64, error) { - rows, err := s.db.QueryContext(ctx, "SELECT TransferSize FROM StorageTagged") + return s.totalTagged(ctx, "") +} + +func (s *StorageDB) totalTagged(ctx context.Context, host string) (uint64, error) { + qry := "SELECT TransferSize FROM StorageTagged" + var args []interface{} + if host != "" { + qry += " WHERE TransferHost = ?" + args = append(args, host) + } + rows, err := s.db.QueryContext(ctx, qry, args...) if err != nil { return 0, fmt.Errorf("getting total tagged: %w", err) } diff --git a/db/storage_test.go b/db/storage_test.go index 543673283..26d859801 100644 --- a/db/storage_test.go +++ b/db/storage_test.go @@ -5,8 +5,8 @@ import ( "errors" "testing" + "github.com/filecoin-project/boost/db/migrations" "github.com/google/uuid" - "github.com/stretchr/testify/require" ) @@ -16,6 +16,7 @@ func TestStorageDB(t *testing.T) { sqldb := CreateTestTmpDB(t) require.NoError(t, CreateAllBoostTables(ctx, sqldb, sqldb)) + req.NoError(migrations.Migrate(sqldb)) db := NewStorageDB(sqldb) @@ -28,12 +29,20 @@ func TestStorageDB(t *testing.T) { req.True(errors.Is(err, ErrNotFound)) req.Equal(uint64(0), amt) - err = db.Tag(ctx, dealUUID, 1111) + err = db.Tag(ctx, dealUUID, 1111, "foo.bar:1234") + req.NoError(err) + + dealUUID2 := uuid.New() + err = db.Tag(ctx, dealUUID2, 2222, "my.host:5678") req.NoError(err) total, err := db.TotalTagged(ctx) req.NoError(err) - req.Equal(uint64(1111), total) + req.Equal(uint64(3333), total) + + total, err = db.TotalTaggedForHost(ctx, "my.host:5678") + req.NoError(err) + req.Equal(uint64(2222), total) amt, err = db.Untag(ctx, dealUUID) req.NoError(err) diff --git a/docker/devnet/.env b/docker/devnet/.env new file mode 100644 index 000000000..e1ebb96a2 --- /dev/null +++ b/docker/devnet/.env @@ -0,0 +1,6 @@ +DOCKER_USER=filecoin +LOTUS_IMAGE=${DOCKER_USER}/lotus-dev:1.17.1-rc2 +LOTUS_MINER_IMAGE=${DOCKER_USER}/lotus-miner-dev:1.17.1-rc2 +BOOST_IMAGE=${DOCKER_USER}/boost-dev:dev +BOOSTER_HTTP_IMAGE=${DOCKER_USER}/booster-http-dev:dev +BOOST_GUI_IMAGE=${DOCKER_USER}/boost-gui:dev diff --git a/docker/devnet/.env-stable b/docker/devnet/.env-stable new file mode 100644 index 000000000..6db690f48 --- /dev/null +++ b/docker/devnet/.env-stable @@ -0,0 +1,5 @@ +DOCKER_USER=filecoin +LOTUS_IMAGE=${DOCKER_USER}/lotus-dev:1.17.1-rc2 +LOTUS_MINER_IMAGE=${DOCKER_USER}/lotus-miner-dev:1.17.1-rc2 +BOOST_IMAGE=${DOCKER_USER}/boost-dev:1.3.0-rc1 +BOOST_GUI_IMAGE=${DOCKER_USER}/boost-gui:1.3.0-rc1 diff --git a/docker/devnet/.gitignore b/docker/devnet/.gitignore new file mode 100644 index 000000000..1269488f7 --- /dev/null +++ b/docker/devnet/.gitignore @@ -0,0 +1 @@ +data diff --git a/docker/devnet/Makefile b/docker/devnet/Makefile new file mode 100644 index 000000000..7cad4af29 --- /dev/null +++ b/docker/devnet/Makefile @@ -0,0 +1,57 @@ +################################################################################## +lotus_version?=1.17.1-rc2 +boost_version?=1.3.0-rc1 +docker_user?=filecoin + +lotus_test_image=$(docker_user)/lotus-test:$(lotus_version) +################################################################################## +lotus-$(lotus_version): + git clone --depth 1 --branch v$(lotus_version) https://github.com/filecoin-project/lotus $@ + +prepare/lotus-test: | lotus-$(lotus_version) + cd lotus-$(lotus_version) && docker build -f Dockerfile.lotus --target lotus-test -t $(lotus_test_image) . +.PHONY: prepare/lotus-test +################################################################################## +build/%: prepare/lotus-test + cd $* && make dbuild +../../build/.update-modules: + cd ../../ && make build/.update-modules +build/boost: ../../build/.update-modules prepare/lotus-test + cd boost && make dbuild +push/%: prepare/lotus-test + cd $* && make dpush +################################################################################## +build/all: build/lotus build/lotus-miner build/boost build/boost-gui build/booster-http +.PHONY: build/all +################################################################################## +push/all: push/lotus push/lotus-miner push/boost push/boost-gui push/booster-http +.PHONY: push/all +################################################################################## +clean: clean/lotus-test +.PHONY: clean + +clean/lotus-test: + rm -rf lotus-$(lotus_version) +.PHONY: clean/lotus-test + +.EXPORT_ALL_VARIABLES: +################################################################################## +start: + docker compose up -d + docker compose logs -f +.PHONY: start +################################################################################## +ssh/boost: + docker compose exec boost /bin/bash +.PHONY: ssh/boost +################################################################################## +clean-stack: clean/docker + rm -rf data +.PHONY: clean +clean/all: clean + rm -rf /var/tmp/filecoin-proof-parameters +.PHONY: clean/all +clean/docker: + docker compose down +.PHONY: clean/docker +################################################################################## diff --git a/docker/devnet/boost/Dockerfile.source b/docker/devnet/boost/Dockerfile.source new file mode 100644 index 000000000..9c4315ee4 --- /dev/null +++ b/docker/devnet/boost/Dockerfile.source @@ -0,0 +1,77 @@ +######################################################################################### +## docker will invoke this file from ../../.. dir in order to access the code +######################################################################################### +ARG LOTUS_TEST_IMAGE=filecoin/lotus-test:latest +FROM ${LOTUS_TEST_IMAGE} as lotus-dev +######################################################################################### +FROM golang:1.18-bullseye as builder + +RUN apt update && apt install -y \ + build-essential \ + bzr pkg-config \ + clang \ + curl \ + gcc git \ + hwloc \ + jq \ + libhwloc-dev wget \ + mesa-opencl-icd \ + ocl-icd-opencl-dev + +WORKDIR /go/src/ + +COPY Makefile /go/src/ +############################################## +# prebuild filecoin-ffi +COPY extern /go/src/extern +COPY build /go/src/build +COPY .git/modules/extern/filecoin-ffi /go/src/.git/modules/extern/filecoin-ffi + +RUN make build/.filecoin-install +############################################## +COPY . /go/src +############################################## +RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ + make debug +######################################################################################### +FROM ubuntu:20.04 as runner + +RUN apt update && apt install -y \ + curl \ + hwloc \ + jq + +ARG BUILD_VERSION=0.1 + +LABEL org.opencontainers.image.version=$BUILD_VERSION \ + org.opencontainers.image.authors="Boost Dev Team" \ + name="boost-dev" \ + maintainer="Boost Dev Team" \ + vendor="Boost Dev Team" \ + version=$BUILD_VERSION \ + release=$BUILD_VERSION \ + summary="This image is used to host the boost-dev storage provider" \ + description="This image is used to host the boost-dev storage provider" + +WORKDIR /app +ENV BOOST_PATH /var/lib/boost +VOLUME /var/lib/boost +EXPOSE 8080 + +## Fix missing lib libhwloc.so.5 +RUN ls -1 /lib/*/libhwloc.so.* | head -n 1 | xargs -n1 -I {} ln -s {} /lib/libhwloc.so.5 + +COPY --from=lotus-dev /usr/local/bin/lotus /usr/local/bin/ +COPY --from=lotus-dev /usr/local/bin/lotus-miner /usr/local/bin/ + +COPY --from=builder /go/src/boostd /usr/local/bin/ +COPY --from=builder /go/src/boost /usr/local/bin/ +COPY --from=builder /go/src/boostx /usr/local/bin/ +## Smoke test for the boost and lotus +RUN lotus -v && boost -v + +COPY docker/devnet/boost/entrypoint.sh /app/ +COPY docker/devnet/boost/sample/* /app/sample/ + +ENTRYPOINT ["./entrypoint.sh"] diff --git a/docker/devnet/boost/Dockerfile.stable b/docker/devnet/boost/Dockerfile.stable new file mode 100644 index 000000000..186e2c034 --- /dev/null +++ b/docker/devnet/boost/Dockerfile.stable @@ -0,0 +1,64 @@ +######################################################################################### +######################################################################################### +ARG LOTUS_TEST_IMAGE=filecoin/lotus-test:latest +FROM ${LOTUS_TEST_IMAGE} as lotus-dev +######################################################################################### +FROM golang:1.18-bullseye as builder + +RUN apt update && apt install -y \ + build-essential \ + bzr pkg-config \ + clang \ + curl \ + gcc git \ + hwloc \ + jq \ + libhwloc-dev wget \ + mesa-opencl-icd \ + ocl-icd-opencl-dev + +WORKDIR /go/src/ + +ARG BUILD_VERSION=0.1 +RUN git clone --depth 1 --branch v${BUILD_VERSION} https://github.com/filecoin-project/boost + +RUN cd boost && make debug +######################################################################################### +FROM ubuntu:20.04 as runner + +RUN apt update && apt install -y \ + curl \ + hwloc \ + jq + +ARG BUILD_VERSION=0.1 + +LABEL org.opencontainers.image.version=$BUILD_VERSION \ + org.opencontainers.image.authors="Boost Dev Team" \ + name="boost-dev" \ + maintainer="Boost Dev Team" \ + vendor="Boost Dev Team" \ + version=$BUILD_VERSION \ + release=$BUILD_VERSION \ + summary="This image is used to host the boost-dev storage provider" \ + description="This image is used to host the boost-dev storage provider" + +WORKDIR /app +ENV BOOST_PATH /var/lib/boost +VOLUME /var/lib/boost +EXPOSE 8080 + +COPY --from=builder /go/src/boost/boostd /usr/local/bin/ +COPY --from=builder /go/src/boost/boost /usr/local/bin/ +COPY --from=builder /go/src/boost/boostx /usr/local/bin/ +COPY --from=lotus-dev /usr/local/bin/lotus /usr/local/bin/ +COPY --from=lotus-dev /usr/local/bin/lotus-miner /usr/local/bin/ +## Fix missing lib libhwloc.so.5 +RUN ls -1 /lib/x86_64-linux-gnu/libhwloc.so.* | head -n 1 | xargs -n1 -I {} ln -s {} /lib/x86_64-linux-gnu/libhwloc.so.5 +## Smoke test for the boost and lotus +RUN lotus -v && boost -v + +COPY entrypoint.sh /app/ +COPY sample/* /app/sample/ + +ENTRYPOINT ["./entrypoint.sh"] diff --git a/docker/devnet/boost/Makefile b/docker/devnet/boost/Makefile new file mode 100644 index 000000000..ec4d22a89 --- /dev/null +++ b/docker/devnet/boost/Makefile @@ -0,0 +1,19 @@ +##################################################################################### +service=$(docker_user)/boost-dev +version=dev +########### DOCKER ################################################################## +tag=$(service):$(version) +rootdir=$(realpath .) + +dbuild: + DOCKER_BUILDKIT=1 docker build --build-arg LOTUS_TEST_IMAGE=$(lotus_test_image) \ + -t $(tag) $(docker_args) -f Dockerfile.source $(rootdir)/../../../ + +dpush: dbuild + docker push $(tag) + +dscan: dbuild + docker scan --accept-license $(tag) +##################################################################################### +.PHONY: + dbuild dpush dscan diff --git a/docker/devnet/boost/entrypoint.sh b/docker/devnet/boost/entrypoint.sh new file mode 100755 index 000000000..a76c7362e --- /dev/null +++ b/docker/devnet/boost/entrypoint.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +set -e + +echo Wait for lotus is ready ... +lotus wait-api +echo Wait for lotus-miner is ready ... +lotus-miner wait-api +echo BOOST_PATH=$BOOST_PATH +export DEFAULT_WALLET=`lotus wallet default` +export FULLNODE_API_INFO=`lotus auth api-info --perm=admin | cut -f2 -d=` +export MINER_API_INFO=`lotus-miner auth api-info --perm=admin | cut -f2 -d=` + +if [ ! -f $BOOST_PATH/.init.boost ]; then + echo Init wallets ... + export COLLAT_WALLET=`lotus wallet new bls` + export PUBMSG_WALLET=`lotus wallet new bls` + export CLIENT_WALLET=`lotus wallet new bls` + echo MINER_API_INFO=$MINER_API_INFO + echo FULLNODE_API_INFO=$FULLNODE_API_INFO + echo PUBMSG_WALLET=$PUBMSG_WALLET + echo COLLAT_WALLET=$COLLAT_WALLET + + lotus send --from $DEFAULT_WALLET $COLLAT_WALLET 10 + lotus send --from $DEFAULT_WALLET $PUBMSG_WALLET 10 + lotus send --from $DEFAULT_WALLET $CLIENT_WALLET 10 + lotus wallet market add --from $DEFAULT_WALLET --address $CLIENT_WALLET 5 + lotus wallet market add --address $COLLAT_WALLET 5 + + until lotus-miner actor control set --really-do-it ${PUBMSG_WALLET}; do echo Waiting for storage miner API ready ...; sleep 1; done + + echo Init boost on first run ... + + boostd -vv --boost-repo $BOOST_PATH init --api-sealer=$MINER_API_INFO \ + --api-sector-index=$MINER_API_INFO \ + --wallet-publish-storage-deals=$PUBMSG_WALLET \ + --wallet-deal-collateral=$COLLAT_WALLET \ + --max-staging-deals-bytes=2000000000 + + # echo exit code: $? + + echo Setting port in boost config... + sed -i 's|ip4/0.0.0.0/tcp/0|ip4/0.0.0.0/tcp/50000|g' $BOOST_PATH/config.toml + + echo Done + touch $BOOST_PATH/.init.boost +fi + +if [ ! -f $BOOST_PATH/.register.boost ]; then + echo Temporary starting boost to get maddr... + + boostd -vv run &> $BOOST_PATH/boostd.log & + BOOST_PID=`echo $!` + echo Got boost PID = $BOOST_PID + + until cat $BOOST_PATH/boostd.log | grep maddr; do echo "Waiting for boost..."; sleep 1; done + echo Looks like boost started and initialized... + + echo Registering to lotus-miner... + MADDR=`cat $BOOST_PATH/boostd.log | grep maddr | cut -f3 -d"{" | cut -f1 -d:` + echo Got maddr=${MADDR} + + lotus-miner actor set-peer-id ${MADDR} + lotus-miner actor set-addrs /dns/boost/tcp/50000 + echo Registered + + touch $BOOST_PATH/.register.boost + echo Try to stop boost... + kill -15 $BOOST_PID || kill -9 $BOOST_PID + rm -f $BOOST_PATH/boostd.log + echo Super. DONE! Boostd is now configured and will be started soon +fi + +echo Starting boost in dev mode... +exec boostd -vv run + diff --git a/docker/devnet/boost/sample/make-a-deal.sh b/docker/devnet/boost/sample/make-a-deal.sh new file mode 100755 index 000000000..e95277d76 --- /dev/null +++ b/docker/devnet/boost/sample/make-a-deal.sh @@ -0,0 +1,138 @@ +#!/usr/bin/env bash +################################################################################### +# sample demo script for making a deal with boost client +################################################################################### +set -e +# colors +cb="\e[1m" +ci="\e[3m" +cn="\e[0m" +################################################################################### +printf "\n +###################################################################################\n \ +Hello to the demo script that makes a storage deal using the boost client\n \ +###################################################################################\n \ +1. The boost client needs to know how to connect to the lotus instance. \ +We need to set ${cb}FULLNODE_API_INFO${cn} env var. We have the lotus client here that will provide a connection token.\n \ + : ${ci}lotus auth api-info --perm=admin${cn} - returns lotus connection token \n\n" +read -rsp $'Press any key to export variable...\n' -n1 key +export `lotus auth api-info --perm=admin` + +printf "\nExported FULLNODE_API_INFO=$FULLNODE_API_INFO\n \ +###################################################################################\n" +################################################################################### +printf "2. The boost client needs to be initialized by calling \n${ci}boost init${cn} \n\n" +read -rsp $'Press any key to execute it...\n\n' -n1 key + +boost init + +printf "\n\nGreat. Boost client has been initialized.\n \ +###################################################################################\n" +################################################################################### +printf "3. Now add some funds from lotus to boost wallet. We will use the lotus client:\n\n \ + : ${ci}lotus wallet default${cn} - returns default lotus wallet\n \ + : ${ci}boost wallet default${cn} - returns default wallet for the current boost client actor\n \ + : ${ci}lotus send --from=`lotus wallet default` `boost wallet default` 10${cn} - sends 10 FIL\n" +read -rsp $'Press any key to execute it...\n\n' -n1 key + +lotus send --from=`lotus wallet default` `boost wallet default` 10 + +printf "\n\nDone. Funds transfer was initiated\n \ +###################################################################################\n" +################################################################################### +printf "4. Now add some funds to the market actor\n \ + : ${ci}boostx market-add 1${cn}\n\n" +read -rsp $'Press any key to execute it...\n' -n1 key + +until boostx market-add 1; do printf "\nOpps, maybe funds not added yet.\nNeed to wait some time. \n"; read -rsp $'Press any key to try again...\n' -n1 key; done + +printf "\n\nYes. We can make a deal now.\n \ +###################################################################################\n" +################################################################################### +printf "5. Let's generate a sample file in ${ci}/app/public/sample.txt${cn}. We will use it as a demo file.\n\n" +read -rsp $'Press any key to generate it...\n\n' -n1 key +rm -f /app/public/sample.txt +for i in {1..57}; do echo "Hi Boost, $i times" >> /app/public/sample.txt; done + +printf "\n\nFile content:\n\n" +cat /app/public/sample.txt +printf "\n\n \ +###################################################################################\n" +################################################################################### + +printf "6. After that, you need to generate a car file for data you want to store on Filecoin (${ci}/app/public/sample.txt${cn}), \ +and note down its ${ci}payload-cid${cn}. \ +We will use the ${ci}boostx${cn} utility\n \ + : ${ci}boostx generate-car /app/public/sample.txt /app/public/sample.car${cn}\n\n" +read -rsp $'Press any key to execute it...\n\n' -n1 key + +boostx generate-car /app/public/sample.txt /app/public/sample.car + +PAYLOAD_CID=`boostx generate-car /app/public/sample.txt /app/public/sample.car | grep CID | cut -d: -f2 | xargs` +printf "\n\nDone. We noted payload-cid = ${ci}$PAYLOAD_CID${cn}\n \ +###################################################################################\n" +################################################################################### +printf "7. Then you need to calculate the commp and piece size for the generated car file:\n \ + : ${ci}boostx commp /app/public/sample.car${cn}\n\n" +read -rsp $'Press any key to execute it...\n\n' -n1 key + +boostx commp /app/public/sample.car + +COMMP_CID=`boostx commp /app/public/sample.car 2> /dev/null | grep CID | cut -d: -f2 | xargs` +PIECE=`boostx commp /app/public/sample.car 2> /dev/null | grep Piece | cut -d: -f2 | xargs` +CAR=`boostx commp /app/public/sample.car 2> /dev/null | grep Car | cut -d: -f2 | xargs` +printf "\n\nYes. We also have remembered these values:\n \ +Commp-cid = $COMMP_CID \n \ +Piece size = $PIECE \n \ +Car size = $CAR \n \ +###################################################################################\n" +################################################################################### +printf "8. That's it. We are ready to make the deal. \n \ + : ${ci}boost deal --verified=false --provider=t01000 \ +--http-url=http://demo-http-server/sample.car \ +--commp=$COMMP_CID --car-size=$CAR --piece-size=$PIECE \ +--payload-cid=$PAYLOAD_CID --storage-price 20000000000\n\n${cn}" +read -rsp $'Press any key to make the deal...\n\n' -n1 key + +until boost deal --verified=false \ + --provider=t01000 \ + --http-url=http://demo-http-server/sample.car \ + --commp=$COMMP_CID \ + --car-size=$CAR \ + --piece-size=$PIECE \ + --payload-cid=$PAYLOAD_CID --storage-price 20000000000 +do + printf "\nThe error has occured. Perhaps we should wait some time for funds to arrive into the market account.\n\n" + read -rsp $'Press any key to check the boost wallet...\n\n' -n1 key + boost init + read -rsp $'\n\nPress any key to try making the deal again...\n' -n1 key +done + +printf "\n\n ${cb}Congrats! You have made it.${cn}\n\n \ +###################################################################################\n" +###################################################################################" +printf "9. Deal has been made, and it will be published automatically after some time, but you can do it manually using boost's graphql API\n \ +: ${ci}curl -X POST -H \"Content-Type: application/json\" -d '{\"query\":\"mutation { dealPublishNow }\"}' http://localhost:8080/graphql/query ${cn}\n\n" +read -rsp $'Press any key to publish the deal...\n\n' -n1 key + +curl -X POST -H "Content-Type: application/json" -d '{"query":"mutation { dealPublishNow }"}' http://localhost:8080/graphql/query | jq +printf "\nDone.\n\n \ +###################################################################################\n" +################################################################################### +printf "10. To retrieve the file from the ${cb}lotus${cn} system you can use \n\ +${ci}lotus client retrieve${cn} or ${ci}lotus client cat${cn} commands.\n\ +: ${ci}lotus client cat --miner t01000 $PAYLOAD_CID ${cn}\n\n" + +read -rsp $'Press any key to show the file content...\n\n' -n1 key +until lotus client cat --miner t01000 $PAYLOAD_CID +do + printf "\nFile publishing may take time, please wait some time until the deal is finished and try again.\n\n" + read -rsp $'Press any key to try again...\n' -n1 key +done + +printf "\n\nIf you see a file content you have just completed the demo. You have succesfully:\n\n\ + 1) initiated the boost client\n\ + 2) prepared sample file\n\ + 3) sent the sample file to the Filecoin devnet\n\ + 4) retrieved the content of the file from it.\n\n\ +More info at ${cb}https://boost.filecoin.io${cn} or ${cb}https://github.com/filecoin-project/boost${cn}.\n\n\n" diff --git a/docker/devnet/booster-http/Dockerfile.source b/docker/devnet/booster-http/Dockerfile.source new file mode 100644 index 000000000..6ff59554c --- /dev/null +++ b/docker/devnet/booster-http/Dockerfile.source @@ -0,0 +1,59 @@ +######################################################################################### +## docker will invoke this file from ../../.. dir in order to access the code +######################################################################################### +ARG LOTUS_TEST_IMAGE=filecoin/lotus-test:latest +FROM ${LOTUS_TEST_IMAGE} as lotus-dev +######################################################################################### +FROM golang:1.18-bullseye as builder + +RUN apt update && apt install -y \ + build-essential \ + bzr pkg-config \ + clang \ + curl \ + gcc git \ + hwloc \ + jq \ + libhwloc-dev wget \ + mesa-opencl-icd \ + ocl-icd-opencl-dev + +WORKDIR /go/src/ + +# copy src +COPY . /go/src/ + +RUN make booster-http debug +######################################################################################### +FROM ubuntu:20.04 as runner + +RUN apt update && apt install -y \ + curl \ + hwloc \ + jq + +ARG BUILD_VERSION=0.1 + +LABEL org.opencontainers.image.version=$BUILD_VERSION \ + org.opencontainers.image.authors="Boost Dev Team" \ + name="booster-http-dev" \ + maintainer="Boost Dev Team" \ + vendor="Boost Dev Team" \ + version=$BUILD_VERSION \ + release=$BUILD_VERSION \ + summary="This image is used to host booster-http-dev" \ + description="This image is used to host booster-http-dev" + +WORKDIR /app +EXPOSE 7777 + +COPY --from=builder /go/src/booster-http /usr/local/bin/ +COPY --from=builder /go/src/boostd /usr/local/bin/ +COPY --from=lotus-dev /usr/local/bin/lotus /usr/local/bin/ +COPY --from=lotus-dev /usr/local/bin/lotus-miner /usr/local/bin/ +## Fix missing lib libhwloc.so.5 +RUN ls -1 /lib/x86_64-linux-gnu/libhwloc.so.* | head -n 1 | xargs -n1 -I {} ln -s {} /lib/x86_64-linux-gnu/libhwloc.so.5 + +COPY docker/devnet/booster-http/entrypoint.sh /app/ + +ENTRYPOINT ["./entrypoint.sh"] diff --git a/docker/devnet/booster-http/Makefile b/docker/devnet/booster-http/Makefile new file mode 100644 index 000000000..3259f9b5c --- /dev/null +++ b/docker/devnet/booster-http/Makefile @@ -0,0 +1,19 @@ +##################################################################################### +service=$(docker_user)/booster-http-dev +version=dev +########### DOCKER ################################################################## +tag=$(service):$(version) +rootdir=$(realpath .) + +dbuild: + docker build --build-arg LOTUS_TEST_IMAGE=$(lotus_test_image) \ + -t $(tag) -f Dockerfile.source $(rootdir)/../../../ + +dpush: dbuild + docker push $(tag) + +dscan: dbuild + docker scan --accept-license $(tag) +##################################################################################### +.PHONY: + dbuild dpush dscan diff --git a/docker/devnet/booster-http/entrypoint.sh b/docker/devnet/booster-http/entrypoint.sh new file mode 100755 index 000000000..7bc51736f --- /dev/null +++ b/docker/devnet/booster-http/entrypoint.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +set -e + +export FULLNODE_API_INFO=`lotus auth api-info --perm=admin | cut -f2 -d=` +export MINER_API_INFO=`lotus-miner auth api-info --perm=admin | cut -f2 -d=` +export BOOST_API_INFO=`boostd auth api-info --perm=admin | cut -f2 -d=` + +echo $FULLNODE_API_INFO +echo $MINER_API_INFO +echo $BOOST_API_INFO + +echo Starting booster-http... +exec booster-http run --api-boost=$BOOST_API_INFO --api-fullnode=$FULLNODE_API_INFO --api-storage=$MINER_API_INFO --tracing diff --git a/docker/devnet/docker-compose.yaml b/docker/devnet/docker-compose.yaml new file mode 100644 index 000000000..d00843b75 --- /dev/null +++ b/docker/devnet/docker-compose.yaml @@ -0,0 +1,91 @@ +version: '3.8' + +x-logging: + &default-logging + options: + max-size: '20m' + max-file: '3' + driver: json-file + +networks: + default: + name: devnet + +services: + lotus: + container_name: lotus + image: ${LOTUS_IMAGE} + # ports: + # - "1234:1234" + environment: + - LOTUS_API_LISTENADDRESS=/dns/lotus/tcp/1234/http + restart: unless-stopped + logging: *default-logging + volumes: + - ./data/lotus:/var/lib/lotus:rw + - ./data/genesis:/var/lib/genesis:rw + - /var/tmp/filecoin-proof-parameters:/var/tmp/filecoin-proof-parameters:rw + + lotus-miner: + container_name: lotus-miner + image: ${LOTUS_MINER_IMAGE} + # ports: + # - "2345:2345" + environment: + - LOTUS_API_LISTENADDRESS=/dns/lotus-miner/tcp/2345/http + - LOTUS_API_REMOTELISTENADDRESS=lotus-miner:2345 + - LOTUS_SEALING_BATCHPRECOMMITS=false + - LOTUS_SEALING_AGGREGATECOMMITS=false + - LOTUS_SUBSYSTEMS_ENABLEMARKETS=false + - LOTUS_SEALING_WAITDEALSDELAY=20s + restart: unless-stopped + logging: *default-logging + volumes: + - ./data/lotus-miner:/var/lib/lotus-miner:rw + - ./data/lotus:/var/lib/lotus:ro + - ./data/genesis:/var/lib/genesis:ro + - /var/tmp/filecoin-proof-parameters:/var/tmp/filecoin-proof-parameters:rw + + boost: + container_name: boost + image: ${BOOST_IMAGE} + ports: + - "8080:8080" + environment: + - LOTUS_API_LISTENADDRESS=/dns/boost/tcp/1288/http + - LOTUS_PATH=/var/lib/lotus + - LOTUS_MINER_PATH=/var/lib/lotus-miner + - LOTUS_TRACING_ENABLED=true + - LOTUS_TRACING_SERVICENAME=boostd + - LOTUS_TRACING_ENDPOINT=http://tempo:14268/api/traces + restart: unless-stopped + logging: *default-logging + volumes: + - ./data/boost:/var/lib/boost:rw + - ./data/lotus:/var/lib/lotus:ro + - ./data/lotus-miner:/var/lib/lotus-miner:ro + - ./data/sample:/app/public:rw + + booster-http: + container_name: booster-http + image: ${BOOSTER_HTTP_IMAGE} + ports: + - "7777:7777" + environment: + - BOOST_PATH=/var/lib/boost + - LOTUS_PATH=/var/lib/lotus + - LOTUS_MINER_PATH=/var/lib/lotus-miner + restart: unless-stopped + logging: *default-logging + volumes: + - ./data/boost:/var/lib/boost:ro + - ./data/lotus:/var/lib/lotus:ro + - ./data/lotus-miner:/var/lib/lotus-miner:ro + + demo-http-server: + container_name: demo-http-server + image: nginx:1.23-alpine + restart: unless-stopped + logging: *default-logging + volumes: + - ./data/sample:/usr/share/nginx/html:ro diff --git a/docker/devnet/lotus-miner/Dockerfile b/docker/devnet/lotus-miner/Dockerfile new file mode 100644 index 000000000..4507e492d --- /dev/null +++ b/docker/devnet/lotus-miner/Dockerfile @@ -0,0 +1,32 @@ +ARG LOTUS_TEST_IMAGE=filecoin/lotus-test:latest +############################################################################# +FROM ${LOTUS_TEST_IMAGE} + +ARG BUILD_VERSION=0.1 + +LABEL org.opencontainers.image.version=$BUILD_VERSION \ + org.opencontainers.image.authors="Boost Dev Team" \ + name="lotus-miner-dev" \ + maintainer="Boost Dev Team" \ + vendor="Boost Dev Team" \ + version=$BUILD_VERSION \ + release=$BUILD_VERSION \ + summary="This image is used to host the lotus-miner dev service" \ + description="This image is used to host the lotus-miner dev service" + +EXPOSE 2345 +ENV LOTUS_SKIP_GENESIS_CHECK=_yes_ +ENV GENESIS_PATH=/var/lib/genesis +ENV SECTOR_SIZE=8388608 + +VOLUME /var/tmp/filecoin-proof-parameters +VOLUME /var/lib/genesis + +WORKDIR /app +RUN mkdir -p /app + +COPY entrypoint.sh /app + +USER root + +ENTRYPOINT ["./entrypoint.sh"] diff --git a/docker/devnet/lotus-miner/Makefile b/docker/devnet/lotus-miner/Makefile new file mode 100644 index 000000000..74f8bd1ee --- /dev/null +++ b/docker/devnet/lotus-miner/Makefile @@ -0,0 +1,16 @@ +##################################################################################### +service=$(docker_user)/lotus-miner-dev +version?=$(lotus_version) +########### DOCKER ################################################################## +tag=$(service):$(version) +dbuild: + docker build -t $(tag) --build-arg LOTUS_TEST_IMAGE=$(lotus_test_image) --build-arg BUILD_VERSION=$(version) . + +dpush: dbuild + docker push $(tag) + +dscan: dbuild + docker scan --accept-license $(tag) +##################################################################################### +.PHONY: + dbuild dpush dscan diff --git a/docker/devnet/lotus-miner/entrypoint.sh b/docker/devnet/lotus-miner/entrypoint.sh new file mode 100755 index 000000000..e1041ebaa --- /dev/null +++ b/docker/devnet/lotus-miner/entrypoint.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -e +echo Wait for lotus is ready ... +lotus wait-api +echo Lotus ready. Lets go +if [ ! -f $LOTUS_MINER_PATH/.init.miner ]; then + echo Import the genesis miner key ... + lotus wallet import --as-default $GENESIS_PATH/pre-seal-t01000.key + echo Set up the genesis miner ... + lotus-miner init --genesis-miner --actor=t01000 --sector-size=$SECTOR_SIZE --pre-sealed-sectors=$GENESIS_PATH --pre-sealed-metadata=$GENESIS_PATH/pre-seal-t01000.json --nosync + touch $LOTUS_MINER_PATH/.init.miner + echo Done +fi + +echo Starting lotus miner ... +exec lotus-miner run --nosync diff --git a/docker/devnet/lotus/Dockerfile b/docker/devnet/lotus/Dockerfile new file mode 100644 index 000000000..94fc9037d --- /dev/null +++ b/docker/devnet/lotus/Dockerfile @@ -0,0 +1,32 @@ +ARG LOTUS_TEST_IMAGE=filecoin/lotus-test:latest +############################################################################# +FROM ${LOTUS_TEST_IMAGE} + +ARG BUILD_VERSION=0.1 + +LABEL org.opencontainers.image.version=$BUILD_VERSION \ + org.opencontainers.image.authors="Boost Dev Team" \ + name="lotus-dev" \ + maintainer="Boost Dev Team" \ + vendor="Boost Dev Team" \ + version=$BUILD_VERSION \ + release=$BUILD_VERSION \ + summary="This image is used to host the lotus dev service" \ + description="This image is used to host the lotus dev service" + +EXPOSE 1234 +ENV LOTUS_SKIP_GENESIS_CHECK=_yes_ +ENV GENESIS_PATH=/var/lib/genesis +ENV SECTOR_SIZE=8388608 + +VOLUME /var/tmp/filecoin-proof-parameters +VOLUME /var/lib/genesis + +WORKDIR /app +RUN mkdir -p /app + +COPY entrypoint.sh /app + +USER root + +ENTRYPOINT ["./entrypoint.sh"] diff --git a/docker/devnet/lotus/Makefile b/docker/devnet/lotus/Makefile new file mode 100644 index 000000000..8ac9eb49a --- /dev/null +++ b/docker/devnet/lotus/Makefile @@ -0,0 +1,16 @@ +##################################################################################### +service=$(docker_user)/lotus-dev +version?=$(lotus_version) +########### DOCKER ################################################################## +tag=$(service):$(version) +dbuild: + docker build -t $(tag) --build-arg LOTUS_TEST_IMAGE=$(lotus_test_image) --build-arg BUILD_VERSION=$(version) . + +dpush: dbuild + docker push $(tag) + +dscan: dbuild + docker scan --accept-license $(tag) +##################################################################################### +.PHONY: + dbuild dpush dscan diff --git a/docker/devnet/lotus/entrypoint.sh b/docker/devnet/lotus/entrypoint.sh new file mode 100755 index 000000000..8c2552275 --- /dev/null +++ b/docker/devnet/lotus/entrypoint.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -e +if [ ! -f $LOTUS_PATH/.init.params ]; then + echo Initializing fetch params ... + lotus fetch-params $SECTOR_SIZE + touch $LOTUS_PATH/.init.params + echo Done +fi + +if [ ! -f $LOTUS_PATH/.init.genesis ]; then + echo Initializing pre seal ... + lotus-seed --sector-dir $GENESIS_PATH pre-seal --sector-size $SECTOR_SIZE --num-sectors 1 + echo Initializing genesis ... + lotus-seed --sector-dir $GENESIS_PATH genesis new $LOTUS_PATH/localnet.json + echo Initializing address ... + lotus-seed --sector-dir $GENESIS_PATH genesis add-miner $LOTUS_PATH/localnet.json $GENESIS_PATH/pre-seal-t01000.json + touch $LOTUS_PATH/.init.genesis + echo Done +fi + +echo Starting lotus deamon ... +exec lotus daemon --lotus-make-genesis=$LOTUS_PATH/devgen.car --genesis-template=$LOTUS_PATH/localnet.json --bootstrap=false diff --git a/docker/monitoring/.gitignore b/docker/monitoring/.gitignore new file mode 100644 index 000000000..1aed7719e --- /dev/null +++ b/docker/monitoring/.gitignore @@ -0,0 +1 @@ +tempo-data diff --git a/docker/monitoring/docker-compose.yaml b/docker/monitoring/docker-compose.yaml new file mode 100644 index 000000000..75dff22e0 --- /dev/null +++ b/docker/monitoring/docker-compose.yaml @@ -0,0 +1,67 @@ +version: "3.8" + +networks: + default: + name: monitoring + +services: + tempo: + container_name: tempo + image: grafana/tempo:latest + command: [ "-search.enabled=true", "-config.file=/etc/tempo.yaml" ] + volumes: + - ./tempo-local.yaml:/etc/tempo.yaml + - ./overrides.yaml:/etc/overrides.yaml + - ./tempo-data:/tmp/tempo + ports: + - "14268:14268" # jaeger ingest + logging: + driver: loki + options: + loki-url: 'http://localhost:3100/loki/api/v1/push' + + loki: + container_name: loki + image: grafana/loki:2.3.0 + command: [ "-config.file=/etc/loki/local-config.yaml" ] + ports: + - "3100:3100" # loki needs to be exposed so it receives logs + environment: + - JAEGER_AGENT_HOST=tempo + - JAEGER_ENDPOINT=http://tempo:14268/api/traces # send traces to Tempo + - JAEGER_SAMPLER_TYPE=const + - JAEGER_SAMPLER_PARAM=1 + logging: + driver: loki + options: + loki-url: 'http://localhost:3100/loki/api/v1/push' + + prometheus: + container_name: prometheus + image: prom/prometheus:latest + command: [ "--config.file=/etc/prometheus.yaml" ] + volumes: + - ./prometheus.yaml:/etc/prometheus.yaml + ports: + - "9090:9090" + logging: + driver: loki + options: + loki-url: 'http://localhost:3100/loki/api/v1/push' + + grafana: + container_name: grafana + image: grafana/grafana:main + volumes: + - ./grafana.ini:/etc/grafana/grafana.ini + - ./grafana-datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml + #environment: + #- GF_AUTH_ANONYMOUS_ENABLED=true + #- GF_AUTH_ANONYMOUS_ORG_ROLE=Admin + #- GF_AUTH_DISABLE_LOGIN_FORM=true + ports: + - "3333:3000" + logging: + driver: loki + options: + loki-url: 'http://localhost:3100/loki/api/v1/push' diff --git a/docker/monitoring/grafana-datasources.yaml b/docker/monitoring/grafana-datasources.yaml new file mode 100644 index 000000000..1a34fc7ea --- /dev/null +++ b/docker/monitoring/grafana-datasources.yaml @@ -0,0 +1,39 @@ +apiVersion: 1 + +datasources: +- name: Prometheus + type: prometheus + access: proxy + orgId: 1 + url: http://prometheus:9090 + basicAuth: false + isDefault: false + version: 1 + editable: false +- name: Tempo + type: tempo + access: proxy + orgId: 1 + url: http://tempo:3200 + basicAuth: false + isDefault: true + version: 1 + editable: false + apiVersion: 1 + uid: tempo +- name: Loki + type: loki + access: proxy + orgId: 1 + url: http://loki:3100 + basicAuth: false + isDefault: false + version: 1 + editable: false + apiVersion: 1 + jsonData: + derivedFields: + - datasourceUid: tempo + matcherRegex: (?:traceID|trace_id)=(\w+) + name: TraceID + url: $${__value.raw} \ No newline at end of file diff --git a/docker/monitoring/grafana.ini b/docker/monitoring/grafana.ini new file mode 100644 index 000000000..8da2d7db4 --- /dev/null +++ b/docker/monitoring/grafana.ini @@ -0,0 +1,3 @@ +[feature_toggles] +enable = tempoSearch tempoBackendSearch + diff --git a/docker/monitoring/overrides.yaml b/docker/monitoring/overrides.yaml new file mode 100644 index 000000000..a07dcff49 --- /dev/null +++ b/docker/monitoring/overrides.yaml @@ -0,0 +1,14 @@ +overrides: + "single-tenant": + search_tags_allow_list: + - "instance" + ingestion_rate_strategy: "local" + ingestion_rate_limit_bytes: 15000000 + ingestion_burst_size_bytes: 20000000 + max_traces_per_user: 10000 + max_global_traces_per_user: 0 + max_bytes_per_trace: 50000 + max_search_bytes_per_trace: 0 + max_bytes_per_tag_values_query: 5000000 + block_retention: 0s + diff --git a/docker/monitoring/prometheus.yaml b/docker/monitoring/prometheus.yaml new file mode 100644 index 000000000..2ac68598a --- /dev/null +++ b/docker/monitoring/prometheus.yaml @@ -0,0 +1,11 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + +scrape_configs: + - job_name: 'prometheus' + static_configs: + - targets: [ 'localhost:9090' ] + - job_name: 'tempo' + static_configs: + - targets: [ 'tempo:3200' ] \ No newline at end of file diff --git a/docker/monitoring/tempo-local.yaml b/docker/monitoring/tempo-local.yaml new file mode 100644 index 000000000..3abc9c2a5 --- /dev/null +++ b/docker/monitoring/tempo-local.yaml @@ -0,0 +1,65 @@ +metrics_generator_enabled: true + +server: + http_listen_port: 3200 + +distributor: + search_tags_deny_list: + - "instance" + - "version" + receivers: # this configuration will listen on all ports and protocols that tempo is capable of. + jaeger: # the receives all come from the OpenTelemetry collector. more configuration information can + protocols: # be found there: https://github.com/open-telemetry/opentelemetry-collector/tree/main/receiver + thrift_http: # + grpc: # for a production deployment you should only enable the receivers you need! + thrift_binary: + thrift_compact: + zipkin: + otlp: + protocols: + http: + grpc: + opencensus: + +ingester: + trace_idle_period: 10s # the length of time after a trace has not received spans to consider it complete and flush it + max_block_bytes: 1_000_000 # cut the head block when it hits this size or ... + max_block_duration: 5m # this much time passes + +compactor: + compaction: + compaction_window: 1h # blocks in this time window will be compacted together + max_block_bytes: 100_000_000 # maximum size of compacted blocks + block_retention: 1h + compacted_block_retention: 10m + +metrics_generator: + registry: + external_labels: + source: tempo + cluster: docker-compose + storage: + path: /tmp/tempo/generator/wal + remote_write: + - url: http://prometheus:9090/api/v1/write + send_exemplars: true + +storage: + trace: + backend: local # backend configuration to use + block: + bloom_filter_false_positive: .05 # bloom filter false positive rate. lower values create larger filters but fewer false positives + index_downsample_bytes: 1000 # number of bytes per index record + encoding: zstd # block encoding/compression. options: none, gzip, lz4-64k, lz4-256k, lz4-1M, lz4, snappy, zstd, s2 + wal: + path: /tmp/tempo/wal # where to store the the wal locally + encoding: snappy # wal encoding/compression. options: none, gzip, lz4-64k, lz4-256k, lz4-1M, lz4, snappy, zstd, s2 + local: + path: /tmp/tempo/blocks + pool: + max_workers: 100 # worker pool determines the number of parallel requests to the object store backend + queue_depth: 10000 + +overrides: + per_tenant_override_config: /etc/overrides.yaml + metrics_generator_processors: [service-graphs, span-metrics] diff --git a/documentation/devnet.md b/documentation/devnet.md deleted file mode 100644 index aedea9424..000000000 --- a/documentation/devnet.md +++ /dev/null @@ -1,181 +0,0 @@ -# Devnet - -To run Boost on your development machine, you will need to set up a devnet: - -1. Remove any existing lotus repo and boost repo -``` -rm -rf ~/.lotusmarkets && rm -rf ~/.lotus && rm -rf ~/.lotusminer && rm -rf ~/.genesis_sectors -rm -rf ~/.boost -``` - -2. Build lotus in debug mode - -The version of lotus needs to match the version in boost's go.mod -``` -cd lotus -git checkout -make debug -``` - -3. Install lotus - -The devnet script uses the installed lotus to run the miner and daemon. -``` -make install -install -C ./lotus-seed /usr/local/bin/lotus-seed -``` - -4. Build boost in debug mode - -Double check if environment variables are set: -``` -export LIBRARY_PATH=/opt/homebrew/lib -export PATH="$(brew --prefix coreutils)/libexec/gnubin:/usr/local/bin:$PATH" -``` - -Build & Install -``` -cd boost -make debug -make install -``` - -3. Start the devnet - -The following command will use the lotus binaries that you built and installed above, and especially it will run `lotus`, `lotus-miner` and `lotus-seed`. So the lotus version must match the version in boost's go.mod. -``` -cd boost -./devnet -``` - -The first time you run it, it will install a lot of metadata-related proofs. It will take at least 30 minutes depending on your connection speed. You may need to restart the command multiple times as your terminal will probably timeout before it finishes downloading everything. - -You can also use `./devnet &` instead of `./devnet` to run the process in the background - that would prevent you from having to manually restart the command when your terminal times out. - -NOTE: -The devnet isn't designed to be restartable unfortunately. After it has been successfully run once, you'll have to clear out the previous data before re-running `./devnet`: -``` -rm -rf ~/.lotusmarkets && rm -rf ~/.lotus && rm -rf ~/.lotusminer && rm -rf ~/.genesis_sectors -``` - -4. Wait for lotus-miner to come up (through the command above) - -Unset these variables as they interfere with the `lotus-miner` command. -``` -unset MINER_API_INFO -unset FULLNODE_API_INFO -``` -Then repeatedly run this command until it succeeds: -``` -lotus-miner auth api-info --perm=admin -``` - -5. Get the auth tokens to connect to the lotus daemon and miner: -``` -export ENV_MINER_API_INFO=`lotus-miner auth api-info --perm=admin` -export ENV_FULLNODE_API_INFO=`lotus auth api-info --perm=admin` - -export MINER_API_INFO=`echo $ENV_MINER_API_INFO | awk '{split($0,a,"="); print a[2]}'` -export FULLNODE_API_INFO=`echo $ENV_FULLNODE_API_INFO | awk '{split($0,a,"="); print a[2]}'` - -echo MINER_API_INFO=$MINER_API_INFO -echo FULLNODE_API_INFO=$FULLNODE_API_INFO -``` - -6. Create the wallets needed for boost -``` -export DEFAULT_WALLET=`lotus wallet list | tail -1 | awk '{print $1}'` -export COLLAT_WALLET=`lotus wallet new bls` -export PUBMSG_WALLET=`lotus wallet new bls` -export CLIENT_WALLET=`lotus wallet new bls` -``` - -7. Add funds to the wallets -``` -lotus send --from $DEFAULT_WALLET $COLLAT_WALLET 10 -lotus send --from $DEFAULT_WALLET $PUBMSG_WALLET 10 -lotus send --from $DEFAULT_WALLET $CLIENT_WALLET 10 -``` - -Run this command repeatedly until each wallet you created has 10 FIL: -``` -lotus wallet list -``` -This should take about 10 seconds. - -8. Set the Publish Message Wallet as a control address on the miner -``` -lotus-miner actor control set --really-do-it $PUBMSG_WALLET -``` - -9. Add funds into the Market Actor escrow for the client and Collateral wallets -``` -lotus wallet market add --from $DEFAULT_WALLET --address $CLIENT_WALLET 5 -lotus wallet market add --address $COLLAT_WALLET 5 -``` - -10. Initialize boost -``` -boostd -vv init \ - --api-sealer=$MINER_API_INFO \ - --api-sector-index=$MINER_API_INFO \ - --wallet-publish-storage-deals=$PUBMSG_WALLET \ - --wallet-deal-collateral=$COLLAT_WALLET \ - --max-staging-deals-bytes=2000000000 -``` - -11. Build the Web UI -``` -make react -``` - -12. Edit config to set a fixed listen address - -Open `~/.boost/config.toml` - -Set the port in the `ListenAddresses` key to `50000` -``` -[Libp2p] - ListenAddresses = ["/ip4/0.0.0.0/tcp/50000", "/ip6/::/tcp/0"] -``` - -13. Run boost -``` -boostd -vv run -``` - -Note down the peer ID of the boost instance: -``` -2022-06-10T09:32:28.819Z INFO boostd boostd/run.go:114 Boost libp2p node listening {"maddr": "{12D3KooWQNNWNiJ1mieEk9EHjDVF2qBc1FSjJGEzwjnMJzteApaW: [/ip4/172.17.0.2/tcp/50000 /ip4/127.0.0.1/tcp/50000]}"} -``` -In this example: `12D3KooWQNNWNiJ1mieEk9EHjDVF2qBc1FSjJGEzwjnMJzteApaW` - -14. Set the peer ID and multi-address of the miner on chain -``` -lotus-miner actor set-peer-id -lotus-miner actor set-addrs /ip4/127.0.0.1/tcp/50000 -``` - -15. Open the Web UI - -Open http://localhost:8080 to see the Boost UI - -## Make a deal with Boost - -2. Initialize the boost client -``` -boost init -``` - -This will output the address of the wallet (it's safe to run the init command repeatedly). - -2. Send funds to the client wallet -``` -lotus send --from=$DEFAULT_WALLET 10 -``` - -3. Follow the guide at https://boost.filecoin.io/tutorials/how-to-store-files-with-boost-on-filecoin - -Note that above you already ran a command to export FULLNODE_API (and point it to your local devnet lotus daemon). - -Note also that the provider address is `t01000` and you will need to supply an appropriate `--storage-price` when using `boost deal` since the devnet has a minimum price. Alternatively, using "Settings" in the Boost web UI to set the deal price to zero. diff --git a/go.mod b/go.mod index 53fe09eed..47023beb6 100644 --- a/go.mod +++ b/go.mod @@ -98,11 +98,19 @@ require ( github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc github.com/whyrusleeping/cbor-gen v0.0.0-20220323183124-98fa8256a799 go.opencensus.io v0.23.0 + go.opentelemetry.io/otel v1.9.0 + go.opentelemetry.io/otel/bridge/opencensus v0.31.0 + go.opentelemetry.io/otel/exporters/jaeger v1.2.0 + go.opentelemetry.io/otel/sdk v1.9.0 + go.opentelemetry.io/otel/trace v1.9.0 go.uber.org/atomic v1.9.0 go.uber.org/fx v1.15.0 go.uber.org/multierr v1.8.0 + golang.org/x/exp v0.0.0-20220426173459-3bcf042a4bf5 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c golang.org/x/tools v0.1.11 golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f + google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1 // indirect + google.golang.org/grpc v1.46.2 // indirect gopkg.in/cheggaaa/pb.v1 v1.0.28 ) diff --git a/go.sum b/go.sum index 24e7fffea..a2b8bebe3 100644 --- a/go.sum +++ b/go.sum @@ -215,6 +215,7 @@ github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XP github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= @@ -334,6 +335,7 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= +github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/etclabscore/go-jsonschema-walk v0.0.6 h1:DrNzoKWKd8f8XB5nFGBY00IcjakRE22OTI12k+2LkyY= github.com/etclabscore/go-jsonschema-walk v0.0.6/go.mod h1:VdfDY72AFAiUhy0ZXEaWSpveGjMT5JcDIm903NGqFwQ= @@ -2066,6 +2068,7 @@ github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5J github.com/streadway/handy v0.0.0-20200128134331-0f66f006fb2e/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -2230,24 +2233,37 @@ go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= go.opentelemetry.io/otel v1.2.0/go.mod h1:aT17Fk0Z1Nor9e0uisf98LrntPGMnk4frBO9+dkf69I= go.opentelemetry.io/otel v1.3.0/go.mod h1:PWIKzi6JCp7sM0k9yZ43VX+T345uNbAkDKwHVjb2PTs= -go.opentelemetry.io/otel v1.7.0 h1:Z2lA3Tdch0iDcrhJXDIlC94XE+bxok1F9B+4Lz/lGsM= go.opentelemetry.io/otel v1.7.0/go.mod h1:5BdUoMIz5WEs0vt0CUEMtSSaTSHBBVwrhnz7+nrD5xk= +go.opentelemetry.io/otel v1.8.0/go.mod h1:2pkj+iMj0o03Y+cW6/m8Y4WkRdYN3AvCXCnzRMp9yvM= +go.opentelemetry.io/otel v1.9.0 h1:8WZNQFIB2a71LnANS9JeyidJKKGOOremcUtb/OtHISw= +go.opentelemetry.io/otel v1.9.0/go.mod h1:np4EoPGzoPs3O67xUVNoPPcmSvsfOxNlNA4F4AC+0Eo= go.opentelemetry.io/otel/bridge/opencensus v0.25.0/go.mod h1:dkZDdaNwLlIutxK2Kc2m3jwW2M1ISaNf8/rOYVwuVHs= +go.opentelemetry.io/otel/bridge/opencensus v0.31.0 h1:8Wk5/hHtEdTpYwUz0Ny8I+ccqfIMbp8Ce8/ZXlN4WPc= +go.opentelemetry.io/otel/bridge/opencensus v0.31.0/go.mod h1:hgI/HZyVIo3QL0RBmlOy2/VIQCc27OC1c/4ifwqTv4A= +go.opentelemetry.io/otel/exporters/jaeger v1.2.0 h1:C/5Egj3MJBXRJi22cSl07suqPqtZLnLFmH//OxETUEc= go.opentelemetry.io/otel/exporters/jaeger v1.2.0/go.mod h1:KJLFbEMKTNPIfOxcg/WikIozEoKcPgJRz3Ce1vLlM8E= go.opentelemetry.io/otel/internal/metric v0.25.0/go.mod h1:Nhuw26QSX7d6n4duoqAFi5KOQR4AuzyMcl5eXOgwxtc= go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= go.opentelemetry.io/otel/metric v0.25.0/go.mod h1:E884FSpQfnJOMMUaq+05IWlJ4rjZpk2s/F1Ju+TEEm8= +go.opentelemetry.io/otel/metric v0.31.0 h1:6SiklT+gfWAwWUR0meEMxQBtihpiEs4c+vL9spDTqUs= +go.opentelemetry.io/otel/metric v0.31.0/go.mod h1:ohmwj9KTSIeBnDBm/ZwH2PSZxZzoOaG2xZeekTRzL5A= go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw= go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= -go.opentelemetry.io/otel/sdk v1.2.0 h1:wKN260u4DesJYhyjxDa7LRFkuhH7ncEVKU37LWcyNIo= go.opentelemetry.io/otel/sdk v1.2.0/go.mod h1:jNN8QtpvbsKhgaC6V5lHiejMoKD+V8uadoSafgHPx1U= +go.opentelemetry.io/otel/sdk v1.8.0/go.mod h1:uPSfc+yfDH2StDM/Rm35WE8gXSNdvCg023J6HeGNO0c= +go.opentelemetry.io/otel/sdk v1.9.0 h1:LNXp1vrr83fNXTHgU8eO89mhzxb/bbWAsHG6fNf3qWo= +go.opentelemetry.io/otel/sdk v1.9.0/go.mod h1:AEZc8nt5bd2F7BC24J5R0mrjYnpEgYHyTcM/vrSple4= go.opentelemetry.io/otel/sdk/export/metric v0.25.0/go.mod h1:Ej7NOa+WpN49EIcr1HMUYRvxXXCCnQCg2+ovdt2z8Pk= go.opentelemetry.io/otel/sdk/metric v0.25.0/go.mod h1:G4xzj4LvC6xDDSsVXpvRVclQCbofGGg4ZU2VKKtDRfg= +go.opentelemetry.io/otel/sdk/metric v0.31.0 h1:2sZx4R43ZMhJdteKAlKoHvRgrMp53V1aRxvEf5lCq8Q= +go.opentelemetry.io/otel/sdk/metric v0.31.0/go.mod h1:fl0SmNnX9mN9xgU6OLYLMBMrNAsaZQi7qBwprwO3abk= go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= go.opentelemetry.io/otel/trace v1.2.0/go.mod h1:N5FLswTubnxKxOJHM7XZC074qpeEdLy3CgAVsdMucK0= go.opentelemetry.io/otel/trace v1.3.0/go.mod h1:c/VDhno8888bvQYmbYLqe41/Ldmr/KKunbvWM4/fEjk= -go.opentelemetry.io/otel/trace v1.7.0 h1:O37Iogk1lEkMRXewVtZ1BBTVn5JEp8GrJvP92bJqC6o= go.opentelemetry.io/otel/trace v1.7.0/go.mod h1:fzLSB9nqR2eXzxPXb2JW9IKE+ScyXA48yyE4TNvoHqU= +go.opentelemetry.io/otel/trace v1.8.0/go.mod h1:0Bt3PXY8w+3pheS3hQUt+wow8b1ojPaTBoTCh2zIFI4= +go.opentelemetry.io/otel/trace v1.9.0 h1:oZaCNJUjWcg60VXWee8lJKlqhPbXAPB51URuR47pQYc= +go.opentelemetry.io/otel/trace v1.9.0/go.mod h1:2737Q0MuG8q1uILYm2YYVkAyLtOofiTNGg6VODnOiPo= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= @@ -2784,8 +2800,9 @@ google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210917145530-b395a37504d4 h1:ysnBoUyeL/H6RCvNRhWHjKoDEmguI+mPU+qHgK8qv/w= google.golang.org/genproto v0.0.0-20210917145530-b395a37504d4/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1 h1:b9mVrqYfq3P4bCdaLg1qtBnPzUYgglsIdjZkL/fQVOE= +google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= @@ -2813,8 +2830,9 @@ google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.45.0 h1:NEpgUqV3Z+ZjkqMsxMg11IaDrXY4RY6CQukSGK0uI1M= google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= +google.golang.org/grpc v1.46.2 h1:u+MLGgVf7vRdjEYZ8wDFhAVNmhkbJ5hmrA1LMWK1CAQ= +google.golang.org/grpc v1.46.2/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= diff --git a/gql/resolver.go b/gql/resolver.go index 4884fda73..2cabdbbec 100644 --- a/gql/resolver.go +++ b/gql/resolver.go @@ -102,7 +102,7 @@ func (r *resolver) Deal(ctx context.Context, args struct{ ID graphql.ID }) (*dea return nil, err } - return newDealResolver(deal, r.dealsDB, r.logsDB, r.spApi), nil + return newDealResolver(deal, r.provider, r.dealsDB, r.logsDB, r.spApi), nil } type dealsArgs struct { @@ -135,7 +135,7 @@ func (r *resolver) Deals(ctx context.Context, args dealsArgs) (*dealListResolver resolvers := make([]*dealResolver, 0, len(deals)) for _, deal := range deals { - resolvers = append(resolvers, newDealResolver(&deal, r.dealsDB, r.logsDB, r.spApi)) + resolvers = append(resolvers, newDealResolver(&deal, r.provider, r.dealsDB, r.logsDB, r.spApi)) } return &dealListResolver{ @@ -168,7 +168,7 @@ func (r *resolver) DealUpdate(ctx context.Context, args struct{ ID graphql.ID }) } net := make(chan *dealResolver, 1) - net <- newDealResolver(deal, r.dealsDB, r.logsDB, r.spApi) + net <- newDealResolver(deal, r.provider, r.dealsDB, r.logsDB, r.spApi) // Updates to deal state are broadcast on pubsub. Pipe these updates to the // client @@ -180,7 +180,7 @@ func (r *resolver) DealUpdate(ctx context.Context, args struct{ ID graphql.ID }) } return nil, fmt.Errorf("%s: subscribing to deal updates: %w", args.ID, err) } - sub := &subLastUpdate{sub: dealUpdatesSub, dealsDB: r.dealsDB, logsDB: r.logsDB, spApi: r.spApi} + sub := &subLastUpdate{sub: dealUpdatesSub, provider: r.provider, dealsDB: r.dealsDB, logsDB: r.logsDB, spApi: r.spApi} go func() { sub.Pipe(ctx, net) // blocks until connection is closed close(net) @@ -219,7 +219,7 @@ func (r *resolver) DealNew(ctx context.Context) (<-chan *dealNewResolver, error) case evti := <-sub.Out(): // Pipe the deal to the new deal channel di := evti.(types.ProviderDealState) - rsv := newDealResolver(&di, r.dealsDB, r.logsDB, r.spApi) + rsv := newDealResolver(&di, r.provider, r.dealsDB, r.logsDB, r.spApi) totalCount, err := r.dealsDB.Count(ctx, "") if err != nil { log.Errorf("getting total deal count: %w", err) @@ -330,15 +330,17 @@ func (r *resolver) dealList(ctx context.Context, query string, cursor *graphql.I type dealResolver struct { types.ProviderDealState + provider *storagemarket.Provider transferred uint64 dealsDB *db.DealsDB logsDB *db.LogsDB spApi sealingpipeline.API } -func newDealResolver(deal *types.ProviderDealState, dealsDB *db.DealsDB, logsDB *db.LogsDB, spApi sealingpipeline.API) *dealResolver { +func newDealResolver(deal *types.ProviderDealState, provider *storagemarket.Provider, dealsDB *db.DealsDB, logsDB *db.LogsDB, spApi sealingpipeline.API) *dealResolver { return &dealResolver{ ProviderDealState: *deal, + provider: provider, transferred: uint64(deal.NBytesReceived), dealsDB: dealsDB, logsDB: logsDB, @@ -502,13 +504,17 @@ func (dr *dealResolver) message(ctx context.Context, checkpoint dealcheckpoints. if dr.IsOffline { return "Awaiting Offline Data Import" } - switch dr.transferred { - case 0: + switch { + case dr.transferred == 0 && !dr.provider.IsTransferStalled(dr.DealUuid): return "Transfer Queued" - case 100: + case dr.transferred == 100: return "Transfer Complete" default: pct := (100 * dr.transferred) / dr.ProviderDealState.Transfer.Size + isStalled := dr.provider.IsTransferStalled(dr.DealUuid) + if isStalled { + return fmt.Sprintf("Transfer stalled at %d%% ", pct) + } return fmt.Sprintf("Transferring %d%%", pct) } case dealcheckpoints.Transferred: @@ -533,6 +539,22 @@ func (dr *dealResolver) message(ctx context.Context, checkpoint dealcheckpoints. return checkpoint.String() } +func (dr *dealResolver) TransferSamples() []*transferPoint { + points := dr.provider.Transfer(dr.DealUuid) + pts := make([]*transferPoint, 0, len(points)) + for _, pt := range points { + pts = append(pts, &transferPoint{ + At: graphql.Time{Time: pt.At}, + Bytes: gqltypes.Uint64(pt.Bytes), + }) + } + return pts +} + +func (dr *dealResolver) IsTransferStalled() bool { + return dr.provider.IsTransferStalled(dr.DealUuid) +} + func (dr *dealResolver) sealingState(ctx context.Context) string { si, err := dr.spApi.SectorsStatus(ctx, dr.SectorID, false) if err != nil { @@ -594,10 +616,11 @@ func toUuid(id graphql.ID) (uuid.UUID, error) { } type subLastUpdate struct { - sub event.Subscription - dealsDB *db.DealsDB - logsDB *db.LogsDB - spApi sealingpipeline.API + sub event.Subscription + provider *storagemarket.Provider + dealsDB *db.DealsDB + logsDB *db.LogsDB + spApi sealingpipeline.API } func (s *subLastUpdate) Pipe(ctx context.Context, net chan *dealResolver) { @@ -636,7 +659,7 @@ func (s *subLastUpdate) Pipe(ctx context.Context, net chan *dealResolver) { loop: for { di := lastUpdate.(types.ProviderDealState) - rsv := newDealResolver(&di, s.dealsDB, s.logsDB, s.spApi) + rsv := newDealResolver(&di, s.provider, s.dealsDB, s.logsDB, s.spApi) select { case <-ctx.Done(): diff --git a/gql/resolver_transfers.go b/gql/resolver_transfers.go index a4c7091a3..e7a20f306 100644 --- a/gql/resolver_transfers.go +++ b/gql/resolver_transfers.go @@ -6,6 +6,8 @@ import ( "time" gqltypes "github.com/filecoin-project/boost/gql/types" + "github.com/filecoin-project/boost/storagemarket" + "github.com/google/uuid" "github.com/graph-gophers/graphql-go" ) @@ -15,15 +17,62 @@ type transferPoint struct { } // query: transfers: [TransferPoint] -func (r *resolver) Transfers(_ context.Context) ([]*transferPoint, error) { - deals := r.provider.Transfers() +func (r *resolver) Transfers(_ context.Context) []*transferPoint { + return r.getTransferSamples(r.provider.Transfers(), nil) +} + +type transferStats struct { + HttpMaxConcurrentDownloads int32 + Stats []*hostTransferStats +} + +type hostTransferStats struct { + Host string + Total int32 + Started int32 + Stalled int32 + TransferSamples []*transferPoint +} + +// query: transferStats: TransferStats +func (r *resolver) TransferStats(_ context.Context) *transferStats { + transfersByDeal := r.provider.Transfers() + stats := r.provider.TransferStats() + gqlStats := make([]*hostTransferStats, 0, len(stats)) + for _, s := range stats { + gqlStats = append(gqlStats, &hostTransferStats{ + Host: s.Host, + Total: int32(s.Total), + Started: int32(s.Started), + Stalled: int32(s.Stalled), + TransferSamples: r.getTransferSamples(transfersByDeal, s.DealUuids), + }) + } + return &transferStats{ + HttpMaxConcurrentDownloads: int32(r.cfg.Dealmaking.HttpTransferMaxConcurrentDownloads), + Stats: gqlStats, + } +} + +func (r *resolver) getTransferSamples(deals map[uuid.UUID][]storagemarket.TransferPoint, filter []uuid.UUID) []*transferPoint { + // If filter is nil, include all deals + if filter == nil { + for dealUuid := range deals { + filter = append(filter, dealUuid) + } + } // We have // dealUUID -> [At: