diff --git a/docs/2.develop/lake/lake-primitives.md b/docs/2.develop/lake/lake-primitives.md new file mode 100644 index 00000000000..392027584a0 --- /dev/null +++ b/docs/2.develop/lake/lake-primitives.md @@ -0,0 +1,369 @@ +--- +id: primitives +title: NEAR Lake Primitive Types +sidebar_label: Lake Primitive Types +--- + +# NEAR Lake Primitive Types + +This article contains the primitive types used by the [NEAR Lake Framework package](https://www.npmjs.com/package/@near-lake/framework). These types are used to define the data structures used by the framework as well as provide some popular helper functions. + +## `Block` + +:::info Important Notes on `Block` + +- All the entities located on different shards were merged into one single list without differentiation. +- `Block` is not the fairest name for this structure either. NEAR Protocol is a sharded blockchain, so its block is actually an ephemeral structure that represents a collection of real blocks called chunks in NEAR Protocol. + +::: + +### `Block` Structure Definition + +The `Block` type is used to represent a block in the NEAR Lake Framework. It is comprised by the following structure: + +```ts +export class Block { + constructor( + readonly streamerMessage: StreamerMessage, + private executedReceipts: Receipt[], + readonly postponedReceipts: Receipt[], + readonly transactions: Transaction[], + private _actions: Map, + private _events: Map, + private _stateChanges: StateChange[]) { + + } + ... // helper methods and getters omitted for brevity +} +``` + +#### `streamerMessage` + +Low-level structure for backward compatibility. As implemented in previous versions of [`near-lake-framework`](https://www.npmjs.com/package/near-lake-framework). + +#### `postponedReceipts` + +Receipts included on the chain but not executed yet marked as “postponed”: they are represented by the same structure `Receipt` (see the corresponding section in this doc for more details). + +#### `transactions` + +List of included `Transactions`, converted into `Receipts`. + +:::info Heads up! + +**Note:** You might want to know about `Transactions` to know where the action chain has begun. Unlike Ethereum, where a Transaction contains everything you may want to know about a particular interaction on the Ethereum blockchain, Near Protocol because of its asynchronous nature converts a `Transaction` into a `Receipt` before executing it. Thus, On NEAR, `Receipts` are more important for figuring out what happened on-chain as a result of a Transaction signed by a user. Read more about [Transactions on Near](https://nomicon.io/RuntimeSpec/Transactions) here. + +::: + +### `Block` Helper Methods + +```ts +export class Block { + ... // constructor omitted for brevity + get blockHash(): string {} + get prevBlockHash(): string {} + get blockHeight(): number {} + + header(): BlockHeader {} + receipts(): Receipt[] {} + actions(): Action[] {} + events(): Event[] {} + stateChanges(): StateChange[] {} + + actionByReceiptId(receipt_id: string): Action | undefined {} + eventsByReceiptId(receipt_id: string): Event[] {} + eventsByAccountId(account_id: string): Event[] {} + + private buildActionsHashmap() {} + private buildEventsHashmap(): Map {} + + static fromStreamerMessage(streamerMessage: StreamerMessage): Block {} +} +``` + +#### `blockHash` + +Returns the block hash. A shortcut to get the data from the block header. + +#### `prevBlockHash` + +Returns the previous block hash. A shortcut to get the data from the block header. + +#### `blockHeight` + +Returns the block height. A shortcut to get the data from the block header. + +#### `header(): BlockHeader` + +Returns a `BlockHeader` structure of the block + +See `BlockHeader` structure sections for details. + +#### `receipts(): Receipt[]` + +Returns a slice of `Receipts` executed in the block. + +Basically is a getter for the `executedReceipts` field. + +#### `actions(): Action[]` + +Returns an Array of `Actions` executed in the block. + +#### `events(): Event[]` + +Returns `Events` emitted in the block. + +#### `stateChanges(): StateChange[]` + +Returns an Array of `StateChange` occurred in the block. + +#### `actionByReceiptId(receipt_id: string): Action | undefined` + +Returns `Action`s of the provided `receipt_id` from the block if any. Returns `undefined` if there is no corresponding `Action`. + +This method uses the internal `Block` `action` field which is empty by default and will be filled with the block’s actions on the first call to optimize memory usage. + +The result is either `Action | undefined` since there might be a request for an `Action` by `receipt_id` from another block, in which case this method will be unable to find the `Action` in the current block. In the other case, the request might be for an `Action` for a `receipt_id` that belongs to a `DataReceipt` where an action does not exist. + +#### `eventsByReceiptId(receipt_id: string): Event[]` + +Returns an Array of Events emitted by `ExecutionOutcome` for the given `receipt_id`. There might be more than one `Event` for the `Receipt` or there might be none of them. In the latter case, this method returns an empty Array. + +#### `eventsByAccountId(account_id: string): Event[]` + +Returns an Array of Events emitted by `ExecutionOutcome` for the given `account_id`. There might be more than one `Event` for the `Receipt` or there might be none of them. In the latter case, this method returns an empty Array. + +--- + +## `BlockHeader` + +Replacement for `BlockHeaderView` from `near-primitives`. Shrunken and simplified. + +:::note +The original `BlockHeaderView` is still accessible via the `.streamerMessage` attribute. +::: + +### `BlockHeader` Structure Definition + +```ts +export class BlockHeader { + constructor( + readonly height: number, + readonly hash: string, + readonly prevHash: string, + readonly author: string, + readonly timestampNanosec: string, + readonly epochId: string, + readonly nextEpochId: string, + readonly gasPrice: string, + readonly totalSupply: string, + readonly latestProtocolVersion: number, + readonly randomValue: string, + readonly chunksIncluded: number, + readonly validatorProposals: ValidatorStakeView[]) { + } + ... // helper method omitted for brevity +} +``` + +--- + +## `Receipt` + +This field is a simplified representation of the `ReceiptView` structure from `near-primitives`. + +### `Receipt` Structure Definition + +```ts +export class Receipt implements Events { + constructor( + readonly receiptKind: ReceiptKind, + readonly receiptId: string, + readonly receiverId: string, + readonly predecessorId: string, + readonly status: ExecutionStatus, + readonly executionOutcomeId?: string | undefined, + readonly logs: string[] = []) { + } + ... // helper methods omitted for brevity +} +``` + +### `Receipt` Fields + +#### `receiptKind` + +Defined the type of the `Receipt`: `Action` or `Data` representing the `ActionReceipt` and `DataReceipt`. + +#### `receiptId` + +The ID of the `Receipt` of the `CryptoHash` type. + +#### `receiverId` + +The receiver account id of the `Receipt`. + +#### `predecessorId` + +The predecessor account id of the `Receipt`. + +#### `status` + +Represents the status of `ExecutionOutcome` of the `Receipt`. + +See the `ExecutionStatus` enum section for the details. + +#### `executionOutcomeId` + +The id of the `ExecutionOutcome` for the `Receipt`. Returns `null` if the `Receipt` isn’t executed yet and has a postponed status. + +#### `logs` + +The original logs of the corresponding `ExecutionOutcome` of the `Receipt`. + +Note: not all of the logs might be parsed as JSON Events (`Events`). + +### `Receipt` Helper Methods + +```ts +export class Receipt { + ... // constructor omitted for brevity + get events(): Event[] {} + + static fromOutcomeWithReceipt(outcomeWithReceipt: OutcomeWithReceipt): Receipt {} +} +``` + +#### `Receipt.events(): Events[]` + +Returns an Array of `Events` for the `Receipt`, if any. This might be empty if the `logs` field is empty or doesn’t contain JSON Events compatible log records. + +--- + +## `Event` + +This structure is an ephemeral entity to provide access to the [Events Standard](https://github.com/near/NEPs/blob/master/neps/nep-0297.md) structure and keep data about the related `Receipt` for convenience. + +### Interface for Capturing Data About an Event in `handleStreamerMessage()` + +The interface to capture data about an event has the following arguments: + +- `standard`: name of standard, e.g. nep171 +- `version`: e.g. 1.0.0 +- `event`: type of the event, e.g. nft_mint +- `data`: associate event data. Strictly typed for each set {standard, version, event} inside corresponding NEP + +### `Event` Structure Definition + +```ts +export class Event { + constructor( + readonly relatedReceiptId: string, + readonly rawEvent: RawEvent) { + } + ... // helper methods omitted for brevity +} +``` + +### `Event` Methods + +```ts +export class Event { + ... // constructor omitted for brevity + static fromLog(log: string): Event {} +} +``` + +--- + +## `Transaction` + +A representation of the `IndexerTransactionWithOutcome` from `near-indexer-primitives` which is an ephemeral structure combining `SignedTransactionView` from `near-primitives` and `IndexerExecutionOutcomeWithOptionalReceipt` from `near-indexer-primitives`. + +This structure is very similar to `Receipt`. Unlike `Receipt`, a `Transaction` has a few additional fields like `signerId`, `signature`, and `operations`. + +### `Transaction` Structure Definition + +```ts +export class Transaction { + constructor( + readonly transactionHash: string, + readonly signerId: string, + readonly signerPublicKey: string, + readonly signature: string, + readonly receiverId: string, + readonly status: ExecutionStatus, + readonly executionOutcomeId: string, + readonly operations: Operation[]) { + } +} +``` + +#### `Transaction.transactionHash` + +Returns the hash of the `Transaction` in `CryptoHash`. + +#### `Transaction.signerId` + +Returns the signer account id of the `Transaction`. + +#### `Transaction.signerPublicKey` + +Returns the `PublicKey` of the signer of the `Transaction`. + +#### `Transaction.signature` + +Returns the `Signature` the `Transaction` was signed with. + +#### `Transaction.receiverId` + +Returns the receiver account id of the `Transaction`. + +#### `Transaction.status` + +Returns the status of the `Transaction` as `ExecutionStatus`. + +#### `Transaction.executionOutcomeId` + +Returns the id of the `ExecutionOutcome` for the `Transaction`. + +#### `Transaction.operations` + +Returns an Array of `Operation` for the `Transaction`. + +--- + +## `StateChange` + +This structure is almost an identical copy of the `StateChangeWithCauseView` from `near-primitives` with a propagated additional field `affectedAccountId`. + +### `StateChange` Structure Definition + +```ts +export class StateChange { + constructor( + readonly cause: StateChangeCause, + readonly value: StateChangeValue + ) {} + + get affectedAccountId(): string {} + + static fromStateChangeView(stateChangeView: StateChangeWithCauseView) {} +} +``` + +#### `StateChange.cause` + +Returns the `cause` of the `StateChange`. + +#### `StateChange.value` + +Returns the `value` of the `StateChange`. + +#### `StateChange.affectedAccountId(): string` + +Returns the account id of the `StateChange`. + +#### `StateChange.fromStateChangeView(stateChangeView: StateChangeWithCauseView): StateChange` + +Returns the `StateChange` from the `StateChangeWithCauseView`. Created for backward compatibility. diff --git a/docs/2.develop/lake/structures/chunk.mdx b/docs/2.develop/lake/structures/chunk.mdx index c569d4fbb0f..35a2cc712ef 100644 --- a/docs/2.develop/lake/structures/chunk.mdx +++ b/docs/2.develop/lake/structures/chunk.mdx @@ -10,11 +10,11 @@ import TabItem from '@theme/TabItem'; ## Definition -`Chunk` of a [`Block`](./block.mdx) is a part of a [`Block`](./block.mdx) from a [Shard](./shard.mdx). The collection of Chunks of the Block forms the NEAR Protocol [`Block`](./block.mdx) +`Chunk` of a [`Block`](block.mdx) is a part of a [`Block`](block.mdx) from a [Shard](shard.mdx). The collection of Chunks of the Block forms the NEAR Protocol [`Block`](block.mdx) Chunk contains all the structures that make the Block: -- [Transactions](./transaction.mdx) -- [Receipts](./receipt.mdx) +- [Transactions](transaction.mdx) +- [Receipts](receipt.mdx) - [ChunkHeader](#chunkheaderview) ## `IndexerChunkView` diff --git a/docs/2.develop/lake/structures/execution_outcome.mdx b/docs/2.develop/lake/structures/execution_outcome.mdx index 8af9b9b1a6f..c43a2878c9b 100644 --- a/docs/2.develop/lake/structures/execution_outcome.mdx +++ b/docs/2.develop/lake/structures/execution_outcome.mdx @@ -11,7 +11,7 @@ import TabItem from '@theme/TabItem'; ## Definition -ExecutionOutcome is the result of execution of [Transaction](./transaction.mdx) or [Receipt](./receipt.mdx) +ExecutionOutcome is the result of execution of [Transaction](transaction.mdx) or [Receipt](receipt.mdx) :::info Transaction's ExecutionOutcome diff --git a/docs/2.develop/lake/structures/shard.mdx b/docs/2.develop/lake/structures/shard.mdx index 4b6223854b1..fe169faed5e 100644 --- a/docs/2.develop/lake/structures/shard.mdx +++ b/docs/2.develop/lake/structures/shard.mdx @@ -13,9 +13,9 @@ import TabItem from '@theme/TabItem'; `IndexerShard` struct is ephemeral structure, there is no such entity in `nearcore`. We've introduces it as a container in [`near-indexer-primitives`](https://crates.io/crates/near-indexer-primitives). This container includes: - shard ID -- [Chunk](./chunk.mdx) that might be absent -- [ExecutionOutcomes](./execution_outcome.mdx) for [Receipts](./receipt.mdx) (these belong to a Shard not to a [Chunk](./chunk.mdx) or a [Block](./block.mdx)) -- [StateChanges](./state_change.mdx) for a Shard +- [Chunk](chunk.mdx) that might be absent +- [ExecutionOutcomes](execution_outcome.mdx) for [Receipts](receipt.mdx) (these belong to a Shard not to a [Chunk](chunk.mdx) or a [Block](block.mdx)) +- [StateChanges](state_change.mdx) for a Shard ## `IndexerShard` diff --git a/docs/2.develop/lake/structures/transaction.mdx b/docs/2.develop/lake/structures/transaction.mdx index 5c0b8b72261..093fec57137 100644 --- a/docs/2.develop/lake/structures/transaction.mdx +++ b/docs/2.develop/lake/structures/transaction.mdx @@ -55,7 +55,7 @@ export type Transaction = { ## `ActionView` -`ActionView` is an Enum with possible actions along with parameters. This structure is used in Transactions and in [Receipts](./receipt.mdx) +`ActionView` is an Enum with possible actions along with parameters. This structure is used in Transactions and in [Receipts](receipt.mdx) diff --git a/docs/4.tools/cli-rs.md b/docs/4.tools/cli-rs.md index 31b8d7bbe35..3e258ea2e6d 100644 --- a/docs/4.tools/cli-rs.md +++ b/docs/4.tools/cli-rs.md @@ -30,7 +30,7 @@ To utilize the commands that involve transactions, sending tokens, deploying con Run... ``` -near-cli +near ``` Using the arrow keys navigate to... @@ -78,7 +78,7 @@ Now you can use `near-cli-rs` to it's full capacity. To use the `near-cli-rs` simply run the following in your terminal. ```bash -$ near-cli +$ near ``` You should then see the following. Use the arrow keys and hit `enter` or simply type out one of the available options to select an option diff --git a/docs/4.tools/events.md b/docs/4.tools/events.md index e05906c3d37..7d471847040 100644 --- a/docs/4.tools/events.md +++ b/docs/4.tools/events.md @@ -69,70 +69,3 @@ that listens for **all** `nft_mint` and `nft_transfer` events in the NEAR networ ::: --- - -## NEAR Lake Indexer - -NEAR Lake is an indexer built on top of [NEAR Indexer Framework](https://near-indexers.io/docs/projects/near-indexer-framework) to watch the network and store all the events as JSON files on AWS S3. - -:::info GitHub repo - -You can find the Lake Indexer source code in [this GitHub repository](https://github.com/near/near-lake-indexer/). - -::: - -### How it works - -:::tip - -[Pagoda Inc.](https://pagoda.co) runs NEAR Lake nodes to store the data in JSON format on AWS S3. -There is no need to run your own NEAR Lake unless you have specific reasons to do that. - -::: - -There are AWS S3 buckets created: - -- `near-lake-data-testnet` (`eu-central-1` region) -- `near-lake-data-mainnet` (`eu-central-1` region) - -All the buckets are set up the way the requester pays for the access. Anyone can read from these buckets by connecting to them with their own AWS credentials to be charged by Amazon. - -### Data structure - -The data structure used by Lake Indexer is the following: - -``` - / - block.json - shard_0.json - shard_1.json - ... - shard_N.json -``` - -`` is a 12-character-long [`u64`](https://doc.rust-lang.org/std/primitive.u64.html) string with leading zeros (e.g "000042839521"). See [this issue for reasoning](https://github.com/near/near-lake/issues/23). - -`block_json` contains JSON-serialized `BlockView` struct. **NB!** this struct might change in the future, we will announce it - -`shard_N.json` where N is [`u64`](https://doc.rust-lang.org/std/primitive.u64.html) starting from `0`. Represents the index number of the shard. In order to find out the expected number of shards in the block you can look in `block.json` at `.header.chunks_included` - - -### How to use it - -We have created the [NEAR Lake Framework](https://near-indexers.io/docs/projects/near-lake-framework) to have a simple straightforward way to create an indexer on top of the data stored by NEAR Lake itself. - -:::info NEAR Lake Framework - -You can check the NEAR Lake Framework release announcement on the [NEAR Governance Forum](https://gov.near.org/t/announcement-near-lake-framework-brand-new-word-in-indexer-building-approach/17668). - -::: - -We have prepared this video tutorial with a simple example to give you an overview and some practical ideas. - - diff --git a/docs/4.tools/indexer4explorer.md b/docs/4.tools/indexer4explorer.md index b3ab4ebbd97..626eb6faac3 100644 --- a/docs/4.tools/indexer4explorer.md +++ b/docs/4.tools/indexer4explorer.md @@ -1,7 +1,7 @@ --- id: indexer-for-explorer title: NEAR Indexer for Explorer -sidebar_label: Query the Blockchain History +sidebar_label: Indexer for Explorer --- import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; @@ -13,7 +13,6 @@ While developing a decentralized app you might want to query usage information f 3. Which transactions failed? In order to simplify asking these questions is that we developed the [NEAR Indexer for Explorer](https://github.com/near/near-indexer-for-explorer). -The NEAR Indexer for Explorer is a **public-access** **read-only** PostgreSQL database where **all** blockchain actions are stored. Both `testnet` and `mainnet` networks have active instances that fill the database with all the data from the network starting from the genesis as [Explorer](https://explorer.near.org/) requires. @@ -24,15 +23,10 @@ You can find the source code on [this GitHub repository](https://github.com/near ::: - -:::warning -The database could go down or take time to reflect the most current blockchain info. Do not use it in production. -For a reliable source of real-time and past information, please run your own [indexer](https://near-indexers.io/). -::: - --- ## Example Queries + ### Transactions Calling a Method Query for all transactions that called `contribute` in the `v1.faucet.nonofficial.testnet` testnet account. @@ -72,3 +66,52 @@ from receipts r, action_receipt_actions ara where r.predecessor_account_id ='v1.faucet.nonofficial.testnet' and ara.receipt_id = r.receipt_id and ara.action_kind = 'TRANSFER' ``` + +--- + +## NEAR Explorer sunsetting + +Pagoda made a decision to sunset NEAR Explorer as a product. This means that `explorer.near.org` (and `explorer.testnet.near.org`) will become a landing page with a list of available alternatives, and the existing [explorer.near.org](https://explorer.near.org) will be hosted under a new domain name and will be transitioned to the community through DevHub. [Read more here](https://near.social/devgovgigs.near/widget/gigs-board.pages.Post?id=635). + + +### What exactly is being shut down? + +You will lose access to databases with these URLs, or other Replicas you might have been given access to: +* `testnet.db.explorer.indexer.near.dev/testnet_explorer` +* `mainnet.db.explorer.indexer.near.dev/mainnet_explorer` + + +There is no plan to shut down any other data products, such as [NEAR Lake](https://docs.near.org/concepts/advanced/near-lake-framework) or [EnhancedAPI](https://www.pagoda.co/enhanced-api) at the moment. + +### What is the timeline? + +Postgres users will lose access to data on the 30th of November, Thursday (12:00 pm Pacific Time Zone). Please migrate to one of the options listed below instead. + +### What does this mean for me? + +If you are using the public Postgres Explorer Database, you will need to migrate to other solutions, depending on your use-case. You should start planning for it right now and reach out to [this Telegram group](https://nearbuilders.com/tg-data) to get help. + +### What are the alternatives? + +There are two major use-cases that you might be using Explorer DB for: analytics and real-time queries from web user interfaces. + +#### Analytics Use-Case + +This is if you use Explorer DB to build internal or external dashboards. Pagoda is working with Google Web3 team to enable BigQuery public dataset that has a compatible schema with Explorer DB. This will be enabled early September 2023. Please follow the announcements on [near.org](https://near.org) and in [Telegram chat](https://nearbuilders.com/tg-data). + +#### Web or API usage + +This is if you make queries to Explorer DB in response to API requests that your users make on your application. There are various options that you can explore: +1. If you are working with token balances, including $NEAR, fungible or non-fungible tokens, consider using [Enhanced API](https://www.pagoda.co/enhanced-api) hosted by Pagoda, or run it yourself using https://github.com/near/near-enhanced-api-server and https://github.com/near/near-microindexers +2. Use NEAR QueryAPI – serverless indexers and GraphQL endpoints: https://near.org/s/p?a=nearpavel.near&b=97029570 +3. Use NEAR Lake Indexer. Create an indexer using [Rust](https://github.com/near/near-lake-framework-rs), [JavaScript](https://github.com/near/near-lake-framework-js). There are other languages supported by community, try this search: https://github.com/search?q=near-lake-framework&type=repositories +4. Consider other indexer solutions built by the community + +### Can I still continue using Explorer Database? + +No, you won’t be able to continue using Public Explorer Database after the sunset. However you can start your own instance of https://github.com/near/near-indexer-for-explorer and reindex the history from scratch or use a latest backup. We will share a backup of Explorer DB in September if you want to run your own instance. + + +To run your own infra, you will need: +* **Indexer services:** We use two e2-medium instances on GCP for redundancy, with 2 vCPUs and 4GB of RAM. +* **A database:** We use Postgres version 11, with 8+ vCPUs, 52GB+ RAM and ~8TB of SSD storage. In addition, we recommend running an extra read-replica on a similar or more powerful machine. diff --git a/docs/4.tools/near-lake.md b/docs/4.tools/near-lake.md new file mode 100644 index 00000000000..9e2891f7c62 --- /dev/null +++ b/docs/4.tools/near-lake.md @@ -0,0 +1,71 @@ +--- +id: near-lake +sidebar_label: Lake Overview +title: NEAR Lake Indexer +--- + + +NEAR Lake is an indexer built on top of [NEAR Indexer Framework](https://near-indexers.io/docs/projects/near-indexer-framework) to watch the network and store all the [events](events.md) as JSON files on AWS S3. + +:::info GitHub repo + +You can find the Lake Indexer source code in [this GitHub repository](https://github.com/near/near-lake-indexer/). + +::: + +### How it works + +:::tip + +[Pagoda Inc.](https://pagoda.co) runs NEAR Lake nodes to store the data in JSON format on AWS S3. +There is no need to run your own NEAR Lake unless you have specific reasons to do that. + +::: + +There are AWS S3 buckets created: + +- `near-lake-data-testnet` (`eu-central-1` region) +- `near-lake-data-mainnet` (`eu-central-1` region) + +All the buckets are set up the way the requester pays for the access. Anyone can read from these buckets by connecting to them with their own AWS credentials to be charged by Amazon. + +### Data structure + +The data structure used by Lake Indexer is the following: + +``` + / + block.json + shard_0.json + shard_1.json + ... + shard_N.json +``` + +`` is a 12-character-long [`u64`](https://doc.rust-lang.org/std/primitive.u64.html) string with leading zeros (e.g "000042839521"). See [this issue for reasoning](https://github.com/near/near-lake/issues/23). + +`block_json` contains JSON-serialized `BlockView` struct. **NB!** this struct might change in the future, we will announce it + +`shard_N.json` where N is [`u64`](https://doc.rust-lang.org/std/primitive.u64.html) starting from `0`. Represents the index number of the shard. In order to find out the expected number of shards in the block you can look in `block.json` at `.header.chunks_included` + + +### How to use it + +We have created the [NEAR Lake Framework](/concepts/advanced/near-lake-framework) to have a simple straightforward way to create an indexer on top of the data stored by NEAR Lake itself. + +:::info NEAR Lake Framework + +You can check the NEAR Lake Framework release announcement on the [NEAR Governance Forum](https://gov.near.org/t/announcement-near-lake-framework-brand-new-word-in-indexer-building-approach/17668). + +::: + +We have prepared this video tutorial with a simple example to give you an overview and some practical ideas. + + diff --git a/docs/bos/community/indexers.md b/docs/bos/community/indexers.md index 7585a5349a6..993be808f48 100644 --- a/docs/bos/community/indexers.md +++ b/docs/bos/community/indexers.md @@ -1,7 +1,7 @@ --- id: indexers title: QueryAPI Indexing Example -sidebar_label: QueryAPI Example +sidebar_label: Getting Started --- With QueryAPI you can quickly create your own indexer by: @@ -26,7 +26,7 @@ You can request access through [this link](https://near.org/dev-queryapi.datapla ## How it Works -This works by: +QueryAPI works by: 1. Writing the indexer name to the blockchain, registering its creation; 2. Creating the tables as specified in the schema to the GraphQL database, exposing a GraphQL endpoint to query the data; @@ -58,6 +58,17 @@ This is the interface through which you can create a new Indexer. On here you ca * the indexer name on Indexer Name * from which block to start indexing on Specific Block Height or From Latest Block Height (selected by default) +### Design Workflow + +To design and create your indexer, you can follow this recommended workflow: + +1. Using [nearblocks.io](https://nearblocks.io), find transactions to smart contracts that you want to index +2. Take the block `height` and put it into the [Debug Mode filter](../queryapi/index-function.md#local-debug-mode), open your browser's _Developer Console_, and hit Play +4. Inspect the block and write JavaScript code using [NEAR Lake Primitives](../../2.develop/lake/lake-primitives.md) to extract data from a `block` object. (This JS code will be your [`IndexingLogic.js`](#indexinglogicjs)) + > **Tip:** Use `context.log` for debugging to ensure you are getting the right results +4. Add more blocks for debugging, or start following the blockchain to see how new blocks are handled +5. Create tables that you need to store the data using Postgres [CREATE table syntax](https://www.postgresql.org/docs/current/sql-createtable.html). (This SQL code will be your [`schema.sql`](#schemasql)) + :::tip Video Walkthrough diff --git a/docs/bos/queryapi/big-query.md b/docs/bos/queryapi/big-query.md new file mode 100644 index 00000000000..8d4b3deb8ba --- /dev/null +++ b/docs/bos/queryapi/big-query.md @@ -0,0 +1,104 @@ +--- +id: big-query +title: BigQuery Public Dataset +sidebar_label: BigQuery +--- + +Blockchain data indexing in NEAR Public Lakehouse is for anyone wanting to understand blockchain data. This includes: + +- **Users**: create queries to track NEAR assets, monitor transactions, or analyze on-chain events at a massive scale. +- **Researchers**: use indexed data for data science tasks, including on-chain activities, identifying trends, or feeding AI/ML pipelines for predictive analysis. +- **Startups**: can use NEAR's indexed data for deep insights on user engagement, smart contract utilization, or insights across tokens and NFT adoption. + +Benefits: + +- **NEAR instant insights**: Historical on-chain data queried at scale. +- **Cost-effective**: eliminate the need to store and process bulk NEAR protocol data; query as little or as much data as preferred. +- **Easy to use**: no prior experience with blockchain technology is required; bring a general knowledge of SQL to unlock insights. + + +## Getting started + +1. Login into your [Google Cloud Account](https://console.cloud.google.com/). +2. Open the [NEAR Protocol BigQuery Public Dataset](https://console.cloud.google.com/marketplace/product/bigquery-public-data/crypto-near-mainnet). +3. Click in the [VIEW DATASET](https://console.cloud.google.com/bigquery?p=bigquery-public-data&d=crypto_near_mainnet_us&page=dataset) button. +4. Click in the + to create a new tab and write your query, click in the RUN button, and check the `Query results` below the query. +5. Done :) + +:::info + +The [NEAR Public Lakehouse repository](https://github.com/near/near-public-lakehouse) contains the source code for ingesting NEAR Protocol data stored as JSON files in AWS S3 by [NEAR Lake Indexer](https://github.com/near/near-lake-indexer). + +::: + +### Example Queries + +- _How many unique users do I have for my smart contract per day?_ + +```sql +SELECT + r.block_date collected_for_day, + COUNT(DISTINCT r.transaction_signer_account_id) +FROM `bigquery-public-data.crypto_near_mainnet_us.receipt_actions` ra + INNER JOIN `bigquery-public-data.crypto_near_mainnet_us.receipts` r ON r.receipt_id = ra.receipt_id +WHERE ra.action_kind = 'FUNCTION_CALL' + AND ra.receipt_receiver_account_id = 'near.social' -- change to your contract +GROUP BY 1 +ORDER BY 1 DESC; +``` + +## How much it costs? + +- NEAR pays for the storage and doesn't charge you to use the public dataset. + > To learn more about BigQuery public datasets [check this page](https://cloud.google.com/bigquery/public-data). +- Google GCP charges for the queries that you perform on the data. For example, in today's price "Sep 1st, 2023" the On-demand (per TB) query pricing is $6.25 per TB where the first 1 TB per month is free. + > Check [Google's pricing page](https://cloud.google.com/bigquery/pricing#analysis_pricing_models) for detailed pricing info, options, and best practices. + +:::tip +You can check how much data it will query before running it in the BigQuery console UI. Again, since BigQuery uses a columnar data structure and partitions, it's recommended to select only the columns and partitions (`block_date`) needed to avoid unnecessary query costs. +::: + +![Query Costs](/docs/BQ_Query_Cost.png "BQ Query Costs") + +## Architecture + +The data is loaded in a streaming fashion using [Databricks Autoloader](https://docs.gcp.databricks.com/ingestion/auto-loader/index.html) into raw/bronze tables, and transformed with [Databricks Delta Live Tables](https://www.databricks.com/product/delta-live-tables) streaming jobs into cleaned/enriched/silver tables. + +The silver tables are also copied into the [GCP BigQuery Public Dataset](https://cloud.google.com/bigquery/public-data). + +![Architecture](/docs/Architecture.png "Architecture") + +:::info + +[Databricks Medallion Architecture](https://www.databricks.com/glossary/medallion-architecture). + +::: + +## Available Data + +The current data that NEAR is providing was inspired by [NEAR Indexer for Explorer](https://github.com/near/near-indexer-for-explorer/). + +:::info +NEAR plans to improve the data available in the NEAR Public Lakehouse making it easier to consume by denormalizing some tables. +::: + +The tables available in the NEAR Public Lakehouse are: + +- **blocks**: A structure that represents an entire block in the NEAR blockchain. `Block` is the main entity in NEAR Protocol blockchain. Blocks are produced in NEAR Protocol every second. +- **chunks**: A structure that represents a chunk in the NEAR blockchain. `Chunk` of a `Block` is a part of a `Block` from a `Shard`. The collection of `Chunks` of the `Block` forms the NEAR Protocol Block. `Chunk` contains all the structures that make the `Block`: `Transactions`, [`Receipts`](https://nomicon.io/RuntimeSpec/Receipts), and `Chunk Header`. +- **transactions**: [`Transaction`](../../2.develop/lake/structures/transaction.mdx#definition) is the main way of interaction between a user and a blockchain. Transaction contains: Signer account ID, Receiver account ID, and Actions. +- **execution_outcomes**: Execution outcome is the result of execution of `Transaction` or `Receipt`. In the result of the Transaction execution will always be a Receipt. +- **receipt_details**: All cross-contract (we assume that each account lives in its own shard) communication in Near happens through Receipts. Receipts are stateful in a sense that they serve not only as messages between accounts but also can be stored in the account storage to await `DataReceipts`. Each receipt has a `predecessor_id` (who sent it) and `receiver_id` the current account. +- **receipt_origin**: Tracks the transaction that originated the receipt. +- **receipt_actions**: Action Receipt represents a request to apply actions on the `receiver_id` side. It could be derived as a result of a `Transaction` execution or another `ACTION` Receipt processing. Action kind can be: `ADD_KEY`, `CREATE_ACCOUNT`, `DELEGATE_ACTION`, `DELETE_ACCOUNT`, `DELETE_KEY`, `DEPLOY_CONTRACT`, `FUNCTION_CALL`, `STAKE`, `TRANSFER`. +- **receipts (view)**: It's recommended to select only the columns and partitions (`block_date`) needed to avoid unnecessary query costs. This view join the receipt details, the transaction that originated the receipt and the receipt execution outcome. +- **account_changes**: Each account has an associated state where it stores its metadata and all the contract-related data (contract's code + storage). + +:::note References + +- [Protocol documentation](../../1.concepts/welcome.md) +- [Near Data flow](../../1.concepts/data-flow/near-data-flow.md) +- [Lake Data structures](../../2.develop/lake/structures/toc.mdx) +- [Protocol specification](https://nomicon.io/) + +::: diff --git a/website/docusaurus.config.js b/website/docusaurus.config.js index 17a9097e50f..0dd078e1d6e 100644 --- a/website/docusaurus.config.js +++ b/website/docusaurus.config.js @@ -117,7 +117,7 @@ module.exports = { items: [ { label: "Smart Contracts", href: "/develop/contracts/welcome" }, { label: "Web3 Applications", href: "/develop/integrate/welcome" }, - { label: "Monitor the Chain", href: "/tools/realtime" }, + { label: "Monitor the Chain", href: "/bos/queryapi/intro" }, { type: 'html', value: '
', diff --git a/website/sidebars.json b/website/sidebars.json index d312193878a..ceb974b777e 100644 --- a/website/sidebars.json +++ b/website/sidebars.json @@ -37,7 +37,6 @@ }, { "Data Flow": [ - "concepts/data-flow/data-storage", "concepts/data-flow/near-data-flow", "concepts/data-flow/token-transfer-flow" ] @@ -64,16 +63,27 @@ }, { "type": "html", - "value": " Advanced Topics " + "value": " Blockchain Data & Indexing " }, [ + "concepts/data-flow/data-storage", { - "Indexers": [ + "Data Indexing": [ "concepts/advanced/indexers", "concepts/advanced/near-indexer-framework", "concepts/advanced/near-lake-framework" ] - }, + } + ], + { + "type": "html", + "value": "
" + }, + { + "type": "html", + "value": " Advanced Topics " + }, + [ "concepts/basics/runtime", "concepts/advanced/specification", "concepts/advanced/papers" @@ -646,9 +656,63 @@ ], "indexers": [ "tools/realtime", + { + "type": "html", + "value": "
" + }, + { + "type": "html", + "value": " Data Analytics " + }, + "bos/queryapi/big-query", "tools/indexer-for-explorer", { - "Lake Indexer": [ + "type": "html", + "value": "
" + }, + { + "type": "html", + "value": " QueryAPI Indexing " + }, + "bos/queryapi/intro", + "bos/queryapi/index-functions", + "bos/community/indexers", + { + "Examples": [ + "bos/tutorial/indexer-tutorials/posts-indexer", + "bos/tutorial/indexer-tutorials/hype-indexer", + "bos/tutorial/indexer-tutorials/feed-indexer" + ] + }, + { + "type": "html", + "value": "
" + }, + { + "type": "html", + "value": " NEAR Lake Framework " + }, + "tools/near-lake", + "tutorials/indexer/near-lake-state-changes-indexer", + "tutorials/indexer/migrating-to-near-lake-framework", + { + "Building Indexers": [ + "develop/lake/primitives", + "tutorials/indexer/js-lake-indexer", + "tutorials/indexer/python-lake-indexer", + "tutorials/indexer/nft-indexer", + "tutorials/indexer/python-nft-indexer" + ] + }, + { + "Running NEAR Lake": [ + "tutorials/indexer/run-lake-indexer", + "tutorials/indexer/lake-start-options", + "tutorials/indexer/credentials" + ] + }, + { + "Lake Data Structures": [ "develop/lake/structures/toc", "develop/lake/structures/block", "develop/lake/structures/chunk", @@ -665,16 +729,9 @@ }, { "type": "html", - "value": " Tutorials " + "value": " Ecosystem Tools " }, - "tutorials/indexer/migrating-to-near-lake-framework", - "tutorials/indexer/near-lake-state-changes-indexer", - "tutorials/indexer/js-lake-indexer", - "tutorials/indexer/lake-start-options", - "tutorials/indexer/python-lake-indexer", - "tutorials/indexer/nft-indexer", - "tutorials/indexer/python-nft-indexer", - "tutorials/indexer/credentials" + "tools/indexing" ], "api": [ "api/rpc/introduction", diff --git a/website/static/docs/Architecture.png b/website/static/docs/Architecture.png new file mode 100644 index 00000000000..00c9493209c Binary files /dev/null and b/website/static/docs/Architecture.png differ diff --git a/website/static/docs/BQ_Query_Cost.png b/website/static/docs/BQ_Query_Cost.png new file mode 100644 index 00000000000..42cc5325c70 Binary files /dev/null and b/website/static/docs/BQ_Query_Cost.png differ