Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(s3): add S3Storage adapter #220

Open
wants to merge 14 commits into
base: main
Choose a base branch
from
Open
4 changes: 2 additions & 2 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -31,10 +31,10 @@ yarn-error.log*
.pnpm-debug.log
dist
eggs-debug.log
.npmrc

.pnpm-store
.idea
lerna-debug.log
# pnpm-lock.yaml
.env
.wrangler
.wrangler
1 change: 1 addition & 0 deletions .npmrc
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
@jsr:registry=https://npm.jsr.io
2 changes: 1 addition & 1 deletion libs/utils/src/deps.deno.ts
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
export { Bot, Context } from 'https://lib.deno.dev/x/grammy@1.x/mod.ts';
export type { SessionFlavor } from 'https://lib.deno.dev/x/grammy@1.x/mod.ts';
export type { SessionFlavor, LazySessionFlavor } from 'https://lib.deno.dev/x/grammy@1.x/mod.ts';
2 changes: 1 addition & 1 deletion libs/utils/src/deps.node.ts
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
export { Bot, Context } from 'grammy';
export type { SessionFlavor } from 'grammy';
export type { SessionFlavor, LazySessionFlavor } from 'grammy';
22 changes: 12 additions & 10 deletions libs/utils/src/mod.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,13 @@ interface StringSessionFlavor {
}

type JsonBot = Deps.Context & Deps.SessionFlavor<JsonSessionData>
type LazyJsonBot = Deps.Context & Deps.LazySessionFlavor<JsonSessionData>
type StringBot = Deps.Context & StringSessionFlavor

export function createBot(json?: true): Deps.Bot<JsonBot>
export function createBot(json?: false): Deps.Bot<StringBot>
export function createBot(json = true) {
export function createBot(json: true): Deps.Bot<JsonBot>
export function createBot(json: true, lazy: true): Deps.Bot<LazyJsonBot>
export function createBot(json: false): Deps.Bot<StringBot>
export function createBot(json = true, lazy:boolean = false): Deps.Bot<any> {
const botInfo = {
id: 42,
first_name: 'Test Bot',
Expand All @@ -26,7 +28,7 @@ export function createBot(json = true) {
};

if (json) {
return new Deps.Bot<JsonBot>('fake-token', { botInfo });
return new Deps.Bot<typeof lazy extends true ? LazyJsonBot : JsonBot>('fake-token', { botInfo });
} else {
return new Deps.Bot<StringBot>('fake-token', { botInfo });
}
Expand All @@ -35,12 +37,12 @@ export function createBot(json = true) {
export function createMessage(bot: Deps.Bot<any>, text = 'Test Text') {
const createRandomNumber = () => Math.floor(Math.random() * (123456789 - 1) + 1);

const ctx = new Deps.Context({
update_id: createRandomNumber(),
message: {
const ctx = new Deps.Context({
update_id: createRandomNumber(),
message: {
text,
message_id: createRandomNumber(),
chat: {
chat: {
id: 1,
type: 'private',
first_name: 'Test User',
Expand All @@ -53,9 +55,9 @@ export function createMessage(bot: Deps.Bot<any>, text = 'Test Text') {
},
},
},
bot.api,
bot.api,
bot.botInfo
);

return ctx;
}
}
3 changes: 2 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -31,5 +31,6 @@
"engines": {
"pnpm": ">=7.0.0",
"node": ">=12.0.0"
}
},
"packageManager": "pnpm@9.0.6+sha1.648f6014eb363abb36618f2ba59282a9eeb3e879"
}
2 changes: 1 addition & 1 deletion packages/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ Bug reports and pull requests are welcome.
│ │ │
│ │ └─⫸ Summary in present tense. Not capitalized. No period at the end.
│ │
│ └─⫸ Commit Scope: utils|file|mongodb|psql|redis|typeorm|supabase|free|firestore|deta|denodb|denokv|cloudflare
│ └─⫸ Commit Scope: cloudflare|denodb|denokv|deta|file|firestore|free|mongodb|psql|redis|s3|supabase|typeorm|utils
Expand Down
2 changes: 1 addition & 1 deletion packages/file/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@
},
"homepage": "https://github.com/grammyjs/storages/tree/main/packages/file#readme",
"devDependencies": {
"@grammyjs/storage-utils": "^2.4.2",
"@grammyjs/storage-utils": "workspace:*",
"grammy": "^1.21.1"
},
"gitHead": "a7758c4f957f103a14832088c6858d693c444576"
Expand Down
2 changes: 1 addition & 1 deletion packages/mongodb/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@
},
"homepage": "https://github.com/grammyjs/storages/tree/main/packages/mongodb#readme",
"devDependencies": {
"@grammyjs/storage-utils": "^2.4.2",
"@grammyjs/storage-utils": "workspace:*",
"grammy": "^1.21.1",
"mongodb": "^6.3.0",
"mongodb-memory-server": "^9.1.6"
Expand Down
2 changes: 1 addition & 1 deletion packages/prisma/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@
},
"homepage": "https://github.com/grammyjs/storages/tree/main/packages/typeorm#readme",
"devDependencies": {
"@grammyjs/storage-utils": "^2.4.2",
"@grammyjs/storage-utils": "workspace:*",
"@prisma/client": "^5.10.2",
"grammy": "^1.21.1",
"prisma": "^5.10.2",
Expand Down
2 changes: 1 addition & 1 deletion packages/psql/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@
},
"homepage": "https://github.com/grammyjs/storages/tree/main/packages/psql#readme",
"devDependencies": {
"@grammyjs/storage-utils": "^2.4.2",
"@grammyjs/storage-utils": "workspace:*",
"@types/pg": "^8.11.2",
"grammy": "^1.21.1",
"pg": "^8.11.3",
Expand Down
2 changes: 1 addition & 1 deletion packages/redis/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@
},
"homepage": "https://github.com/grammyjs/storages/tree/main/packages/redis#readme",
"devDependencies": {
"@grammyjs/storage-utils": "^2.4.2",
"@grammyjs/storage-utils": "workspace:*",
"@types/ioredis": "^5.0.0",
"grammy": "^1.21.1",
"ioredis": "^5.3.2"
Expand Down
21 changes: 21 additions & 0 deletions packages/s3/LICENSE
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
MIT License

Copyright (c) 2024 Christian Bewernitz

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
121 changes: 121 additions & 0 deletions packages/s3/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
# S3 Storage Adapter for grammY

Session storage adapter that can be used to
[store your session data](https://grammy.dev/plugins/session.html) via an
[S3 compatible object storage](https://en.wikipedia.org/wiki/Amazon_S3#S3_API_and_competing_services).

The most prominent options are:

- AWS S3 (12 months limited free tier)
- Cloudflare R2 (unlimited free tier, no egress fee, but needs account with
payment connected, in case the use exceeds free tier)
- https://github.com/minio/minio your own S3 in docker
- ... <!-- is there a stable external list that compares the options? -->

## Pros and Cons

The biggest restriction of the current setup is that it only works with deno.
For it to work with pnpm / node, we would need to add the following line to
`.npmrc` (which is currently `.gitignore`d)

```
@jsr:registry=https://npm.jsr.io
# The above doesn't work and prevents us from adding
# "@bradenmacdonald/s3-lite-client": "npm:@jsr/bradenmacdonald__s3-lite-client@0.7.4"
# to packages/s3/package.json
# the error reported by "pnpm i":
# ERR_PNPM_FETCH_404 GET https://registry.npmjs.org/@grammyjs%2Fstorage-utils: Not Found - 404
# This error happened while installing a direct dependency of /run/media/karfau/hdd-data/dev/storages/packages/file
# @grammyjs/storage-utils is not in the npm registry, or you have no permission to fetch it.
```

which would allow us to add `packages/s3/package.json` with the following
`dependency`:
`"@bradenmacdonald/s3-lite-client": "npm:@jsr/bradenmacdonald__s3-lite-client@0.7.4"`
it would also require some changes to the imports, to provide the bare
specifiers as listed in `package.json` in `deno.json` "imports". And we would
need to add related tests

1. It is not the fastest way to get your data (benchmarks?), so it currently
does not implement the methods for loading all sessions. In a webhook
approach it works best using `LazySession`
[and the `serialize` middleware from `@grammyjs/runner`](https://grammy.dev/advanced/deployment#webhooks).
2. You should consider limiting the key to
["safe characters"](https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html).
3. The setup requires 3-5 parameters (depending on the provider) that you need
to pass as env variables, which could be passed as one JSON string in a
single env variable. But by using individual andcommon variable names like
`AWS_SECRET_KEY`, more tools will pick them up. The process of getting all
the right parameters from your provider can be very different.
4. Each provider can have different limitations regarding the storage, check
them out.
5. You can use the same storage to even store the raw Updates to process them
later
6. You can use the same storage to also store and access other data using the
`client`. It helps to think of the objects that you store as files on disk,
where you have at least one ID as a path element (or filename). Examples:
assets, or each update as a json file for async processing.
7. You can use the aws cli, `mc`, `rclone` and similar CLI tools to access the
data or to get a local copy of some or all files.

## Instructions

1. Import the adapter

```ts
import { S3DBAdapter } from "https://deno.land/x/grammy_storages/s3/src/mod.ts";
```

2. Get the credentials from you provider and pass them from one or multiple env
variables.

```ts
const clientOptions: S3ClientOptions = JSON.parse(
Deno.env.get("S3_CLIENT_OPTS") ?? "{}",
);
```

3. Define lazy session structure

```ts
interface SessionData {
count: number;
}
type MyContext = Context & LazySessionFlavor<SessionData>;
```

4. Define method to create session key from context

```ts
function getSessionKey(ctx: MyContext) {
// it could be user based
return `/chat/${ctx.from?.id ?? 0}/session.json`;
// or if group chats are relevant, it could be chat based
// return `/chat/${ctx.chat?.id ?? 0}/session.json`;
}
```

5. Register adapter's middleware

```ts
const bot = new Bot<MyContext>("<Token>");

bot.use(sequentialize(getSessionKey)).use(
lazySession({
getSessionKey,
initial() {
return { count: 0 };
},
storage: new S3Adapter(clientOptions),
}),
);
```

Use `await ctx.session` as explained in
[session plugin](https://grammy.dev/plugins/session.html#lazy-sessions)'s docs.

<!--
## More examples

can be found in the [examples](./examples) folder.
-->
66 changes: 66 additions & 0 deletions packages/s3/adapter.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
import {
S3Client,
type S3ClientOptions,
type StorageAdapter,
} from './deps.deno.ts';
export { S3Client, type S3ClientOptions } from './deps.deno.ts';

export type S3StorageClient = Pick<
S3Client,
'exists' | 'deleteObject' | 'getObject' | 'host' | 'region' | 'putObject'
>;
function isS3StorageClient(
maybeClient: S3StorageClient | S3ClientOptions,
): maybeClient is S3StorageClient {
return ['exists', 'deleteObject', 'getObject', 'putObject'].every(
(required) =>
typeof maybeClient[required as keyof typeof maybeClient] === 'function',
);
}

export function isObjectSession(maybeSession: unknown): maybeSession is object {
return !!maybeSession && typeof maybeSession === 'object';
}

export class S3Storage<T> implements StorageAdapter<T> {
readonly client: S3StorageClient;
constructor(
clientOrOptions: S3StorageClient | S3ClientOptions,
readonly validateSession: (data: T | unknown) => boolean,
) {
this.client = isS3StorageClient(clientOrOptions)
? clientOrOptions
: new S3Client(clientOrOptions);
}

/**
* @see https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html
*/
isSafe(key: string): boolean {
return /^[-0-9a-zA-Z!_.()]+$/.test(key);
}

async delete(key: string): Promise<void> {
return await this.client.deleteObject(key);
}

async has(key: string): Promise<boolean> {
return await this.client.exists(key);
}

async read(key: string): Promise<T | undefined> {
try {
const res = await this.client.getObject(key);
const data = (await res.json()) as T;
return this.validateSession(data) ? data : undefined;
} catch {
return undefined;
}
}

async write(key: string, value: T): Promise<void> {
// the client has a mismatching return type
// to make type checks happy we await it, and intentionally do not return it
await this.client.putObject(key, JSON.stringify(value));
}
}
15 changes: 15 additions & 0 deletions packages/s3/deno.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
{
"tasks": {
"check": "deno check *.ts test/*.ts",
"test": "deno test test"
},
"lock": false,
"nodeModulesDir": false,
"imports": {
"@bradenmacdonald/s3-lite-client": "jsr:@bradenmacdonald/s3-lite-client@0.7.5",
"@grammyjs/storage-utils": "../../libs/utils/src/mod.ts",
"@std/assert": "jsr:@std/assert",
"@std/testing/mock": "jsr:@std/testing/mock",
"grammy": "https://lib.deno.dev/x/grammy@1.x/mod.ts"
}
}
5 changes: 5 additions & 0 deletions packages/s3/deps.deno.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
export type { StorageAdapter } from 'grammy';
export {
S3Client,
type S3ClientOptions,
} from '@bradenmacdonald/s3-lite-client';
5 changes: 5 additions & 0 deletions packages/s3/deps.node.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
export type { StorageAdapter } from 'grammy';
export {
S3Client,
type S3ClientOptions,
} from '@bradenmacdonald/s3-lite-client';
10 changes: 10 additions & 0 deletions packages/s3/package.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
{
"name": "s3",
"private": true,
"scripts": {
"test:deno": "deno task test"
},
"dependencies": {
"@bradenmacdonald/s3-lite-client": "@jsr/bradenmacdonald__s3-lite-client@0.7.5"
}
}
Loading