Merge branch 'master' into feat/recommendations

# Conflicts:
#	.env
#	Makefile
#	db/migrations/1704717446430-Data.js
#	package-lock.json
#	package.json
#	src/mappings/content/channel.ts
#	src/mappings/content/video.ts
#	src/processor.ts
#	src/server-extension/resolvers/VideosResolver/index.ts
#	src/server-extension/resolvers/VideosResolver/types.ts
#	src/utils/language.ts

.env

@@ -11,6 +11,8 @@ DB_ADMIN_PASS=admin
 DB_PORT=23798
 INTERACTIONS_DB_PORT=23799
 
+# Processor service host
+PROCESSOR_HOST=localhost
 # Processor service prometheus port
 PROCESSOR_PROMETHEUS_PORT=3337
 # Graphql server port
@@ -74,6 +76,8 @@ SENDGRID_FROM_EMAIL=gateway@example.com
 
 # Debug settings
 SQD_DEBUG=api:*
+
+# playground
 OPENAPI_PLAYGROUND=true
 
 # max number of attempts to deliver email notification
@@ -83,3 +87,11 @@ EMAIL_NOTIFICATION_DELIVERY_MAX_ATTEMPTS=5
 APP_ASSET_STORAGE=https://raw.githubusercontent.com/Joystream/atlas-notification-assets/main/logos/gleev
 APP_NAME_ALT=Gleev.xyz
 NOTIFICATION_ASSET_ROOT=https://raw.githubusercontent.com/Joystream/atlas-notification-assets/main/icons
+
+# =====================================================================================
+# Telemetry
+# =====================================================================================
+
+# yes/no
+TELEMETRY_ENABLED=no
+TELEMETRY_ENDPOINT=http://apm-server:8200

.github/workflows/checks.yml

@@ -36,3 +36,26 @@ jobs:
  node-version: ${{matrix.node-version}}
  - name: Build docker image
  run: make build-docker
+
+ migrations:
+ name: Run migrations
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v2
+ - name: Use Node.js 16.x
+ uses: actions/setup-node@v1
+ with:
+ node-version: 16.x
+ - name: Prepare workspace
+ run: make prepare
+ - name: create joystream_default network
+ run: docker network create joystream_default
+ - name: Start db
+ run: docker-compose up -d orion_db
+ - name: Wait for db
+ run: |
+ while ! docker-compose exec -T orion_db pg_isready -U postgres -p 23798; do
+ sleep 1
+ done
+ - name: Run migrations
+ run: make migrate

.github/workflows/docker.yml

@@ -1,22 +1,58 @@
-name: Build and publish to Docker Hub
+name: Build & Publish Orion Docker image
 
 on:
- push:
- branches:
- - 'master'
+ workflow_dispatch:
 
 jobs:
- docker:
+ build_and_publish:
+ name: Build and Publish
  runs-on: ubuntu-latest
  steps:
+ - name: Checkout
+ uses: actions/checkout@v3
+
+ - uses: actions/setup-node@v3
+ with:
+ node-version: '18.x'
+
+ - name: Extract Package Version
+ id: extract_version
+ shell: bash
+ run: |
+ echo "orion_version=$(cat package.json | jq -r '.version')" >> $GITHUB_OUTPUT
+
+ - name: Make some space
+ shell: bash
+ run: |
+ sudo rm -rf /usr/share/dotnet
+ sudo rm -rf /opt/ghc
+ sudo rm -rf "/usr/local/share/boost"
+ sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+
+ - name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@v2
  - name: Login to DockerHub
- uses: docker/login-action@v1
+ if: github.event_name == 'workflow_dispatch'
+ uses: docker/login-action@v2
  with:
  username: ${{ secrets.DOCKERHUB_USERNAME }}
- password: ${{ secrets.DOCKERHUB_TOKEN }}
- - name: Build and push
-  id: docker_build
- uses: docker/build-push-action@v2
+ password: ${{ secrets.DOCKERHUB_PASSWORD }}
+
+ - name: Build orion
+ uses: docker/build-push-action@v3
  with:
- push: true
+ context: .
+ file: Dockerfile
+ push: false
+ load: true
  tags: joystream/orion:latest
+
+ - name: Push Orion Image
+ run: |
+ docker image tag joystream/orion:latest joystream/orion:${{ steps.extract_version.outputs.orion_version }}
+ docker push joystream/orion:${{ steps.extract_version.outputs.orion_version }}
+
+ - name: Push Orion latest tag
+ if: github.ref == 'refs/heads/master'
+ run: |
+ docker push joystream/orion:latest

.gitignore

@@ -8,9 +8,20 @@
 /.vscode
 /.idea
 src/model/generated
+src/types/
 src/auth-server/emails/templates/*.mst
 src/auth-server/emails/templates/preview
 /schema.graphql
 /db/persisted
 /scripts/orion-v1-migration/data
 /db/export
+/db/migrations_dir*
+.DS_STORE
+
+# network tests
+network-tests/graphql/generated
+network-tests/node_modules
+network-tests/lib
+
+# LOGS
+chron_mail_scheduler.log

.prettierignore

@@ -5,7 +5,8 @@ lib
 # Autogenerated stuff
 src/types
 src/model/generated
-db/migrations/*.js
+db/migrations/*-Data.js
+db/migrations/*-Views.js
 schema.graphql
 /scripts/orion-v1-migration/data
-/db/export
+/db/export

CHANGELOG.md

@@ -1,3 +1,105 @@
+# 4.0.2
+
+## Bug Fixes:
+- Fixed: unstake tokens atfer revenue share has been finalized.
+- Fixed: transform raw json objects to `jsonb` properties while importing the offchain state.
+- Fixed: `bigint` to `number` conversion in `getCumulativeHistoricalShareAllocation` custom resolver.
+- Fixed duplicate notifications being received by users for featured NFTs.
+
+# 4.0.1
+
+## Misc
+
+- Enable both in App and email notifications for `previousTopBidder` in NFT Open Auction.
+
+# 4.0.0
+
+This is Creator Tokens (CRT) release. It introduces the CRT mappings, custom resolvers and mutations.
+
+# 3.7.0
+
+## Schema changes
+- Added `isShortDerived` field to `Video` entity indicating whether a video is a short format, vertical video or not. This field is computed in the mappings based on the video dimensions and duration when `isShort` is not set in the metadata.
+
+## Misc
+
+- update `setOrionLanguage` Custom migration script.
+
+## Bug Fixes:
+- Added fix to improve the accuracy of `Video.orionLanguage` field by reworking the `predictVideoLanguage` function in `src/utils/language.ts`
+- Use UTC midnight epoch instead of current epoch to calculate video relevance score in `VideoRelevanceManager`
+
+
+# 3.6.0
+
+## Schema changes
+- Added `includeInHomeFeed` field to `Video` entity indicating if the video should be included in the home feed/page.
+
+## Mutations
+### Additions
+- `setOrUnsetPublicFeedVideos`: mutation to set or unset the `includeInHomeFeed` field of a video by the Operator. 
+
+### Queries
+#### Additions
+- `dumbPublicFeedVideos`: resolver to retrieve random `N` videos from list of all homepage videos.
+
+## DB Migrations
+- Added partial index on `Video` entity to include only videos that are included in the home feed (in `db/migrations/2200000000000-Indexes.js`)
+
+# 3.5.0
+
+## Schema changes
+- Added `isShort` field to `Video` entity indicating whether a video is a short format, vertical video or not.
+
+## Misc
+- Disable both in App and email notifications for `videoPosted` notifiations type by default.
+- Remove unused `@joystream/metadata-protobuf` patch from assets/patches directory.
+
+# 3.4.0
+
+## Schema changes
+- Added `@schema(name: "admin")` directive to hide entities (from public GRAPHQL API) in Graphql schema definitions.
+
+## Misc
+- Patch `@subsquid/typeorm-config` & `@subsquid/typeorm-migration` packages to change `squid-typeorm-migration apply` command to apply a single migrations file too using `--filename` option instead of applying the whole `db/migrations` directory.
+- Patch `@subsquid/openreader` and `@subsquid/typeorm-codegen` dependencies to include the db schema `name` too in the generated typeorm/postgres migrations, and an optional `schema` directive to specify the schema of any GRAPHQL entity.
+
+## DB Migrations
+- Update `generate-migrations` makefile command. Now the existing `*-Data.js` will not be overwritten, instead a new `*-Data.js` migration file (containing only changes compared to the previous DB state) will be added whenever there are GRAPHQL schema changes. The `*-Views.js` migration file will also be updated whenever the GRAPHQL schema changes.
+- Create `generateViewsMigration.js` script to create new `*-Views.js` migration file.
+- Separate the view definitions(in `db/viewDefinitions.js`) from views migration file(`*-Views.js`).
+- Add `*-Admin.js` migration file to create an `admin` schema & user, previously the `admin` schema and user was being created in the `*-Views.js` migration.
+- Regenerate the postgres db migrations.
+
+## Documentation
+- Updated documentation for [upgrading-orion.md](docs/operator-guide/tutorials/upgrading-orion.md)
+- Updated documentation for [entity-visibility.md#managing-entity-visibility](docs/developer-guide/tutorials/entity-visibility.md)
+
+### Bug Fixes:
+- read/write `export.json` file, containing the offchain state, using `big-json` package, instead using Javascript native `JSON.stringify` function which does not work on large JSON objects
+
+# 3.3.0
+
+## Schema
+- `orionLanguage` property has been added.
+
+## Mappings 
+- Language detection is used to populate new property on video update and creation.
+
+# 3.2.1
+
+## Misc
+- Added index on timestamp fields on `Video`, `Event` and `OwnedNft` entities
+- update `docker.yml` github workflow to build and publish docker images.
+- added `generate-migrations` makefile command to generate migrations
+- adds OpenTelemetry tracing integration with `graphql-server` and `auth-server`
+- update nodejs version to `node:18`
+
+### Bug Fixes:
+- Extend session based on activity after initial expiry is over.
+- Fix/notification email title.
+- Improves off-chain import/export script
+
 # 3.2.0
 This release adds notifications to the orion infrastructure...
 ## Schema changes

Dockerfile

@@ -1,4 +1,4 @@
-FROM node:16-alpine AS node
+FROM node:18-alpine AS node
 
 FROM node AS node-with-gyp
 RUN apk add g++ make python3
@@ -10,10 +10,13 @@ ADD package-lock.json .
 ADD assets assets
 RUN npm ci
 ADD tsconfig.json .
+ADD typegen.json .
+ADD joystream.jsonl .
 ADD src src
 ADD schema schema
 ADD scripts scripts
 RUN npx squid-typeorm-codegen
+RUN npx squid-substrate-typegen typegen.json
 RUN npm run build
 
 FROM node-with-gyp AS deps
@@ -45,4 +48,4 @@ CMD ["npm", "run", "processor-start"]
 
 
 FROM squid AS query-node
-CMD ["npm", "run", "query-node-start"]
+CMD ["npm", "run", "graphql-server-start"]

Makefile

@@ -3,6 +3,7 @@ process: migrate
 
 install:
  @rm -rf node_modules # clean up node_modules to avoid issues with patch-package
+ @rm -rf network-tests/node_modules # clean up node_modules to avoid issues with patch-package
  @npm install
 
 build:
@@ -12,7 +13,7 @@ build-docker:
  @docker build . -t joystream/orion
 
 serve:
- @npx squid-graphql-server --subscriptions
+ @npm run graphql-server-start
 
 serve-auth-api:
  @npm run auth-server-start
@@ -26,17 +27,41 @@ migrate:
 dbgen:
  @npx squid-typeorm-migration generate
 
+generate-migrations:
+ @docker run -d --name temp_migrations_db \
+ -e POSTGRES_DB=squid \
+ -e POSTGRES_HOST_AUTH_METHOD=trust \
+ -v temp_migrations_db_volume:/var/lib/postgresql/data \
+ -v ./db/postgres.conf:/etc/postgresql/postgresql.conf \
+ -p 5555:5555 postgres:14 postgres -p 5555 || true
+ @sleep 5
+ @export DB_PORT=5555 && npx squid-typeorm-migration apply --filename *-Admin.js
+ @ls db/migrations > db/migrations_dir_before.txt
+ @export DB_PORT=5555 && npx squid-typeorm-migration apply --filename *-Data.js
+ @export DB_PORT=5555 && npx squid-typeorm-migration generate || true
+ @ls db/migrations > db/migrations_dir_after.txt
+ @if ! diff db/migrations_dir_before.txt db/migrations_dir_after.txt > /dev/null ; then \
+ rm db/migrations/*-Views.js; \
+ node db/generateViewsMigration.js; \
+ fi;
+ @docker rm temp_migrations_db -vf || true
+ @docker volume rm temp_migrations_db_volume || true
+ @rm db/migrations_dir_before.txt db/migrations_dir_after.txt || true
+
 codegen:
  @npm run generate:schema || true
  @npx squid-typeorm-codegen
 
 setup-recommendations:
  @npm run recommendations:setup
 
+network-tests:
+ @npm run generate:network-tests || true
+
 typegen:
  @npx squid-substrate-typegen typegen.json
 
-prepare: install codegen build setup-recommendations
+prepare: install typegen codegen build setup-recommendations
 
 up-squid:
  @set -a; \