fix(tk:shared): defined metadata filters based on nature type

packages/shared/src/endpoints/MinimalEndpoint.ts

@@ -22,10 +22,10 @@ export interface TypeOfEndpointInstance<E extends MinimalEndpointInstance> {
     ? // eslint-disable-next-line @typescript-eslint/no-invalid-void-type
       void
     : {
-        [k in keyof NonNullable<E['Input']>]: NonNullable<
+        [K in keyof NonNullable<E['Input']>]: NonNullable<
           E['Input']
-        >[k] extends Codec<any, any, any>
-          ? runtimeType<NonNullable<E['Input']>[k]>
+        >[K] extends Codec<any, any, any>
+          ? runtimeType<NonNullable<E['Input']>[K]>
           : never;
       };
 }

platforms/tktrex/backend/io/metadata.io.ts

@@ -0,0 +1,134 @@
+import { APIError, toAPIError } from '@shared/errors/APIError';
+import { toValidationError } from '@shared/errors/ValidationError';
+import { Metadata } from '@shared/models/Metadata';
+import { string2Food } from '@shared/utils/food.utils';
+import {
+  FollowingVideoMetadata,
+  ForYouMetadata,
+  NativeMetadata,
+  ProfileMetadata,
+  SearchMetadata,
+  TKMetadata,
+} from '@tktrex/shared/models/metadata';
+import {
+  CreatorType,
+  FollowingType,
+  ForYouType,
+  NativeType,
+  ProfileType,
+  SearchType,
+} from '@tktrex/shared/models/Nature';
+import * as E from 'fp-ts/Either';
+import { pipe } from 'fp-ts/function';
+import {
+  FollowingVideoMetadataDB,
+  ForYouVideoMetadataDB,
+  NativeMetadataDB,
+  ProfileMetadataDB,
+  SearchMetadataDB,
+  TKMetadataDB,
+} from '../models/metadata';
+
+type SpecificM<M> = Omit<M, keyof Metadata | '_id' | 'publicKey'>;
+
+export const toTKNativeMetadata = (
+  { author, ...m }: SpecificM<NativeMetadataDB>,
+  meta: Metadata
+): NativeMetadata => {
+  return {
+    ...m,
+    ...meta,
+    author: author ?? undefined,
+  };
+};
+
+export const toProfileMetadata = (
+  m: SpecificM<ProfileMetadataDB>,
+  meta: Metadata
+): ProfileMetadata => {
+  return {
+    ...m,
+    ...meta,
+  };
+};
+
+export const toForYouMetadata = (
+  { author, music, hashtags, ...m }: SpecificM<ForYouVideoMetadataDB>,
+  meta: Metadata
+): ForYouMetadata => {
+  return {
+    ...m,
+    ...meta,
+    author: author ?? undefined,
+    music: music ?? undefined,
+    hashtags: hashtags ?? [],
+  };
+};
+
+export const toSearchMetadata = (
+  m: SpecificM<SearchMetadataDB>,
+  meta: Metadata
+): SearchMetadata => {
+  return { ...m, ...meta };
+};
+
+export const toFollowingMetadata = (
+  m: SpecificM<FollowingVideoMetadataDB>,
+  meta: Metadata
+): FollowingVideoMetadata => {
+  return { ...m, ...meta };
+};
+
+export const toTKMetadata = ({
+  publicKey,
+  _id,
+  id,
+  blang,
+  href,
+  savingTime,
+  clientTime,
+  experimentId,
+  researchTag,
+  ...m
+}: TKMetadataDB): E.Either<APIError, TKMetadata> => {
+  const meta: Metadata = {
+    id: id.substring(0, 10),
+    blang,
+    href,
+    supporter: string2Food(publicKey),
+    savingTime,
+    clientTime,
+    researchTag,
+    experimentId,
+  };
+
+  return pipe(
+    E.tryCatch(() => {
+      const mm: any = m;
+      switch (m.nature.type) {
+        case SearchType.value: {
+          return toSearchMetadata(mm, meta);
+        }
+        case ForYouType.value: {
+          return toForYouMetadata(mm, meta);
+        }
+        case FollowingType.value: {
+          return toFollowingMetadata(mm, meta);
+        }
+        case CreatorType.value:
+        case ProfileType.value: {
+          return toProfileMetadata(mm, meta);
+        }
+        case NativeType.value: {
+          return toTKNativeMetadata(mm, meta);
+        }
+      }
+    }, toAPIError),
+    E.chain((e) =>
+      pipe(
+        TKMetadata.decode(e),
+        E.mapLeft((e) => toValidationError(TKMetadata.name, e))
+      )
+    )
+  );
+};

platforms/tktrex/backend/models/metadata/ForYouMetadata.ts

@@ -1,16 +1,16 @@
-import { ForYouVideoMetadata } from '@tktrex/shared/models/metadata/ForYouMetadata';
+import { ForYouMetadata } from '@tktrex/shared/models/metadata/ForYouMetadata';
 import * as t from 'io-ts';
 
-const { supporter, ...metadataBaseProps } = ForYouVideoMetadata.types[0].props;
-export const ForYouVideoMetadataDB = t.strict(
+const { supporter, ...metadataBaseProps } = ForYouMetadata.types[0].props;
+export const ForYouMetadataDB = t.strict(
   {
     ...metadataBaseProps,
-    ...ForYouVideoMetadata.types[1].type.props,
-    ...ForYouVideoMetadata.types[2].props,
-    ...ForYouVideoMetadata.types[3].props,
+    ...ForYouMetadata.types[1].type.props,
+    ...ForYouMetadata.types[2].props,
+    ...ForYouMetadata.types[3].props,
     _id: t.any,
     publicKey: t.string,
   },
-  'ForYouVideoMetadataDB'
+  'ForYouMetadataDB'
 );
-export type ForYouVideoMetadataDB = t.TypeOf<typeof ForYouVideoMetadataDB>;
+export type ForYouMetadataDB = t.TypeOf<typeof ForYouMetadataDB>;

platforms/tktrex/backend/models/metadata/index.ts

@@ -1,6 +1,6 @@
 import * as t from 'io-ts';
 import { FollowingVideoMetadataDB } from './FollowingMetadata';
-import { ForYouVideoMetadataDB } from './ForYouMetadata';
+import { ForYouMetadataDB } from './ForYouMetadata';
 import { NativeMetadataDB } from './NativeMetadata';
 import { ProfileMetadataDB } from './ProfileMetadata';
 import { SearchMetadataDB } from './SearchMetadata';
@@ -9,10 +9,18 @@ export const TKMetadataDB = t.union(
   [
     NativeMetadataDB,
     SearchMetadataDB,
-    ForYouVideoMetadataDB,
+    ForYouMetadataDB,
     FollowingVideoMetadataDB,
     ProfileMetadataDB,
   ],
   'MetadataDB'
 );
 export type TKMetadataDB = t.TypeOf<typeof TKMetadataDB>;
+
+export {
+  ProfileMetadataDB,
+  NativeMetadataDB,
+  SearchMetadataDB,
+  FollowingVideoMetadataDB,
+  ForYouMetadataDB as ForYouVideoMetadataDB,
+};

platforms/tktrex/backend/routes/metadata.ts

@@ -1,30 +1,51 @@
+import { toAPIError } from '@shared/errors/APIError';
 import { decodeOrThrowRequest } from '@shared/endpoints/helper';
-import * as foodUtils from '@shared/utils/food.utils';
-import endpoints, {
-  ListMetadataResponse,
-} from '@tktrex/shared/endpoints/v2/metadata.endpoints';
+import endpoints from '@tktrex/shared/endpoints/v2/metadata.endpoints';
 import createDebug from 'debug';
+import * as A from 'fp-ts/Array';
+import { pipe } from 'fp-ts/lib/function';
+import * as TE from 'fp-ts/TaskEither';
+import * as E from 'fp-ts/lib/Either';
+import { toTKMetadata } from '../io/metadata.io';
 import _ from 'lodash';
 import * as automo from '../lib/automo';
+import { throwTE } from '@shared/utils/task.utils';
+import { AppError } from '@shared/errors/AppError';
+import moment from 'moment';
+import CSV from '../lib/CSV';
+import { ListMetadataOutput } from '@tktrex/shared/models/http/metadata/output/ListMetadata.output';
+import { ListMetadataQuery } from '@tktrex/shared/models/http/metadata/query/ListMetadata.query';
 
 const debug = createDebug('routes:public');
 
 // This variables is used as cap in every readLimit below
 const PUBLIC_AMOUNT_ELEMS = 100;
 
-const listMetadata = async (
-  req: any
-): Promise<{ json: ListMetadataResponse }> => {
+type ListMetadataResponse =
+  | { json: ListMetadataOutput }
+  | { headers: any; text: string };
+
+const listMetadata = async (req: any): Promise<ListMetadataResponse> => {
+  const { query } = decodeOrThrowRequest(
+    endpoints.ListMetadata,
+    req
+  ) as any as {
+    query: ListMetadataQuery;
+  };
+
+  debug('Filter metadata with query %O', query);
+
   const {
-    query: {
-      researchTag,
-      experimentId,
-      publicKey,
-      nature,
-      amount = PUBLIC_AMOUNT_ELEMS,
-      skip = 0,
-    },
-  } = decodeOrThrowRequest(endpoints.ListMetadata, req);
+    researchTag,
+    experimentId,
+    publicKey,
+    nature,
+    amount = PUBLIC_AMOUNT_ELEMS,
+    skip = 0,
+    format,
+  } = query;
+
+  debug('Filter metadata with query %O', query);
 
   const filter = {} as any;
   if (publicKey) {
@@ -36,29 +57,71 @@ const listMetadata = async (
   if (researchTag) {
     filter.researchTag = researchTag;
   }
+
   if (nature) {
     filter['nature.type'] = nature;
+    switch (nature) {
+      case 'search': {
+        const { query: q } = query;
+        if (q) {
+          filter.query = {
+            $regex: q,
+          };
+        }
+        break;
+      }
+    }
   }
 
   debug('Filtering metadata for %O (%d, %d)', filter, amount, skip);
 
-  const metadata = await automo
-    .getMetadataByFilter(filter, {
-      amount,
-      skip,
-    })
-    .then(({ totals, data }) => ({
-      totals,
-      data: data.map(({ publicKey, _id, id, ...m }) => ({
-        ...m,
-        id: id.substring(0, 10),
-        supporter: foodUtils.string2Food(publicKey),
-      })),
-    }));
-
-  debug('Fetched %d evidences of %d', _.size(metadata.data), metadata.totals);
-
-  return { json: metadata };
+  return pipe(
+    TE.tryCatch(
+      () =>
+        automo.getMetadataByFilter(filter, {
+          amount,
+          skip,
+        }),
+      toAPIError
+    ),
+    TE.chain(({ totals, data }) => {
+      debug('Metadata by %O, %d evidences', filter, _.size(data));
+      return pipe(
+        data.map(toTKMetadata),
+        A.sequence(E.Applicative),
+        E.map((d) => ({ data: d, totals })),
+        TE.fromEither
+      );
+    }),
+    TE.chain((metadata): TE.TaskEither<AppError, ListMetadataResponse> => {
+      if (format === 'csv') {
+        const csv = CSV.produceCSVv1(metadata.data);
+        let filename = `metadata`;
+        filename += experimentId ? `-experiment-${experimentId}` : '';
+        filename += researchTag ? `-research_tag-${researchTag}` : '';
+        filename += '-' + moment().format('YY-MM-DD') + '.csv';
+
+        debug(
+          'VideoCSV: produced %d bytes, returning %s',
+          _.size(csv),
+          filename
+        );
+
+        // if (!_.size(csv)) return { text: 'Error, Zorry: 🤷' };
+
+        return TE.right({
+          headers: {
+            'Content-Type': 'csv/text',
+            'Content-Disposition': `attachment; filename=${filename}`,
+          },
+          text: csv,
+        });
+      }
+
+      return TE.right({ json: metadata });
+    }),
+    throwTE
+  );
 };
 
 export { listMetadata };

platforms/tktrex/backend/tsconfig.json

@@ -28,6 +28,7 @@
   "include": [
     "./bin",
     "./lib",
+    "./io",
     "./models",
     "./routes",
     "./test",

platforms/tktrex/shared/src/arbitraries/Metadata.arb.ts

@@ -1,9 +1,9 @@
 import { getArbitrary } from 'fast-check-io-ts';
 import {
   SearchMetadata,
-  ForYouVideoMetadata,
+  ForYouMetadata,
   FollowingVideoMetadata,
-  MetadataBase,
+  TKMetadataBase,
   SearchMetadataResult,
   NativeMetadata,
 } from '../models/metadata';
@@ -12,7 +12,7 @@ import { fc } from '@shared/test';
 import { subDays } from 'date-fns';
 import * as t from 'io-ts';
 
-const metadataBaseProps = propsOmitType(MetadataBase, [
+const metadataBaseProps = propsOmitType(TKMetadataBase, [
   'id',
   'clientTime',
   'savingTime',
@@ -101,7 +101,7 @@ export const SearchMetaDataArb = (opts: {
  * ForYouMetadata arbitrary
  *
  **/
-const forYouMetadataProps = propsOmitType(ForYouVideoMetadata.types[2], []);
+const forYouMetadataProps = propsOmitType(ForYouMetadata.types[2], []);
 
 export const ForYouVideoMetaDataArb = getArbitrary(
   t.intersection([t.type(metadataBaseProps), t.type(forYouMetadataProps)]),