defaultConfig.js

// @flow
import path from "path";
import os from "os";

import EnvironmentHelper from "./helpers/EnvironmentHelper";
import { DELETE_MODE_TRASH } from "./types/DeleteModeTypes";
import {
  TYPE_ARCHIVE,
  TYPE_AUDIO,
  TYPE_IMAGE,
  TYPE_VIDEO,
  TYPE_TEXT,
  TYPE_SCRAP
} from "./types/ClassifyTypes";
import type { DeepLearningConfig, DefaultConfig } from "./types";
import {
  // MODEL_FACE_RECOGNITION,
  MODEL_AGE_GENDER,
  // eslint-disable-next-line no-unused-vars
  MODEL_FACE_LANDMARK_68,
  // eslint-disable-next-line no-unused-vars
  MODEL_FACE_EXPRESSION,
  MODEL_SSD_MOBILENETV1,
  CLASS_NAME_PORN,
  CLASS_NAME_SEXY,
  CLASS_NAME_HENTAI
} from "./types/DeepLearningTypes";

const dbTableName = "hash";
const facePPDbTableName = "facepp";

const facePPConfig = {
  facePPJudgeFunction: result => {
    return result.faces.some(face => {
      if (face.attributes.gender.value === "Male") {
        return false;
      }
      return true;
    });
  },
  /*
  facePPJudgeFunction: result => {
    // eslint-disable-next-line complexity
    return result.faces.some(face => {
      const { attributes: a } = face;
      if (a.gender.value === "Male") {
        return false;
      }
      if (face.face_rectangle.width < 300 && face.face_rectangle.height < 300) {
        return false;
      }
      if (a.age.value > 40) {
        return false;
      }
      if (a.blur.blurness.value > a.blur.blurness.threshold) {
        return false;
      }
      if (a.blur.motionblur.value > a.blur.motionblur.threshold) {
        return false;
      }
      if (a.blur.gaussianblur.value > a.blur.gaussianblur.threshold) {
        return false;
      }
      if (a.beauty.male_score < 70) {
        return false;
      }
      return true;
    });
  },
  */
  facePPResizedImageSize: 640,
  facePPDomain: "api-us.faceplusplus.com",
  facePPDetectApiPath: "facepp/v3/detect",
  facePPApiKey: "dummyapikeypleaseset",
  facePPApiSecret: "",
  facePPFaceAttributes: [
    "gender",
    "age",
    "smiling",
    "headpose",
    "facequality",
    "blur",
    "eyestatus",
    "emotion",
    "ethnicity",
    "beauty",
    "mouthstatus",
    "eyegaze",
    "skinstatus"
  ],
  facePPDbVersion: 2,
  facePPDbTableName: "facepp",
  facePPDbCreateTableSql: `CREATE TABLE IF NOT EXISTS ${facePPDbTableName} (${[
    "hash text",
    "image_id text",
    "face_token text",
    "face_num integer",
    "version text",
    "landmark text",
    "emotion_sadness real",
    "emotion_neutral real",
    "emotion_disgust real",
    "emotion_anger real",
    "emotion_surprise real",
    "emotion_fear real",
    "emotion_happiness real",
    "beauty_female_score real",
    "beauty_male_score real",
    "gender text",
    "age integer",
    "mouth_close real",
    "mouth_surgical_mask_or_respirator real",
    "mouth_open real",
    "mouth_other_occlusion real",
    "glass text",
    "skin_dark_circle real",
    "skin_stain real",
    "skin_acne real",
    "skin_health real",
    "headpose_yaw_angle real",
    "headpose_pitch_angle real",
    "headpose_roll_angle real",
    "gaussianblur real",
    "motionblur real",
    "blurness real",
    "smile real",
    "eye_status_left_normal_glass_eye_open real",
    "eye_status_left_normal_glass_eye_close real",
    "eye_status_left_no_glass_eye_close real",
    "eye_status_left_no_glass_eye_open real",
    "eye_status_left_occlusion real",
    "eye_status_left_dark_glasses real",
    "eye_status_right_normal_glass_eye_open real",
    "eye_status_right_normal_glass_eye_close real",
    "eye_status_right_no_glass_eye_close real",
    "eye_status_right_no_glass_eye_open real",
    "eye_status_right_occlusion real",
    "eye_status_right_dark_glasses real",
    "eyegaze_right_position_x_coordinate real",
    "eyegaze_right_position_y_coordinate real",
    "eyegaze_right_vector_z real",
    "eyegaze_right_vector_x real",
    "eyegaze_right_vector_y real",
    "eyegaze_left_position_x_coordinate real",
    "eyegaze_left_position_y_coordinate real",
    "eyegaze_left_vector_z real",
    "eyegaze_left_vector_x real",
    "eyegaze_left_vector_y real",
    "facequality integer",
    "ethnicity text",
    "top integer",
    "left integer",
    "width integer",
    "height integer",
    "PRIMARY KEY (face_token, hash)"
  ].join(",")})`,
  facePPDbCreateIndexSqls: [
    `CREATE INDEX IF NOT EXISTS facepp_hash_idx ON ${facePPDbTableName} (hash);`,
    `CREATE INDEX IF NOT EXISTS age_idx ON ${facePPDbTableName} (age);`,
    `CREATE INDEX IF NOT EXISTS facequality_idx ON ${facePPDbTableName} (facequality);`,
    `CREATE INDEX IF NOT EXISTS smile_idx ON ${facePPDbTableName} (smile);`,
    `CREATE INDEX IF NOT EXISTS glass_idx ON ${facePPDbTableName} (glass);`,
    `CREATE INDEX IF NOT EXISTS gender_idx ON ${facePPDbTableName} (gender);`,
    `CREATE INDEX IF NOT EXISTS beauty_female_idx ON ${facePPDbTableName} (beauty_female_score);`,
    `CREATE INDEX IF NOT EXISTS beauty_male_idx ON ${facePPDbTableName} (beauty_male_score);`,
    `CREATE INDEX IF NOT EXISTS height_idx ON ${facePPDbTableName} (height);`,
    `CREATE INDEX IF NOT EXISTS width_idx ON ${facePPDbTableName} (width);`,
    `CREATE INDEX IF NOT EXISTS ethnicity_idx ON ${facePPDbTableName} (ethnicity);`
  ]
};

const log4jsConfig = {
  appenders: {
    out: {
      type: "stdout",
      layout: { type: "pattern", pattern: "%[[%p] %c%] - %m" }
    },
    file: {
      type: "dateFile",
      filename: path.join(
        EnvironmentHelper.getHomeDir(),
        ".dedupper",
        "log",
        "process"
      ),
      pattern: ".yyyy-MM-dd.log",
      alwaysIncludePattern: true,
      // daysToKeep: 365,
      layout: {
        type: "pattern",
        pattern: "[%d][%z][%p] %c - %m"
      }
    }
  },
  categories: {
    default: { appenders: ["out", "file"], level: "info" }
  }
};

const faceApiDbTableName = "face";
const nsfwJsDbTableName = "nsfw_js";
const faceApiModelBaseUrl =
  "https://github.com/justadudewhohacks/face-api.js-models/raw/master/";
const deepLearningFaceApiConfig = {
  faceApiDbCreateTableSql: `CREATE TABLE IF NOT EXISTS ${faceApiDbTableName} (${[
    "id integer primary key",
    "hash text",
    "version text",
    "gender integer",
    "gender_probability real",
    "age real",
    "box_x integer",
    "box_y integer",
    "box_w integer",
    "box_h integer"
  ].join(",")})`,
  faceApiDbCreateIndexSqls: [
    `CREATE INDEX IF NOT EXISTS face_api_hash_idx ON ${faceApiDbTableName} (hash);`,
    `CREATE INDEX IF NOT EXISTS age_idx ON ${faceApiDbTableName} (age);`
  ],
  faceApiUseModels: [
    // MODEL_FACE_RECOGNITION,
    MODEL_FACE_EXPRESSION,
    MODEL_AGE_GENDER,
    MODEL_FACE_LANDMARK_68,
    MODEL_SSD_MOBILENETV1
  ],
  faceApiModelBasePath: path.join(
    EnvironmentHelper.getHomeDir(),
    ".dedupper/model"
  ),
  faceApiModelUrlsByName: {
    face_expression: [
      `${faceApiModelBaseUrl}face_expression/face_expression_model-shard1`,
      `${faceApiModelBaseUrl}face_expression/face_expression_model-weights_manifest.json`
    ],
    age_gender_model: [
      `${faceApiModelBaseUrl}age_gender_model/age_gender_model-shard1`,
      `${faceApiModelBaseUrl}age_gender_model/age_gender_model-weights_manifest.json`
    ],
    face_landmark_68: [
      `${faceApiModelBaseUrl}face_landmark_68/face_landmark_68_model-shard1`,
      `${faceApiModelBaseUrl}face_landmark_68/face_landmark_68_model-weights_manifest.json`
    ],
    face_landmark_68_tiny: [
      `${faceApiModelBaseUrl}face_landmark_68_tiny/face_landmark_68_tiny_model-shard1`,
      `${faceApiModelBaseUrl}face_landmark_68_tiny/face_landmark_68_tiny_model-weights_manifest.json`
    ],
    face_recognition: [
      `${faceApiModelBaseUrl}face_recognition/face_recognition_model-shard1`,
      `${faceApiModelBaseUrl}face_recognition/face_recognition_model-shard2`,
      `${faceApiModelBaseUrl}face_recognition/face_recognition_model-weights_manifest.json`
    ],
    mtcnn: [
      `${faceApiModelBaseUrl}mtcnn/mtcnn_model-shard1`,
      `${faceApiModelBaseUrl}mtcnn/mtcnn_model-weights_manifest.json`
    ],
    ssd_mobilenetv1: [
      `${faceApiModelBaseUrl}ssd_mobilenetv1/ssd_mobilenetv1_model-shard1`,
      `${faceApiModelBaseUrl}ssd_mobilenetv1/ssd_mobilenetv1_model-shard2`,
      `${faceApiModelBaseUrl}ssd_mobilenetv1/ssd_mobilenetv1_model-weights_manifest.json`
    ],
    tiny_face_detector: [
      `${faceApiModelBaseUrl}ssd_mobilenetv1/ssd_mobilenetv1_model-shard1`,
      `${faceApiModelBaseUrl}ssd_mobilenetv1/ssd_mobilenetv1_model-weights_manifest.json`
    ]
  }
};

const deepLearningApiConfig = {
  faceSpinnerApi: [
    "http://localhost:7000/detect",
    "http://localhost:7001/detect",
    "http://localhost:7002/detect",
    "http://localhost:7003/detect"
  ],
  nsfwApi: [
    "http://localhost:6000/image",
    "http://localhost:6001/image",
    "http://localhost:6002/image",
    "http://localhost:6003/image",
    "http://localhost:6004/image",
    "http://localhost:6005/image"
  ],
  faceDetectWithGenderApi: [
    "http://localhost:5100/face/detect",
    "http://localhost:5101/face/detect"
  ],
  facePredictAgeApi: [
    "http://localhost:5000/face/predict",
    "http://localhost:5001/face/predict",
    "http://localhost:5002/face/predict",
    "http://localhost:5003/face/predict",
    "http://localhost:5004/face/predict",
    "http://localhost:5005/face/predict"
  ]
};

// eslint-disable-next-line no-unused-vars
const deepLearningConfigSfwAndNoFace = {
  ...deepLearningApiConfig,
  instantDelete: false,
  logicalOperation: "and",
  nsfwType: "nsfw",
  nsfwMode: "disallow",
  nsfwModeNoneDefault: true,
  nsfwThreshold: 0.1,
  nsfwPostJudgeFunction: fileInfo => {
    if (fileInfo.nsfwJs) {
      let score = 0;
      fileInfo.nsfwJs.results.forEach(({ className, probability }) => {
        if (
          [CLASS_NAME_PORN, CLASS_NAME_SEXY, CLASS_NAME_HENTAI].includes(
            className
          )
        ) {
          score += probability;
        }
      });
      const isLowScore = score < 0.003;
      if (isLowScore) {
        return true;
      }
    }
    return false;
  },
  faceCategories: [
    ["M", "(4, 6)"],
    ["M", "(8, 12)"],
    ["M", "(15, 20)"],
    ["M", "(25, 32)"],
    ["M", "(38, 43)"],
    ["M", "(48, 53)"],
    ["M", "(60, 100)"],
    ["F", "(4, 6)"],
    ["F", "(8, 12)"],
    ["F", "(15, 20)"],
    ["F", "(25, 32)"],
    ["F", "(38, 43)"],
    ["F", "(48, 53)"],
    ["F", "(60, 100)"]
  ],
  faceMode: "disallow",
  faceModeNoneDefault: true,
  faceMinLongSide: 450
};

// eslint-disable-next-line no-unused-vars
const deepLearningConfigNsfwOrFemaleFace = {
  ...deepLearningApiConfig,
  instantDelete: false,
  logicalOperation: "or",
  nsfwType: "nsfw",
  nsfwMode: "allow",
  nsfwThreshold: 0.1,
  faceCategories: [
    ["F", "(4, 6)"],
    ["F", "(8, 12)"],
    ["F", "(15, 20)"],
    ["F", "(25, 32)"],
    ["F", "(38, 43)"],
    ["F", "(48, 53)"]
  ],
  faceMode: "allow",
  faceModeNoneDefault: true,
  faceMinLongSide: 450
};

const deepLearningConfig: DeepLearningConfig = {
  // nsfwBackEnd: "NSFWJS",
  nsfwJsDbVersion: 1,
  nsfwJsDbTableName,
  nsfwJsDbCreateTableSql: `CREATE TABLE IF NOT EXISTS ${nsfwJsDbTableName} (${[
    "hash text primary key",
    "neutral real",
    "drawing real",
    "hentai real",
    "porn real",
    "sexy real",
    "porn_sexy real",
    "hentai_porn_sexy real",
    "hentai_sexy real",
    "hentai_porn real",
    "version integer"
  ].join(",")})`,
  nsfwJsDbCreateIndexSqls: [
    // `CREATE INDEX IF NOT EXISTS hash_idx ON ${nsfwJsDbTableName} (hash);`,
    `CREATE INDEX IF NOT EXISTS porn_idx ON ${nsfwJsDbTableName} (porn);`,
    `CREATE INDEX IF NOT EXISTS sexy_idx ON ${nsfwJsDbTableName} (sexy);`,
    `CREATE INDEX IF NOT EXISTS porn_sexy_idx ON ${nsfwJsDbTableName} (porn_sexy);`,
    `CREATE INDEX IF NOT EXISTS hentai_porn_sexy_idx ON ${nsfwJsDbTableName} (hentai_porn_sexy);`,
    `CREATE INDEX IF NOT EXISTS hentai_sexy_idx ON ${nsfwJsDbTableName} (hentai_sexy);`,
    `CREATE INDEX IF NOT EXISTS hentai_porn_idx ON ${nsfwJsDbTableName} (hentai_porn);`
  ],
  nsfwBackEnd: "OpenNSFW",
  nsfwJsJudgeFunction: results => {
    let score = 0;
    results.forEach(({ className, probability }) => {
      if (
        [CLASS_NAME_PORN, CLASS_NAME_SEXY, CLASS_NAME_HENTAI].includes(
          className
        )
      ) {
        score += probability;
      }
    });
    return score < 0.5; // accept sfw image only
  },
  savePredictionResults: true,
  faceBackEnd: "face-api.js",
  tfjsBackEnd: "cpu",
  ...deepLearningFaceApiConfig,
  ...deepLearningConfigSfwAndNoFace,
  ...facePPConfig
};

const tagDbName = "tag";
const processStateDbName = "process_state";
const tagDbLength = 128;
const tagDbColumns = [
  "hash text primary key",
  ...Array.from({ length: tagDbLength }).map((n, i) => `t${i + 1} integer`)
];
const tagDbCreateIndexSqls = Array.from({ length: tagDbLength }).map(
  (n, i) =>
    `CREATE INDEX IF NOT EXISTS tag_t${i + 1}_idx ON ${tagDbName} (t${i + 1});`
);
const defaultConfig: DefaultConfig = {
  recovery: true,
  amazonDriveApiUrl: "https://www.amazon.com/drive/v1/",
  amazonLoginUrl: "https://www.amazon.com/clouddrive?mgh=1",
  amazonDriveBaseDir: "/Backup",
  amazonUser: "",
  amazonPassword: "",
  amazonBaseDir: path.join(EnvironmentHelper.getHomeDir(), ".dedupper/amazon"),
  serverHttpsPort: 8443,
  serverPort: 8080,
  channelDbName: "channel",
  channelDbCreateTableSql: `CREATE TABLE IF NOT EXISTS channel (${[
    "id text primary key",
    "name text",
    "sql text"
  ].join(",")})`,
  tagDbLength,
  tagDbName,
  tagDbCreateTableSql: `CREATE TABLE IF NOT EXISTS ${tagDbName} (${tagDbColumns.join(
    ","
  )})`,
  tagDbCreateIndexSqls,
  processStateDbName,
  processStateDbCreateTableSql: `CREATE TABLE IF NOT EXISTS ${processStateDbName} (${[
    "hash text primary key",
    "meta text",
    "missing integer",
    "orientation integer",
    "trim text",
    "view_date integer",
    "view_count integer",
    "rating integer",
    "score integer",
    "feature integer",
    "detect integer",
    "nsfwjs integer",
    "facepp integer",
    "facepp_face_count integer",
    "acd_id text",
    "acd_md5 text"
  ].join(",")})`,
  processStateSkipFunction: () => false,
  processStateDbCreateIndexSqls: [
    `CREATE INDEX IF NOT EXISTS process_state_missing_idx ON ${processStateDbName} (missing);`,
    `CREATE INDEX IF NOT EXISTS process_state_view_date_idx ON ${processStateDbName} (view_date);`,
    `CREATE INDEX IF NOT EXISTS process_state_rating_idx ON ${processStateDbName} (rating);`,
    `CREATE INDEX IF NOT EXISTS process_state_score_idx ON ${processStateDbName} (score);`,
    `CREATE INDEX IF NOT EXISTS process_state_feature_idx ON ${processStateDbName} (feature);`,
    `CREATE INDEX IF NOT EXISTS process_state_detect_idx ON ${processStateDbName} (detect);`,
    `CREATE INDEX IF NOT EXISTS process_state_nsfwjs_idx ON ${processStateDbName} (nsfwjs);`,
    `CREATE INDEX IF NOT EXISTS process_state_facepp_idx ON ${processStateDbName} (facepp);`,
    `CREATE INDEX IF NOT EXISTS process_state_facepp_face_count_idx ON ${processStateDbName} (facepp_face_count);`
  ],
  cacheVersion: 2,
  noTransfer: true,
  archiveExtract: false,
  archiveExtractCommand: '"C:\\Program Files (x86)\\LhaForge\\LhaForge.exe" /e',
  instantDelete: false,
  sortMarksFunction: () => [],
  deepLearningConfig,
  useFileName: false,
  fileNameWhiteList: [
    /^[ 0-9_-]+$/,
    "mov",
    "movie",
    "video",
    /^(video|mov|movie|)[ 0-9_-]+$/i
  ],
  useImageMagickHash: true,
  dummyPath: "?",
  log4jsConfig,
  maxCpuLoadPercent: 60,
  maxWorkers: os.cpus().length,
  hashAlgorithm: "sha1",
  defaultLogLevel: "debug",
  deleteMode: DELETE_MODE_TRASH,
  dbBasePath: path.join(EnvironmentHelper.getHomeDir(), ".dedupper/db"),
  dbTableName,
  dbCreateTableSql: `CREATE TABLE IF NOT EXISTS ${dbTableName} (${[
    "hash text primary key",
    "p_hash text",
    "d_hash text",
    "width integer",
    "height integer",
    "ratio real",
    "timestamp integer not null",
    "name text not null",
    "to_path text not null",
    "from_path text not null",
    "size integer not null",
    "state integer not null",
    "process_state text"
  ].join(",")})`,
  dbCreateIndexSqls: [
    `CREATE INDEX IF NOT EXISTS p_hash_idx ON ${dbTableName} (p_hash);`,
    `CREATE INDEX IF NOT EXISTS ratio_state_idx ON ${dbTableName} (ratio, state);`,
    `CREATE INDEX IF NOT EXISTS name_idx ON ${dbTableName} (name);`,
    `CREATE INDEX IF NOT EXISTS hash_width_idx ON ${dbTableName} (width);`,
    `CREATE INDEX IF NOT EXISTS hash_height_idx ON ${dbTableName} (height);`,
    `CREATE INDEX IF NOT EXISTS timestamp_idx ON ${dbTableName} (timestamp);`,
    `CREATE INDEX IF NOT EXISTS size_idx ON ${dbTableName} (size);`,
    `CREATE INDEX IF NOT EXISTS state_idx ON ${dbTableName} (state);`,
    `CREATE INDEX IF NOT EXISTS process_state_idx ON ${dbTableName} (process_state);`,
    `CREATE INDEX IF NOT EXISTS to_path_idx ON ${dbTableName} (to_path);`
  ],
  ignoreVideoDamage: false,
  ignoreAudioDamage: false,
  pHashIgnoreSameDir: true,
  dHashExactThreshold: 8,
  pHashExactThreshold: 3,
  pHashSearchThreshold: 11,
  pHashSearchRatioRangeOffset: 0.2,
  meanExactThreshold: 3500,
  relativeResolutionRatioThreshold: 0.95,
  relativeFileSizeRatioThreshold: 0.66,
  forceTransfer: false,
  renameRules: [
    [/(src|sandbox|projects)\\dedupper\\/, "\\"],
    [/\\new folder\\/gi, ""],
    [
      new RegExp(
        `${["\\\\Users", process.env.USERNAME].join("\\\\")}\\\\`,
        "i"
      ),
      "\\"
    ]
  ],
  ngDirPathPatterns: [/\.bak\\/],
  ngFileNamePatterns: [".DS_store", "Thumbs.db", ".BridgeSort"],
  baseLibraryPathByType: {
    [TYPE_IMAGE]: "B:\\Image",
    [TYPE_VIDEO]: "B:\\Video",
    [TYPE_AUDIO]: "B:\\Audio",
    [TYPE_TEXT]: "B:\\Document"
  },
  libraryPathHourOffset: 0,
  minFileSizeByType: {
    [TYPE_IMAGE]: 1024 * 25,
    [TYPE_VIDEO]: 1024 * 1024 * 2
  },
  minResolutionByType: {
    [TYPE_IMAGE]: 500 * 500,
    [TYPE_VIDEO]: 320 * 240
  },
  minLongSideByType: {
    [TYPE_IMAGE]: 540,
    [TYPE_VIDEO]: 320
  },
  classifyTypeByExtension: (() => {
    const lookup = {};
    const assignFn = (ext, type) => {
      lookup[ext] = type;
    };
    `aac
ac3
ape
flac
m1a
m2a
m4a
mka
mp3
ogg
wav
wma
tak
tta
alac
wv`
      .split("\n")
      .filter(Boolean)
      .forEach(e => assignFn(e, TYPE_AUDIO));
    `jpg
jpeg
png`
      .split("\n")
      .filter(Boolean)
      .forEach(e => assignFn(e, TYPE_IMAGE));

    `3gp
asf
avi
divx
flv
m1v
m2v
m4v
mts
mkv
mod
mov
mp4
mpeg
mpg
ogm
rm
rmvb
ts
vob
webm
wmv`
      .split("\n")
      .filter(Boolean)
      .forEach(e => assignFn(e, TYPE_VIDEO));

    `lnk
!ut
1
url
log
xmp`
      .split("\n")
      .filter(Boolean)
      .forEach(e => assignFn(e, TYPE_SCRAP));

    `zip
rar
tgz
gz
lzh
lha
bz2
7z`
      .split("\n")
      .filter(Boolean)
      .forEach(e => assignFn(e, TYPE_ARCHIVE));

    /*
    `doc
docx
xls
xlsx
ppt
pptx
rtf
html
htm
vsd
ai
xps`
      .split("\n")
      .filter(Boolean)
      .forEach(e => assignFn(e, TYPE_TEXT));
    */

    return lookup;
  })()
};

export default defaultConfig;