From fb837d73e37b4c45f78890573342bddd03c8d3b7 Mon Sep 17 00:00:00 2001 From: Akos Balasko Date: Tue, 20 Aug 2024 12:43:22 +0200 Subject: [PATCH] option to remove utf tags --- README.md | 1 + config.json | 1 + config.logseq.json | 1 + config.tana.json | 1 + package-lock.json | 58 ++++++++++------------------- src/YarleOptions.ts | 1 + src/ui/index.html | 13 +++++++ src/ui/settingsMapper.ts | 1 + src/utils/content-utils.ts | 2 +- src/yarle.ts | 1 + test/data/test-invalid-tags-keep.md | 12 ++++++ test/data/test-invalid-tags.enex | 2 +- test/yarle-special-cases.spec.ts | 27 ++++++++++++++ 13 files changed, 80 insertions(+), 41 deletions(-) create mode 100644 test/data/test-invalid-tags-keep.md diff --git a/README.md b/README.md index d07ee40f..e01baa9e 100644 --- a/README.md +++ b/README.md @@ -229,6 +229,7 @@ The following configurational properties are available: |```useLevenshteinForLinks```| boolean| it applies the link to the note with the filename that has the closest Levenshtein distance to the text of the link |```keepEvernoteLinkIfNoNoteFound```| boolean | it keeps the link to the evernote note if no such note found among the converted files |```convertColorsToMDHighlight```| boolean | it converts colored highlights and custom font spans to standard markdown highlights +|```removeUnicodeCharsFromTags```| boolean | it removes the unicode characters (like éáűőú) from tags Metadata settings can be set via the template. diff --git a/config.json b/config.json index 682cc475..1be42adb 100644 --- a/config.json +++ b/config.json @@ -20,6 +20,7 @@ "skipWebClips": true, "skipTags": false, "useHashTags": true, + "removeUnicodeCharsFromTags": false, "outputFormat": "ObsidianMD", "taskOutputFormat": "ObsidianMD", "imageSizeFormat": "ObsidianMD", diff --git a/config.logseq.json b/config.logseq.json index ef3e2355..957ad34a 100644 --- a/config.logseq.json +++ b/config.logseq.json @@ -9,6 +9,7 @@ "plainTextNotesOnly": false, "skipWebClips": true, "useHashTags": false, + "removeUnicodeCharsFromTags": false, "nestedTags": { "separatorInEN": "_", "replaceSeparatorWith": "---", diff --git a/config.tana.json b/config.tana.json index 916a993b..466893a4 100644 --- a/config.tana.json +++ b/config.tana.json @@ -35,6 +35,7 @@ "useLevenshteinForLinks": false, "keepEvernoteLinkIfNoNoteFound": false, "convertColorsToMDHighlight": false, + "removeUnicodeCharsFromTags": false, "replacementChar": "_", "replacementCharacterMap": { "<": "_", diff --git a/package-lock.json b/package-lock.json index f53cdc6a..f7724392 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1822,7 +1822,6 @@ "version": "2.10.0", "resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.10.0.tgz", "integrity": "sha512-Cn6WYCm0tXv8p6k+A8PvbDG763EDpBoTzHdA+Q/MF6H3sapGjCm9NzoaJncJS9tUKSuCoDs9XHxYYsQDgxR6kw==", - "dev": true, "optional": true, "dependencies": { "@types/node": "*" @@ -2684,7 +2683,6 @@ "version": "3.2.0", "resolved": "https://registry.npmjs.org/boolean/-/boolean-3.2.0.tgz", "integrity": "sha512-d0II/GO9uf9lfUHH2BQsjxzRJZBdsjgsBiW4BvhWk/3qoKwQFjIDVN19PfX8F2D/r9PCMTtLWjYVCFrpeYUzsw==", - "dev": true, "optional": true }, "node_modules/bottleneck": { @@ -4472,7 +4470,7 @@ "version": "1.1.4", "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.4.tgz", "integrity": "sha512-uckOqKcfaVvtBdsVkdPv3XjveQJsNQqmhXgRi8uhvWWuPYZCNlzT8qAyblUgNoXdHdjMTzAqeGjAoli8f+bzPA==", - "dev": true, + "devOptional": true, "dependencies": { "has-property-descriptors": "^1.0.0", "object-keys": "^1.1.1" @@ -4561,7 +4559,6 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/detect-node/-/detect-node-2.1.0.tgz", "integrity": "sha512-T0NIuQpnTvFDATNuHN5roPwSBG83rFsuO+MXXH9/3N1eFbn4wcPjttvjMLEPWJ0RGUYgQE7cGgS3tNxbqCGM7g==", - "dev": true, "optional": true }, "node_modules/diff": { @@ -5419,7 +5416,7 @@ "version": "4.1.1", "resolved": "https://registry.npmjs.org/es6-error/-/es6-error-4.1.1.tgz", "integrity": "sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg==", - "dev": true + "devOptional": true }, "node_modules/escalade": { "version": "3.1.1", @@ -6619,7 +6616,7 @@ "version": "1.2.0", "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.0.tgz", "integrity": "sha512-L049y6nFOuom5wGyRc3/gdTLO94dySVKRACj1RmJZBQXlbTMhtNIgkWkUHq+jYmZvKf14EW1EoJnnjbmoHij0Q==", - "dev": true, + "devOptional": true, "dependencies": { "function-bind": "^1.1.1", "has": "^1.0.3", @@ -6732,7 +6729,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/global-agent/-/global-agent-3.0.0.tgz", "integrity": "sha512-PT6XReJ+D07JvGoxQMkT6qji/jVNfX/h364XHZOWeRzy64sSFr+xJ5OX7LI3b4MPQzdL4H8Y8M0xzPpsVMwA8Q==", - "dev": true, "optional": true, "dependencies": { "boolean": "^3.0.1", @@ -6801,7 +6797,7 @@ "version": "1.0.3", "resolved": "https://registry.npmjs.org/globalthis/-/globalthis-1.0.3.tgz", "integrity": "sha512-sFdI5LyBiNTHjRd7cGPWapiHWMOXKyuBNX/cWJ3NfzrZQVa8GI/8cofCl74AOVqq9W5kNmguTIzJ/1s2gyI9wA==", - "dev": true, + "devOptional": true, "dependencies": { "define-properties": "^1.1.3" }, @@ -6988,7 +6984,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.0.tgz", "integrity": "sha512-62DVLZGoiEBDHQyqG4w9xCuZ7eJEwNmJRWw2VY84Oedb7WFcA27fiEVe8oUQx9hAUJ4ekurquucTGwsyO1XGdQ==", - "dev": true, + "devOptional": true, "dependencies": { "get-intrinsic": "^1.1.1" }, @@ -7012,7 +7008,7 @@ "version": "1.0.3", "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==", - "dev": true, + "devOptional": true, "engines": { "node": ">= 0.4" }, @@ -8161,7 +8157,7 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", "integrity": "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==", - "dev": true + "devOptional": true }, "node_modules/json5": { "version": "2.2.3", @@ -8691,7 +8687,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/matcher/-/matcher-3.0.0.tgz", "integrity": "sha512-OkeDaAZ/bQCxeFAozM55PKcKU0yJMPGifLwV4Qgjitu+5MoAfSQN4lsLJeXZ1b8w0x+/Emda6MZgXS1jvsapng==", - "dev": true, "optional": true, "dependencies": { "escape-string-regexp": "^4.0.0" @@ -8704,7 +8699,6 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", - "dev": true, "optional": true, "engines": { "node": ">=10" @@ -13029,7 +13023,7 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", - "dev": true, + "devOptional": true, "engines": { "node": ">= 0.4" } @@ -14291,7 +14285,6 @@ "version": "2.15.4", "resolved": "https://registry.npmjs.org/roarr/-/roarr-2.15.4.tgz", "integrity": "sha512-CHhPh+UNHD2GTXNYhPWLnU8ONHdI+5DI+4EYIAOaiD63rHeYlZvyh8P+in5999TTSFgUYuKUAjzRI4mdh/p+2A==", - "dev": true, "optional": true, "dependencies": { "boolean": "^3.0.1", @@ -14629,7 +14622,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/semver-compare/-/semver-compare-1.0.0.tgz", "integrity": "sha512-YM3/ITh2MJ5MtzaM429anh+x2jiLVjqILF4m4oyQB18W7Ggea7BfqdH/wGMK7dDiMghv/6WG7znWMwUDzJiXow==", - "dev": true, "optional": true }, "node_modules/semver-diff": { @@ -14669,7 +14661,6 @@ "version": "7.0.1", "resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-7.0.1.tgz", "integrity": "sha512-8I8TjW5KMOKsZQTvoxjuSIa7foAwPWGOts+6o7sgjz41/qMD9VQHEDxi6PBvK2l0MXUmqZyNpUK+T2tQaaElvw==", - "dev": true, "optional": true, "dependencies": { "type-fest": "^0.13.1" @@ -14685,7 +14676,6 @@ "version": "0.13.1", "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.13.1.tgz", "integrity": "sha512-34R7HTnG0XIJcBSn5XhDd7nNFPRcXYRZrBB2O2jdKqYODldSzBAqzsWoZYYvduky73toYS/ESqxPvkDf/F0XMg==", - "dev": true, "optional": true, "engines": { "node": ">=10" @@ -14954,6 +14944,7 @@ "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "devOptional": true, "engines": { "node": ">=0.10.0" } @@ -15048,7 +15039,6 @@ "version": "1.1.2", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.2.tgz", "integrity": "sha512-VE0SOVEHCk7Qc8ulkWw3ntAzXuqf7S2lvwQaDLRnUeIEaKNQJzV6BwmLKhOqT61aGhfUMrXeaBk+oDGCzvhcug==", - "dev": true, "optional": true }, "node_modules/ssri": { @@ -18275,7 +18265,6 @@ "version": "2.10.0", "resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.10.0.tgz", "integrity": "sha512-Cn6WYCm0tXv8p6k+A8PvbDG763EDpBoTzHdA+Q/MF6H3sapGjCm9NzoaJncJS9tUKSuCoDs9XHxYYsQDgxR6kw==", - "dev": true, "optional": true, "requires": { "@types/node": "*" @@ -18900,7 +18889,6 @@ "version": "3.2.0", "resolved": "https://registry.npmjs.org/boolean/-/boolean-3.2.0.tgz", "integrity": "sha512-d0II/GO9uf9lfUHH2BQsjxzRJZBdsjgsBiW4BvhWk/3qoKwQFjIDVN19PfX8F2D/r9PCMTtLWjYVCFrpeYUzsw==", - "dev": true, "optional": true }, "bottleneck": { @@ -20245,7 +20233,7 @@ "version": "1.1.4", "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.4.tgz", "integrity": "sha512-uckOqKcfaVvtBdsVkdPv3XjveQJsNQqmhXgRi8uhvWWuPYZCNlzT8qAyblUgNoXdHdjMTzAqeGjAoli8f+bzPA==", - "dev": true, + "devOptional": true, "requires": { "has-property-descriptors": "^1.0.0", "object-keys": "^1.1.1" @@ -20309,7 +20297,6 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/detect-node/-/detect-node-2.1.0.tgz", "integrity": "sha512-T0NIuQpnTvFDATNuHN5roPwSBG83rFsuO+MXXH9/3N1eFbn4wcPjttvjMLEPWJ0RGUYgQE7cGgS3tNxbqCGM7g==", - "dev": true, "optional": true }, "diff": { @@ -20996,7 +20983,7 @@ "version": "4.1.1", "resolved": "https://registry.npmjs.org/es6-error/-/es6-error-4.1.1.tgz", "integrity": "sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg==", - "dev": true + "devOptional": true }, "escalade": { "version": "3.1.1", @@ -21913,7 +21900,7 @@ "version": "1.2.0", "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.0.tgz", "integrity": "sha512-L049y6nFOuom5wGyRc3/gdTLO94dySVKRACj1RmJZBQXlbTMhtNIgkWkUHq+jYmZvKf14EW1EoJnnjbmoHij0Q==", - "dev": true, + "devOptional": true, "requires": { "function-bind": "^1.1.1", "has": "^1.0.3", @@ -21998,7 +21985,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/global-agent/-/global-agent-3.0.0.tgz", "integrity": "sha512-PT6XReJ+D07JvGoxQMkT6qji/jVNfX/h364XHZOWeRzy64sSFr+xJ5OX7LI3b4MPQzdL4H8Y8M0xzPpsVMwA8Q==", - "dev": true, "optional": true, "requires": { "boolean": "^3.0.1", @@ -22047,7 +22033,7 @@ "version": "1.0.3", "resolved": "https://registry.npmjs.org/globalthis/-/globalthis-1.0.3.tgz", "integrity": "sha512-sFdI5LyBiNTHjRd7cGPWapiHWMOXKyuBNX/cWJ3NfzrZQVa8GI/8cofCl74AOVqq9W5kNmguTIzJ/1s2gyI9wA==", - "dev": true, + "devOptional": true, "requires": { "define-properties": "^1.1.3" } @@ -22183,7 +22169,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.0.tgz", "integrity": "sha512-62DVLZGoiEBDHQyqG4w9xCuZ7eJEwNmJRWw2VY84Oedb7WFcA27fiEVe8oUQx9hAUJ4ekurquucTGwsyO1XGdQ==", - "dev": true, + "devOptional": true, "requires": { "get-intrinsic": "^1.1.1" } @@ -22198,7 +22184,7 @@ "version": "1.0.3", "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==", - "dev": true + "devOptional": true }, "has-tostringtag": { "version": "1.0.0", @@ -23031,7 +23017,7 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", "integrity": "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==", - "dev": true + "devOptional": true }, "json5": { "version": "2.2.3", @@ -23462,7 +23448,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/matcher/-/matcher-3.0.0.tgz", "integrity": "sha512-OkeDaAZ/bQCxeFAozM55PKcKU0yJMPGifLwV4Qgjitu+5MoAfSQN4lsLJeXZ1b8w0x+/Emda6MZgXS1jvsapng==", - "dev": true, "optional": true, "requires": { "escape-string-regexp": "^4.0.0" @@ -23472,7 +23457,6 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", - "dev": true, "optional": true } } @@ -26602,7 +26586,7 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", - "dev": true + "devOptional": true }, "object.assign": { "version": "4.1.4", @@ -27551,7 +27535,6 @@ "version": "2.15.4", "resolved": "https://registry.npmjs.org/roarr/-/roarr-2.15.4.tgz", "integrity": "sha512-CHhPh+UNHD2GTXNYhPWLnU8ONHdI+5DI+4EYIAOaiD63rHeYlZvyh8P+in5999TTSFgUYuKUAjzRI4mdh/p+2A==", - "dev": true, "optional": true, "requires": { "boolean": "^3.0.1", @@ -27798,7 +27781,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/semver-compare/-/semver-compare-1.0.0.tgz", "integrity": "sha512-YM3/ITh2MJ5MtzaM429anh+x2jiLVjqILF4m4oyQB18W7Ggea7BfqdH/wGMK7dDiMghv/6WG7znWMwUDzJiXow==", - "dev": true, "optional": true }, "semver-diff": { @@ -27828,7 +27810,6 @@ "version": "7.0.1", "resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-7.0.1.tgz", "integrity": "sha512-8I8TjW5KMOKsZQTvoxjuSIa7foAwPWGOts+6o7sgjz41/qMD9VQHEDxi6PBvK2l0MXUmqZyNpUK+T2tQaaElvw==", - "dev": true, "optional": true, "requires": { "type-fest": "^0.13.1" @@ -27838,7 +27819,6 @@ "version": "0.13.1", "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.13.1.tgz", "integrity": "sha512-34R7HTnG0XIJcBSn5XhDd7nNFPRcXYRZrBB2O2jdKqYODldSzBAqzsWoZYYvduky73toYS/ESqxPvkDf/F0XMg==", - "dev": true, "optional": true } } @@ -28046,7 +28026,8 @@ "source-map": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", - "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==" + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "devOptional": true }, "source-map-support": { "version": "0.5.21", @@ -28132,7 +28113,6 @@ "version": "1.1.2", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.2.tgz", "integrity": "sha512-VE0SOVEHCk7Qc8ulkWw3ntAzXuqf7S2lvwQaDLRnUeIEaKNQJzV6BwmLKhOqT61aGhfUMrXeaBk+oDGCzvhcug==", - "dev": true, "optional": true }, "ssri": { diff --git a/src/YarleOptions.ts b/src/YarleOptions.ts index 6d90c120..30d0e697 100644 --- a/src/YarleOptions.ts +++ b/src/YarleOptions.ts @@ -22,6 +22,7 @@ export interface YarleOptions { skipUpdateTime?: boolean; skipSourceUrl?: boolean; skipWebClips?: boolean; + removeUnicodeCharsFromTags?: boolean skipReminderTime?: boolean; skipReminderOrder?: boolean; skipReminderDoneTime?: boolean; diff --git a/src/ui/index.html b/src/ui/index.html index c67b98fc..fbf28e42 100644 --- a/src/ui/index.html +++ b/src/ui/index.html @@ -586,6 +586,19 @@
Tags
+
+ + + + +
+