From d709b895c9ce1f2100ec48f457946ab9e5ec29d0 Mon Sep 17 00:00:00 2001
From: Daulet Zhanguzin <daulet@zhanguzin.kz>
Date: Mon, 4 Nov 2024 20:09:47 -0800
Subject: [PATCH] fix: bump lib version otherwise bazel gets it confused with
 hf's tokenizers

---
 Cargo.Bazel.lock  | 22 +++++++++++-----------
 Cargo.lock        | 20 ++++++++++----------
 Cargo.toml        |  2 +-
 MODULE.bazel.lock |  8 ++++----
 4 files changed, 26 insertions(+), 26 deletions(-)

diff --git a/Cargo.Bazel.lock b/Cargo.Bazel.lock
index e319f995..ac0f8ff9 100644
--- a/Cargo.Bazel.lock
+++ b/Cargo.Bazel.lock
@@ -1,5 +1,5 @@
 {
-  "checksum": "274eda904d8c182522e7565c734cf8c608671ee2bdb346dccb07c10d44904563",
+  "checksum": "8c24cec6818168bbb4b50e04f24841ad2a219f25d0f3aab050fcb832fef53eef",
   "crates": {
     "aho-corasick 1.1.2": {
       "name": "aho-corasick",
@@ -4486,13 +4486,13 @@
       },
       "license": "Apache-2.0 OR MIT"
     },
-    "tokenizers 0.20.0": {
+    "tokenizers 0.20.2": {
       "name": "tokenizers",
-      "version": "0.20.0",
+      "version": "0.20.2",
       "repository": {
         "Http": {
-          "url": "https://static.crates.io/crates/tokenizers/0.20.0/download",
-          "sha256": "c8a24d7f7d6be5b9d1377418b893ab1808af0074f5d1bb2c64784452ddd2aa70"
+          "url": "https://static.crates.io/crates/tokenizers/0.20.2/download",
+          "sha256": "fef9821871de9277bd7d8a7568e798e775db7eede56e290b4b03f1dfb9ac6769"
         }
       },
       "targets": [
@@ -4628,13 +4628,13 @@
           ],
           "selects": {}
         },
-        "version": "0.20.0"
+        "version": "0.20.2"
       },
       "license": "Apache-2.0"
     },
-    "tokenizers 0.9.0": {
+    "tokenizers 1.20.2": {
       "name": "tokenizers",
-      "version": "0.9.0",
+      "version": "1.20.2",
       "repository": null,
       "targets": [],
       "library_target_name": null,
@@ -4649,7 +4649,7 @@
               "target": "libc"
             },
             {
-              "id": "tokenizers 0.20.0",
+              "id": "tokenizers 0.20.2",
               "target": "tokenizers"
             }
           ],
@@ -4669,7 +4669,7 @@
           "selects": {}
         },
         "edition": "2021",
-        "version": "0.9.0"
+        "version": "1.20.2"
       },
       "license": null
     },
@@ -6375,7 +6375,7 @@
   },
   "binary_crates": [],
   "workspace_members": {
-    "tokenizers 0.9.0": ""
+    "tokenizers 1.20.2": ""
   },
   "conditions": {
     "aarch64-apple-darwin": [
diff --git a/Cargo.lock b/Cargo.lock
index 0d84f02a..a146675b 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -821,16 +821,6 @@ dependencies = [
  "serde_json",
 ]
 
-[[package]]
-name = "tokenizers"
-version = "0.20.2"
-dependencies = [
- "criterion",
- "libc",
- "rand",
- "tokenizers 0.20.2 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
 [[package]]
 name = "tokenizers"
 version = "0.20.2"
@@ -863,6 +853,16 @@ dependencies = [
  "unicode_categories",
 ]
 
+[[package]]
+name = "tokenizers"
+version = "1.20.2"
+dependencies = [
+ "criterion",
+ "libc",
+ "rand",
+ "tokenizers 0.20.2",
+]
+
 [[package]]
 name = "unicode-ident"
 version = "1.0.8"
diff --git a/Cargo.toml b/Cargo.toml
index 5557e009..08ddffca 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -2,7 +2,7 @@
 name = "tokenizers"
 # Bump major version every time we change the behavior of the library.
 # Bump minor.patch version every time we bump tokenizers dependency version.
-version = "0.20.2"
+version = "1.20.2"
 edition = "2021"
 
 [lib]
diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock
index f5cd5310..21b358eb 100644
--- a/MODULE.bazel.lock
+++ b/MODULE.bazel.lock
@@ -34,8 +34,8 @@
     "https://bcr.bazel.build/modules/rules_cc/0.0.9/MODULE.bazel": "836e76439f354b89afe6a911a7adf59a6b2518fafb174483ad78a2a2fde7b1c5",
     "https://bcr.bazel.build/modules/rules_cc/0.0.9/source.json": "1f1ba6fea244b616de4a554a0f4983c91a9301640c8fe0dd1d410254115c8430",
     "https://bcr.bazel.build/modules/rules_java/4.0.0/MODULE.bazel": "5a78a7ae82cd1a33cef56dc578c7d2a46ed0dca12643ee45edbb8417899e6f74",
-    "https://bcr.bazel.build/modules/rules_java/7.6.1/MODULE.bazel": "2f14b7e8a1aa2f67ae92bc69d1ec0fa8d9f827c4e17ff5e5f02e91caa3b2d0fe",
-    "https://bcr.bazel.build/modules/rules_java/7.6.1/source.json": "8f3f3076554e1558e8e468b2232991c510ecbcbed9e6f8c06ac31c93bcf38362",
+    "https://bcr.bazel.build/modules/rules_java/7.6.5/MODULE.bazel": "481164be5e02e4cab6e77a36927683263be56b7e36fef918b458d7a8a1ebadb1",
+    "https://bcr.bazel.build/modules/rules_java/7.6.5/source.json": "a805b889531d1690e3c72a7a7e47a870d00323186a9904b36af83aa3d053ee8d",
     "https://bcr.bazel.build/modules/rules_jvm_external/4.4.2/MODULE.bazel": "a56b85e418c83eb1839819f0b515c431010160383306d13ec21959ac412d2fe7",
     "https://bcr.bazel.build/modules/rules_jvm_external/4.4.2/source.json": "a075731e1b46bc8425098512d038d416e966ab19684a10a34f4741295642fc35",
     "https://bcr.bazel.build/modules/rules_license/0.0.3/MODULE.bazel": "627e9ab0247f7d1e05736b59dbb1b6871373de5ad31c3011880b4133cafd4bd0",
@@ -56,8 +56,8 @@
     "https://bcr.bazel.build/modules/upb/0.0.0-20220923-a547704/source.json": "f1ef7d3f9e0e26d4b23d1c39b5f5de71f584dd7d1b4ef83d9bbba6ec7a6a6459",
     "https://bcr.bazel.build/modules/zlib/1.2.11/MODULE.bazel": "07b389abc85fdbca459b69e2ec656ae5622873af3f845e1c9d80fe179f3effa0",
     "https://bcr.bazel.build/modules/zlib/1.2.12/MODULE.bazel": "3b1a8834ada2a883674be8cbd36ede1b6ec481477ada359cd2d3ddc562340b27",
-    "https://bcr.bazel.build/modules/zlib/1.3/MODULE.bazel": "6a9c02f19a24dcedb05572b2381446e27c272cd383aed11d41d99da9e3167a72",
-    "https://bcr.bazel.build/modules/zlib/1.3/source.json": "b6b43d0737af846022636e6e255fd4a96fee0d34f08f3830e6e0bac51465c37c"
+    "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.3/MODULE.bazel": "af322bc08976524477c79d1e45e241b6efbeb918c497e8840b8ab116802dda79",
+    "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.3/source.json": "2be409ac3c7601245958cd4fcdff4288be79ed23bd690b4b951f500d54ee6e7d"
   },
   "selectedYankedVersions": {},
   "moduleExtensions": {