diff --git a/src/open_clip/tokenizer.py b/src/open_clip/tokenizer.py index 3e651aed5..954ebb6d2 100644 --- a/src/open_clip/tokenizer.py +++ b/src/open_clip/tokenizer.py @@ -250,7 +250,7 @@ def random_mask_tokenize(texts: Union[str, List[str]], context_length: int = 77) if len(tokens) > context_length - 2: # 2 for sot and eot token indices = np.random.permutation(len(tokens)).tolist() indices = indices[:context_length - 2] - tokens = tokens[indices] + tokens = [tokens[i] for i in indices] tokens = [sot_token,] + tokens + [eot_token,] result[i, :len(tokens)] = torch.tensor(tokens) @@ -350,4 +350,4 @@ def get_order(x): tokens[-1] = eot_token result[i, :len(tokens)] = torch.tensor(tokens) - return result \ No newline at end of file + return result