Commit
•
d6e48cb
1
Parent(s):
e81b7b3
[Added] - Special Tokens to indicate Farsi and English text has been added
Browse files- tokenizer.json +2 -2
- tokenizer_config.json +2 -2
tokenizer.json
CHANGED
|
@@ -50,7 +50,7 @@
|
|
| 50 |
},
|
| 51 |
{
|
| 52 |
"id": 36945,
|
| 53 |
-
"content": "<|
|
| 54 |
"single_word": false,
|
| 55 |
"lstrip": false,
|
| 56 |
"rstrip": true,
|
|
@@ -59,7 +59,7 @@
|
|
| 59 |
},
|
| 60 |
{
|
| 61 |
"id": 36946,
|
| 62 |
-
"content": "<|
|
| 63 |
"single_word": false,
|
| 64 |
"lstrip": false,
|
| 65 |
"rstrip": true,
|
|
|
|
| 50 |
},
|
| 51 |
{
|
| 52 |
"id": 36945,
|
| 53 |
+
"content": "<|EN|>",
|
| 54 |
"single_word": false,
|
| 55 |
"lstrip": false,
|
| 56 |
"rstrip": true,
|
|
|
|
| 59 |
},
|
| 60 |
{
|
| 61 |
"id": 36946,
|
| 62 |
+
"content": "<|FA|>",
|
| 63 |
"single_word": false,
|
| 64 |
"lstrip": false,
|
| 65 |
"rstrip": true,
|
tokenizer_config.json
CHANGED
|
@@ -43,7 +43,7 @@
|
|
| 43 |
"special": true
|
| 44 |
},
|
| 45 |
"36945": {
|
| 46 |
-
"content": "<|
|
| 47 |
"lstrip": false,
|
| 48 |
"normalized": false,
|
| 49 |
"rstrip": true,
|
|
@@ -51,7 +51,7 @@
|
|
| 51 |
"special": true
|
| 52 |
},
|
| 53 |
"36946": {
|
| 54 |
-
"content": "<|
|
| 55 |
"lstrip": false,
|
| 56 |
"normalized": false,
|
| 57 |
"rstrip": true,
|
|
|
|
| 43 |
"special": true
|
| 44 |
},
|
| 45 |
"36945": {
|
| 46 |
+
"content": "<|EN|>",
|
| 47 |
"lstrip": false,
|
| 48 |
"normalized": false,
|
| 49 |
"rstrip": true,
|
|
|
|
| 51 |
"special": true
|
| 52 |
},
|
| 53 |
"36946": {
|
| 54 |
+
"content": "<|FA|>",
|
| 55 |
"lstrip": false,
|
| 56 |
"normalized": false,
|
| 57 |
"rstrip": true,
|