leveldevai commited on
Commit
13a0de3
1 Parent(s): 388af87

Upload 4 files

Browse files
Files changed (3) hide show
  1. special_tokens_map.json +1 -1
  2. tokenizer.json +0 -18
  3. tokenizer_config.json +2 -21
special_tokens_map.json CHANGED
@@ -7,7 +7,7 @@
7
  "single_word": false
8
  },
9
  "eos_token": {
10
- "content": "<|im_end|>",
11
  "lstrip": false,
12
  "normalized": false,
13
  "rstrip": false,
 
7
  "single_word": false
8
  },
9
  "eos_token": {
10
+ "content": "</s>",
11
  "lstrip": false,
12
  "normalized": false,
13
  "rstrip": false,
tokenizer.json CHANGED
@@ -29,24 +29,6 @@
29
  "rstrip": false,
30
  "normalized": false,
31
  "special": true
32
- },
33
- {
34
- "id": 32000,
35
- "content": "<|im_end|>",
36
- "single_word": false,
37
- "lstrip": false,
38
- "rstrip": false,
39
- "normalized": false,
40
- "special": true
41
- },
42
- {
43
- "id": 32001,
44
- "content": "<|im_start|>",
45
- "single_word": false,
46
- "lstrip": false,
47
- "rstrip": false,
48
- "normalized": false,
49
- "special": true
50
  }
51
  ],
52
  "normalizer": {
 
29
  "rstrip": false,
30
  "normalized": false,
31
  "special": true
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
32
  }
33
  ],
34
  "normalizer": {
tokenizer_config.json CHANGED
@@ -23,37 +23,18 @@
23
  "rstrip": false,
24
  "single_word": false,
25
  "special": true
26
- },
27
- "32000": {
28
- "content": "<|im_end|>",
29
- "lstrip": false,
30
- "normalized": false,
31
- "rstrip": false,
32
- "single_word": false,
33
- "special": true
34
- },
35
- "32001": {
36
- "content": "<|im_start|>",
37
- "lstrip": false,
38
- "normalized": false,
39
- "rstrip": false,
40
- "single_word": false,
41
- "special": true
42
  }
43
  },
44
  "additional_special_tokens": [],
45
  "bos_token": "<s>",
46
- "chat_template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
47
  "clean_up_tokenization_spaces": false,
48
- "eos_token": "<|im_end|>",
49
  "legacy": true,
50
  "model_max_length": 1000000000000000019884624838656,
51
  "pad_token": null,
52
  "sp_model_kwargs": {},
53
  "spaces_between_special_tokens": false,
54
  "tokenizer_class": "LlamaTokenizer",
55
- "trust_remote_code": false,
56
  "unk_token": "<unk>",
57
- "use_default_system_prompt": true,
58
- "use_fast": true
59
  }
 
23
  "rstrip": false,
24
  "single_word": false,
25
  "special": true
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
26
  }
27
  },
28
  "additional_special_tokens": [],
29
  "bos_token": "<s>",
 
30
  "clean_up_tokenization_spaces": false,
31
+ "eos_token": "</s>",
32
  "legacy": true,
33
  "model_max_length": 1000000000000000019884624838656,
34
  "pad_token": null,
35
  "sp_model_kwargs": {},
36
  "spaces_between_special_tokens": false,
37
  "tokenizer_class": "LlamaTokenizer",
 
38
  "unk_token": "<unk>",
39
+ "use_default_system_prompt": false
 
40
  }