drrobot9 commited on
Commit
7fdc060
·
verified ·
1 Parent(s): 22c55cf

Update special_tokens_map.json

Browse files
Files changed (1) hide show
  1. special_tokens_map.json +5 -0
special_tokens_map.json CHANGED
@@ -18,3 +18,8 @@
18
  "single_word": false
19
  }
20
  }
 
 
 
 
 
 
18
  "single_word": false
19
  }
20
  }
21
+ {text = re.sub(r'\*\*(.*?)\*\*', r'\1', text)
22
+ text = re.sub(r'(\*|_)(.*?)\1', r'\2', text)
23
+ text = re.sub(r'`(.*?)`', r'\1', text)
24
+ text = re.sub(r'^#+\s+', '', text, flags=re.MULTILINE)
25
+ return text}