Update file dataset.cli.py
Browse files- dataset.cli.py +4 -2
dataset.cli.py
CHANGED
|
@@ -17,9 +17,11 @@ if __name__ == '__main__':
|
|
| 17 |
|
| 18 |
parser.add_argument('-p', '--remote_path')
|
| 19 |
parser.add_argument('-o', '--output_file', default='dataset.txt')
|
|
|
|
|
|
|
| 20 |
|
| 21 |
-
dataset = load_dataset(
|
| 22 |
text = ''.join(s for s in dataset['train']['text']).encode('ascii', 'ignore').decode('ascii')
|
| 23 |
|
| 24 |
-
with open(
|
| 25 |
f.write(text)
|
|
|
|
| 17 |
|
| 18 |
parser.add_argument('-p', '--remote_path')
|
| 19 |
parser.add_argument('-o', '--output_file', default='dataset.txt')
|
| 20 |
+
|
| 21 |
+
args = parser.parse_args()
|
| 22 |
|
| 23 |
+
dataset = load_dataset(args.remote_path)
|
| 24 |
text = ''.join(s for s in dataset['train']['text']).encode('ascii', 'ignore').decode('ascii')
|
| 25 |
|
| 26 |
+
with open(args.output_file, 'w') as f:
|
| 27 |
f.write(text)
|