flopml
/

mamba

flpelerin commited on Aug 23, 2024

Commit

415566b

1 Parent(s): 7bfd78e

Update file dataset.cli.py

Files changed (1) hide show

dataset.cli.py CHANGED Viewed

@@ -18,7 +18,7 @@ if __name__ == '__main__':
     parser.add_argument('-p', '--remote_path')
     parser.add_argument('-o', '--output_file', default='dataset.txt')
-    dataset = load_dataset(parser.dataset_path)
     text = ''.join(s for s in dataset['train']['text']).encode('ascii', 'ignore').decode('ascii')
     with open(parser.output_file, 'w') as f:

     parser.add_argument('-p', '--remote_path')
     parser.add_argument('-o', '--output_file', default='dataset.txt')
+    dataset = load_dataset(parser.remote_path)
     text = ''.join(s for s in dataset['train']['text']).encode('ascii', 'ignore').decode('ascii')
     with open(parser.output_file, 'w') as f: