102 lines
3.2 KiB
Python
102 lines
3.2 KiB
Python
import numpy as np
|
||
import datasets
|
||
from typing import Dict, Tuple
|
||
from datasets import Dataset, load_from_disk
|
||
from torch.utils.data import DataLoader
|
||
|
||
from datasets import load_dataset
|
||
|
||
# dataset = load_dataset("liwu/MNBVC", "wikipedia", split="train", streaming=True)
|
||
# dataset = load_dataset("liwu/MNBVC", "wikipedia", split="train")
|
||
|
||
# dataset = load_dataset("./liwu/MNBVC/MNBVC.py", "wikipedia", split="train")
|
||
dataset = load_dataset("json", data_files="./liwu/MNBVC/wiki/20230197/0.jsonl.gz")
|
||
|
||
# dataset = load_dataset("./liwu/MNBVC/wiki", split="train", streaming=True)
|
||
# dataset = load_dataset("./liwu/MNBVC/wiki", split="train")
|
||
|
||
|
||
from functools import partial
|
||
|
||
|
||
def format_inputs(examples):
|
||
p = examples["段落"]
|
||
mergeLine = ""
|
||
for line in p:
|
||
mergeLine += line["内容"] + "\n"
|
||
return {"text": mergeLine}
|
||
|
||
|
||
dataset = dataset.map(
|
||
partial(format_inputs),
|
||
batched=False,
|
||
num_proc=12,
|
||
remove_columns="段落",
|
||
)
|
||
|
||
d = dataset["train"][0]
|
||
|
||
# def split_raw_dataset(
|
||
# raw_dataset: datasets.DatasetDict,
|
||
# ) -> Tuple[datasets.Dataset, datasets.Dataset]:
|
||
# if 'validation' in raw_dataset:
|
||
# train_dataset, val_dataset = raw_dataset['train'], raw_dataset['validation']
|
||
# else:
|
||
# raw_dataset = raw_dataset['train'].train_test_split(test_size=0.05, seed=1234)
|
||
# train_dataset, val_dataset = raw_dataset['train'], raw_dataset['test']
|
||
# return train_dataset, val_dataset
|
||
|
||
# dataset = split_raw_dataset(dataset)
|
||
|
||
dataloader = DataLoader(dataset, batch_size=32, num_workers=4)
|
||
|
||
|
||
count = 0
|
||
for _ in dataloader:
|
||
count += 1
|
||
|
||
print(next(iter(dataset))) # get the first line
|
||
|
||
|
||
# {
|
||
# "文件名": "cleaned/zhwiki-20230320/folder_0/486404.txt",
|
||
# "是否待查文件": False,
|
||
# "是否重复文件": False,
|
||
# "文件大小": 388,
|
||
# "simhash": "5350070205913746051",
|
||
# "最长段落长度": 150,
|
||
# "段落数": 4,
|
||
# "去重段落数": 4,
|
||
# "低质量段落数": 0,
|
||
# "段落": [
|
||
# {
|
||
# "行号": 0,
|
||
# "是否重复": False,
|
||
# "是否跨文件重复": False,
|
||
# "md5": "b687acdad842a48d260f709e93e7df9b",
|
||
# "内容": "【昂斯尼区】",
|
||
# },
|
||
# {
|
||
# "行号": 2,
|
||
# "是否重复": False,
|
||
# "是否跨文件重复": False,
|
||
# "md5": "275fffb0a600559e5c37ee01f2f3cea3",
|
||
# "内容": "昂斯尼区（arrondissement de Ancenis；arondisamant Ankiniz）是法国大西洋卢瓦尔省所辖的一个旧区。总面积785平方公里，总人口61457，人口密度78人/平方公里（2012年）。主要城镇为昂斯尼。2017年1月，该区与沙托布里扬区合并为沙托布里扬-昂斯尼区。",
|
||
# },
|
||
# {
|
||
# "行号": 4,
|
||
# "是否重复": False,
|
||
# "是否跨文件重复": False,
|
||
# "md5": "3cf5a8dd2ea5a3feeafec5c5abeeea07",
|
||
# "内容": "== 辖区 ==",
|
||
# },
|
||
# {
|
||
# "行号": 5,
|
||
# "是否重复": False,
|
||
# "是否跨文件重复": False,
|
||
# "md5": "45ff2c3113b4cf9b4b7f0fe1e22131a7",
|
||
# "内容": "昂斯尼区曾辖有24个市镇。",
|
||
# },
|
||
# ],
|
||
# }
|