446 lines
18 KiB
Python
446 lines
18 KiB
Python
# This dict is all that matters in this file
|
|
# Each key in this dict corresponds to a sub-dataset
|
|
# Each key corresponds to a dict, which specify the config info for this sub-dataset
|
|
# Something like:
|
|
# {
|
|
# "law_judgement": {
|
|
# "files": [...],
|
|
# "feature_type": "TEXT_CORPUS_LEGACY",
|
|
# "description": "Texts from open law suits",
|
|
# },
|
|
# ...
|
|
# }
|
|
MNBVC_META = {}
|
|
|
|
base_url = 'https://huggingface.co/datasets/liwu/MNBVC/resolve/main'
|
|
def _flatten_list(input):
|
|
res = []
|
|
for i in input:
|
|
if type(i) == list:
|
|
res += _flatten_list(i)
|
|
else:
|
|
res.append(i)
|
|
return res
|
|
|
|
# =============================
|
|
# law_judgement
|
|
MNBVC_META['law_judgement'] = {}
|
|
MNBVC_META['law_judgement']['files'] = [
|
|
[f'{base_url}/law/judgement/{folder}/{i}.jsonl.gz' for i in range(1, count+1)]
|
|
for folder, count in [
|
|
('20230134', 49),
|
|
('20230135', 44),
|
|
('20230136', 44),
|
|
('20230137', 49),
|
|
('20230138', 41),
|
|
('20230139', 46),
|
|
('20230140', 46),
|
|
('20230141', 50),
|
|
]
|
|
]
|
|
MNBVC_META['law_judgement']['files'] = _flatten_list(MNBVC_META['law_judgement']['files'])
|
|
MNBVC_META['law_judgement']['feature_type'] = "TEXT_CORPUS_LEGACY"
|
|
MNBVC_META['law_judgement']['description'] = "Texts from open law suits"
|
|
|
|
_CONFIG2FORMAT = {
|
|
"law_judgement": "TEXT_CORPUS_LEGACY",
|
|
"gov_xuexiqiangguo": "TEXT_CORPUS",
|
|
"gov_report": "TEXT_CORPUS_LEGACY",
|
|
"co_ann_report": "TEXT_CORPUS_LEGACY",
|
|
"code_metadata": "TEXT_CORPUS_LEGACY",
|
|
}
|
|
|
|
|
|
# =============================
|
|
# gov_xuexiqiangguo
|
|
MNBVC_META['gov_xuexiqiangguo'] = {}
|
|
MNBVC_META['gov_xuexiqiangguo']['files'] = ['https://huggingface.co/datasets/liwu/MNBVC/resolve/main/gov/20230172/XueXiQiangGuo.jsonl.gz']
|
|
MNBVC_META['gov_xuexiqiangguo']['feature_type'] = "TEXT_CORPUS"
|
|
MNBVC_META['gov_xuexiqiangguo']['description'] = "Texts from government files (XueXiQiangGuo)"
|
|
|
|
|
|
# =============================
|
|
# gov_report
|
|
MNBVC_META['gov_report'] = {}
|
|
MNBVC_META['gov_report']['files'] = ['https://huggingface.co/datasets/liwu/MNBVC/resolve/main/gov/20230172/GovReport.jsonl.gz']
|
|
MNBVC_META['gov_report']['feature_type'] = "TEXT_CORPUS_LEGACY"
|
|
MNBVC_META['gov_report']['description'] = "Texts from government files (Gov Report)"
|
|
|
|
|
|
# =============================
|
|
# co_ann_report
|
|
MNBVC_META['co_ann_report'] = {}
|
|
MNBVC_META['co_ann_report']['files'] = [
|
|
[f'{base_url}/co_ann_report/{folder}/{i}.jsonl.gz' for i in range(count)]
|
|
for folder, count in [
|
|
('20230125', 49)
|
|
]
|
|
]
|
|
MNBVC_META['co_ann_report']['files'] = _flatten_list(MNBVC_META['co_ann_report']['files'])
|
|
MNBVC_META['co_ann_report']['feature_type'] = "TEXT_CORPUS_LEGACY"
|
|
MNBVC_META['co_ann_report']['description'] = "Texts from company annual reports"
|
|
|
|
|
|
# =============================
|
|
# code_metadata
|
|
MNBVC_META['code_metadata'] = {}
|
|
MNBVC_META['code_metadata']['files'] = [
|
|
[f'{base_url}/code/metadata/{folder}/{(start+i)*1000000}-{(start+i+1)*1000000}.jsonl.gz' for i in range(start, start+count)]
|
|
for folder, start, count in [
|
|
('20230301', 0, 10),
|
|
('20230302', 10, 10),
|
|
('20230303', 20, 10),
|
|
('20230304', 30, 10),
|
|
('20230305', 40, 10),
|
|
('20230306', 50, 10),
|
|
('20230307', 60, 10),
|
|
('20230308', 70, 10),
|
|
('20230309', 80, 10),
|
|
('20230310', 90, 10),
|
|
('20230311', 100, 10),
|
|
('20230312', 110, 10),
|
|
('20230313', 120, 10),
|
|
('20230314', 130, 10),
|
|
('20230315', 140, 10),
|
|
('20230316', 150, 10),
|
|
('20230317', 160, 10),
|
|
('20230318', 170, 10),
|
|
('20230319', 180, 10),
|
|
('20230320', 190, 10),
|
|
]
|
|
]
|
|
MNBVC_META['code_metadata']['files'] = _flatten_list(MNBVC_META['code_metadata']['files'])
|
|
MNBVC_META['code_metadata']['feature_type'] = "TEXT_CORPUS_LEGACY"
|
|
MNBVC_META['code_metadata']['description'] = "Meta data for the code corpus"
|
|
|
|
|
|
# =============================
|
|
# qa_zhihu
|
|
MNBVC_META['qa_zhihu'] = {}
|
|
MNBVC_META['qa_zhihu']['files'] = [
|
|
[f'{base_url}/qa/{folder}/zhihu/{i}.jsonl.gz' for i in range(count)]
|
|
for folder, count in [('20230196', 5)]
|
|
]
|
|
MNBVC_META['qa_zhihu']['files'] = _flatten_list(MNBVC_META['qa_zhihu']['files'])
|
|
MNBVC_META['qa_zhihu']['feature_type'] = "QA_CORPUS"
|
|
MNBVC_META['qa_zhihu']['description'] = "QA data from zhihu"
|
|
|
|
|
|
# =============================
|
|
# qa_wikihow
|
|
MNBVC_META['qa_wikihow'] = {}
|
|
MNBVC_META['qa_wikihow']['files'] = [
|
|
[f'{base_url}/qa/{folder}/wikihow/wikihow_en.{i}.jsonl.gz' for i in range(count)]
|
|
for folder, count in [('20230196', 4)]
|
|
]
|
|
MNBVC_META['qa_wikihow']['files'] += [
|
|
[f'{base_url}/qa/{folder}/wikihow/wikihow_zh.{i}.jsonl.gz' for i in range(count)]
|
|
for folder, count in [('20230196', 1)]
|
|
]
|
|
MNBVC_META['qa_wikihow']['files'] = _flatten_list(MNBVC_META['qa_wikihow']['files'])
|
|
MNBVC_META['qa_wikihow']['feature_type'] = "QA_CORPUS"
|
|
MNBVC_META['qa_wikihow']['description'] = "QA data from wikihow (both in zh and en)"
|
|
|
|
|
|
# =============================
|
|
# qa_mfa
|
|
MNBVC_META['qa_mfa'] = {}
|
|
MNBVC_META['qa_mfa']['files'] = [
|
|
[f'{base_url}/qa/{folder}/mfa/{i}.jsonl.gz' for i in range(count)]
|
|
for folder, count in [('20230196', 1)]
|
|
]
|
|
MNBVC_META['qa_mfa']['files'] = _flatten_list(MNBVC_META['qa_mfa']['files'])
|
|
MNBVC_META['qa_mfa']['feature_type'] = "QA_CORPUS"
|
|
MNBVC_META['qa_mfa']['description'] = "QA data from Ministry of Foreign Affairs"
|
|
|
|
|
|
# =============================
|
|
# news_peoples_daily
|
|
MNBVC_META['news_peoples_daily'] = {}
|
|
MNBVC_META['news_peoples_daily']['files'] = [
|
|
[f'{base_url}/news/{folder}/{i}.jsonl.gz' for i in range(count)]
|
|
for folder, count in [('20230196', 12)]
|
|
]
|
|
MNBVC_META['news_peoples_daily']['files'] = _flatten_list(MNBVC_META['news_peoples_daily']['files'])
|
|
MNBVC_META['news_peoples_daily']['feature_type'] = "TEXT_CORPUS"
|
|
MNBVC_META['news_peoples_daily']['description'] = "News data from people's daily"
|
|
|
|
|
|
# =============================
|
|
# wikipedia
|
|
MNBVC_META['wikipedia'] = {}
|
|
MNBVC_META['wikipedia']['files'] = [
|
|
[f'{base_url}/wiki/{folder}/{i}.jsonl.gz' for i in range(count)]
|
|
for folder, count in [('20230197', 46)]
|
|
]
|
|
MNBVC_META['wikipedia']['files'] += [
|
|
[f'{base_url}/wiki/{folder}/{i}.jsonl.gz' for i in range(beg, end)]
|
|
for folder, beg, end in [('20230198', 46, 78)]
|
|
]
|
|
MNBVC_META['wikipedia']['files'] = _flatten_list(MNBVC_META['wikipedia']['files'])
|
|
MNBVC_META['wikipedia']['feature_type'] = "TEXT_CORPUS"
|
|
MNBVC_META['wikipedia']['description'] = "Texts from wikipedia"
|
|
|
|
|
|
# =============================
|
|
# qa_stackexchange
|
|
MNBVC_META['qa_stackexchange'] = {}
|
|
MNBVC_META['qa_stackexchange']['files'] = []
|
|
|
|
MNBVC_META['qa_stackexchange']['files'] += [
|
|
[f'{base_url}/qa/stackexchange/{folder}/{name}/{i}.jsonl.gz' for i in range(1, count+1)]
|
|
for folder, name, count in [
|
|
("20230204", "conlang.20230204.33.问答", 1),
|
|
("20230204", "aviation.20230204.10.问答", 1),
|
|
("20230204", "drupal.20230204.46.问答", 1),
|
|
("20230204", "gamedev.20230204.66.问答", 1),
|
|
("20230204", "linguistics.20230204.95.问答", 1),
|
|
("20230204", "datascience.20230204.41.问答", 1),
|
|
("20230204", "coffee.20230204.30.问答", 1),
|
|
("20230204", "german.20230204.70.问答", 1),
|
|
("20230204", "italian.20230204.85.问答", 1),
|
|
("20230204", "health.20230204.75.问答", 1),
|
|
("20230204", "gaming.20230204.67.问答", 2),
|
|
("20230204", "dba.20230204.42.问答", 2),
|
|
("20230204", "bitcoin.20230204.17.问答", 1),
|
|
("20230204", "anime.20230204.5.问答", 1),
|
|
("20230204", "iot.20230204.82.问答", 1),
|
|
("20230204", "avpaviation.20230204.11.问答", 1),
|
|
("20230204", "hsm.20230204.80.问答", 1),
|
|
("20230204", "english.20230204.56.问答", 3),
|
|
("20230204", "eosio.20230204.57.问答", 1),
|
|
("20230204", "astronomy.20230204.9.问答", 1),
|
|
("20230204", "materials.20230204.99.问答", 1),
|
|
("20230204", "homebrew.20230204.79.问答", 1),
|
|
("20230204", "elementaryos.20230204.52.问答", 1),
|
|
("20230204", "korean.20230204.90.问答", 1),
|
|
("20230204", "academia.20230204.2.问答", 2),
|
|
("20230204", "expatriates.20230204.61.问答", 1),
|
|
("20230204", "islam.20230204.84.问答", 1),
|
|
("20230204", "cogsci.20230204.31.问答", 1),
|
|
("20230204", "ethereum.20230204.60.问答", 1),
|
|
("20230204", "cardano.20230204.22.问答", 1),
|
|
("20230204", "bioinformatics.20230204.15.问答", 1),
|
|
("20230204", "magento.20230204.97.问答", 2),
|
|
("20230204", "literature.20230204.96.问答", 1),
|
|
("20230204", "ebooks.20230204.49.问答", 1),
|
|
("20230204", "devops.20230204.43.问答", 1),
|
|
("20230204", "economics.20230204.50.问答", 1),
|
|
("20230204", "chemistry.20230204.23.问答", 1),
|
|
("20230204", "judaism.20230204.89.问答", 1),
|
|
("20230204", "iota.20230204.83.问答", 1),
|
|
("20230204", "ell.20230204.53.问答", 2),
|
|
("20230204", "diy.20230204.44.问答", 2),
|
|
("20230204", "chess.20230204.24.问答", 1),
|
|
("20230204", "boardgames.20230204.19.问答", 1),
|
|
("20230204", "freelancing.20230204.64.问答", 1),
|
|
("20230204", "computergraphics.20230204.32.问答", 1),
|
|
("20230204", "engineering.20230204.55.问答", 1),
|
|
("20230204", "hermeneutics.20230204.76.问答", 1),
|
|
("20230204", "codereview.20230204.29.问答", 4),
|
|
("20230204", "civicrm.20230204.27.问答", 1),
|
|
("20230204", "expressionengine.20230204.62.问答", 1),
|
|
("20230204", "hinduism.20230204.77.问答", 1),
|
|
("20230204", "lifehacks.20230204.94.问答", 1),
|
|
("20230204", "chinese.20230204.25.问答", 1),
|
|
("20230204", "interpersonal.20230204.81.问答", 1),
|
|
("20230204", "blender.20230204.18.问答", 1),
|
|
("20230204", "emacs.20230204.54.问答", 1),
|
|
("20230204", "cstheory.20230204.40.问答", 1),
|
|
("20230204", "history.20230204.78.问答", 1),
|
|
("20230204", "earthscience.20230204.48.问答", 1),
|
|
("20230204", "askubuntu.20230204.8.问答", 5),
|
|
("20230204", "gis.20230204.71.问答", 2),
|
|
("20230204", "japanese.20230204.87.问答", 1),
|
|
("20230204", "gardening.20230204.68.问答", 1),
|
|
("20230204", "biology.20230204.16.问答", 1),
|
|
("20230204", "joomla.20230204.88.问答", 1),
|
|
("20230204", "drones.20230204.45.问答", 1),
|
|
("20230204", "latin.20230204.92.问答", 1),
|
|
("20230204", "buddhism.20230204.21.问答", 1),
|
|
("20230204", "apple.20230204.6.问答", 2),
|
|
("20230204", "ai.20230204.3.问答", 1),
|
|
("20230204", "es.20230204.58.问答", 4),
|
|
("20230204", "cooking.20230204.34.问答", 1),
|
|
("20230204", "crypto.20230204.37.问答", 1),
|
|
("20230204", "christianity.20230204.26.问答", 1),
|
|
("20230204", "bricks.20230204.20.问答", 1),
|
|
("20230204", "bicycles.20230204.13.问答", 1),
|
|
("20230204", "codegolf.20230204.28.问答", 4),
|
|
("20230204", "martialarts.20230204.98.问答", 1),
|
|
("20230204", "dsp.20230204.47.问答", 1),
|
|
("20230204", "bioacoustics.20230204.14.问答", 1),
|
|
("20230204", "electronics.20230204.51.问答", 4),
|
|
("20230204", "crafts.20230204.36.问答", 1),
|
|
("20230204", "ham.20230204.73.问答", 1),
|
|
("20230204", "cs.20230204.38.问答", 1),
|
|
("20230204", "beer.20230204.12.问答", 1),
|
|
("20230204", "graphicdesign.20230204.72.问答", 1),
|
|
("20230204", "french.20230204.65.问答", 1),
|
|
("20230204", "languagelearning.20230204.91.问答", 1),
|
|
("20230204", "genealogy.20230204.69.问答", 1),
|
|
("20230204", "3dprinting.20230204.1.问答", 1),
|
|
("20230204", "hardwarerecs.20230204.74.问答", 1),
|
|
("20230204", "law.20230204.93.问答", 1),
|
|
("20230204", "fitness.20230204.63.问答", 1),
|
|
("20230204", "esperanto.20230204.59.问答", 1),
|
|
("20230204", "craftcms.20230204.35.问答", 1),
|
|
("20230204", "android.20230204.4.问答", 1),
|
|
("20230204", "arduino.20230204.7.问答", 1),
|
|
("20230204", "ja.20230204.86.问答", 1),
|
|
("20230204", "cseducators.20230204.39.问答", 1),
|
|
("20230205", "writers.20230205.80.问答", 1),
|
|
("20230205", "scifi.20230205.43.问答", 2),
|
|
("20230205", "opensource.20230205.15.问答", 1),
|
|
("20230205", "outdoors.20230205.17.问答", 1),
|
|
("20230205", "quantumcomputing.20230205.32.问答", 1),
|
|
("20230205", "salesforce.20230205.41.问答", 2),
|
|
("20230205", "philosophy.20230205.21.问答", 1),
|
|
("20230205", "or.20230205.16.问答", 1),
|
|
("20230205", "tex.20230205.63.问答", 5),
|
|
("20230205", "music.20230205.10.问答", 1),
|
|
("20230205", "russian.20230205.40.问答", 1),
|
|
("20230205", "superuser.20230205.61.问答", 6),
|
|
("20230205", "tridion.20230205.67.问答", 1),
|
|
("20230205", "stackapps.20230205.57.问答", 1),
|
|
("20230205", "webmasters.20230205.74.问答", 1),
|
|
("20230205", "musicfans.20230205.11.问答", 1),
|
|
("20230205", "pm.20230205.24.问答", 1),
|
|
("20230205", "sitecore.20230205.47.问答", 1),
|
|
("20230205", "space.20230205.53.问答", 1),
|
|
("20230205", "sharepoint.20230205.46.问答", 2),
|
|
("20230205", "windowsphone.20230205.75.问答", 1),
|
|
("20230205", "unix.20230205.69.问答", 4),
|
|
("20230205", "mathematica.20230205.3.问答", 2),
|
|
("20230205", "ux.20230205.70.问答", 1),
|
|
("20230205", "moderators.20230205.6.问答", 1),
|
|
("20230205", "vi.20230205.72.问答", 1),
|
|
("20230205", "sustainability.20230205.62.问答", 1),
|
|
("20230205", "rus.20230205.39.问答", 1),
|
|
("20230205", "money.20230205.8.问答", 1),
|
|
("20230205", "travel.20230205.66.问答", 1),
|
|
("20230205", "tezos.20230205.64.问答", 1),
|
|
("20230205", "movies.20230205.9.问答", 1),
|
|
("20230205", "retrocomputing.20230205.34.问答", 1),
|
|
("20230205", "robotics.20230205.36.问答", 1),
|
|
("20230205", "mathoverflow.20230205.4.问答", 3),
|
|
("20230205", "rpg.20230205.37.问答", 2),
|
|
("20230205", "networkengineering.20230205.13.问答", 1),
|
|
("20230205", "monero.20230205.7.问答", 1),
|
|
("20230205", "math.20230205.1.问答", 20),
|
|
("20230205", "quant.20230205.31.问答", 1),
|
|
("20230205", "photo.20230205.22.问答", 1),
|
|
("20230205", "scicomp.20230205.42.问答", 1),
|
|
("20230205", "softwarerecs.20230205.50.问答", 1),
|
|
("20230205", "patents.20230205.19.问答", 1),
|
|
("20230205", "workplace.20230205.78.问答", 2),
|
|
("20230205", "sqa.20230205.56.问答", 1),
|
|
("20230205", "portuguese.20230205.27.问答", 1),
|
|
("20230205", "woodworking.20230205.76.问答", 1),
|
|
("20230205", "tor.20230205.65.问答", 1),
|
|
("20230205", "worldbuilding.20230205.79.问答", 3),
|
|
("20230205", "reverseengineering.20230205.35.问答", 1),
|
|
("20230205", "proofassistants.20230205.28.问答", 1),
|
|
("20230205", "spanish.20230205.54.问答", 1),
|
|
("20230205", "security.20230205.44.问答", 2),
|
|
("20230205", "skeptics.20230205.48.问答", 1),
|
|
("20230205", "sound.20230205.52.问答", 1),
|
|
("20230205", "wordpress.20230205.77.问答", 2),
|
|
("20230205", "mythology.20230205.12.问答", 1),
|
|
("20230205", "serverfault.20230205.45.问答", 5),
|
|
("20230205", "pets.20230205.20.问答", 1),
|
|
("20230205", "solana.20230205.51.问答", 1),
|
|
("20230205", "webapps.20230205.73.问答", 1),
|
|
("20230205", "vegetarianism.20230205.71.问答", 1),
|
|
("20230205", "puzzling.20230205.30.问答", 1),
|
|
("20230205", "politics.20230205.26.问答", 1),
|
|
("20230205", "pt.20230205.29.问答", 3),
|
|
("20230205", "matheducators.20230205.2.问答", 1),
|
|
("20230205", "ru.20230205.38.问答", 6),
|
|
("20230205", "physics.20230205.23.问答", 4),
|
|
("20230205", "substrate.20230205.60.问答", 1),
|
|
("20230205", "parenting.20230205.18.问答", 1),
|
|
("20230205", "stats.20230205.58.问答", 3),
|
|
("20230205", "sports.20230205.55.问答", 1),
|
|
("20230205", "mechanics.20230205.5.问答", 1),
|
|
("20230205", "raspberrypi.20230205.33.问答", 1),
|
|
("20230205", "stellar.20230205.59.问答", 1),
|
|
("20230205", "ukrainian.20230205.68.问答", 1),
|
|
("20230205", "opendata.20230205.14.问答", 1),
|
|
("20230205", "poker.20230205.25.问答", 1),
|
|
("20230205", "softwareengineering.20230205.49.问答", 3), ]
|
|
]
|
|
|
|
MNBVC_META['qa_stackexchange']['files'] += [
|
|
[f'{base_url}/qa/stackexchange/{folder}/{i}.jsonl.gz' for i in range(beg, end)]
|
|
for folder, beg, end in [('20230199', 1, 81)]
|
|
]
|
|
|
|
MNBVC_META['qa_stackexchange']['files'] = _flatten_list(MNBVC_META['qa_stackexchange']['files'])
|
|
MNBVC_META['qa_stackexchange']['feature_type'] = "QA_CORPUS"
|
|
MNBVC_META['qa_stackexchange']['description'] = "QA data from StackExchange"
|
|
|
|
|
|
# =============================
|
|
# qa_chatgpt
|
|
MNBVC_META['qa_chatgpt'] = {}
|
|
MNBVC_META['qa_chatgpt']['files'] = [
|
|
f'{base_url}/qa/chatgpt/20230211/chatgpt_baiduzhidao_2023-10-04-08-35-40_01.jsonl.gz',
|
|
f'{base_url}/qa/chatgpt/20230211/chatgpt_baiduzhidao_2023-10-04-08-35-40_02.jsonl.gz',
|
|
f'{base_url}/qa/chatgpt/20230211/chatgpt_baiduzhidao_2023-10-04-08-35-40_03.jsonl.gz',
|
|
f'{base_url}/qa/chatgpt/20230211/chatgpt_baiduzhidao_2023-10-04-08-35-40_04.jsonl.gz',
|
|
f'{base_url}/qa/chatgpt/20230211/chatgpt_baiduzhidao_2023-10-04-08-35-40_05.jsonl.gz',
|
|
f'{base_url}/qa/chatgpt/20230211/chatgpt_baiduzhidao_2023-10-04-08-35-40_06.jsonl.gz',
|
|
f'{base_url}/qa/chatgpt/20230211/chatgpt_baiduzhidao_2023-10-04-08-35-40_07.jsonl.gz',
|
|
f'{base_url}/qa/chatgpt/20230211/chatgpt_baiduzhidao_2023-10-04-08-35-40_08.jsonl.gz',
|
|
f'{base_url}/qa/chatgpt/20230211/chatgpt_baiduzhidao_2023-10-04-08-35-40_09.jsonl.gz',
|
|
f'{base_url}/qa/chatgpt/20230211/chatgpt_baiduzhidao_2023-10-04-08-35-40_10.jsonl.gz',
|
|
f'{base_url}/qa/chatgpt/20230211/chatgpt_baiduzhidao_2023-10-04-08-35-40_11.jsonl.gz',
|
|
]
|
|
|
|
MNBVC_META['qa_chatgpt']['feature_type'] = "QA_CORPUS"
|
|
MNBVC_META['qa_chatgpt']['description'] = "QA data made by ChatGPT"
|
|
|
|
|
|
# =============================
|
|
# math_qa
|
|
MNBVC_META['math_qa'] = {}
|
|
MNBVC_META['math_qa']['files'] = [
|
|
f'{base_url}/math/qa/20230206/math_qa.jsonl.gz'
|
|
]
|
|
|
|
MNBVC_META['math_qa']['feature_type'] = "QA_CORPUS"
|
|
MNBVC_META['math_qa']['description'] = "QA data in the math domain"
|
|
|
|
|
|
# =============================
|
|
# math_chat
|
|
MNBVC_META['math_chat'] = {}
|
|
MNBVC_META['math_chat']['files'] = [
|
|
f'{base_url}/math/conv/20230206/math_cot.jsonl.gz'
|
|
]
|
|
|
|
MNBVC_META['math_chat']['feature_type'] = "FORUM_DIALOGUE"
|
|
MNBVC_META['math_chat']['description'] = "Chain of thought conv. data in the math domain"
|
|
|
|
|
|
# =============================
|
|
# crawler_oscar
|
|
MNBVC_META['crawler_oscar'] = {}
|
|
MNBVC_META['crawler_oscar']['files'] = []
|
|
MNBVC_META['crawler_oscar']['files'] += [
|
|
f'{base_url}/crawler/oscar/20230207/oscar_202201.part_00{i:02}.jsonl.gz' for i in range(25)
|
|
]
|
|
MNBVC_META['crawler_oscar']['files'] += [
|
|
f'{base_url}/crawler/oscar/20230208/oscar_202201.part_00{i:02}.jsonl.gz' for i in range(25, 50)
|
|
]
|
|
MNBVC_META['crawler_oscar']['files'] += [
|
|
f'{base_url}/crawler/oscar/20230209/oscar_202201.part_00{i:02}.jsonl.gz' for i in range(50, 75)
|
|
]
|
|
MNBVC_META['crawler_oscar']['files'] += [
|
|
f'{base_url}/crawler/oscar/20230210/oscar_202201.part_00{i:02}.jsonl.gz' for i in range(75, 100)
|
|
]
|
|
|
|
MNBVC_META['crawler_oscar']['feature_type'] = "TEXT_CORPUS"
|
|
MNBVC_META['crawler_oscar']['description'] = "General text corpus from common crawl"
|