{ "datasets": [ { "id": "audioset", "name": "AudioSet", "slug": "audioset", "category": "audio", "description": "AudioSet is Google's 2-million-clip audio event dataset, distributing only YouTube IDs and labels (527 categories) for audio classification research.", "provider": "Google", "size": "2,084,320 10s clips (5.8 thousand hours of audio)", "license": "CC-BY 4.0", "commercialUse": false, "format": [ "CSV" ], "languages": [ "English" ], "useCases": [ "Pretraining", "Fine-tuning", "Evaluation" ], "downloadUrl": "https://research.google.com/audioset/", "lastVerified": "2026-09-01" }, { "id": "bigbench", "name": "BIG-Bench", "slug": "bigbench", "category": "benchmark", "description": "BIG-Bench is a collaborative benchmark of 204 diverse tasks designed to probe language model capabilities beyond standard evaluations.", "provider": "BIG-bench Collaboration", "size": "200+ tasks", "license": "Apache 2.0", "commercialUse": true, "format": [ "JSON", "Markdown" ], "languages": [ "English" ], "useCases": [ "Model evaluation", "Reasoning benchmarks", "Capability tracking" ], "downloadUrl": "https://github.com/google/BIG-bench", "lastVerified": "2026-09-01" }, { "id": "c4", "name": "C4 (Colossal Clean Crawled Corpus)", "slug": "c4", "category": "text", "description": "C4 (Colossal Clean Crawled Corpus) is a 156-billion-token cleaned English web corpus created by Google Research for T5 pretraining.", "provider": "Google Research", "size": "305GB (English subset), 1.7TB (full dataset)", "license": "ODC-By 1.0", "commercialUse": true, "format": [ "Parquet", "JSON" ], "languages": [ "English" ], "useCases": [ "Pretraining", "Fine-tuning", "Evaluation" ], "downloadUrl": "https://www.tensorflow.org/datasets/catalog/c4", "lastVerified": "2026-09-01" }, { "id": "coco", "name": "MS COCO", "slug": "coco", "category": "image", "description": "MS COCO is a benchmark dataset of 330K images with object detection, segmentation, and English caption annotations for vision model training and evaluation.", "provider": "Microsoft", "size": "330K images (>200K labeled)", "license": "CC-BY 4.0 (annotations); mixed (Flickr images)", "commercialUse": true, "format": [ "JPEG", "JSON" ], "languages": [ "English" ], "useCases": [ "Pretraining", "Fine-tuning", "Evaluation" ], "downloadUrl": "https://cocodataset.org/", "lastVerified": "2026-09-01" }, { "id": "common-voice", "name": "Common Voice", "slug": "common-voice", "category": "audio", "description": "Mozilla Common Voice is a crowd-sourced multilingual speech dataset with validated audio recordings in 100+ languages for speech recognition training.", "provider": "Mozilla", "size": "30K+ hours", "license": "CC0-1.0", "commercialUse": true, "format": [ "WAV", "JSON" ], "languages": [ "Multilingual" ], "useCases": [ "Pretraining", "Fine-tuning", "Evaluation" ], "downloadUrl": "https://commonvoice.mozilla.org/en/datasets", "lastVerified": "2026-09-04" }, { "id": "cosmopedia", "name": "Cosmopedia", "slug": "cosmopedia", "category": "text", "description": "Cosmopedia is a 25-billion-token synthetic English dataset of textbooks and educational content generated by Mixtral for LLM pretraining.", "provider": "HuggingFace", "size": "25B tokens", "license": "Apache 2.0", "commercialUse": true, "format": [ "Parquet", "JSON" ], "languages": [ "English" ], "useCases": [ "Pretraining", "Fine-tuning", "Evaluation" ], "huggingFaceUrl": "https://huggingface.co/datasets/HuggingFaceTB/cosmopedia", "downloadUrl": "https://huggingface.co/datasets/HuggingFaceTB/cosmopedia", "lastVerified": "2026-08-31" }, { "id": "dolma", "name": "Dolma", "slug": "dolma", "category": "text", "description": "Dolma is a 3-trillion-token open English corpus from AI2, combining web, academic, code, and social media text for LLM pretraining.", "provider": "AI2 (Allen Institute)", "size": "4.5TB (v1.7 gzip), 5.4TB (v1.6), 16.4GB (v1.6-sample), 6.4TB (v1.5), 2.9TB (v1.5-sample), 6.0TB (v1)", "license": "ODC-By-1.0", "commercialUse": true, "format": [ "Parquet", "JSON" ], "languages": [ "English" ], "useCases": [ "Pretraining", "Fine-tuning", "Evaluation" ], "huggingFaceUrl": "https://huggingface.co/datasets/allenai/dolma", "downloadUrl": "https://huggingface.co/datasets/allenai/dolma", "lastVerified": "2026-08-31" }, { "id": "fineweb", "name": "FineWeb", "slug": "fineweb", "category": "text", "description": "FineWeb is a 15-trillion-token English web dataset curated by HuggingFace from CommonCrawl, designed for high-quality LLM pretraining.", "provider": "HuggingFace", "size": "15T tokens", "license": "ODC-By 1.0", "commercialUse": true, "format": [ "Parquet", "JSON" ], "languages": [ "English" ], "useCases": [ "Pretraining", "Fine-tuning", "Evaluation" ], "huggingFaceUrl": "https://huggingface.co/datasets/HuggingFaceFW/fineweb", "downloadUrl": "https://huggingface.co/datasets/HuggingFaceFW/fineweb", "lastVerified": "2026-08-31" }, { "id": "fineweb-edu", "name": "FineWeb-Edu", "slug": "fineweb-edu", "category": "text", "description": "FineWeb-Edu is a filtered subset of FineWeb focused on educational content, containing 1.3T high-quality English tokens for pretraining.", "provider": "HuggingFace", "size": "1.3T tokens (original release; later snapshots add data)", "license": "ODC-By 1.0", "commercialUse": true, "format": [ "Parquet", "JSON" ], "languages": [ "English" ], "useCases": [ "Pretraining", "Fine-tuning", "Evaluation" ], "huggingFaceUrl": "https://huggingface.co/datasets/HuggingFaceFW/fineweb-edu", "downloadUrl": "https://huggingface.co/datasets/HuggingFaceFW/fineweb-edu", "lastVerified": "2026-08-31" }, { "id": "imagenet", "name": "ImageNet", "slug": "imagenet", "category": "image", "description": "ImageNet is a benchmark dataset of 1.2 million labeled images across 1,000 categories, the foundation of modern computer vision research.", "provider": "Stanford/Princeton", "size": "14,197,122 images, 21841 synsets indexed", "license": "Custom (research)", "commercialUse": false, "format": [ "JPEG", "XML" ], "languages": [ "English" ], "useCases": [ "Pretraining", "Fine-tuning", "Evaluation" ], "downloadUrl": "https://www.image-net.org/", "lastVerified": "2026-09-01" }, { "id": "laion-5b", "name": "LAION-5B", "slug": "laion-5b", "category": "image", "description": "LAION-5B is a 5-billion image-text pair dataset curated from Common Crawl, widely used for training image generation and vision-language models.", "provider": "LAION", "size": "5.85B image-text pairs", "license": "CC-BY 4.0 (metadata only; images carry original licenses)", "commercialUse": false, "format": [ "JPEG", "JSON" ], "languages": [ "Multilingual" ], "useCases": [ "Pretraining", "Fine-tuning", "Evaluation" ], "downloadUrl": "https://laion.ai/blog/laion-5b/", "lastVerified": "2026-09-01" }, { "id": "librispeech", "name": "LibriSpeech", "slug": "librispeech", "category": "audio", "description": "LibriSpeech is a 1,000-hour English speech corpus derived from LibriVox audiobooks, the standard benchmark for automatic speech recognition.", "provider": "OpenSLR", "size": "1000 hours", "license": "CC BY 4.0", "commercialUse": true, "format": [ "WAV", "JSON" ], "languages": [ "English" ], "useCases": [ "Pretraining", "Fine-tuning", "Evaluation" ], "downloadUrl": "https://www.openslr.org/12", "lastVerified": "2026-09-01" }, { "id": "msmarco", "name": "MS MARCO", "slug": "msmarco", "category": "text", "description": "MS MARCO is Microsoft's 1-million-query English question answering and passage ranking dataset, widely used for information retrieval research.", "provider": "Microsoft", "size": "1M+ passages (100K QnA, 1M QnA, 180K NLG, 148K KeyPhrase, Crawling dataset, Conversational search)", "license": "Custom (non-commercial research only, no license granted)", "commercialUse": false, "format": [ "Parquet", "JSON" ], "languages": [ "English" ], "useCases": [ "Pretraining", "Fine-tuning", "Evaluation" ], "downloadUrl": "https://microsoft.github.io/msmarco/", "lastVerified": "2026-09-01" }, { "id": "natural-questions", "name": "Natural Questions", "slug": "natural-questions", "category": "text", "description": "Natural Questions is Google's 300K-query QA dataset built from real Google Search queries with Wikipedia-passage answers, used for open-domain QA research.", "provider": "Google", "size": "300K+ questions", "license": "CC-BY-SA 3.0", "commercialUse": true, "format": [ "Parquet", "JSON" ], "languages": [ "English" ], "useCases": [ "Pretraining", "Fine-tuning", "Evaluation" ], "downloadUrl": "https://ai.google.com/research/NaturalQuestions", "lastVerified": "2026-09-01" }, { "id": "open-images-v7", "name": "Open Images V7", "slug": "open-images-v7", "category": "image", "description": "Open Images V7 is Google's large-scale image dataset with 9M images, 600 object categories, and rich annotations for detection and segmentation.", "provider": "Google", "size": "15,851,536 boxes on 600 classes; 2,785,498 instance segmentations on 350 classes; 3,284,280 relationship annotations on 1,466 relationships; 675,155 localized narratives; 66,391,027 point-level annotations on 5,827 classes; 61,404,966 image-level labels on 20,638 classes; Extension - 478,000 crowdsourced images with 6,000+ classes", "license": "CC-BY 4.0", "commercialUse": true, "format": [ "JPEG", "JSON" ], "languages": [ "English" ], "useCases": [ "Pretraining", "Fine-tuning", "Evaluation" ], "downloadUrl": "https://storage.googleapis.com/openimages/web/index.html", "lastVerified": "2026-09-01" }, { "id": "openassistant", "name": "OpenAssistant Conversations", "slug": "openassistant", "category": "text", "description": "OpenAssistant Conversations is a crowd-sourced dataset of 161K human-annotated conversation trees in multiple languages, designed for RLHF training.", "provider": "LAION", "size": "88.8k rows", "license": "Apache 2.0", "commercialUse": true, "format": [ "Parquet", "JSON" ], "languages": [ "English", "Multilingual" ], "useCases": [ "Pretraining", "Fine-tuning", "Evaluation" ], "huggingFaceUrl": "https://huggingface.co/datasets/OpenAssistant/oasst1", "downloadUrl": "https://huggingface.co/datasets/OpenAssistant/oasst1", "lastVerified": "2026-08-31" }, { "id": "pile", "name": "The Pile", "slug": "pile", "category": "text", "description": "The Pile is an 800GB diverse English text dataset from EleutherAI, combining 22 curated sources including books, code, and web for LLM pretraining.", "provider": "EleutherAI", "size": "825 GiB", "license": "mixed (constituent data)", "commercialUse": true, "format": [ "Parquet", "JSON" ], "languages": [ "English" ], "useCases": [ "Pretraining", "Fine-tuning", "Evaluation" ], "downloadUrl": "https://pile.eleuther.ai/", "lastVerified": "2026-09-01" }, { "id": "redpajama-v2", "name": "RedPajama v2", "slug": "redpajama-v2", "category": "text", "description": "RedPajama V2 is a 30-trillion-token open dataset for LLM pretraining, containing web data in 5 European languages with quality annotations.", "provider": "Together AI", "size": "100B+ documents (30T+ tokens, 20B deduplicated)", "license": "CC-BY 4.0", "commercialUse": true, "format": [ "Parquet", "JSON" ], "languages": [ "English", "German", "French", "Spanish", "Italian" ], "useCases": [ "Pretraining", "Fine-tuning", "Evaluation" ], "huggingFaceUrl": "https://huggingface.co/datasets/togethercomputer/RedPajama-Data-V2", "downloadUrl": "https://huggingface.co/datasets/togethercomputer/RedPajama-Data-V2", "lastVerified": "2026-08-31" }, { "id": "slim-pajama", "name": "SlimPajama", "slug": "slim-pajama", "category": "text", "description": "SlimPajama is a cleaned and deduplicated version of the RedPajama dataset, containing 627B tokens of high-quality English pretraining data.", "provider": "Cerebras", "size": "627B tokens", "license": "Apache 2.0", "commercialUse": true, "format": [ "Parquet", "JSON" ], "languages": [ "English" ], "useCases": [ "Pretraining", "Fine-tuning", "Evaluation" ], "huggingFaceUrl": "https://huggingface.co/datasets/cerebras/SlimPajama-627B", "downloadUrl": "https://huggingface.co/datasets/cerebras/SlimPajama-627B", "lastVerified": "2026-04-08" }, { "id": "squad-2", "name": "SQuAD 2.0", "slug": "squad-2", "category": "text", "description": "SQuAD 2.0 is Stanford's reading comprehension benchmark with 150K questions on Wikipedia, including 50K unanswerable questions for robustness testing.", "provider": "Stanford", "size": "150K questions", "license": "CC-BY-SA 4.0", "commercialUse": true, "format": [ "Parquet", "JSON" ], "languages": [ "English" ], "useCases": [ "Fine-tuning", "Evaluation" ], "downloadUrl": "https://rajpurkar.github.io/SQuAD-explorer/", "lastVerified": "2026-09-01" }, { "id": "starcoderdata", "name": "StarCoderData", "slug": "starcoderdata", "category": "code", "description": "StarCoderData is the training dataset for StarCoder, containing filtered permissively-licensed code from The Stack in 80+ programming languages.", "provider": "BigCode", "size": "783GB", "license": "Mixed original repository licenses; The Stack terms of use apply", "commercialUse": true, "format": [ "Parquet", "JSONL" ], "languages": [ "Programming Languages", "Multilingual comments" ], "useCases": [ "Pretraining", "Fine-tuning", "Evaluation" ], "huggingFaceUrl": "https://huggingface.co/datasets/bigcode/starcoderdata", "downloadUrl": "https://huggingface.co/datasets/bigcode/starcoderdata", "lastVerified": "2026-08-31" }, { "id": "the-stack-v2", "name": "The Stack v2", "slug": "the-stack-v2", "category": "code", "description": "The Stack v2 is a large dataset of permissively licensed source code in 600+ programming languages, created by BigCode for training code models.", "provider": "BigCode / HuggingFace", "size": "67.5TB (full), 32.1TB (dedup), ~900B tokens (train)", "license": "Mixed per-repo", "commercialUse": false, "format": [ "Parquet", "JSONL" ], "languages": [ "Programming Languages", "Multilingual comments" ], "useCases": [ "Pretraining", "Fine-tuning", "Evaluation" ], "huggingFaceUrl": "https://huggingface.co/datasets/bigcode/the-stack-v2", "downloadUrl": "https://huggingface.co/datasets/bigcode/the-stack-v2", "lastVerified": "2026-08-31" }, { "id": "ultrachat", "name": "UltraChat", "slug": "ultrachat", "category": "text", "description": "UltraChat is a large-scale synthetic dialogue dataset of 1.5M high-quality English conversations generated by ChatGPT for instruction fine-tuning.", "provider": "Tsinghua NLP", "size": "~949k rows", "license": "MIT", "commercialUse": true, "format": [ "Parquet", "JSON" ], "languages": [ "English" ], "useCases": [ "Pretraining", "Fine-tuning", "Evaluation" ], "huggingFaceUrl": "https://huggingface.co/datasets/stingning/ultrachat", "downloadUrl": "https://huggingface.co/datasets/stingning/ultrachat", "lastVerified": "2026-08-31" }, { "id": "webvid", "name": "WebVid", "slug": "webvid", "category": "video", "description": "WebVid is a large-scale video-text dataset of 10M English-captioned videos scraped from the web, used for video-language model training.", "provider": "University of Oxford", "size": "10 million video-text pairs", "license": "CC-BY 4.0 (metadata); mixed (videos from web)", "commercialUse": false, "format": [ "MP4", "CSV" ], "languages": [ "English" ], "useCases": [ "Pretraining", "Fine-tuning", "Evaluation" ], "downloadUrl": "https://github.com/m-bain/webvid", "lastVerified": "2026-09-01" }, { "id": "wikipedia-dump", "name": "Wikipedia Dump", "slug": "wikipedia-dump", "category": "text", "description": "Wikipedia Dumps are snapshots of Wikipedia's full article content in 300+ languages, widely used for pretraining and knowledge retrieval.", "provider": "Wikimedia", "size": "22GB (English)", "license": "CC-BY-SA-3.0 AND GFDL", "commercialUse": true, "format": [ "Parquet", "JSON" ], "languages": [ "English", "Multilingual" ], "useCases": [ "Pretraining", "Fine-tuning", "Evaluation" ], "downloadUrl": "https://dumps.wikimedia.org/", "lastVerified": "2026-09-01" } ], "count": 25, "exportedAt": "2026-11-02T20:23:36.092Z" }