workloads.csv

#Trace Name,Misc notes,Collection Year,Publication,"Trace Type (Block Level, File Level, Aggregate, etc)",Trace Origin,Trace Length (# days),Trace Length (# Accesses/Entries),Platform (if known),R/W ratio,max IOPs,avg IOPs,Available?,Grouped / Labeled?,Snapshot or Dynamic?,Corpus size (GB/Entries),System size (GB),"System Use (e.g. Personal, Scientific, Enterprise, etc)","Num files (clarify if observed, at end, etc)",File Size Information MAY BE ESTIMATED OR EYEBALLED,median file size,Data read (Passive NFS Margo) GB/day,"Read Operations (Passive NFS Margo) (million per day, averaged)",Data Written (Passive NFS Margo) (GB/day),"Write Operations (Passive NFS Margo)(million per day, averaged)",R/W byte ratio (Passive NFS Margo),R/W Ops ratio (Passive NFS Margo),Length of time until cliff (10-minutes?)

fiu-dedup,0,1,0,Block Level,University server,20.96,17836701,,96/4,11897,37,Y,Has PID data for accesses; groups well with neighborhood partitioning,dyn,5.9GB,TODO,Research,Unknown,Unknown,Unkown,0.736,0.034593,3.1,0.814773,0.236,0.042,

On Sheet 2,,2008(?),Characterizing HEC Storage Systems at Rest. Shobbhit Dayal CMU-PDL-08-109,File Level,Various HPC stores,1,NULL,Various,NULL,NULL,NULL,N,NULL,Snapshot,TODO,TODO,TODO,TODO,TODO,TODO,,,,,,,

,,2007-2010,"Analysis of Workload Behavior in Scientific and Historical Long-Term Data Repositories, Adams et al.",File/Record Level,LANL HPC Archive (SCI) Washington State Digital Archive (HIST) California Dept. of Water Res. (WATER),SCI-13 Months HIST-33 Months WATER updates 51 Month WATER  accesses 33 Month ,SCI-4716 (Histograms) HIST Accesses 5.8 Million HIST metadata 28.3 Million WATER update 900 thousand WATER access 100 thousand,Unknown,NULL,NULL,NULL,Y,NULL,Dynamic and Snapshot,SCI 1.3 PB HIST 10+ TB SCI 4.6 GB,NULL,"SCI-Mixed HPC, HIST public archive, WATER public archive","SCI 60,000,000 at end HIST 28 Million records at end WATER 56,000 files at end",SCI-TODO,SCI-1 MB,,,,,,,

,,2003(?),File System Workload Analysis for Large-Scale Scientific Computing Feng Wang et al.,File/IO level,Lawrence Livermore Labs,"Unclear, most results are very short (static or in seconds)","Non-Static information is benchmark based (io2,f1 and m1)","ASCI Linux Cluster, Luster Lite FS",NULL,NULL,NULL,Maybe(?),NULL,Benchmark and Snapshot,"~30000 GB,",~45000 GB,Scientific, ~11.2 million files,60% between 2 and 8,2-4 MB est.,,,,,,,

,,1999,Analysis and characterization of large-scale Web server access patterns and performance.  Iyengar et al.,File/Access Request level,"1998 Olympic Web Site Access Logs, 2/13/1998",1,56830964,Multiple SP2 machine frames.  Each frame had 10 uniprocessor nodes for web requests and one multiprocessor node for data updates,N/A Read only trace,1840,657.76,Unlikely,Yes: Filetype,Snapshot,N/A: Many pages created dynamically,NULL,Web Server,NULL,12-300 bytes,unknown,,,,,,,

namespace metadata traces1 from two Hadoop clusters at Yahoo!,,,"A Storage-Centric Analysis of MapReduce Workloads: File Popularity, Temporal Locality and Arrival Patterns

Cristina L. Abad et a.","namespace metadata trace ( snapshot of the namespace (file and directory hierarchy) and a set of events that operate atop that namespace (e.g., open a file, list directory contents))",,,,,,,,,,,,,,,,,,,,,,,

PROD,,2011,"""""",,MapReduce jobs,182 (plus 1 initial point),"9.71 billion file opens, ~722 million creates",Yahoo! HDFS,n/a : No I/O,n/a : No I/O,n/a : No I/O,n,n/a : No I/O,Snapshots,Unknown,Unknown,Mapreduce ,"~51million beginning, ~54 million end","Ranging from 2MB (smallest bin size) to 10^8 MB (953TB ... which seems odd, but it's Yahoo!?) ",unclear,,,,,,,

R&D,,2011,"""""",,Misc R&D tasks,182 (plus 1 initial point),"2.23 billion file opens, ~227 million creates",Yahoo! HDFS,n/a : No I/O,n/a : No I/O,n/a : No I/O,n,n/a : No I/O,Snapshots,Unknown,Unknown,Enterprise,"~38 million beginning, ~51million end","Ranging from 2MB (smallest bin size) to 10^8 MB (953TB ... which seems odd, but it's Yahoo!?) ",unclear,,,,,,,

Margo Passive NFS  - CAMPUS,,2001,Passive NFS Tracing of Email and Research Workloads,NFS,"EECS

Storage for the campus SMTP, POP and login servers.  E-mail heavy",91 days / 7 days,(29.9 / 26.7) million per day (averaged) ,NFS,3.25 / 3.01,4719.44444444444,3091.66666666667,maybe,n/a ,dynamic,unclear,742 GB,educational,--,--,--,(135.7/119.6),(19.29 / 17.29) ,(45/44.57),(5.93/5.73),(3.01/2.68),(3.25/3.01),

Margo Passive NFS  - EECS,,2001,Passive NFS Tracing of Email and Research Workloads,NFS,Research servers: Storage for the EECS department home directories,91 days / 7 days,(2.3 / 4.44) million per day (averaged) ,NFS,.8/.69,741.666666666667,0.514166666666667,maybe,n/a,dynamic,unclear,unclear,research,--,--,--,(2.27/5.10) ,(.35/ .461) ,(2.94/9.086),(.438/.667),(.77/.56),(.8/.69),

INS,,2000,"Drew Roselli, Jacob Lorch, and Thomas Anderson. A Comparison of File System Workloads. In USENIX 2000 Technical Conference, pages 41–54, San Diego, CA, 2000.",,,31,,,,,,,,,,,instructional,,,,3.05,2.32,0.542,0.15,5.6,15.4,

RES,,2000,"Drew Roselli, Jacob Lorch, and Thomas Anderson. A Comparison of File System Workloads. In USENIX 2000 Technical Conference, pages 41–54, San Diego, CA, 2000.",,,31,,,,,,,,,,,research,,,,1.7,0.303,0.455,0.071,3.7,4.27,

NT,,2000,"Drew Roselli, Jacob Lorch, and Thomas Anderson. A Comparison of File System Workloads. In USENIX 2000 Technical Conference, pages 41–54, San Diego, CA, 2000.",,,31,,,,,,,,,,,Windows NT Desktop,,,,4.04,1.27,0.639,0.231,6.3,4.49,

SPRITE,,1991,"Mary G. Baker, John H. Hartman, Michael D. Kupfer, Ken W. Shirriff, and John K. Ousterhout. Measurements of a Distributed File System. In Proceedings of 13th ACM Symposium on Operating Systems Principles, pages 198– 212, Monterey, CA, October 1991.",,,8,,,,,,,,,,,,,,,5.36,0.207,1.16,0.057,4.6,3.61,

ncar01,,2010,,,"10.46 % reads, 2.25% writes, ",,,,73.4,,,,,,,,Archival,,,,,,,,,,

ncar2010,"All possible syscall types: 'BF', 'CMPT', 'REDCMP', 'READ', 'DA', 'BS', 'DV', 'REDCPY', 'CREATE', 'NG', 'BL', 'CRTEND', 'WRTSTR', 'PE', 'CH', 'WRTEND', 'REDVDT', 'AN', 'WRITE', 'CRTSTR', 'COPY', 'SH', 'MS'",2010,,,"8.9% reads, 4% writes, 13.34% creates",,14771864,,~2/1,,,,,,,,Archival,,,,,1326137,,595661,,,

argonne,See Sheet3,2010?,"Understanding and Improving Computational Science Storage Access through Continuous Characterization

PHILIP CARNS, KEVIN HARMS, WILLIAM ALLCOCK, CHARLES BACON, SAMUEL LANG, ROBERT LATHAM, and ROBERT ROSS, Argonne National Laboratory",fsstat,Scientific jobs,varied,varied,,,,,,,,,,,,,,,,,,,,

hadoop,,,Workload Characterization on a Production Hadoop Cluster: A Case Study on Taobao,,,,,,,,,,,,,,,,,,,,,,,,

msr-cambridge-rsrch01,,2007,Write Offloading (stats not from paper),Block I/O,Research Machines,7,433655,WinNT,9/91,3925,6.53,y,n,dyn,75MB,--,research,-,--,--,0.155,0.014847,1.21,0.144447,0.128,0.103,

ent-storage,,2010,HANDS (stats not from paper),block I/O,Enterprise,3,2161328,IBM XIV,,342142,75997,n,n,dyn,2.8GB,-,ent,-,-,--,,,,,,,