{ "meta": { "project_title": "NumPy versus PyArrow with Lahman Baseball Data", "project_slug": "numpy-versus-pyarrow", "test_slug": "small-medium-large-csv-files", "test_title": "Small, medium, and large CSV file performance", "generated_at": "2026-06-16T17:37:13.718741+00:00", "data_dir": "C:\\code\\projects\\data-reports\\numpy-vs-pyarrow\\data", "output_path": "C:\\code\\projects\\data-reports\\numpy-vs-pyarrow\\output\\test2\\data.json", "runs_per_reader": 100, "warmups_per_reader": 2, "environment": { "python_version": "3.14.6 (tags/v3.14.6:c63aec6, Jun 10 2026, 10:26:10) [MSC v.1944 64 bit (AMD64)]", "python_executable": "C:\\code\\projects\\data-reports\\numpy-vs-pyarrow\\venv\\Scripts\\python.exe", "pandas_version": "3.0.3", "pyarrow_version": "24.0.0", "platform": "Windows-11-10.0.26200-SP0", "machine": "AMD64", "processor": "Intel64 Family 6 Model 186 Stepping 2, GenuineIntel", "system": "Windows", "release": "11" } }, "summary": { "bucket_count": 3, "files_tested": 27, "bucket_definitions": [ { "key": "small", "label": "Small CSV files", "description": "Files smaller than 1 MB.", "min_bytes": 0, "max_bytes": 1000000 }, { "key": "medium", "label": "Medium CSV files", "description": "Files from 1 MB up to 10 MB.", "min_bytes": 1000000, "max_bytes": 10000000 }, { "key": "large", "label": "Large CSV files", "description": "Files 10 MB or larger.", "min_bytes": 10000000, "max_bytes": null } ] }, "methodology_notes": [ "This test reuses the raw CSV benchmark results and groups files by file size.", "Small files are under 1 MB, medium files are from 1 MB up to 10 MB, and large files are 10 MB or larger.", "The point is to see where PyArrow begins to matter, not to pretend every Lahman table has the same shape." ], "buckets": [ { "key": "small", "label": "Small CSV files", "description": "Files smaller than 1 MB.", "min_bytes": 0, "max_bytes": 1000000, "summary": { "files_tested": 20, "files_with_results": 20, "total_size_bytes": 5786661, "total_size_mb": 5.786661, "total_rows": 129290, "total_columns_across_files": 216, "total_cells": 1293430, "largest_file_by_size": { "file": "FieldingPost.csv", "size_mb": 0.905643, "rows": 17934, "columns": 17, "pandas_default_avg_ms": 23.1803, "pyarrow_avg_ms": 5.4958, "speedup_default_over_pyarrow": 4.2178, "winner": "pyarrow" }, "largest_file_by_rows": { "file": "Salaries.csv", "size_mb": 0.800653, "rows": 26428, "columns": 5, "pandas_default_avg_ms": 13.1203, "pyarrow_avg_ms": 3.4301, "speedup_default_over_pyarrow": 3.825, "winner": "pyarrow" }, "widest_file": { "file": "Teams.csv", "size_mb": 0.701384, "rows": 3614, "columns": 48, "pandas_default_avg_ms": 13.0053, "pyarrow_avg_ms": 4.3277, "speedup_default_over_pyarrow": 3.0051, "winner": "pyarrow" }, "pyarrow_faster_count": 13, "pandas_default_faster_count": 7, "tie_count": 0, "no_result_count": 0, "median_speedup_default_over_pyarrow": 2.2127, "average_speedup_default_over_pyarrow": 2.0487, "total_pandas_default_avg_ms": 140.7927, "total_pyarrow_avg_ms": 59.8357, "overall_speedup_default_over_pyarrow": 2.353, "largest_pyarrow_win": { "file": "FieldingPost.csv", "size_mb": 0.905643, "rows": 17934, "columns": 17, "pandas_default_avg_ms": 23.1803, "pyarrow_avg_ms": 5.4958, "speedup_default_over_pyarrow": 4.2178, "winner": "pyarrow" }, "largest_pandas_default_win": { "file": "ManagersHalf.csv", "size_mb": 0.003568, "rows": 93, "columns": 10, "pandas_default_avg_ms": 1.0375, "pyarrow_avg_ms": 2.022, "speedup_default_over_pyarrow": 0.5131, "winner": "pandas_default" } }, "files": [ { "file": "AllstarFull.csv", "size_bytes": 255977, "size_mb": 0.255977, "rows": 6425, "columns": 8, "column_count": 8, "default_profile_error": null, "pandas_default": { "label": "pandas_default_read_csv", "ok": true, "error": null, "warmup_errors": [], "run_errors": [], "runs_requested": 100, "runs_completed": 100, "avg_ms": 6.6879, "median_ms": 6.6346, "min_ms": 6.2833, "max_ms": 7.3574, "stdev_ms": 0.245 }, "pyarrow": { "label": "pandas_read_csv_engine_pyarrow", "ok": true, "error": null, "warmup_errors": [], "run_errors": [], "runs_requested": 100, "runs_completed": 100, "avg_ms": 2.5639, "median_ms": 2.5188, "min_ms": 2.2341, "max_ms": 3.8813, "stdev_ms": 0.2206 }, "speedup_default_over_pyarrow": 2.6085, "winner": "pyarrow" }, { "file": "AwardsManagers.csv", "size_bytes": 10986, "size_mb": 0.010986, "rows": 232, "columns": 6, "column_count": 6, "default_profile_error": null, "pandas_default": { "label": "pandas_default_read_csv", "ok": true, "error": null, "warmup_errors": [], "run_errors": [], "runs_requested": 100, "runs_completed": 100, "avg_ms": 1.8335, "median_ms": 1.784, "min_ms": 1.1528, "max_ms": 3.8764, "stdev_ms": 0.474 }, "pyarrow": { "label": "pandas_read_csv_engine_pyarrow", "ok": true, "error": null, "warmup_errors": [], "run_errors": [], "runs_requested": 100, "runs_completed": 100, "avg_ms": 3.1165, "median_ms": 3.0305, "min_ms": 2.0521, "max_ms": 6.073, "stdev_ms": 0.6626 }, "speedup_default_over_pyarrow": 0.5883, "winner": "pandas_default" }, { "file": "AwardsPlayers.csv", "size_bytes": 590678, "size_mb": 0.590678, "rows": 12667, "columns": 6, "column_count": 6, "default_profile_error": null, "pandas_default": { "label": "pandas_default_read_csv", "ok": true, "error": null, "warmup_errors": [], "run_errors": [], "runs_requested": 100, "runs_completed": 100, "avg_ms": 14.8096, "median_ms": 13.8912, "min_ms": 12.2579, "max_ms": 28.4252, "stdev_ms": 2.6791 }, "pyarrow": { "label": "pandas_read_csv_engine_pyarrow", "ok": true, "error": null, "warmup_errors": [], "run_errors": [], "runs_requested": 100, "runs_completed": 100, "avg_ms": 4.3027, "median_ms": 4.3312, "min_ms": 3.0511, "max_ms": 5.5621, "stdev_ms": 0.4553 }, "speedup_default_over_pyarrow": 3.4419, "winner": "pyarrow" }, { "file": "AwardsShareManagers.csv", "size_bytes": 29062, "size_mb": 0.029062, "rows": 539, "columns": 7, "column_count": 7, "default_profile_error": null, "pandas_default": { "label": "pandas_default_read_csv", "ok": true, "error": null, "warmup_errors": [], "run_errors": [], "runs_requested": 100, "runs_completed": 100, "avg_ms": 2.0702, "median_ms": 1.9461, "min_ms": 1.3424, "max_ms": 3.5982, "stdev_ms": 0.4882 }, "pyarrow": { "label": "pandas_read_csv_engine_pyarrow", "ok": true, "error": null, "warmup_errors": [], "run_errors": [], "runs_requested": 100, "runs_completed": 100, "avg_ms": 2.8933, "median_ms": 2.8813, "min_ms": 1.9789, "max_ms": 4.1527, "stdev_ms": 0.4235 }, "speedup_default_over_pyarrow": 0.7155, "winner": "pandas_default" }, { "file": "AwardsSharePlayers.csv", "size_bytes": 361165, "size_mb": 0.361165, "rows": 7613, "columns": 7, "column_count": 7, "default_profile_error": null, "pandas_default": { "label": "pandas_default_read_csv", "ok": true, "error": null, "warmup_errors": [], "run_errors": [], "runs_requested": 100, "runs_completed": 100, "avg_ms": 8.5613, "median_ms": 7.195, "min_ms": 6.3805, "max_ms": 19.0179, "stdev_ms": 2.9962 }, "pyarrow": { "label": "pandas_read_csv_engine_pyarrow", "ok": true, "error": null, "warmup_errors": [], "run_errors": [], "runs_requested": 100, "runs_completed": 100, "avg_ms": 2.8507, "median_ms": 2.8499, "min_ms": 2.4579, "max_ms": 3.6368, "stdev_ms": 0.2315 }, "speedup_default_over_pyarrow": 3.0032, "winner": "pyarrow" }, { "file": "CollegePlaying.csv", "size_bytes": 429375, "size_mb": 0.429375, "rows": 17687, "columns": 3, "column_count": 3, "default_profile_error": null, "pandas_default": { "label": "pandas_default_read_csv", "ok": true, "error": null, "warmup_errors": [], "run_errors": [], "runs_requested": 100, "runs_completed": 100, "avg_ms": 8.478, "median_ms": 8.1732, "min_ms": 7.6902, "max_ms": 14.6991, "stdev_ms": 0.9195 }, "pyarrow": { "label": "pandas_read_csv_engine_pyarrow", "ok": true, "error": null, "warmup_errors": [], "run_errors": [], "runs_requested": 100, "runs_completed": 100, "avg_ms": 2.6666, "median_ms": 2.6124, "min_ms": 2.3582, "max_ms": 4.0061, "stdev_ms": 0.2508 }, "speedup_default_over_pyarrow": 3.1793, "winner": "pyarrow" }, { "file": "FieldingOF.csv", "size_bytes": 298472, "size_mb": 0.298472, "rows": 12028, "columns": 6, "column_count": 6, "default_profile_error": null, "pandas_default": { "label": "pandas_default_read_csv", "ok": true, "error": null, "warmup_errors": [], "run_errors": [], "runs_requested": 100, "runs_completed": 100, "avg_ms": 7.0466, "median_ms": 7.0103, "min_ms": 6.6048, "max_ms": 8.372, "stdev_ms": 0.3048 }, "pyarrow": { "label": "pandas_read_csv_engine_pyarrow", "ok": true, "error": null, "warmup_errors": [], "run_errors": [], "runs_requested": 100, "runs_completed": 100, "avg_ms": 2.6181, "median_ms": 2.5795, "min_ms": 2.3724, "max_ms": 3.3246, "stdev_ms": 0.1633 }, "speedup_default_over_pyarrow": 2.6915, "winner": "pyarrow" }, { "file": "FieldingPost.csv", "size_bytes": 905643, "size_mb": 0.905643, "rows": 17934, "columns": 17, "column_count": 17, "default_profile_error": null, "pandas_default": { "label": "pandas_default_read_csv", "ok": true, "error": null, "warmup_errors": [], "run_errors": [], "runs_requested": 100, "runs_completed": 100, "avg_ms": 23.1803, "median_ms": 23.0825, "min_ms": 18.4664, "max_ms": 40.7668, "stdev_ms": 3.7514 }, "pyarrow": { "label": "pandas_read_csv_engine_pyarrow", "ok": true, "error": null, "warmup_errors": [], "run_errors": [], "runs_requested": 100, "runs_completed": 100, "avg_ms": 5.4958, "median_ms": 5.5046, "min_ms": 4.2937, "max_ms": 7.4085, "stdev_ms": 0.821 }, "speedup_default_over_pyarrow": 4.2178, "winner": "pyarrow" }, { "file": "HallOfFame.csv", "size_bytes": 314822, "size_mb": 0.314822, "rows": 6426, "columns": 9, "column_count": 9, "default_profile_error": null, "pandas_default": { "label": "pandas_default_read_csv", "ok": true, "error": null, "warmup_errors": [], "run_errors": [], "runs_requested": 100, "runs_completed": 100, "avg_ms": 9.3481, "median_ms": 9.2303, "min_ms": 6.8384, "max_ms": 14.9774, "stdev_ms": 1.5813 }, "pyarrow": { "label": "pandas_read_csv_engine_pyarrow", "ok": true, "error": null, "warmup_errors": [], "run_errors": [], "runs_requested": 100, "runs_completed": 100, "avg_ms": 3.7826, "median_ms": 3.8804, "min_ms": 2.6618, "max_ms": 5.0595, "stdev_ms": 0.5841 }, "speedup_default_over_pyarrow": 2.4713, "winner": "pyarrow" }, { "file": "HomeGames.csv", "size_bytes": 178043, "size_mb": 0.178043, "rows": 3303, "columns": 9, "column_count": 9, "default_profile_error": null, "pandas_default": { "label": "pandas_default_read_csv", "ok": true, "error": null, "warmup_errors": [], "run_errors": [], "runs_requested": 100, "runs_completed": 100, "avg_ms": 5.8979, "median_ms": 5.7682, "min_ms": 4.1181, "max_ms": 9.6967, "stdev_ms": 1.1317 }, "pyarrow": { "label": "pandas_read_csv_engine_pyarrow", "ok": true, "error": null, "warmup_errors": [], "run_errors": [], "runs_requested": 100, "runs_completed": 100, "avg_ms": 3.0182, "median_ms": 2.7921, "min_ms": 2.4303, "max_ms": 4.9532, "stdev_ms": 0.564 }, "speedup_default_over_pyarrow": 1.9541, "winner": "pyarrow" }, { "file": "Managers.csv", "size_bytes": 169548, "size_mb": 0.169548, "rows": 4410, "columns": 10, "column_count": 10, "default_profile_error": null, "pandas_default": { "label": "pandas_default_read_csv", "ok": true, "error": null, "warmup_errors": [], "run_errors": [], "runs_requested": 100, "runs_completed": 100, "avg_ms": 4.8435, "median_ms": 4.5952, "min_ms": 4.224, "max_ms": 7.2098, "stdev_ms": 0.6636 }, "pyarrow": { "label": "pandas_read_csv_engine_pyarrow", "ok": true, "error": null, "warmup_errors": [], "run_errors": [], "runs_requested": 100, "runs_completed": 100, "avg_ms": 2.5402, "median_ms": 2.5603, "min_ms": 2.1767, "max_ms": 2.9898, "stdev_ms": 0.1676 }, "speedup_default_over_pyarrow": 1.9067, "winner": "pyarrow" }, { "file": "ManagersHalf.csv", "size_bytes": 3568, "size_mb": 0.003568, "rows": 93, "columns": 10, "column_count": 10, "default_profile_error": null, "pandas_default": { "label": "pandas_default_read_csv", "ok": true, "error": null, "warmup_errors": [], "run_errors": [], "runs_requested": 100, "runs_completed": 100, "avg_ms": 1.0375, "median_ms": 1.005, "min_ms": 0.9508, "max_ms": 1.4247, "stdev_ms": 0.0953 }, "pyarrow": { "label": "pandas_read_csv_engine_pyarrow", "ok": true, "error": null, "warmup_errors": [], "run_errors": [], "runs_requested": 100, "runs_completed": 100, "avg_ms": 2.022, "median_ms": 1.9759, "min_ms": 1.778, "max_ms": 2.4755, "stdev_ms": 0.1629 }, "speedup_default_over_pyarrow": 0.5131, "winner": "pandas_default" }, { "file": "Parks.csv", "size_bytes": 22638, "size_mb": 0.022638, "rows": 345, "columns": 7, "column_count": 7, "default_profile_error": null, "pandas_default": { "label": "pandas_default_read_csv", "ok": true, "error": null, "warmup_errors": [], "run_errors": [], "runs_requested": 100, "runs_completed": 100, "avg_ms": 1.4478, "median_ms": 1.4008, "min_ms": 1.315, "max_ms": 2.5888, "stdev_ms": 0.156 }, "pyarrow": { "label": "pandas_read_csv_engine_pyarrow", "ok": true, "error": null, "warmup_errors": [], "run_errors": [], "runs_requested": 100, "runs_completed": 100, "avg_ms": 1.9848, "median_ms": 1.9524, "min_ms": 1.7651, "max_ms": 2.4534, "stdev_ms": 0.1305 }, "speedup_default_over_pyarrow": 0.7294, "winner": "pandas_default" }, { "file": "PitchingPost.csv", "size_bytes": 624914, "size_mb": 0.624914, "rows": 7474, "columns": 30, "column_count": 30, "default_profile_error": null, "pandas_default": { "label": "pandas_default_read_csv", "ok": true, "error": null, "warmup_errors": [], "run_errors": [], "runs_requested": 100, "runs_completed": 100, "avg_ms": 13.6285, "median_ms": 13.3337, "min_ms": 12.0286, "max_ms": 21.644, "stdev_ms": 1.5348 }, "pyarrow": { "label": "pandas_read_csv_engine_pyarrow", "ok": true, "error": null, "warmup_errors": [], "run_errors": [], "runs_requested": 100, "runs_completed": 100, "avg_ms": 4.229, "median_ms": 4.2076, "min_ms": 3.8615, "max_ms": 4.7461, "stdev_ms": 0.1798 }, "speedup_default_over_pyarrow": 3.2226, "winner": "pyarrow" }, { "file": "Salaries.csv", "size_bytes": 800653, "size_mb": 0.800653, "rows": 26428, "columns": 5, "column_count": 5, "default_profile_error": null, "pandas_default": { "label": "pandas_default_read_csv", "ok": true, "error": null, "warmup_errors": [], "run_errors": [], "runs_requested": 100, "runs_completed": 100, "avg_ms": 13.1203, "median_ms": 12.9306, "min_ms": 12.1456, "max_ms": 15.6877, "stdev_ms": 0.7001 }, "pyarrow": { "label": "pandas_read_csv_engine_pyarrow", "ok": true, "error": null, "warmup_errors": [], "run_errors": [], "runs_requested": 100, "runs_completed": 100, "avg_ms": 3.4301, "median_ms": 3.4161, "min_ms": 3.0599, "max_ms": 3.9051, "stdev_ms": 0.1928 }, "speedup_default_over_pyarrow": 3.825, "winner": "pyarrow" }, { "file": "Schools.csv", "size_bytes": 66713, "size_mb": 0.066713, "rows": 1287, "columns": 5, "column_count": 5, "default_profile_error": null, "pandas_default": { "label": "pandas_default_read_csv", "ok": true, "error": null, "warmup_errors": [], "run_errors": [], "runs_requested": 100, "runs_completed": 100, "avg_ms": 2.2595, "median_ms": 2.1617, "min_ms": 2.0603, "max_ms": 5.0534, "stdev_ms": 0.3227 }, "pyarrow": { "label": "pandas_read_csv_engine_pyarrow", "ok": true, "error": null, "warmup_errors": [], "run_errors": [], "runs_requested": 100, "runs_completed": 100, "avg_ms": 1.9975, "median_ms": 1.9748, "min_ms": 1.7803, "max_ms": 2.3156, "stdev_ms": 0.1238 }, "speedup_default_over_pyarrow": 1.1312, "winner": "pyarrow" }, { "file": "SeriesPost.csv", "size_bytes": 13628, "size_mb": 0.013628, "rows": 440, "columns": 9, "column_count": 9, "default_profile_error": null, "pandas_default": { "label": "pandas_default_read_csv", "ok": true, "error": null, "warmup_errors": [], "run_errors": [], "runs_requested": 100, "runs_completed": 100, "avg_ms": 1.3226, "median_ms": 1.2806, "min_ms": 1.2105, "max_ms": 2.0515, "stdev_ms": 0.1133 }, "pyarrow": { "label": "pandas_read_csv_engine_pyarrow", "ok": true, "error": null, "warmup_errors": [], "run_errors": [], "runs_requested": 100, "runs_completed": 100, "avg_ms": 2.0891, "median_ms": 2.0458, "min_ms": 1.7931, "max_ms": 2.6227, "stdev_ms": 0.1834 }, "speedup_default_over_pyarrow": 0.6331, "winner": "pandas_default" }, { "file": "Teams.csv", "size_bytes": 701384, "size_mb": 0.701384, "rows": 3614, "columns": 48, "column_count": 48, "default_profile_error": null, "pandas_default": { "label": "pandas_default_read_csv", "ok": true, "error": null, "warmup_errors": [], "run_errors": [], "runs_requested": 100, "runs_completed": 100, "avg_ms": 13.0053, "median_ms": 12.7435, "min_ms": 12.1998, "max_ms": 18.1552, "stdev_ms": 0.7551 }, "pyarrow": { "label": "pandas_read_csv_engine_pyarrow", "ok": true, "error": null, "warmup_errors": [], "run_errors": [], "runs_requested": 100, "runs_completed": 100, "avg_ms": 4.3277, "median_ms": 4.2984, "min_ms": 4.0304, "max_ms": 4.859, "stdev_ms": 0.1842 }, "speedup_default_over_pyarrow": 3.0051, "winner": "pyarrow" }, { "file": "TeamsFranchises.csv", "size_bytes": 5610, "size_mb": 0.00561, "rows": 203, "columns": 4, "column_count": 4, "default_profile_error": null, "pandas_default": { "label": "pandas_default_read_csv", "ok": true, "error": null, "warmup_errors": [], "run_errors": [], "runs_requested": 100, "runs_completed": 100, "avg_ms": 1.0878, "median_ms": 1.0627, "min_ms": 0.9475, "max_ms": 1.5473, "stdev_ms": 0.1147 }, "pyarrow": { "label": "pandas_read_csv_engine_pyarrow", "ok": true, "error": null, "warmup_errors": [], "run_errors": [], "runs_requested": 100, "runs_completed": 100, "avg_ms": 1.8359, "median_ms": 1.7968, "min_ms": 1.5649, "max_ms": 2.2781, "stdev_ms": 0.1389 }, "speedup_default_over_pyarrow": 0.5925, "winner": "pandas_default" }, { "file": "TeamsHalf.csv", "size_bytes": 3782, "size_mb": 0.003782, "rows": 142, "columns": 10, "column_count": 10, "default_profile_error": null, "pandas_default": { "label": "pandas_default_read_csv", "ok": true, "error": null, "warmup_errors": [], "run_errors": [], "runs_requested": 100, "runs_completed": 100, "avg_ms": 1.1265, "median_ms": 1.0772, "min_ms": 0.9663, "max_ms": 2.3633, "stdev_ms": 0.1814 }, "pyarrow": { "label": "pandas_read_csv_engine_pyarrow", "ok": true, "error": null, "warmup_errors": [], "run_errors": [], "runs_requested": 100, "runs_completed": 100, "avg_ms": 2.071, "median_ms": 2.0042, "min_ms": 1.8293, "max_ms": 2.6029, "stdev_ms": 0.1723 }, "speedup_default_over_pyarrow": 0.5439, "winner": "pandas_default" } ] }, { "key": "medium", "label": "Medium CSV files", "description": "Files from 1 MB up to 10 MB.", "min_bytes": 1000000, "max_bytes": 10000000, "summary": { "files_tested": 7, "files_with_results": 7, "total_size_bytes": 36361131, "total_size_mb": 36.361131, "total_rows": 577176, "total_columns_across_files": 156, "total_cells": 12225294, "largest_file_by_size": { "file": "Fielding.csv", "size_mb": 8.767478, "rows": 174332, "columns": 18, "pandas_default_avg_ms": 153.9081, "pyarrow_avg_ms": 20.5, "speedup_default_over_pyarrow": 7.5077, "winner": "pyarrow" }, "largest_file_by_rows": { "file": "Fielding.csv", "size_mb": 8.767478, "rows": 174332, "columns": 18, "pandas_default_avg_ms": 153.9081, "pyarrow_avg_ms": 20.5, "speedup_default_over_pyarrow": 7.5077, "winner": "pyarrow" }, "widest_file": { "file": "Pitching.csv", "size_mb": 5.112472, "rows": 57630, "columns": 30, "pandas_default_avg_ms": 84.5593, "pyarrow_avg_ms": 10.2096, "speedup_default_over_pyarrow": 8.2823, "winner": "pyarrow" }, "pyarrow_faster_count": 7, "pandas_default_faster_count": 0, "tie_count": 0, "no_result_count": 0, "median_speedup_default_over_pyarrow": 7.5077, "average_speedup_default_over_pyarrow": 7.4177, "total_pandas_default_avg_ms": 701.45, "total_pyarrow_avg_ms": 93.286, "overall_speedup_default_over_pyarrow": 7.5193, "largest_pyarrow_win": { "file": "People.csv", "size_mb": 3.223128, "rows": 24270, "columns": 25, "pandas_default_avg_ms": 69.9977, "pyarrow_avg_ms": 8.2688, "speedup_default_over_pyarrow": 8.4653, "winner": "pyarrow" }, "largest_pandas_default_win": null }, "files": [ { "file": "Appearances.csv", "size_bytes": 7833413, "size_mb": 7.833413, "rows": 128512, "columns": 21, "column_count": 21, "default_profile_error": null, "pandas_default": { "label": "pandas_default_read_csv", "ok": true, "error": null, "warmup_errors": [], "run_errors": [], "runs_requested": 100, "runs_completed": 100, "avg_ms": 169.3614, "median_ms": 142.9781, "min_ms": 130.5391, "max_ms": 483.976, "stdev_ms": 57.2118 }, "pyarrow": { "label": "pandas_read_csv_engine_pyarrow", "ok": true, "error": null, "warmup_errors": [], "run_errors": [], "runs_requested": 100, "runs_completed": 100, "avg_ms": 20.1427, "median_ms": 19.8103, "min_ms": 16.3903, "max_ms": 32.922, "stdev_ms": 2.1971 }, "speedup_default_over_pyarrow": 8.4081, "winner": "pyarrow" }, { "file": "Batting.csv", "size_bytes": 7999073, "size_mb": 7.999073, "rows": 128598, "columns": 22, "column_count": 22, "default_profile_error": null, "pandas_default": { "label": "pandas_default_read_csv", "ok": true, "error": null, "warmup_errors": [], "run_errors": [], "runs_requested": 100, "runs_completed": 100, "avg_ms": 146.7359, "median_ms": 142.2553, "min_ms": 135.1172, "max_ms": 205.2244, "stdev_ms": 12.1893 }, "pyarrow": { "label": "pandas_read_csv_engine_pyarrow", "ok": true, "error": null, "warmup_errors": [], "run_errors": [], "runs_requested": 100, "runs_completed": 100, "avg_ms": 22.2018, "median_ms": 22.0769, "min_ms": 18.9362, "max_ms": 25.7383, "stdev_ms": 1.2511 }, "speedup_default_over_pyarrow": 6.6092, "winner": "pyarrow" }, { "file": "BattingPost.csv", "size_bytes": 1155184, "size_mb": 1.155184, "rows": 18687, "columns": 22, "column_count": 22, "default_profile_error": null, "pandas_default": { "label": "pandas_default_read_csv", "ok": true, "error": null, "warmup_errors": [], "run_errors": [], "runs_requested": 100, "runs_completed": 100, "avg_ms": 28.7525, "median_ms": 24.7356, "min_ms": 22.3711, "max_ms": 48.0821, "stdev_ms": 6.4032 }, "pyarrow": { "label": "pandas_read_csv_engine_pyarrow", "ok": true, "error": null, "warmup_errors": [], "run_errors": [], "runs_requested": 100, "runs_completed": 100, "avg_ms": 5.1037, "median_ms": 5.0743, "min_ms": 4.7253, "max_ms": 5.5126, "stdev_ms": 0.1619 }, "speedup_default_over_pyarrow": 5.6337, "winner": "pyarrow" }, { "file": "Fielding.csv", "size_bytes": 8767478, "size_mb": 8.767478, "rows": 174332, "columns": 18, "column_count": 18, "default_profile_error": null, "pandas_default": { "label": "pandas_default_read_csv", "ok": true, "error": null, "warmup_errors": [], "run_errors": [], "runs_requested": 100, "runs_completed": 100, "avg_ms": 153.9081, "median_ms": 152.5401, "min_ms": 144.2111, "max_ms": 170.0556, "stdev_ms": 5.8101 }, "pyarrow": { "label": "pandas_read_csv_engine_pyarrow", "ok": true, "error": null, "warmup_errors": [], "run_errors": [], "runs_requested": 100, "runs_completed": 100, "avg_ms": 20.5, "median_ms": 19.207, "min_ms": 16.1863, "max_ms": 35.7065, "stdev_ms": 4.3084 }, "speedup_default_over_pyarrow": 7.5077, "winner": "pyarrow" }, { "file": "FieldingOFsplit.csv", "size_bytes": 2270383, "size_mb": 2.270383, "rows": 45147, "columns": 18, "column_count": 18, "default_profile_error": null, "pandas_default": { "label": "pandas_default_read_csv", "ok": true, "error": null, "warmup_errors": [], "run_errors": [], "runs_requested": 100, "runs_completed": 100, "avg_ms": 48.1351, "median_ms": 46.8266, "min_ms": 40.418, "max_ms": 73.2686, "stdev_ms": 5.4742 }, "pyarrow": { "label": "pandas_read_csv_engine_pyarrow", "ok": true, "error": null, "warmup_errors": [], "run_errors": [], "runs_requested": 100, "runs_completed": 100, "avg_ms": 6.8594, "median_ms": 6.5676, "min_ms": 5.9595, "max_ms": 9.3686, "stdev_ms": 0.7822 }, "speedup_default_over_pyarrow": 7.0174, "winner": "pyarrow" }, { "file": "People.csv", "size_bytes": 3223128, "size_mb": 3.223128, "rows": 24270, "columns": 25, "column_count": 25, "default_profile_error": null, "pandas_default": { "label": "pandas_default_read_csv", "ok": true, "error": null, "warmup_errors": [], "run_errors": [], "runs_requested": 100, "runs_completed": 100, "avg_ms": 69.9977, "median_ms": 68.8832, "min_ms": 67.3183, "max_ms": 80.9916, "stdev_ms": 2.8704 }, "pyarrow": { "label": "pandas_read_csv_engine_pyarrow", "ok": true, "error": null, "warmup_errors": [], "run_errors": [], "runs_requested": 100, "runs_completed": 100, "avg_ms": 8.2688, "median_ms": 8.1879, "min_ms": 7.7085, "max_ms": 9.2485, "stdev_ms": 0.3203 }, "speedup_default_over_pyarrow": 8.4653, "winner": "pyarrow" }, { "file": "Pitching.csv", "size_bytes": 5112472, "size_mb": 5.112472, "rows": 57630, "columns": 30, "column_count": 30, "default_profile_error": null, "pandas_default": { "label": "pandas_default_read_csv", "ok": true, "error": null, "warmup_errors": [], "run_errors": [], "runs_requested": 100, "runs_completed": 100, "avg_ms": 84.5593, "median_ms": 83.9022, "min_ms": 81.6429, "max_ms": 90.5135, "stdev_ms": 2.0852 }, "pyarrow": { "label": "pandas_read_csv_engine_pyarrow", "ok": true, "error": null, "warmup_errors": [], "run_errors": [], "runs_requested": 100, "runs_completed": 100, "avg_ms": 10.2096, "median_ms": 10.1544, "min_ms": 9.5849, "max_ms": 11.5154, "stdev_ms": 0.3945 }, "speedup_default_over_pyarrow": 8.2823, "winner": "pyarrow" } ] }, { "key": "large", "label": "Large CSV files", "description": "Files 10 MB or larger.", "min_bytes": 10000000, "max_bytes": null, "summary": { "files_tested": 0, "files_with_results": 0, "total_size_bytes": 0, "total_size_mb": 0.0, "total_rows": 0, "total_columns_across_files": 0, "total_cells": 0, "largest_file_by_size": null, "largest_file_by_rows": null, "widest_file": null, "pyarrow_faster_count": 0, "pandas_default_faster_count": 0, "tie_count": 0, "no_result_count": 0, "median_speedup_default_over_pyarrow": null, "average_speedup_default_over_pyarrow": null, "total_pandas_default_avg_ms": 0.0, "total_pyarrow_avg_ms": 0.0, "overall_speedup_default_over_pyarrow": null, "largest_pyarrow_win": null, "largest_pandas_default_win": null }, "files": [] } ], "quality_checks": { "csv_file_count": 27, "ignored_file_count": 0, "ignored_files": [], "profile_error_count": 0, "profile_errors": [], "pandas_default_error_count": 0, "pandas_default_errors": [], "pyarrow_error_count": 0, "pyarrow_errors": [] } }