{ "artifact_version": 1, "dataset": { "source": "nyc_tlc_trip_records", "source_url": "https://www.nyc.gov/site/tlc/about/tlc-trip-record-data.page", "taxi_type": "yellow", "year": 2024, "months": [ 1, 2, 3, 4 ], "sample_size": 50000, "dataset_hash": "5110541e767f70aabdc8ee1eb158beaab347fb8d3fede1cd1a47e2637d3825f2", "task_rows": { "duration": 50000, "fare": 50000 } }, "git_commit": "4050b4bb9a61919ba90a8818dd0032b62a696805", "dataset_hash": "5110541e767f70aabdc8ee1eb158beaab347fb8d3fede1cd1a47e2637d3825f2", "source_file_hashes": { "/Users/ryanculligan/GitHub/CartoBoost/data/nyc_taxi/taxi_zone_lookup.csv": "1a99e105092230f8620f301edcca7f80d3080642ff404d28ed957d3fa222c8ed", "/Users/ryanculligan/GitHub/CartoBoost/data/nyc_taxi/taxi_zones.zip": "f6d711917bb4340f8f644d5366c51665489eb2d426dd1a4a55677721ae5adf17", "/Users/ryanculligan/GitHub/CartoBoost/data/nyc_taxi/yellow_tripdata_2024-01.parquet": "c4d59da7bbc8abaeeeb1727947ee93d9891a71acb42854bd80db1571b2030510", "/Users/ryanculligan/GitHub/CartoBoost/data/nyc_taxi/yellow_tripdata_2024-02.parquet": "c76c43c18c6c6664080dd920baab4928988d5786a6b65980792ca7cd796f9f20", "/Users/ryanculligan/GitHub/CartoBoost/data/nyc_taxi/yellow_tripdata_2024-03.parquet": "2d4cdc8fb96726cdd3803b13b02d2e61e71d45720aff0ebc693a8bdd1f249823", "/Users/ryanculligan/GitHub/CartoBoost/data/nyc_taxi/yellow_tripdata_2024-04.parquet": "ce37f736c7cf9e92164c850ef2b502a8f7bb590fc1804b6edbcb80293da01b89" }, "benchmark_integrity": { "command_argv": [ "scripts/run_nyc_taxi_quality_benchmarks.py", "--no-plots", "--sample-size", "50000", "--months", "1,2,3,4", "--output-dir", "docs/assets/nyc_taxi_benchmarks", "--models", "cartoboost,lightgbm,xgboost,catboost,hist_gradient_boosting", "--n-estimators", "24", "--cartoboost-n-estimators", "24", "--tasks", "duration,fare", "--model-workers", "1" ], "seed": 42, "synthetic_smoke": false, "no_download": false, "no_plots": true, "model_roster": [ "cartoboost", "lightgbm", "xgboost", "catboost", "hist_gradient_boosting" ], "split_modes": [ "random", "spatial_holdout", "out_of_time" ], "split_modes_by_task": { "duration": [ "random", "spatial_holdout", "out_of_time" ], "fare": [ "random", "spatial_holdout", "out_of_time" ], "pickup_demand": [ "random", "spatial_holdout" ] }, "hpo": "fixed_settings_no_hpo", "target_transforms": { "duration": "log_trip_duration_seconds", "fare": "log_total_amount", "pickup_demand": "log_pickup_trip_count" }, "selection_policy": { "global_hyperparameters": "fixed_before_holdout_scoring; no model family uses test labels for tuning", "primary_cartoboost_row": "single configured cartoboost run; no internal candidate is selected on test metrics", "zone_target_encoding": "fit on training rows for each outer split before transforming holdout rows", "graph_feature_gate": "uses deterministic inner train/validation rows inside the training split only", "neural_feature_gate": "uses deterministic inner train/validation rows inside the training split only", "segment_diagnostics": "computed after prediction and excluded from fitting, tuning, and model selection" }, "feature_access_policy": { "zone_treatment": "target_mean", "zone_target_encoding": "train_only_smoothed_target_mean", "zone_target_smoothing": 20.0, "cartoboost_splitters": [ "axis_histogram:512", "diagonal_2d", "gaussian_2d", "periodic:24", "periodic:7", "sparse_set" ], "graph_features": "train_split_graph_augmented_features_for_graph_rows", "neural_features": "train_split_residual_embedding_features_for_neural_rows", "baseline_feature_access": "same_dense_rows_plus_train_only_zone_features_when_enabled" } }, "resource_usage": { "cpu": "arm", "threads": 10, "os": "macOS-26.5.2-arm64-arm-64bit", "python": "3.12.13", "numpy": "1.26.4", "rustc": "rustc 1.94.0 (4a4ef493e 2026-03-02)" }, "baseline_environment": { "sklearn": { "package": "scikit-learn", "import_name": "sklearn", "version": "1.9.0", "module_importable": true }, "xgboost": { "package": "xgboost", "import_name": "xgboost", "version": "3.2.0", "module_importable": true, "required_class": "XGBRegressor", "required_class_available": true }, "lightgbm": { "package": "lightgbm", "import_name": "lightgbm", "version": "4.6.0", "module_importable": true, "required_class": "LGBMRegressor", "required_class_available": true }, "catboost": { "package": "catboost", "import_name": "catboost", "version": "1.2.10", "module_importable": true, "required_class": "CatBoostRegressor", "required_class_available": true } }, "models_requested": [ "cartoboost", "lightgbm", "xgboost", "catboost", "hist_gradient_boosting" ], "model_roster": [ "cartoboost", "lightgbm", "xgboost", "catboost", "hist_gradient_boosting" ], "split_definitions": { "random": { "kind": "seeded_row_shuffle", "train_fraction": "0.8", "purpose": "interpolation across rows drawn from the same task distribution" }, "spatial_holdout": { "kind": "pickup_zone_holdout", "train_fraction": "zone_blocked_approximately_0.8_by_row", "purpose": "generalization to held-out pickup zones where eligible" }, "out_of_time": { "kind": "chronological_timestamp_holdout", "train_fraction": "earlier_timestamp_groups_approximately_0.8_by_row", "purpose": "forward generalization to later pickup times without timestamp overlap", "scope": "duration_and_fare_row_tasks_only", "tie_policy": "all_rows_at_boundary_timestamp_remain_in_test" } }, "feature_access_policy": { "zone_treatment": "target_mean", "zone_target_encoding": "train_only_smoothed_target_mean", "zone_target_smoothing": 20.0, "cartoboost_splitters": [ "axis_histogram:512", "diagonal_2d", "gaussian_2d", "periodic:24", "periodic:7", "sparse_set" ], "graph_features": "train_split_graph_augmented_features_for_graph_rows", "neural_features": "train_split_residual_embedding_features_for_neural_rows", "baseline_feature_access": "same_dense_rows_plus_train_only_zone_features_when_enabled" }, "model_config": { "baseline_n_estimators": 24, "cartoboost_n_estimators": 24, "learning_rate": 0.08, "baseline_max_depth": 4, "cartoboost_max_depth": 5, "neural_dim": 12, "graph_dim": 8, "graph_epochs": 8, "graph_family": "graphsage", "model_workers": 1, "zone_treatment": "target_mean", "zone_target_smoothing": 20.0 }, "tasks": { "duration": { "display_name": "Trip duration", "description": "Predict log trip duration from zone, trip, passenger, and time features.", "row_count": 50000, "feature_names": [ "trip_distance", "log_trip_distance", "passenger_count", "hour", "dayofweek", "PULocationID", "DOLocationID" ], "zone_treatment": "target_mean", "splits": { "random": { "train_rows": 40000, "test_rows": 10000, "split_manifest": { "split_id": "duration_random", "split_kind": "seeded_row_shuffle", "dataset_fingerprint": "5110541e767f70aabdc8ee1eb158beaab347fb8d3fede1cd1a47e2637d3825f2", "coordinate_crs_note": "No coordinate CRS applies to this random-row diagnostic split.", "model_version": "0.3.0", "dependency_versions": { "cartoboost": "0.3.0", "numpy": "1.26.4", "scikit-learn": "1.9.0", "xgboost": "3.2.0", "lightgbm": "4.6.0", "catboost": "1.2.10" }, "random_seed": 42, "row_count": 50000, "train_index_sha256": "sha256:5656dfaa20d5c5d6eca74ec12ee802f587b040049ad44667e7d6e3b4a4254055", "test_index_sha256": "sha256:bee73ca172c57c21b478d46bb81c1524faff5ec9465e8c98e26c07687ed49f32", "random_row_split_allowed_for_geo_claim": true, "leakage_safe": false, "split_manifest_hash": "sha256:c925c81ec67ed734883e447b0ed8cff585e6e3cf38e1e817725157ee58b9bf95" }, "holdout_pickup_zones": [ 2, 4, 7, 10, 12, 13, 14, 17, 21, 24, 25, 28, 33, 35, 36, 37, 38, 39, 40, 41, 42, 43, 45, 48, 49, 50, 52, 56, 61, 62, 64, 65, 66, 68, 69, 70, 71, 74, 75, 76, 77, 78, 79, 80, 85, 87, 88, 89, 90, 91, 92, 93, 95, 97, 100, 106, 107, 112, 113, 114, 116, 119, 123, 125, 126, 127, 129, 130, 131, 132, 134, 135, 137, 138, 140, 141, 142, 143, 144, 145, 146, 147, 148, 151, 152, 158, 159, 161, 162, 163, 164, 166, 167, 168, 170, 174, 177, 179, 181, 186, 188, 189, 195, 196, 209, 211, 215, 216, 218, 223, 224, 225, 226, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 241, 242, 244, 246, 247, 249, 252, 255, 256, 259, 260, 261, 262, 263 ], "models": { "cartoboost": { "status": "ok", "metrics": { "rmse": 0.3257288492367098, "mae": 0.24955148606010977, "r2": 0.7824932571843731, "wape": 0.037797039666660454 }, "timing": { "train_seconds": 78.43002683301165, "predict_seconds": 0.031028083991259336, "fit_predict_seconds": 78.46105491700291, "prediction_rows": 10000.0, "predict_rows_per_second": 322288.6725076875 }, "backend": "rust", "config": { "n_estimators": 24, "learning_rate": 0.08, "max_depth": 5, "min_samples_leaf": 20, "constant_l2_regularization": 0.0, "leaf_predictor": "constant", "init": "constant", "calibration": "none", "splitters": [ "axis_histogram:512", "diagonal_2d", "gaussian_2d", "periodic:24", "periodic:7", "sparse_set" ], "sparse_sets": false, "zone_treatment": "target_mean", "feature_count": 22, "preset": "candidate", "predict_path": "model.predict" }, "calibration_report_fields": { "status": "not_computed", "coverage_by_horizon": {}, "coverage_by_spatial_block": {}, "width_by_horizon": {}, "residual_morans_i_after_calibration": null }, "segment_diagnostics": { "pickup_zone": { "segment_key": "pickup_zone", "segment_count": 138, "row_count_min": 1, "row_count_max": 464, "rmse_quantiles": { "p50": 0.34272843390743557, "p90": 0.786005977187939, "max": 1.1832381637091705 }, "mae_quantiles": { "p50": 0.26702581495085925, "p90": 0.772412996310766, "max": 1.0688099647582394 }, "wape_quantiles": { "p50": 0.040715552895833645, "p90": 0.09706382204494131, "max": 0.15481591733880798 }, "worst_rmse_segments": [ { "pickup_zone": 62, "rows": 3, "rmse": 1.1832381637091705, "mae": 1.013110520101743, "bias": -1.013110520101743, "wape": 0.13016012887012915 }, { "pickup_zone": 14, "rows": 2, "rmse": 1.1194998399226108, "mae": 0.8998180554460711, "bias": 0.6660385564515034, "wape": 0.14730335759576682 }, { "pickup_zone": 235, "rows": 1, "rmse": 1.0688099647582394, "mae": 1.0688099647582394, "bias": -1.0688099647582394, "wape": 0.15481591733880798 }, { "pickup_zone": 196, "rows": 1, "rmse": 1.0572777092373045, "mae": 1.0572777092373045, "bias": -1.0572777092373045, "wape": 0.1279960921098252 }, { "pickup_zone": 77, "rows": 1, "rmse": 1.0025817633940397, "mae": 1.0025817633940397, "bias": -1.0025817633940397, "wape": 0.1162805473660904 } ], "largest_abs_bias_segments": [ { "pickup_zone": 235, "rows": 1, "rmse": 1.0688099647582394, "mae": 1.0688099647582394, "bias": -1.0688099647582394, "wape": 0.15481591733880798 }, { "pickup_zone": 196, "rows": 1, "rmse": 1.0572777092373045, "mae": 1.0572777092373045, "bias": -1.0572777092373045, "wape": 0.1279960921098252 }, { "pickup_zone": 62, "rows": 3, "rmse": 1.1832381637091705, "mae": 1.013110520101743, "bias": -1.013110520101743, "wape": 0.13016012887012915 }, { "pickup_zone": 77, "rows": 1, "rmse": 1.0025817633940397, "mae": 1.0025817633940397, "bias": -1.0025817633940397, "wape": 0.1162805473660904 }, { "pickup_zone": 106, "rows": 1, "rmse": 0.9538557987985392, "mae": 0.9538557987985392, "bias": -0.9538557987985392, "wape": 0.12230593093679092 } ] } } }, "lightgbm": { "status": "ok", "metrics": { "rmse": 0.34554841927097274, "mae": 0.26681776722097245, "r2": 0.7552187793738043, "wape": 0.040412188645479366 }, "timing": { "train_seconds": 0.11698941700160503, "predict_seconds": 0.0014513749920297414, "fit_predict_seconds": 0.11844079199363478, "prediction_rows": 10000.0, "predict_rows_per_second": 6890018.124134167 }, "config": { "n_estimators": 24, "learning_rate": 0.08, "max_depth": 4, "num_leaves": 16, "zone_treatment": "target_mean", "feature_count": 22 }, "calibration_report_fields": { "status": "not_computed", "coverage_by_horizon": {}, "coverage_by_spatial_block": {}, "width_by_horizon": {}, "residual_morans_i_after_calibration": null }, "segment_diagnostics": { "pickup_zone": { "segment_key": "pickup_zone", "segment_count": 138, "row_count_min": 1, "row_count_max": 464, "rmse_quantiles": { "p50": 0.35631711006941796, "p90": 0.8109146488988981, "max": 1.176191654213742 }, "mae_quantiles": { "p50": 0.2851010056169543, "p90": 0.8107959144272964, "max": 1.176191654213742 }, "wape_quantiles": { "p50": 0.04234682895771308, "p90": 0.10220249632201465, "max": 0.17037003388580801 }, "worst_rmse_segments": [ { "pickup_zone": 235, "rows": 1, "rmse": 1.176191654213742, "mae": 1.176191654213742, "bias": -1.176191654213742, "wape": 0.17037003388580801 }, { "pickup_zone": 14, "rows": 2, "rmse": 1.1213538047560856, "mae": 0.9085618172562739, "bias": 0.6572288639925414, "wape": 0.1487347419349286 }, { "pickup_zone": 62, "rows": 3, "rmse": 1.0957200676649717, "mae": 0.969980279485906, "bias": -0.969980279485906, "wape": 0.12461893907358729 }, { "pickup_zone": 28, "rows": 1, "rmse": 1.0305509259429346, "mae": 1.0305509259429346, "bias": 1.0305509259429346, "wape": 0.16943667124259273 }, { "pickup_zone": 77, "rows": 1, "rmse": 1.0103047130188862, "mae": 1.0103047130188862, "bias": -1.0103047130188862, "wape": 0.1171762636482396 } ], "largest_abs_bias_segments": [ { "pickup_zone": 235, "rows": 1, "rmse": 1.176191654213742, "mae": 1.176191654213742, "bias": -1.176191654213742, "wape": 0.17037003388580801 }, { "pickup_zone": 28, "rows": 1, "rmse": 1.0305509259429346, "mae": 1.0305509259429346, "bias": 1.0305509259429346, "wape": 0.16943667124259273 }, { "pickup_zone": 77, "rows": 1, "rmse": 1.0103047130188862, "mae": 1.0103047130188862, "bias": -1.0103047130188862, "wape": 0.1171762636482396 }, { "pickup_zone": 252, "rows": 2, "rmse": 1.002220310931705, "mae": 0.9892164267926784, "bias": 0.9892164267926784, "wape": 0.16610549114899506 }, { "pickup_zone": 196, "rows": 1, "rmse": 0.9869420485625318, "mae": 0.9869420485625318, "bias": -0.9869420485625318, "wape": 0.1194811204768491 } ] } } }, "xgboost": { "status": "ok", "metrics": { "rmse": 0.34568455768438744, "mae": 0.26743546671349017, "r2": 0.7550258646351776, "wape": 0.04050574534028962 }, "timing": { "train_seconds": 0.06890129200473893, "predict_seconds": 0.0006425000028684735, "fit_predict_seconds": 0.06954379200760741, "prediction_rows": 10000.0, "predict_rows_per_second": 15564202.265143188 }, "config": { "n_estimators": 24, "learning_rate": 0.08, "max_depth": 4, "tree_method": "hist", "max_bin": 256, "subsample": 1.0, "colsample_bytree": 1.0, "zone_treatment": "target_mean", "feature_count": 22 }, "calibration_report_fields": { "status": "not_computed", "coverage_by_horizon": {}, "coverage_by_spatial_block": {}, "width_by_horizon": {}, "residual_morans_i_after_calibration": null }, "segment_diagnostics": { "pickup_zone": { "segment_key": "pickup_zone", "segment_count": 138, "row_count_min": 1, "row_count_max": 464, "rmse_quantiles": { "p50": 0.35771072576704155, "p90": 0.7855415052671939, "max": 1.1942078441324497 }, "mae_quantiles": { "p50": 0.2834446098842369, "p90": 0.7855415052671939, "max": 1.1942078441324497 }, "wape_quantiles": { "p50": 0.042695482820920866, "p90": 0.10258202320012548, "max": 0.17378312714747735 }, "worst_rmse_segments": [ { "pickup_zone": 235, "rows": 1, "rmse": 1.1942078441324497, "mae": 1.1942078441324497, "bias": -1.1942078441324497, "wape": 0.17297965866587434 }, { "pickup_zone": 14, "rows": 2, "rmse": 1.115836349167756, "mae": 0.9066758774770896, "bias": 0.6504072657383033, "wape": 0.14842600700788874 }, { "pickup_zone": 62, "rows": 3, "rmse": 1.1054840435228726, "mae": 0.9802418380215482, "bias": -0.9802418380215482, "wape": 0.12593730045164644 }, { "pickup_zone": 28, "rows": 1, "rmse": 1.0569870222407411, "mae": 1.0569870222407411, "bias": 1.0569870222407411, "wape": 0.17378312714747735 }, { "pickup_zone": 252, "rows": 2, "rmse": 1.0261479687436834, "mae": 1.008593177566885, "bias": 1.008593177566885, "wape": 0.16935916205158696 } ], "largest_abs_bias_segments": [ { "pickup_zone": 235, "rows": 1, "rmse": 1.1942078441324497, "mae": 1.1942078441324497, "bias": -1.1942078441324497, "wape": 0.17297965866587434 }, { "pickup_zone": 28, "rows": 1, "rmse": 1.0569870222407411, "mae": 1.0569870222407411, "bias": 1.0569870222407411, "wape": 0.17378312714747735 }, { "pickup_zone": 196, "rows": 1, "rmse": 1.0200728726925021, "mae": 1.0200728726925021, "bias": -1.0200728726925021, "wape": 0.12349200236716455 }, { "pickup_zone": 252, "rows": 2, "rmse": 1.0261479687436834, "mae": 1.008593177566885, "bias": 1.008593177566885, "wape": 0.16935916205158696 }, { "pickup_zone": 77, "rows": 1, "rmse": 0.982667846694488, "mae": 0.982667846694488, "bias": -0.982667846694488, "wape": 0.11397090917140831 } ] } } }, "catboost": { "status": "ok", "metrics": { "rmse": 0.3613229938563958, "mae": 0.27992388550207076, "r2": 0.7323597203356293, "wape": 0.04239723982817317 }, "timing": { "train_seconds": 0.4297059579985216, "predict_seconds": 0.007824042011634447, "fit_predict_seconds": 0.4375300000101561, "prediction_rows": 10000.0, "predict_rows_per_second": 1278111.746477061 }, "config": { "iterations": 24, "learning_rate": 0.08, "depth": 4, "zone_treatment": "target_mean", "feature_count": 22 }, "calibration_report_fields": { "status": "not_computed", "coverage_by_horizon": {}, "coverage_by_spatial_block": {}, "width_by_horizon": {}, "residual_morans_i_after_calibration": null }, "segment_diagnostics": { "pickup_zone": { "segment_key": "pickup_zone", "segment_count": 138, "row_count_min": 1, "row_count_max": 464, "rmse_quantiles": { "p50": 0.37105262997982147, "p90": 0.8501819807020898, "max": 1.2104345976753947 }, "mae_quantiles": { "p50": 0.29171404261170125, "p90": 0.8501819807020898, "max": 1.2104345976753947 }, "wape_quantiles": { "p50": 0.043784805460713894, "p90": 0.1091907567587479, "max": 0.18444354226508664 }, "worst_rmse_segments": [ { "pickup_zone": 235, "rows": 1, "rmse": 1.2104345976753947, "mae": 1.2104345976753947, "bias": -1.2104345976753947, "wape": 0.17533008560613028 }, { "pickup_zone": 14, "rows": 2, "rmse": 1.1404983932892088, "mae": 0.9156052513481705, "bias": 0.6800026535234402, "wape": 0.149887776689523 }, { "pickup_zone": 28, "rows": 1, "rmse": 1.1218260006615273, "mae": 1.1218260006615273, "bias": 1.1218260006615273, "wape": 0.18444354226508664 }, { "pickup_zone": 62, "rows": 3, "rmse": 1.1163794910968812, "mae": 0.969871753212133, "bias": -0.969871753212133, "wape": 0.12460499607970872 }, { "pickup_zone": 252, "rows": 2, "rmse": 1.079165798509348, "mae": 1.0680235584797995, "bias": 1.0680235584797995, "wape": 0.17933848744827333 } ], "largest_abs_bias_segments": [ { "pickup_zone": 235, "rows": 1, "rmse": 1.2104345976753947, "mae": 1.2104345976753947, "bias": -1.2104345976753947, "wape": 0.17533008560613028 }, { "pickup_zone": 28, "rows": 1, "rmse": 1.1218260006615273, "mae": 1.1218260006615273, "bias": 1.1218260006615273, "wape": 0.18444354226508664 }, { "pickup_zone": 252, "rows": 2, "rmse": 1.079165798509348, "mae": 1.0680235584797995, "bias": 1.0680235584797995, "wape": 0.17933848744827333 }, { "pickup_zone": 71, "rows": 1, "rmse": 1.0331446768503172, "mae": 1.0331446768503172, "bias": -1.0331446768503172, "wape": 0.12290179894902763 }, { "pickup_zone": 196, "rows": 1, "rmse": 1.0283253867820763, "mae": 1.0283253867820763, "bias": -1.0283253867820763, "wape": 0.1244910677445182 } ] } } }, "hist_gradient_boosting": { "status": "ok", "metrics": { "rmse": 0.3418852385950508, "mae": 0.2635626211170737, "r2": 0.7603811526807877, "wape": 0.039919164587188645 }, "timing": { "train_seconds": 5.471721959009301, "predict_seconds": 0.076582875000895, "fit_predict_seconds": 5.548304834010196, "prediction_rows": 10000.0, "predict_rows_per_second": 130577.4952936036 }, "config": { "n_estimators": 24, "learning_rate": 0.08, "max_leaf_nodes": 16, "zone_treatment": "target_mean", "feature_count": 22 }, "calibration_report_fields": { "status": "not_computed", "coverage_by_horizon": {}, "coverage_by_spatial_block": {}, "width_by_horizon": {}, "residual_morans_i_after_calibration": null }, "segment_diagnostics": { "pickup_zone": { "segment_key": "pickup_zone", "segment_count": 138, "row_count_min": 1, "row_count_max": 464, "rmse_quantiles": { "p50": 0.3557998094418424, "p90": 0.7953674083240778, "max": 1.182169139134703 }, "mae_quantiles": { "p50": 0.28119177252344296, "p90": 0.7953674083240778, "max": 1.182169139134703 }, "wape_quantiles": { "p50": 0.04241561936654764, "p90": 0.10104460975750498, "max": 0.17952445026185962 }, "worst_rmse_segments": [ { "pickup_zone": 235, "rows": 1, "rmse": 1.182169139134703, "mae": 1.182169139134703, "bias": -1.182169139134703, "wape": 0.17123586583155223 }, { "pickup_zone": 28, "rows": 1, "rmse": 1.0919070062576184, "mae": 1.0919070062576184, "bias": 1.0919070062576184, "wape": 0.17952445026185962 }, { "pickup_zone": 62, "rows": 3, "rmse": 1.0759027548696989, "mae": 0.9410157511940117, "bias": -0.9410157511940117, "wape": 0.12089769972177712 }, { "pickup_zone": 14, "rows": 2, "rmse": 1.066729277927043, "mae": 0.8469846336955644, "bias": 0.6484815977885114, "wape": 0.1386543419753092 }, { "pickup_zone": 252, "rows": 2, "rmse": 1.0465319156227586, "mae": 1.032152701842489, "bias": 1.032152701842489, "wape": 0.17331518850347688 } ], "largest_abs_bias_segments": [ { "pickup_zone": 235, "rows": 1, "rmse": 1.182169139134703, "mae": 1.182169139134703, "bias": -1.182169139134703, "wape": 0.17123586583155223 }, { "pickup_zone": 28, "rows": 1, "rmse": 1.0919070062576184, "mae": 1.0919070062576184, "bias": 1.0919070062576184, "wape": 0.17952445026185962 }, { "pickup_zone": 252, "rows": 2, "rmse": 1.0465319156227586, "mae": 1.032152701842489, "bias": 1.032152701842489, "wape": 0.17331518850347688 }, { "pickup_zone": 196, "rows": 1, "rmse": 0.9899576238196186, "mae": 0.9899576238196186, "bias": -0.9899576238196186, "wape": 0.11984619187199716 }, { "pickup_zone": 77, "rows": 1, "rmse": 0.9843861450935227, "mae": 0.9843861450935227, "bias": -0.9843861450935227, "wape": 0.11417019932975074 } ] } } } } }, "spatial_holdout": { "train_rows": 40772, "test_rows": 9228, "split_manifest": { "split_id": "duration_spatial_holdout", "split_kind": "group_spatial_cv", "dataset_fingerprint": "5110541e767f70aabdc8ee1eb158beaab347fb8d3fede1cd1a47e2637d3825f2", "coordinate_crs_note": "NYC TLC pickup/dropoff zone identifiers are treated as spatial groups; distance-buffered claims require projected taxi zone geometry.", "model_version": "0.3.0", "dependency_versions": { "cartoboost": "0.3.0", "numpy": "1.26.4", "scikit-learn": "1.9.0", "xgboost": "3.2.0", "lightgbm": "4.6.0", "catboost": "1.2.10" }, "random_seed": 42, "row_count": 50000, "train_index_sha256": "sha256:f1ed6d3eeb63ae16c0f13fe796c8db8678cd29cf5fb07b258860f873653004df", "test_index_sha256": "sha256:2cc5ef67dbed99aff0b6de977e3d150e32de539c1b72494d42b1a71c708dbb61", "random_row_split_allowed_for_geo_claim": false, "leakage_safe": true, "split_manifest_hash": "sha256:69a310148327f58cf791c5df7765fa86ec8c8961aa88594fc63cf561ce37286a" }, "holdout_pickup_zones": [ 2, 12, 18, 26, 35, 40, 47, 52, 57, 64, 69, 74, 79, 85, 90, 95, 106, 114, 122, 127, 132, 137, 142, 147, 154, 160, 165, 171, 178, 185, 191, 197, 205, 212, 219, 225, 230, 235, 241, 247, 254, 260 ], "models": { "cartoboost": { "status": "ok", "metrics": { "rmse": 0.32839469015295164, "mae": 0.2501317849928507, "r2": 0.8351327436446745, "wape": 0.03686223453532374 }, "timing": { "train_seconds": 85.53559308400145, "predict_seconds": 0.15608933400653768, "fit_predict_seconds": 85.69168241800799, "prediction_rows": 9228.0, "predict_rows_per_second": 59119.99086121728 }, "backend": "rust", "config": { "n_estimators": 24, "learning_rate": 0.08, "max_depth": 5, "min_samples_leaf": 20, "constant_l2_regularization": 0.0, "leaf_predictor": "constant", "init": "constant", "calibration": "none", "splitters": [ "axis_histogram:512", "diagonal_2d", "gaussian_2d", "periodic:24", "periodic:7", "sparse_set" ], "sparse_sets": false, "zone_treatment": "target_mean", "feature_count": 22, "preset": "candidate", "predict_path": "model.predict" }, "calibration_report_fields": { "status": "not_computed", "coverage_by_horizon": {}, "coverage_by_spatial_block": {}, "width_by_horizon": {}, "residual_morans_i_after_calibration": null }, "segment_diagnostics": { "pickup_zone": { "segment_key": "pickup_zone", "segment_count": 42, "row_count_min": 1, "row_count_max": 2076, "rmse_quantiles": { "p50": 0.40682175844285906, "p90": 0.883649031564081, "max": 1.2078904089649265 }, "mae_quantiles": { "p50": 0.3531336870692403, "p90": 0.8739612764079894, "max": 1.0801545187329769 }, "wape_quantiles": { "p50": 0.04946430943519892, "p90": 0.10738752668017935, "max": 0.17726136886834662 }, "worst_rmse_segments": [ { "pickup_zone": 171, "rows": 2, "rmse": 1.2078904089649265, "mae": 0.9091472314927462, "bias": -0.9091472314927462, "wape": 0.1175609452306828 }, { "pickup_zone": 154, "rows": 1, "rmse": 1.0801545187329769, "mae": 1.0801545187329769, "bias": 1.0801545187329769, "wape": 0.17726136886834662 }, { "pickup_zone": 26, "rows": 2, "rmse": 1.0258714958170754, "mae": 1.0211608813596316, "bias": -1.0211608813596316, "wape": 0.11749918672587337 }, { "pickup_zone": 254, "rows": 2, "rmse": 0.9256308585658165, "mae": 0.9148795296553565, "bias": -0.9148795296553565, "wape": 0.10985602838931702 }, { "pickup_zone": 57, "rows": 1, "rmse": 0.8838526595943765, "mae": 0.8838526595943765, "bias": -0.8838526595943765, "wape": 0.10422650456146058 } ], "largest_abs_bias_segments": [ { "pickup_zone": 154, "rows": 1, "rmse": 1.0801545187329769, "mae": 1.0801545187329769, "bias": 1.0801545187329769, "wape": 0.17726136886834662 }, { "pickup_zone": 26, "rows": 2, "rmse": 1.0258714958170754, "mae": 1.0211608813596316, "bias": -1.0211608813596316, "wape": 0.11749918672587337 }, { "pickup_zone": 254, "rows": 2, "rmse": 0.9256308585658165, "mae": 0.9148795296553565, "bias": -0.9148795296553565, "wape": 0.10985602838931702 }, { "pickup_zone": 171, "rows": 2, "rmse": 1.2078904089649265, "mae": 0.9091472314927462, "bias": -0.9091472314927462, "wape": 0.1175609452306828 }, { "pickup_zone": 57, "rows": 1, "rmse": 0.8838526595943765, "mae": 0.8838526595943765, "bias": -0.8838526595943765, "wape": 0.10422650456146058 } ] } } }, "lightgbm": { "status": "ok", "metrics": { "rmse": 0.3613323138160299, "mae": 0.27994404472995155, "r2": 0.800402194459886, "wape": 0.04125570460346566 }, "timing": { "train_seconds": 0.11230220900324639, "predict_seconds": 0.0026406249962747097, "fit_predict_seconds": 0.1149428339995211, "prediction_rows": 9228.0, "predict_rows_per_second": 3494627.2238649945 }, "config": { "n_estimators": 24, "learning_rate": 0.08, "max_depth": 4, "num_leaves": 16, "zone_treatment": "target_mean", "feature_count": 22 }, "calibration_report_fields": { "status": "not_computed", "coverage_by_horizon": {}, "coverage_by_spatial_block": {}, "width_by_horizon": {}, "residual_morans_i_after_calibration": null }, "segment_diagnostics": { "pickup_zone": { "segment_key": "pickup_zone", "segment_count": 42, "row_count_min": 1, "row_count_max": 2076, "rmse_quantiles": { "p50": 0.4428737874365276, "p90": 0.9085340700803185, "max": 1.181194700022279 }, "mae_quantiles": { "p50": 0.41211266506286726, "p90": 0.813766888640123, "max": 1.1811901224455683 }, "wape_quantiles": { "p50": 0.05730263463278157, "p90": 0.1072954788226204, "max": 0.17702958527834903 }, "worst_rmse_segments": [ { "pickup_zone": 26, "rows": 2, "rmse": 1.181194700022279, "mae": 1.1811901224455683, "bias": -1.1811901224455683, "wape": 0.1359128432056638 }, { "pickup_zone": 171, "rows": 2, "rmse": 1.123830169537754, "mae": 0.8146533162743168, "bias": -0.7741667935571215, "wape": 0.10534202885848332 }, { "pickup_zone": 154, "rows": 1, "rmse": 1.078742129255775, "mae": 1.078742129255775, "bias": 1.078742129255775, "wape": 0.17702958527834903 }, { "pickup_zone": 57, "rows": 1, "rmse": 0.9760699264327277, "mae": 0.9760699264327277, "bias": -0.9760699264327277, "wape": 0.11510103582913114 }, { "pickup_zone": 254, "rows": 2, "rmse": 0.9107530313232991, "mae": 0.9107513041329649, "bias": -0.9107513041329649, "wape": 0.10936032327680216 } ], "largest_abs_bias_segments": [ { "pickup_zone": 26, "rows": 2, "rmse": 1.181194700022279, "mae": 1.1811901224455683, "bias": -1.1811901224455683, "wape": 0.1359128432056638 }, { "pickup_zone": 154, "rows": 1, "rmse": 1.078742129255775, "mae": 1.078742129255775, "bias": 1.078742129255775, "wape": 0.17702958527834903 }, { "pickup_zone": 57, "rows": 1, "rmse": 0.9760699264327277, "mae": 0.9760699264327277, "bias": -0.9760699264327277, "wape": 0.11510103582913114 }, { "pickup_zone": 254, "rows": 2, "rmse": 0.9107530313232991, "mae": 0.9107513041329649, "bias": -0.9107513041329649, "wape": 0.10936032327680216 }, { "pickup_zone": 171, "rows": 2, "rmse": 1.123830169537754, "mae": 0.8146533162743168, "bias": -0.7741667935571215, "wape": 0.10534202885848332 } ] } } }, "xgboost": { "status": "ok", "metrics": { "rmse": 0.35584197363545306, "mae": 0.27478222351173504, "r2": 0.806421773518329, "wape": 0.04049500054348085 }, "timing": { "train_seconds": 0.09603837500617374, "predict_seconds": 0.0010308329947292805, "fit_predict_seconds": 0.09706920800090302, "prediction_rows": 9228.0, "predict_rows_per_second": 8951983.53873362 }, "config": { "n_estimators": 24, "learning_rate": 0.08, "max_depth": 4, "tree_method": "hist", "max_bin": 256, "subsample": 1.0, "colsample_bytree": 1.0, "zone_treatment": "target_mean", "feature_count": 22 }, "calibration_report_fields": { "status": "not_computed", "coverage_by_horizon": {}, "coverage_by_spatial_block": {}, "width_by_horizon": {}, "residual_morans_i_after_calibration": null }, "segment_diagnostics": { "pickup_zone": { "segment_key": "pickup_zone", "segment_count": 42, "row_count_min": 1, "row_count_max": 2076, "rmse_quantiles": { "p50": 0.4372197514183717, "p90": 0.9122731734268481, "max": 1.197605771493759 }, "mae_quantiles": { "p50": 0.4104176943043443, "p90": 0.815605811370245, "max": 1.1963633773723732 }, "wape_quantiles": { "p50": 0.05691512135334017, "p90": 0.10849113730002442, "max": 0.17216085120412525 }, "worst_rmse_segments": [ { "pickup_zone": 26, "rows": 2, "rmse": 1.197605771493759, "mae": 1.1963633773723732, "bias": -1.1963633773723732, "wape": 0.13765874352992039 }, { "pickup_zone": 171, "rows": 2, "rmse": 1.111940265415341, "mae": 0.8046294369785651, "bias": -0.7674520330284467, "wape": 0.10404585076535755 }, { "pickup_zone": 154, "rows": 1, "rmse": 1.0490741584826964, "mae": 1.0490741584826964, "bias": 1.0490741584826964, "wape": 0.17216085120412525 }, { "pickup_zone": 57, "rows": 1, "rmse": 0.964002809273449, "mae": 0.964002809273449, "bias": -0.964002809273449, "wape": 0.11367804589071488 }, { "pickup_zone": 254, "rows": 2, "rmse": 0.9140103789786249, "mae": 0.9139703286258563, "bias": -0.9139703286258563, "wape": 0.10974685421843355 } ], "largest_abs_bias_segments": [ { "pickup_zone": 26, "rows": 2, "rmse": 1.197605771493759, "mae": 1.1963633773723732, "bias": -1.1963633773723732, "wape": 0.13765874352992039 }, { "pickup_zone": 154, "rows": 1, "rmse": 1.0490741584826964, "mae": 1.0490741584826964, "bias": 1.0490741584826964, "wape": 0.17216085120412525 }, { "pickup_zone": 57, "rows": 1, "rmse": 0.964002809273449, "mae": 0.964002809273449, "bias": -0.964002809273449, "wape": 0.11367804589071488 }, { "pickup_zone": 254, "rows": 2, "rmse": 0.9140103789786249, "mae": 0.9139703286258563, "bias": -0.9139703286258563, "wape": 0.10974685421843355 }, { "pickup_zone": 171, "rows": 2, "rmse": 1.111940265415341, "mae": 0.8046294369785651, "bias": -0.7674520330284467, "wape": 0.10404585076535755 } ] } } }, "catboost": { "status": "ok", "metrics": { "rmse": 0.39450887730055706, "mae": 0.30785402185721783, "r2": 0.7620664313594809, "wape": 0.04536883290009623 }, "timing": { "train_seconds": 0.3283463749976363, "predict_seconds": 0.007524625005316921, "fit_predict_seconds": 0.3358710000029532, "prediction_rows": 9228.0, "predict_rows_per_second": 1226373.406446098 }, "config": { "iterations": 24, "learning_rate": 0.08, "depth": 4, "zone_treatment": "target_mean", "feature_count": 22 }, "calibration_report_fields": { "status": "not_computed", "coverage_by_horizon": {}, "coverage_by_spatial_block": {}, "width_by_horizon": {}, "residual_morans_i_after_calibration": null }, "segment_diagnostics": { "pickup_zone": { "segment_key": "pickup_zone", "segment_count": 42, "row_count_min": 1, "row_count_max": 2076, "rmse_quantiles": { "p50": 0.5074794306842394, "p90": 0.9758802912628242, "max": 1.3280852539024757 }, "mae_quantiles": { "p50": 0.447936472921011, "p90": 0.8739384622160187, "max": 1.3278517640733583 }, "wape_quantiles": { "p50": 0.06154777504255488, "p90": 0.11335605202722421, "max": 0.17899797819279542 }, "worst_rmse_segments": [ { "pickup_zone": 26, "rows": 2, "rmse": 1.3280852539024757, "mae": 1.3278517640733583, "bias": -1.3278517640733583, "wape": 0.15278836588745937 }, { "pickup_zone": 171, "rows": 2, "rmse": 1.1483152847351847, "mae": 0.8771253089622713, "bias": -0.741133716365872, "wape": 0.11342022153887188 }, { "pickup_zone": 154, "rows": 1, "rmse": 1.0907366688148166, "mae": 1.0907366688148166, "bias": 1.0907366688148166, "wape": 0.17899797819279542 }, { "pickup_zone": 57, "rows": 1, "rmse": 0.9974084359827247, "mae": 0.9974084359827247, "bias": -0.9974084359827247, "wape": 0.11761733562051062 }, { "pickup_zone": 254, "rows": 2, "rmse": 0.9809912968772644, "mae": 0.9793181150277501, "bias": -0.9793181150277501, "wape": 0.117593623159531 } ], "largest_abs_bias_segments": [ { "pickup_zone": 26, "rows": 2, "rmse": 1.3280852539024757, "mae": 1.3278517640733583, "bias": -1.3278517640733583, "wape": 0.15278836588745937 }, { "pickup_zone": 154, "rows": 1, "rmse": 1.0907366688148166, "mae": 1.0907366688148166, "bias": 1.0907366688148166, "wape": 0.17899797819279542 }, { "pickup_zone": 57, "rows": 1, "rmse": 0.9974084359827247, "mae": 0.9974084359827247, "bias": -0.9974084359827247, "wape": 0.11761733562051062 }, { "pickup_zone": 254, "rows": 2, "rmse": 0.9809912968772644, "mae": 0.9793181150277501, "bias": -0.9793181150277501, "wape": 0.117593623159531 }, { "pickup_zone": 241, "rows": 5, "rmse": 0.91194235738827, "mae": 0.8371501655448925, "bias": -0.8371501655448925, "wape": 0.10155608113425929 } ] } } }, "hist_gradient_boosting": { "status": "ok", "metrics": { "rmse": 0.352825943273662, "mae": 0.2715776213331416, "r2": 0.8096893114571468, "wape": 0.040022734305493156 }, "timing": { "train_seconds": 0.5137142919993494, "predict_seconds": 0.005016833994886838, "fit_predict_seconds": 0.5187311259942362, "prediction_rows": 9228.0, "predict_rows_per_second": 1839407.0861035439 }, "config": { "n_estimators": 24, "learning_rate": 0.08, "max_leaf_nodes": 16, "zone_treatment": "target_mean", "feature_count": 22 }, "calibration_report_fields": { "status": "not_computed", "coverage_by_horizon": {}, "coverage_by_spatial_block": {}, "width_by_horizon": {}, "residual_morans_i_after_calibration": null }, "segment_diagnostics": { "pickup_zone": { "segment_key": "pickup_zone", "segment_count": 42, "row_count_min": 1, "row_count_max": 2076, "rmse_quantiles": { "p50": 0.44174131380023035, "p90": 0.9230048369622919, "max": 1.1844971095666241 }, "mae_quantiles": { "p50": 0.40339795160715225, "p90": 0.8413372636573763, "max": 1.1844862315897755 }, "wape_quantiles": { "p50": 0.057345715225043156, "p90": 0.11013572974442103, "max": 0.16799487804019653 }, "worst_rmse_segments": [ { "pickup_zone": 26, "rows": 2, "rmse": 1.1844971095666241, "mae": 1.1844862315897755, "bias": -1.1844862315897755, "wape": 0.1362921077768726 }, { "pickup_zone": 171, "rows": 2, "rmse": 1.114043076555919, "mae": 0.8007613035795598, "bias": -0.7745148875985159, "wape": 0.10354566619358165 }, { "pickup_zone": 154, "rows": 1, "rmse": 1.023688510348161, "mae": 1.023688510348161, "bias": 1.023688510348161, "wape": 0.16799487804019653 }, { "pickup_zone": 57, "rows": 1, "rmse": 0.9842269477540535, "mae": 0.9842269477540535, "bias": -0.9842269477540535, "wape": 0.11606293576881709 }, { "pickup_zone": 254, "rows": 2, "rmse": 0.9233224298329578, "mae": 0.9233068744282971, "bias": -0.9233068744282971, "wape": 0.11086795902784763 } ], "largest_abs_bias_segments": [ { "pickup_zone": 26, "rows": 2, "rmse": 1.1844971095666241, "mae": 1.1844862315897755, "bias": -1.1844862315897755, "wape": 0.1362921077768726 }, { "pickup_zone": 154, "rows": 1, "rmse": 1.023688510348161, "mae": 1.023688510348161, "bias": 1.023688510348161, "wape": 0.16799487804019653 }, { "pickup_zone": 57, "rows": 1, "rmse": 0.9842269477540535, "mae": 0.9842269477540535, "bias": -0.9842269477540535, "wape": 0.11606293576881709 }, { "pickup_zone": 254, "rows": 2, "rmse": 0.9233224298329578, "mae": 0.9233068744282971, "bias": -0.9233068744282971, "wape": 0.11086795902784763 }, { "pickup_zone": 171, "rows": 2, "rmse": 1.114043076555919, "mae": 0.8007613035795598, "bias": -0.7745148875985159, "wape": 0.10354566619358165 } ] } } } } }, "out_of_time": { "train_rows": 40000, "test_rows": 10000, "split_manifest": { "split_id": "duration_out_of_time", "split_kind": "chronological_timestamp_holdout", "dataset_fingerprint": "5110541e767f70aabdc8ee1eb158beaab347fb8d3fede1cd1a47e2637d3825f2", "coordinate_crs_note": "No coordinate CRS applies to this chronological timestamp split.", "model_version": "0.3.0", "dependency_versions": { "cartoboost": "0.3.0", "numpy": "1.26.4", "scikit-learn": "1.9.0", "xgboost": "3.2.0", "lightgbm": "4.6.0", "catboost": "1.2.10" }, "random_seed": 42, "row_count": 50000, "train_index_sha256": "sha256:aa149280a2f2fc3b1d53da10fdd409517e69cd13c3cd215b1d548b09b4e74088", "test_index_sha256": "sha256:93189ab389199f948dbfce5cbac71d88ece6d66d64ab3da3e4a48e0ac0721098", "random_row_split_allowed_for_geo_claim": false, "timestamp_column": "tpep_pickup_datetime", "timestamp_tie_policy": "grouped_at_boundary_no_timestamp_overlap", "train_timestamp_min": "2024-01-01T00:07:08.000000000", "train_timestamp_max": "2024-04-08T13:45:40.000000000", "test_timestamp_min": "2024-04-08T13:48:01.000000000", "test_timestamp_max": "2024-04-30T23:57:36.000000000", "temporal_boundary": "2024-04-08T13:48:01.000000000", "leakage_safe": true, "split_manifest_hash": "sha256:c68ad145848792a952729e58fe59e75d38affd822e29a7fe460f0ed0a67ae25f" }, "holdout_pickup_zones": [ 3, 4, 7, 10, 12, 13, 17, 24, 25, 28, 33, 35, 36, 37, 39, 40, 41, 42, 43, 45, 47, 48, 49, 50, 52, 53, 57, 61, 62, 65, 66, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 79, 80, 82, 85, 87, 88, 89, 90, 91, 92, 93, 95, 97, 100, 107, 112, 113, 114, 116, 119, 121, 125, 126, 127, 129, 130, 132, 134, 137, 138, 140, 141, 142, 143, 144, 145, 146, 148, 151, 152, 153, 155, 157, 158, 159, 161, 162, 163, 164, 166, 168, 170, 174, 179, 181, 182, 186, 188, 189, 191, 195, 197, 198, 203, 207, 209, 210, 211, 212, 213, 215, 216, 218, 219, 222, 223, 224, 225, 226, 227, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 241, 242, 243, 244, 246, 249, 254, 255, 256, 259, 260, 261, 262, 263 ], "models": { "cartoboost": { "status": "ok", "metrics": { "rmse": 0.3308252158984261, "mae": 0.25564556251668336, "r2": 0.7803005380508858, "wape": 0.0385541672576472 }, "timing": { "train_seconds": 73.03090333400178, "predict_seconds": 0.02660199999809265, "fit_predict_seconds": 73.05750533399987, "prediction_rows": 10000.0, "predict_rows_per_second": 375911.5856220207 }, "backend": "rust", "config": { "n_estimators": 24, "learning_rate": 0.08, "max_depth": 5, "min_samples_leaf": 20, "constant_l2_regularization": 0.0, "leaf_predictor": "constant", "init": "constant", "calibration": "none", "splitters": [ "axis_histogram:512", "diagonal_2d", "gaussian_2d", "periodic:24", "periodic:7", "sparse_set" ], "sparse_sets": false, "zone_treatment": "target_mean", "feature_count": 22, "preset": "candidate", "predict_path": "model.predict" }, "calibration_report_fields": { "status": "not_computed", "coverage_by_horizon": {}, "coverage_by_spatial_block": {}, "width_by_horizon": {}, "residual_morans_i_after_calibration": null }, "segment_diagnostics": { "pickup_zone": { "segment_key": "pickup_zone", "segment_count": 146, "row_count_min": 1, "row_count_max": 464, "rmse_quantiles": { "p50": 0.3304198836357529, "p90": 0.6446079822511843, "max": 0.9644447434077188 }, "mae_quantiles": { "p50": 0.26392400843667785, "p90": 0.5882161171846216, "max": 0.9644447434077188 }, "wape_quantiles": { "p50": 0.03986887117895434, "p90": 0.07992679262182306, "max": 0.15651815766968377 }, "worst_rmse_segments": [ { "pickup_zone": 254, "rows": 1, "rmse": 0.9644447434077188, "mae": 0.9644447434077188, "bias": -0.9644447434077188, "wape": 0.11917243257362611 }, { "pickup_zone": 197, "rows": 1, "rmse": 0.9454095392647686, "mae": 0.9454095392647686, "bias": 0.9454095392647686, "wape": 0.15651815766968377 }, { "pickup_zone": 61, "rows": 3, "rmse": 0.9398075879417205, "mae": 0.6697902863586392, "bias": -0.6697902863586392, "wape": 0.08388305084545132 }, { "pickup_zone": 222, "rows": 1, "rmse": 0.8759115339426682, "mae": 0.8759115339426682, "bias": -0.8759115339426682, "wape": 0.10099291710390157 }, { "pickup_zone": 57, "rows": 1, "rmse": 0.8538399188401478, "mae": 0.8538399188401478, "bias": -0.8538399188401478, "wape": 0.10068731392016281 } ], "largest_abs_bias_segments": [ { "pickup_zone": 254, "rows": 1, "rmse": 0.9644447434077188, "mae": 0.9644447434077188, "bias": -0.9644447434077188, "wape": 0.11917243257362611 }, { "pickup_zone": 197, "rows": 1, "rmse": 0.9454095392647686, "mae": 0.9454095392647686, "bias": 0.9454095392647686, "wape": 0.15651815766968377 }, { "pickup_zone": 222, "rows": 1, "rmse": 0.8759115339426682, "mae": 0.8759115339426682, "bias": -0.8759115339426682, "wape": 0.10099291710390157 }, { "pickup_zone": 57, "rows": 1, "rmse": 0.8538399188401478, "mae": 0.8538399188401478, "bias": -0.8538399188401478, "wape": 0.10068731392016281 }, { "pickup_zone": 126, "rows": 2, "rmse": 0.7525012396465396, "mae": 0.7473060671322576, "bias": -0.7473060671322576, "wape": 0.09809988012277722 } ] } } }, "lightgbm": { "status": "ok", "metrics": { "rmse": 0.34962507714941177, "mae": 0.27153388975898196, "r2": 0.7546212585900887, "wape": 0.04095030204642859 }, "timing": { "train_seconds": 0.0958430419996148, "predict_seconds": 0.0017165419994853437, "fit_predict_seconds": 0.09755958399910014, "prediction_rows": 10000.0, "predict_rows_per_second": 5825665.787961038 }, "config": { "n_estimators": 24, "learning_rate": 0.08, "max_depth": 4, "num_leaves": 16, "zone_treatment": "target_mean", "feature_count": 22 }, "calibration_report_fields": { "status": "not_computed", "coverage_by_horizon": {}, "coverage_by_spatial_block": {}, "width_by_horizon": {}, "residual_morans_i_after_calibration": null }, "segment_diagnostics": { "pickup_zone": { "segment_key": "pickup_zone", "segment_count": 146, "row_count_min": 1, "row_count_max": 464, "rmse_quantiles": { "p50": 0.34654630667911246, "p90": 0.6650031112721446, "max": 0.9551281671495451 }, "mae_quantiles": { "p50": 0.27858539179548547, "p90": 0.5894824211512286, "max": 0.9551281671495451 }, "wape_quantiles": { "p50": 0.041666697750061515, "p90": 0.0796086715808377, "max": 0.15470087785468817 }, "worst_rmse_segments": [ { "pickup_zone": 222, "rows": 1, "rmse": 0.9551281671495451, "mae": 0.9551281671495451, "bias": -0.9551281671495451, "wape": 0.11012662360357639 }, { "pickup_zone": 57, "rows": 1, "rmse": 0.9509376273483792, "mae": 0.9509376273483792, "bias": -0.9509376273483792, "wape": 0.11213736122033721 }, { "pickup_zone": 197, "rows": 1, "rmse": 0.934432706300532, "mae": 0.934432706300532, "bias": 0.934432706300532, "wape": 0.15470087785468817 }, { "pickup_zone": 61, "rows": 3, "rmse": 0.9047937781850359, "mae": 0.6665910587170168, "bias": -0.6665910587170168, "wape": 0.08348238666683602 }, { "pickup_zone": 254, "rows": 1, "rmse": 0.8762201638915483, "mae": 0.8762201638915483, "bias": -0.8762201638915483, "wape": 0.10827088759078143 } ], "largest_abs_bias_segments": [ { "pickup_zone": 222, "rows": 1, "rmse": 0.9551281671495451, "mae": 0.9551281671495451, "bias": -0.9551281671495451, "wape": 0.11012662360357639 }, { "pickup_zone": 57, "rows": 1, "rmse": 0.9509376273483792, "mae": 0.9509376273483792, "bias": -0.9509376273483792, "wape": 0.11213736122033721 }, { "pickup_zone": 197, "rows": 1, "rmse": 0.934432706300532, "mae": 0.934432706300532, "bias": 0.934432706300532, "wape": 0.15470087785468817 }, { "pickup_zone": 254, "rows": 1, "rmse": 0.8762201638915483, "mae": 0.8762201638915483, "bias": -0.8762201638915483, "wape": 0.10827088759078143 }, { "pickup_zone": 259, "rows": 1, "rmse": 0.7745012455357667, "mae": 0.7745012455357667, "bias": -0.7745012455357667, "wape": 0.09031200452791391 } ] } } }, "xgboost": { "status": "ok", "metrics": { "rmse": 0.3492241515118479, "mae": 0.27121271749755005, "r2": 0.7551837023499789, "wape": 0.04090186573107125 }, "timing": { "train_seconds": 0.04941254200821277, "predict_seconds": 0.0005512920033652335, "fit_predict_seconds": 0.049963834011578, "prediction_rows": 10000.0, "predict_rows_per_second": 18139207.42357468 }, "config": { "n_estimators": 24, "learning_rate": 0.08, "max_depth": 4, "tree_method": "hist", "max_bin": 256, "subsample": 1.0, "colsample_bytree": 1.0, "zone_treatment": "target_mean", "feature_count": 22 }, "calibration_report_fields": { "status": "not_computed", "coverage_by_horizon": {}, "coverage_by_spatial_block": {}, "width_by_horizon": {}, "residual_morans_i_after_calibration": null }, "segment_diagnostics": { "pickup_zone": { "segment_key": "pickup_zone", "segment_count": 146, "row_count_min": 1, "row_count_max": 464, "rmse_quantiles": { "p50": 0.350441443958156, "p90": 0.6633923125001098, "max": 0.9848169096637482 }, "mae_quantiles": { "p50": 0.2779071517503072, "p90": 0.588156403960624, "max": 0.9848169096637482 }, "wape_quantiles": { "p50": 0.041189410397883555, "p90": 0.07982069992410773, "max": 0.16304228161521947 }, "worst_rmse_segments": [ { "pickup_zone": 197, "rows": 1, "rmse": 0.9848169096637482, "mae": 0.9848169096637482, "bias": 0.9848169096637482, "wape": 0.16304228161521947 }, { "pickup_zone": 222, "rows": 1, "rmse": 0.9530360919868102, "mae": 0.9530360919868102, "bias": -0.9530360919868102, "wape": 0.10988540657959889 }, { "pickup_zone": 57, "rows": 1, "rmse": 0.9468528840415154, "mae": 0.9468528840415154, "bias": -0.9468528840415154, "wape": 0.11165567627852731 }, { "pickup_zone": 254, "rows": 1, "rmse": 0.9230102148552781, "mae": 0.9230102148552781, "bias": -0.9230102148552781, "wape": 0.11405253991633549 }, { "pickup_zone": 61, "rows": 3, "rmse": 0.9118431424520607, "mae": 0.6681692630185806, "bias": -0.6681692630185806, "wape": 0.08368003747540823 } ], "largest_abs_bias_segments": [ { "pickup_zone": 197, "rows": 1, "rmse": 0.9848169096637482, "mae": 0.9848169096637482, "bias": 0.9848169096637482, "wape": 0.16304228161521947 }, { "pickup_zone": 222, "rows": 1, "rmse": 0.9530360919868102, "mae": 0.9530360919868102, "bias": -0.9530360919868102, "wape": 0.10988540657959889 }, { "pickup_zone": 57, "rows": 1, "rmse": 0.9468528840415154, "mae": 0.9468528840415154, "bias": -0.9468528840415154, "wape": 0.11165567627852731 }, { "pickup_zone": 254, "rows": 1, "rmse": 0.9230102148552781, "mae": 0.9230102148552781, "bias": -0.9230102148552781, "wape": 0.11405253991633549 }, { "pickup_zone": 259, "rows": 1, "rmse": 0.7647331818623986, "mae": 0.7647331818623986, "bias": -0.7647331818623986, "wape": 0.08917298323416774 } ] } } }, "catboost": { "status": "ok", "metrics": { "rmse": 0.36673747677420515, "mae": 0.28543625596563443, "r2": 0.7300133007551008, "wape": 0.043046932031834115 }, "timing": { "train_seconds": 0.049258250001003034, "predict_seconds": 0.000781249997089617, "fit_predict_seconds": 0.05003949999809265, "prediction_rows": 10000.0, "predict_rows_per_second": 12800000.047683716 }, "config": { "iterations": 24, "learning_rate": 0.08, "depth": 4, "zone_treatment": "target_mean", "feature_count": 22 }, "calibration_report_fields": { "status": "not_computed", "coverage_by_horizon": {}, "coverage_by_spatial_block": {}, "width_by_horizon": {}, "residual_morans_i_after_calibration": null }, "segment_diagnostics": { "pickup_zone": { "segment_key": "pickup_zone", "segment_count": 146, "row_count_min": 1, "row_count_max": 464, "rmse_quantiles": { "p50": 0.36319561784305593, "p90": 0.6687535246743557, "max": 1.1347309823839753 }, "mae_quantiles": { "p50": 0.28550174871243716, "p90": 0.5932156893494553, "max": 1.1347309823839753 }, "wape_quantiles": { "p50": 0.04314005005034077, "p90": 0.07960859841547993, "max": 0.1878614456879416 }, "worst_rmse_segments": [ { "pickup_zone": 197, "rows": 1, "rmse": 1.1347309823839753, "mae": 1.1347309823839753, "bias": 1.1347309823839753, "wape": 0.1878614456879416 }, { "pickup_zone": 222, "rows": 1, "rmse": 1.0742644321515584, "mae": 1.0742644321515584, "bias": -1.0742644321515584, "wape": 0.1238630781074445 }, { "pickup_zone": 57, "rows": 1, "rmse": 0.9960222721339411, "mae": 0.9960222721339411, "bias": -0.9960222721339411, "wape": 0.11745387510348908 }, { "pickup_zone": 61, "rows": 3, "rmse": 0.9756170792848411, "mae": 0.700791628193708, "bias": -0.700791628193708, "wape": 0.08776559018110819 }, { "pickup_zone": 259, "rows": 1, "rmse": 0.9211412288435179, "mae": 0.9211412288435179, "bias": -0.9211412288435179, "wape": 0.10741120341596949 } ], "largest_abs_bias_segments": [ { "pickup_zone": 197, "rows": 1, "rmse": 1.1347309823839753, "mae": 1.1347309823839753, "bias": 1.1347309823839753, "wape": 0.1878614456879416 }, { "pickup_zone": 222, "rows": 1, "rmse": 1.0742644321515584, "mae": 1.0742644321515584, "bias": -1.0742644321515584, "wape": 0.1238630781074445 }, { "pickup_zone": 57, "rows": 1, "rmse": 0.9960222721339411, "mae": 0.9960222721339411, "bias": -0.9960222721339411, "wape": 0.11745387510348908 }, { "pickup_zone": 259, "rows": 1, "rmse": 0.9211412288435179, "mae": 0.9211412288435179, "bias": -0.9211412288435179, "wape": 0.10741120341596949 }, { "pickup_zone": 254, "rows": 1, "rmse": 0.8624556734607864, "mae": 0.8624556734607864, "bias": -0.8624556734607864, "wape": 0.10657006665834068 } ] } } }, "hist_gradient_boosting": { "status": "ok", "metrics": { "rmse": 0.34577174947338457, "mae": 0.2679223069223183, "r2": 0.7600002447937049, "wape": 0.040405635566092196 }, "timing": { "train_seconds": 0.24266266598715447, "predict_seconds": 0.004723500009276904, "fit_predict_seconds": 0.24738616599643137, "prediction_rows": 10000.0, "predict_rows_per_second": 2117074.1992929196 }, "config": { "n_estimators": 24, "learning_rate": 0.08, "max_leaf_nodes": 16, "zone_treatment": "target_mean", "feature_count": 22 }, "calibration_report_fields": { "status": "not_computed", "coverage_by_horizon": {}, "coverage_by_spatial_block": {}, "width_by_horizon": {}, "residual_morans_i_after_calibration": null }, "segment_diagnostics": { "pickup_zone": { "segment_key": "pickup_zone", "segment_count": 146, "row_count_min": 1, "row_count_max": 464, "rmse_quantiles": { "p50": 0.34352095823468304, "p90": 0.6454001493928133, "max": 1.059634039806431 }, "mae_quantiles": { "p50": 0.2738949285146197, "p90": 0.6009005224243933, "max": 1.059634039806431 }, "wape_quantiles": { "p50": 0.041491757761747225, "p90": 0.08186912024304008, "max": 0.1754287013473205 }, "worst_rmse_segments": [ { "pickup_zone": 197, "rows": 1, "rmse": 1.059634039806431, "mae": 1.059634039806431, "bias": 1.059634039806431, "wape": 0.1754287013473205 }, { "pickup_zone": 222, "rows": 1, "rmse": 1.0188663175590884, "mae": 1.0188663175590884, "bias": -1.0188663175590884, "wape": 0.11747565543067452 }, { "pickup_zone": 57, "rows": 1, "rmse": 0.9559772596391847, "mae": 0.9559772596391847, "bias": -0.9559772596391847, "wape": 0.11273164947895577 }, { "pickup_zone": 254, "rows": 1, "rmse": 0.940344810032868, "mae": 0.940344810032868, "bias": -0.940344810032868, "wape": 0.11619450386928651 }, { "pickup_zone": 61, "rows": 3, "rmse": 0.8975119383872967, "mae": 0.6728996792337067, "bias": -0.6728996792337067, "wape": 0.08427246431702579 } ], "largest_abs_bias_segments": [ { "pickup_zone": 197, "rows": 1, "rmse": 1.059634039806431, "mae": 1.059634039806431, "bias": 1.059634039806431, "wape": 0.1754287013473205 }, { "pickup_zone": 222, "rows": 1, "rmse": 1.0188663175590884, "mae": 1.0188663175590884, "bias": -1.0188663175590884, "wape": 0.11747565543067452 }, { "pickup_zone": 57, "rows": 1, "rmse": 0.9559772596391847, "mae": 0.9559772596391847, "bias": -0.9559772596391847, "wape": 0.11273164947895577 }, { "pickup_zone": 254, "rows": 1, "rmse": 0.940344810032868, "mae": 0.940344810032868, "bias": -0.940344810032868, "wape": 0.11619450386928651 }, { "pickup_zone": 259, "rows": 1, "rmse": 0.8068947305103649, "mae": 0.8068947305103649, "bias": -0.8068947305103649, "wape": 0.09408930066341215 } ] } } } } } } }, "fare": { "display_name": "Fare amount", "description": "Predict log total amount from zone, trip, passenger, and time features.", "row_count": 50000, "feature_names": [ "trip_distance", "log_trip_distance", "passenger_count", "hour", "dayofweek", "PULocationID", "DOLocationID" ], "zone_treatment": "target_mean", "splits": { "random": { "train_rows": 40000, "test_rows": 10000, "split_manifest": { "split_id": "fare_random", "split_kind": "seeded_row_shuffle", "dataset_fingerprint": "5110541e767f70aabdc8ee1eb158beaab347fb8d3fede1cd1a47e2637d3825f2", "coordinate_crs_note": "No coordinate CRS applies to this random-row diagnostic split.", "model_version": "0.3.0", "dependency_versions": { "cartoboost": "0.3.0", "numpy": "1.26.4", "scikit-learn": "1.9.0", "xgboost": "3.2.0", "lightgbm": "4.6.0", "catboost": "1.2.10" }, "random_seed": 42, "row_count": 50000, "train_index_sha256": "sha256:5656dfaa20d5c5d6eca74ec12ee802f587b040049ad44667e7d6e3b4a4254055", "test_index_sha256": "sha256:bee73ca172c57c21b478d46bb81c1524faff5ec9465e8c98e26c07687ed49f32", "random_row_split_allowed_for_geo_claim": true, "leakage_safe": false, "split_manifest_hash": "sha256:ed57b397be0b8074fcec43717f8f1b4e040f671b61a31e64e38e58ab92c19d78" }, "holdout_pickup_zones": [ 2, 4, 7, 10, 12, 13, 14, 17, 21, 24, 25, 28, 33, 35, 36, 37, 38, 39, 40, 41, 42, 43, 45, 48, 49, 50, 52, 56, 61, 62, 64, 65, 66, 68, 69, 70, 71, 74, 75, 76, 77, 78, 79, 80, 85, 87, 88, 89, 90, 91, 92, 93, 95, 97, 100, 106, 107, 112, 113, 114, 116, 119, 123, 125, 126, 127, 129, 130, 131, 132, 134, 135, 137, 138, 140, 141, 142, 143, 144, 145, 146, 147, 148, 151, 152, 158, 159, 161, 162, 163, 164, 166, 167, 168, 170, 174, 177, 179, 181, 186, 188, 189, 195, 196, 209, 211, 215, 216, 218, 223, 224, 225, 226, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 241, 242, 244, 246, 247, 249, 252, 255, 256, 259, 260, 261, 262, 263 ], "models": { "cartoboost": { "status": "ok", "metrics": { "rmse": 0.17294850472898857, "mae": 0.1316499317331205, "r2": 0.8870946806164656, "wape": 0.04136462720028184 }, "timing": { "train_seconds": 84.1884249170107, "predict_seconds": 0.03486954199615866, "fit_predict_seconds": 84.22329445900687, "prediction_rows": 10000.0, "predict_rows_per_second": 286783.2333760401 }, "backend": "rust", "config": { "n_estimators": 24, "learning_rate": 0.08, "max_depth": 5, "min_samples_leaf": 20, "constant_l2_regularization": 0.0, "leaf_predictor": "constant", "init": "constant", "calibration": "none", "splitters": [ "axis_histogram:512", "diagonal_2d", "gaussian_2d", "periodic:24", "periodic:7", "sparse_set" ], "sparse_sets": false, "zone_treatment": "target_mean", "feature_count": 22, "preset": "candidate", "predict_path": "model.predict" }, "calibration_report_fields": { "status": "not_computed", "coverage_by_horizon": {}, "coverage_by_spatial_block": {}, "width_by_horizon": {}, "residual_morans_i_after_calibration": null }, "segment_diagnostics": { "pickup_zone": { "segment_key": "pickup_zone", "segment_count": 138, "row_count_min": 1, "row_count_max": 464, "rmse_quantiles": { "p50": 0.1736415796715071, "p90": 0.3326078825580504, "max": 0.979602604513452 }, "mae_quantiles": { "p50": 0.14002902205197876, "p90": 0.32470400061249954, "max": 0.979602604513452 }, "wape_quantiles": { "p50": 0.043086487000742996, "p90": 0.09873181547439289, "max": 0.2338146374418538 }, "worst_rmse_segments": [ { "pickup_zone": 106, "rows": 1, "rmse": 0.979602604513452, "mae": 0.979602604513452, "bias": -0.979602604513452, "wape": 0.2338146374418538 }, { "pickup_zone": 216, "rows": 5, "rmse": 0.6185656102127242, "mae": 0.45496584788028455, "bias": -0.02228234090931718, "wape": 0.1255727592553027 }, { "pickup_zone": 130, "rows": 4, "rmse": 0.5790655702613812, "mae": 0.3736173869880005, "bias": -0.1839229425383967, "wape": 0.10986086349470627 }, { "pickup_zone": 145, "rows": 8, "rmse": 0.5511307421936625, "mae": 0.4698689073592709, "bias": -0.09712503904860204, "wape": 0.12376076064586912 }, { "pickup_zone": 78, "rows": 2, "rmse": 0.5208109455058726, "mae": 0.5016750641735774, "bias": -0.139879129769805, "wape": 0.14852623581487637 } ], "largest_abs_bias_segments": [ { "pickup_zone": 106, "rows": 1, "rmse": 0.979602604513452, "mae": 0.979602604513452, "bias": -0.979602604513452, "wape": 0.2338146374418538 }, { "pickup_zone": 77, "rows": 1, "rmse": 0.5160441080985838, "mae": 0.5160441080985838, "bias": 0.5160441080985838, "wape": 0.13561093075902889 }, { "pickup_zone": 223, "rows": 4, "rmse": 0.4661422364107865, "mae": 0.45273364958974815, "bias": -0.45273364958974815, "wape": 0.11375655799136869 }, { "pickup_zone": 40, "rows": 2, "rmse": 0.4738479192494172, "mae": 0.45043123236509763, "bias": -0.45043123236509763, "wape": 0.11576708967436063 }, { "pickup_zone": 92, "rows": 1, "rmse": 0.3813646943575222, "mae": 0.3813646943575222, "bias": -0.3813646943575222, "wape": 0.08476385678156262 } ] } } }, "lightgbm": { "status": "ok", "metrics": { "rmse": 0.18154241732013726, "mae": 0.13957498083508607, "r2": 0.8755952369326658, "wape": 0.04385469079037379 }, "timing": { "train_seconds": 0.11702554200019222, "predict_seconds": 0.002057457997580059, "fit_predict_seconds": 0.11908299999777228, "prediction_rows": 10000.0, "predict_rows_per_second": 4860366.535677424 }, "config": { "n_estimators": 24, "learning_rate": 0.08, "max_depth": 4, "num_leaves": 16, "zone_treatment": "target_mean", "feature_count": 22 }, "calibration_report_fields": { "status": "not_computed", "coverage_by_horizon": {}, "coverage_by_spatial_block": {}, "width_by_horizon": {}, "residual_morans_i_after_calibration": null }, "segment_diagnostics": { "pickup_zone": { "segment_key": "pickup_zone", "segment_count": 138, "row_count_min": 1, "row_count_max": 464, "rmse_quantiles": { "p50": 0.19230850555453657, "p90": 0.36175655461827494, "max": 1.0045271164831533 }, "mae_quantiles": { "p50": 0.15072780862064555, "p90": 0.3499003541663528, "max": 1.0045271164831533 }, "wape_quantiles": { "p50": 0.0453472147238673, "p90": 0.10826473946978285, "max": 0.2397636985231127 }, "worst_rmse_segments": [ { "pickup_zone": 106, "rows": 1, "rmse": 1.0045271164831533, "mae": 1.0045271164831533, "bias": -1.0045271164831533, "wape": 0.2397636985231127 }, { "pickup_zone": 216, "rows": 5, "rmse": 0.6364141573037558, "mae": 0.49613611203608876, "bias": 0.030425047940155457, "wape": 0.13693594990664665 }, { "pickup_zone": 77, "rows": 1, "rmse": 0.5579639829806191, "mae": 0.5579639829806191, "bias": 0.5579639829806191, "wape": 0.14662703027618262 }, { "pickup_zone": 145, "rows": 8, "rmse": 0.5543652921727478, "mae": 0.4589218721885544, "bias": -0.1184537490510773, "wape": 0.12087737470922735 }, { "pickup_zone": 78, "rows": 2, "rmse": 0.541449461630517, "mae": 0.5345947190801337, "bias": -0.08588367616497128, "wape": 0.15827244960297845 } ], "largest_abs_bias_segments": [ { "pickup_zone": 106, "rows": 1, "rmse": 1.0045271164831533, "mae": 1.0045271164831533, "bias": -1.0045271164831533, "wape": 0.2397636985231127 }, { "pickup_zone": 77, "rows": 1, "rmse": 0.5579639829806191, "mae": 0.5579639829806191, "bias": 0.5579639829806191, "wape": 0.14662703027618262 }, { "pickup_zone": 223, "rows": 4, "rmse": 0.4785628530403919, "mae": 0.4655827695517214, "bias": -0.4655827695517214, "wape": 0.11698510453615674 }, { "pickup_zone": 40, "rows": 2, "rmse": 0.4561279722885312, "mae": 0.4385006945806782, "bias": -0.4385006945806782, "wape": 0.11270077557731167 }, { "pickup_zone": 64, "rows": 1, "rmse": 0.3911970578573212, "mae": 0.3911970578573212, "bias": 0.3911970578573212, "wape": 0.11815401347726967 } ] } } }, "xgboost": { "status": "ok", "metrics": { "rmse": 0.1815853434596485, "mae": 0.13941737467051524, "r2": 0.8755363983686808, "wape": 0.0438051706717067 }, "timing": { "train_seconds": 0.06769837500178255, "predict_seconds": 0.0007357910071732476, "fit_predict_seconds": 0.0684341660089558, "prediction_rows": 10000.0, "predict_rows_per_second": 13590815.737770254 }, "config": { "n_estimators": 24, "learning_rate": 0.08, "max_depth": 4, "tree_method": "hist", "max_bin": 256, "subsample": 1.0, "colsample_bytree": 1.0, "zone_treatment": "target_mean", "feature_count": 22 }, "calibration_report_fields": { "status": "not_computed", "coverage_by_horizon": {}, "coverage_by_spatial_block": {}, "width_by_horizon": {}, "residual_morans_i_after_calibration": null }, "segment_diagnostics": { "pickup_zone": { "segment_key": "pickup_zone", "segment_count": 138, "row_count_min": 1, "row_count_max": 464, "rmse_quantiles": { "p50": 0.1896475267724423, "p90": 0.35756096356693595, "max": 1.003395949149228 }, "mae_quantiles": { "p50": 0.1478749598550527, "p90": 0.34984313586020194, "max": 1.003395949149228 }, "wape_quantiles": { "p50": 0.044922160515502435, "p90": 0.10924146339491989, "max": 0.239493707938309 }, "worst_rmse_segments": [ { "pickup_zone": 106, "rows": 1, "rmse": 1.003395949149228, "mae": 1.003395949149228, "bias": -1.003395949149228, "wape": 0.239493707938309 }, { "pickup_zone": 216, "rows": 5, "rmse": 0.6376616742868348, "mae": 0.5090615235020692, "bias": 0.041200645810198465, "wape": 0.14050342555312723 }, { "pickup_zone": 77, "rows": 1, "rmse": 0.5588565896576116, "mae": 0.5588565896576116, "bias": 0.5588565896576116, "wape": 0.14686159786521044 }, { "pickup_zone": 130, "rows": 4, "rmse": 0.5573142198191042, "mae": 0.36283786182344746, "bias": -0.18099090383678063, "wape": 0.106691182468382 }, { "pickup_zone": 145, "rows": 8, "rmse": 0.5492561926311651, "mae": 0.45359621052676846, "bias": -0.10361913233150133, "wape": 0.1194746261385474 } ], "largest_abs_bias_segments": [ { "pickup_zone": 106, "rows": 1, "rmse": 1.003395949149228, "mae": 1.003395949149228, "bias": -1.003395949149228, "wape": 0.239493707938309 }, { "pickup_zone": 77, "rows": 1, "rmse": 0.5588565896576116, "mae": 0.5588565896576116, "bias": 0.5588565896576116, "wape": 0.14686159786521044 }, { "pickup_zone": 40, "rows": 2, "rmse": 0.4823013866706859, "mae": 0.46863796897645926, "bias": -0.46863796897645926, "wape": 0.12044647413644102 }, { "pickup_zone": 223, "rows": 4, "rmse": 0.46366000557110065, "mae": 0.45182531556461425, "bias": -0.45182531556461425, "wape": 0.11352832456471858 }, { "pickup_zone": 134, "rows": 1, "rmse": 0.3985085300937157, "mae": 0.3985085300937157, "bias": 0.3985085300937157, "wape": 0.1563118652789218 } ] } } }, "catboost": { "status": "ok", "metrics": { "rmse": 0.19259074078258653, "mae": 0.14852123030734304, "r2": 0.8599924074958962, "wape": 0.04666561723286377 }, "timing": { "train_seconds": 0.05580479200580157, "predict_seconds": 0.000830165998195298, "fit_predict_seconds": 0.05663495800399687, "prediction_rows": 10000.0, "predict_rows_per_second": 12045783.640547853 }, "config": { "iterations": 24, "learning_rate": 0.08, "depth": 4, "zone_treatment": "target_mean", "feature_count": 22 }, "calibration_report_fields": { "status": "not_computed", "coverage_by_horizon": {}, "coverage_by_spatial_block": {}, "width_by_horizon": {}, "residual_morans_i_after_calibration": null }, "segment_diagnostics": { "pickup_zone": { "segment_key": "pickup_zone", "segment_count": 138, "row_count_min": 1, "row_count_max": 464, "rmse_quantiles": { "p50": 0.19369165563026472, "p90": 0.3686926969452859, "max": 0.9976731413620556 }, "mae_quantiles": { "p50": 0.15337149827897706, "p90": 0.35888065008815717, "max": 0.9976731413620556 }, "wape_quantiles": { "p50": 0.04744091938688928, "p90": 0.11280767136941784, "max": 0.2381277701368556 }, "worst_rmse_segments": [ { "pickup_zone": 106, "rows": 1, "rmse": 0.9976731413620556, "mae": 0.9976731413620556, "bias": -0.9976731413620556, "wape": 0.2381277701368556 }, { "pickup_zone": 216, "rows": 5, "rmse": 0.6363963551690577, "mae": 0.5231730441262702, "bias": 0.07109019833049146, "wape": 0.14439827302426156 }, { "pickup_zone": 130, "rows": 4, "rmse": 0.5597210528177954, "mae": 0.3855890969610881, "bias": -0.15191471568801662, "wape": 0.11338110222276586 }, { "pickup_zone": 145, "rows": 8, "rmse": 0.559586748376577, "mae": 0.458039930864958, "bias": -0.11126296539143155, "wape": 0.12064507645040713 }, { "pickup_zone": 78, "rows": 2, "rmse": 0.5059952109516971, "mae": 0.5022836765618022, "bias": -0.06117402852200704, "wape": 0.14870642198227585 } ], "largest_abs_bias_segments": [ { "pickup_zone": 106, "rows": 1, "rmse": 0.9976731413620556, "mae": 0.9976731413620556, "bias": -0.9976731413620556, "wape": 0.2381277701368556 }, { "pickup_zone": 64, "rows": 1, "rmse": 0.48913961563800834, "mae": 0.48913961563800834, "bias": 0.48913961563800834, "wape": 0.14773579600754183 }, { "pickup_zone": 223, "rows": 4, "rmse": 0.46633207019279943, "mae": 0.45602426969628806, "bias": -0.45602426969628806, "wape": 0.11458337883253297 }, { "pickup_zone": 134, "rows": 1, "rmse": 0.44812200827418325, "mae": 0.44812200827418325, "bias": 0.44812200827418325, "wape": 0.17577236544824118 }, { "pickup_zone": 77, "rows": 1, "rmse": 0.4396914866085635, "mae": 0.4396914866085635, "bias": 0.4396914866085635, "wape": 0.11554626980532717 } ] } } }, "hist_gradient_boosting": { "status": "ok", "metrics": { "rmse": 0.17915465884646392, "mae": 0.13673145631972836, "r2": 0.8788462131871305, "wape": 0.04296125066500347 }, "timing": { "train_seconds": 0.2711835000081919, "predict_seconds": 0.005650334001984447, "fit_predict_seconds": 0.27683383401017636, "prediction_rows": 10000.0, "predict_rows_per_second": 1769806.8815910548 }, "config": { "n_estimators": 24, "learning_rate": 0.08, "max_leaf_nodes": 16, "zone_treatment": "target_mean", "feature_count": 22 }, "calibration_report_fields": { "status": "not_computed", "coverage_by_horizon": {}, "coverage_by_spatial_block": {}, "width_by_horizon": {}, "residual_morans_i_after_calibration": null }, "segment_diagnostics": { "pickup_zone": { "segment_key": "pickup_zone", "segment_count": 138, "row_count_min": 1, "row_count_max": 464, "rmse_quantiles": { "p50": 0.186731280627679, "p90": 0.3657075450062455, "max": 0.9941997819188799 }, "mae_quantiles": { "p50": 0.1450123260750933, "p90": 0.3445733391556954, "max": 0.9941997819188799 }, "wape_quantiles": { "p50": 0.044801642092915095, "p90": 0.10548337192560794, "max": 0.23729873775661328 }, "worst_rmse_segments": [ { "pickup_zone": 106, "rows": 1, "rmse": 0.9941997819188799, "mae": 0.9941997819188799, "bias": -0.9941997819188799, "wape": 0.23729873775661328 }, { "pickup_zone": 216, "rows": 5, "rmse": 0.6381440075869832, "mae": 0.5074742848243934, "bias": 0.038896940561459384, "wape": 0.14006534005444396 }, { "pickup_zone": 130, "rows": 4, "rmse": 0.5586097114214629, "mae": 0.37140505614199437, "bias": -0.17179644030311647, "wape": 0.10921033547983629 }, { "pickup_zone": 77, "rows": 1, "rmse": 0.5568430862267415, "mae": 0.5568430862267415, "bias": 0.5568430862267415, "wape": 0.14633247047074624 }, { "pickup_zone": 145, "rows": 8, "rmse": 0.5497085220602381, "mae": 0.4487429980493213, "bias": -0.1183618443524902, "wape": 0.11819631795859024 } ], "largest_abs_bias_segments": [ { "pickup_zone": 106, "rows": 1, "rmse": 0.9941997819188799, "mae": 0.9941997819188799, "bias": -0.9941997819188799, "wape": 0.23729873775661328 }, { "pickup_zone": 77, "rows": 1, "rmse": 0.5568430862267415, "mae": 0.5568430862267415, "bias": 0.5568430862267415, "wape": 0.14633247047074624 }, { "pickup_zone": 223, "rows": 4, "rmse": 0.4893436743347984, "mae": 0.47926887821344977, "bias": -0.47926887821344977, "wape": 0.12042395785546461 }, { "pickup_zone": 40, "rows": 2, "rmse": 0.461671557209689, "mae": 0.4382599748026239, "bias": -0.4382599748026239, "wape": 0.11263890724729805 }, { "pickup_zone": 64, "rows": 1, "rmse": 0.416187100350784, "mae": 0.416187100350784, "bias": 0.416187100350784, "wape": 0.12570180495030026 } ] } } } } }, "spatial_holdout": { "train_rows": 40772, "test_rows": 9228, "split_manifest": { "split_id": "fare_spatial_holdout", "split_kind": "group_spatial_cv", "dataset_fingerprint": "5110541e767f70aabdc8ee1eb158beaab347fb8d3fede1cd1a47e2637d3825f2", "coordinate_crs_note": "NYC TLC pickup/dropoff zone identifiers are treated as spatial groups; distance-buffered claims require projected taxi zone geometry.", "model_version": "0.3.0", "dependency_versions": { "cartoboost": "0.3.0", "numpy": "1.26.4", "scikit-learn": "1.9.0", "xgboost": "3.2.0", "lightgbm": "4.6.0", "catboost": "1.2.10" }, "random_seed": 42, "row_count": 50000, "train_index_sha256": "sha256:f1ed6d3eeb63ae16c0f13fe796c8db8678cd29cf5fb07b258860f873653004df", "test_index_sha256": "sha256:2cc5ef67dbed99aff0b6de977e3d150e32de539c1b72494d42b1a71c708dbb61", "random_row_split_allowed_for_geo_claim": false, "leakage_safe": true, "split_manifest_hash": "sha256:27f918c994550d46489b2daa0df7ac5f2c3c4a99cb9b97675e3039fddc42c74a" }, "holdout_pickup_zones": [ 2, 12, 18, 26, 35, 40, 47, 52, 57, 64, 69, 74, 79, 85, 90, 95, 106, 114, 122, 127, 132, 137, 142, 147, 154, 160, 165, 171, 178, 185, 191, 197, 205, 212, 219, 225, 230, 235, 241, 247, 254, 260 ], "models": { "cartoboost": { "status": "ok", "metrics": { "rmse": 0.2740822914348948, "mae": 0.19832947924231076, "r2": 0.8347378763514225, "wape": 0.05872393068742097 }, "timing": { "train_seconds": 109.66248483301024, "predict_seconds": 0.04000287500093691, "fit_predict_seconds": 109.70248770801118, "prediction_rows": 9228.0, "predict_rows_per_second": 230683.4196238113 }, "backend": "rust", "config": { "n_estimators": 24, "learning_rate": 0.08, "max_depth": 5, "min_samples_leaf": 20, "constant_l2_regularization": 0.0, "leaf_predictor": "constant", "init": "constant", "calibration": "none", "splitters": [ "axis_histogram:512", "diagonal_2d", "gaussian_2d", "periodic:24", "periodic:7", "sparse_set" ], "sparse_sets": false, "zone_treatment": "target_mean", "feature_count": 22, "preset": "candidate", "predict_path": "model.predict" }, "calibration_report_fields": { "status": "not_computed", "coverage_by_horizon": {}, "coverage_by_spatial_block": {}, "width_by_horizon": {}, "residual_morans_i_after_calibration": null }, "segment_diagnostics": { "pickup_zone": { "segment_key": "pickup_zone", "segment_count": 42, "row_count_min": 1, "row_count_max": 2076, "rmse_quantiles": { "p50": 0.19456620371954916, "p90": 0.4571363597852067, "max": 0.6613932019530568 }, "mae_quantiles": { "p50": 0.1707104569178089, "p90": 0.39873603954552694, "max": 0.5509605217727386 }, "wape_quantiles": { "p50": 0.04934061310743522, "p90": 0.10724632821918846, "max": 0.16407851442712654 }, "worst_rmse_segments": [ { "pickup_zone": 106, "rows": 3, "rmse": 0.6613932019530568, "mae": 0.5509605217727386, "bias": -0.18371044692852503, "wape": 0.16407851442712654 }, { "pickup_zone": 197, "rows": 3, "rmse": 0.5633914058455747, "mae": 0.40438745037364354, "bias": -0.3732805405386732, "wape": 0.10865651150151662 }, { "pickup_zone": 191, "rows": 3, "rmse": 0.540093098720968, "mae": 0.33137801233332737, "bias": -0.31535605771613023, "wape": 0.0846818213350062 }, { "pickup_zone": 132, "rows": 2076, "rmse": 0.48394191584372814, "mae": 0.44404369642250907, "bias": -0.4238101243264531, "wape": 0.10239980686995803 }, { "pickup_zone": 154, "rows": 1, "rmse": 0.45740120397941286, "mae": 0.45740120397941286, "bias": 0.45740120397941286, "wape": 0.15398565058216113 } ], "largest_abs_bias_segments": [ { "pickup_zone": 154, "rows": 1, "rmse": 0.45740120397941286, "mae": 0.45740120397941286, "bias": 0.45740120397941286, "wape": 0.15398565058216113 }, { "pickup_zone": 132, "rows": 2076, "rmse": 0.48394191584372814, "mae": 0.44404369642250907, "bias": -0.4238101243264531, "wape": 0.10239980686995803 }, { "pickup_zone": 26, "rows": 2, "rmse": 0.45475276203735165, "mae": 0.4096181543572124, "bias": 0.4096181543572124, "wape": 0.11944718823233345 }, { "pickup_zone": 197, "rows": 3, "rmse": 0.5633914058455747, "mae": 0.40438745037364354, "bias": -0.3732805405386732, "wape": 0.10865651150151662 }, { "pickup_zone": 160, "rows": 1, "rmse": 0.3282712017143359, "mae": 0.3282712017143359, "bias": 0.3282712017143359, "wape": 0.10683500372292833 } ] } } }, "lightgbm": { "status": "ok", "metrics": { "rmse": 0.2602095814351218, "mae": 0.19341984656956238, "r2": 0.8510440195561513, "wape": 0.05727022380593955 }, "timing": { "train_seconds": 0.13305358399520628, "predict_seconds": 0.0020750830008182675, "fit_predict_seconds": 0.13512866699602455, "prediction_rows": 9228.0, "predict_rows_per_second": 4447051.0318677 }, "config": { "n_estimators": 24, "learning_rate": 0.08, "max_depth": 4, "num_leaves": 16, "zone_treatment": "target_mean", "feature_count": 22 }, "calibration_report_fields": { "status": "not_computed", "coverage_by_horizon": {}, "coverage_by_spatial_block": {}, "width_by_horizon": {}, "residual_morans_i_after_calibration": null }, "segment_diagnostics": { "pickup_zone": { "segment_key": "pickup_zone", "segment_count": 42, "row_count_min": 1, "row_count_max": 2076, "rmse_quantiles": { "p50": 0.21737182727867096, "p90": 0.44473787812647536, "max": 0.6791388309896915 }, "mae_quantiles": { "p50": 0.17520064617438985, "p90": 0.4059806284654708, "max": 0.5725294182299921 }, "wape_quantiles": { "p50": 0.04921631087880996, "p90": 0.11800145104666586, "max": 0.17050182852802043 }, "worst_rmse_segments": [ { "pickup_zone": 106, "rows": 3, "rmse": 0.6791388309896915, "mae": 0.5725294182299921, "bias": -0.19269914753898906, "wape": 0.17050182852802043 }, { "pickup_zone": 197, "rows": 3, "rmse": 0.6022225724997028, "mae": 0.4068084098471652, "bias": -0.377685219909029, "wape": 0.10930700896536313 }, { "pickup_zone": 191, "rows": 3, "rmse": 0.6021828719699714, "mae": 0.41134372703678723, "bias": -0.2762834730728178, "wape": 0.10511661819362515 }, { "pickup_zone": 26, "rows": 2, "rmse": 0.4601991051624861, "mae": 0.43792096068180664, "bias": 0.43792096068180664, "wape": 0.12770046167394197 }, { "pickup_zone": 154, "rows": 1, "rmse": 0.445444257542996, "mae": 0.445444257542996, "bias": 0.445444257542996, "wape": 0.14996030443097222 } ], "largest_abs_bias_segments": [ { "pickup_zone": 154, "rows": 1, "rmse": 0.445444257542996, "mae": 0.445444257542996, "bias": 0.445444257542996, "wape": 0.14996030443097222 }, { "pickup_zone": 26, "rows": 2, "rmse": 0.4601991051624861, "mae": 0.43792096068180664, "bias": 0.43792096068180664, "wape": 0.12770046167394197 }, { "pickup_zone": 122, "rows": 1, "rmse": 0.3894378864399308, "mae": 0.3894378864399308, "bias": 0.3894378864399308, "wape": 0.12037734061108178 }, { "pickup_zone": 197, "rows": 3, "rmse": 0.6022225724997028, "mae": 0.4068084098471652, "bias": -0.377685219909029, "wape": 0.10930700896536313 }, { "pickup_zone": 160, "rows": 1, "rmse": 0.36555064242795465, "mae": 0.36555064242795465, "bias": 0.36555064242795465, "wape": 0.11896750016681062 } ] } } }, "xgboost": { "status": "ok", "metrics": { "rmse": 0.26071202992298564, "mae": 0.19400382502171054, "r2": 0.8504682147374272, "wape": 0.057443135620551825 }, "timing": { "train_seconds": 0.07354495899926405, "predict_seconds": 0.0007588749867863953, "fit_predict_seconds": 0.07430383398605045, "prediction_rows": 9228.0, "predict_rows_per_second": 12160105.630938994 }, "config": { "n_estimators": 24, "learning_rate": 0.08, "max_depth": 4, "tree_method": "hist", "max_bin": 256, "subsample": 1.0, "colsample_bytree": 1.0, "zone_treatment": "target_mean", "feature_count": 22 }, "calibration_report_fields": { "status": "not_computed", "coverage_by_horizon": {}, "coverage_by_spatial_block": {}, "width_by_horizon": {}, "residual_morans_i_after_calibration": null }, "segment_diagnostics": { "pickup_zone": { "segment_key": "pickup_zone", "segment_count": 42, "row_count_min": 1, "row_count_max": 2076, "rmse_quantiles": { "p50": 0.2207123367651576, "p90": 0.43902735587317976, "max": 0.6745276328378078 }, "mae_quantiles": { "p50": 0.18445877054457746, "p90": 0.3975859782782617, "max": 0.57032819108597 }, "wape_quantiles": { "p50": 0.05213441768914112, "p90": 0.11462303930638407, "max": 0.16984629321208575 }, "worst_rmse_segments": [ { "pickup_zone": 106, "rows": 3, "rmse": 0.6745276328378078, "mae": 0.57032819108597, "bias": -0.19105388531404902, "wape": 0.16984629321208575 }, { "pickup_zone": 191, "rows": 3, "rmse": 0.6020135253012974, "mae": 0.40918646988986335, "bias": -0.27788512213175637, "wape": 0.10456534304110925 }, { "pickup_zone": 197, "rows": 3, "rmse": 0.5144704050966914, "mae": 0.3624802866989292, "bias": -0.3255968750741962, "wape": 0.09739630496540815 }, { "pickup_zone": 26, "rows": 2, "rmse": 0.4489892349783677, "mae": 0.424244074858956, "bias": 0.424244074858956, "wape": 0.12371219714529144 }, { "pickup_zone": 132, "rows": 2076, "rmse": 0.4395727110724945, "mae": 0.4001580600151847, "bias": -0.3617970324288957, "wape": 0.09227944995760752 } ], "largest_abs_bias_segments": [ { "pickup_zone": 154, "rows": 1, "rmse": 0.43411915907934695, "mae": 0.43411915907934695, "bias": 0.43411915907934695, "wape": 0.14614767202060688 }, { "pickup_zone": 26, "rows": 2, "rmse": 0.4489892349783677, "mae": 0.424244074858956, "bias": 0.424244074858956, "wape": 0.12371219714529144 }, { "pickup_zone": 122, "rows": 1, "rmse": 0.3744372426459548, "mae": 0.3744372426459548, "bias": 0.3744372426459548, "wape": 0.11574056111363684 }, { "pickup_zone": 132, "rows": 2076, "rmse": 0.4395727110724945, "mae": 0.4001580600151847, "bias": -0.3617970324288957, "wape": 0.09227944995760752 }, { "pickup_zone": 160, "rows": 1, "rmse": 0.3559167717509206, "mae": 0.3559167717509206, "bias": 0.3559167717509206, "wape": 0.11583218216062502 } ] } } }, "catboost": { "status": "ok", "metrics": { "rmse": 0.24593357946134559, "mae": 0.1876002682579834, "r2": 0.8669401474577558, "wape": 0.05554708857306956 }, "timing": { "train_seconds": 0.055603959001018666, "predict_seconds": 0.0010069169948110357, "fit_predict_seconds": 0.0566108759958297, "prediction_rows": 9228.0, "predict_rows_per_second": 9164608.450899951 }, "config": { "iterations": 24, "learning_rate": 0.08, "depth": 4, "zone_treatment": "target_mean", "feature_count": 22 }, "calibration_report_fields": { "status": "not_computed", "coverage_by_horizon": {}, "coverage_by_spatial_block": {}, "width_by_horizon": {}, "residual_morans_i_after_calibration": null }, "segment_diagnostics": { "pickup_zone": { "segment_key": "pickup_zone", "segment_count": 42, "row_count_min": 1, "row_count_max": 2076, "rmse_quantiles": { "p50": 0.24742014311519234, "p90": 0.4817754521542761, "max": 0.6944035152797199 }, "mae_quantiles": { "p50": 0.20334367124334618, "p90": 0.38792078547851444, "max": 0.65043513281531 }, "wape_quantiles": { "p50": 0.059694443275670424, "p90": 0.11886129037680082, "max": 0.18967090915252827 }, "worst_rmse_segments": [ { "pickup_zone": 106, "rows": 3, "rmse": 0.6944035152797199, "mae": 0.5840692190854049, "bias": -0.17655183078391046, "wape": 0.17393843297846057 }, { "pickup_zone": 26, "rows": 2, "rmse": 0.6595144590371659, "mae": 0.65043513281531, "bias": 0.65043513281531, "wape": 0.18967090915252827 }, { "pickup_zone": 191, "rows": 3, "rmse": 0.6112007848342461, "mae": 0.4159686505159527, "bias": -0.27807477607720693, "wape": 0.10629849185202833 }, { "pickup_zone": 197, "rows": 3, "rmse": 0.5225275945923753, "mae": 0.3561808176155609, "bias": -0.303567170957136, "wape": 0.09570367495357664 }, { "pickup_zone": 154, "rows": 1, "rmse": 0.4913255330192303, "mae": 0.4913255330192303, "bias": 0.4913255330192303, "wape": 0.16540638982008127 } ], "largest_abs_bias_segments": [ { "pickup_zone": 26, "rows": 2, "rmse": 0.6595144590371659, "mae": 0.65043513281531, "bias": 0.65043513281531, "wape": 0.18967090915252827 }, { "pickup_zone": 154, "rows": 1, "rmse": 0.4913255330192303, "mae": 0.4913255330192303, "bias": 0.4913255330192303, "wape": 0.16540638982008127 }, { "pickup_zone": 122, "rows": 1, "rmse": 0.3890490749065201, "mae": 0.3890490749065201, "bias": 0.3890490749065201, "wape": 0.12025715687955334 }, { "pickup_zone": 160, "rows": 1, "rmse": 0.37776618062646383, "mae": 0.37776618062646383, "bias": 0.37776618062646383, "wape": 0.12294301511329368 }, { "pickup_zone": 197, "rows": 3, "rmse": 0.5225275945923753, "mae": 0.3561808176155609, "bias": -0.303567170957136, "wape": 0.09570367495357664 } ] } } }, "hist_gradient_boosting": { "status": "ok", "metrics": { "rmse": 0.2411672224353955, "mae": 0.1815549435180951, "r2": 0.8720477464308415, "wape": 0.05375711144831535 }, "timing": { "train_seconds": 0.27435399999376386, "predict_seconds": 0.004518625006312504, "fit_predict_seconds": 0.27887262500007637, "prediction_rows": 9228.0, "predict_rows_per_second": 2042214.1662803427 }, "config": { "n_estimators": 24, "learning_rate": 0.08, "max_leaf_nodes": 16, "zone_treatment": "target_mean", "feature_count": 22 }, "calibration_report_fields": { "status": "not_computed", "coverage_by_horizon": {}, "coverage_by_spatial_block": {}, "width_by_horizon": {}, "residual_morans_i_after_calibration": null }, "segment_diagnostics": { "pickup_zone": { "segment_key": "pickup_zone", "segment_count": 42, "row_count_min": 1, "row_count_max": 2076, "rmse_quantiles": { "p50": 0.22232820962655475, "p90": 0.4659390074324044, "max": 0.6640822328681739 }, "mae_quantiles": { "p50": 0.18084167549483326, "p90": 0.40077672986270013, "max": 0.5534700539778291 }, "wape_quantiles": { "p50": 0.05215315945926109, "p90": 0.1192921197994899, "max": 0.16482586437298732 }, "worst_rmse_segments": [ { "pickup_zone": 106, "rows": 3, "rmse": 0.6640822328681739, "mae": 0.5534700539778291, "bias": -0.18256161953074304, "wape": 0.16482586437298732 }, { "pickup_zone": 191, "rows": 3, "rmse": 0.5994364021423273, "mae": 0.40827670831131685, "bias": -0.2757754374399384, "wape": 0.10433285849299616 }, { "pickup_zone": 197, "rows": 3, "rmse": 0.5859465143585096, "mae": 0.3970298634421443, "bias": -0.35398691516300723, "wape": 0.10667957149433493 }, { "pickup_zone": 26, "rows": 2, "rmse": 0.507721998094011, "mae": 0.4774490894903194, "bias": 0.4774490894903194, "wape": 0.1392271086060623 }, { "pickup_zone": 154, "rows": 1, "rmse": 0.4731330028855747, "mae": 0.4731330028855747, "bias": 0.4731330028855747, "wape": 0.15928181348754364 } ], "largest_abs_bias_segments": [ { "pickup_zone": 26, "rows": 2, "rmse": 0.507721998094011, "mae": 0.4774490894903194, "bias": 0.4774490894903194, "wape": 0.1392271086060623 }, { "pickup_zone": 154, "rows": 1, "rmse": 0.4731330028855747, "mae": 0.4731330028855747, "bias": 0.4731330028855747, "wape": 0.15928181348754364 }, { "pickup_zone": 122, "rows": 1, "rmse": 0.40119304835387304, "mae": 0.40119304835387304, "bias": 0.40119304835387304, "wape": 0.12401092424257909 }, { "pickup_zone": 160, "rows": 1, "rmse": 0.3708541537651562, "mae": 0.3708541537651562, "bias": 0.3708541537651562, "wape": 0.12069351405561826 }, { "pickup_zone": 197, "rows": 3, "rmse": 0.5859465143585096, "mae": 0.3970298634421443, "bias": -0.35398691516300723, "wape": 0.10667957149433493 } ] } } } } }, "out_of_time": { "train_rows": 40000, "test_rows": 10000, "split_manifest": { "split_id": "fare_out_of_time", "split_kind": "chronological_timestamp_holdout", "dataset_fingerprint": "5110541e767f70aabdc8ee1eb158beaab347fb8d3fede1cd1a47e2637d3825f2", "coordinate_crs_note": "No coordinate CRS applies to this chronological timestamp split.", "model_version": "0.3.0", "dependency_versions": { "cartoboost": "0.3.0", "numpy": "1.26.4", "scikit-learn": "1.9.0", "xgboost": "3.2.0", "lightgbm": "4.6.0", "catboost": "1.2.10" }, "random_seed": 42, "row_count": 50000, "train_index_sha256": "sha256:aa149280a2f2fc3b1d53da10fdd409517e69cd13c3cd215b1d548b09b4e74088", "test_index_sha256": "sha256:93189ab389199f948dbfce5cbac71d88ece6d66d64ab3da3e4a48e0ac0721098", "random_row_split_allowed_for_geo_claim": false, "timestamp_column": "tpep_pickup_datetime", "timestamp_tie_policy": "grouped_at_boundary_no_timestamp_overlap", "train_timestamp_min": "2024-01-01T00:07:08.000000000", "train_timestamp_max": "2024-04-08T13:45:40.000000000", "test_timestamp_min": "2024-04-08T13:48:01.000000000", "test_timestamp_max": "2024-04-30T23:57:36.000000000", "temporal_boundary": "2024-04-08T13:48:01.000000000", "leakage_safe": true, "split_manifest_hash": "sha256:35f1b4539497448393eb1e98daa23782145e79eac800f622f740048c8db4214e" }, "holdout_pickup_zones": [ 3, 4, 7, 10, 12, 13, 17, 24, 25, 28, 33, 35, 36, 37, 39, 40, 41, 42, 43, 45, 47, 48, 49, 50, 52, 53, 57, 61, 62, 65, 66, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 79, 80, 82, 85, 87, 88, 89, 90, 91, 92, 93, 95, 97, 100, 107, 112, 113, 114, 116, 119, 121, 125, 126, 127, 129, 130, 132, 134, 137, 138, 140, 141, 142, 143, 144, 145, 146, 148, 151, 152, 153, 155, 157, 158, 159, 161, 162, 163, 164, 166, 168, 170, 174, 179, 181, 182, 186, 188, 189, 191, 195, 197, 198, 203, 207, 209, 210, 211, 212, 213, 215, 216, 218, 219, 222, 223, 224, 225, 226, 227, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 241, 242, 243, 244, 246, 249, 254, 255, 256, 259, 260, 261, 262, 263 ], "models": { "cartoboost": { "status": "ok", "metrics": { "rmse": 0.17968685993558708, "mae": 0.13552224666716578, "r2": 0.8811444777544205, "wape": 0.04239213803172058 }, "timing": { "train_seconds": 78.61442762499792, "predict_seconds": 0.035961916990345344, "fit_predict_seconds": 78.65038954198826, "prediction_rows": 10000.0, "predict_rows_per_second": 278071.93934307474 }, "backend": "rust", "config": { "n_estimators": 24, "learning_rate": 0.08, "max_depth": 5, "min_samples_leaf": 20, "constant_l2_regularization": 0.0, "leaf_predictor": "constant", "init": "constant", "calibration": "none", "splitters": [ "axis_histogram:512", "diagonal_2d", "gaussian_2d", "periodic:24", "periodic:7", "sparse_set" ], "sparse_sets": false, "zone_treatment": "target_mean", "feature_count": 22, "preset": "candidate", "predict_path": "model.predict" }, "calibration_report_fields": { "status": "not_computed", "coverage_by_horizon": {}, "coverage_by_spatial_block": {}, "width_by_horizon": {}, "residual_morans_i_after_calibration": null }, "segment_diagnostics": { "pickup_zone": { "segment_key": "pickup_zone", "segment_count": 146, "row_count_min": 1, "row_count_max": 464, "rmse_quantiles": { "p50": 0.1723043217838913, "p90": 0.3925164408804538, "max": 1.013570733319069 }, "mae_quantiles": { "p50": 0.13658753402900853, "p90": 0.3398579014058538, "max": 1.013570733319069 }, "wape_quantiles": { "p50": 0.04329914657832057, "p90": 0.10160469022020983, "max": 0.2670947985663295 }, "worst_rmse_segments": [ { "pickup_zone": 197, "rows": 1, "rmse": 1.013570733319069, "mae": 1.013570733319069, "bias": -1.013570733319069, "wape": 0.2253588845755213 }, { "pickup_zone": 207, "rows": 1, "rmse": 0.8418585335814508, "mae": 0.8418585335814508, "bias": -0.8418585335814508, "wape": 0.1861200958626946 }, { "pickup_zone": 28, "rows": 3, "rmse": 0.6949425140199585, "mae": 0.5569717284903183, "bias": -0.16867644476793378, "wape": 0.1548184617994913 }, { "pickup_zone": 198, "rows": 1, "rmse": 0.614473777095025, "mae": 0.614473777095025, "bias": 0.614473777095025, "wape": 0.2670947985663295 }, { "pickup_zone": 226, "rows": 8, "rmse": 0.5974924542669001, "mae": 0.4792250596524133, "bias": -0.10538859616830559, "wape": 0.14252551019103726 } ], "largest_abs_bias_segments": [ { "pickup_zone": 197, "rows": 1, "rmse": 1.013570733319069, "mae": 1.013570733319069, "bias": -1.013570733319069, "wape": 0.2253588845755213 }, { "pickup_zone": 207, "rows": 1, "rmse": 0.8418585335814508, "mae": 0.8418585335814508, "bias": -0.8418585335814508, "wape": 0.1861200958626946 }, { "pickup_zone": 198, "rows": 1, "rmse": 0.614473777095025, "mae": 0.614473777095025, "bias": 0.614473777095025, "wape": 0.2670947985663295 }, { "pickup_zone": 215, "rows": 2, "rmse": 0.4561356140454439, "mae": 0.45068021432485716, "bias": -0.45068021432485716, "wape": 0.10023713544513664 }, { "pickup_zone": 73, "rows": 1, "rmse": 0.45020834651594877, "mae": 0.45020834651594877, "bias": 0.45020834651594877, "wape": 0.1577689685921702 } ] } } }, "lightgbm": { "status": "ok", "metrics": { "rmse": 0.1882977225302735, "mae": 0.14309536983606794, "r2": 0.8694800615272908, "wape": 0.04476105450560239 }, "timing": { "train_seconds": 0.12374783300037961, "predict_seconds": 0.002047249989118427, "fit_predict_seconds": 0.12579508298949804, "prediction_rows": 10000.0, "predict_rows_per_second": 4884601.3203820465 }, "config": { "n_estimators": 24, "learning_rate": 0.08, "max_depth": 4, "num_leaves": 16, "zone_treatment": "target_mean", "feature_count": 22 }, "calibration_report_fields": { "status": "not_computed", "coverage_by_horizon": {}, "coverage_by_spatial_block": {}, "width_by_horizon": {}, "residual_morans_i_after_calibration": null }, "segment_diagnostics": { "pickup_zone": { "segment_key": "pickup_zone", "segment_count": 146, "row_count_min": 1, "row_count_max": 464, "rmse_quantiles": { "p50": 0.1836327293910901, "p90": 0.41401012286739636, "max": 1.0109833414485263 }, "mae_quantiles": { "p50": 0.14617993345690508, "p90": 0.3534125910703207, "max": 1.0109833414485263 }, "wape_quantiles": { "p50": 0.04601558796172338, "p90": 0.11377411550736746, "max": 0.2807687520790615 }, "worst_rmse_segments": [ { "pickup_zone": 197, "rows": 1, "rmse": 1.0109833414485263, "mae": 1.0109833414485263, "bias": -1.0109833414485263, "wape": 0.22478359986500496 }, { "pickup_zone": 207, "rows": 1, "rmse": 0.8724737021684441, "mae": 0.8724737021684441, "bias": -0.8724737021684441, "wape": 0.1928885704756712 }, { "pickup_zone": 28, "rows": 3, "rmse": 0.6537847592251764, "mae": 0.560309975333927, "bias": -0.08910792272488226, "wape": 0.1557463764763015 }, { "pickup_zone": 198, "rows": 1, "rmse": 0.6459318433242842, "mae": 0.6459318433242842, "bias": 0.6459318433242842, "wape": 0.2807687520790615 }, { "pickup_zone": 226, "rows": 8, "rmse": 0.607842157444739, "mae": 0.4961765080487186, "bias": -0.10678139008148024, "wape": 0.14756701163696084 } ], "largest_abs_bias_segments": [ { "pickup_zone": 197, "rows": 1, "rmse": 1.0109833414485263, "mae": 1.0109833414485263, "bias": -1.0109833414485263, "wape": 0.22478359986500496 }, { "pickup_zone": 207, "rows": 1, "rmse": 0.8724737021684441, "mae": 0.8724737021684441, "bias": -0.8724737021684441, "wape": 0.1928885704756712 }, { "pickup_zone": 198, "rows": 1, "rmse": 0.6459318433242842, "mae": 0.6459318433242842, "bias": 0.6459318433242842, "wape": 0.2807687520790615 }, { "pickup_zone": 73, "rows": 1, "rmse": 0.48052229296243576, "mae": 0.48052229296243576, "bias": 0.48052229296243576, "wape": 0.16839205033161797 }, { "pickup_zone": 215, "rows": 2, "rmse": 0.4270990643522245, "mae": 0.4015715166250349, "bias": -0.4015715166250349, "wape": 0.08931472299744234 } ] } } }, "xgboost": { "status": "ok", "metrics": { "rmse": 0.1885978959452272, "mae": 0.1433243225771378, "r2": 0.8690635950584086, "wape": 0.04483267223952331 }, "timing": { "train_seconds": 0.07103254199319053, "predict_seconds": 0.0008176660048775375, "fit_predict_seconds": 0.07185020799806807, "prediction_rows": 10000.0, "predict_rows_per_second": 12229932.442278443 }, "config": { "n_estimators": 24, "learning_rate": 0.08, "max_depth": 4, "tree_method": "hist", "max_bin": 256, "subsample": 1.0, "colsample_bytree": 1.0, "zone_treatment": "target_mean", "feature_count": 22 }, "calibration_report_fields": { "status": "not_computed", "coverage_by_horizon": {}, "coverage_by_spatial_block": {}, "width_by_horizon": {}, "residual_morans_i_after_calibration": null }, "segment_diagnostics": { "pickup_zone": { "segment_key": "pickup_zone", "segment_count": 146, "row_count_min": 1, "row_count_max": 464, "rmse_quantiles": { "p50": 0.1832164539861641, "p90": 0.41704164659292864, "max": 1.0004736548125486 }, "mae_quantiles": { "p50": 0.14620159074289754, "p90": 0.3490325984994326, "max": 1.0004736548125486 }, "wape_quantiles": { "p50": 0.04603510849236257, "p90": 0.11282331692885922, "max": 0.2770745766938518 }, "worst_rmse_segments": [ { "pickup_zone": 197, "rows": 1, "rmse": 1.0004736548125486, "mae": 1.0004736548125486, "bias": -1.0004736548125486, "wape": 0.22244685988261972 }, { "pickup_zone": 207, "rows": 1, "rmse": 0.8836666265388091, "mae": 0.8836666265388091, "bias": -0.8836666265388091, "wape": 0.19536312893614521 }, { "pickup_zone": 28, "rows": 3, "rmse": 0.6628274218018162, "mae": 0.5708552066519208, "bias": -0.09108182546663827, "wape": 0.15867757820245876 }, { "pickup_zone": 198, "rows": 1, "rmse": 0.6374330859003821, "mae": 0.6374330859003821, "bias": 0.6374330859003821, "wape": 0.2770745766938518 }, { "pickup_zone": 226, "rows": 8, "rmse": 0.6070662239677247, "mae": 0.4970905668645641, "bias": -0.11392977176230956, "wape": 0.1478388603152572 } ], "largest_abs_bias_segments": [ { "pickup_zone": 197, "rows": 1, "rmse": 1.0004736548125486, "mae": 1.0004736548125486, "bias": -1.0004736548125486, "wape": 0.22244685988261972 }, { "pickup_zone": 207, "rows": 1, "rmse": 0.8836666265388091, "mae": 0.8836666265388091, "bias": -0.8836666265388091, "wape": 0.19536312893614521 }, { "pickup_zone": 198, "rows": 1, "rmse": 0.6374330859003821, "mae": 0.6374330859003821, "bias": 0.6374330859003821, "wape": 0.2770745766938518 }, { "pickup_zone": 73, "rows": 1, "rmse": 0.4714547481695095, "mae": 0.4714547481695095, "bias": 0.4714547481695095, "wape": 0.1652144611093964 }, { "pickup_zone": 215, "rows": 2, "rmse": 0.4272731756901983, "mae": 0.41988344915440345, "bias": -0.41988344915440345, "wape": 0.09338753472262147 } ] } } }, "catboost": { "status": "ok", "metrics": { "rmse": 0.19949495276943813, "mae": 0.15289294278951293, "r2": 0.8534956395524064, "wape": 0.047825791663025266 }, "timing": { "train_seconds": 0.057544542010873556, "predict_seconds": 0.0009105829958571121, "fit_predict_seconds": 0.05845512500673067, "prediction_rows": 10000.0, "predict_rows_per_second": 10981975.33393122 }, "config": { "iterations": 24, "learning_rate": 0.08, "depth": 4, "zone_treatment": "target_mean", "feature_count": 22 }, "calibration_report_fields": { "status": "not_computed", "coverage_by_horizon": {}, "coverage_by_spatial_block": {}, "width_by_horizon": {}, "residual_morans_i_after_calibration": null }, "segment_diagnostics": { "pickup_zone": { "segment_key": "pickup_zone", "segment_count": 146, "row_count_min": 1, "row_count_max": 464, "rmse_quantiles": { "p50": 0.19722401812400348, "p90": 0.422098775153724, "max": 0.97872275155681 }, "mae_quantiles": { "p50": 0.15599717545672395, "p90": 0.36441419820776166, "max": 0.97872275155681 }, "wape_quantiles": { "p50": 0.04970014487953092, "p90": 0.11560475958215485, "max": 0.2874944917469535 }, "worst_rmse_segments": [ { "pickup_zone": 197, "rows": 1, "rmse": 0.97872275155681, "mae": 0.97872275155681, "bias": -0.97872275155681, "wape": 0.21761073040976894 }, { "pickup_zone": 207, "rows": 1, "rmse": 0.8530780573379366, "mae": 0.8530780573379366, "bias": -0.8530780573379366, "wape": 0.188600534979 }, { "pickup_zone": 28, "rows": 3, "rmse": 0.6635384539630597, "mae": 0.58823640862819, "bias": -0.04932888497427953, "wape": 0.1635089382456087 }, { "pickup_zone": 198, "rows": 1, "rmse": 0.6614049662741537, "mae": 0.6614049662741537, "bias": 0.6614049662741537, "wape": 0.2874944917469535 }, { "pickup_zone": 226, "rows": 8, "rmse": 0.5962101719869115, "mae": 0.49144101784841726, "bias": -0.07932828578432699, "wape": 0.1461586375479845 } ], "largest_abs_bias_segments": [ { "pickup_zone": 197, "rows": 1, "rmse": 0.97872275155681, "mae": 0.97872275155681, "bias": -0.97872275155681, "wape": 0.21761073040976894 }, { "pickup_zone": 207, "rows": 1, "rmse": 0.8530780573379366, "mae": 0.8530780573379366, "bias": -0.8530780573379366, "wape": 0.188600534979 }, { "pickup_zone": 198, "rows": 1, "rmse": 0.6614049662741537, "mae": 0.6614049662741537, "bias": 0.6614049662741537, "wape": 0.2874944917469535 }, { "pickup_zone": 73, "rows": 1, "rmse": 0.5122074777003518, "mae": 0.5122074777003518, "bias": 0.5122074777003518, "wape": 0.1794956625912283 }, { "pickup_zone": 127, "rows": 1, "rmse": 0.43081632376913914, "mae": 0.43081632376913914, "bias": 0.43081632376913914, "wape": 0.12005449235613003 } ] } } }, "hist_gradient_boosting": { "status": "ok", "metrics": { "rmse": 0.18605638580603123, "mae": 0.1407664098069287, "r2": 0.8725687665665424, "wape": 0.044032542416601206 }, "timing": { "train_seconds": 0.3933904579898808, "predict_seconds": 0.006202708013006486, "fit_predict_seconds": 0.3995931660028873, "prediction_rows": 10000.0, "predict_rows_per_second": 1612199.05548205 }, "config": { "n_estimators": 24, "learning_rate": 0.08, "max_leaf_nodes": 16, "zone_treatment": "target_mean", "feature_count": 22 }, "calibration_report_fields": { "status": "not_computed", "coverage_by_horizon": {}, "coverage_by_spatial_block": {}, "width_by_horizon": {}, "residual_morans_i_after_calibration": null }, "segment_diagnostics": { "pickup_zone": { "segment_key": "pickup_zone", "segment_count": 146, "row_count_min": 1, "row_count_max": 464, "rmse_quantiles": { "p50": 0.18058560598997908, "p90": 0.40690784067692737, "max": 1.0396795750200223 }, "mae_quantiles": { "p50": 0.14407202576816017, "p90": 0.3467827457027871, "max": 1.0396795750200223 }, "wape_quantiles": { "p50": 0.04503660592763123, "p90": 0.11303319866883964, "max": 0.28701664266025445 }, "worst_rmse_segments": [ { "pickup_zone": 197, "rows": 1, "rmse": 1.0396795750200223, "mae": 1.0396795750200223, "bias": -1.0396795750200223, "wape": 0.23116396482287435 }, { "pickup_zone": 207, "rows": 1, "rmse": 0.8815350818388779, "mae": 0.8815350818388779, "bias": -0.8815350818388779, "wape": 0.19489188194147608 }, { "pickup_zone": 28, "rows": 3, "rmse": 0.6614730542543718, "mae": 0.5655065167467822, "bias": -0.09847211732200749, "wape": 0.15719083138678008 }, { "pickup_zone": 198, "rows": 1, "rmse": 0.6603056347455674, "mae": 0.6603056347455674, "bias": 0.6603056347455674, "wape": 0.28701664266025445 }, { "pickup_zone": 226, "rows": 8, "rmse": 0.6026850321123924, "mae": 0.49523029925691275, "bias": -0.09876968694115995, "wape": 0.147285601288977 } ], "largest_abs_bias_segments": [ { "pickup_zone": 197, "rows": 1, "rmse": 1.0396795750200223, "mae": 1.0396795750200223, "bias": -1.0396795750200223, "wape": 0.23116396482287435 }, { "pickup_zone": 207, "rows": 1, "rmse": 0.8815350818388779, "mae": 0.8815350818388779, "bias": -0.8815350818388779, "wape": 0.19489188194147608 }, { "pickup_zone": 198, "rows": 1, "rmse": 0.6603056347455674, "mae": 0.6603056347455674, "bias": 0.6603056347455674, "wape": 0.28701664266025445 }, { "pickup_zone": 73, "rows": 1, "rmse": 0.47682088600878236, "mae": 0.47682088600878236, "bias": 0.47682088600878236, "wape": 0.16709494608657907 }, { "pickup_zone": 215, "rows": 2, "rmse": 0.416204347568975, "mae": 0.40632315749788495, "bias": -0.40632315749788495, "wape": 0.09037154966659627 } ] } } } } } } } }, "external_baseline_comparison": [ { "task": "duration", "split": "random", "cartoboost_model": "cartoboost", "cartoboost_rmse": 0.3257288492367098, "cartoboost_wape": 0.037797039666660454, "cartoboost_r2": 0.7824932571843731, "best_external_baseline": "hist_gradient_boosting", "best_external_rmse": 0.3418852385950508, "best_external_wape": 0.039919164587188645, "best_external_r2": 0.7603811526807877, "rmse_delta_vs_external": -0.016156389358341017, "r2_delta_vs_external": 0.022112104503585472, "status": "cartoboost_lower_rmse" }, { "task": "duration", "split": "spatial_holdout", "cartoboost_model": "cartoboost", "cartoboost_rmse": 0.32839469015295164, "cartoboost_wape": 0.03686223453532374, "cartoboost_r2": 0.8351327436446745, "best_external_baseline": "hist_gradient_boosting", "best_external_rmse": 0.352825943273662, "best_external_wape": 0.040022734305493156, "best_external_r2": 0.8096893114571468, "rmse_delta_vs_external": -0.024431253120710372, "r2_delta_vs_external": 0.025443432187527648, "status": "cartoboost_lower_rmse" }, { "task": "duration", "split": "out_of_time", "cartoboost_model": "cartoboost", "cartoboost_rmse": 0.3308252158984261, "cartoboost_wape": 0.0385541672576472, "cartoboost_r2": 0.7803005380508858, "best_external_baseline": "hist_gradient_boosting", "best_external_rmse": 0.34577174947338457, "best_external_wape": 0.040405635566092196, "best_external_r2": 0.7600002447937049, "rmse_delta_vs_external": -0.014946533574958454, "r2_delta_vs_external": 0.020300293257180968, "status": "cartoboost_lower_rmse" }, { "task": "fare", "split": "random", "cartoboost_model": "cartoboost", "cartoboost_rmse": 0.17294850472898857, "cartoboost_wape": 0.04136462720028184, "cartoboost_r2": 0.8870946806164656, "best_external_baseline": "hist_gradient_boosting", "best_external_rmse": 0.17915465884646392, "best_external_wape": 0.04296125066500347, "best_external_r2": 0.8788462131871305, "rmse_delta_vs_external": -0.006206154117475354, "r2_delta_vs_external": 0.008248467429335049, "status": "cartoboost_lower_rmse" }, { "task": "fare", "split": "spatial_holdout", "cartoboost_model": "cartoboost", "cartoboost_rmse": 0.2740822914348948, "cartoboost_wape": 0.05872393068742097, "cartoboost_r2": 0.8347378763514225, "best_external_baseline": "hist_gradient_boosting", "best_external_rmse": 0.2411672224353955, "best_external_wape": 0.05375711144831535, "best_external_r2": 0.8720477464308415, "rmse_delta_vs_external": 0.03291506899949931, "r2_delta_vs_external": -0.037309870079418994, "status": "external_lower_or_tied_rmse" }, { "task": "fare", "split": "out_of_time", "cartoboost_model": "cartoboost", "cartoboost_rmse": 0.17968685993558708, "cartoboost_wape": 0.04239213803172058, "cartoboost_r2": 0.8811444777544205, "best_external_baseline": "hist_gradient_boosting", "best_external_rmse": 0.18605638580603123, "best_external_wape": 0.044032542416601206, "best_external_r2": 0.8725687665665424, "rmse_delta_vs_external": -0.006369525870444148, "r2_delta_vs_external": 0.008575711187878143, "status": "cartoboost_lower_rmse" } ], "model_status_summary": { "models": { "cartoboost": { "ok": 6, "skipped": 0, "skip_reasons": [] }, "lightgbm": { "ok": 6, "skipped": 0, "skip_reasons": [] }, "xgboost": { "ok": 6, "skipped": 0, "skip_reasons": [] }, "catboost": { "ok": 6, "skipped": 0, "skip_reasons": [] }, "hist_gradient_boosting": { "ok": 6, "skipped": 0, "skip_reasons": [] } }, "completed_external_baselines": [ "catboost", "hist_gradient_boosting", "lightgbm", "xgboost" ], "skipped_requested_external_baselines": [], "completed_cartoboost_family_models": [ "cartoboost" ], "skipped_cartoboost_family_models": [] }, "comparability_audit": { "same_outer_splits": true, "same_metrics": [ "mae", "rmse", "r2", "wape" ], "primary_metric": "rmse", "selection_mode": "fixed_settings_no_hpo", "selection_uses_outer_test_labels": false, "same_feature_access_policy": true, "train_only_target_encoding": true, "segment_diagnostics_used_for_selection": false, "completed_external_baselines": [ "catboost", "hist_gradient_boosting", "lightgbm", "xgboost" ], "skipped_requested_external_baselines": [], "completed_cartoboost_family_models": [ "cartoboost" ], "skipped_cartoboost_family_models": [], "cartoboost_external_comparison_count": 6, "status_summary": { "cartoboost": { "ok": 6, "skipped": 0, "skip_reasons": [] }, "lightgbm": { "ok": 6, "skipped": 0, "skip_reasons": [] }, "xgboost": { "ok": 6, "skipped": 0, "skip_reasons": [] }, "catboost": { "ok": 6, "skipped": 0, "skip_reasons": [] }, "hist_gradient_boosting": { "ok": 6, "skipped": 0, "skip_reasons": [] } } }, "output_artifacts": { "results.json": { "size_bytes": 198602 }, "results.jsonl": { "size_bytes": 32079 }, "results.md": { "size_bytes": 11462 } } }