{ "generated_at": "2026-05-12T03:05:01+00:00", "status": "materialized_for_real_pipeline", "source_staging_manifest": "data/staging/modern_ids_pipeline_candidates/manifests/additional10_candidates_manifest.json", "retained_23d_definition": "research/docs/dctabgan_iteration_map/planning_docs/PRESERVED_RATIO_TRAIN500_TEST500_SINGLE_LABEL_23D_BENCHMARK_DEFINITION_2026-04-18.json", "base_30d_definition": "research/docs/dctabgan_iteration_map/planning_docs/PRESERVED_RATIO_TRAIN500_TEST500_SINGLE_LABEL_30D_BENCHMARK_DEFINITION_2026-05-11.json", "additional10_definition": "research/docs/dctabgan_iteration_map/planning_docs/PRESERVED_RATIO_TRAIN500_TEST500_SINGLE_LABEL_ADDITIONAL10_BENCHMARK_DEFINITION_2026-05-11.json", "final40d_definition": "research/docs/dctabgan_iteration_map/planning_docs/PRESERVED_RATIO_TRAIN500_TEST500_SINGLE_LABEL_40D_BENCHMARK_DEFINITION_2026-05-11.json", "candidate_count": 10, "candidates": [ { "candidate_id": "5g_nidd_tcp_connect_scan_vs_benign", "dataset": "5G-NIDD", "year": 2022, "positive_class": "TCPConnectScan", "negative_class": "Benign", "ratio_floor": 23, "pipeline_status": "materialized_for_benchmark_definition", "provisional_source_caveat": "Official IEEE DataPort route remains gated; local artifact was sourced from the open Fairdata route.", "input_train_csv": "data/staging/modern_ids_pipeline_candidates/5g_nidd_tcp_connect_scan_vs_benign/train.csv", "input_test_csv": "data/staging/modern_ids_pipeline_candidates/5g_nidd_tcp_connect_scan_vs_benign/test.csv", "output_dataset": "data/preserved_ratio_experiments/processed/train500_test500_modern_ids_additional10/5g_nidd_tcp_connect_scan_vs_benign/5g_nidd_tcp_connect_scan_vs_benign_preservedratio_train500minority_11500majority_test500minority_11500majority_additional10.csv", "split_metadata": "data/preserved_ratio_experiments/processed/train500_test500_modern_ids_additional10/5g_nidd_tcp_connect_scan_vs_benign/split_metadata.json", "train_counts": { "0": 11500, "1": 500 }, "test_counts": { "0": 11500, "1": 500 }, "train_test_source_row_index_overlap": 0, "input_column_count": 49, "output_column_count": 46, "feature_count": 44, "dropped_columns": [ "Offset", "SrcTCPBase", "DstTCPBase" ], "configured_drop_columns_missing": [], "risk_note": "5G Offset and TCP sequence-base artifacts filtered at benchmark materialization; remaining low-card categorical handling should be monitored." }, { "candidate_id": "5g_nidd_syn_scan_vs_benign", "dataset": "5G-NIDD", "year": 2022, "positive_class": "SYNScan", "negative_class": "Benign", "ratio_floor": 23, "pipeline_status": "materialized_for_benchmark_definition", "provisional_source_caveat": "Official IEEE DataPort route remains gated; local artifact was sourced from the open Fairdata route.", "input_train_csv": "data/staging/modern_ids_pipeline_candidates/5g_nidd_syn_scan_vs_benign/train.csv", "input_test_csv": "data/staging/modern_ids_pipeline_candidates/5g_nidd_syn_scan_vs_benign/test.csv", "output_dataset": "data/preserved_ratio_experiments/processed/train500_test500_modern_ids_additional10/5g_nidd_syn_scan_vs_benign/5g_nidd_syn_scan_vs_benign_preservedratio_train500minority_11500majority_test500minority_11500majority_additional10.csv", "split_metadata": "data/preserved_ratio_experiments/processed/train500_test500_modern_ids_additional10/5g_nidd_syn_scan_vs_benign/split_metadata.json", "train_counts": { "0": 11500, "1": 500 }, "test_counts": { "0": 11500, "1": 500 }, "train_test_source_row_index_overlap": 0, "input_column_count": 49, "output_column_count": 46, "feature_count": 44, "dropped_columns": [ "Offset", "SrcTCPBase", "DstTCPBase" ], "configured_drop_columns_missing": [], "risk_note": "5G Offset and TCP sequence-base artifacts filtered at benchmark materialization; remaining low-card categorical handling should be monitored." }, { "candidate_id": "5g_nidd_udp_scan_vs_benign", "dataset": "5G-NIDD", "year": 2022, "positive_class": "UDPScan", "negative_class": "Benign", "ratio_floor": 30, "pipeline_status": "materialized_for_benchmark_definition", "provisional_source_caveat": "Official IEEE DataPort route remains gated; local artifact was sourced from the open Fairdata route.", "input_train_csv": "data/staging/modern_ids_pipeline_candidates/5g_nidd_udp_scan_vs_benign/train.csv", "input_test_csv": "data/staging/modern_ids_pipeline_candidates/5g_nidd_udp_scan_vs_benign/test.csv", "output_dataset": "data/preserved_ratio_experiments/processed/train500_test500_modern_ids_additional10/5g_nidd_udp_scan_vs_benign/5g_nidd_udp_scan_vs_benign_preservedratio_train500minority_15000majority_test500minority_15000majority_additional10.csv", "split_metadata": "data/preserved_ratio_experiments/processed/train500_test500_modern_ids_additional10/5g_nidd_udp_scan_vs_benign/split_metadata.json", "train_counts": { "0": 15000, "1": 500 }, "test_counts": { "0": 15000, "1": 500 }, "train_test_source_row_index_overlap": 0, "input_column_count": 49, "output_column_count": 46, "feature_count": 44, "dropped_columns": [ "Offset", "SrcTCPBase", "DstTCPBase" ], "configured_drop_columns_missing": [], "risk_note": "5G Offset and TCP sequence-base artifacts filtered at benchmark materialization; remaining low-card categorical handling should be monitored." }, { "candidate_id": "edge_iiotset_vulnerability_scanner_vs_normal", "dataset": "Edge-IIoTset", "year": 2022, "positive_class": "Vulnerability_scanner_attack", "negative_class": "Normal", "ratio_floor": 32, "pipeline_status": "materialized_for_benchmark_definition", "provisional_source_caveat": "Materialized from public Kaggle mirror; official IEEE DataPort remains login/subscription-gated.", "input_train_csv": "data/staging/modern_ids_pipeline_candidates/edge_iiotset_vulnerability_scanner_vs_normal/train.csv", "input_test_csv": "data/staging/modern_ids_pipeline_candidates/edge_iiotset_vulnerability_scanner_vs_normal/test.csv", "output_dataset": "data/preserved_ratio_experiments/processed/train500_test500_modern_ids_additional10/edge_iiotset_vulnerability_scanner_vs_normal/edge_iiotset_vulnerability_scanner_vs_normal_preservedratio_train500minority_16000majority_test500minority_16000majority_additional10.csv", "split_metadata": "data/preserved_ratio_experiments/processed/train500_test500_modern_ids_additional10/edge_iiotset_vulnerability_scanner_vs_normal/split_metadata.json", "train_counts": { "0": 16000, "1": 500 }, "test_counts": { "0": 16000, "1": 500 }, "train_test_source_row_index_overlap": 0, "input_column_count": 63, "output_column_count": 43, "feature_count": 41, "dropped_columns": [ "frame.time", "ip.src_host", "ip.dst_host", "arp.dst.proto_ipv4", "arp.src.proto_ipv4", "http.file_data", "http.request.uri.query", "http.referer", "http.request.full_uri", "tcp.ack", "tcp.ack_raw", "tcp.checksum", "tcp.options", "tcp.payload", "tcp.seq", "tcp.srcport", "udp.stream", "mqtt.msg", "mqtt.topic", "mbtcp.trans_id" ], "configured_drop_columns_missing": [], "risk_note": "High-cardinality Edge timestamp/IP/payload/text/sequence artifacts filtered at benchmark materialization; remaining risk is moderate runtime and retained destination-port cardinality." }, { "candidate_id": "edge_iiotset_backdoor_vs_normal", "dataset": "Edge-IIoTset", "year": 2022, "positive_class": "Backdoor_attack", "negative_class": "Normal", "ratio_floor": 64, "pipeline_status": "materialized_for_benchmark_definition", "provisional_source_caveat": "Materialized from public Kaggle mirror; official IEEE DataPort remains login/subscription-gated.", "input_train_csv": "data/staging/modern_ids_pipeline_candidates/edge_iiotset_backdoor_vs_normal/train.csv", "input_test_csv": "data/staging/modern_ids_pipeline_candidates/edge_iiotset_backdoor_vs_normal/test.csv", "output_dataset": "data/preserved_ratio_experiments/processed/train500_test500_modern_ids_additional10/edge_iiotset_backdoor_vs_normal/edge_iiotset_backdoor_vs_normal_preservedratio_train500minority_32000majority_test500minority_32000majority_additional10.csv", "split_metadata": "data/preserved_ratio_experiments/processed/train500_test500_modern_ids_additional10/edge_iiotset_backdoor_vs_normal/split_metadata.json", "train_counts": { "0": 32000, "1": 500 }, "test_counts": { "0": 32000, "1": 500 }, "train_test_source_row_index_overlap": 0, "input_column_count": 63, "output_column_count": 43, "feature_count": 41, "dropped_columns": [ "frame.time", "ip.src_host", "ip.dst_host", "arp.dst.proto_ipv4", "arp.src.proto_ipv4", "http.file_data", "http.request.uri.query", "http.referer", "http.request.full_uri", "tcp.ack", "tcp.ack_raw", "tcp.checksum", "tcp.options", "tcp.payload", "tcp.seq", "tcp.srcport", "udp.stream", "mqtt.msg", "mqtt.topic", "mbtcp.trans_id" ], "configured_drop_columns_missing": [], "risk_note": "High-cardinality Edge timestamp/IP/payload/text/sequence artifacts filtered at benchmark materialization; remaining risk is moderate runtime and retained destination-port cardinality." }, { "candidate_id": "edge_iiotset_port_scanning_vs_normal", "dataset": "Edge-IIoTset", "year": 2022, "positive_class": "Port_Scanning_attack", "negative_class": "Normal", "ratio_floor": 71, "pipeline_status": "materialized_for_benchmark_definition", "provisional_source_caveat": "Materialized from public Kaggle mirror; official IEEE DataPort remains login/subscription-gated.", "input_train_csv": "data/staging/modern_ids_pipeline_candidates/edge_iiotset_port_scanning_vs_normal/train.csv", "input_test_csv": "data/staging/modern_ids_pipeline_candidates/edge_iiotset_port_scanning_vs_normal/test.csv", "output_dataset": "data/preserved_ratio_experiments/processed/train500_test500_modern_ids_additional10/edge_iiotset_port_scanning_vs_normal/edge_iiotset_port_scanning_vs_normal_preservedratio_train500minority_35500majority_test500minority_35500majority_additional10.csv", "split_metadata": "data/preserved_ratio_experiments/processed/train500_test500_modern_ids_additional10/edge_iiotset_port_scanning_vs_normal/split_metadata.json", "train_counts": { "0": 35500, "1": 500 }, "test_counts": { "0": 35500, "1": 500 }, "train_test_source_row_index_overlap": 0, "input_column_count": 63, "output_column_count": 43, "feature_count": 41, "dropped_columns": [ "frame.time", "ip.src_host", "ip.dst_host", "arp.dst.proto_ipv4", "arp.src.proto_ipv4", "http.file_data", "http.request.uri.query", "http.referer", "http.request.full_uri", "tcp.ack", "tcp.ack_raw", "tcp.checksum", "tcp.options", "tcp.payload", "tcp.seq", "tcp.srcport", "udp.stream", "mqtt.msg", "mqtt.topic", "mbtcp.trans_id" ], "configured_drop_columns_missing": [], "risk_note": "High-cardinality Edge timestamp/IP/payload/text/sequence artifacts filtered at benchmark materialization; remaining risk is moderate runtime and retained destination-port cardinality." }, { "candidate_id": "rt_iot2022_nmap_udp_scan_vs_benign", "dataset": "RT-IoT2022", "year": 2023, "positive_class": "NMAP_UDP_SCAN", "negative_class": "benign_control", "ratio_floor": 4, "pipeline_status": "materialized_for_benchmark_definition", "provisional_source_caveat": "", "input_train_csv": "data/staging/modern_ids_pipeline_candidates/rt_iot2022_nmap_udp_scan_vs_benign/train.csv", "input_test_csv": "data/staging/modern_ids_pipeline_candidates/rt_iot2022_nmap_udp_scan_vs_benign/test.csv", "output_dataset": "data/preserved_ratio_experiments/processed/train500_test500_modern_ids_additional10/rt_iot2022_nmap_udp_scan_vs_benign/rt_iot2022_nmap_udp_scan_vs_benign_preservedratio_train500minority_2000majority_test500minority_2000majority_additional10.csv", "split_metadata": "data/preserved_ratio_experiments/processed/train500_test500_modern_ids_additional10/rt_iot2022_nmap_udp_scan_vs_benign/split_metadata.json", "train_counts": { "0": 2000, "1": 500 }, "test_counts": { "0": 2000, "1": 500 }, "train_test_source_row_index_overlap": 0, "input_column_count": 85, "output_column_count": 85, "feature_count": 83, "dropped_columns": [], "configured_drop_columns_missing": [], "risk_note": "No extra materialization drops; remaining risk is small train size relative to feature count and retained port-like fields." }, { "candidate_id": "rt_iot2022_nmap_xmas_tree_scan_vs_benign", "dataset": "RT-IoT2022", "year": 2023, "positive_class": "NMAP_XMAS_TREE_SCAN", "negative_class": "benign_control", "ratio_floor": 6, "pipeline_status": "materialized_for_benchmark_definition", "provisional_source_caveat": "", "input_train_csv": "data/staging/modern_ids_pipeline_candidates/rt_iot2022_nmap_xmas_tree_scan_vs_benign/train.csv", "input_test_csv": "data/staging/modern_ids_pipeline_candidates/rt_iot2022_nmap_xmas_tree_scan_vs_benign/test.csv", "output_dataset": "data/preserved_ratio_experiments/processed/train500_test500_modern_ids_additional10/rt_iot2022_nmap_xmas_tree_scan_vs_benign/rt_iot2022_nmap_xmas_tree_scan_vs_benign_preservedratio_train500minority_3000majority_test500minority_3000majority_additional10.csv", "split_metadata": "data/preserved_ratio_experiments/processed/train500_test500_modern_ids_additional10/rt_iot2022_nmap_xmas_tree_scan_vs_benign/split_metadata.json", "train_counts": { "0": 3000, "1": 500 }, "test_counts": { "0": 3000, "1": 500 }, "train_test_source_row_index_overlap": 0, "input_column_count": 85, "output_column_count": 85, "feature_count": 83, "dropped_columns": [], "configured_drop_columns_missing": [], "risk_note": "No extra materialization drops; remaining risk is small train size relative to feature count and retained port-like fields." }, { "candidate_id": "rt_iot2022_nmap_os_detection_vs_benign", "dataset": "RT-IoT2022", "year": 2023, "positive_class": "NMAP_OS_DETECTION", "negative_class": "benign_control", "ratio_floor": 6, "pipeline_status": "materialized_for_benchmark_definition", "provisional_source_caveat": "", "input_train_csv": "data/staging/modern_ids_pipeline_candidates/rt_iot2022_nmap_os_detection_vs_benign/train.csv", "input_test_csv": "data/staging/modern_ids_pipeline_candidates/rt_iot2022_nmap_os_detection_vs_benign/test.csv", "output_dataset": "data/preserved_ratio_experiments/processed/train500_test500_modern_ids_additional10/rt_iot2022_nmap_os_detection_vs_benign/rt_iot2022_nmap_os_detection_vs_benign_preservedratio_train500minority_3000majority_test500minority_3000majority_additional10.csv", "split_metadata": "data/preserved_ratio_experiments/processed/train500_test500_modern_ids_additional10/rt_iot2022_nmap_os_detection_vs_benign/split_metadata.json", "train_counts": { "0": 3000, "1": 500 }, "test_counts": { "0": 3000, "1": 500 }, "train_test_source_row_index_overlap": 0, "input_column_count": 85, "output_column_count": 85, "feature_count": 83, "dropped_columns": [], "configured_drop_columns_missing": [], "risk_note": "No extra materialization drops; remaining risk is small train size relative to feature count and retained port-like fields." }, { "candidate_id": "rt_iot2022_nmap_tcp_scan_vs_benign", "dataset": "RT-IoT2022", "year": 2023, "positive_class": "NMAP_TCP_scan", "negative_class": "benign_control", "ratio_floor": 12, "pipeline_status": "materialized_for_benchmark_definition", "provisional_source_caveat": "", "input_train_csv": "data/staging/modern_ids_pipeline_candidates/rt_iot2022_nmap_tcp_scan_vs_benign/train.csv", "input_test_csv": "data/staging/modern_ids_pipeline_candidates/rt_iot2022_nmap_tcp_scan_vs_benign/test.csv", "output_dataset": "data/preserved_ratio_experiments/processed/train500_test500_modern_ids_additional10/rt_iot2022_nmap_tcp_scan_vs_benign/rt_iot2022_nmap_tcp_scan_vs_benign_preservedratio_train500minority_6000majority_test500minority_6000majority_additional10.csv", "split_metadata": "data/preserved_ratio_experiments/processed/train500_test500_modern_ids_additional10/rt_iot2022_nmap_tcp_scan_vs_benign/split_metadata.json", "train_counts": { "0": 6000, "1": 500 }, "test_counts": { "0": 6000, "1": 500 }, "train_test_source_row_index_overlap": 0, "input_column_count": 85, "output_column_count": 85, "feature_count": 83, "dropped_columns": [], "configured_drop_columns_missing": [], "risk_note": "No extra materialization drops; remaining risk is attack support just above the 1000-row floor plus retained port-like fields." } ] }