Ingest raw data into BDF#
This notebook demonstrates bdf.ingest, which:
converts raw vendor files into BDF artifacts
validates existing BDF files
writes outputs to parquet by default (csv optional)
from pathlib import Path
import bdf
Pick a source directory#
By default this uses the local data/ folder in the repo.
source_dir = Path("./in/ingest/example_raw_data_collection/g20m7")
source_dir.exists()
True
Run ingest#
The ingest function converts all the raw files in a directory and serializes them as .bdf in a subdirectory data/ . It also reads the metadata files to serialize RDF linked data descriptions.
summary = bdf.ingest(
source_dir,
format="parquet",
recursive=True,
validate_existing=True,
validate_converted=True,
raise_on_error=False,
layout="nested"
)
summary
/tmp/ipykernel_2376/2481481659.py:1: UserWarning: Configured raw_dir not found: /home/runner/work/battery-data-format/battery-data-format/docs/examples/in/ingest/example_raw_data_collection/g20m7/timeseries/raw. Falling back to auto-discovery.
summary = bdf.ingest(
{'converted': [{'path': '/home/runner/work/battery-data-format/battery-data-format/docs/examples/in/ingest/example_raw_data_collection/g20m7/SINTEF__google-g20m7-202512-c5hnd9__20260112__HPPC__25degC.nda',
'output': '/home/runner/work/battery-data-format/battery-data-format/docs/examples/in/ingest/example_raw_data_collection/g20m7/timeseries/SINTEF__google-g20m7-202512-c5hnd9__20260112__HPPC__25degC.bdf.parquet'}],
'validated': [],
'failed': [],
'skipped': [{'path': '/home/runner/work/battery-data-format/battery-data-format/docs/examples/in/ingest/example_raw_data_collection/g20m7/person.json',
'reason': 'metadata_file'},
{'path': '/home/runner/work/battery-data-format/battery-data-format/docs/examples/in/ingest/example_raw_data_collection/g20m7/battery.json',
'reason': 'metadata_file'},
{'path': '/home/runner/work/battery-data-format/battery-data-format/docs/examples/in/ingest/example_raw_data_collection/g20m7/contribution.json',
'reason': 'metadata_file'}],
'metadata': [{'path': '/home/runner/work/battery-data-format/battery-data-format/docs/examples/in/ingest/example_raw_data_collection/g20m7',
'metadata': '/home/runner/work/battery-data-format/battery-data-format/docs/examples/in/ingest/example_raw_data_collection/g20m7/metadata.jsonld'},
{'path': '/home/runner/work/battery-data-format/battery-data-format/docs/examples/in/ingest/example_raw_data_collection/g20m7/batteries/c5hnd9',
'metadata': '/home/runner/work/battery-data-format/battery-data-format/docs/examples/in/ingest/example_raw_data_collection/g20m7/batteries/c5hnd9/metadata.jsonld'}],
'metadata_failed': []}
Inspect results#
print("converted:", len(summary.get("converted", [])))
print("validated:", len(summary.get("validated", [])))
print("failed:", len(summary.get("failed", [])))
print("skipped:", len(summary.get("skipped", [])))
converted: 1
validated: 0
failed: 0
skipped: 3
summary.get("failed", [])[:5]
[]
Ingest a repository#
Use bdf.ingest with discover_collections=True to ingest every folder
that contains contribution.json.
repo_root = Path("./in/ingest/example_raw_data_collection")
repo_summary = bdf.ingest(repo_root, layout="nested", format="parquet", discover_collections=True)
repo_summary
/home/runner/work/battery-data-format/battery-data-format/src/bdf/_ingest.py:283: UserWarning: Configured raw_dir not found: /home/runner/work/battery-data-format/battery-data-format/docs/examples/in/ingest/example_raw_data_collection/g20m7/timeseries/raw. Falling back to auto-discovery.
summary = ingest(
/home/runner/work/battery-data-format/battery-data-format/src/bdf/_ingest.py:283: UserWarning: Configured raw_dir not found: /home/runner/work/battery-data-format/battery-data-format/docs/examples/in/ingest/example_raw_data_collection/nacr32140mp10/timeseries/raw. Falling back to auto-discovery.
summary = ingest(
{'roots': [{'path': '/home/runner/work/battery-data-format/battery-data-format/docs/examples/in/ingest/example_raw_data_collection/g20m7',
'summary': {'converted': [{'path': '/home/runner/work/battery-data-format/battery-data-format/docs/examples/in/ingest/example_raw_data_collection/g20m7/timeseries/SINTEF__google-g20m7-202512-c5hnd9__20260112__HPPC__25degC.bdf.parquet',
'output': '/home/runner/work/battery-data-format/battery-data-format/docs/examples/in/ingest/example_raw_data_collection/g20m7/timeseries/SINTEF__google-g20m7-202512-c5hnd9__20260112__HPPC__25degC.bdf.parquet',
'existing_bdf': True}],
'validated': [{'path': '/home/runner/work/battery-data-format/battery-data-format/docs/examples/in/ingest/example_raw_data_collection/g20m7/timeseries/SINTEF__google-g20m7-202512-c5hnd9__20260112__HPPC__25degC.bdf.parquet',
'ok': True,
'report': {'ok': True,
'missing': [],
'extras': [],
'required': ('Voltage / V', 'Test Time / s', 'Current / A'),
'optional': ('Absolute Impedance / ohm',
'AC Internal Resistance / ohm',
'Ambient Pressure / Pa',
'Ambient Temperature / degC',
'Applied Pressure / Pa',
'Cumulative Capacity / Ah',
'Cumulative Energy / Wh',
'Cycle Cumulative Capacity / Ah',
'Cycle Cumulative Energy / Wh',
'Cycle Net Capacity / Ah',
'Cycle Net Energy / Wh',
'DC Internal Resistance / ohm',
'Frequency / Hz',
'Net Capacity / Ah',
'Net Energy / Wh',
'Phase / deg',
'Record Index / 1',
'Schedule Charging Capacity / Ah',
'Schedule Charging Energy / Wh',
'Schedule Discharging Capacity / Ah',
'Schedule Discharging Energy / Wh',
'Step Count / 1',
'Step ID',
'Step Net Capacity / Ah',
'Step Net Energy / Wh',
'Step Type',
'Surface Pressure / Pa',
'Surface Temperature / degC',
'Temperature T1 / degC',
'Temperature T2 / degC',
'Temperature T3 / degC',
'Temperature T4 / degC',
'Temperature T5 / degC',
'Charging Capacity / Ah',
'Charging Energy / Wh',
'Cycle Charging Capacity / Ah',
'Cycle Charging Energy / Wh',
'Cycle Discharging Capacity / Ah',
'Cycle Discharging Energy / Wh',
'Discharging Capacity / Ah',
'Discharging Energy / Wh',
'Step Charging Capacity / Ah',
'Step Charging Energy / Wh',
'Step Cumulative Capacity / Ah',
'Step Cumulative Energy / Wh',
'Step Discharging Capacity / Ah',
'Step Discharging Energy / Wh',
'Step Record Index / 1',
'Unix Time / s',
'Imaginary Impedance / ohm',
'Internal Resistance / ohm',
'Real Impedance / ohm',
'Cycle Count / 1',
'Step Time / s',
'Power / W'),
'legacy_labels': [],
'n_rows': 95417,
'n_cols': 12,
'time_stats': {'present': True,
'monotonic': True,
'violations': 0,
'min_drop': 0.0,
'first_bad_index': None,
'epsilon': 0.1},
'derived': {'issues': [], 'details': []}}}],
'failed': [],
'skipped': [{'path': '/home/runner/work/battery-data-format/battery-data-format/docs/examples/in/ingest/example_raw_data_collection/g20m7/.bdf.state.json',
'reason': 'metadata_file'},
{'path': '/home/runner/work/battery-data-format/battery-data-format/docs/examples/in/ingest/example_raw_data_collection/g20m7/person.json',
'reason': 'metadata_file'},
{'path': '/home/runner/work/battery-data-format/battery-data-format/docs/examples/in/ingest/example_raw_data_collection/g20m7/SINTEF__google-g20m7-202512-c5hnd9__20260112__HPPC__25degC.nda',
'reason': 'unchanged'},
{'path': '/home/runner/work/battery-data-format/battery-data-format/docs/examples/in/ingest/example_raw_data_collection/g20m7/battery.json',
'reason': 'metadata_file'},
{'path': '/home/runner/work/battery-data-format/battery-data-format/docs/examples/in/ingest/example_raw_data_collection/g20m7/contribution.json',
'reason': 'metadata_file'},
{'path': '/home/runner/work/battery-data-format/battery-data-format/docs/examples/in/ingest/example_raw_data_collection/g20m7/metadata.jsonld',
'reason': 'metadata_file'},
{'path': '/home/runner/work/battery-data-format/battery-data-format/docs/examples/in/ingest/example_raw_data_collection/g20m7/batteries/c5hnd9/metadata.jsonld',
'reason': 'metadata_file'}],
'metadata': [{'path': '/home/runner/work/battery-data-format/battery-data-format/docs/examples/in/ingest/example_raw_data_collection/g20m7',
'metadata': '/home/runner/work/battery-data-format/battery-data-format/docs/examples/in/ingest/example_raw_data_collection/g20m7/metadata.jsonld'},
{'path': '/home/runner/work/battery-data-format/battery-data-format/docs/examples/in/ingest/example_raw_data_collection/g20m7/batteries/c5hnd9',
'metadata': '/home/runner/work/battery-data-format/battery-data-format/docs/examples/in/ingest/example_raw_data_collection/g20m7/batteries/c5hnd9/metadata.jsonld'}],
'metadata_failed': []}},
{'path': '/home/runner/work/battery-data-format/battery-data-format/docs/examples/in/ingest/example_raw_data_collection/nacr32140mp10',
'summary': {'converted': [{'path': '/home/runner/work/battery-data-format/battery-data-format/docs/examples/in/ingest/example_raw_data_collection/nacr32140mp10/SINTEF__hina-nacr32140mp10-202410-3gvjz6__20250617__2C__25degC.mpt',
'output': '/home/runner/work/battery-data-format/battery-data-format/docs/examples/in/ingest/example_raw_data_collection/nacr32140mp10/timeseries/SINTEF__hina-nacr32140mp10-202410-3gvjz6__20250617__2C__25degC.bdf.parquet'}],
'validated': [],
'failed': [],
'skipped': [{'path': '/home/runner/work/battery-data-format/battery-data-format/docs/examples/in/ingest/example_raw_data_collection/nacr32140mp10/person.json',
'reason': 'metadata_file'},
{'path': '/home/runner/work/battery-data-format/battery-data-format/docs/examples/in/ingest/example_raw_data_collection/nacr32140mp10/battery.json',
'reason': 'metadata_file'},
{'path': '/home/runner/work/battery-data-format/battery-data-format/docs/examples/in/ingest/example_raw_data_collection/nacr32140mp10/contribution.json',
'reason': 'metadata_file'}],
'metadata': [{'path': '/home/runner/work/battery-data-format/battery-data-format/docs/examples/in/ingest/example_raw_data_collection/nacr32140mp10',
'metadata': '/home/runner/work/battery-data-format/battery-data-format/docs/examples/in/ingest/example_raw_data_collection/nacr32140mp10/metadata.jsonld'},
{'path': '/home/runner/work/battery-data-format/battery-data-format/docs/examples/in/ingest/example_raw_data_collection/nacr32140mp10/batteries/wv5jgv',
'metadata': '/home/runner/work/battery-data-format/battery-data-format/docs/examples/in/ingest/example_raw_data_collection/nacr32140mp10/batteries/wv5jgv/metadata.jsonld'},
{'path': '/home/runner/work/battery-data-format/battery-data-format/docs/examples/in/ingest/example_raw_data_collection/nacr32140mp10/batteries/r30yqg',
'metadata': '/home/runner/work/battery-data-format/battery-data-format/docs/examples/in/ingest/example_raw_data_collection/nacr32140mp10/batteries/r30yqg/metadata.jsonld'},
{'path': '/home/runner/work/battery-data-format/battery-data-format/docs/examples/in/ingest/example_raw_data_collection/nacr32140mp10/batteries/9peqy1',
'metadata': '/home/runner/work/battery-data-format/battery-data-format/docs/examples/in/ingest/example_raw_data_collection/nacr32140mp10/batteries/9peqy1/metadata.jsonld'},
{'path': '/home/runner/work/battery-data-format/battery-data-format/docs/examples/in/ingest/example_raw_data_collection/nacr32140mp10/batteries/e6yezv',
'metadata': '/home/runner/work/battery-data-format/battery-data-format/docs/examples/in/ingest/example_raw_data_collection/nacr32140mp10/batteries/e6yezv/metadata.jsonld'},
{'path': '/home/runner/work/battery-data-format/battery-data-format/docs/examples/in/ingest/example_raw_data_collection/nacr32140mp10/batteries/mhpggp',
'metadata': '/home/runner/work/battery-data-format/battery-data-format/docs/examples/in/ingest/example_raw_data_collection/nacr32140mp10/batteries/mhpggp/metadata.jsonld'},
{'path': '/home/runner/work/battery-data-format/battery-data-format/docs/examples/in/ingest/example_raw_data_collection/nacr32140mp10/batteries/z1nxja',
'metadata': '/home/runner/work/battery-data-format/battery-data-format/docs/examples/in/ingest/example_raw_data_collection/nacr32140mp10/batteries/z1nxja/metadata.jsonld'},
{'path': '/home/runner/work/battery-data-format/battery-data-format/docs/examples/in/ingest/example_raw_data_collection/nacr32140mp10/batteries/3gvjz6',
'metadata': '/home/runner/work/battery-data-format/battery-data-format/docs/examples/in/ingest/example_raw_data_collection/nacr32140mp10/batteries/3gvjz6/metadata.jsonld'},
{'path': '/home/runner/work/battery-data-format/battery-data-format/docs/examples/in/ingest/example_raw_data_collection/nacr32140mp10/batteries/v0pygf',
'metadata': '/home/runner/work/battery-data-format/battery-data-format/docs/examples/in/ingest/example_raw_data_collection/nacr32140mp10/batteries/v0pygf/metadata.jsonld'}],
'metadata_failed': []}}],
'errors': []}
Notes#
Use
format="csv"if you need.bdf.csvoutput instead of parquet.CSV ingest writes machine headers (
skos:notation) by default. Passlabels="preferred"to writeprefLabelheaders.