Update NovonixDatapath with improved parsing capability (#687)

* bugfix novonix ingestion when a cycle has no charge steps, allow discharge energy/capacity convention to work based on highest discharge value within the cycle
also include tests

* better description during interpolation

* docs got corrupted somehow, rebuild

* allow for CV segment to be nan, though will not resolve missing cycles which are at the end of a cycler run and are incomplete (#

* account for new novonix firmware and formatting

* update tests with new novonix formatting capability

beep/conversion_schemas/novonix_conversion.yaml

@@ -39,6 +39,9 @@ data_columns:
   Step position:
     beep_name: step_position
     data_type: int
+
+# These two variables need to be accounted for in the loading python file
+# As windows can't handle parts of the unicode deg symbol
 #  Temperature (°C):
 #    beep_name: temperature
 #    data_type: float

beep/structure/base.py

@@ -887,9 +887,13 @@ def summarize_cycles(
             CV = get_CV_segment_from_charge(charge)
             if CV.empty:
                 logger.debug(f"Failed to extract CV segment for cycle {cycle}!")
-            CV_time.append(get_CV_time(CV))
-            CV_current.append(get_CV_current(CV))
-            CV_capacity.append(get_CV_capacity(CV))
+                CV_time.append(np.nan)
+                CV_current.append(np.nan)
+                CV_capacity.append(np.nan)
+            else:
+                CV_time.append(get_CV_time(CV))
+                CV_current.append(get_CV_current(CV))
+                CV_capacity.append(get_CV_capacity(CV))
         summary["CV_time"] = CV_time
         summary["CV_current"] = CV_current
         summary["CV_capacity"] = CV_capacity

beep/structure/novonix.py

@@ -74,7 +74,7 @@ def from_file(cls, path, summary_path=None):
 
         # format columns
         map = cls.conversion_config['data_columns']
-        type_map = {j: map[j]['data_type'] for j in map}
+        type_map = {j: map[j]['data_type'] for j in map if j in data.columns}
         data = data.astype(type_map)
         name_map = {i: map[i]['beep_name'] for i in map}
         data.rename(name_map, axis="columns", inplace=True)
@@ -120,8 +120,23 @@ def from_file(cls, path, summary_path=None):
         data['discharge_capacity'] = data[dchg_ix]['capacity'].astype('float')
         data['charge_energy'] = data[chg_ix]['energy'].astype('float')
         data['discharge_energy'] = data[dchg_ix]['energy'].astype('float')
-        data['date_time_iso'] = data['date_time'].map(
-            lambda x: datetime.strptime(x, '%Y-%m-%d %I:%M:%S %p').isoformat())
+
+        # account for multiple novonix datetime formats
+        converted = False
+        for fmt in (
+                '%Y-%m-%d %I:%M:%S %p',
+                '%m/%d/%Y %I:%M:%S %p',
+        ):
+            try:
+                data['date_time_iso'] = data['date_time'].map(
+                    lambda x: datetime.strptime(x, fmt).isoformat())
+                converted = True
+            except ValueError:
+                logger.warning(f"{cls.__name__} could not load date-time in format '{fmt}'! Trying alternative...")
+
+        if not converted:
+            raise ValueError("Date-time was not parsable, aborting.")
+
         data.fillna(0)
 
         # Correct discharge capacities and energies for convention

beep/structure/tests/test_cyclerpaths.py

@@ -493,6 +493,9 @@ def setUp(self) -> None:
         self.file_long = os.path.join(
             TEST_FILE_DIR, "raw", "XC_Formation_Test_040722.csv"
         )
+        self.file_alternative = os.path.join(
+            TEST_FILE_DIR, "raw", "Nova_Formation_138.csv"
+        )
 
     def test_from_file_w_metadata_and_summary(self):
         dp = NovonixDatapath.from_file(self.file_short, summary_path=self.file_short_summary)
@@ -574,6 +577,14 @@ def test_from_file_long(self):
         is_valid, reason = dp.validate()
         self.assertTrue(is_valid)
 
+    def test_from_file_alternate_format(self):
+        # Test with alternate format for date-time in the source file.
+        dp = NovonixDatapath.from_file(self.file_alternative)
+        self.assertEqual(dp.paths.get("raw"), self.file_alternative)
+        self.assertTupleEqual(dp.raw_data.shape, (32331, 24))
+        self.assertEqual(dp.raw_data["date_time"].iloc[100], "9/21/2022 7:31:43 PM")
+        self.assertEqual(dp.raw_data["date_time_iso"].iloc[100], "2022-09-21T19:31:43")
+
     def test_structure_novonix(self):
         dp = NovonixDatapath.from_file(self.file_long)
         dp.structure(