Skip to content

NotImplementedError: Unknown calibration type #3281

@toufikshit

Description

@toufikshit

I just wanted to report this issue:

if I load files before 2008-11-30 → works fine.

Files after 2008-11-30 → fail immediately at calibration.

Likely cause: metadata in the file specifies an unsupported value of dataset_id["calibration"].

Could be related to MSG reprocessing phase or calibration table change.


RemoteTraceback                           Traceback (most recent call last)
RemoteTraceback: 
"""
Traceback (most recent call last):
  File "/opt/conda/lib/python3.11/multiprocessing/pool.py", line 125, in worker
    result = (True, func(*args, **kwds))
                    ^^^^^^^^^^^^^^^^^^^
 


 File "/tmp/ipykernel_930349/299977156.py", line 122, in process_product
    scn.load(CHANNELS)
  File "/opt/conda/lib/python3.11/site-packages/satpy/scene.py", line 1474, in load
    self._read_datasets_from_storage(**kwargs)
  File "/opt/conda/lib/python3.11/site-packages/satpy/scene.py", line 1497, in _read_datasets_from_storage
    return self._read_dataset_nodes_from_storage(nodes, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/conda/lib/python3.11/site-packages/satpy/scene.py", line 1503, in _read_dataset_nodes_from_storage
    loaded_datasets = self._load_datasets_by_readers(reader_datasets, **kwargs)
                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/conda/lib/python3.11/site-packages/satpy/scene.py", line 1528, in _load_datasets_by_readers
    new_datasets = reader_instance.load(ds_ids, **kwargs)
                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/conda/lib/python3.11/site-packages/satpy/readers/core/yaml_reader.py", line 950, in load
    ds = self._load_dataset_with_area(dsid, coords, **kwargs)
         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/conda/lib/python3.11/site-packages/satpy/readers/core/yaml_reader.py", line 1147, in _load_dataset_with_area
    ds = super(GEOFlippableFileYAMLReader, self)._load_dataset_with_area(dsid, coords, **kwargs)
         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/conda/lib/python3.11/site-packages/satpy/readers/core/yaml_reader.py", line 838, in _load_dataset_with_area
    ds = self._load_dataset_data(file_handlers, dsid, **kwargs)
         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/conda/lib/python3.11/site-packages/satpy/readers/core/yaml_reader.py", line 738, in _load_dataset_data
    proj = self._load_dataset(dsid, ds_info, file_handlers, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/conda/lib/python3.11/site-packages/satpy/readers/core/yaml_reader.py", line 714, in _load_dataset
    projectable = fh.get_dataset(dsid, ds_info)
                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/conda/lib/python3.11/site-packages/satpy/readers/seviri_l1b_native.py", line 585, in get_dataset
    dataset = self.calibrate(xarr, dataset_id)
              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/conda/lib/python3.11/site-packages/satpy/readers/seviri_l1b_native.py", line 627, in calibrate
    res = calib.calibrate(data, dataset_id["calibration"])
          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/conda/lib/python3.11/site-packages/satpy/readers/core/seviri.py", line 669, in calibrate
    res = self._algo.ir_calibrate(
          ^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/conda/lib/python3.11/site-packages/satpy/readers/core/seviri.py", line 592, in ir_calibrate
    raise NotImplementedError("Unknown calibration type")
NotImplementedError: Unknown calibration type
"""

The above exception was the direct cause of the following exception:

NotImplementedError                       Traceback (most recent call last)
Cell In[1], line 175
    173 if products:
    174     with Pool(processes=cpu_count() - 1) as pool:
--> 175         for _ in tqdm(pool.imap_unordered(process_product, products), total=len(products)):
    176             pass
    178 current += datetime.timedelta(days=2)

File /opt/conda/lib/python3.11/site-packages/tqdm/std.py:1182, in tqdm.__iter__(self)
   1179 time = self._time
   1181 try:
-> 1182     for obj in iterable:
   1183         yield obj
   1184         # Update and possibly print the progressbar.
   1185         # Note: does not call self.update(1) for speed optimisation.

File /opt/conda/lib/python3.11/multiprocessing/pool.py:873, in IMapIterator.next(self, timeout)
    871 if success:
    872     return value
--> 873 raise value

File /opt/conda/lib/python3.11/multiprocessing/pool.py:125, in worker()
    123 job, i, func, args, kwds = task
    124 try:
--> 125     result = (True, func(*args, **kwds))
    126 except Exception as e:
    127     if wrap_exception and func is not _helper_reraises_exception:

Cell In[1], line 122, in process_product()
    120 try:
    121     scn = Scene(reader="seviri_l1b_native", filenames=[nat_path])
--> 122     scn.load(CHANNELS)
    123 except ValueError as e:
    124     print(f"❌ Corrupted or partial NAT file: {nat_file} — skipping. ({e})")

File /opt/conda/lib/python3.11/site-packages/satpy/scene.py:1474, in load()
   1470 self._update_dependency_tree(needed_datasets, query)
   1472 self._wishlist |= needed_datasets
-> 1474 self._read_datasets_from_storage(**kwargs)
   1475 if generate:
   1476     self.generate_possible_composites(unload)

File /opt/conda/lib/python3.11/site-packages/satpy/scene.py:1497, in _read_datasets_from_storage()
   1487 """Load datasets from the necessary reader.
   1488 
   1489 Args:
   (...)
   1494 
   1495 """
   1496 nodes = self._dependency_tree.leaves(limit_nodes_to=self.missing_datasets)
-> 1497 return self._read_dataset_nodes_from_storage(nodes, **kwargs)

File /opt/conda/lib/python3.11/site-packages/satpy/scene.py:1503, in _read_dataset_nodes_from_storage()
   1501 # Sort requested datasets by reader
   1502 reader_datasets = self._sort_dataset_nodes_by_reader(reader_nodes)
-> 1503 loaded_datasets = self._load_datasets_by_readers(reader_datasets, **kwargs)
   1504 self._datasets.update(loaded_datasets)
   1505 return loaded_datasets

File /opt/conda/lib/python3.11/site-packages/satpy/scene.py:1528, in _load_datasets_by_readers()
   1526 for reader_name, ds_ids in reader_datasets.items():
   1527     reader_instance = self._readers[reader_name]
-> 1528     new_datasets = reader_instance.load(ds_ids, **kwargs)
   1529     loaded_datasets.update(new_datasets)
   1530 return loaded_datasets

File /opt/conda/lib/python3.11/site-packages/satpy/readers/core/yaml_reader.py:950, in load()
    947     continue
    948 coords = [all_datasets.get(cid, None)
    949           for cid in coordinates.get(dsid, [])]
--> 950 ds = self._load_dataset_with_area(dsid, coords, **kwargs)
    951 if ds is not None:
    952     all_datasets[dsid] = ds

File /opt/conda/lib/python3.11/site-packages/satpy/readers/core/yaml_reader.py:1147, in _load_dataset_with_area()
   1146 def _load_dataset_with_area(self, dsid, coords, upper_right_corner="native", **kwargs):
-> 1147     ds = super(GEOFlippableFileYAMLReader, self)._load_dataset_with_area(dsid, coords, **kwargs)
   1149     if ds is not None:
   1150         ds = _set_orientation(ds, upper_right_corner)

File /opt/conda/lib/python3.11/site-packages/satpy/readers/core/yaml_reader.py:838, in _load_dataset_with_area()
    835     return
    837 try:
--> 838     ds = self._load_dataset_data(file_handlers, dsid, **kwargs)
    839 except (KeyError, ValueError) as err:
    840     logger.exception("Could not load dataset '%s': %s", dsid, str(err))

File /opt/conda/lib/python3.11/site-packages/satpy/readers/core/yaml_reader.py:738, in _load_dataset_data()
    736 def _load_dataset_data(self, file_handlers, dsid, **kwargs):
    737     ds_info = self.all_ids[dsid]
--> 738     proj = self._load_dataset(dsid, ds_info, file_handlers, **kwargs)
    739     # FIXME: areas could be concatenated here
    740     # Update the metadata
    741     proj.attrs["start_time"] = file_handlers[0].start_time

File /opt/conda/lib/python3.11/site-packages/satpy/readers/core/yaml_reader.py:714, in _load_dataset()
    712 for fh in file_handlers:
    713     try:
--> 714         projectable = fh.get_dataset(dsid, ds_info)
    715         if projectable is not None:
    716             slice_list.append(projectable)

File /opt/conda/lib/python3.11/site-packages/satpy/readers/seviri_l1b_native.py:585, in get_dataset()
    582 if xarr is None:
    583     return None
--> 585 dataset = self.calibrate(xarr, dataset_id)
    586 self._add_scanline_acq_time(dataset, dataset_id)
    587 self._update_attrs(dataset, dataset_info)

File /opt/conda/lib/python3.11/site-packages/satpy/readers/seviri_l1b_native.py:627, in calibrate()
    625 tic = dt.datetime.now()
    626 calib = self._get_calibration_handler(dataset_id)
--> 627 res = calib.calibrate(data, dataset_id["calibration"])
    628 logger.debug("Calibration time " + str(dt.datetime.now() - tic))
    629 return res

File /opt/conda/lib/python3.11/site-packages/satpy/readers/core/seviri.py:669, in calibrate()
    667     res = self._algo.vis_calibrate(res, solar_irradiance)
    668 elif calibration == "brightness_temperature":
--> 669     res = self._algo.ir_calibrate(
    670         res, self._scan_params.channel_name, self._calib_params.radiance_type
    671     )
    672 return res

File /opt/conda/lib/python3.11/site-packages/satpy/readers/core/seviri.py:592, in ir_calibrate()
    590     return self._erads2bt(data, channel_name)
    591 else:
--> 592     raise NotImplementedError("Unknown calibration type")

NotImplementedError: Unknown calibration type 


Metadata

Metadata

Assignees

No one assigned

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions