-
Notifications
You must be signed in to change notification settings - Fork 320
Labels
Description
I just wanted to report this issue:
if I load files before 2008-11-30 → works fine.
Files after 2008-11-30 → fail immediately at calibration.
Likely cause: metadata in the file specifies an unsupported value of dataset_id["calibration"].
Could be related to MSG reprocessing phase or calibration table change.
RemoteTraceback Traceback (most recent call last)
RemoteTraceback:
"""
Traceback (most recent call last):
File "/opt/conda/lib/python3.11/multiprocessing/pool.py", line 125, in worker
result = (True, func(*args, **kwds))
^^^^^^^^^^^^^^^^^^^
File "/tmp/ipykernel_930349/299977156.py", line 122, in process_product
scn.load(CHANNELS)
File "/opt/conda/lib/python3.11/site-packages/satpy/scene.py", line 1474, in load
self._read_datasets_from_storage(**kwargs)
File "/opt/conda/lib/python3.11/site-packages/satpy/scene.py", line 1497, in _read_datasets_from_storage
return self._read_dataset_nodes_from_storage(nodes, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/lib/python3.11/site-packages/satpy/scene.py", line 1503, in _read_dataset_nodes_from_storage
loaded_datasets = self._load_datasets_by_readers(reader_datasets, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/lib/python3.11/site-packages/satpy/scene.py", line 1528, in _load_datasets_by_readers
new_datasets = reader_instance.load(ds_ids, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/lib/python3.11/site-packages/satpy/readers/core/yaml_reader.py", line 950, in load
ds = self._load_dataset_with_area(dsid, coords, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/lib/python3.11/site-packages/satpy/readers/core/yaml_reader.py", line 1147, in _load_dataset_with_area
ds = super(GEOFlippableFileYAMLReader, self)._load_dataset_with_area(dsid, coords, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/lib/python3.11/site-packages/satpy/readers/core/yaml_reader.py", line 838, in _load_dataset_with_area
ds = self._load_dataset_data(file_handlers, dsid, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/lib/python3.11/site-packages/satpy/readers/core/yaml_reader.py", line 738, in _load_dataset_data
proj = self._load_dataset(dsid, ds_info, file_handlers, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/lib/python3.11/site-packages/satpy/readers/core/yaml_reader.py", line 714, in _load_dataset
projectable = fh.get_dataset(dsid, ds_info)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/lib/python3.11/site-packages/satpy/readers/seviri_l1b_native.py", line 585, in get_dataset
dataset = self.calibrate(xarr, dataset_id)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/lib/python3.11/site-packages/satpy/readers/seviri_l1b_native.py", line 627, in calibrate
res = calib.calibrate(data, dataset_id["calibration"])
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/lib/python3.11/site-packages/satpy/readers/core/seviri.py", line 669, in calibrate
res = self._algo.ir_calibrate(
^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/lib/python3.11/site-packages/satpy/readers/core/seviri.py", line 592, in ir_calibrate
raise NotImplementedError("Unknown calibration type")
NotImplementedError: Unknown calibration type
"""
The above exception was the direct cause of the following exception:
NotImplementedError Traceback (most recent call last)
Cell In[1], line 175
173 if products:
174 with Pool(processes=cpu_count() - 1) as pool:
--> 175 for _ in tqdm(pool.imap_unordered(process_product, products), total=len(products)):
176 pass
178 current += datetime.timedelta(days=2)
File /opt/conda/lib/python3.11/site-packages/tqdm/std.py:1182, in tqdm.__iter__(self)
1179 time = self._time
1181 try:
-> 1182 for obj in iterable:
1183 yield obj
1184 # Update and possibly print the progressbar.
1185 # Note: does not call self.update(1) for speed optimisation.
File /opt/conda/lib/python3.11/multiprocessing/pool.py:873, in IMapIterator.next(self, timeout)
871 if success:
872 return value
--> 873 raise value
File /opt/conda/lib/python3.11/multiprocessing/pool.py:125, in worker()
123 job, i, func, args, kwds = task
124 try:
--> 125 result = (True, func(*args, **kwds))
126 except Exception as e:
127 if wrap_exception and func is not _helper_reraises_exception:
Cell In[1], line 122, in process_product()
120 try:
121 scn = Scene(reader="seviri_l1b_native", filenames=[nat_path])
--> 122 scn.load(CHANNELS)
123 except ValueError as e:
124 print(f"❌ Corrupted or partial NAT file: {nat_file} — skipping. ({e})")
File /opt/conda/lib/python3.11/site-packages/satpy/scene.py:1474, in load()
1470 self._update_dependency_tree(needed_datasets, query)
1472 self._wishlist |= needed_datasets
-> 1474 self._read_datasets_from_storage(**kwargs)
1475 if generate:
1476 self.generate_possible_composites(unload)
File /opt/conda/lib/python3.11/site-packages/satpy/scene.py:1497, in _read_datasets_from_storage()
1487 """Load datasets from the necessary reader.
1488
1489 Args:
(...)
1494
1495 """
1496 nodes = self._dependency_tree.leaves(limit_nodes_to=self.missing_datasets)
-> 1497 return self._read_dataset_nodes_from_storage(nodes, **kwargs)
File /opt/conda/lib/python3.11/site-packages/satpy/scene.py:1503, in _read_dataset_nodes_from_storage()
1501 # Sort requested datasets by reader
1502 reader_datasets = self._sort_dataset_nodes_by_reader(reader_nodes)
-> 1503 loaded_datasets = self._load_datasets_by_readers(reader_datasets, **kwargs)
1504 self._datasets.update(loaded_datasets)
1505 return loaded_datasets
File /opt/conda/lib/python3.11/site-packages/satpy/scene.py:1528, in _load_datasets_by_readers()
1526 for reader_name, ds_ids in reader_datasets.items():
1527 reader_instance = self._readers[reader_name]
-> 1528 new_datasets = reader_instance.load(ds_ids, **kwargs)
1529 loaded_datasets.update(new_datasets)
1530 return loaded_datasets
File /opt/conda/lib/python3.11/site-packages/satpy/readers/core/yaml_reader.py:950, in load()
947 continue
948 coords = [all_datasets.get(cid, None)
949 for cid in coordinates.get(dsid, [])]
--> 950 ds = self._load_dataset_with_area(dsid, coords, **kwargs)
951 if ds is not None:
952 all_datasets[dsid] = ds
File /opt/conda/lib/python3.11/site-packages/satpy/readers/core/yaml_reader.py:1147, in _load_dataset_with_area()
1146 def _load_dataset_with_area(self, dsid, coords, upper_right_corner="native", **kwargs):
-> 1147 ds = super(GEOFlippableFileYAMLReader, self)._load_dataset_with_area(dsid, coords, **kwargs)
1149 if ds is not None:
1150 ds = _set_orientation(ds, upper_right_corner)
File /opt/conda/lib/python3.11/site-packages/satpy/readers/core/yaml_reader.py:838, in _load_dataset_with_area()
835 return
837 try:
--> 838 ds = self._load_dataset_data(file_handlers, dsid, **kwargs)
839 except (KeyError, ValueError) as err:
840 logger.exception("Could not load dataset '%s': %s", dsid, str(err))
File /opt/conda/lib/python3.11/site-packages/satpy/readers/core/yaml_reader.py:738, in _load_dataset_data()
736 def _load_dataset_data(self, file_handlers, dsid, **kwargs):
737 ds_info = self.all_ids[dsid]
--> 738 proj = self._load_dataset(dsid, ds_info, file_handlers, **kwargs)
739 # FIXME: areas could be concatenated here
740 # Update the metadata
741 proj.attrs["start_time"] = file_handlers[0].start_time
File /opt/conda/lib/python3.11/site-packages/satpy/readers/core/yaml_reader.py:714, in _load_dataset()
712 for fh in file_handlers:
713 try:
--> 714 projectable = fh.get_dataset(dsid, ds_info)
715 if projectable is not None:
716 slice_list.append(projectable)
File /opt/conda/lib/python3.11/site-packages/satpy/readers/seviri_l1b_native.py:585, in get_dataset()
582 if xarr is None:
583 return None
--> 585 dataset = self.calibrate(xarr, dataset_id)
586 self._add_scanline_acq_time(dataset, dataset_id)
587 self._update_attrs(dataset, dataset_info)
File /opt/conda/lib/python3.11/site-packages/satpy/readers/seviri_l1b_native.py:627, in calibrate()
625 tic = dt.datetime.now()
626 calib = self._get_calibration_handler(dataset_id)
--> 627 res = calib.calibrate(data, dataset_id["calibration"])
628 logger.debug("Calibration time " + str(dt.datetime.now() - tic))
629 return res
File /opt/conda/lib/python3.11/site-packages/satpy/readers/core/seviri.py:669, in calibrate()
667 res = self._algo.vis_calibrate(res, solar_irradiance)
668 elif calibration == "brightness_temperature":
--> 669 res = self._algo.ir_calibrate(
670 res, self._scan_params.channel_name, self._calib_params.radiance_type
671 )
672 return res
File /opt/conda/lib/python3.11/site-packages/satpy/readers/core/seviri.py:592, in ir_calibrate()
590 return self._erads2bt(data, channel_name)
591 else:
--> 592 raise NotImplementedError("Unknown calibration type")
NotImplementedError: Unknown calibration type