diff --git a/message_ix_models/data/report/global.yaml b/message_ix_models/data/report/global.yaml index b274ad8102..566684b335 100644 --- a/message_ix_models/data/report/global.yaml +++ b/message_ix_models/data/report/global.yaml @@ -18,11 +18,6 @@ units: '-': '' apply: - GDP: billion USD_2005 / year - PRICE_COMMODITY: USD_2010 / kWa - # These were initially "carbon", which is not a unit. - # TODO check that Mt (rather than t or kt) is correct for all values. - PRICE_EMISSION: USD_2005 / Mt tax_emission: USD_2005 / Mt # Inconsistent units. These quantities (and others computed from them) # must be split apart into separate quantities with consistent units. @@ -454,100 +449,6 @@ combine: - quantity: land_out::CH4_0+1 weight: 0.025 - -# Prices - -- key: price_carbon:n-y - # TODO PRICE_EMISSION has dimension "y", tax_emission has dimension - # "type_year". Implement a dimension rename so that the two can be - # combined in this way. - inputs: - - quantity: PRICE_EMISSION - select: {type_emission: [TCE], type_tec: [all]} - # - quantity: tax_emission - # select: {type_emission: [TCE], type_tec: [all]} - -# Commodity price minus emission price -# NB This is only for illustration. -# TODO use emission factor must be used to convert the following to compatible -# units: -# - PRICE_COMMODITY with (c, l) dimensions and units [currency] / [energy] -# - 'price emission' with (e, t) dimensions and units [currency] / [mass] -- key: price ex carbon:n-t-y-c-l-e - inputs: - - quantity: PRICE_COMMODITY:n-c-l-y - - quantity: price emission:n-e-t-y - weight: -1 - -# TODO remove these entries once the one-step conversion is checked. -# - The following entries subset the components of PRICE_COMMODITY used in the -# legacy reporting. The preferred method is to convert the entire variable to -# IAMC format in one step; see below in the "iamc:" section. -# - l: [import] is sometimes included to pick up prices at the global node(s). -# - In general, PRICE_COMMODITY has data for n=GLB and level=import OR for -# other nodes and l=primary or secondary—but not otherwise. -- key: price_c:n-y-h - inputs: - - quantity: PRICE_COMMODITY - select: {c: [coal], l: [primary, import]} -- key: price_g_w:n-y-h - # Only includes 11 regions; no data for c="gas" for global regions, instead - # c="LNG" is used. The name "LNG" is replaced later. - inputs: - - quantity: PRICE_COMMODITY - select: {c: [gas], l: [primary]} -- key: price_o_w:n-y-h - # l="import" is used for the global region. - inputs: - - quantity: PRICE_COMMODITY - select: {c: [crudeoil], l: [primary, import]} -- key: price_b_w:n-y-h - inputs: - - quantity: PRICE_COMMODITY - select: {c: [biomass], l: [primary]} -- key: price_e_w:n-y-h - inputs: - - quantity: PRICE_COMMODITY - select: {c: [electr], l: [secondary]} -- key: price_h_w:n-y-h - # For the global region: l="import", c="l2h". - inputs: - - quantity: PRICE_COMMODITY - select: {c: [hydrogen], l: [secondary]} -- key: price_liq_o_w:n-y-h - # l="import" is used for the global region. - inputs: - - quantity: PRICE_COMMODITY - select: {c: [lightoil], l: [secondary, import]} -- key: price_liq_b_w:n-y-h - # l="import" is used for the global region. - inputs: - - quantity: PRICE_COMMODITY - select: {c: [ethanol], l: [secondary, import]} -- key: price_final_e:n-y-h - inputs: - - quantity: PRICE_COMMODITY - select: {c: [electr], l: [final]} -- key: price_final_sol_c:n-y-h - inputs: - - quantity: PRICE_COMMODITY - select: {c: [coal], l: [final]} -- key: price_final_sol_b:n-y-h - inputs: - - quantity: PRICE_COMMODITY - select: {c: [biomass], l: [final]} -- key: price_final_liq_o:n-y-h - inputs: - - quantity: PRICE_COMMODITY - select: {c: [lightoil], l: [final]} -- key: price_final_liq_b:n-y-h - inputs: - - quantity: PRICE_COMMODITY - select: {c: [ethanol], l: [final]} -- key: price_final_gas:n-y-h - inputs: - - quantity: PRICE_COMMODITY - select: {c: [gas], l: [final]} # TODO complete or replace this # - key: land_out:n-s-y-h # inputs: @@ -747,12 +648,6 @@ general: args: units: 'GWa / year' -- key: gdp_ppp - comp: product - inputs: - - GDP - - MERtoPPP - # CH4 emissions from GLOBIOM: select only the subset - key: land_out:n-s-y-c-l-h:CH4_0 comp: select @@ -839,14 +734,6 @@ _iamc formats: iamc: -- variable: GDP|MER - base: GDP:n-y - unit: billion USD_2010 / year - -- variable: GDP|PPP - base: gdp_ppp:n-y - unit: billion USD_2010 / year - - variable: Primary Energy|Coal base: coal:nl-ya <<: *pe_iamc @@ -973,73 +860,8 @@ iamc: - gwp metric unit: Mt / year # Species captured in 'e equivalent' -# Prices -# Preferred method: convert all the contents of the variable at once. -- variable: Price - base: PRICE_COMMODITY:n-c-l-y - var: [l, c] - <<: *price_iamc -- variable: Price|Carbon - base: price_carbon:n-y - # This was initially "carbon_dioxide", which is not a unit. - # TODO check that Mt (rather than t or kt) is correct. - # TODO check whether there is a species / GWP conversion here. - unit: USD_2010 / Mt - rename: {y: year} -# commented: see above -# - variable: Price w/o carbon -# base: price ex carbon:n-t-y-c-e -# var: [t, c, l, e] # rename: {y: year} -# TODO ensure these are covered by the preferred method, above, and then -# remove these separate conversions. -- variable: Price (legacy)|Primary Energy wo carbon price|Biomass - base: price_b_w:n-y-h - <<: *price_iamc -- variable: Price (legacy)|Primary Energy wo carbon price|Coal - base: price_c:n-y-h - <<: *price_iamc -- variable: Price (legacy)|Primary Energy wo carbon price|Gas - base: price_g_w:n-y-h - <<: *price_iamc -- variable: Price (legacy)|Primary Energy wo carbon price|Oil - base: price_o_w:n-y-h - <<: *price_iamc -- variable: Price (legacy)|Secondary Energy wo carbon price|Electricity - base: price_e_w:n-y-h - <<: *price_iamc -- variable: Price (legacy)|Secondary Energy wo carbon price|Hydrogen - base: price_h_w:n-y-h - <<: *price_iamc -- variable: Price (legacy)|Secondary Energy wo carbon price|Liquids|Biomass - base: price_liq_b_w:n-y-h - <<: *price_iamc -- variable: Price (legacy)|Secondary Energy wo carbon price|Liquids|Oil - base: price_liq_o_w:n-y-h - <<: *price_iamc -- variable: Price (legacy)|Final Energy wo carbon price|Residential|Electricity - base: price_final_e:n-y-h - <<: *price_iamc -- variable: Price (legacy)|Final Energy wo carbon price|Residential|Gases|Natural Gas - base: price_final_gas:n-y-h - <<: *price_iamc -- variable: Price (legacy)|Final Energy wo carbon price|Residential|Liquids|Biomass - base: price_final_liq_b:n-y-h - <<: *price_iamc -- variable: Price (legacy)|Final Energy wo carbon price|Residential|Liquids|Oil - base: price_final_liq_o:n-y-h - <<: *price_iamc -- variable: Price (legacy)|Final Energy wo carbon price|Residential|Solids|Biomass - base: price_final_sol_b:n-y-h - <<: *price_iamc -- variable: Price (legacy)|Final Energy wo carbon price|Residential|Solids|Coal - base: price_final_sol_c:n-y-h - <<: *price_iamc -#- variable: Price (legacy)|Agriculture|Non-Energy Crops and Livestock|Index -# base: price_agriculture:n-y-h -# rename: {y: year} - report: - key: pe test members: @@ -1051,11 +873,6 @@ report: - Primary Energy|Solar::iamc - Primary Energy|Wind::iamc -- key: gdp test - members: - - GDP|MER::iamc - - GDP|PPP::iamc - - key: se test members: - Secondary Energy::iamc @@ -1080,28 +897,3 @@ report: # - Emissions|CH4|Energy|Supply|Solids|Biomass|Fugitive::iamc # - Emissions|CH4|Energy|Supply|Solids|Coal|Fugitive::iamc -- key: price test - members: - - Price::iamc - - Price|Carbon::iamc - # commented: see above - # - Price w/o carbon::iamc - - # TODO ensure these are covered by the preferred method, above, then remove - # these - - Price (legacy)|Primary Energy wo carbon price|Biomass::iamc - - Price (legacy)|Primary Energy wo carbon price|Coal::iamc - - Price (legacy)|Primary Energy wo carbon price|Gas::iamc - - Price (legacy)|Primary Energy wo carbon price|Oil::iamc - - Price (legacy)|Secondary Energy wo carbon price|Electricity::iamc - - Price (legacy)|Secondary Energy wo carbon price|Hydrogen::iamc - - Price (legacy)|Secondary Energy wo carbon price|Liquids|Biomass::iamc - - Price (legacy)|Secondary Energy wo carbon price|Liquids|Oil::iamc - # NB for "Price|Secondary Energy|Liquids|Oil", the legacy reporting inserts a - # zero matrix. - - Price (legacy)|Final Energy wo carbon price|Residential|Electricity::iamc - - Price (legacy)|Final Energy wo carbon price|Residential|Gases|Natural Gas::iamc - - Price (legacy)|Final Energy wo carbon price|Residential|Liquids|Biomass::iamc - - Price (legacy)|Final Energy wo carbon price|Residential|Liquids|Oil::iamc - - Price (legacy)|Final Energy wo carbon price|Residential|Solids|Biomass::iamc - - Price (legacy)|Final Energy wo carbon price|Residential|Solids|Coal::iamc diff --git a/message_ix_models/data/technology.yaml b/message_ix_models/data/technology.yaml index 0669af9679..77d92c2cd8 100644 --- a/message_ix_models/data/technology.yaml +++ b/message_ix_models/data/technology.yaml @@ -1139,7 +1139,7 @@ gas_extr_1: type: primary sector: extraction input: [resource] - output: [primary] + output: [gas, primary] gas_extr_2: name: gas_extr_2 @@ -1147,7 +1147,7 @@ gas_extr_2: type: primary sector: extraction input: [resource] - output: [primary] + output: [gas, primary] gas_extr_3: name: gas_extr_3 @@ -1155,7 +1155,7 @@ gas_extr_3: type: primary sector: extraction input: [resource] - output: [primary] + output: [gas, primary] gas_extr_4: name: gas_extr_4 @@ -1163,7 +1163,7 @@ gas_extr_4: type: primary sector: extraction input: [resource] - output: [primary] + output: [gas, primary] gas_extr_5: name: gas_extr_5 @@ -1171,7 +1171,7 @@ gas_extr_5: type: primary sector: extraction input: [resource] - output: [primary] + output: [gas, primary] gas_extr_6: name: gas_extr_6 @@ -1179,7 +1179,7 @@ gas_extr_6: type: primary sector: extraction input: [resource] - output: [primary] + output: [gas, primary] gas_extr_mpen: name: gas_extr_mpen diff --git a/message_ix_models/model/structure.py b/message_ix_models/model/structure.py index ada09c838a..ffd28af217 100644 --- a/message_ix_models/model/structure.py +++ b/message_ix_models/model/structure.py @@ -1,10 +1,11 @@ import logging import re -from collections import ChainMap -from collections.abc import Mapping, MutableMapping +from collections import ChainMap, defaultdict +from collections.abc import Iterator, Mapping, MutableMapping from copy import copy from functools import cache -from itertools import product +from itertools import product, starmap +from typing import Literal import click import pandas as pd @@ -15,7 +16,7 @@ from sdmx.model.v21 import Annotation from message_ix_models.util import load_package_data, package_data_path -from message_ix_models.util.sdmx import as_codes +from message_ix_models.util.sdmx import as_codes, leaf_ids log = logging.getLogger(__name__) @@ -217,6 +218,57 @@ def generate_set_elements(data: MutableMapping, name) -> None: data[name]["indexers"] = indexers +@cache +def get_technology_groups() -> dict[Literal["t"], dict[str, list[str]]]: + """Return a mapping of technology groups. + + The mapping includes 1 group for every unique combination of (input level, + input commodity, output level, output commodity) in :ref:`technology-yaml`. Group + keys are either as given by :func:`technology_group_keys` or, if an explicit name + is given in the code list, that name. Group members are lists of IDs of + technologies belonging to the respective group. + + Groups with only 1 member are omitted; these **should** be referred to be the ID + of the technology directly. + + See also + -------- + get_codelist + technology_group_keys + """ + cl = get_codelist("technology") + + group0: MutableMapping[str, set[str]] = defaultdict(set) + rename = {} + + for tech in cl: + if tech.eval_annotation(id="report-only") is True: + if len(tech.child): + # Groups based on parent/child hierarchy + group0[str(tech.name)] = set(leaf_ids(tech)) + else: + # A reporting name. Map the longest associated key to the group name + key = sorted(technology_group_keys(tech), key=len, reverse=True)[0] + rename[key] = str(tech.name) + else: + for key in technology_group_keys(tech): + group0[key].add(tech.id) + + # Iterate over the completed mapping + group1 = {} + for key, v in group0.items(): + # Omit: + # - The "* * → * *" catch-all key + # - Entries with only 1 technology + if key == "* * → * *" or len(v) == 1: + continue + + # Apply name mapping + group1[rename.get(key, key)] = sorted(v) + + return dict(t=group1) + + def process_units_anno(set_name: str, code: Code, quiet: bool = False) -> None: """Process an annotation on `code` with id="units". @@ -305,6 +357,53 @@ def process_technology_codes(codes): code.annotations.append(anno) +def technology_group_keys(tech: Code) -> Iterator[str]: + """Generate group keys for a `tech` based on its ``input``/``output`` annotations. + + For a technology annotated like: + + .. code-block:: yaml + + tech: + input: [c_foo, l_bar] + output: [c_baz, l_qux] + + …this yields 8 strings with either the given commodity/level or a "*" character, + including: + + - "l_bar c_foo → l_qux c_baz" + - "* c_foo → l_qux c_baz" + - "l_bar * → l_qux c_baz" + - "* * → l_qux c_baz" + - … + - "* * → * *" + """ + # Starting lists of coords + coords = dict(c_in=["*"], c_out=["*"], l_in=["*"], l_out=["*"]) + + # Process input and output annotations + for io in ("input", "output"): + # Retrieve annotation + c_l = tech.eval_annotation(id=io) + + # Transform to an iterable of (c, l) tuples. Some techs are annotated with a + # list of 2-tuples, representing multiple inputs or outputs. + if c_l is None or len(c_l) != 2: + c_l = [] # No annotation or length-1 collection → do nothing + elif isinstance(c_l[0], str): + c_l = [c_l] # Single (c, l) tuple + + # Extend each of the coords + for commodity, level in c_l: + coords[f"c_{io[:-3]}"].append(commodity) + coords[f"l_{io[:-3]}"].append(level) + + yield from starmap( + "{} {} → {} {}".format, + product(coords["l_in"], coords["c_in"], coords["l_out"], coords["c_out"]), + ) + + @click.command(name="techs") @click.pass_obj def cli(ctx): diff --git a/message_ix_models/model/transport/report.py b/message_ix_models/model/transport/report.py index e0c1caa1a0..9491e8740a 100644 --- a/message_ix_models/model/transport/report.py +++ b/message_ix_models/model/transport/report.py @@ -1,7 +1,6 @@ """Reporting/postprocessing for MESSAGEix-Transport.""" import logging -from copy import deepcopy from pathlib import Path from typing import TYPE_CHECKING, Any @@ -12,7 +11,12 @@ from message_ix import Reporter from message_ix_models import Context, ScenarioInfo -from message_ix_models.report.util import add_replacements +from message_ix_models.report.key import all_iamc +from message_ix_models.report.util import ( + IAMCConversion, + add_replacements, + store_write_ts, +) from . import Config from .key import exo, pop @@ -34,48 +38,43 @@ CONVERT_IAMC = ( # NB these are currently tailored to produce the variable names expected for the # NGFS project - dict( - variable="transport activity", - base="out:nl-t-ya-c:transport+units", - var=["Energy Service|Transportation", "t", "c"], + IAMCConversion( + base=Key("out:nl-t-ya-c:transport+units"), + var_parts=["Energy Service|Transportation", "t", "c"], + unit="", sums=["c", "t", "c-t"], ), - dict( - variable="transport stock", - base="CAP:nl-t-ya:ldv+units", - var=["Transport|Stock|Road|Passenger|LDV", "t"], + IAMCConversion( + base=Key("CAP:nl-t-ya:ldv+units"), + var_parts=["Transport|Stock|Road|Passenger|LDV", "t"], unit="Mvehicle", ), - dict( - variable="transport sales", - base="CAP_NEW:nl-t-yv:ldv+units", - var=["Transport|Sales|Road|Passenger|LDV", "t"], + IAMCConversion( + base=Key("CAP_NEW:nl-t-yv:ldv+units"), + var_parts=["Transport|Sales|Road|Passenger|LDV", "t"], unit="Mvehicle", ), # Final energy # # The following are 4 different partial sums of in::transport, in which # individual technologies are already aggregated to modes - dict( - variable="transport fe", - base="in:nl-t-ya-c:transport+units", - var=["Final Energy|Transportation", "t", "c"], - sums=["c", "t", "c-t"], + IAMCConversion( + base=Key("in:nl-t-ya-c:transport+units"), + var_parts=["Final Energy|Transportation", "t", "c"], unit=_FE_UNIT, + sums=["c", "t", "c-t"], ), - dict( - variable="transport fe ldv", - base="in:nl-t-ya-c:ldv+units", - var=["Final Energy|Transportation|Road|Passenger|LDV", "t", "c"], + IAMCConversion( + base=Key("in:nl-t-ya-c:ldv+units"), + var_parts=["Final Energy|Transportation|Road|Passenger|LDV", "t", "c"], unit="EJ/yr", ), # Emissions using MESSAGEix emission_factor parameter # base: auto-sum over dimensions yv, m, h # var: Same as in data/report/global.yaml - # dict( - # variable="transport emi 0", + # IAMCConversion( # base="emi:nl-t-ya-e-gwp metric-e equivalent:gwpe+agg", - # var=[ + # var_parts=[ # "Emissions|CO2|Energy|Demand|Transportation", # "t", # "e", @@ -83,20 +82,17 @@ # "gwp metric", # ], # ), - # dict( - # variable="transport emi 1", + # IAMCConversion( # base="emi:nl-t-ya-e:transport+units", - # var=["Emissions", "e", "Energy|Demand|Transportation", "t"], - # sums=["t"], + # var_parts=["Emissions", "e", "Energy|Demand|Transportation", "t"], # unit="Mt/yr", + # sums=["t"], # ), # # # For debugging - # dict(variable="debug ACT", base="ACT:nl-t-ya", var=["DEBUG", "t"], unit="-"), - # dict(variable="debug CAP", base="CAP:nl-t-ya", var=["DEBUG", "t"], unit="-"), - # dict( - # variable="debug CAP_NEW", base="CAP_NEW:nl-t-yv", var=["DEBUG", "t"], unit="-" - # ), + # IAMCConversion(base="ACT:nl-t-ya", var=["DEBUG", "t"], unit="-"), + # IAMCConversion(base="CAP:nl-t-ya", var=["DEBUG", "t"], unit="-"), + # IAMCConversion(base="CAP_NEW:nl-t-yv", var=["DEBUG", "t"], unit="-"), ) @@ -115,52 +111,6 @@ ] -# TODO Type c as (string) "Computer" once genno supports this -def add_iamc_store_write(c: Computer, base_key) -> "Key": - """Write `base_key` to CSV, XLSX, and/or both; and/or store on "scenario". - - If `base_key` is, for instance, "foo::iamc", this function adds the following keys: - - - "foo::iamc+all": both of: - - - "foo::iamc+file": both of: - - - "foo::iamc+csv": write the data in `base_key` to a file named :file:`foo.csv`. - - "foo::iamc+xlsx": write the data in `base_key` to a file named - :file:`foo.xlsx`. - - The files are created in a subdirectory using :func:`make_output_path`—that is, - including a path component given by the scenario URL. - - - "foo::iamc+store" store the data in `base_key` as time series data on the - scenario identified by the key "scenario". - - .. todo:: Move upstream, to :mod:`message_ix_models`. - """ - k = KeySeq(base_key) - - file_keys = [] - for suffix in ("csv", "xlsx"): - # Create the path - path = c.add( - k[f"{suffix} path"], - "make_output_path", - "config", - name=f"{k.base.name}.{suffix}", - ) - # Write `key` to the path - file_keys.append(c.add(k[suffix], "write_report", base_key, path)) - - # Write all files - c.add(k["file"], file_keys) - - # Store data on "scenario" - c.add(k["store"], "store_ts", "scenario", base_key) - - # Both write and store - return single_key(c.add(k["all"], [k["file"], k["store"]])) - - def aggregate(c: "Computer") -> None: """Aggregate individual transport technologies to modes.""" from genno.operator import aggregate as func @@ -313,24 +263,24 @@ def configure_legacy_reporting(config: dict) -> None: def convert_iamc(c: "Computer") -> None: """Add tasks from :data:`.CONVERT_IAMC`.""" - from message_ix_models.report import iamc as handle_iamc from message_ix_models.report import util from .key import report as k_report util.REPLACE_VARS.update({r"^CAP\|(Transport)": r"\1"}) - keys = [] - for info in CONVERT_IAMC: - handle_iamc(c, deepcopy(info)) - keys.append(f"{info['variable']}::iamc") + # List of keys in all::iamc + keys_pre = set(c.graph[all_iamc][1:]) + for conversion in CONVERT_IAMC: + conversion.add_tasks(c) + added = set(c.graph[all_iamc][1:]) - keys_pre # Concatenate IAMC-format tables - k = Key("transport", tag="iamc") - c.add(k, "concat", *keys) + k = Key("transport::iamc") + c.add(k, "concat", *added) # Add tasks for writing IAMC-structured data to file and storing on the scenario - c.apply(add_iamc_store_write, k) + c.apply(store_write_ts, k) c.graph[k_report.all].append( # Use ths line to both store and write to file IAMC structured-data diff --git a/message_ix_models/model/transport/workflow.py b/message_ix_models/model/transport/workflow.py index 385f847958..71d1ed4acf 100644 --- a/message_ix_models/model/transport/workflow.py +++ b/message_ix_models/model/transport/workflow.py @@ -1,13 +1,12 @@ import logging from copy import deepcopy -from hashlib import blake2s from typing import TYPE_CHECKING, Literal from genno import KeyExistsError from message_ix_models.model.workflow import Config as WorkflowConfig from message_ix_models.tools.policy import single_policy_of_type -from message_ix_models.util import minimum_version +from message_ix_models.util import minimum_version, short_hash if TYPE_CHECKING: from message_ix import Scenario @@ -135,11 +134,6 @@ def scenario_url(context: "Context", label: str | None = None) -> str: ) -def short_hash(value: str) -> str: - """Return a short (length 3) hash of `value`.""" - return blake2s(value.encode()).hexdigest()[:3] - - def tax_emission(context: "Context", scenario: "Scenario", price: float) -> "Scenario": """Add emission tax. diff --git a/message_ix_models/report/__init__.py b/message_ix_models/report/__init__.py index 058f3892e3..0ce7c09462 100644 --- a/message_ix_models/report/__init__.py +++ b/message_ix_models/report/__init__.py @@ -1,7 +1,6 @@ import logging from contextlib import nullcontext from copy import deepcopy -from functools import partial from pathlib import Path from re import escape from typing import TYPE_CHECKING @@ -10,7 +9,6 @@ import genno.config import yaml from genno import Key -from genno.compat.pyam import iamc as handle_iamc from genno.core.key import single_key from ixmp.util import discard_on_error from message_ix import Reporter, Scenario @@ -19,6 +17,7 @@ from message_ix_models.util._logging import mark_time, silence_log from .config import Config +from .util import IAMCConversion, add_replacements, store_write_ts if TYPE_CHECKING: from genno.core.key import KeyLike # TODO Import from genno.types @@ -29,6 +28,7 @@ "NOT_IMPLEMENTED_IAMC", "NOT_IMPLEMENTED_MEASURE", "Config", + "IAMCConversion", "defaults", "prepare_reporter", "register", @@ -135,58 +135,7 @@ def iamc(c: Reporter, info): over "y", and over both "x" and "y". The corresponding dimensions are omitted from "var". All data are concatenated. """ - # FIXME the upstream key "variable" for the configuration is confusing; choose a - # better name - from message_ix_models.report.util import collapse - - # Common - base_key = Key(info["base"]) - - # First part of the 'Variable' name - name = info.pop("variable", base_key.name) - # Parts (string literals or dimension names) to concatenate into variable name - var_parts = info.pop("var", [name]) - - # Use message_ix_models custom collapse() method - info.setdefault("collapse", {}) - - # Add standard renames - info.setdefault("rename", {}) - for dim, target in ( - ("n", "region"), - ("nl", "region"), - ("y", "year"), - ("ya", "year"), - ("yv", "year"), - ): - info["rename"].setdefault(dim, target) - - # Iterate over partial sums - # TODO move some or all of this logic upstream - keys = [] # Resulting keys - for dims in [""] + info.pop("sums", []): - # Dimensions to partial - # TODO allow iterable of str - dims = dims.split("-") - - label = f"{name} {'-'.join(dims) or 'full'}" - - # Modified copy of `info` for this invocation - _info = info.copy() - # Base key: use the partial sum over any `dims`. Use a distinct variable name. - _info.update(base=base_key.drop(*dims), variable=label) - # Exclude any summed dimensions from the IAMC Variable to be constructed - _info["collapse"].update( - callback=partial(collapse, var=[v for v in var_parts if v not in dims]) - ) - - # Invoke the genno built-in handler - handle_iamc(c, _info) - - keys.append(f"{label}::iamc") - - # Concatenate together the multiple tables - c.add("concat", f"{name}::iamc", *keys) + IAMCConversion(**info).add_tasks(c) def register(name_or_callback: "Callback | str") -> str | None: @@ -444,20 +393,23 @@ def defaults(rep: Reporter, context: Context) -> None: - Call :func:`.add_replacements` for members of the :ref:`commodity-yaml` and :ref:`technology-yaml` code lists """ - from message_ix_models.model.structure import get_codes + from message_ix_models.model.structure import get_codes, get_technology_groups from . import key as k - from .util import add_replacements # Add tasks to return coordinates for data manpulation, e.g. expand_dims, select rep.add(k.coords.n_glb, "node_glb", "n") # Add tasks to return groups of codes for aggregation rep.add(k.groups.c, "get_commodity_groups") + rep.add(k.groups.t, get_technology_groups) # Add a placeholder task to concatenate IAMC-structured data rep.add(k.all_iamc, "concat") + # Add tasks to store and write IAMC-structured data + rep.apply(store_write_ts, k.all_iamc) + # Add mappings for conversions to IAMC data structures add_replacements("c", get_codes("commodity")) add_replacements("t", get_codes("technology")) diff --git a/message_ix_models/report/compat.py b/message_ix_models/report/compat.py index d4a70bf69e..37a728cd06 100644 --- a/message_ix_models/report/compat.py +++ b/message_ix_models/report/compat.py @@ -9,6 +9,8 @@ from genno import Key, Quantity, quote from genno.core.key import iter_keys, single_key +from .util import IAMCConversion + if TYPE_CHECKING: from genno import Computer from ixmp import Reporter @@ -261,8 +263,6 @@ def callback(rep: "Reporter", context: "Context") -> None: """ from message_ix_models.model.bare import get_spec - from . import iamc - N = len(rep.graph) # Structure information @@ -384,13 +384,13 @@ def full(name: str) -> Key: # TODO Identify where to sum on "h", "m", "yv" dimensions # Convert to IAMC structure - var = "Emissions|CO2|Energy|Demand|Transportation|Road Rail and Domestic Shipping" - info = dict(variable="transport emissions", base=k1.drop("h", "m", "yv"), var=[var]) - iamc(rep, info) - - # Append to the "all::iamc" task - # TODO Use a helper function for this - rep.graph["all::iamc"] += ("transport emissions::iamc",) + IAMCConversion( + base=k1.drop("h", "m", "yv"), + var_parts=[ + "Emissions|CO2|Energy|Demand|Transportation|Road Rail and Domestic Shipping" + ], + unit="Mt/yr", + ).add_tasks(rep) # TODO use store_ts() to store on scenario diff --git a/message_ix_models/report/config.py b/message_ix_models/report/config.py index 8b909891dd..6cf08c1d82 100644 --- a/message_ix_models/report/config.py +++ b/message_ix_models/report/config.py @@ -24,12 +24,16 @@ def _default_callbacks() -> list[Callback]: - from message_ix_models.report import plot - - from . import defaults, extraction + from . import defaults, extraction, gdp, plot, price # NB When updating this list, also update the docstring of Config.callback - return [defaults, extraction.callback, plot.callback] + return [ + defaults, + extraction.callback, + gdp.callback, + price.callback, + plot.callback, + ] @dataclass @@ -54,7 +58,9 @@ class Config(ConfigHelper): #: #: 1. :func:`.report.defaults` #: 2. :func:`.report.extraction.callback` - #: 3. :func:`.report.plot.callback` + #: 3. :func:`.report.gdp.callback` + #: 4. :func:`.report.plot.callback` + #: 5. :func:`.report.price.callback` #: #: A callback function **must** take two arguments: the Computer/Reporter, and a #: :class:`.Context`: diff --git a/message_ix_models/report/gdp.py b/message_ix_models/report/gdp.py new file mode 100644 index 0000000000..56c5889737 --- /dev/null +++ b/message_ix_models/report/gdp.py @@ -0,0 +1,31 @@ +"""Report GDP.""" + +from typing import TYPE_CHECKING + +from genno import Key + +from .key import GDP +from .util import IAMCConversion + +if TYPE_CHECKING: + from message_ix import Reporter + + from message_ix_models import Context + +K = Key("gdp_ppp:n-y") + +U = "billion USD_2010 / year" + +CONV = ( + IAMCConversion(base=GDP["units"], var_parts=["GDP|MER"], unit=U), + IAMCConversion(base=K, var_parts=["GDP|PPP"], unit=U), +) + + +def callback(r: "Reporter", context: "Context") -> None: + """Prepare reporting of GDP.""" + r.add(GDP["units"], "apply_units", GDP, units="billion USD_2005 / year") + r.add(K, "mul", GDP["units"], "MERtoPPP:n-y") + + for c in CONV: + c.add_tasks(r) diff --git a/message_ix_models/report/key.py b/message_ix_models/report/key.py index 7d06d21f8d..4679dd76ab 100644 --- a/message_ix_models/report/key.py +++ b/message_ix_models/report/key.py @@ -24,6 +24,8 @@ #: Identifiers for grouping/aggregation mappings, including: #: #: - :py:`.c`: the output of :func:`.get_commodity_groups`. +#: - :py:`.t`: the output of :func:`.get_technology_groups`. groups = Keys( c="c::groups", + t="t::groups", ) diff --git a/message_ix_models/report/operator.py b/message_ix_models/report/operator.py index 1f4f2454ab..eeb66eafb6 100644 --- a/message_ix_models/report/operator.py +++ b/message_ix_models/report/operator.py @@ -12,7 +12,7 @@ Sequence, ) from functools import cache, reduce -from itertools import filterfalse, product +from itertools import chain, filterfalse, product from typing import TYPE_CHECKING, Any, Literal import genno @@ -234,6 +234,12 @@ def gwp_factors() -> "AnyQuantity": ) +def groups_to_selectors(groups: dict, dim: str, keys: Iterable[str], *key_args) -> dict: + return { + dim: list(chain(*[groups[dim][k] for k in sorted(set(keys) | set(key_args))])) + } + + def make_output_path(config: Mapping, name: "str | Path") -> "Path": """Return a path under the "output_dir" Path from the reporter configuration.""" return config["output_dir"].joinpath(name) @@ -419,7 +425,10 @@ def select_allow_empty( try: return genno.operator.select(qty, indexers=indexers, inverse=inverse, drop=drop) except KeyError: - return genno.Quantity([], coords={d: [] for d in qty.dims}) + # Dimensions to retain: all those of `qty` *except* those for which `indexers` + # contains a scalar. These dimensions would be dropped by the select() operator. + dims = {d for d in qty.dims if not isinstance(indexers.get(d, []), str)} + return type(qty)([], coords={d: [] for d in dims}, units=qty.units) def select_expand( diff --git a/message_ix_models/report/price.py b/message_ix_models/report/price.py new file mode 100644 index 0000000000..809cdfb298 --- /dev/null +++ b/message_ix_models/report/price.py @@ -0,0 +1,63 @@ +"""Report prices. + +.. todo:: Extend with the following: + + - Subtract PRICE_EMISSION from PRICE_COMMODITY to produce IAMC variables like + "Price|* Energy wo carbon price". This requires transforming the dimensions (e, t) + and units [currency] / [mass] on the former to (c, l) and [currency] / [energy] + (or other units) on the latter. + - Add the MESSAGE parameter ``tax_emission``. +""" + +from typing import TYPE_CHECKING + +from genno import Keys + +from . import util +from .util import IAMCConversion + +if TYPE_CHECKING: + from message_ix import Reporter + + from message_ix_models import Context + +K = Keys( + PC="PRICE_COMMODITY:n-c-l-y-h", + PE="PRICE_EMISSION:n-type_emission-type_tec-y", + carbon="carbon price:n-y", + c="commodity price:n-c-l-y", +) + +CONV = ( + IAMCConversion(base=K.c, var_parts=["Price", "l", "c"], unit="USD_2010 / GJ"), + IAMCConversion(base=K.carbon, var_parts=["Price|Carbon"], unit="USD_2010 / Mt"), +) + + +def callback(rep: "Reporter", context: "Context") -> None: + """Prepare reporting of prices.""" + # Add replacements for fully constructed variable names + util.REPLACE_VARS.update( + { + r"^(?:PRICE_COMMODITY\|)?(Price\|Final Energy\|Residential)": r"\1", + r"^(?:PRICE_COMMODITY\|)?(Price\|(Primary|Secondary) Energy)\|": ( + r"\1 w carbon price|" + ), + } + ) + + # Apply units that are not present in the scenario + rep.add(K.PC["units"], "apply_units", K.PC, "USD_2010 / kWa", sums=True) + rep.add(K.PE["units"], "apply_units", K.PE, "USD_2005 / Mt") + + # Prepare commodity prices: select only certain levels + idx = dict(h="year", l=["final", "primary", "secondary"]) + rep.add(K.c, "select", K.PC["units"], indexers=idx) + + # Prepare carbon prices: select and drop 2 dimensions + idx = dict(type_emission="TCE", type_tec="all") + rep.add(K.carbon, "select_allow_empty", K.PE["units"], indexers=idx, drop=True) + + # Convert to IAMC structure + for c in CONV: + c.add_tasks(rep) diff --git a/message_ix_models/report/sim.py b/message_ix_models/report/sim.py index 90a98cb91c..e78d65183a 100644 --- a/message_ix_models/report/sim.py +++ b/message_ix_models/report/sim.py @@ -213,7 +213,7 @@ def data_from_file(path: Path, *, name: str, dims: Sequence[str]) -> "AnyQuantit cols = list(dims) + ["Val", "Marginal", "Lower", "Upper", "Scale"] return genno.Quantity( - pd.read_csv(path, engine="pyarrow") + pd.read_csv(path, engine="pyarrow", na_values=["Eps"]) .set_axis(cols, axis=1) .set_index(cols[:-5])["Val"], name=name, diff --git a/message_ix_models/report/util.py b/message_ix_models/report/util.py index c91bc3b2cc..48b10c6d24 100644 --- a/message_ix_models/report/util.py +++ b/message_ix_models/report/util.py @@ -1,20 +1,17 @@ import logging -from collections.abc import Iterable +from collections import Counter +from collections.abc import Iterable, Mapping from dataclasses import dataclass, field from itertools import count -from typing import TYPE_CHECKING import pandas as pd from dask.core import quote -from genno import Key, Keys +from genno import Computer, Key, Keys from genno.compat.pyam.util import collapse as genno_collapse from genno.core.key import single_key from message_ix import Reporter from sdmx.model.v21 import Code -if TYPE_CHECKING: - from genno import Computer - log = logging.getLogger(__name__) @@ -77,9 +74,6 @@ } -_RENAME = {"n": "region", "nl": "region", "y": "year", "ya": "year", "yv": "year"} - - @dataclass class IAMCConversion: """Description of a conversion to IAMC data structure. @@ -98,6 +92,9 @@ class IAMCConversion: #: Exact unit string for output. unit: str + #: Explicit dimension renaming. + rename: Mapping[str, str] = field(default_factory=dict) + #: Dimension(s) to sum over. sums: list[str] = field(default_factory=list) @@ -156,12 +153,32 @@ def add_tasks(self, c: "Computer") -> None: k = Keys(base=self.base, glb=self.base + "glb") + # Common keyword arguments for genno.compat.pyam.iamc + args: dict = dict(rename=self.rename, unit=self.unit) + + # Populate rename + for d in set(self.base.dims) - set(self.var_parts): + if d in {"n", "nd", "nl", "no"}: + args["rename"].setdefault(d, "region") + elif d in {"y", "ya", "yv"}: + args["rename"].setdefault(d, "year") + + # Check rename arg + assert dict(region=1, year=1) == Counter(args["rename"].values()), ( + f"Expected 1 region and 1 year dimension; got {args['rename']}" + ) + + # Identify node/region dimension + dim_n = next(kv[0] for kv in args["rename"].items() if kv[1] == "region") + if self.GLB_zeros: # Quantity of zeros in the same shape as self.base, without an 'n' dimension - c.add(k.glb[0], "zeros_like", self.base, drop=["n"]) + c.add(k.glb[0], "zeros_like", self.base, drop=[dim_n]) # Add the 'n' dimension - c.add(k.glb[1], "expand_dims", k.glb[0], coords.n_glb) + if dim_n != "n": + c.add(f"{dim_n}::glb", lambda d: {dim_n: d["n"]}, coords.n_glb) + c.add(k.glb[1], "expand_dims", k.glb[0], f"{dim_n}::glb") # Add zeros to base data & update the base key for next steps c.add(k.base[0], "add", self.base, k.glb[1]) @@ -170,10 +187,10 @@ def add_tasks(self, c: "Computer") -> None: c.add(k.base[0], k.base) # Convert to target units - c.add(k.base[1], "convert_units", k.base[0], units=self.unit, sums=True) - - # Common keyword arguments for genno.compat.pyam.iamc - args: dict = dict(rename=_RENAME, unit=self.unit) + if self.unit is not None: + c.add(k.base[1], "convert_units", k.base[0], units=self.unit) + else: + c.add(k.base[1], k.base[0]) # Identify a `start` value that does not duplicate existing keys label = self.var_parts[0] @@ -187,6 +204,10 @@ def add_tasks(self, c: "Computer") -> None: for i, dims in enumerate( map(lambda s: s.split("-"), [""] + self.sums), start=start ): + k.sum = k.base[1].drop(*dims) + if k.sum != k.base[1]: + c.add(k.sum, "sum", k.base[1], dimensions=dims) + # Parts (string literals or dimension IDs) to concatenate into ‘variable’. # Exclude any summed dimensions from the expression. var_parts = [v for v in self.var_parts if v not in dims] @@ -200,7 +221,7 @@ def add_tasks(self, c: "Computer") -> None: c, args | dict( - base=k.base[1].drop(*dims), + base=k.sum, variable=f"{label} {i}", collapse=dict(callback=collapse, var=var_parts), ), @@ -208,7 +229,7 @@ def add_tasks(self, c: "Computer") -> None: keys.append(f"{label} {i}::iamc") # Concatenate each of `keys` into all::iamc - c.graph[all_iamc] += tuple(keys) + c.graph[all_iamc] = c.graph.pop(all_iamc, ("concat",)) + tuple(keys) def collapse(df: pd.DataFrame, var=[]) -> pd.DataFrame: @@ -344,3 +365,44 @@ def add_replacements(dim: str, codes: Iterable[Code]) -> None: pass else: REPLACE_DIMS[dim][f"{code.id.title()}$"] = label + + +# TODO Type c as (string) "Computer" once genno supports this +def store_write_ts(c: Computer, base_key: Key) -> "Key": + """Write `base_key` to CSV, XLSX, and/or both; and/or store on "scenario". + + If `base_key` is, for instance, "foo::iamc", this function adds the following keys: + + - "foo::iamc+all": both of: + + - "foo::iamc+file": both of: + + - "foo::iamc+csv": write the data in `base_key` to a file named :file:`foo.csv`. + - "foo::iamc+xlsx": write the data in `base_key` to a file named + :file:`foo.xlsx`. + + The files are created in a subdirectory using :func:`make_output_path`—that is, + including a path component given by the scenario URL. + + - "foo::iamc+store" store the data in `base_key` as time series data on the + scenario identified by the key "scenario". + """ + k = Key(base_key) + + file_keys = [] + for suffix in ("csv", "xlsx"): + # Create the path + path = c.add( + k[f"{suffix} path"], "make_output_path", "config", name=f"{k.name}.{suffix}" + ) + # Write `key` to the path + file_keys.append(c.add(k[suffix], "write_report", base_key, path)) + + # Write all files + c.add(k["file"], file_keys) + + # Store data on "scenario" + c.add(k["store"], "store_ts", "scenario", base_key) + + # Both write and store + return single_key(c.add(k["all"], [k["file"], k["store"]])) diff --git a/message_ix_models/tests/model/test_structure.py b/message_ix_models/tests/model/test_structure.py index d274750fe1..5b4b59ce96 100644 --- a/message_ix_models/tests/model/test_structure.py +++ b/message_ix_models/tests/model/test_structure.py @@ -13,6 +13,7 @@ generate_set_elements, get_codes, get_region_codes, + get_technology_groups, process_commodity_codes, process_units_anno, ) @@ -273,6 +274,23 @@ def test_generate_set_elements(colour_expr, expected): assert expected == set(map(str, data["technology"]["add"])) +def test_get_technology_groups() -> None: + # Function runs + result = get_technology_groups() + + # Top-level: single key "t" + assert {"t"} == set(result) + + # Next level: expecte number of groups + assert 524 == len(result["t"]) + + # Certain keys are present + assert "* * → secondary electr" in result["t"] + + # Keys contain expected members + assert ["h2_smr", "h2_smr_ccs"] == result["t"]["* gas → secondary hydrogen"] + + def test_process_units_anno(): # Prepare 2 codes: the parent has a units annotation, the child has none codes = as_codes({"foo": {"units": "kg"}, "bar": {"parent": "foo"}}) diff --git a/message_ix_models/tests/report/test_util.py b/message_ix_models/tests/report/test_util.py new file mode 100644 index 0000000000..8524d248bf --- /dev/null +++ b/message_ix_models/tests/report/test_util.py @@ -0,0 +1,81 @@ +import pytest +from genno import Key +from message_ix import Reporter + +from message_ix_models.report.key import all_iamc +from message_ix_models.report.util import IAMCConversion + +M = pytest.mark.xfail(raises=AssertionError) + +foo = Key("foo:nl-nd-yv-ya-a-b") + + +class TestIAMCConversion: + @pytest.fixture + def rep(self) -> Reporter: + r = Reporter() + r.add(all_iamc, "concat") + return r + + @pytest.mark.parametrize( + "var_parts, d_region, d_year", + [ + (["Foo", "nd", "ya", "a", "b"], "nl", "yv"), + (["Foo", "nd", "yv", "a", "b"], "nl", "ya"), + (["Foo", "nl", "ya", "a", "b"], "nd", "yv"), + (["Foo", "nl", "yv", "a", "b"], "nd", "ya"), + # + # Invalid arguments + # No dimension mapped to "region" + pytest.param(["Foo", "nd", "nl", "yv", "a", "b"], "", "", marks=M), + # No dimension mapped to "year" + pytest.param(["Foo", "nd", "ya", "yv", "a", "b"], "", "", marks=M), + # Both of the above combined + pytest.param(["Foo", "nd", "nl", "ya", "yv", "a", "b"], "", "", marks=M), + ], + ) + def test_add_tasks0( + self, rep: Reporter, var_parts: list[str], d_region: str, d_year: str + ) -> None: + """Test behaviour of :meth:`.IAMCConversion.add_tasks`. + + See https://github.com/iiasa/message-ix-models/issues/454. + """ + # IAMCConversion can be instantiated with these keyword arguments + c = IAMCConversion( + base=foo, var_parts=var_parts, unit="kg", sums=["a", "b", "a-b"] + ) + + keys_pre = set(rep.graph) + c.add_tasks(rep) + + # Set of all added keys + keys_post = set(rep.graph) + added = keys_post - keys_pre + assert 4 <= len(added) + + # Task for iamc::all has 4 input keys: base quantity + 3 sums + task_all = rep.graph[all_iamc] + assert 5 == len(task_all) + + for k in task_all[1:]: + # Retrieve the final key added by add_tasks(), and the task it refers to + task = rep.graph[k] + + # Description of the same task + desc = rep.describe(k) + + # The task is connected to the original key + assert f"- {foo!s}" in desc, desc + + # Retrieve the 'rename' keyword argument passed to partial(as_pyam, …) + rename = task[0].keywords["rename"] + + # `d_region` is mapped to the IAMC `region` dimension + assert "region" == rename[d_region] + + # `d_year` is mapped to the IAMC `year` dimension + assert "year" == rename[d_year] + + # No dimensions referenced in "var_parts" appear in `rename` + assert not (set(rename) & set(var_parts)) diff --git a/message_ix_models/tools/iamc.py b/message_ix_models/tools/iamc.py index 9ced84a56b..c0f23c5626 100644 --- a/message_ix_models/tools/iamc.py +++ b/message_ix_models/tools/iamc.py @@ -15,6 +15,7 @@ if TYPE_CHECKING: import pathlib + import numpy as np from genno.types import AnyQuantity __all__ = [ @@ -59,54 +60,62 @@ def compare( list of str A collection of messages describing differences between `left` and `right`. """ + prefix = "" result = [] matching = 0 _ignore = [re.compile(pattern) for pattern in ignore] - def record(message: str, condition: bool | None = True) -> None: + def record(message: str, condition: "bool | np.bool | None" = True) -> bool: + message = (f"{prefix} " if prefix else "") + message if not condition or any(p.match(message) for p in _ignore): - return + return False result.append(message) + return True def checks(df: pd.DataFrame): - nonlocal matching + nonlocal prefix, matching prefix = f"variable={df.variable.iloc[0]!r}:" if df.value_left.isna().all(): - record(f"{prefix} no left data") + record("no left data") return elif df.value_right.isna().all(): - record(f"{prefix} no right data") + record("no right data") return - tmp = df.eval("value_diff = value_right - value_left").eval( + tmp: pd.DataFrame = df.eval("value_diff = value_right - value_left").eval( # type: ignore [assignment,union-attr] "value_rel = value_diff / value_left" ) + # Entries in `left` with NA values or units na_left = tmp.isna()[["unit_left", "value_left"]] - if na_left.any(axis=None): - record(f"{prefix} {na_left.sum(axis=0).max()} missing left entries") - tmp = tmp[~na_left.any(axis=1)] + record( + f"{na_left.sum(axis=0).max()} missing left entries", + condition=na_left.any(axis=None), + ) + tmp = tmp[~na_left.any(axis=1)] # Omit these rows from further processing + + # Entries in `right` with NA values or units na_right = tmp.isna()[["unit_right", "value_right"]] - if na_right.any(axis=None): - record(f"{prefix} {na_right.sum(axis=0).max()} missing right entries") - tmp = tmp[~na_right.any(axis=1)] + record( + f"{na_right.sum(axis=0).max()} missing right entries", + condition=na_right.any(axis=None), + ) + tmp = tmp[~na_right.any(axis=1)] # Omit these rows from further processing units_left = set(tmp.unit_left.unique()) units_right = set(tmp.unit_right.unique()) record( condition=units_left != units_right, - message=f"{prefix} units mismatch: {units_left} != {units_right}", + message=f"units mismatch: {units_left} != {units_right}", ) N0 = len(df) - mask1 = tmp.query("abs(value_diff) > @atol") - record( - condition=bool(len(mask1)), - message=f"{prefix} {len(mask1)} of {N0} values with |diff| > {atol}", - ) + record(f"{len(mask1)} of {N0} values with |diff| > {atol}", bool(len(mask1))) + + # Increment number of matching rows matching += N0 - len(mask1) for (model, scenario), group_0 in left.merge( @@ -115,6 +124,7 @@ def checks(df: pd.DataFrame): on=["model", "scenario", "variable", "region", "year"], suffixes=("_left", "_right"), ).groupby(["model", "scenario"]): + prefix = "" if group_0.value_left.isna().all(): record(f"No left data for model={model!r}, scenario={scenario!r}") elif group_0.value_right.isna().all(): @@ -176,7 +186,7 @@ def _cl(dim: str) -> common.Codelist: group_units = group_data["UNIT"].unique() cl_variable.append( common.Code( - id=variable, + id=str(variable), annotations=[ v21.Annotation( id="preferred-unit-measure", text=", ".join(group_units) diff --git a/message_ix_models/util/__init__.py b/message_ix_models/util/__init__.py index dd21ec9784..1c5db232f7 100644 --- a/message_ix_models/util/__init__.py +++ b/message_ix_models/util/__init__.py @@ -3,6 +3,7 @@ from collections.abc import Collection, Iterable, Mapping, MutableMapping, Sequence from datetime import datetime from functools import partial, singledispatch +from hashlib import blake2s from itertools import count from pathlib import Path from typing import TYPE_CHECKING, Any, Literal, Protocol @@ -82,6 +83,7 @@ "same_node", "same_time", "show_versions", + "short_hash", "silence_log", "strip_par_data", ] @@ -781,6 +783,11 @@ def _(data: "MutableParameterData") -> "MutableParameterData": return data +def short_hash(value: Any, len: int = 3) -> str: + """Return a short (length `len`) hash of `value`.""" + return blake2s(str(value).encode()).hexdigest()[:len] + + def show_versions() -> str: """Output of :func:`ixmp.show_versions`, as a :class:`str`.""" from io import StringIO