Question: """     if keep_attrs is None:         keep_attrs = _get_keep_attrs(default=False)     if keep_attrs is True:         # keep the attributes of x, the second parameter, by default to         # be consistent with the `where` method of `DataArray` and `Dataset`        keep_attrs = lambda attrs, context: attrs[1]        keep_attrs = lambda attrs, context: getattr(x, "attrs", {})      # alignment for three arguments is complicated, so don't support it yet     return apply_ufunc(

with scalar as second argument fails with keepattrs=True fails with The workaround is to pass No response No response No response No response xarray 2022.3.0


Question: import numpy as npimport xarray as xrfrom . import parameterized, randn, requires_daskNDEGS = (2, 5, 20)NX = (10**2, 10**6)class Polyval:    def setup(self, *args, **kwargs):        self.xs = {nx: xr.DataArray(randn((nx,)), dims="x", name="x") for nx in NX}        self.coeffs = {            ndeg: xr.DataArray(                randn((ndeg,)), dims="degree", coords={"degree": np.arange(ndeg)}            )            for ndeg in NDEGS        }    @parameterized(["nx", "ndeg"], [NX, NDEGS])    def time_polyval(self, nx, ndeg):        x = self.xs[nx]        c = self.coeffs[ndeg]        xr.polyval(x, c).compute()    @parameterized(["nx", "ndeg"], [NX, NDEGS])    def peakmem_polyval(self, nx, ndeg):        x = self.xs[nx]        c = self.coeffs[ndeg]        xr.polyval(x, c).compute()class PolyvalDask(Polyval):    def setup(self, *args, **kwargs):        requires_dask()        super().setup(*args, **kwargs)        self.xs = {k: v.chunk({"x": 10000}) for k, v in self.xs.items()}diff --git a/xarray/core/computation.py b/xarray/core/computation.py-- a/xarray/core/computation.py++ b/xarray/core/computation.py

xr.polyval first arg requires name attribute I have some polynomial coefficients and want to evaluate them at some values using . As described in the docstring/docu I created a 1D coordinate DataArray and pass it to but it raises a KeyError (see example). I expected that the polynomial would be evaluated at the given points. No response I assume that the "standard" workflow is to obtain the argument from an existing DataArrays coordinate, where the name would be correctly set already. However, that is not clear from the description, and also prevents my "manual" workflow. It could be that the problem will be solved by replacing the coord DataArray argument by an explicit Index in the future. <detailsINSTALLED VERSIONS commit: None python: 3.9.10 (main, Mar 15 2022, 15:56:56) [GCC 7.5.0] python-bits: 64 OS: Linux OS-release: 3.10.0-1160.49.1.el7.x8664 machine: x8664 processor: x8664 byteorder: little LCALL: None LANG: enUS.UTF-8 LOCALE: ('enUS', 'UTF-8') libhdf5: 1.12.0 libnetcdf: 4.7.4 xarray: 2022.3.0 pandas: 1.4.2 numpy: 1.22.3 scipy: None netCDF4: 1.5.8 pydap: None h5netcdf: None h5py: None Nio: None zarr: None cftime: 1.6.0 nctimeaxis: None PseudoNetCDF: None rasterio: None cfgrib: None iris: None bottleneck: None dask: None distributed: None matplotlib: 3.5.1 cartopy: 0.20.2 seaborn: None numbagg: None fsspec: None cupy: None pint: None sparse: None setuptools: 58.1.0 pip: 22.0.4 conda: None pytest: None IPython: 8.2.0 sphinx: None </details


Question: Iterable,     Mapping,     Sequence,    overload, )  import numpy as np  from . import dtypes, duck_array_ops, utils from .alignment import align, deep_alignfrom .common import zeros_likefrom .duck_array_ops import datetime_to_numeric from .indexes import Index, filter_indexes_from_coords from .merge import merge_attrs, merge_coordinates_without_align from .options import OPTIONS, _get_keep_attrs


Question: )  def polyval(coord, coeffs, degree_dim="degree"):@overloaddef polyval(coord: DataArray, coeffs: DataArray, degree_dim: Hashable) -> DataArray:    ...@overloaddef polyval(coord: T_Xarray, coeffs: Dataset, degree_dim: Hashable) -> Dataset:    ...@overloaddef polyval(coord: Dataset, coeffs: T_Xarray, degree_dim: Hashable) -> Dataset:    ...def polyval(    coord: T_Xarray, coeffs: T_Xarray, degree_dim: Hashable = "degree") -> T_Xarray:     """Evaluate a polynomial at specific values      Parameters     ----------    coord : DataArray        The 1D coordinate along which to evaluate the polynomial.    coeffs : DataArray        Coefficients of the polynomials.    degree_dim : str, default: "degree"    coord : DataArray or Dataset        Values at which to evaluate the polynomial.    coeffs : DataArray or Dataset        Coefficients of the polynomial.    degree_dim : Hashable, default: "degree"         Name of the polynomial degree dimension in `coeffs`.     Returns    -------    DataArray or Dataset        Evaluated polynomial.     See Also     --------     xarray.DataArray.polyfit    numpy.polyval    numpy.polynomial.polynomial.polyval     """    from .dataarray import DataArray    from .missing import get_clean_interp_index     x = get_clean_interp_index(coord, coord.name, strict=False)    if degree_dim not in coeffs._indexes:        raise ValueError(            f"Dimension `{degree_dim}` should be a coordinate variable with labels."        )    if not np.issubdtype(coeffs[degree_dim].dtype, int):        raise ValueError(            f"Dimension `{degree_dim}` should be of integer dtype. Received {coeffs[degree_dim].dtype} instead."        )    max_deg = coeffs[degree_dim].max().item()    coeffs = coeffs.reindex(        {degree_dim: np.arange(max_deg + 1)}, fill_value=0, copy=False    )    coord = _ensure_numeric(coord)    # using Horner's method    # https://en.wikipedia.org/wiki/Horner%27s_method    res = coeffs.isel({degree_dim: max_deg}, drop=True) + zeros_like(coord)    for deg in range(max_deg - 1, -1, -1):        res *= coord        res += coeffs.isel({degree_dim: deg}, drop=True)     deg_coord = coeffs[degree_dim]    return res     lhs = DataArray(        np.vander(x, int(deg_coord.max()) + 1),        dims=(coord.name, degree_dim),        coords={coord.name: coord, degree_dim: np.arange(deg_coord.max() + 1)[::-1]},    )    return (lhs * coeffs).sum(degree_dim)def _ensure_numeric(data: T_Xarray) -> T_Xarray:    """Converts all datetime64 variables to float64    Parameters    ----------    data : DataArray or Dataset        Variables with possible datetime dtypes.    Returns    -------    DataArray or Dataset        Variables with datetime64 dtypes converted to float64.    """    from .dataset import Dataset    def to_floatable(x: DataArray) -> DataArray:        if x.dtype.kind in "mM":            return x.copy(                data=datetime_to_numeric(                    x.data,                    offset=np.datetime64("1970-01-01"),                    datetime_unit="ns",                ),            )        return x    if isinstance(data, Dataset):        return data.map(to_floatable)    else:        return to_floatable(data)   def _calc_idxminmax(


Question: pd.to_timedelta(flat_num_dates.max(), delta) + ref_date      # To avoid integer overflow when converting to nanosecond units for integer    # dtypes smaller than np.int64 cast all integer-dtype arrays to np.int64    # (GH 2002).    if flat_num_dates.dtype.kind == "i":    # dtypes smaller than np.int64 cast all integer and unsigned integer dtype    # arrays to np.int64 (GH 2002, GH 6589).  Note this is safe even in the case    # of np.uint64 values, because any np.uint64 value that would lead to    # overflow when converting to np.int64 would not be representable with a    # timedelta64 value, and therefore would raise an error in the lines above.    if flat_num_dates.dtype.kind in "iu":         flat_num_dates = flat_num_dates.astype(np.int64)      # Cast input ordinals to integers of nanoseconds because pd.to_timedelta

xarray improperly decodes times from a NetCDF when it is a uint improperly decodes times from a NetCDF when it is a . The generates a NetCDF file with the right time ('goodtime') and the wrong time ('time') (use ) to be properly decoded (see ). [X] Minimal example — the example is as focused as reasonably possible to demonstrate the underlying issue in xarray. [X] Complete example — the example is self-contained, including all data and the text of any traceback. , returning the result. [X] New issue — a search of GitHub Issues suggests this is not a duplicate.


Question: pd.to_timedelta(flat_num_dates.max(), delta) + ref_date      # To avoid integer overflow when converting to nanosecond units for integer    # dtypes smaller than np.int64 cast all integer-dtype arrays to np.int64    # (GH 2002).    if flat_num_dates.dtype.kind == "i":    # dtypes smaller than np.int64 cast all integer and unsigned integer dtype    # arrays to np.int64 (GH 2002, GH 6589).  Note this is safe even in the case    # of np.uint64 values, because any np.uint64 value that would lead to    # overflow when converting to np.int64 would not be representable with a    # timedelta64 value, and therefore would raise an error in the lines above.    if flat_num_dates.dtype.kind in "iu":         flat_num_dates = flat_num_dates.astype(np.int64)      # Cast input ordinals to integers of nanoseconds because pd.to_timedelta

No response <detailsINSTALLED VERSIONS commit: None python: 3.8.13 (default, Mar 25 2022, 06:04:10) [GCC 10.3.0] python-bits: 64 OS: Linux OS-release: 3.10.0-1160.62.1.el7.x8664 machine: x8664 processor: x8664 byteorder: little LCALL: None LANG: enUS.UTF-8 LOCALE: ('enUS', 'UTF-8') libhdf5: 1.10.6 libnetcdf: 4.8.0 xarray: 2022.3.0 pandas: 1.4.2 numpy: 1.22.3 scipy: 1.7.0 netCDF4: 1.5.7 pydap: None h5netcdf: 1.0.0 h5py: 3.3.0 Nio: None zarr: 2.11.3 cftime: 1.6.0 nctimeaxis: None PseudoNetCDF: None rasterio: None cfgrib: 0.9.10.1 iris: None bottleneck: 1.3.4 dask: 2022.04.1 distributed: 2022.4.1 matplotlib: 3.5.1 cartopy: None seaborn: 0.11.2 numbagg: None fsspec: 2021.06.1 cupy: None pint: 0.19.1 sparse: None setuptools: 62.1.0 pip: 22.0.4 conda: 4.12.0 pytest: 7.1.1 IPython: 8.2.0 sphinx: 4.5.0 </details


Question: from .dataset import Dataset      def to_floatable(x: DataArray) -> DataArray:        if x.dtype.kind in "mM":        if x.dtype.kind == "M":            # datetimes             return x.copy(                 data=datetime_to_numeric(                     x.data,

with timedelta64 coordinates produces wrong results I'm not sure if this is a bug or an expected breaking change, but I'm not able to reproduce the results generated by using a timedelta64 coordinate. The results are correct in , whereas they are wrong in the latest unreleased version (, ). Both the stable and latest functions should return the same results. [X] Minimal example — the example is as focused as reasonably possible to demonstrate the underlying issue in xarray. [X] Complete example — the example is self-contained, including all data and the text of any traceback. , returning the result. [X] New issue — a search of GitHub Issues suggests this is not a duplicate. No response <detailsINSTALLED VERSIONS commit: None python: 3.10.4 (main, Mar 24 2022, 17:43:32) [Clang 12.0.1 ] python-bits: 64 OS: Darwin OS-release: 21.4.0 machine: x8664 processor: i386 byteorder: little LCALL: None LANG: None LOCALE: (None, 'UTF-8') libhdf5: None libnetcdf: None xarray: 2022.3.0 or 2022.3.1.dev102+g6bb2b855 pandas: 1.4.2 numpy: 1.22.3 scipy: 1.8.0 netCDF4: None pydap: None h5netcdf: None h5py: None Nio: None zarr: 2.11.3 cftime: None nctime_axis: None PseudoNetCDF: None rasterio: 1.2.10 cfgrib: None iris: None bottleneck: None dask: 2022.05.0 distributed: 2022.5.0 matplotlib: None cartopy: None seaborn: None numbagg: None fsspec: 2022.3.0 cupy: None pint: None sparse: None setuptools: 62.2.0 pip: 22.1 conda: None pytest: 7.1.2 IPython: None sphinx: None </details


Question: datetime_unit="ns",                 ),             )<ins>        elif x.dtype.kind == "m":            # timedeltas            return x.astype(float)</ins>         return x      if isinstance(data, Dataset):


Question: # using Horner's method     # https://en.wikipedia.org/wiki/Horner%27s_method    res = coeffs.isel({degree_dim: max_deg}, drop=True) + zeros_like(coord)    res = zeros_like(coord) + coeffs.isel({degree_dim: max_deg}, drop=True)     for deg in range(max_deg - 1, -1, -1):         res *= coord         res += coeffs.isel({degree_dim: deg}, drop=True)

returns objects with different dimension order I noticed that the dimension order of the object returned by the latest (, unreleased) is different compared to . For example, the following code returns different results. Is this the expected behaviour? If yes, is it worth mentioning this change in what's new/breaking changes? cc:


Question: chunks: dict[Any, tuple[int, ...]] = {}     for v in variables:        if hasattr(v.data, "chunks"):        if hasattr(v._data, "chunks"):             for dim, c in v.chunksizes.items():                 if dim in chunks and c != chunks[dim]:                     raise ValueError(

Accessing chunks on zarr backed xarray seems to load entire array into memory When running the following example it appears the entire dataset is loaded into memory when accessing the attribute: According to accessing the chunks attribute should simply inspect the attribute on the underlying DataArrays. No response No response <detailsINSTALLED VERSIONS commit: None python: 3.9.12 (main, Mar 24 2022, 23:24:38) [Clang 12.0.1 ] python-bits: 64 OS: Darwin OS-release: 21.2.0 machine: arm64 processor: arm byteorder: little LCALL: None LANG: enUS.UTF-8 LOCALE: ('enUS', 'UTF-8') libhdf5: None libnetcdf: None xarray: 2022.3.0 pandas: 1.4.2 numpy: 1.21.2 scipy: 1.8.0 netCDF4: None pydap: None h5netcdf: None h5py: None Nio: None zarr: 2.8.1 cftime: None nctime_axis: None PseudoNetCDF: None rasterio: None cfgrib: None iris: None bottleneck: 1.3.4 dask: 2022.04.0 distributed: 2022.4.0 matplotlib: 3.4.3 cartopy: None seaborn: None numbagg: None fsspec: 2022.3.0 cupy: None pint: None sparse: None setuptools: 62.0.0 pip: 22.0.4 conda: None pytest: 7.1.1 IPython: 8.2.0 sphinx: None </details


Question: # TODO legacy attribute         self.window_labels = self.obj[self.dim[0]]     def __iter__(self) -> Iterator[tuple[RollingKey, DataArray]]:    def __iter__(self) -> Iterator[tuple[DataArray, DataArray]]:         if self.ndim > 1:             raise ValueError("__iter__ is only supported for 1d-rolling")        stops = np.arange(1, len(self.window_labels) + 1)        starts = stops - int(self.window[0])        starts[: int(self.window[0])] = 0        dim0 = self.dim[0]        window0 = int(self.window[0])        offset = (window0 + 1) // 2 if self.center[0] else 1        stops = np.arange(offset, self.obj.sizes[dim0] + offset)        starts = stops - window0        starts[: window0 - offset] = 0         for (label, start, stop) in zip(self.window_labels, starts, stops):            window = self.obj.isel({self.dim[0]: slice(start, stop)})            window = self.obj.isel({dim0: slice(start, stop)})             counts = window.count(dim=self.dim[0])            counts = window.count(dim=dim0)             window = window.where(counts >= self.min_periods)              yield (label, window)

"center" kwarg ignored when manually iterating over DataArrayRolling <div type='discussions-op-text'<supOriginally posted by ckingdon95 June 29, 2022</supHello, I am trying to manually iterate over a DataArrayRolling object, as described in the documentation. I am confused why the following two code chunks do not produce the same sequence of values. I would like to be able to manually iterate over a DataArrayRolling object, and still be given center-justified windows. Is there a way to do this? This returns the following values, as expected: Whereas when I do it manually, it is not equivalent: This returns Is this an issue with the window iterator? If it is not an issue, then is there a way for me to get the center-justified windows in the manual iteration? </div


Question: from __future__ import annotations <ins>import warnings</ins> from contextlib import contextmanager from typing import TYPE_CHECKING, Any, Hashable, Iterator, Mapping, Sequence, cast

Dropping a MultiIndex variable raises an error after explicit indexes refactor With the latest released version of Xarray, it is possible to delete all variables corresponding to a MultiIndex by simply deleting the name of the MultiIndex. After the explicit indexes refactor (i.e,. using the "main" development branch) this now raises error about how this would "corrupt" index state. This comes up when using and and possibly some other methods. This is not hard to work around, but we may want to consider this bug a blocker for the next Xarray release. I found the issue surfaced in several projects when attempting to use the new version of Xarray inside Google's codebase. CC in case you have any thoughts to share. For now, we should preserve the behavior of deleting the variables corresponding to MultiIndex levels, but should issue a deprecation warning encouraging users to explicitly delete everything.


Question: from __future__ import annotations <ins>import warnings</ins> from contextlib import contextmanager from typing import TYPE_CHECKING, Any, Hashable, Iterator, Mapping, Sequence, cast

No response <detailsINSTALLED VERSIONS commit: python: 3.10.4 (main, Mar 24 2022, 17:42:03) [Clang 12.0.1 ] python-bits: 64 OS: Darwin OS-release: 21.4.0 machine: arm64 processor: arm byteorder: little LCALL: None LANG: enUS.UTF-8 LOCALE: ('enUS', 'UTF-8') libhdf5: 1.12.1 libnetcdf: 4.8.1 xarray: 0.18.3.dev137+g96c56836 pandas: 1.4.2 numpy: 1.22.3 scipy: 1.8.0 netCDF4: 1.5.8 pydap: None h5netcdf: None h5py: None Nio: None zarr: 2.11.3 cftime: 1.6.0 nctimeaxis: None PseudoNetCDF: None rasterio: None cfgrib: None iris: None bottleneck: None dask: 2022.04.1 distributed: 2022.4.1 matplotlib: None cartopy: None seaborn: None numbagg: None fsspec: 2022.3.0 cupy: None pint: None sparse: None setuptools: 62.1.0 pip: 22.0.4 conda: None pytest: 7.1.1 IPython: 8.2.0 sphinx: None </details


Question: import pandas as pd  from . import formattingfrom .indexes import Index, Indexes, assert_no_index_corruptedfrom .indexes import Index, Indexes, PandasMultiIndex, assert_no_index_corrupted from .merge import merge_coordinates_without_align, merge_coords from .utils import Frozen, ReprObject from .variable import Variable, calculate_dimensions


Question: import pandas as pd  from . import formattingfrom .indexes import Index, Indexes, assert_no_index_corruptedfrom .indexes import Index, Indexes, PandasMultiIndex, assert_no_index_corrupted from .merge import merge_coordinates_without_align, merge_coords from .utils import Frozen, ReprObject from .variable import Variable, calculate_dimensions


Question: def _update_coords(self, coords, indexes):         raise NotImplementedError() <ins>    def _maybe_drop_multiindex_coords(self, coords):        raise NotImplementedError()</ins>     def __iter__(self) -> Iterator[Hashable]:         # needs to be in the same order as the dataset variables         for k in self.variables:


Question: def _update_coords(self, coords, indexes):         raise NotImplementedError() <ins>    def _maybe_drop_multiindex_coords(self, coords):        raise NotImplementedError()</ins>     def __iter__(self) -> Iterator[Hashable]:         # needs to be in the same order as the dataset variables         for k in self.variables:


Question: def update(self, other: Mapping[Any, Any]) -> None:         other_vars = getattr(other, "variables", other)<ins>        self._maybe_drop_multiindex_coords(set(other_vars))</ins>         coords, indexes = merge_coords(             [self.variables, other_vars], priority_arg=1, indexes=self.xindexes         )


Question: def update(self, other: Mapping[Any, Any]) -> None:         other_vars = getattr(other, "variables", other)<ins>        self._maybe_drop_multiindex_coords(set(other_vars))</ins>         coords, indexes = merge_coords(             [self.variables, other_vars], priority_arg=1, indexes=self.xindexes         )


Question: original_indexes.update(indexes)         self._data._indexes = original_indexes <ins>    def _maybe_drop_multiindex_coords(self, coords: set[Hashable]) -> None:        """Drops variables in coords, and any associated variables as well."""        assert self._data.xindexes is not None        variables, indexes = drop_coords(            coords, self._data._variables, self._data.xindexes        )        self._data._variables = variables        self._data._indexes = indexes</ins>     def __delitem__(self, key: Hashable) -> None:         if key in self:             del self._data[key]


Question: original_indexes.update(indexes)         self._data._indexes = original_indexes <ins>    def _maybe_drop_multiindex_coords(self, coords: set[Hashable]) -> None:        """Drops variables in coords, and any associated variables as well."""        assert self._data.xindexes is not None        variables, indexes = drop_coords(            coords, self._data._variables, self._data.xindexes        )        self._data._variables = variables        self._data._indexes = indexes</ins>     def __delitem__(self, key: Hashable) -> None:         if key in self:             del self._data[key]


Question: original_indexes.update(indexes)         self._data._indexes = original_indexes <ins>    def _maybe_drop_multiindex_coords(self, coords: set[Hashable]) -> None:        """Drops variables in coords, and any associated variables as well."""        variables, indexes = drop_coords(            coords, self._data._coords, self._data.xindexes        )        self._data._coords = variables        self._data._indexes = indexes</ins>     @property     def variables(self):         return Frozen(self._data._coords)


Question: original_indexes.update(indexes)         self._data._indexes = original_indexes <ins>    def _maybe_drop_multiindex_coords(self, coords: set[Hashable]) -> None:        """Drops variables in coords, and any associated variables as well."""        variables, indexes = drop_coords(            coords, self._data._coords, self._data.xindexes        )        self._data._coords = variables        self._data._indexes = indexes</ins>     @property     def variables(self):         return Frozen(self._data._coords)


Question: return self._data._ipython_key_completions_()  def drop_coords(    coords_to_drop: set[Hashable], variables, indexes: Indexes) -> tuple[dict, dict]:    """Drop index variables associated with variables in coords_to_drop."""    # Only warn when we're dropping the dimension with the multi-indexed coordinate    # If asked to drop a subset of the levels in a multi-index, we raise an error    # later but skip the warning here.    new_variables = dict(variables.copy())    new_indexes = dict(indexes.copy())    for key in coords_to_drop & set(indexes):        maybe_midx = indexes[key]        idx_coord_names = set(indexes.get_all_coords(key))        if (            isinstance(maybe_midx, PandasMultiIndex)            and key == maybe_midx.dim            and (idx_coord_names - coords_to_drop)        ):            warnings.warn(                f"Updating MultiIndexed coordinate {key!r} would corrupt indices for "                f"other variables: {list(maybe_midx.index.names)!r}. "                f"This will raise an error in the future. Use `.drop_vars({idx_coord_names!r})` before "                "assigning new coordinate values.",                DeprecationWarning,                stacklevel=4,            )            for k in idx_coord_names:                del new_variables[k]                del new_indexes[k]    return new_variables, new_indexes def assert_coordinate_consistent(     obj: DataArray | Dataset, coords: Mapping[Any, Variable] ) -> None:diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py-- a/xarray/core/dataset.py++ b/xarray/core/dataset.py


Question: return self._data._ipython_key_completions_()  def drop_coords(    coords_to_drop: set[Hashable], variables, indexes: Indexes) -> tuple[dict, dict]:    """Drop index variables associated with variables in coords_to_drop."""    # Only warn when we're dropping the dimension with the multi-indexed coordinate    # If asked to drop a subset of the levels in a multi-index, we raise an error    # later but skip the warning here.    new_variables = dict(variables.copy())    new_indexes = dict(indexes.copy())    for key in coords_to_drop & set(indexes):        maybe_midx = indexes[key]        idx_coord_names = set(indexes.get_all_coords(key))        if (            isinstance(maybe_midx, PandasMultiIndex)            and key == maybe_midx.dim            and (idx_coord_names - coords_to_drop)        ):            warnings.warn(                f"Updating MultiIndexed coordinate {key!r} would corrupt indices for "                f"other variables: {list(maybe_midx.index.names)!r}. "                f"This will raise an error in the future. Use `.drop_vars({idx_coord_names!r})` before "                "assigning new coordinate values.",                DeprecationWarning,                stacklevel=4,            )            for k in idx_coord_names:                del new_variables[k]                del new_indexes[k]    return new_variables, new_indexes def assert_coordinate_consistent(     obj: DataArray | Dataset, coords: Mapping[Any, Variable] ) -> None:diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py-- a/xarray/core/dataset.py++ b/xarray/core/dataset.py


Question: data = self.copy()         # do all calculations first...         results: CoercibleMapping = data._calc_assign_results(variables)        data.coords._maybe_drop_multiindex_coords(set(results.keys()))         # ... and then assign         data.update(results)         return datadiff --git a/xarray/core/indexes.py b/xarray/core/indexes.py-- a/xarray/core/indexes.py++ b/xarray/core/indexes.py


Question: data = self.copy()         # do all calculations first...         results: CoercibleMapping = data._calc_assign_results(variables)        data.coords._maybe_drop_multiindex_coords(set(results.keys()))         # ... and then assign         data.update(results)         return datadiff --git a/xarray/core/indexes.py b/xarray/core/indexes.py-- a/xarray/core/indexes.py++ b/xarray/core/indexes.py


Question: return Frozen(self._dims) <ins>    def copy(self):        return type(self)(dict(self._indexes), dict(self._variables))</ins>     def get_unique(self) -> list[T_PandasOrXarrayIndex]:         """Return a list of unique indexes, preserving order."""


Question: return Frozen(self._dims) <ins>    def copy(self):        return type(self)(dict(self._indexes), dict(self._variables))</ins>     def get_unique(self) -> list[T_PandasOrXarrayIndex]:         """Return a list of unique indexes, preserving order."""


Question: if name in ["sum", "prod"]:                 kwargs.pop("min_count", None)             func = getattr(np, name)            if hasattr(values, "__array_namespace__"):                xp = values.__array_namespace__()                func = getattr(xp, name)            else:                func = getattr(np, name)          try:             with warnings.catch_warnings():diff --git a/xarray/core/indexing.py b/xarray/core/indexing.py-- a/xarray/core/indexing.py++ b/xarray/core/indexing.py

Use pytorch as backend for xarrays I would be interested in using pytorch as a backend for xarrays - because: a) pytorch is very similar to numpy - so the conceptual overhead is small b) [most helpful] enable having a GPU as the underlying hardware for compute - which would provide non-trivial speed up c) it would allow seamless integration with deep-learning algorithms and techniques Any thoughts on what the interest for such a feature might be ? I would be open to implementing parts of it - so any suggestions on where I could start ? Thanks


Question: return DaskIndexingAdapter(array)     if hasattr(array, "__array_function__"):         return NdArrayLikeIndexingAdapter(array)<ins>    if hasattr(array, "__array_namespace__"):        return ArrayApiIndexingAdapter(array)</ins>      raise TypeError(f"Invalid array type: {type(array)}")


Question: self.array = array  class ArrayApiIndexingAdapter(ExplicitlyIndexedNDArrayMixin):    """Wrap an array API array to use explicit indexing."""    __slots__ = ("array",)    def __init__(self, array):        if not hasattr(array, "__array_namespace__"):            raise TypeError(                "ArrayApiIndexingAdapter must wrap an object that "                "implements the __array_namespace__ protocol"            )        self.array = array    def __getitem__(self, key):        if isinstance(key, BasicIndexer):            return self.array[key.tuple]        elif isinstance(key, OuterIndexer):            # manual orthogonal indexing (implemented like DaskIndexingAdapter)            key = key.tuple            value = self.array            for axis, subkey in reversed(list(enumerate(key))):                value = value[(slice(None),) * axis + (subkey, Ellipsis)]            return value        else:            if isinstance(key, VectorizedIndexer):                raise TypeError("Vectorized indexing is not supported")            else:                raise TypeError(f"Unrecognized indexer: {key}")    def __setitem__(self, key, value):        if isinstance(key, (BasicIndexer, OuterIndexer)):            self.array[key.tuple] = value        else:            if isinstance(key, VectorizedIndexer):                raise TypeError("Vectorized indexing is not supported")            else:                raise TypeError(f"Unrecognized indexer: {key}")    def transpose(self, order):        xp = self.array.__array_namespace__()        return xp.permute_dims(self.array, order) class DaskIndexingAdapter(ExplicitlyIndexedNDArrayMixin):     """Wrap a dask array to support explicit indexing.""" diff --git a/xarray/core/utils.py b/xarray/core/utils.py-- a/xarray/core/utils.py++ b/xarray/core/utils.py


Question: hasattr(value, "ndim")         and hasattr(value, "shape")         and hasattr(value, "dtype")        and hasattr(value, "__array_function__")        and hasattr(value, "__array_ufunc__")        and (            (hasattr(value, "__array_function__") and hasattr(value, "__array_ufunc__"))            or hasattr(value, "__array_namespace__")        )     )


Question: or not (             isinstance(value, (Iterable,) + NON_NUMPY_SUPPORTED_ARRAY_TYPES)             or hasattr(value, "__array_function__")            or hasattr(value, "__array_namespace__")         )     ) diff --git a/xarray/core/variable.py b/xarray/core/variable.py-- a/xarray/core/variable.py++ b/xarray/core/variable.py


Question: else:             data = np.asarray(data)     if not isinstance(data, np.ndarray) and hasattr(data, "__array_function__"):    if not isinstance(data, np.ndarray) and (        hasattr(data, "__array_function__") or hasattr(data, "__array_namespace__")    ):         return data      # validate whether the data is valid data types.


Question:
return f"
{escape(repr(self))}
"         return formatting_html.array_repr(self)     def __format__(self: Any, format_spec: str) -> str:        # we use numpy: scalars will print fine and arrays will raise        return self.values.__format__(format_spec)    def __format__(self: Any, format_spec: str = "") -> str:        if format_spec != "":            if self.shape == ():                # Scalar values might be ok use format_spec with instead of repr:                return self.data.__format__(format_spec)            else:                # TODO: If it's an array the formatting.array_repr(self) should                # take format_spec as an input. If we'd only use self.data we                # lose all the information about coords for example which is                # important information:                raise NotImplementedError(                    "Using format_spec is only supported"                    f" when shape is (). Got shape = {self.shape}."                )        else:            return self.__repr__()      def _iter(self: Any) -> Iterator[Any]:         for n in range(len(self)):

RuntimeError when formatting sparse-backed DataArray in f-string On upgrading from xarray 2022.3.0 to 2022.6.0, f-string formatting of sparse-backed DataArray raises an exception. Code does not error, or A breaking change is listed in the section of the docs.[X] Minimal example — the example is as focused as reasonably possible to demonstrate the underlying issue in xarray. [X] Complete example — the example is self-contained, including all data and the text of any traceback. , returning the result. [X] New issue — a search of GitHub Issues suggests this is not a duplicate. Along with the versions below, I have confirmed the error occurs with both sparse 0.12 and sparse 0.13.


Question:
return f"
{escape(repr(self))}
"         return formatting_html.array_repr(self)     def __format__(self: Any, format_spec: str) -> str:        # we use numpy: scalars will print fine and arrays will raise        return self.values.__format__(format_spec)    def __format__(self: Any, format_spec: str = "") -> str:        if format_spec != "":            if self.shape == ():                # Scalar values might be ok use format_spec with instead of repr:                return self.data.__format__(format_spec)            else:                # TODO: If it's an array the formatting.array_repr(self) should                # take format_spec as an input. If we'd only use self.data we                # lose all the information about coords for example which is                # important information:                raise NotImplementedError(                    "Using format_spec is only supported"                    f" when shape is (). Got shape = {self.shape}."                )        else:            return self.__repr__()      def _iter(self: Any) -> Iterator[Any]:         for n in range(len(self)):

<detailsINSTALLED VERSIONS commit: None python: 3.10.4 (main, Jun 29 2022, 12:14:53) [GCC 11.2.0] python-bits: 64 OS: Linux OS-release: 5.15.0-41-generic machine: x8664 processor: x8664 byteorder: little LCALL: None LANG: enCA.UTF-8 LOCALE: ('enCA', 'UTF-8') libhdf5: 1.10.7 libnetcdf: 4.8.1 xarray: 2022.6.0 pandas: 1.4.2 numpy: 1.22.4 scipy: 1.8.0 netCDF4: 1.5.8 pydap: None h5netcdf: 0.12.0 h5py: 3.6.0 Nio: None zarr: None cftime: 1.5.2 nctimeaxis: None PseudoNetCDF: None rasterio: None cfgrib: None iris: None bottleneck: 1.3.2 dask: 2022.01.0+dfsg distributed: 2022.01.0+ds.1 matplotlib: 3.5.1 cartopy: 0.20.2 seaborn: 0.11.2 numbagg: None fsspec: 2022.01.0 cupy: None pint: 0.18 sparse: 0.13.0 flox: None numpy_groupies: None setuptools: 62.1.0 pip: 22.0.2 conda: None pytest: 6.2.5 IPython: 7.31.1 sphinx: 4.5.0 </details


Question: if obj_idx is not None:                     for name, var in self.aligned_index_vars[key].items():                         new_indexes[name] = aligned_idx                        new_variables[name] = var                        new_variables[name] = var.copy()              objects[i + 1] = obj._overwrite_indexes(new_indexes, new_variables)

Testing DataArray equality using built-in '==' operator leads to mutilated dictionary In previous versions of xarray, testing numerical equivalence of two DataArrays was possible using the built-in operator '==' and without side affects. Now in version 2022.6.0, when one DataArray lacks an attribute that the other DataArray has, the DataArray with the attribute is mutilated during comparison leading to an empty attrs dictionary. DataArray1 == DataArray2 should not have side affects. [X] Minimal example — the example is as focused as reasonably possible to demonstrate the underlying issue in xarray. [X] Complete example — the example is self-contained, including all data and the text of any traceback. , returning the result. [X] New issue — a search of GitHub Issues suggests this is not a duplicate. No response <detailsINSTALLED VERSIONS commit: None python: 3.9.13 (tags/v3.9.13:, May 17 2022, 16:36:42) [MSC v.1929 64 bit (AMD64)] python-bits: 64 OS: Windows OS-release: 10 machine: AMD64 processor: Intel64 Family 6 Model 85 Stepping 4, GenuineIntel byteorder: little LCALL: None LANG: None LOCALE: ('EnglishUnited States', '1252') libhdf5: None libnetcdf: None xarray: 2022.6.0 pandas: 1.4.3 numpy: 1.23.1 scipy: 1.9.0 netCDF4: None pydap: None h5netcdf: None h5py: None Nio: None zarr: None cftime: None nctimeaxis: None PseudoNetCDF: None rasterio: None cfgrib: None iris: None bottleneck: None dask: None distributed: None matplotlib: 3.5.2 cartopy: None seaborn: None numbagg: None fsspec: None cupy: None pint: None sparse: None flox: None numpygroupies: None setuptools: 63.2.0 pip: 22.2.1 conda: None pytest: 7.1.2 IPython: 8.4.0 sphinx: None </details


Question: if obj_idx is not None:                 for name, var in index_vars.items():                     new_indexes[name] = aligned_idx                    new_variables[name] = var                    new_variables[name] = var.copy()          return new_indexes, new_variables


Question: def _decode_datetime_with_cftime(num_dates, units, calendar):     if cftime is None:         raise ModuleNotFoundError("No module named 'cftime'")    return np.asarray(        cftime.num2date(num_dates, units, calendar, only_use_cftime_datetimes=True)    )    if num_dates.size > 0:        return np.asarray(            cftime.num2date(num_dates, units, calendar, only_use_cftime_datetimes=True)        )    else:        return np.array([], dtype=object)   def _decode_datetime_with_pandas(flat_num_dates, units, calendar):

Cannot open NetCDF file if dimension with time coordinate has length 0 ( when decoding CF datetime) If a data set has a zero-sized coordinate that is a time index, reading fails. A is triggered when xarray tries to decode the array, as shown below:


Question: with warnings.catch_warnings():         warnings.filterwarnings("ignore", "invalid value encountered", RuntimeWarning)        pd.to_timedelta(flat_num_dates.min(), delta) + ref_date        pd.to_timedelta(flat_num_dates.max(), delta) + ref_date        if flat_num_dates.size > 0:            # avoid size 0 datetimes GH1329            pd.to_timedelta(flat_num_dates.min(), delta) + ref_date            pd.to_timedelta(flat_num_dates.max(), delta) + ref_date      # To avoid integer overflow when converting to nanosecond units for integer     # dtypes smaller than np.int64 cast all integer and unsigned integer dtype


Question: )      # determine which variables to merge, and then merge them according to compat    variables_to_merge = (coord_names | data_names) - concat_over - dim_names    variables_to_merge = (coord_names | data_names) - concat_over - unlabeled_dims      result_vars = {}     result_indexes = {}

Alignment of dataset with MultiIndex fails after applying After applying the function to a dataset with a Multiindex, a lot of functions related to indexing are broken. For example, it is not possible to apply to itself anymore. The error is raised in the alignment module. It seems that the function does not find indexes that belong to the same dimension. I expected the alignment to be functional and that these basic functions work. [X] Minimal example — the example is as focused as reasonably possible to demonstrate the underlying issue in xarray. [X] Complete example — the example is self-contained, including all data and the text of any traceback. , returning the result. [X] New issue — a search of GitHub Issues suggests this is not a duplicate.


Question: )      # determine which variables to merge, and then merge them according to compat    variables_to_merge = (coord_names | data_names) - concat_over - dim_names    variables_to_merge = (coord_names | data_names) - concat_over - unlabeled_dims      result_vars = {}     result_indexes = {}

No response <detailsINSTALLED VERSIONS commit: None python: 3.10.5 (main, Jun 14 2022, 07:04:59) [GCC 10.3.0] python-bits: 64 OS: Linux OS-release: 5.15.0-41-generic machine: x8664 processor: x8664 byteorder: little LCALL: None LANG: enUS.UTF-8 LOCALE: ('enUS', 'UTF-8') libhdf5: 1.12.1 libnetcdf: 4.8.1 xarray: 2022.6.0 pandas: 1.4.2 numpy: 1.21.6 scipy: 1.8.1 netCDF4: 1.6.0 pydap: None h5netcdf: None h5py: 3.6.0 Nio: None zarr: None cftime: 1.5.1.1 nctimeaxis: None PseudoNetCDF: None rasterio: 1.2.10 cfgrib: None iris: None bottleneck: 1.3.4 dask: 2022.6.1 distributed: 2022.6.1 matplotlib: 3.5.1 cartopy: 0.20.2 seaborn: 0.11.2 numbagg: None fsspec: 2022.3.0 cupy: None pint: None sparse: 0.13.0 flox: None numpygroupies: None setuptools: 61.2.0 pip: 22.1.2 conda: 4.13.0 pytest: 7.1.2 IPython: 7.33.0 sphinx: 5.0.2 /home/fabian/.miniconda3/lib/python3.10/site- UserWarning: Setuptools is replacing distutils. ("Setuptools is replacing distutils.") </details


Question: indexes: dict[Hashable, Index] = {}         for k, v in self.variables.items():             dims = tuple(dims_dict.get(dim, dim) for dim in v.dims)            var: Variable             if k in result_dims:                 var = v.to_index_variable()                 var.dims = dimsdiff --git a/xarray/core/variable.py b/xarray/core/variable.py-- a/xarray/core/variable.py++ b/xarray/core/variable.py

can modify original object This is kind of a convoluted example, but something I ran into. It appears that in certain cases can modify the original object, here the of a data variable that was swapped into being a dimension coordinate variable. I expected it not to modify the original object. [X] Minimal example — the example is as focused as reasonably possible to demonstrate the underlying issue in xarray. [X] Complete example — the example is self-contained, including all data and the text of any traceback. , returning the result. [X] New issue — a search of GitHub Issues suggests this is not a duplicate. No response_ More experiments in . <details</details


Question: Iterable,     Literal,     Mapping,<ins>    NoReturn,</ins>     Sequence, )


Question: def values(self, values):         self.data = values     def to_base_variable(self):    def to_base_variable(self) -> Variable:         """Return this variable as a base xarray.Variable"""         return Variable(            self.dims, self._data, self._attrs, encoding=self._encoding, fastpath=True            self._dims, self._data, self._attrs, encoding=self._encoding, fastpath=True         )      to_variable = utils.alias(to_base_variable, "to_variable")     def to_index_variable(self):    def to_index_variable(self) -> IndexVariable:         """Return this variable as an xarray.IndexVariable"""         return IndexVariable(            self.dims, self._data, self._attrs, encoding=self._encoding, fastpath=True            self._dims, self._data, self._attrs, encoding=self._encoding, fastpath=True         )      to_coord = utils.alias(to_index_variable, "to_coord")     def to_index(self):    def to_index(self) -> pd.Index:         """Convert this variable to a pandas.Index"""         return self.to_index_variable().to_index()


Question: def _data_equals(self, other):         return self.to_index().equals(other.to_index())     def to_index_variable(self):    def to_index_variable(self) -> IndexVariable:         """Return this variable as an xarray.IndexVariable"""        return self        return self.copy()      to_coord = utils.alias(to_index_variable, "to_coord")     def to_index(self):    def to_index(self) -> pd.Index:         """Convert this variable to a pandas.Index"""         # n.b. creating a new pandas.Index from an old pandas.Index is         # basically free as pandas.Index objects are immutable


Question: return index      @property    def level_names(self):    def level_names(self) -> list[str] | None:         """Return MultiIndex level names or None if this IndexVariable has no         MultiIndex.         """


Question: return type(self)(self.dims, index.get_level_values(level))      @property    def name(self):    def name(self) -> Hashable:         return self.dims[0]      @name.setter    def name(self, value):    def name(self, value) -> NoReturn:         raise AttributeError("cannot modify name of IndexVariable in-place")      def _inplace_binary_op(self, other, f):


Question: dim_coords = either_dict_or_kwargs(indexes, indexes_kwargs, "set_index")          new_indexes: dict[Hashable, Index] = {}        new_variables: dict[Hashable, IndexVariable] = {}        maybe_drop_indexes: list[Hashable] = []        drop_variables: list[Hashable] = []        new_variables: dict[Hashable, Variable] = {}        drop_indexes: set[Hashable] = set()        drop_variables: set[Hashable] = set()         replace_dims: dict[Hashable, Hashable] = {}        all_var_names: set[Hashable] = set()          for dim, _var_names in dim_coords.items():             if isinstance(_var_names, str) or not isinstance(_var_names, Sequence):

index refactor: more than on Dataset assumes that everything that is in and not in is a "data variable". However, since the index refactor we can end up with more than which breaks a number of stuff (e.g. the repr). Well it seems this assumption is now wrong. [ ] Minimal example — the example is as focused as reasonably possible to demonstrate the underlying issue in xarray. [ ] Complete example — the example is self-contained, including all data and the text of any traceback. , returning the result. [ ] New issue — a search of GitHub Issues suggests this is not a duplicate. The error comes from here Bisected to - which probably does not help too much. <details</details


Question: + " variable(s) do not exist"                 )             current_coord_names = self.xindexes.get_all_coords(dim, errors="ignore")            all_var_names.update(var_names)            drop_variables.update(var_names)             # drop any pre-existing index involved            maybe_drop_indexes += list(current_coord_names) + var_names            # drop any pre-existing index involved and its corresponding coordinates            index_coord_names = self.xindexes.get_all_coords(dim, errors="ignore")            all_index_coord_names = set(index_coord_names)             for k in var_names:                maybe_drop_indexes += list(                all_index_coord_names.update(                     self.xindexes.get_all_coords(k, errors="ignore")                 )             drop_variables += var_names            drop_indexes.update(all_index_coord_names)            drop_variables.update(all_index_coord_names)              if len(var_names) == 1 and (not append or dim not in self._indexes):                 var_name = var_names[0]


Question: )                 idx = PandasIndex.from_variables({dim: var})                 idx_vars = idx.create_variables({var_name: var})                # trick to preserve coordinate order in this case                if dim in self._coord_names:                    drop_variables.remove(dim)             else:                 if append:                     current_variables = {                        k: self._variables[k] for k in current_coord_names                        k: self._variables[k] for k in index_coord_names                     }                 else:                     current_variables = {}


Question: new_indexes.update({k: idx for k in idx_vars})             new_variables.update(idx_vars)         # re-add deindexed coordinates (convert to base variables)        for k in drop_variables:            if (                k not in new_variables                and k not in all_var_names                and k in self._coord_names            ):                new_variables[k] = self._variables[k].to_base_variable()         indexes_: dict[Any, Index] = {            k: v for k, v in self._indexes.items() if k not in maybe_drop_indexes            k: v for k, v in self._indexes.items() if k not in drop_indexes         }         indexes_.update(new_indexes)


Question: new_dims = [replace_dims.get(d, d) for d in v.dims]                 variables[k] = v._replace(dims=new_dims)         coord_names = self._coord_names - set(drop_variables) | set(new_variables)        coord_names = self._coord_names - drop_variables | set(new_variables)          return self._replace_with_new_dims(             variables, coord_names=coord_names, indexes=indexes_


Question: f"{tuple(invalid_coords)} are not coordinates with an index"             )         drop_indexes: list[Hashable] = []        drop_variables: list[Hashable] = []        replaced_indexes: list[PandasMultiIndex] = []        drop_indexes: set[Hashable] = set()        drop_variables: set[Hashable] = set()        seen: set[Index] = set()         new_indexes: dict[Hashable, Index] = {}        new_variables: dict[Hashable, IndexVariable] = {}        new_variables: dict[Hashable, Variable] = {}        def drop_or_convert(var_names):            if drop:                drop_variables.update(var_names)            else:                base_vars = {                    k: self._variables[k].to_base_variable() for k in var_names                }                new_variables.update(base_vars)          for name in dims_or_levels:             index = self._indexes[name]            drop_indexes += list(self.xindexes.get_all_coords(name))            if isinstance(index, PandasMultiIndex) and name not in self.dims:                # special case for pd.MultiIndex (name is an index level):                # replace by a new index with dropped level(s) instead of just drop the index                if index not in replaced_indexes:                    level_names = index.index.names                    level_vars = {                        k: self._variables[k]                        for k in level_names                        if k not in dims_or_levels                    }                    if level_vars:                        idx = index.keep_levels(level_vars)                        idx_vars = idx.create_variables(level_vars)                        new_indexes.update({k: idx for k in idx_vars})                        new_variables.update(idx_vars)                replaced_indexes.append(index)             if drop:                drop_variables.append(name)            if index in seen:                continue            seen.add(index)            idx_var_names = set(self.xindexes.get_all_coords(name))            drop_indexes.update(idx_var_names)            if isinstance(index, PandasMultiIndex):                # special case for pd.MultiIndex                level_names = index.index.names                keep_level_vars = {                    k: self._variables[k]                    for k in level_names                    if k not in dims_or_levels                }                if index.dim not in dims_or_levels and keep_level_vars:                    # do not drop the multi-index completely                    # instead replace it by a new (multi-)index with dropped level(s)                    idx = index.keep_levels(keep_level_vars)                    idx_vars = idx.create_variables(keep_level_vars)                    new_indexes.update({k: idx for k in idx_vars})                    new_variables.update(idx_vars)                    if not isinstance(idx, PandasMultiIndex):                        # multi-index reduced to single index                        # backward compatibility: unique level coordinate renamed to dimension                        drop_variables.update(keep_level_vars)                    drop_or_convert(                        [k for k in level_names if k not in keep_level_vars]                    )                else:                    # always drop the multi-index dimension variable                    drop_variables.add(index.dim)                    drop_or_convert(level_names)            else:                drop_or_convert(idx_var_names)          indexes = {k: v for k, v in self._indexes.items() if k not in drop_indexes}         indexes.update(new_indexes)


Question: }         variables.update(new_variables)         coord_names = set(new_variables) | self._coord_names        coord_names = self._coord_names - drop_variables         return self._replace(variables, coord_names=coord_names, indexes=indexes)        return self._replace_with_new_dims(            variables, coord_names=coord_names, indexes=indexes        )      def reorder_levels(         self: T_Dataset,diff --git a/xarray/core/indexes.py b/xarray/core/indexes.py-- a/xarray/core/indexes.py++ b/xarray/core/indexes.py


Question: level_coords_dtype = {k: self.level_coords_dtype[k] for k in index.names}             return self._replace(index, level_coords_dtype=level_coords_dtype)         else:            # backward compatibility: rename the level coordinate to the dimension name             return PandasIndex(                index, self.dim, coord_dtype=self.level_coords_dtype[index.name]                index.rename(self.dim),                self.dim,                coord_dtype=self.level_coords_dtype[index.name],             )      def reorder_levels(


Question: if utils.is_dict_like(new_name_or_name_dict) or new_name_or_name_dict is None:             # change dims/coords             name_dict = either_dict_or_kwargs(new_name_or_name_dict, names, "rename")            dataset = self._to_temp_dataset().rename(name_dict)            dataset = self._to_temp_dataset()._rename(name_dict)             return self._from_temp_dataset(dataset)         if utils.hashable(new_name_or_name_dict) and names:             # change name + dims/coords            dataset = self._to_temp_dataset().rename(names)            dataset = self._to_temp_dataset()._rename(names)             dataarray = self._from_temp_dataset(dataset)             return dataarray._replace(name=new_name_or_name_dict)         # only change namediff --git a/xarray/core/dataset.py b/xarray/core/dataset.py-- a/xarray/core/dataset.py++ b/xarray/core/dataset.py

[Bug]: renamevars to dimension coordinate does not create an index We used to make a coordinate a dimension coordinate (instead of ). This results in the coordinate correctly being displayed as a dimension coordinate (with the *) but it does not create an index, such that further operations like fail with a strange . I expect one of two things to be true: does not allow setting dimension coordinates (raises Error and tells you to use setindex) checks for this occasion and sets the index correctly No response No response INSTALLED VERSIONS commit: None python: 3.9.1 (default, Jan 13 2021, 15:21:08) [GCC 4.8.5 (Red Hat 4.8.5-44)] python-bits: 64 OS: Linux OS-release: 3.10.0-1160.49.1.el7.x8664 machine: x8664 processor: x8664 byteorder: little LCALL: None LANG: enUS.UTF-8 LOCALE: ('enUS', 'UTF-8') libhdf5: 1.12.0 libnetcdf: 4.7.4 xarray: 0.20.2 pandas: 1.3.5 numpy: 1.21.5 scipy: 1.7.3 netCDF4: 1.5.8 pydap: None h5netcdf: None h5py: None Nio: None zarr: None cftime: 1.5.1.1 nctime_axis: None PseudoNetCDF: None rasterio: None cfgrib: None iris: None bottleneck: None dask: None distributed: None matplotlib: 3.5.1 cartopy: None seaborn: None numbagg: None fsspec: None cupy: None pint: None sparse: None setuptools: 49.2.1 pip: 22.0.2 conda: None pytest: 6.2.5 IPython: 8.0.0 sphinx: None


Question: return variables, coord_names, dims, indexes <ins>    def _rename(        self: T_Dataset,        name_dict: Mapping[Any, Hashable] | None = None,        **names: Hashable,    ) -> T_Dataset:        """Also used internally by DataArray so that the warning (if any)        is raised at the right stack level.        """        name_dict = either_dict_or_kwargs(name_dict, names, "rename")        for k in name_dict.keys():            if k not in self and k not in self.dims:                raise ValueError(                    f"cannot rename {k!r} because it is not a "                    "variable or dimension in this dataset"                )            create_dim_coord = False            new_k = name_dict[k]            if k in self.dims and new_k in self._coord_names:                coord_dims = self._variables[name_dict[k]].dims                if coord_dims == (k,):                    create_dim_coord = True            elif k in self._coord_names and new_k in self.dims:                coord_dims = self._variables[k].dims                if coord_dims == (new_k,):                    create_dim_coord = True            if create_dim_coord:                warnings.warn(                    f"rename {k!r} to {name_dict[k]!r} does not create an index "                    "anymore. Try using swap_dims instead or use set_index "                    "after rename to create an indexed coordinate.",                    UserWarning,                    stacklevel=3,                )        variables, coord_names, dims, indexes = self._rename_all(            name_dict=name_dict, dims_dict=name_dict        )        return self._replace(variables, coord_names, dims=dims, indexes=indexes)</ins>     def rename(         self: T_Dataset,         name_dict: Mapping[Any, Hashable] | None = None,


Question: Dataset.rename_dims         DataArray.rename         """        name_dict = either_dict_or_kwargs(name_dict, names, "rename")        for k in name_dict.keys():            if k not in self and k not in self.dims:                raise ValueError(                    f"cannot rename {k!r} because it is not a "                    "variable or dimension in this dataset"                )        variables, coord_names, dims, indexes = self._rename_all(            name_dict=name_dict, dims_dict=name_dict        )        return self._replace(variables, coord_names, dims=dims, indexes=indexes)        return self._rename(name_dict=name_dict, **names)      def rename_dims(         self: T_Dataset,


Question: """Return a list of unique indexes, preserving order."""          unique_indexes: list[T_PandasOrXarrayIndex] = []        seen: set[T_PandasOrXarrayIndex] = set()        seen: set[int] = set()          for index in self._indexes.values():            if index not in seen:            index_id = id(index)            if index_id not in seen:                 unique_indexes.append(index)                seen.add(index)                seen.add(index_id)          return unique_indexes

Indexes.getunique() TypeError with pandas indexes I also just tested the method that you mentioned and maybe noticed a related issue here, which I'm not sure is wanted / expected. Taking the above dataset , accessing this function results in an error: However, for it works: Originally posted by in


Question: """         new_indexes = {}         new_index_vars = {}         for idx, coords in self.group_by_index():            if isinstance(idx, pd.Index):                convert_new_idx = True                dim = next(iter(coords.values())).dims[0]                if isinstance(idx, pd.MultiIndex):                    idx = PandasMultiIndex(idx, dim)                else:                    idx = PandasIndex(idx, dim)            else:                convert_new_idx = False             new_idx = idx.copy(deep=deep)             idx_vars = idx.create_variables(coords)            if convert_new_idx:                new_idx = cast(PandasIndex, new_idx).index             new_indexes.update({k: new_idx for k in coords})             new_index_vars.update(idx_vars)


Question: variables, indexes = drop_coords(             coords, self._data._variables, self._data.xindexes         )<ins>        self._data._coord_names.intersection_update(variables)</ins>         self._data._variables = variables         self._data._indexes = indexes

Broken state when using assigncoords with multiindex I was trying to assign coordinates on a dataset that had been created by using stack. After assigning the coordinates, the dataset was in a state where its length was coming out as less than zero, which caused all sorts of issues. I think the issue is with the updating of , perhaps in I expected to just be able to assign the coords and then print the array to see the result. [X] Minimal example — the example is as focused as reasonably possible to demonstrate the underlying issue in xarray. [X] Complete example — the example is self-contained, including all data and the text of any traceback. , returning the result. [X] New issue — a search of GitHub Issues suggests this is not a duplicate. Here's a test (I put it in but maybe there is a better spot) <detailsINSTALLED VERSIONS commit: python: 3.8.13 (default, Mar 25 2022, 06:04:14) [Clang 12.0.1 ] python-bits: 64 OS: Darwin OS-release: 21.5.0 machine: arm64 processor: arm byteorder: little LCALL: None LANG: enAU.UTF-8 LOCALE: ('enAU', 'UTF-8') libhdf5: 1.12.2 libnetcdf: 4.8.1 xarray: 0.1.dev4312+ge678a1d.


Question: variables, indexes = drop_coords(             coords, self._data._variables, self._data.xindexes         )<ins>        self._data._coord_names.intersection_update(variables)</ins>         self._data._variables = variables         self._data._indexes = indexes

pandas: 1.5.0 numpy: 1.22.4 scipy: 1.9.1 netCDF4: 1.6.1 pydap: installed h5netcdf: 1.0.2 h5py: 3.7.0 Nio: None zarr: 2.13.2 cftime: 1.6.2 nctimeaxis: 1.4.1 PseudoNetCDF: 3.2.2 rasterio: 1.3.1 cfgrib: 0.9.10.1 iris: 3.3.0 bottleneck: 1.3.5 dask: 2022.9.1 distributed: 2022.9.1 matplotlib: 3.6.0 cartopy: 0.21.0 seaborn: 0.12.0 numbagg: 0.2.1 fsspec: 2022.8.2 cupy: None pint: 0.19.2 sparse: 0.13.0 flox: 0.5.9 numpy_groupies: 0.9.19 setuptools: 65.4.0 pip: 22.2.2 conda: None pytest: 7.1.3 IPython: None sphinx: None </details


Question: f"other variables: {list(maybe_midx.index.names)!r}. "                 f"This will raise an error in the future. Use `.drop_vars({idx_coord_names!r})` before "                 "assigning new coordinate values.",                DeprecationWarning,                FutureWarning,                 stacklevel=4,             )             for k in idx_coord_names:


Question: f"other variables: {list(maybe_midx.index.names)!r}. "                 f"This will raise an error in the future. Use `.drop_vars({idx_coord_names!r})` before "                 "assigning new coordinate values.",                DeprecationWarning,                FutureWarning,                 stacklevel=4,             )             for k in idx_coord_names:


Question: numpy.transpose         DataArray.transpose         """<ins>        # Raise error if list is passed as dims        if (len(dims) > 0) and (isinstance(dims[0], list)):            list_fix = [f"{repr(x)}" if isinstance(x, str) else f"{x}" for x in dims[0]]            raise TypeError(                f'transpose requires dims to be passed as multiple arguments. Expected `{", ".join(list_fix)}`. Received `{dims[0]}` instead'            )</ins>         # Use infix_dims to check once for missing dimensions         if len(dims) != 0:             _ = list(infix_dims(dims, self.dims, missing_dims))

Raise nicer error if passing a list of dimension names to transpose Hello, in xarray 0.20.1, I am getting the following error The expected result is No response No response No response <detailsINSTALLED VERSIONS commit: None python: 3.9.12 (main, Apr 5 2022, 06:56:58) [GCC 7.5.0] python-bits: 64 OS: Linux OS-release: 3.10.0-1160.42.2.el7.x8664 machine: x8664 processor: x8664 byteorder: little LCALL: None LANG: enUS LOCALE: ('enUS', 'ISO8859-1') libhdf5: 1.12.1 libnetcdf: 4.8.1 xarray: 0.20.1 pandas: 1.4.1 numpy: 1.21.5 scipy: 1.8.0 netCDF4: 1.5.7 pydap: None h5netcdf: 999 h5py: 3.6.0 Nio: None zarr: None cftime: 1.5.1.1 nctimeaxis: 1.4.0 PseudoNetCDF: None rasterio: None cfgrib: None iris: None bottleneck: 1.3.4 dask: 2022.02.1 distributed: 2022.2.1 matplotlib: 3.5.1 cartopy: 0.18.0 seaborn: 0.11.2 numbagg: None fsspec: 2022.02.0 cupy: None pint: 0.18 sparse: 0.13.0 setuptools: 61.2.0 pip: 21.2.4 conda: None pytest: None IPython: 8.2.0 sphinx: None </details


Question: and v.ndim > 0             and stackable(v.dims[-1])         )        new_vars[k] = decode_cf_variable(            k,            v,            concat_characters=concat_characters,            mask_and_scale=mask_and_scale,            decode_times=decode_times,            stack_char_dim=stack_char_dim,            use_cftime=use_cftime,            decode_timedelta=decode_timedelta,        )        try:            new_vars[k] = decode_cf_variable(                k,                v,                concat_characters=concat_characters,                mask_and_scale=mask_and_scale,                decode_times=decode_times,                stack_char_dim=stack_char_dim,                use_cftime=use_cftime,                decode_timedelta=decode_timedelta,            )        except Exception as e:            raise type(e)(f"Failed to decode variable {k!r}: {e}")         if decode_coords in [True, "coordinates", "all"]:             var_attrs = new_vars[k].attrs             if "coordinates" in var_attrs:

Time decoding error message does not include the problematic variable's name If any variable in a Dataset has times that cannot be represented as objects, an error message will be raised. However, this error message will not indicate the problematic variable's name. It would be nice if it did, because it would make it easier for users to determine the source of the error. cc: xref: Unidata/cftime This is a minimal example of the issue. The error message gives no indication that is the problem:


Question: def _protect_dataset_variables_inplace(dataset, cache):     for name, variable in dataset.variables.items():        if name not in variable.dims:        if name not in dataset._indexes:             # no need to protect IndexVariable objects             data = indexing.CopyOnWriteArray(variable._data)             if cache:

xarray.opendataset has issues if the dataset returned by the backend contains a multiindex As a follow up of this comment: I'm currently trying to implement a custom backend that allows me to also handle multiindices when loading a NetCDF dataset using . I'm using the following two functions to convert the dataset to a NetCDF compatible version and back again: Here is a small code example: While the manual patching for saving is currently still required, I tried to at least work around the function call in by creating a custom NetCDF Backend: but this works: So I'm guessing is performing some operation on the dataset returned by the backend, and one of those leads to a failure if there is a multiindex already contained. I expected that it doesn't matter wheter is called inside the backend or afterwards, and the same dataset will be returned each time. [X] Minimal example — the example is as focused as reasonably possible to demonstrate the underlying issue in xarray. [X] Complete example — the example is self-contained, including all data and the text of any traceback. , returning the result. [X] New issue — a search of GitHub Issues suggests this is not a duplicate. No response I'm also open to other suggestions how I could simplify the usage of multiindices, maybe there is an approach that doesn't require a custom backend at all?


Question: def _protect_dataset_variables_inplace(dataset, cache):     for name, variable in dataset.variables.items():        if name not in variable.dims:        if name not in dataset._indexes:             # no need to protect IndexVariable objects             data = indexing.CopyOnWriteArray(variable._data)             if cache:

<detailsINSTALLED VERSIONS commit: None python: 3.8.10 (default, Jan 28 2022, 09:41:12) [GCC 9.3.0] python-bits: 64 OS: Linux OS-release: 5.10.102.1-microsoft-standard-WSL2 machine: x8664 processor: x8664 byteorder: little LCALL: None LANG: C.UTF-8 LOCALE: ('enUS', 'UTF-8') libhdf5: 1.10.5 libnetcdf: 4.6.3 xarray: 2022.9.0 pandas: 1.5.0 numpy: 1.23.3 scipy: 1.9.1 netCDF4: 1.5.4 pydap: None h5netcdf: None h5py: 3.7.0 Nio: None zarr: None cftime: 1.6.2 nctimeaxis: None PseudoNetCDF: None rasterio: 1.3.2 cfgrib: None iris: None bottleneck: None dask: None distributed: None matplotlib: 3.6.0 cartopy: 0.19.0.post1 seaborn: None numbagg: None fsspec: None cupy: None pint: None sparse: 0.13.0 flox: None numpy_groupies: None setuptools: 65.3.0 pip: 22.2.2 conda: None pytest: 7.1.3 IPython: 8.5.0 sphinx: 4.5.0 </details


Question: return short_numpy_repr(array)     elif is_duck_array(internal_data):         return limit_lines(repr(array.data), limit=40)    elif array._in_memory or array.size < 1e5:    elif array._in_memory:         return short_numpy_repr(array)     else:         # internal xarray array type

Avoid loading any data for reprs For "small" datasets, we load in to memory when displaying the repr. For cloud backed datasets with large number of "small" variables, this can use a lot of time sequentially loading O(100) variables just for a repr. Fast reprs! This dataset has 48 "small" variables On this repr takes 36.4s If I comment the condition I get 6μs. [x] Minimal example — the example is as focused as reasonably possible to demonstrate the underlying issue in xarray. [x] Complete example — the example is self-contained, including all data and the text of any traceback. , returning the result. [x] New issue — a search of GitHub Issues suggests this is not a duplicate. No response No response <detailsINSTALLED VERSIONS commit: None python: 3.10.4 (main, Mar 24 2022, 17:43:32) [Clang 12.0.1 ] python-bits: 64 OS: Darwin OS-release: 21.5.0 machine: x8664 processor: i386 byteorder: little LCALL: None LANG: enUS.UTF-8 LOCALE: ('enUS', 'UTF-8') libhdf5: None libnetcdf: None xarray: 2022.3.0 pandas: 1.4.2 numpy: 1.22.4 scipy: 1.8.1 netCDF4: None pydap: None h5netcdf: None h5py: None Nio: None zarr: 2.11.3 cftime: None nctime_axis: None PseudoNetCDF: None rasterio: 1.2.10 cfgrib: None iris: None bottleneck: None dask: 2022.05.2 distributed: None matplotlib: 3.5.2 cartopy: 0.20.2 seaborn: 0.11.2 numbagg: None fsspec: 2022.5.0 cupy: None pint: None sparse: None setuptools: 62.3.2 pip: 22.1.2 conda: None pytest: None IPython: 8.4.0 sphinx: 4.5.0 </details


Question: Dataset.where, DataArray.where :         equivalent methods     """    from .dataset import Dataset     if keep_attrs is None:         keep_attrs = _get_keep_attrs(default=False)    if keep_attrs is True:        # keep the attributes of x, the second parameter, by default to        # be consistent with the `where` method of `DataArray` and `Dataset`        keep_attrs = lambda attrs, context: getattr(x, "attrs", {})      # alignment for three arguments is complicated, so don't support it yet    return apply_ufunc(    result = apply_ufunc(         duck_array_ops.where,         cond,         x,

overwrites coordinate attributes had some unintended consequences for , where coordinate attributes are getting overwritten by variable attributes. I guess this has been broken since . Coordinate attributes should be preserved. [X] Minimal example — the example is as focused as reasonably possible to demonstrate the underlying issue in xarray. [X] Complete example — the example is self-contained, including all data and the text of any traceback. , returning the result. [X] New issue — a search of GitHub Issues suggests this is not a duplicate. I'm struggling to figure out how the simple change in brought this about. I tried tracing my way through the various merge functions but there are a lot of layers. Happy to submit a PR if someone has an idea for an obvious fix.


Question: Dataset.where, DataArray.where :         equivalent methods     """    from .dataset import Dataset     if keep_attrs is None:         keep_attrs = _get_keep_attrs(default=False)    if keep_attrs is True:        # keep the attributes of x, the second parameter, by default to        # be consistent with the `where` method of `DataArray` and `Dataset`        keep_attrs = lambda attrs, context: getattr(x, "attrs", {})      # alignment for three arguments is complicated, so don't support it yet    return apply_ufunc(    result = apply_ufunc(         duck_array_ops.where,         cond,         x,

<detailsINSTALLED VERSIONS commit: None python: 3.9.13 (main, May 27 2022, 16:56:21) [GCC 10.3.0] python-bits: 64 OS: Linux OS-release: 5.15.0-52-generic machine: x8664 processor: x8664 byteorder: little LCALL: None LANG: enUS.UTF-8 LOCALE: ('enUS', 'UTF-8') libhdf5: 1.12.2 libnetcdf: 4.8.1 xarray: 2022.10.0 pandas: 1.4.3 numpy: 1.23.4 scipy: 1.9.3 netCDF4: 1.6.1 pydap: None h5netcdf: 1.0.2 h5py: 3.7.0 Nio: None zarr: 2.13.3 cftime: 1.6.2 nctimeaxis: 1.4.1 PseudoNetCDF: None rasterio: 1.3.3 cfgrib: 0.9.10.2 iris: None bottleneck: 1.3.5 dask: 2022.10.0 distributed: 2022.10.0 matplotlib: 3.6.1 cartopy: 0.21.0 seaborn: None numbagg: None fsspec: 2022.10.0 cupy: None pint: 0.19.2 sparse: 0.13.0 flox: 0.6.1 numpy_groupies: 0.9.19 setuptools: 65.5.0 pip: 22.3 conda: None pytest: 7.1.3 IPython: 8.5.0 sphinx: None </details


Question: keep_attrs=keep_attrs,     ) <ins>    # keep the attributes of x, the second parameter, by default to    # be consistent with the `where` method of `DataArray` and `Dataset`    # rebuild the attrs from x at each level of the output, which could be    # Dataset, DataArray, or Variable, and also handle coords    if keep_attrs is True:        if isinstance(y, Dataset) and not isinstance(x, Dataset):            # handle special case where x gets promoted to Dataset            result.attrs = {}            if getattr(x, "name", None) in result.data_vars:                result[x.name].attrs = getattr(x, "attrs", {})        else:            # otherwise, fill in global attrs and variable attrs (if they exist)            result.attrs = getattr(x, "attrs", {})            for v in getattr(result, "data_vars", []):                result[v].attrs = getattr(getattr(x, v, None), "attrs", {})        for c in getattr(result, "coords", []):            # always fill coord attrs of x            result[c].attrs = getattr(getattr(x, c, None), "attrs", {})    return result</ins>  @overload def polyval(


Question: keep_attrs=keep_attrs,     ) <ins>    # keep the attributes of x, the second parameter, by default to    # be consistent with the `where` method of `DataArray` and `Dataset`    # rebuild the attrs from x at each level of the output, which could be    # Dataset, DataArray, or Variable, and also handle coords    if keep_attrs is True:        if isinstance(y, Dataset) and not isinstance(x, Dataset):            # handle special case where x gets promoted to Dataset            result.attrs = {}            if getattr(x, "name", None) in result.data_vars:                result[x.name].attrs = getattr(x, "attrs", {})        else:            # otherwise, fill in global attrs and variable attrs (if they exist)            result.attrs = getattr(x, "attrs", {})            for v in getattr(result, "data_vars", []):                result[v].attrs = getattr(getattr(x, v, None), "attrs", {})        for c in getattr(result, "coords", []):            # always fill coord attrs of x            result[c].attrs = getattr(getattr(x, c, None), "attrs", {})    return result</ins>  @overload def polyval(


Question: else:                 reshaped[key] = var         should_be_coords = set(window_dim) & set(self.obj.coords)        # should handle window_dim being unindexed        should_be_coords = (set(window_dim) & set(self.obj.coords)) | set(            self.obj.coords        )         result = reshaped.set_coords(should_be_coords)         if isinstance(self.obj, DataArray):             return self.obj._from_temp_dataset(result)

ds.Coarsen.construct demotes non-dimensional coordinates to variables demotes non-dimensional coordinates to variables All variables that were coordinates before the coarsen.construct stay as coordinates afterwards. [X] Minimal example — the example is as focused as reasonably possible to demonstrate the underlying issue in xarray. [X] Complete example — the example is self-contained, including all data and the text of any traceback. , returning the result. [X] New issue — a search of GitHub Issues suggests this is not a duplicate. No response No response_


Question: for name, variable in mapping.items():             if isinstance(variable, DataArray):                coords = variable._coords.copy()  # use private API for speed                indexes = dict(variable._indexes)                coords_ = variable._coords.copy()  # use private API for speed                indexes_ = dict(variable._indexes)                 # explicitly overwritten variables should take precedence                coords.pop(name, None)                indexes.pop(name, None)                append_all(coords, indexes)                coords_.pop(name, None)                indexes_.pop(name, None)                append_all(coords_, indexes_)              variable = as_variable(variable, name=name)             if name in indexes:

assigncoords reset all dimension coords to default (pandas) index See should preserve the index of coordinates that are not updated or not part of a dropped multi-coordinate index. See [X] Minimal example — the example is as focused as reasonably possible to demonstrate the underlying issue in xarray. [X] Complete example — the example is self-contained, including all data and the text of any traceback. , returning the result. [X] New issue — a search of GitHub Issues suggests this is not a duplicate. No response No response_ <detailsXarray version 2022.11.0 </details


Question: aligned = deep_align(         coerced, join=join, copy=False, indexes=indexes, fill_value=fill_value     )    collected = collect_variables_and_indexes(aligned)    collected = collect_variables_and_indexes(aligned, indexes=indexes)     prioritized = _get_priority_vars_and_indexes(aligned, priority_arg, compat=compat)     variables, out_indexes = merge_collected(collected, prioritized, compat=compat)     return variables, out_indexes


Question: self, other = align(self, other, join=align_type, copy=False)  # type: ignore[assignment]         g = f if not reflexive else lambda x, y: f(y, x)         ds = self._calculate_binary_op(g, other, join=align_type)<ins>        keep_attrs = _get_keep_attrs(default=False)        if keep_attrs:            ds.attrs = self.attrs</ins>         return ds      def _inplace_binary_op(self: T_Dataset, other, f) -> T_Dataset:

binary ops ignore option When doing arithmetic operations on two Dataset operands, the option is ignored and therefore attributes not kept. Minimal example: This is not true for DataArrays/Variables which do take into account. Datasets to behave the same as DataArray/Variables, and keep attributes during binary operations when option is set. PR is inbound.


Question: self.level = level      def __array__(self, dtype: DTypeLike = None) -> np.ndarray:        if dtype is None:            dtype = self.dtype         if self.level is not None:            return self.array.get_level_values(self.level).values            return np.asarray(                self.array.get_level_values(self.level).values, dtype=dtype            )         else:             return super().__array__(dtype)

stack casts int32 dtype coordinate to int64 The code example below results in , because the data type of the coordinate is changed from 'i4' to 'i8'. I expect the result to be . Creating a MultiIndex should not change the data type of the Indexes from which it is built. [X] Minimal example — the example is as focused as reasonably possible to demonstrate the underlying issue in xarray. [X] Complete example — the example is self-contained, including all data and the text of any traceback. , returning the result. [X] New issue — a search of GitHub Issues suggests this is not a duplicate. No response No response <detailsINSTALLED VERSIONS commit: None python: 3.10.8 (main, Oct 13 2022, 10:17:43) [Clang 14.0.0 (clang-1400.0.29.102)] python-bits: 64 OS: Darwin OS-release: 21.6.0 machine: x8664 processor: i386 byteorder: little LCALL: None LANG: None LOCALE: (None, 'UTF-8') libhdf5: 1.12.2 libnetcdf: 4.9.0 xarray: 2022.10.0 pandas: 1.5.1 numpy: 1.23.4 scipy: 1.9.3 netCDF4: 1.6.1 pydap: None h5netcdf: None h5py: 3.7.0 Nio: None zarr: None cftime: 1.6.2 nctimeaxis: None PseudoNetCDF: None rasterio: None cfgrib: None iris: None bottleneck: None dask: 2022.10.2 distributed: None matplotlib: 3.6.1 cartopy: 0.21.0 seaborn: None numbagg: None fsspec: 2022.10.0 cupy: None pint: None sparse: None flox: None numpygroupies: None setuptools: 65.5.0 pip: 22.1.2 conda: None pytest: None IPython: 8.6.0 sphinx: None ("Setuptools is replacing distutils.") </details


Question: import pandas as pd  from xarray.core import dtypes, utilsfrom xarray.core.alignment import alignfrom xarray.core.alignment import align, reindex_variables from xarray.core.duck_array_ops import lazy_array_equiv from xarray.core.indexes import Index, PandasIndex from xarray.core.merge import (

Add defaults during concat 508 [x] [x] Tests [x] Passes [x] Fully documented, including for all changes and for new API Contined on issue by removing exception when concat two datasets with disjoint variables and instead add the missing variable with


Question: elif opt == "all":                 concat_over.update(                    set(getattr(datasets[0], subset)) - set(datasets[0].dims)                    set().union(                        *list(set(getattr(d, subset)) - set(d.dims) for d in datasets)                    )                 )             elif opt == "minimal":                 pass


Question: # determine dimensional coordinate names and a dict mapping name to DataArray def _parse_datasets(    datasets: Iterable[T_Dataset],) -> tuple[dict[Hashable, Variable], dict[Hashable, int], set[Hashable], set[Hashable]]:    datasets: list[T_Dataset],) -> tuple[    dict[Hashable, Variable],    dict[Hashable, int],    set[Hashable],    set[Hashable],    list[Hashable],]:     dims: set[Hashable] = set()     all_coord_names: set[Hashable] = set()     data_vars: set[Hashable] = set()  # list of data_vars     dim_coords: dict[Hashable, Variable] = {}  # maps dim name to variable     dims_sizes: dict[Hashable, int] = {}  # shared dimension sizes to expand variables    variables_order: dict[Hashable, Variable] = {}  # variables in order of appearance      for ds in datasets:         dims_sizes.update(ds.dims)         all_coord_names.update(ds.coords)         data_vars.update(ds.data_vars)        variables_order.update(ds.variables)          # preserves ordering of dimensions         for dim in ds.dims:


Question: dim_coords[dim] = ds.coords[dim].variable         dims = dims | set(ds.dims)     return dim_coords, dims_sizes, all_coord_names, data_vars    return dim_coords, dims_sizes, all_coord_names, data_vars, list(variables_order)   def _dataset_concat(


Question: coords: str | list[str],     compat: CompatOptions,     positions: Iterable[Iterable[int]] | None,    fill_value: object = dtypes.NA,    fill_value: Any = dtypes.NA,     join: JoinOptions = "outer",     combine_attrs: CombineAttrsOptions = "override", ) -> T_Dataset:


Question: align(*datasets, join=join, copy=False, exclude=[dim], fill_value=fill_value)     )     dim_coords, dims_sizes, coord_names, data_names = _parse_datasets(datasets)    dim_coords, dims_sizes, coord_names, data_names, vars_order = _parse_datasets(        datasets    )     dim_names = set(dim_coords)     unlabeled_dims = dim_names - coord_names


Question: # we've already verified everything is consistent; now, calculate     # shared dimension sizes so we can expand the necessary variables    def ensure_common_dims(vars):    def ensure_common_dims(vars, concat_dim_lengths):         # ensure each variable with the given name shares the same         # dimensions and the same shape for all of them except along the         # concat dimension


Question: data = var.set_dims(dim).values                     yield PandasIndex(data, dim, coord_dtype=var.dtype)     # create concatenation index, needed for later reindexing    concat_index = list(range(sum(concat_dim_lengths)))     # stack up each variable and/or index to fill-out the dataset (in order)     # n.b. this loop preserves variable order, needed for groupby.    for name in datasets[0].variables:    for name in vars_order:         if name in concat_over and name not in result_indexes:            try:                vars = ensure_common_dims([ds[name].variable for ds in datasets])            except KeyError:                raise ValueError(f"{name!r} is not present in all datasets.")            # Try concatenate the indexes, concatenate the variables when no index            variables = []            variable_index = []            var_concat_dim_length = []            for i, ds in enumerate(datasets):                if name in ds.variables:                    variables.append(ds[name].variable)                    # add to variable index, needed for reindexing                    var_idx = [                        sum(concat_dim_lengths[:i]) + k                        for k in range(concat_dim_lengths[i])                    ]                    variable_index.extend(var_idx)                    var_concat_dim_length.append(len(var_idx))                else:                    # raise if coordinate not in all datasets                    if name in coord_names:                        raise ValueError(                            f"coordinate {name!r} not present in all datasets."                        )            vars = ensure_common_dims(variables, var_concat_dim_length)            # Try to concatenate the indexes, concatenate the variables when no index             # is found on all datasets.             indexes: list[Index] = list(get_indexes(name))             if indexes:


Question: combined_var = concat_vars(                     vars, dim, positions, combine_attrs=combine_attrs                 )<ins>                # reindex if variable is not present in all datasets                if len(variable_index) < len(concat_index):                    combined_var = reindex_variables(                        variables={name: combined_var},                        dim_pos_indexers={                            dim: pd.Index(variable_index).get_indexer(concat_index)                        },                        fill_value=fill_value,                    )[name]</ins>                 result_vars[name] = combined_var          elif name in result_vars:


Question: pool.join()      _merge_mapreduce_data(linter, all_mapreduce_data)    linter.stats = _merge_stats(all_stats)    linter.stats = _merge_stats([linter.stats] + all_stats)      # Insert stats data to local checkers.     for checker in linter.get_checkers():

Pylint 2.7.0 seems to ignore the min-similarity-lines setting <!-- Hi there! Thank you for discovering and submitting an issue. Before you submit this, make sure that the issue doesn't already exist or if it is not closed. Is your issue fixed on the preview release?: pip install pylint astroid --pre -U --two Python source files that share 8 common lines min-similarity-lines=40 in the pylint config pylint 2.7.0 on the source files Before pylint 2.7.0, the min-similarity-lines setting was honored and caused shorter similar lines to be accepted. Starting with pylint 2.7.0, the min-similarity-lines setting seems to be ignored and the common lines are always reported as an issue R0801, even when the min-similarity-lines setting is significantly larger than the number of common lines. The min-similarity-lines setting should be respected again as it was before pylint 2.7.0. pylint 2.7.0 astroid 2.5 Python 3.9.1 (default, Feb 1 2021, 20:41:56) [Clang 12.0.0 (clang-1200.0.32.29)] Pylint 2.7.0 seems to ignore the min-similarity-lines setting <!-- Hi there! Thank you for discovering and submitting an issue. Before you submit this, make sure that the issue doesn't already exist or if it is not closed. Is your issue fixed on the preview release?: pip install pylint astroid --pre -U --two Python source files that share 8 common lines min-similarity-lines=40 in the pylint config pylint 2.7.0 on the source files Before pylint 2.7.0, the min-similarity-lines setting was honored and caused shorter similar lines to be accepted. Starting with pylint 2.7.0, the min-similarity-lines setting seems to be ignored and the common lines are always reported as an issue R0801, even when the min-similarity-lines setting is significantly larger than the number of common lines.


Question: pool.join()      _merge_mapreduce_data(linter, all_mapreduce_data)    linter.stats = _merge_stats(all_stats)    linter.stats = _merge_stats([linter.stats] + all_stats)      # Insert stats data to local checkers.     for checker in linter.get_checkers():

The min-similarity-lines setting should be respected again as it was before pylint 2.7.0. pylint 2.7.0 astroid 2.5 Python 3.9.1 (default, Feb 1 2021, 20:41:56) [Clang 12.0.0 (clang-1200.0.32.29)]


Question: do_exit=UNUSED_PARAM_SENTINEL,     ):  # pylint: disable=redefined-builtin         self._rcfile = None<ins>        self._output = None</ins>         self._version_asked = False         self._plugins = []         self.verbose = None

Support output to a file to support tox on CI servers Currently , which means when running Pylint from within there seems to be no way to output the results to a single (for example HTML) file that a CI server can store for display. used to be close to what is needed, but it , and also created a different file for each module which is difficult to navigate. Here is the that relates to this. this feature for the same reason:


Question: "rcfile": (self.cb_set_rcfile, True),                     "load-plugins": (self.cb_add_plugins, True),                     "verbose": (self.cb_verbose_mode, False),<ins>                    "output": (self.cb_set_output, True),</ins>                 },             )         except ArgumentPreprocessingError as ex:


Question: "help": "Specify a configuration file to load.",                     },                 ),<ins>                (                    "output",                    {                        "action": "callback",                        "callback": Run._return_one,                        "group": "Commands",                        "type": "string",                        "metavar": "<file>",                        "help": "Specify an output file.",                    },                ),</ins>                 (                     "init-hook",                     {


Question: # load plugin specific configuration.         linter.load_plugin_configuration()         linter.check(args)        score_value = linter.generate_reports()        if self._output:            try:                with open(self._output, "w") as output:                    linter.reporter.set_output(output)                    linter.check(args)                    score_value = linter.generate_reports()            except OSError as ex:                print(ex, file=sys.stderr)                sys.exit(32)        else:            linter.check(args)            score_value = linter.generate_reports()          if do_exit is not UNUSED_PARAM_SENTINEL:             warnings.warn(


Question: """callback for option preprocessing (i.e. before option parsing)"""         self._rcfile = value <ins>    def cb_set_output(self, name, value):        """callback for option preprocessing (i.e. before option parsing)"""        self._output = value</ins>     def cb_add_plugins(self, name, value):         """callback for option preprocessing (i.e. before option parsing)"""         self._plugins.extend(utils._splitstrip(value))


Question: "help": "Specify a score threshold to be exceeded before program exits with error.",                 },             ),<ins>            (                "fail-on",                {                    "default": "",                    "type": "csv",                    "metavar": "<msg ids>",                    "help": "Return non-zero exit code if any of these messages/categories are detected,"                    " even if score is above --fail-under value. Syntax same as enable."                    " Messages specified are enabled, while categories only check already-enabled messages.",                },            ),</ins>             (                 "confidence",                 {

Add --fail-on option to always return error code if specific issues (or issue types) are found We're using pylint in CI with two primary goals: there are no error-category issues - this can be achieved by looking at the exit code the overall linting score doesn't deteriorate too much - this can be achieved by using However if is used, and the score is above that, then it passes even if there are error-category issues detected. Essentially it's acting as a "only throw fail (non-zero) codes if under this", instead of a "fail if under this, otherwise change nothing". Two possible solutions I can think of here are: a configuration option to prevent from consuming other exit statuses. I.e. a way to say "if score is under N, return error code X, regardless of all else, otherwise change nothing". a new option like which means that if there are any code (error-category) issues, or any issues, then fail, otherwise change nothing.


Question: self.current_name = None         self.current_file = None         self.stats = None<ins>        self.fail_on_symbols = []</ins>         # init options         self._external_opts = options         self.options = options + PyLinter.make_options()


Question: if not getattr(checker, "enabled", True):             self.disable(checker.name)     def enable_fail_on_messages(self):        """enable 'fail on' msgs        Convert values in config.fail_on (which might be msg category, msg id,        or symbol) to specific msgs, then enable and flag them for later.        """        fail_on_vals = self.config.fail_on        if not fail_on_vals:            return        fail_on_cats = set()        fail_on_msgs = set()        for val in fail_on_vals:            # If value is a cateogry, add category, else add message            if val in MSG_TYPES:                fail_on_cats.add(val)            else:                fail_on_msgs.add(val)        # For every message in every checker, if cat or msg flagged, enable check        for all_checkers in self._checkers.values():            for checker in all_checkers:                for msg in checker.messages:                    if msg.msgid in fail_on_msgs or msg.symbol in fail_on_msgs:                        # message id/symbol matched, enable and flag it                        self.enable(msg.msgid)                        self.fail_on_symbols.append(msg.symbol)                    elif msg.msgid[0] in fail_on_cats:                        # message starts with a cateogry value, flag (but do not enable) it                        self.fail_on_symbols.append(msg.symbol)    def any_fail_on_issues(self):        return any(x in self.fail_on_symbols for x in self.stats["by_msg"])     def disable_noerror_messages(self):         for msgcat, msgids in self.msgs_store._msgs_by_category.items():             # enable only messages with 'error' severity and above ('fatal')diff --git a/pylint/lint/run.py b/pylint/lint/run.py-- a/pylint/lint/run.py++ b/pylint/lint/run.py


Question: # load plugin specific configuration.         linter.load_plugin_configuration() <ins>        # Now that plugins are loaded, get list of all fail_on messages, and enable them        linter.enable_fail_on_messages()</ins>         if self._output:             try:                 with open(self._output, "w") as output:


Question: if linter.config.exit_zero:                 sys.exit(0)             else:                if score_value and score_value >= linter.config.fail_under:                if (                    score_value                    and score_value >= linter.config.fail_under                    # detected messages flagged by --fail-on prevent non-zero exit code                    and not linter.any_fail_on_issues()                ):                     sys.exit(0)                 sys.exit(self.linter.msg_status)


Question: # Licensed under the GPL: https://www.gnu.org/licenses/old-licenses/gpl-2.0.html # For details: https://github.com/PyCQA/pylint/blob/master/LICENSE<ins>from typing import Tuple</ins>  from pkg_resources import DistributionNotFound, get_distribution

Pylint 2.8.2 broke pylint-quotes <!-- Hi there! Thank you for discovering and submitting an issue. Before you submit this, make sure that the issue doesn't already exist or if it is not closed. Is your issue fixed on the preview release?: pip install pylint astroid --pre -U --See See in attached Running this configuration on a file containing: results in an exception: should not throw exceptions, and instead generate report complaining about the use of double quotes () and illegal variable name . Result of output: Additional dependencies: seems to have just changed versioning scheme: broke this line:


Question: except DistributionNotFound:     __version__ = "2.8.2+" # Kept for compatibility reason, see https://github.com/PyCQA/pylint/issues/4399numversion = tuple(__version__.split("."))def get_numversion_from_version(v: str) -> Tuple:    """Kept for compatibility reason    See https://github.com/PyCQA/pylint/issues/4399    https://github.com/PyCQA/pylint/issues/4420,    """    v = v.replace("pylint-", "")    version = []    for n in v.split(".")[0:3]:        try:            version.append(int(n))        except ValueError:            num = ""            for c in n:                if c.isdigit():                    num += c                else:                    break            try:                version.append(int(num))            except ValueError:                version.append(0)    while len(version) != 3:        version.append(0)    return tuple(version)numversion = get_numversion_from_version(__version__)


Question: return sys.stdin.read()  <ins>def _load_reporter_by_class(reporter_class: str) -> type:    qname = reporter_class    module_part = astroid.modutils.get_module_part(qname)    module = astroid.modutils.load_module_from_name(module_part)    class_name = qname.split(".")[-1]    return getattr(module, class_name)</ins> # Python Linter class #########################################################  MSGS = {

Allow parseable output in parallel of human-readable output Currently, when selecting a format like JSON using the argument, it will replace the "normal" output. It would be nice to have it generate an output-file instead so the normal human-readable output is still there. This makes sens in a CI environment like travis or gitlab where you can see the stdout and stderr of each job but where you also want to parse the output. Allowing multiple output formats would solve this. Maybe with a new CLI argument? Or by adding a separator to the current value? For example: pylint -f # Would redirect the JSON data to and still write the normal report to stdout pylint 1.8.1, astroid 1.6.0 Python 3.5.2 (default, Nov 23 2017, 16:37:01) [GCC 5.4.0 ]


Question: messages store / checkers / reporter / astroid manager"""         self.msgs_store = MessageDefinitionStore()         self.reporter = None        self._reporter_name = None        self._reporter_names = None         self._reporters = {}         self._checkers = collections.defaultdict(list)         self._pragma_lineno = {}


Question: # Make sure to load the default reporter, because         # the option has been set before the plugins had been loaded.         if not self.reporter:            self._load_reporter()            self._load_reporters()      def load_plugin_modules(self, modnames):         """take a list of module names which are pylint plugins and load