-
Notifications
You must be signed in to change notification settings - Fork 283
/
util.py
1947 lines (1526 loc) · 61.7 KB
/
util.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
# Copyright Iris contributors
#
# This file is part of Iris and is released under the LGPL license.
# See COPYING and COPYING.LESSER in the root of the repository for full
# licensing details.
"""
Miscellaneous utility functions.
"""
from abc import ABCMeta, abstractmethod
from collections.abc import Hashable, Iterable
from contextlib import contextmanager
import copy
import functools
import inspect
import os
import os.path
import sys
import tempfile
import cf_units
from dask import array as da
import numpy as np
import numpy.ma as ma
from iris._deprecation import warn_deprecated
from iris._lazy_data import is_lazy_data
import iris.exceptions
def broadcast_to_shape(array, shape, dim_map):
"""
Broadcast an array to a given shape.
Each dimension of the array must correspond to a dimension in the
given shape. Striding is used to repeat the array until it matches
the desired shape, returning repeated views on the original array.
If you need to write to the resulting array, make a copy first.
Args:
* array (:class:`numpy.ndarray`-like)
An array to broadcast.
* shape (:class:`list`, :class:`tuple` etc.):
The shape the array should be broadcast to.
* dim_map (:class:`list`, :class:`tuple` etc.):
A mapping of the dimensions of *array* to their corresponding
element in *shape*. *dim_map* must be the same length as the
number of dimensions in *array*. Each element of *dim_map*
corresponds to a dimension of *array* and its value provides
the index in *shape* which the dimension of *array* corresponds
to, so the first element of *dim_map* gives the index of *shape*
that corresponds to the first dimension of *array* etc.
Examples:
Broadcasting an array of shape (2, 3) to the shape (5, 2, 6, 3)
where the first dimension of the array corresponds to the second
element of the desired shape and the second dimension of the array
corresponds to the fourth element of the desired shape::
a = np.array([[1, 2, 3], [4, 5, 6]])
b = broadcast_to_shape(a, (5, 2, 6, 3), (1, 3))
Broadcasting an array of shape (48, 96) to the shape (96, 48, 12)::
# a is an array of shape (48, 96)
result = broadcast_to_shape(a, (96, 48, 12), (1, 0))
"""
if len(dim_map) != array.ndim:
# We must check for this condition here because we cannot rely on
# getting an error from numpy if the dim_map argument is not the
# correct length, we might just get a segfault.
raise ValueError(
"dim_map must have an entry for every "
"dimension of the input array"
)
def _broadcast_helper(a):
strides = [0] * len(shape)
for idim, dim in enumerate(dim_map):
if shape[dim] != a.shape[idim]:
# We'll get garbage values if the dimensions of array are not
# those indicated by shape.
raise ValueError("shape and array are not compatible")
strides[dim] = a.strides[idim]
return np.lib.stride_tricks.as_strided(a, shape=shape, strides=strides)
array_view = _broadcast_helper(array)
if ma.isMaskedArray(array):
if array.mask is ma.nomask:
# Degenerate masks can be applied as-is.
mask_view = array.mask
else:
# Mask arrays need to be handled in the same way as the data array.
mask_view = _broadcast_helper(array.mask)
array_view = ma.array(array_view, mask=mask_view)
return array_view
def delta(ndarray, dimension, circular=False):
"""
Calculates the difference between values along a given dimension.
Args:
* ndarray:
The array over which to do the difference.
* dimension:
The dimension over which to do the difference on ndarray.
* circular:
If not False then return n results in the requested dimension
with the delta between the last and first element included in
the result otherwise the result will be of length n-1 (where n
is the length of ndarray in the given dimension's direction)
If circular is numeric then the value of circular will be added
to the last element of the given dimension if the last element
is negative, otherwise the value of circular will be subtracted
from the last element.
The example below illustrates the process::
original array -180, -90, 0, 90
delta (with circular=360): 90, 90, 90, -270+360
.. note::
The difference algorithm implemented is forward difference:
>>> import numpy as np
>>> import iris.util
>>> original = np.array([-180, -90, 0, 90])
>>> iris.util.delta(original, 0)
array([90, 90, 90])
>>> iris.util.delta(original, 0, circular=360)
array([90, 90, 90, 90])
"""
if circular is not False:
_delta = np.roll(ndarray, -1, axis=dimension)
last_element = [slice(None, None)] * ndarray.ndim
last_element[dimension] = slice(-1, None)
last_element = tuple(last_element)
if not isinstance(circular, bool):
result = np.where(ndarray[last_element] >= _delta[last_element])[0]
_delta[last_element] -= circular
_delta[last_element][result] += 2 * circular
np.subtract(_delta, ndarray, _delta)
else:
_delta = np.diff(ndarray, axis=dimension)
return _delta
def describe_diff(cube_a, cube_b, output_file=None):
"""
Prints the differences that prevent compatibility between two cubes, as
defined by :meth:`iris.cube.Cube.is_compatible()`.
Args:
* cube_a:
An instance of :class:`iris.cube.Cube` or
:class:`iris.cube.CubeMetadata`.
* cube_b:
An instance of :class:`iris.cube.Cube` or
:class:`iris.cube.CubeMetadata`.
* output_file:
A :class:`file` or file-like object to receive output. Defaults to
sys.stdout.
.. seealso::
:meth:`iris.cube.Cube.is_compatible()`
.. note::
Compatibility does not guarantee that two cubes can be merged.
Instead, this function is designed to provide a verbose description
of the differences in metadata between two cubes. Determining whether
two cubes will merge requires additional logic that is beyond the
scope of this function.
"""
if output_file is None:
output_file = sys.stdout
if cube_a.is_compatible(cube_b):
output_file.write("Cubes are compatible\n")
else:
common_keys = set(cube_a.attributes).intersection(cube_b.attributes)
for key in common_keys:
if np.any(cube_a.attributes[key] != cube_b.attributes[key]):
output_file.write(
'"%s" cube_a attribute value "%s" is not '
"compatible with cube_b "
'attribute value "%s"\n'
% (key, cube_a.attributes[key], cube_b.attributes[key])
)
if cube_a.name() != cube_b.name():
output_file.write(
'cube_a name "%s" is not compatible '
'with cube_b name "%s"\n' % (cube_a.name(), cube_b.name())
)
if cube_a.units != cube_b.units:
output_file.write(
'cube_a units "%s" are not compatible with cube_b units "%s"\n'
% (cube_a.units, cube_b.units)
)
if cube_a.cell_methods != cube_b.cell_methods:
output_file.write(
"Cell methods\n%s\nand\n%s\nare not compatible\n"
% (cube_a.cell_methods, cube_b.cell_methods)
)
def guess_coord_axis(coord):
"""
Returns a "best guess" axis name of the coordinate.
Heuristic categorisation of the coordinate into either label
'T', 'Z', 'Y', 'X' or None.
Args:
* coord:
The :class:`iris.coords.Coord`.
Returns:
'T', 'Z', 'Y', 'X', or None.
"""
axis = None
if coord.standard_name in (
"longitude",
"grid_longitude",
"projection_x_coordinate",
):
axis = "X"
elif coord.standard_name in (
"latitude",
"grid_latitude",
"projection_y_coordinate",
):
axis = "Y"
elif coord.units.is_convertible("hPa") or coord.attributes.get(
"positive"
) in ("up", "down"):
axis = "Z"
elif coord.units.is_time_reference():
axis = "T"
return axis
def rolling_window(a, window=1, step=1, axis=-1):
"""
Make an ndarray with a rolling window of the last dimension
Args:
* a : array_like
Array to add rolling window to
Kwargs:
* window : int
Size of rolling window
* step : int
Size of step between rolling windows
* axis : int
Axis to take the rolling window over
Returns:
Array that is a view of the original array with an added dimension
of the size of the given window at axis + 1.
Examples::
>>> x = np.arange(10).reshape((2, 5))
>>> rolling_window(x, 3)
array([[[0, 1, 2], [1, 2, 3], [2, 3, 4]],
[[5, 6, 7], [6, 7, 8], [7, 8, 9]]])
Calculate rolling mean of last dimension::
>>> np.mean(rolling_window(x, 3), -1)
array([[ 1., 2., 3.],
[ 6., 7., 8.]])
"""
# NOTE: The implementation of this function originates from
# https://github.com/numpy/numpy/pull/31#issuecomment-1304851 04/08/2011
if window < 1:
raise ValueError("`window` must be at least 1.")
if window > a.shape[axis]:
raise ValueError("`window` is too long.")
if step < 1:
raise ValueError("`step` must be at least 1.")
axis = axis % a.ndim
num_windows = (a.shape[axis] - window + step) // step
shape = a.shape[:axis] + (num_windows, window) + a.shape[axis + 1 :]
strides = (
a.strides[:axis]
+ (step * a.strides[axis], a.strides[axis])
+ a.strides[axis + 1 :]
)
rw = np.lib.stride_tricks.as_strided(a, shape=shape, strides=strides)
if ma.isMaskedArray(a):
mask = ma.getmaskarray(a)
strides = (
mask.strides[:axis]
+ (step * mask.strides[axis], mask.strides[axis])
+ mask.strides[axis + 1 :]
)
rw = ma.array(
rw,
mask=np.lib.stride_tricks.as_strided(
mask, shape=shape, strides=strides
),
)
return rw
def array_equal(array1, array2, withnans=False):
"""
Returns whether two arrays have the same shape and elements.
Args:
* array1, array2 (arraylike):
args to be compared, after normalising with :func:`np.asarray`.
Kwargs:
* withnans (bool):
When unset (default), the result is False if either input contains NaN
points. This is the normal floating-point arithmetic result.
When set, return True if inputs contain the same value in all elements,
_including_ any NaN values.
This provides much the same functionality as :func:`numpy.array_equal`, but
with additional support for arrays of strings and NaN-tolerant operation.
"""
array1, array2 = np.asarray(array1), np.asarray(array2)
eq = array1.shape == array2.shape
if eq:
eqs = array1 == array2
if withnans and (array1.dtype.kind == "f" or array2.dtype.kind == "f"):
nans1, nans2 = np.isnan(array1), np.isnan(array2)
if not np.all(nans1 == nans2):
eq = False # simply fail
else:
eqs[nans1] = True # fix NaNs; check all the others
if eq:
eq = np.all(eqs) # check equal at all points
return eq
def approx_equal(a, b, max_absolute_error=1e-10, max_relative_error=1e-10):
"""
Returns whether two numbers are almost equal, allowing for the
finite precision of floating point numbers.
"""
# Deal with numbers close to zero
if abs(a - b) < max_absolute_error:
return True
# Ensure we get consistent results if "a" and "b" are supplied in the
# opposite order.
max_ab = max([a, b], key=abs)
relative_error = abs(a - b) / max_ab
return relative_error < max_relative_error
def between(lh, rh, lh_inclusive=True, rh_inclusive=True):
"""
Provides a convenient way of defining a 3 element inequality such as
``a < number < b``.
Arguments:
* lh
The left hand element of the inequality
* rh
The right hand element of the inequality
Keywords:
* lh_inclusive - boolean
Affects the left hand comparison operator to use in the inequality.
True for ``<=`` false for ``<``. Defaults to True.
* rh_inclusive - boolean
Same as lh_inclusive but for right hand operator.
For example::
between_3_and_6 = between(3, 6)
for i in range(10):
print(i, between_3_and_6(i))
between_3_and_6 = between(3, 6, rh_inclusive=False)
for i in range(10):
print(i, between_3_and_6(i))
"""
if lh_inclusive and rh_inclusive:
return lambda c: lh <= c <= rh
elif lh_inclusive and not rh_inclusive:
return lambda c: lh <= c < rh
elif not lh_inclusive and rh_inclusive:
return lambda c: lh < c <= rh
else:
return lambda c: lh < c < rh
def reverse(cube_or_array, coords_or_dims):
"""
Reverse the cube or array along the given dimensions.
Args:
* cube_or_array: :class:`iris.cube.Cube` or :class:`numpy.ndarray`
The cube or array to reverse.
* coords_or_dims: int, str, :class:`iris.coords.Coord` or sequence of these
Identify one or more dimensions to reverse. If cube_or_array is a
numpy array, use int or a sequence of ints, as in the examples below.
If cube_or_array is a Cube, a Coord or coordinate name (or sequence of
these) may be specified instead.
::
>>> import numpy as np
>>> a = np.arange(24).reshape(2, 3, 4)
>>> print(a)
[[[ 0 1 2 3]
[ 4 5 6 7]
[ 8 9 10 11]]
<BLANKLINE>
[[12 13 14 15]
[16 17 18 19]
[20 21 22 23]]]
>>> print(reverse(a, 1))
[[[ 8 9 10 11]
[ 4 5 6 7]
[ 0 1 2 3]]
<BLANKLINE>
[[20 21 22 23]
[16 17 18 19]
[12 13 14 15]]]
>>> print(reverse(a, [1, 2]))
[[[11 10 9 8]
[ 7 6 5 4]
[ 3 2 1 0]]
<BLANKLINE>
[[23 22 21 20]
[19 18 17 16]
[15 14 13 12]]]
"""
from iris.cube import Cube
index = [slice(None, None)] * cube_or_array.ndim
if isinstance(coords_or_dims, Cube):
raise TypeError(
"coords_or_dims must be int, str, coordinate or "
"sequence of these. Got cube."
)
if isinstance(coords_or_dims, str) or not isinstance(
coords_or_dims, Iterable
):
coords_or_dims = [coords_or_dims]
axes = set()
for coord_or_dim in coords_or_dims:
if isinstance(coord_or_dim, int):
axes.add(coord_or_dim)
elif isinstance(cube_or_array, np.ndarray):
raise TypeError(
"To reverse an array, provide an int or sequence of ints."
)
else:
try:
axes.update(cube_or_array.coord_dims(coord_or_dim))
except AttributeError:
raise TypeError(
"coords_or_dims must be int, str, coordinate "
"or sequence of these."
)
axes = np.array(list(axes), ndmin=1)
if axes.ndim != 1 or axes.size == 0:
raise ValueError(
"Reverse was expecting a single axis or a 1d array "
"of axes, got %r" % axes
)
if np.min(axes) < 0 or np.max(axes) > cube_or_array.ndim - 1:
raise ValueError(
"An axis value out of range for the number of "
"dimensions from the given array (%s) was received. "
"Got: %r" % (cube_or_array.ndim, axes)
)
for axis in axes:
index[axis] = slice(None, None, -1)
return cube_or_array[tuple(index)]
def monotonic(array, strict=False, return_direction=False):
"""
Return whether the given 1d array is monotonic.
Note that, the array must not contain missing data.
Kwargs:
* strict (boolean)
Flag to enable strict monotonic checking
* return_direction (boolean)
Flag to change return behaviour to return
(monotonic_status, direction). Direction will be 1 for positive
or -1 for negative. The direction is meaningless if the array is
not monotonic.
Returns:
* monotonic_status (boolean)
Whether the array was monotonic.
If the return_direction flag was given then the returned value
will be:
``(monotonic_status, direction)``
"""
if array.ndim != 1 or len(array) <= 1:
raise ValueError(
"The array to check must be 1 dimensional and have "
"more than 1 element."
)
if ma.isMaskedArray(array) and ma.count_masked(array) != 0:
raise ValueError("The array to check contains missing data.")
# Identify the directions of the largest/most-positive and
# smallest/most-negative steps.
d = np.diff(array)
sign_max_d = np.sign(np.max(d))
sign_min_d = np.sign(np.min(d))
if strict:
monotonic = sign_max_d == sign_min_d and sign_max_d != 0
else:
monotonic = (
(sign_min_d < 0 and sign_max_d <= 0)
or (sign_max_d > 0 and sign_min_d >= 0)
or (sign_min_d == sign_max_d == 0)
)
if return_direction:
if sign_max_d == 0:
direction = sign_min_d
else:
direction = sign_max_d
return monotonic, direction
return monotonic
def column_slices_generator(full_slice, ndims):
"""
Given a full slice full of tuples, return a dictionary mapping old
data dimensions to new and a generator which gives the successive
slices needed to index correctly (across columns).
This routine deals with the special functionality for tuple based
indexing e.g. [0, (3, 5), :, (1, 6, 8)] by first providing a slice
which takes the non tuple slices out first i.e. [0, :, :, :] then
subsequently iterates through each of the tuples taking out the
appropriate slices i.e. [(3, 5), :, :] followed by [:, :, (1, 6, 8)]
This method was developed as numpy does not support the direct
approach of [(3, 5), : , (1, 6, 8)] for column based indexing.
"""
list_of_slices = []
# Map current dimensions to new dimensions, or None
dimension_mapping = {None: None}
_count_current_dim = 0
for i, i_key in enumerate(full_slice):
if isinstance(i_key, (int, np.integer)):
dimension_mapping[i] = None
else:
dimension_mapping[i] = _count_current_dim
_count_current_dim += 1
# Get all of the dimensions for which a tuple of indices were provided
# (numpy.ndarrays are treated in the same way tuples in this case)
def is_tuple_style_index(key):
return isinstance(key, tuple) or (
isinstance(key, np.ndarray) and key.ndim == 1
)
tuple_indices = [
i for i, key in enumerate(full_slice) if is_tuple_style_index(key)
]
# stg1: Take a copy of the full_slice specification, turning all tuples
# into a full slice
if tuple_indices != list(range(len(full_slice))):
first_slice = list(full_slice)
for tuple_index in tuple_indices:
first_slice[tuple_index] = slice(None, None)
# turn first_slice back into a tuple ready for indexing
first_slice = tuple(first_slice)
list_of_slices.append(first_slice)
# stg2 iterate over each of the tuples
for tuple_index in tuple_indices:
# Create a list with the indices to span the whole data array that we
# currently have
spanning_slice_with_tuple = [slice(None, None)] * _count_current_dim
# Replace the slice(None, None) with our current tuple
spanning_slice_with_tuple[dimension_mapping[tuple_index]] = full_slice[
tuple_index
]
# if we just have [(0, 1)] turn it into [(0, 1), ...] as this is
# Numpy's syntax.
if len(spanning_slice_with_tuple) == 1:
spanning_slice_with_tuple.append(Ellipsis)
spanning_slice_with_tuple = tuple(spanning_slice_with_tuple)
list_of_slices.append(spanning_slice_with_tuple)
# return the dimension mapping and a generator of slices
return dimension_mapping, iter(list_of_slices)
def _build_full_slice_given_keys(keys, ndim):
"""
Given the keys passed to a __getitem__ call, build an equivalent
tuple of keys which span ndims.
"""
# Ensure that we always have a tuple of keys
if not isinstance(keys, tuple):
keys = tuple([keys])
# catch the case where an extra Ellipsis has been provided which can be
# discarded iff len(keys)-1 == ndim
if len(keys) - 1 == ndim and Ellipsis in filter(
lambda obj: not isinstance(obj, np.ndarray), keys
):
keys = list(keys)
is_ellipsis = [key is Ellipsis for key in keys]
keys.pop(is_ellipsis.index(True))
keys = tuple(keys)
# for ndim >= 1 appending a ":" to the slice specification is allowable,
# remove this now
if len(keys) > ndim and ndim != 0 and keys[-1] == slice(None, None):
keys = keys[:-1]
if len(keys) > ndim:
raise IndexError(
"More slices requested than dimensions. Requested "
"%r, but there were only %s dimensions." % (keys, ndim)
)
# For each dimension get the slice which has been requested.
# If no slice provided, then default to the whole dimension
full_slice = [slice(None, None)] * ndim
for i, key in enumerate(keys):
if key is Ellipsis:
# replace any subsequent Ellipsis objects in keys with
# slice(None, None) as per Numpy
keys = keys[:i] + tuple(
[
slice(None, None) if key is Ellipsis else key
for key in keys[i:]
]
)
# iterate over the remaining keys in reverse to fill in
# the gaps from the right hand side
for j, key in enumerate(keys[:i:-1]):
full_slice[-j - 1] = key
# we've finished with i now so stop the iteration
break
else:
full_slice[i] = key
# remove any tuples on dimensions, turning them into numpy array's for
# consistent behaviour
full_slice = tuple(
[
np.array(key, ndmin=1) if isinstance(key, tuple) else key
for key in full_slice
]
)
return full_slice
def _slice_data_with_keys(data, keys):
"""
Index an array-like object as "data[keys]", with orthogonal indexing.
Args:
* data (array-like):
array to index.
* keys (list):
list of indexes, as received from a __getitem__ call.
This enforces an orthogonal interpretation of indexing, which means that
both 'real' (numpy) arrays and other array-likes index in the same way,
instead of numpy arrays doing 'fancy indexing'.
Returns (dim_map, data_region), where :
* dim_map (dict) :
A dimension map, as returned by :func:`column_slices_generator`.
i.e. "dim_map[old_dim_index]" --> "new_dim_index" or None.
* data_region (array-like) :
The sub-array.
.. Note::
Avoids copying the data, where possible.
"""
# Combines the use of _build_full_slice_given_keys and
# column_slices_generator.
# By slicing on only one index at a time, this also mostly avoids copying
# the data, except some cases when a key contains a list of indices.
n_dims = len(data.shape)
full_slice = _build_full_slice_given_keys(keys, n_dims)
dims_mapping, slices_iter = column_slices_generator(full_slice, n_dims)
for this_slice in slices_iter:
data = data[this_slice]
if data.ndim > 0 and min(data.shape) < 1:
# Disallow slicings where a dimension has no points, like "[5:5]".
raise IndexError("Cannot index with zero length slice.")
return dims_mapping, data
def _wrap_function_for_method(function, docstring=None):
"""
Returns a wrapper function modified to be suitable for use as a
method.
The wrapper function renames the first argument as "self" and allows
an alternative docstring, thus allowing the built-in help(...)
routine to display appropriate output.
"""
# Generate the Python source for the wrapper function.
# NB. The first argument is replaced with "self".
args, varargs, varkw, defaults = inspect.getargspec(function)
if defaults is None:
basic_args = ["self"] + args[1:]
default_args = []
simple_default_args = []
else:
cutoff = -len(defaults)
basic_args = ["self"] + args[1:cutoff]
default_args = [
"%s=%r" % pair for pair in zip(args[cutoff:], defaults)
]
simple_default_args = args[cutoff:]
var_arg = [] if varargs is None else ["*" + varargs]
var_kw = [] if varkw is None else ["**" + varkw]
arg_source = ", ".join(basic_args + default_args + var_arg + var_kw)
simple_arg_source = ", ".join(
basic_args + simple_default_args + var_arg + var_kw
)
source = "def %s(%s):\n return function(%s)" % (
function.__name__,
arg_source,
simple_arg_source,
)
# Compile the wrapper function
# NB. There's an outstanding bug with "exec" where the locals and globals
# dictionaries must be the same if we're to get closure behaviour.
my_locals = {"function": function}
exec(source, my_locals, my_locals)
# Update the docstring if required, and return the modified function
wrapper = my_locals[function.__name__]
if docstring is None:
wrapper.__doc__ = function.__doc__
else:
wrapper.__doc__ = docstring
return wrapper
class _MetaOrderedHashable(ABCMeta):
"""
A metaclass that ensures that non-abstract subclasses of _OrderedHashable
without an explicit __init__ method are given a default __init__ method
with the appropriate method signature.
Also, an _init method is provided to allow subclasses with their own
__init__ constructors to initialise their values via an explicit method
signature.
NB. This metaclass is used to construct the _OrderedHashable class as well
as all its subclasses.
"""
def __new__(cls, name, bases, namespace):
# We only want to modify concrete classes that have defined the
# "_names" property.
if "_names" in namespace and not getattr(
namespace["_names"], "__isabstractmethod__", False
):
args = ", ".join(namespace["_names"])
# Ensure the class has a constructor with explicit arguments.
if "__init__" not in namespace:
# Create a default __init__ method for the class
method_source = (
"def __init__(self, %s):\n "
"self._init_from_tuple((%s,))" % (args, args)
)
exec(method_source, namespace)
# Ensure the class has a "helper constructor" with explicit
# arguments.
if "_init" not in namespace:
# Create a default _init method for the class
method_source = (
"def _init(self, %s):\n "
"self._init_from_tuple((%s,))" % (args, args)
)
exec(method_source, namespace)
return super().__new__(cls, name, bases, namespace)
@functools.total_ordering
class _OrderedHashable(Hashable, metaclass=_MetaOrderedHashable):
"""
Convenience class for creating "immutable", hashable, and ordered classes.
Instance identity is defined by the specific list of attribute names
declared in the abstract attribute "_names". Subclasses must declare the
attribute "_names" as an iterable containing the names of all the
attributes relevant to equality/hash-value/ordering.
Initial values should be set by using ::
self._init(self, value1, value2, ..)
.. note::
It's the responsibility of the subclass to ensure that the values of
its attributes are themselves hashable.
"""
@property
@abstractmethod
def _names(self):
"""
Override this attribute to declare the names of all the attributes
relevant to the hash/comparison semantics.
"""
pass
def _init_from_tuple(self, values):
for name, value in zip(self._names, values):
object.__setattr__(self, name, value)
def __repr__(self):
class_name = type(self).__name__
attributes = ", ".join(
"%s=%r" % (name, value)
for (name, value) in zip(self._names, self._as_tuple())
)
return "%s(%s)" % (class_name, attributes)
def _as_tuple(self):
return tuple(getattr(self, name) for name in self._names)
# Prevent attribute updates
def __setattr__(self, name, value):
raise AttributeError(
"Instances of %s are immutable" % type(self).__name__
)
def __delattr__(self, name):
raise AttributeError(
"Instances of %s are immutable" % type(self).__name__
)
# Provide hash semantics
def _identity(self):
return self._as_tuple()
def __hash__(self):
return hash(self._identity())
def __eq__(self, other):
return (
isinstance(other, type(self))
and self._identity() == other._identity()
)
def __ne__(self, other):
# Since we've defined __eq__ we should also define __ne__.
return not self == other
# Provide default ordering semantics
def __lt__(self, other):
if isinstance(other, _OrderedHashable):
return self._identity() < other._identity()
else:
return NotImplemented
def create_temp_filename(suffix=""):
"""Return a temporary file name.
Args:
* suffix - Optional filename extension.
"""
temp_file = tempfile.mkstemp(suffix)
os.close(temp_file[0])
return temp_file[1]
def clip_string(the_str, clip_length=70, rider="..."):
"""
Returns a clipped version of the string based on the specified clip
length and whether or not any graceful clip points can be found.
If the string to be clipped is shorter than the specified clip
length, the original string is returned.
If the string is longer than the clip length, a graceful point (a
space character) after the clip length is searched for. If a
graceful point is found the string is clipped at this point and the
rider is added. If no graceful point can be found, then the string
is clipped exactly where the user requested and the rider is added.
Args:
* the_str
The string to be clipped
* clip_length
The length in characters that the input string should be clipped
to. Defaults to a preconfigured value if not specified.
* rider