add ability to request regions with synonyms from IXMP web API (#285)

* Add ability to request regions with synonyms from IXMP web API

* Avoid dropping column if synonym_N column only is partially empty

* Add ability to request regions with synonyms from IXMP web API

- uncomment test

* Fix dtype when reading data from json

- fix e.g. reading model '1' and '1.0' as float (leading to duplicate error)

* Change getting regions from ixmp API

- switch to long format when output with synonyms
- add tests

* Update release notes

RELEASE_NOTES.md

@@ -6,7 +6,7 @@
 - [#295](https://github.com/IAMconsortium/pyam/pull/295) Include `meta` table when writing to or reading from `xlsx` files
 - [#292](https://github.com/IAMconsortium/pyam/pull/292) Add warning message if `data` is empty at initialization (after formatting)
 - [#288](https://github.com/IAMconsortium/pyam/pull/288) Put `pyam` logger in its own namespace (see [here](https://docs.python-guide.org/writing/logging/#logging-in-a-library>))
-
+- [#285](https://github.com/IAMconsortium/pyam/pull/285) Add ability to fetch regions with synonyms from IXMP API
 # Release v0.3.0
 
 ## Highlights

pyam/iiasa.py

@@ -235,13 +235,37 @@ def variables(self):
         return pd.Series(df['variable'].unique(), name='variable')
 
     @lru_cache()
-    def regions(self):
-        """All regions in the connected data source"""
+    def regions(self, include_synonyms=False):
+        """All regions in the connected data source
+
+        :param include_synonyms: whether to include synonyms
+               (possibly leading to duplicate region names for
+               regions with more than one synonym)
+        """
         url = '/'.join([self._base_url, 'nodes?hierarchy=%2A'])
         headers = {'Authorization': 'Bearer {}'.format(self._token)}
-        r = requests.get(url, headers=headers)
+        params = {'includeSynonyms': include_synonyms}
+        r = requests.get(url, headers=headers, params=params)
         _check_response(r)
-        df = pd.read_json(r.content, orient='records')
+        return self.convert_regions_payload(r.content, include_synonyms)
+
+    @staticmethod
+    def convert_regions_payload(response, include_synonyms):
+        df = pd.read_json(response, orient='records')
+        if df.empty:
+            return df
+        if 'synonyms' not in df.columns:
+            df['synonyms'] = [list()] * len(df)
+        df = df.astype({
+            'id': str,
+            'name': str,
+            'hierarchy': str,
+            'parent': str,
+            'synonyms': object
+        })
+        if include_synonyms:
+            df = df[['name', 'synonyms']].explode('synonyms')
+            return df.rename(columns={'name': 'region', 'synonyms': 'synonym'})
         return pd.Series(df['name'].unique(), name='region')
 
     def _query_post_data(self, **kwargs):
@@ -330,7 +354,9 @@ def query(self, **kwargs):
         r = requests.post(url, headers=headers, data=data)
         _check_response(r)
         # refactor returned json object to be castable to an IamDataFrame
-        df = pd.read_json(r.content, orient='records')
+        dtype = dict(model=str, scenario=str, variable=str, unit=str,
+                     region=str, year=int, value=float, version=int)
+        df = pd.read_json(r.content, orient='records', dtype=dtype)
         logger.debug('Response size is {0} bytes, '
                        '{1} records'.format(len(r.content), len(df)))
         columns = ['model', 'scenario', 'variable', 'unit',

tests/test_iiasa.py

@@ -77,6 +77,50 @@ def test_regions():
     assert 'World' in obs
 
 
+def test_regions_with_synonyms():
+    conn = iiasa.Connection('IXSE_SR15')
+    obs = conn.regions(include_synonyms=True)
+    assert 'synonym' in obs.columns
+    assert (obs[obs.region == 'R5ROWO']
+            .synonym == 'Rest of the World (R5)').all()
+
+
+def test_regions_empty_response():
+    obs = iiasa.Connection.convert_regions_payload('[]', include_synonyms=True)
+    assert obs.empty
+
+
+def test_regions_no_synonyms_response():
+    json = '[{"id":1,"name":"World","parent":"World","hierarchy":"common"}]'
+    obs = iiasa.Connection.convert_regions_payload(json, include_synonyms=True)
+    assert not obs.empty
+
+
+def test_regions_with_synonyms_response():
+    json = '''
+    [
+        {
+            "id":1,"name":"World","parent":"World","hierarchy":"common",
+            "synonyms":[]
+        },
+        {
+            "id":2,"name":"USA","parent":"World","hierarchy":"country",
+            "synonyms":["US","United States"]
+        },
+        {
+            "id":3,"name":"Germany","parent":"World","hierarchy":"country",
+            "synonyms":["Deutschland","DE"]
+        }
+    ]
+    '''
+    obs = iiasa.Connection.convert_regions_payload(json, include_synonyms=True)
+    assert not obs.empty
+    assert (obs[obs.region == 'USA']
+            .synonym.isin(['US', 'United States'])).all()
+    assert (obs[obs.region == 'Germany']
+            .synonym.isin(['Deutschland', 'DE'])).all()
+
+
 def test_metadata():
     conn = iiasa.Connection('IXSE_SR15')
     obs = conn.scenario_list()['model'].values