Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Warn on DAL overflow #329

Merged
merged 5 commits into from
May 27, 2022
Merged

Warn on DAL overflow #329

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
7becc23
Warn on DAL overflow
May 13, 2022
3972c68
Added entry in CHANGES.rst
May 13, 2022
7b2bcd9
Fixed style
May 13, 2022
4bca491
Fixed warnings
May 16, 2022
740953d
Moved the overflow status in test
May 26, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGES.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,10 @@
1.4 (unreleased)
================

1.3.1 (unreleased)
==================

- pyvo.dal warns on results with overflow status. [#329]`

1.3 (2022-02-19)
==================
Expand Down
6 changes: 3 additions & 3 deletions pyvo/dal/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
from .scs import search as conesearch
from .tap import search as tablesearch

from .query import DALService, DALQuery, DALResults, Record
from .query import DALService, DALQuery, DALResults, DALOverflowWarning, Record

from .sia import SIAService, SIAQuery, SIAResults, SIARecord
from .ssa import SSAService, SSAQuery, SSAResults, SSARecord
Expand All @@ -17,7 +17,7 @@

from .exceptions import (
DALAccessError, DALProtocolError, DALFormatError, DALServiceError,
DALQueryError)
DALQueryError, DALOverflowWarning)

__all__ = [
"imagesearch", "spectrumsearch", "linesearch", "conesearch", "tablesearch",
Expand All @@ -30,4 +30,4 @@
"SIARecord", "SSARecord", "SLARecord", "SCSRecord",
"AsyncTAPJob",
"DALAccessError", "DALProtocolError", "DALFormatError", "DALServiceError",
"DALQueryError"]
"DALQueryError", "DALOverflowWarning"]
6 changes: 5 additions & 1 deletion pyvo/dal/exceptions.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

__all__ = [
"DALAccessError", "DALProtocolError", "DALFormatError", "DALServiceError",
"DALQueryError"]
"DALQueryError", "DALOverflowWarning"]

import re

Expand Down Expand Up @@ -232,3 +232,7 @@ def label(self):

class PyvoUserWarning(AstropyUserWarning):
pass


class DALOverflowWarning(UserWarning):
pass
13 changes: 11 additions & 2 deletions pyvo/dal/query.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,8 @@
from astropy.utils.exceptions import AstropyDeprecationWarning

from .mimetype import mime_object_maker
from .exceptions import (DALFormatError, DALServiceError, DALQueryError)
from .exceptions import (DALFormatError, DALServiceError, DALQueryError,
DALOverflowWarning)

from .. import samp

Expand Down Expand Up @@ -319,6 +320,10 @@ def __init__(self, votable, url=None, session=None):
if self._status[0].lower() not in ("ok", "overflow"):
raise DALQueryError(self._status[1], self._status[0], url)

if self._status[0].lower() == "overflow":
warn("Partial result set. Potential causes MAXREC, async storage space, etc.",
category=DALOverflowWarning);

self._resultstable = self._findresultstable(votable)
if not self._resultstable:
raise DALFormatError(
Expand Down Expand Up @@ -376,9 +381,13 @@ def _findstatus(self, votable):

def _findstatusinfo(self, infos):
# this can be overridden to specialize for a particular DAL protocol
status = None
# return the last status to catch potential overflow or error sent
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is interesting. I had to go back and look at the DALI standard to see that this would work to the OVERFLOW case (i.e., that OVERFLOW would come last).

# after the results.
for info in infos:
if info.name.lower() == 'query_status':
return info
status = info
return status

def _findinfos(self, votable):
# this can be overridden to specialize for a particular DAL protocol
Expand Down
4 changes: 2 additions & 2 deletions pyvo/dal/tests/data/query/basic.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
<?xml version="1.0" encoding="utf-8"?>
<!-- Produced with astropy.io.votable version 3.1.dev21501
<!-- Produced with astropy.io.votable version 5.0.1
http://www.astropy.org/ -->
<VOTABLE version="1.3" xmlns="http://www.ivoa.net/xml/VOTable/v1.3" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="http://www.ivoa.net/xml/VOTable/v1.3">
<VOTABLE version="1.4" xmlns="http://www.ivoa.net/xml/VOTable/v1.3" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.ivoa.net/xml/VOTable/v1.3 http://www.ivoa.net/xml/VOTable/VOTable-1.4.xsd">
<RESOURCE type="results">
<INFO ID="QUERY_STATUS" name="QUERY_STATUS" value="OK">OK</INFO>
<TABLE>
Expand Down
4 changes: 2 additions & 2 deletions pyvo/dal/tests/data/query/dataset.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
<?xml version="1.0" encoding="utf-8"?>
<!-- Produced with astropy.io.votable version 3.1.dev21501
<!-- Produced with astropy.io.votable version 5.0.1
http://www.astropy.org/ -->
<VOTABLE version="1.3" xmlns="http://www.ivoa.net/xml/VOTable/v1.3" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="http://www.ivoa.net/xml/VOTable/v1.3">
<VOTABLE version="1.4" xmlns="http://www.ivoa.net/xml/VOTable/v1.3" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.ivoa.net/xml/VOTable/v1.3 http://www.ivoa.net/xml/VOTable/VOTable-1.4.xsd">
<RESOURCE type="results">
<INFO ID="QUERY_STATUS" name="QUERY_STATUS" value="OK">OK</INFO>
<TABLE>
Expand Down
4 changes: 2 additions & 2 deletions pyvo/dal/tests/data/query/errorstatus.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
<?xml version="1.0" encoding="utf-8"?>
<!-- Produced with astropy.io.votable version 3.1.dev21501
<!-- Produced with astropy.io.votable version 5.0.1
http://www.astropy.org/ -->
<VOTABLE version="1.3" xmlns="http://www.ivoa.net/xml/VOTable/v1.3" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="http://www.ivoa.net/xml/VOTable/v1.3">
<VOTABLE version="1.4" xmlns="http://www.ivoa.net/xml/VOTable/v1.3" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.ivoa.net/xml/VOTable/v1.3 http://www.ivoa.net/xml/VOTable/VOTable-1.4.xsd">
<RESOURCE type="results">
<INFO ID="QUERY_STATUS" name="QUERY_STATUS" value="ERROR">ERROR</INFO>
<TABLE>
Expand Down
4 changes: 2 additions & 2 deletions pyvo/dal/tests/data/query/firstresource.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
<?xml version="1.0" encoding="utf-8"?>
<!-- Produced with astropy.io.votable version 3.1.dev21501
<!-- Produced with astropy.io.votable version 5.0.1
http://www.astropy.org/ -->
<VOTABLE version="1.3" xmlns="http://www.ivoa.net/xml/VOTable/v1.3" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="http://www.ivoa.net/xml/VOTable/v1.3">
<VOTABLE version="1.4" xmlns="http://www.ivoa.net/xml/VOTable/v1.3" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.ivoa.net/xml/VOTable/v1.3 http://www.ivoa.net/xml/VOTable/VOTable-1.4.xsd">
<RESOURCE type="results">
<INFO ID="QUERY_STATUS" name="QUERY_STATUS" value="OK">OK</INFO>
<TABLE>
Expand Down
4 changes: 2 additions & 2 deletions pyvo/dal/tests/data/query/missingcolumns.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
<?xml version="1.0" encoding="utf-8"?>
<!-- Produced with astropy.io.votable version 3.1.dev21501
<!-- Produced with astropy.io.votable version 5.0.1
http://www.astropy.org/ -->
<VOTABLE version="1.3" xmlns="http://www.ivoa.net/xml/VOTable/v1.3" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="http://www.ivoa.net/xml/VOTable/v1.3">
<VOTABLE version="1.4" xmlns="http://www.ivoa.net/xml/VOTable/v1.3" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.ivoa.net/xml/VOTable/v1.3 http://www.ivoa.net/xml/VOTable/VOTable-1.4.xsd">
<RESOURCE type="results">
<INFO ID="QUERY_STATUS" name="QUERY_STATUS" value="OK">OK</INFO>
<TABLE>
Expand Down
4 changes: 2 additions & 2 deletions pyvo/dal/tests/data/query/missingresource.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
<?xml version="1.0" encoding="utf-8"?>
<!-- Produced with astropy.io.votable version 3.1.dev21501
<!-- Produced with astropy.io.votable version 5.0.1
http://www.astropy.org/ -->
<VOTABLE version="1.3" xmlns="http://www.ivoa.net/xml/VOTable/v1.3" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="http://www.ivoa.net/xml/VOTable/v1.3"/>
<VOTABLE version="1.4" xmlns="http://www.ivoa.net/xml/VOTable/v1.3" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.ivoa.net/xml/VOTable/v1.3 http://www.ivoa.net/xml/VOTable/VOTable-1.4.xsd"/>
4 changes: 2 additions & 2 deletions pyvo/dal/tests/data/query/missingtable.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
<?xml version="1.0" encoding="utf-8"?>
<!-- Produced with astropy.io.votable version 3.1.dev21501
<!-- Produced with astropy.io.votable version 5.0.1
http://www.astropy.org/ -->
<VOTABLE version="1.3" xmlns="http://www.ivoa.net/xml/VOTable/v1.3" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="http://www.ivoa.net/xml/VOTable/v1.3">
<VOTABLE version="1.4" xmlns="http://www.ivoa.net/xml/VOTable/v1.3" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.ivoa.net/xml/VOTable/v1.3 http://www.ivoa.net/xml/VOTable/VOTable-1.4.xsd">
<RESOURCE type="results">
<INFO ID="QUERY_STATUS" name="QUERY_STATUS" value="OK">OK</INFO>
</RESOURCE>
Expand Down
29 changes: 29 additions & 0 deletions pyvo/dal/tests/data/query/overflowstatus.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
<?xml version="1.0" encoding="utf-8"?>
<!-- Produced with astropy.io.votable version 5.0.1
http://www.astropy.org/ -->
<VOTABLE version="1.4" xmlns="http://www.ivoa.net/xml/VOTable/v1.3" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.ivoa.net/xml/VOTable/v1.3 http://www.ivoa.net/xml/VOTable/VOTable-1.4.xsd">
<RESOURCE type="results">
<INFO ID="QUERY_STATUS" name="QUERY_STATUS" value="OK"/>
<INFO ID="QUERY_STATUS" name="QUERY_STATUS" value="OVERFLOW"/>
andamian marked this conversation as resolved.
Show resolved Hide resolved
<TABLE>
<FIELD ID="_1" datatype="long" name="1" ucd="foo;bar"/>
<FIELD ID="_2" arraysize="37" datatype="char" name="2" utype="foobar"/>
<DATA>
<TABLEDATA>
<TR>
<TD>23</TD>
<TD>Illuminatus</TD>
</TR>
<TR>
<TD>42</TD>
<TD>Don't panic, and always carry a towel</TD>
</TR>
<TR>
<TD>1337</TD>
<TD>Elite</TD>
</TR>
</TABLEDATA>
</DATA>
</TABLE>
</RESOURCE>
</VOTABLE>
4 changes: 2 additions & 2 deletions pyvo/dal/tests/data/query/rootinfo.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
<?xml version="1.0" encoding="utf-8"?>
<!-- Produced with astropy.io.votable version 3.1.dev21501
<!-- Produced with astropy.io.votable version 5.0.1
http://www.astropy.org/ -->
<VOTABLE version="1.3" xmlns="http://www.ivoa.net/xml/VOTable/v1.3" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="http://www.ivoa.net/xml/VOTable/v1.3">
<VOTABLE version="1.4" xmlns="http://www.ivoa.net/xml/VOTable/v1.3" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.ivoa.net/xml/VOTable/v1.3 http://www.ivoa.net/xml/VOTable/VOTable-1.4.xsd">
<INFO ID="QUERY_STATUS" name="QUERY_STATUS" value="OK">OK</INFO>
<RESOURCE type="results">
<TABLE>
Expand Down
4 changes: 2 additions & 2 deletions pyvo/dal/tests/data/query/tableinfo.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
<?xml version="1.0" encoding="utf-8"?>
<!-- Produced with astropy.io.votable version 3.1.dev21501
<!-- Produced with astropy.io.votable version 5.0.1
http://www.astropy.org/ -->
<VOTABLE version="1.3" xmlns="http://www.ivoa.net/xml/VOTable/v1.3" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="http://www.ivoa.net/xml/VOTable/v1.3">
<VOTABLE version="1.4" xmlns="http://www.ivoa.net/xml/VOTable/v1.3" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.ivoa.net/xml/VOTable/v1.3 http://www.ivoa.net/xml/VOTable/VOTable-1.4.xsd">
<RESOURCE type="results">
<TABLE>
<FIELD ID="_1" datatype="long" name="1" ucd="foo;bar"/>
Expand Down
Binary file modified pyvo/dal/tests/data/querydata/image.fits
View file
Open in desktop
Binary file not shown.
16 changes: 15 additions & 1 deletion pyvo/dal/tests/make_testdata.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,17 @@ def votablefile_errorstatus():
return votablefile


def votablefile_overflowstatus():
votablefile = _votablefile()

info_ok = Info(name='QUERY_STATUS', value='OK')
info_overflow = Info(name='QUERY_STATUS', value='OVERFLOW')
votablefile.resources[0].infos[0] = info_ok
votablefile.resources[0].infos.append(info_overflow)

return votablefile


def votablefile_missingtable():
votablefile = _votablefile()
del votablefile.resources[0].tables[0]
Expand All @@ -61,7 +72,7 @@ def votablefile_missingcolumns():

def votablefile_firstresource():
votablefile = _votablefile()
votablefile.resources[0]._type = ''
votablefile.resources[0]._type = 'results'
return votablefile


Expand Down Expand Up @@ -126,6 +137,9 @@ def main():
votablefile_errorstatus().to_xml(
str(dirname / 'query/errorstatus.xml'), tabledata_format='tabledata')

votablefile_overflowstatus().to_xml(
str(dirname / 'query/overflowstatus.xml'), tabledata_format='tabledata')

votablefile_missingtable().to_xml(
str(dirname / 'query/missingtable.xml'), tabledata_format='tabledata')

Expand Down
17 changes: 16 additions & 1 deletion pyvo/dal/tests/test_query.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,8 @@
import numpy as np

from pyvo.dal.query import DALService, DALQuery, DALResults, Record
from pyvo.dal.exceptions import DALServiceError, DALQueryError, DALFormatError
from pyvo.dal.exceptions import DALServiceError, DALQueryError,\
DALFormatError, DALOverflowWarning
from pyvo.version import version
from pyvo.utils.compat import ASTROPY_LT_4_1

Expand Down Expand Up @@ -91,6 +92,10 @@ def register_mocks(mocker):
'GET', 'http://example.com/query/errorstatus',
content=get_pkg_data_contents('data/query/errorstatus.xml')
)),
stack.enter_context(mocker.register_uri(
'GET', 'http://example.com/query/overflowstatus',
content=get_pkg_data_contents('data/query/overflowstatus.xml')
)),
]

def verbosetest_callback(request, context):
Expand Down Expand Up @@ -203,6 +208,12 @@ def test_query_exception(self):
with pytest.raises(DALQueryError):
service.search()

def test_query_warning(self):
service = DALService('http://example.com/query/overflowstatus')

with pytest.warns(DALOverflowWarning):
service.search()

@pytest.mark.filterwarnings("ignore::astropy.io.votable.exceptions.W53")
def test_format_exception(self):
with pytest.raises(DALFormatError):
Expand Down Expand Up @@ -283,6 +294,10 @@ def test_init_errorstatus(self):
with pytest.raises(DALQueryError):
DALResults.from_result_url('http://example.com/query/errorstatus')

def test_init_overflowstatus(self):
with pytest.warns(DALOverflowWarning):
DALResults.from_result_url('http://example.com/query/overflowstatus')

def test_init_missingtable(self):
with pytest.raises(DALFormatError):
DALResults.from_result_url('http://example.com/query/missingtable')
Expand Down