[WIP] Test CSW ISO output with SEMIC XSLT to GeoDCAT-AP

setup.py

@@ -45,6 +45,7 @@ def pip(filename):
         'udata.harvesters': [
             'dcat = udata.harvest.backends.dcat:DcatBackend',
             'csw-dcat = udata.harvest.backends.dcat:CswDcatBackend',
+            'csw-iso-xslt-dcat = udata.harvest.backends.dcat:CswIsoXsltDcatBackend'
         ],
         'udata.avatars': [
             'internal = udata.features.identicon.backends:internal',

udata/harvest/backends/dcat.py

@@ -3,6 +3,7 @@
 from rdflib import Graph, URIRef
 from rdflib.namespace import RDF
 import xml.etree.ElementTree as ET
+import lxml.etree as lET
 from typing import List
 
 from udata.rdf import (
@@ -137,7 +138,6 @@ def process(self, item):
         dataset = dataset_from_rdf(graph, dataset, node=node)
         return dataset
 
-
 class CswDcatBackend(DcatBackend):
     display_name = 'CSW-DCAT'
 
@@ -222,3 +222,81 @@ def parse_graph(self, url: str, fmt: str) -> List[Graph]:
                           headers=headers).text)
 
         return graphs
+
+
+class CswIsoXsltDcatBackend(DcatBackend):
+    display_name = 'CSW-ISO-XSLT-DCAT'
+
+    ISO_SCHEMA = 'http://www.isotc211.org/2005/gmd'
+
+    def get_format(self):
+        # TODO: should we redefine get_format here?
+        return 'xml'
+
+    def parse_graph(self, url: str, fmt: str) -> List[Graph]:
+        '''
+        Parse CSW graph querying ISO schema.
+        Use SEMIC GeoDCAT-AP XSLT to map it to a correct version.
+        See https://github.com/SEMICeu/iso-19139-to-dcat-ap for more information on the XSLT.
+        '''
+
+        # Load XSLT
+        xslURL = "https://raw.githubusercontent.com/SEMICeu/iso-19139-to-dcat-ap/master/iso-19139-to-dcat-ap.xsl"
+        xsl = lET.fromstring(self.get(xslURL).content)
+        transform = lET.XSLT(xsl)
+
+        # Start querying and parsing graph
+        body = '''<csw:GetRecords xmlns:csw="http://www.opengis.net/cat/csw/2.0.2"
+                                  xmlns:gmd="http://www.isotc211.org/2005/gmd"
+                                  service="CSW" version="2.0.2" resultType="results"
+                                  startPosition="{start}" maxPosition="10"
+                                  outputSchema="{schema}">
+                      <csw:Query typeNames="csw:Record">
+                        <csw:ElementSetName>full</csw:ElementSetName>
+                    </csw:Query>
+                </csw:GetRecords>'''
+        headers = {'Content-Type': 'application/xml'}
+
+        graphs = []
+        page = 0
+        start = 1
+        page_size = 10
+        response = self.post(url, data=body.format(start=start, schema=self.ISO_SCHEMA),
+                             headers=headers)
+        response.raise_for_status()
+        xml = lET.fromstring(response.content)
+        tree = transform(xml)
+
+        while tree:
+            subgraph = Graph(namespace_manager=namespace_manager)
+            subgraph.parse(data = lET.tostring(tree), format=fmt)
+
+            if not subgraph.subjects(RDF.type, DCAT.Dataset):
+                raise ValueError("Failed to fetch CSW content")
+
+            dataset_found = False
+            for node in subgraph.subjects(RDF.type, DCAT.Dataset):
+                id = subgraph.value(node, DCT.identifier)
+                kwargs = {'nid': str(node), 'page': page}
+                kwargs['type'] = 'uriref' if isinstance(node, URIRef) else 'blank'
+                self.add_item(id, **kwargs)
+                dataset_found = True
+            graphs.append(subgraph)
+
+            # No dataset returned already
+            if not dataset_found:
+                break
+
+            # Enough items have been harvested already
+            if self.max_items and len(self.job.items) >= self.max_items:
+                break
+
+            # TODO: use CSW results to deal with pagination
+            page += 1
+            start = start + page_size
+
+            tree = transform(lET.fromstring(
+                self.post(url, data=body.format(start=start, schema=self.ISO_SCHEMA),
+                          headers=headers).content))
+
+        return graphs