Test a data stream entryΒΆ

In [19]:

import owslib
from owslib.wps import monitorExecution
import uuid
from owslib.wps import WebProcessingService
import sys
import os
import lxml.etree as etree
import requests
import cioppy
ciop = cioppy.Cioppy()
import shapely
import dateutil.parser
from shapely.wkt import loads
import pandas as pd
import geopandas as gp
from datetime import datetime, timedelta
from io import BytesIO
from zipfile import ZipFile
import matplotlib.pyplot as plt
import gdal
import numpy as np
import requests

from nbconvert.preprocessors import ExecutePreprocessor, CellExecutionError
import nbformat as nbf
  • Read the data pipeline configuration information:
In [20]:

nb_config = os.path.join('..', 'configuration.ipynb')

nb = nbf.read(nb_config, 4)

exec(nb['cells'][0]['source']) in globals(), locals()

In [21]:

series = 'https://catalog.terradue.com/%s/series/source-out/search' % data_pipeline

In [49]:

update = '2018-11-21T16:00:00Z/2018-11-21T16:30:00Z'
search_params = dict([('count', '1'),('update', update)])

In [53]:

entry_search = ciop.search(end_point=series,
                           params=search_params,
                           output_fields='self,updated',
                           model='EOP')
for item in entry_search:
    print item

{'self': 'https://catalog.terradue.com//better-wfp-00004/series/source-out/search?format=atomeop&uid=LC08_L1TP_153035_20170930_20171013_01_T1', 'updated': '2018-11-21T16:18:01.0566740+00:00'}

In [54]:

stream = entry_search[0]['self']
  • Read the stream OWS Context document
In [55]:

root = etree.fromstring(requests.get(stream).content)
  • Get the Web Process Service access point:
In [56]:

ns = {'a':'http://www.w3.org/2005/Atom',
      'b':'http://www.opengis.net/owc/1.0',
      'c':'http://www.opengis.net/wps/1.0.0',
      'd':'http://www.opengis.net/ows/1.1'}

In [57]:

wps_url = root.xpath('/a:feed/a:entry/b:offering/b:operation[@code="Execute"]',
                                 namespaces=ns)[0].attrib['href']
  • Get the process identifier:
In [58]:

process_id = root.xpath('/a:feed/a:entry/b:offering/b:operation[@code="Execute"]/b:request/c:Execute/d:Identifier',
                                 namespaces=ns)[0].text
  • Check that the process is deployed
In [59]:

wps = WebProcessingService(wps_url, verbose=False, skip_caps=True)

In [60]:

wps.getcapabilities()

In [61]:

deployed = False

for index, elem in enumerate(wps.processes):
    if elem.identifier == process_id:
        deployed = True
        break

if not deployed:
    raise Exception('Process %s not deployed' % process_id)
  • Print the parameters
In [62]:

identifiers = root.xpath('/a:feed/a:entry/b:offering/b:operation[@code="Execute"]/b:request/c:Execute/c:DataInputs/c:Input/d:Identifier',
                                 namespaces=ns)

values = root.xpath('/a:feed/a:entry/b:offering/b:operation[@code="Execute"]/b:request/c:Execute/c:DataInputs/c:Input/c:Data/c:LiteralData',
                                 namespaces=ns)

params = dict()

for index, elem in enumerate(identifiers):

    params[elem.text] = values[index].text


params

Out[62]:

{'_T2Username': 'better-wfp-00004',
 'data_pipeline': 'better-wfp-00004',
 'source': 'https://catalog.terradue.com/landsat8/search?format=atom&uid=LC08_L1TP_153035_20170930_20171013_01_T1'}
  • Submit the WPS request:
In [63]:

execution = owslib.wps.WPSExecution(url=wps.url)

In [64]:

execution_request = root.xpath('/a:feed/a:entry/b:offering/b:operation[@code="Execute"]/b:request/c:Execute',
                                 namespaces=ns)[0]


print etree.tostring(execution_request)

execution_response = execution.submitRequest(etree.tostring(execution_request))

<wps100:Execute xmlns:wps100="http://www.opengis.net/wps/1.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:owc="http://www.opengis.net/owc/1.0" xmlns="http://www.w3.org/2005/Atom" service="WPS" version="1.0.0" xsi:schemaLocation="http://www.opengis.net/wps/1.0.0 http://schemas.opengis.net/wps/1.0.0/wpsExecute_request.xsd">



          <ows110:Identifier xmlns:ows110="http://www.opengis.net/ows/1.1">ec_better_ewf_wfp_01_02_02_ewf_wfp_01_02_02_0_8</ows110:Identifier><wps100:DataInputs>



            <wps100:Input>


              <ows110:Identifier xmlns:ows110="http://www.opengis.net/ows/1.1">source</ows110:Identifier><wps100:Data>

                <wps100:LiteralData>https://catalog.terradue.com/landsat8/search?format=atom&amp;uid=LC08_L1TP_153035_20170930_20171013_01_T1</wps100:LiteralData></wps100:Data></wps100:Input><wps100:Input>


              <ows110:Identifier xmlns:ows110="http://www.opengis.net/ows/1.1">data_pipeline</ows110:Identifier><wps100:Data>

                <wps100:LiteralData>better-wfp-00004</wps100:LiteralData></wps100:Data></wps100:Input><wps100:Input>


              <ows110:Identifier xmlns:ows110="http://www.opengis.net/ows/1.1">_T2Username</ows110:Identifier><wps100:Data>

                <wps100:LiteralData>better-wfp-00004</wps100:LiteralData></wps100:Data></wps100:Input></wps100:DataInputs><wps100:ResponseForm>

            <wps100:ResponseDocument status="true" storeExecuteResponse="true">

              <wps100:Output asReference="false">

                <ows110:Identifier xmlns:ows110="http://www.opengis.net/ows/1.1">result_osd</ows110:Identifier></wps100:Output></wps100:ResponseDocument></wps100:ResponseForm></wps100:Execute>

In [65]:

execution.parseResponse(execution_response)
  • Status location of the submitted process:
In [66]:

execution.statusLocation

Out[66]:

'http://ec-better-apps-deployer.terradue.com/zoo-bin/zoo_loader.cgi?request=Execute&service=WPS&version=1.0.0&Identifier=GetStatus&DataInputs=sid=05973742-ef24-11e8-ab29-0242ac11000f&RawDataOutput=Result'
  • Monitor the execution
In [67]:

monitorExecution(execution)

In [68]:

print execution.isSucceded()

True
  • Check the output by querying the OpenSearch access points for the results:
In [69]:

for output in execution.processOutputs:
    print(output.identifier)

result_osd
QUOTATION

In [70]:

results_osd = execution.processOutputs[0].reference

In [71]:

results_osd

Out[71]:

'https://recast.terradue.com/t2api/describe/better-wfp-00004/_results/workflows/ec_better_ewf_wfp_01_02_02_ewf_wfp_01_02_02_0_8/run/05973742-ef24-11e8-ab29-0242ac11000f/0024934-180330140554685-oozie-oozi-W'

In [73]:

search_results = ciop.search(end_point=results_osd,
                         params=[],
                         output_fields='identifier,enclosure',
                         model='GeoTime')

In [74]:

for index, elem in enumerate(search_results):

    print(elem['enclosure'])

https://store.terradue.com/better-wfp-00004/_results/workflows/ec_better_ewf_wfp_01_02_02_ewf_wfp_01_02_02_0_8/run/05973742-ef24-11e8-ab29-0242ac11000f/0024934-180330140554685-oozie-oozi-W/78ecd6a2-27f9-4c13-a33f-177218fe0a94/LC08_L1TP_153035_20170930_20171013_01_T1_BQA.TIF
https://store.terradue.com/better-wfp-00004/_results/workflows/ec_better_ewf_wfp_01_02_02_ewf_wfp_01_02_02_0_8/run/05973742-ef24-11e8-ab29-0242ac11000f/0024934-180330140554685-oozie-oozi-W/78ecd6a2-27f9-4c13-a33f-177218fe0a94/LC08_L1TP_153035_20170930_20171013_01_T1_MNDWI.TIF
https://store.terradue.com/better-wfp-00004/_results/workflows/ec_better_ewf_wfp_01_02_02_ewf_wfp_01_02_02_0_8/run/05973742-ef24-11e8-ab29-0242ac11000f/0024934-180330140554685-oozie-oozi-W/78ecd6a2-27f9-4c13-a33f-177218fe0a94/LC08_L1TP_153035_20170930_20171013_01_T1_NDBI.TIF
https://store.terradue.com/better-wfp-00004/_results/workflows/ec_better_ewf_wfp_01_02_02_ewf_wfp_01_02_02_0_8/run/05973742-ef24-11e8-ab29-0242ac11000f/0024934-180330140554685-oozie-oozi-W/78ecd6a2-27f9-4c13-a33f-177218fe0a94/LC08_L1TP_153035_20170930_20171013_01_T1_NDVI.TIF
https://store.terradue.com/better-wfp-00004/_results/workflows/ec_better_ewf_wfp_01_02_02_ewf_wfp_01_02_02_0_8/run/05973742-ef24-11e8-ab29-0242ac11000f/0024934-180330140554685-oozie-oozi-W/78ecd6a2-27f9-4c13-a33f-177218fe0a94/LC08_L1TP_153035_20170930_20171013_01_T1_NDWI.TIF
https://store.terradue.com/better-wfp-00004/_results/workflows/ec_better_ewf_wfp_01_02_02_ewf_wfp_01_02_02_0_8/run/05973742-ef24-11e8-ab29-0242ac11000f/0024934-180330140554685-oozie-oozi-W/78ecd6a2-27f9-4c13-a33f-177218fe0a94/LC08_L1TP_153035_20170930_20171013_01_T1_R1.TIF
https://store.terradue.com/better-wfp-00004/_results/workflows/ec_better_ewf_wfp_01_02_02_ewf_wfp_01_02_02_0_8/run/05973742-ef24-11e8-ab29-0242ac11000f/0024934-180330140554685-oozie-oozi-W/78ecd6a2-27f9-4c13-a33f-177218fe0a94/LC08_L1TP_153035_20170930_20171013_01_T1_R2.TIF
https://store.terradue.com/better-wfp-00004/_results/workflows/ec_better_ewf_wfp_01_02_02_ewf_wfp_01_02_02_0_8/run/05973742-ef24-11e8-ab29-0242ac11000f/0024934-180330140554685-oozie-oozi-W/78ecd6a2-27f9-4c13-a33f-177218fe0a94/LC08_L1TP_153035_20170930_20171013_01_T1_R3.TIF
https://store.terradue.com/better-wfp-00004/_results/workflows/ec_better_ewf_wfp_01_02_02_ewf_wfp_01_02_02_0_8/run/05973742-ef24-11e8-ab29-0242ac11000f/0024934-180330140554685-oozie-oozi-W/78ecd6a2-27f9-4c13-a33f-177218fe0a94/LC08_L1TP_153035_20170930_20171013_01_T1_R4.TIF
https://store.terradue.com/better-wfp-00004/_results/workflows/ec_better_ewf_wfp_01_02_02_ewf_wfp_01_02_02_0_8/run/05973742-ef24-11e8-ab29-0242ac11000f/0024934-180330140554685-oozie-oozi-W/78ecd6a2-27f9-4c13-a33f-177218fe0a94/LC08_L1TP_153035_20170930_20171013_01_T1_R5.TIF
https://store.terradue.com/better-wfp-00004/_results/workflows/ec_better_ewf_wfp_01_02_02_ewf_wfp_01_02_02_0_8/run/05973742-ef24-11e8-ab29-0242ac11000f/0024934-180330140554685-oozie-oozi-W/78ecd6a2-27f9-4c13-a33f-177218fe0a94/LC08_L1TP_153035_20170930_20171013_01_T1_R6.TIF
https://store.terradue.com/better-wfp-00004/_results/workflows/ec_better_ewf_wfp_01_02_02_ewf_wfp_01_02_02_0_8/run/05973742-ef24-11e8-ab29-0242ac11000f/0024934-180330140554685-oozie-oozi-W/78ecd6a2-27f9-4c13-a33f-177218fe0a94/LC08_L1TP_153035_20170930_20171013_01_T1_R7.TIF
https://store.terradue.com/better-wfp-00004/_results/workflows/ec_better_ewf_wfp_01_02_02_ewf_wfp_01_02_02_0_8/run/05973742-ef24-11e8-ab29-0242ac11000f/0024934-180330140554685-oozie-oozi-W/78ecd6a2-27f9-4c13-a33f-177218fe0a94/LC08_L1TP_153035_20170930_20171013_01_T1_R8.TIF
https://store.terradue.com/better-wfp-00004/_results/workflows/ec_better_ewf_wfp_01_02_02_ewf_wfp_01_02_02_0_8/run/05973742-ef24-11e8-ab29-0242ac11000f/0024934-180330140554685-oozie-oozi-W/78ecd6a2-27f9-4c13-a33f-177218fe0a94/LC08_L1TP_153035_20170930_20171013_01_T1_R9.TIF
https://store.terradue.com/better-wfp-00004/_results/workflows/ec_better_ewf_wfp_01_02_02_ewf_wfp_01_02_02_0_8/run/05973742-ef24-11e8-ab29-0242ac11000f/0024934-180330140554685-oozie-oozi-W/78ecd6a2-27f9-4c13-a33f-177218fe0a94/result.ipynb
https://store.terradue.com/better-wfp-00004/_results/workflows/ec_better_ewf_wfp_01_02_02_ewf_wfp_01_02_02_0_8/run/05973742-ef24-11e8-ab29-0242ac11000f/0024934-180330140554685-oozie-oozi-W/78ecd6a2-27f9-4c13-a33f-177218fe0a94/stage-in.ipynb
  • Download the results to the local filesystem
In [75]:

headers = {'Authorization': 'Bearer %s' % access_token,
           'User-Agent': 'curl/t2Client'}

In [76]:

wf = results_osd.split('/')[-1]

os.mkdir('results/%s' %wf)

for index, elem in enumerate(search_results):

    r = requests.get(elem['enclosure'], headers=headers)

    filename = elem['enclosure'][elem['enclosure'].rfind("/")+1:]

    out_path = 'results/%s/%s' %(wf,filename)

    print ('Download %s as %s' % (elem['enclosure'], out_path))

    open(out_path, 'wb').write(r.content)

print "Results downloaded"