Test the data transformation application¶

This Jupyter Notebook to query the catalog for a set of Landsat 8 products, creates a Web Processing Service (WPS) request invoking the data transformation application that was deployed in the Deploy step, monitors the WPS request execution and finally retrieves the data transformation execution results

First do the imports of the Python libraries required

In [61]:

import os
import owslib
from owslib.wps import monitorExecution
from owslib.wps import WebProcessingService
import lxml.etree as etree
import cioppy
import numpy as np
from datetime import datetime, timedelta
import dateutil.parser
import folium
import getpass
from shapely.wkt import loads

from nbconvert.preprocessors import ExecutePreprocessor, CellExecutionError
import nbformat as nbf

Read the data pipeline configuration information:

In [62]:

%store -r

nb_config = os.path.join('../operations', 'configuration.ipynb')

nb = nbf.read(nb_config, 4)

exec(nb['cells'][0]['source']) in globals(), locals()

app = dict([('artifact_id', app_artifact_id),
            ('version', app_version),
            ('repository', repository),
            ('community', community)])

app_process_id = '%s_%s_%s_%s' % (app['community'].replace('-', '_'), app['artifact_id'].replace('-', '_'), app['artifact_id'].replace('-', '_'), app['version'].replace('.', '_'))

Define the search parameters: the catalog series OpenSearch endpoint, the time of interest and the area of interest

In [63]:

series = 'https://catalog.terradue.com/landsat8/search'

start_date = '2017-09-01T00:00:00Z'

stop_date = '2017-09-01T23:59:59Z'
geom = 'MULTIPOLYGON (((6.4788 14.5973, 7.5577 14.5973, 7.5577 13.6328, 6.4788 13.6328, 6.4788 14.5973)), ((67.7116 37.9032, 68.791 37.9032, 68.791 36.9211, 67.7116 36.9211, 67.7116 37.9032)), ((-10.3668 15.3471, -9.351800000000001 15.3471, -9.351800000000001 14.3406, -10.3668 14.3406, -10.3668 15.3471)), ((67.62430000000001 36.7228, 68.116 36.7228, 68.116 35.6923, 67.62430000000001 35.6923, 67.62430000000001 36.7228)))'

Search for Landsat 8 products in one of the polygons of the area of interest

In [64]:

wkt = loads(geom)[0].wkt
wkt

Out[64]:

'POLYGON ((6.4788 14.5973, 7.5577 14.5973, 7.5577 13.6328, 6.4788 13.6328, 6.4788 14.5973))'

In [65]:

aois = []

for index, aoi in enumerate(loads(geom)):

    aois.append(np.asarray([t[::-1] for t in list(aoi.exterior.coords)]).tolist())

In [66]:

ciop = cioppy.Cioppy()

search_params = dict([('geom', wkt),
                      ('start', start_date),
                      ('stop', stop_date),
                      ('pt', 'L1TP')])

search = ciop.search(end_point = series,
                     params = search_params,
                     output_fields='self,enclosure,identifier,wkt,track,startdate',
                     model='EOP')

In [67]:

discovery_locations = []

for index, elem in enumerate(search):
    discovery_locations.append([t[::-1] for t in list(loads(elem['wkt']).exterior.coords)])

    print(index, elem['identifier'] + ' ' + elem['track'] + ' ' + elem['startdate'])

(0, 'LC08_L1TP_190051_20170901_20170915_01_T1 190 2017-09-01T09:55:00.3250000Z')
(1, 'LC08_L1TP_190050_20170901_20170915_01_T1 190 2017-09-01T09:54:36.4210000Z')

In [68]:

lat = (loads(geom).bounds[3]+loads(geom).bounds[1])/2
lon = (loads(geom).bounds[2]+loads(geom).bounds[0])/2

zoom_start = 4

m = folium.Map(location=[lat, lon], zoom_start=zoom_start)

radius = 4
folium.CircleMarker(
    location=[lat, lon],
    radius=radius,
    color='#FF0000',
    stroke=False,
    fill=True,
    fill_opacity=0.6,
    opacity=1,
    popup='{} pixels'.format(radius),
    tooltip='I am in pixels',
).add_to(m)

folium.PolyLine(
    locations=aois,
    color='#FF0000',
    weight=2,
    tooltip='',
).add_to(m)

folium.PolyLine(
    locations=discovery_locations,
    color='orange',
    weight=1,
    opacity=1,
    smooth_factor=0,
).add_to(m)


folium.PolyLine(
    locations=aois[0],
    color='green',
    weight=1,
    opacity=1,
    smooth_factor=0,
).add_to(m)

m.save(os.path.join('results', 'wfp-01-02-02-search.html'))

m

Out[68]:

In [69]:

ls8_index = 1

Connect to the WPS server

In [70]:

wps_url = 'https://ec-better-apps-deployer.terradue.com/zoo-bin/zoo_loader.cgi'

wps = WebProcessingService(wps_url,
                           verbose=False,
                           skip_caps=True)

Do a GetCapabilities WPS request and list the process:

In [71]:

wps.getcapabilities()

In [72]:

for index, elem in enumerate(wps.processes):
    print(index, elem.identifier)

(0, 'coordinator_ec_better_wfp_01_03_02_wfp_01_03_02_1_5')
(1, 'ec_better_ewf_wfp_01_01_02_ewf_wfp_01_01_02_0_10')
(2, 'ec_better_ewf_satcen_01_03_01_ewf_satcen_01_03_01_0_4')
(3, 'ec_better_ewf_satcen_01_02_01_ewf_satcen_01_02_01_0_6')
(4, 'ec_better_ewf_sen2cor_ewf_sen2cor_0_12')
(5, 'ec_better_ewf_ethz_01_02_01_ewf_ethz_01_02_01_0_4')
(6, 'ec_better_wfp_01_03_02_wfp_01_03_02_1_11')
(7, 'ec_better_ewf_satcen_01_03_01_ewf_satcen_01_03_01_0_3')
(8, 'UnDeployProcess')
(9, 'ec_better_dcs_better_wfp_00001_dcs_better_wfp_00001_1_1')
(10, 'ec_better_ewf_wfp_01_02_01_ewf_wfp_01_02_01_0_1')
(11, 'ec_better_ewf_wfp_01_01_01_ewf_wfp_01_01_01_1_14')
(12, 'ec_better_ewf_satcen_01_03_01_ewf_satcen_01_03_01_0_5')
(13, 'ec_better_wfp_01_03_01_wfp_01_03_01_1_0')
(14, 'ec_better_ewf_wfp_01_02_02_ewf_wfp_01_02_02_0_6')
(15, 'ec_better_ewf_wfp_01_01_02_ewf_wfp_01_01_02_0_12')
(16, 'ec_better_ewf_wfp_01_02_03_ewf_wfp_01_02_03_0_5')
(17, 'ec_better_wfp_01_01_01_wfp_01_01_01_1_5')
(18, 'coordinator_ec_better_wfp_01_03_02_wfp_01_03_02_1_10')
(19, 'ec_better_ewf_sen2cor_sen2cor_0_3')
(20, 'coordinator_ec_better_wfp_01_03_02_wfp_01_03_02_1_8')
(21, 'ec_better_ewf_satcen_01_02_01_ewf_satcen_01_02_01_0_11')
(22, 'ec_better_ewf_satcen_01_03_01_ewf_satcen_01_03_01_0_7')
(23, 'coordinator_ec_better_wfp_01_03_02_wfp_01_03_02_1_2')
(24, 'ec_better_ewf_wfp_01_02_02_ewf_wfp_01_02_02_0_10')
(25, 'ec_better_ewf_wfp_01_02_01_ewf_wfp_01_02_01_0_3')
(26, 'ec_better_ewf_ethz_01_01_01_ewf_ethz_01_01_01_0_2')
(27, 'ec_better_ewf_wfp_01_01_01_ewf_wfp_01_01_01_1_9')
(28, 'ec_better_wfp_01_03_02_wfp_01_03_02_1_2')
(29, 'ec_better_ewf_ethz_01_01_01_ewf_ethz_01_01_01_0_5')
(30, 'ec_better_wfp_01_01_01_wfp_01_01_01_0_4')
(31, 'ec_better_wfp_01_01_01_wfp_01_01_01_1_7')
(32, 'ec_better_ewf_satcen_01_03_01_ewf_satcen_01_03_01_0_6')
(33, 'TerradueUnDeployProcess')
(34, 'ec_better_ewf_sen2cor_sen2cor_0_4')
(35, 'ec_better_wfp_01_03_01_wfp_01_03_01_1_2')
(36, 'ec_better_ewf_satcen_01_02_01_ewf_satcen_01_02_01_0_14')
(37, 'ec_better_ewf_wfp_01_01_02_ewf_wfp_01_01_02_0_8')
(38, 'ec_better_ewf_wfp_01_01_02_ewf_wfp_01_01_02_0_5')
(39, 'coordinator_ec_better_wfp_01_03_02_wfp_01_03_02_1_3')
(40, 'ec_better_ewf_ethz_01_01_01_ewf_ethz_01_01_01_0_6')
(41, 'ec_better_ewf_satcen_01_02_01_ewf_satcen_01_02_01_0_8')
(42, 'ec_better_wfp_01_01_01_g_wfp_01_01_01_0_4')
(43, 'ec_better_ewf_wfp_01_01_02_ewf_wfp_01_01_02_0_9')
(44, 'ec_better_ewf_satcen_01_03_01_ewf_satcen_01_03_01_0_2')
(45, 'ec_better_ewf_ethz_01_01_01_ewf_ethz_01_01_01_0_1')
(46, 'ec_better_wfp_01_03_02_wfp_01_03_02_1_1')
(47, 'ec_better_ewf_wfp_01_01_01_ewf_wfp_01_01_01_1_16')
(48, 'ec_better_wfp_01_03_02_wfp_01_03_02_1_3')
(49, 'coordinator_ec_better_wfp_01_03_01_wfp_01_03_01_1_2')
(50, 'ec_better_wfp_01_01_01_wfp_01_01_01_1_1')
(51, 'ec_better_ewf_wfp_01_02_01_ewf_wfp_01_02_01_0_2')
(52, 'ec_better_ewf_ethz_01_02_01_ewf_ethz_01_02_01_0_7')
(53, 'ec_better_wfp_01_03_01_wfp_01_03_01_1_3')
(54, 'coordinator_ec_better_wfp_01_03_02_wfp_01_03_02_1_1')
(55, 'ec_better_ewf_sen2cor_sen2cor_0_1')
(56, 'ec_better_ewf_wfp_01_01_01_ewf_wfp_01_01_01_1_10')
(57, 'ec_better_ewf_satcen_01_02_01_ewf_satcen_01_02_01_0_12')
(58, 'ec_better_wfp_01_03_02_wfp_01_03_02_1_9')
(59, 'ec_better_ewf_wfp_01_02_02_ewf_wfp_01_02_02_0_11')
(60, 'ec_better_ewf_satcen_01_01_01_ewf_satcen_01_01_01_0_11')
(61, 'ec_better_ewf_satcen_01_03_01_ewf_satcen_01_03_01_0_1')
(62, 'ec_better_ewf_satcen_01_02_01_ewf_satcen_01_02_01_0_13')
(63, 'ec_better_dcs_better_wfp_00001_dcs_better_wfp_00001_1_2_1')
(64, 'ec_better_wfp_01_01_01_wfp_01_01_01_0_1')
(65, 'ec_better_wfp_01_03_02_wfp_01_03_02_1_0')
(66, 'ec_better_ewf_wfp_01_01_02_ewf_wfp_01_01_02_0_7')
(67, 'ec_better_ewf_ethz_01_03_01_ewf_ethz_01_03_01_0_2')
(68, 'ec_better_ewf_ethz_01_02_01_ewf_ethz_01_02_01_0_6')
(69, 'ec_better_ewf_wfp_01_01_01_ewf_wfp_01_01_01_1_15')
(70, 'ec_better_wfp_01_03_02_wfp_01_03_02_1_5')
(71, 'ec_better_ewf_wfp_01_01_02_ewf_wfp_01_01_02_0_11')
(72, 'ec_better_ewf_satcen_01_02_01_ewf_satcen_01_02_01_0_7')
(73, 'DeployProcess')
(74, 'ec_better_ewf_sen2cor_ewf_sen2cor_0_11')
(75, 'ec_better_ewf_sen2cor_ewf_sen2cor_0_9')
(76, 'ec_better_wfp_01_01_01_wfp_01_01_01_1_6')
(77, 'GetStatus')
(78, 'ec_better_wfp_01_03_02_wfp_01_03_02_1_8')
(79, 'ec_better_tg_wfp_01_02_02_queue_start_stop_tg_wfp_01_02_02_queue_start_stop_0_1')
(80, 'ec_better_ewf_wfp_01_02_02_ewf_wfp_01_02_02_0_9')
(81, 'ec_better_ewf_wfp_01_02_03_ewf_wfp_01_02_03_0_4')
(82, 'ec_better_ewf_ethz_01_03_01_ewf_ethz_01_03_01_0_5')
(83, 'ec_better_ewf_satcen_01_02_01_ewf_satcen_01_02_01_0_10')
(84, 'coordinator_ec_better_wfp_01_03_01_wfp_01_03_01_1_1')
(85, 'ec_better_wfp_01_03_01_wfp_01_03_01_1_1')
(86, 'coordinator_ec_better_wfp_01_03_01_wfp_01_03_01_1_3')
(87, 'ec_better_ewf_satcen_01_01_01_ewf_satcen_01_01_01_0_10')
(88, 'ec_better_ewf_sen2cor_ewf_sen2cor_0_6')
(89, 'ec_better_wfp_01_01_01_wfp_01_01_01_1_2')
(90, 'ec_better_wfp_01_03_02_wfp_01_03_02_1_10')
(91, 'TerradueDeployProcess')
(92, 'coordinator_ec_better_wfp_01_03_02_wfp_01_03_02_1_0')
(93, 'ec_better_ewf_wfp_01_02_02_ewf_wfp_01_02_02_0_8')
(94, 'ec_better_ewf_sen2cor_sen2cor_0_2')
(95, 'coordinator_ec_better_wfp_01_03_02_wfp_01_03_02_1_9')
(96, 'ec_better_ewf_wfp_01_01_01_ewf_wfp_01_01_01_1_11')
(97, 'ec_better_ewf_wfp_01_01_01_ewf_wfp_01_01_01_1_13')
(98, 'ec_better_ewf_satcen_01_01_01_ewf_satcen_01_01_01_0_12')
(99, 'coordinator_ec_better_wfp_01_03_02_wfp_01_03_02_1_11')
(100, 'ec_better_wfp_01_01_01_a_wfp_01_01_01_0_1')
(101, 'ec_better_ewf_satcen_01_02_01_ewf_satcen_01_02_01_0_9')
(102, 'ec_better_ewf_ethz_01_03_01_ewf_ethz_01_03_01_0_4')
(103, 'ec_better_wfp_01_01_01_wfp_01_01_01_1_0')

Select the process and print the title and abstract after having submited a WPS DescribeProcess request

In [73]:

process = wps.describeprocess(app_process_id)

In [74]:

process.title

Out[74]:

'WFP-01-02-02 Landsat-8 reflectances and vegetation indices'

In [75]:

process.abstract

Out[75]:

'WFP-01-02-02 Landsat-8 reflectances and vegetation indices'

List the WPS process inputs:

In [76]:

for data_input in process.dataInputs:
    print data_input.identifier

source
_T2Username

In [77]:

source = []

for index, elem in enumerate(search):
    source.append(elem['self'])

In [78]:

','.join(source)

Out[78]:

'https://catalog.terradue.com/landsat8/search?format=atom&uid=LC08_L1TP_190051_20170901_20170915_01_T1,https://catalog.terradue.com/landsat8/search?format=atom&uid=LC08_L1TP_190050_20170901_20170915_01_T1'

Create a Python dictionary with the inputs:

In [79]:

inputs = [('source', ','.join(source)),
          ('quotation', 'No'),
          ('_T2Username', 'msagona')]

Submit the Execute WPS request:

In [80]:

execution = owslib.wps.WPSExecution(url=wps.url)

execution_request = execution.buildRequest(app_process_id,
                                           inputs,
                                           output=[('result_osd', False)])

execution_response = execution.submitRequest(etree.tostring(execution_request))

execution.parseResponse(execution_response)

Monitor the request:

In [81]:

execution.statusLocation

Out[81]:

'http://ec-better-apps-deployer.terradue.com/zoo-bin/zoo_loader.cgi?request=Execute&service=WPS&version=1.0.0&Identifier=GetStatus&DataInputs=sid=9466f18a-27a8-11e9-8e2f-0242ac11000f&RawDataOutput=Result'

In [82]:

monitorExecution(execution)

Check the outcome of the processing request

In [83]:

execution.isSucceded()

Out[83]:

True

Search for the results produced

In [84]:

for output in execution.processOutputs:
    print(output.identifier)

result_osd
QUOTATION

In [41]:

results_osd = execution.processOutputs[0].reference

In [42]:

results_osd

In [43]:

wps = WebProcessingService(recast_wps_url,
                           verbose=False,
                           skip_caps=False)

recast_inputs = [('items', results_osd),
                  ('index', data_pipeline),
                  ('_T2ApiKey', datapipeline_api_key),
                  ('_T2Username', data_pipeline)]

recast_execution = wps.execute(recast_process_id,
                               recast_inputs,
                               output = [('result_osd', True)])


monitorExecution(recast_execution, sleepSecs=60)

etree.fromstring(recast_execution.processOutputs[0].data[0]).xpath('./@href')[0]


ExceptionTraceback (most recent call last)
<ipython-input-43-250e5b0c94be> in <module>()
     10 recast_execution = wps.execute(recast_process_id,
     11                                recast_inputs,
---> 12                                output = [('result_osd', True)])
     13
     14

/opt/anaconda/lib/python2.7/site-packages/owslib/wps.pyc in execute(self, identifier, inputs, output, request, response)
    273         # build XML request from parameters
    274         if request is None:
--> 275             requestElement = execution.buildRequest(identifier, inputs, output)
    276             request = etree.tostring(requestElement)
    277             execution.request = request

/opt/anaconda/lib/python2.7/site-packages/owslib/wps.pyc in buildRequest(self, identifier, inputs, output)
    596             else:
    597                 raise Exception(
--> 598                     'input type of "%s" parameter is unknown' % key)
    599
    600         # <wps:ResponseForm>

Exception: input type of "items" parameter is unknown