Test the application¶

This Jupyter Notebook to query the catalog for a pair of Sentinel-1 SLC products, creates a Web Processing Service (WPS) request invoking the data transformation application that was deployed in the Deploy step, monitors the WPS request execution and finally retrieves the data transformation execution results

First do the imports of the Python libraries required

In [1]:

import os
import owslib
from owslib.wps import monitorExecution
from owslib.wps import WebProcessingService
import lxml.etree as etree
import cioppy
from datetime import datetime, timedelta
import dateutil.parser
from shapely.wkt import loads


from nbconvert.preprocessors import ExecutePreprocessor, CellExecutionError
import nbformat as nbf

import geopandas as gp


import numpy as np
import ipyleaflet
from shapely.geometry import box
from shapely.geometry import multipolygon
from shapely.wkt import loads
from ipyleaflet import Map, Polygon

Read the data pipeline configuration information:

In [2]:

%store -r

nb_config = os.path.join('../operations', 'configuration.ipynb')

nb = nbf.read(nb_config, 4)

exec(nb['cells'][2]['source']) in globals(), locals()

app = dict([('artifact_id', app_artifact_id),
            ('version', app_version),
            ('repository', repository),
            ('community', community)])

app_process_id = '%s_%s_%s_%s' % (app['community'].replace('-', '_'), app['artifact_id'].replace('-', '_'), app['artifact_id'].replace('-', '_'), app['version'].replace('.', '_'))

Define the search parameters: the catalog series OpenSearch endpoint, the time of interest and the area of interest

In [3]:

series = 'https://catalog.terradue.com/sentinel1/search'

s1_prd_type = 'SLC'

crop_wkt = 'POLYGON ((10.65344444444444 12.17622222222222, 10.65344444444444 15.48261111111111, 8.398666666666667 15.48261111111111, 8.398666666666667 12.17622222222222, 10.65344444444444 12.17622222222222))'

Search for a master¶

In [4]:

slave_self = 'https://catalog.terradue.com/sentinel1/search?uid=S1B_S1_SLC__1SDV_20180712T051859_20180712T051915_011773_015A8D_A652'
aoi = loads(crop_wkt)

In [5]:

ciop = cioppy.Cioppy()
# get the slave metadata
slave_search = ciop.search(end_point = slave_self,
                           params = [],
                           output_fields='self,productType,track,enclosure,identifier,startdate,wkt',
                           model='EOP')

In [6]:

slave = slave_search[0]

# look for a master up to one cycle in the past
master_search_start_date = (dateutil.parser.parse(slave['startdate']) + timedelta(days=-12)).isoformat()
master_search_stop_date = (dateutil.parser.parse(slave['startdate']) + timedelta(days=-1)).isoformat()

# do the catalogue search
master_search_params = dict([('geom', slave['wkt']),
                             ('track', slave['track']),
                             ('pt',slave['productType']),
                             ('start', master_search_start_date),
                             ('stop', master_search_stop_date),
                             ('q', 'sensorOperationalMode.text:S*')])


series = 'https://catalog.terradue.com/sentinel1/search'

master_search = ciop.search(end_point=series,
                            params=master_search_params,
                            output_fields='identifier,enclosure,self,startdate,wkt',
                            model='EOP')

result = []

# build a data frame
for index, elem in enumerate(master_search):

    master_wkt = loads(elem['wkt'])

    result.append({'self' : elem['self'],
                           'identifier' : elem['identifier'],
                           'enclosure' : elem['enclosure'],
                           'wkt': loads(elem['wkt']),
                           'aoi_intersec' : (master_wkt.intersection(aoi).area/aoi.area) * 100,
                           'slave_intersec' : (master_wkt.intersection(loads(slave['wkt']))).area / loads(slave['wkt']).area * 100,
                           'contains': master_wkt.contains(aoi),
                           'days': (dateutil.parser.parse(slave['startdate'])- dateutil.parser.parse(elem['startdate'])).days
                          })

masters = gp.GeoDataFrame(result)

# select the master
master = master_search[masters.sort_values(['aoi_intersec', 'days'],
                                                   ascending=[False, False]).iloc[0].name]

In [7]:

masters

Out[7]:

	aoi_intersec	contains	days	enclosure	identifier	self	slave_intersec	wkt
0	9.87878	False	12	https://store.terradue.com/download/sentinel1/...	S1B_S1_SLC__1SDV_20180630T051858_20180630T0519...	https://catalog.terradue.com/sentinel1/search?...	99.838596	POLYGON ((9.931101999999999 12.372849, 9.20709...

In [8]:

master

Out[8]:

{'enclosure': 'https://store.terradue.com/download/sentinel1/files/v1/S1B_S1_SLC__1SDV_20180630T051858_20180630T051915_011598_015523_67BF',
 'identifier': 'S1B_S1_SLC__1SDV_20180630T051858_20180630T051915_011598_015523_67BF',
 'self': 'https://catalog.terradue.com/sentinel1/search?format=atom&uid=S1B_S1_SLC__1SDV_20180630T051858_20180630T051915_011598_015523_67BF',
 'startdate': '2018-06-30T05:18:58.9393970Z',
 'wkt': 'POLYGON((9.931102 12.372849,9.207095 12.519132,9.409536 13.493165,10.136349 13.347219,9.931102 12.372849))'}

In [9]:

slave

Out[9]:

{'enclosure': 'https://store.terradue.com/download/sentinel1/files/v1/S1B_S1_SLC__1SDV_20180712T051859_20180712T051915_011773_015A8D_A652',
 'identifier': 'S1B_S1_SLC__1SDV_20180712T051859_20180712T051915_011773_015A8D_A652',
 'productType': 'SLC',
 'self': 'https://catalog.terradue.com/sentinel1/search?format=atom&uid=S1B_S1_SLC__1SDV_20180712T051859_20180712T051915_011773_015A8D_A652',
 'startdate': '2018-07-12T05:18:59.2852070Z',
 'track': '22',
 'wkt': 'POLYGON((9.932179 12.373003,9.208238 12.519269,9.41065 13.493176,10.137398 13.347246,9.932179 12.373003))'}

In [10]:

slave_locations = np.asarray([t[::-1] for t in list(loads(slave['wkt']).exterior.coords)]).tolist()

In [11]:

master_locations= np.asarray([t[::-1] for t in list(loads(master['wkt']).exterior.coords)]).tolist()

In [12]:

aoi_locations = np.asarray([t[::-1] for t in list(loads(crop_wkt).exterior.coords)]).tolist()

In [13]:

map_center_x = loads(crop_wkt).centroid.x
map_center_y = loads(crop_wkt).centroid.y

In [14]:

m = Map(center=(map_center_y,
                map_center_x),
                zoom=7)

m += Polygon(locations=slave_locations,
                 color="blue",
                 fill_color="cyan",
                 weight=2,
                 fill_opacity=0.1)

m += Polygon(locations=master_locations,
                 color="yellow",
                 fill_color="orange",
                 weight=1,
                 fill_opacity=0.1)


m += Polygon(locations=aoi_locations,
             color="green",
             fill_color="green",
             weight=1,
             fill_opacity=0.1)
m

In [15]:

source = '%s,%s' % (master['self'],
                   slave['self'])

In [16]:

source

Out[16]:

'https://catalog.terradue.com/sentinel1/search?format=atom&uid=S1B_S1_SLC__1SDV_20180630T051858_20180630T051915_011598_015523_67BF,https://catalog.terradue.com/sentinel1/search?format=atom&uid=S1B_S1_SLC__1SDV_20180712T051859_20180712T051915_011773_015A8D_A652'

In [17]:

crop_wkt

Out[17]:

'POLYGON ((10.65344444444444 12.17622222222222, 10.65344444444444 15.48261111111111, 8.398666666666667 15.48261111111111, 8.398666666666667 12.17622222222222, 10.65344444444444 12.17622222222222))'

In [18]:

algorithm = 'FULL'

Connect to the WPS server

In [19]:

wps_url = '%s/zoo-bin/zoo_loader.cgi' % apps_deployer

wps = WebProcessingService(wps_url,
                           verbose=False,
                           skip_caps=True)

Do a GetCapabilities WPS request and list the process:

In [20]:

wps.getcapabilities()

In [21]:

app_deployed = False

for index, elem in enumerate(wps.processes):
    if elem.identifier == app_process_id:
        app_deployed = True

if app_deployed:
    print 'Process %s deployed' % app_process_id
else:
    raise Exception('Process %s not deployed' % app_process_id)

Process ec_better_ewf_satcen_01_02_01_ewf_satcen_01_02_01_0_18 deployed

Select the process and print the title and abstract after having submited a WPS DescribeProcess request

In [22]:

process = wps.describeprocess(app_process_id)

print process.title

print process.abstract

SATCEN-01-02-01 Sentinel-1 Multi-temporal SLC and Coherence Stack
SATCEN-01-02-01 Sentinel-1 Multi-temporal SLC and Coherence Stack

List the WPS process inputs:

In [23]:

for data_input in process.dataInputs:
    print data_input.identifier

s1_products
crop_wkt
algorithm
_T2Username

Create a Python dictionary with the inputs:

In [24]:

inputs = [('s1_products', source),
          ('crop_wkt', crop_wkt),
          ('algorithm', algorithm),
          ('quotation', 'No'),
          ('_T2Username', data_pipeline)]

Submit the Execute WPS request:

In [25]:

execution = owslib.wps.WPSExecution(url=wps.url)

execution_request = execution.buildRequest(app_process_id,
                                           inputs,
                                           output=[('result_osd', False)])

execution_response = execution.submitRequest(etree.tostring(execution_request))

execution.parseResponse(execution_response)

Monitor the request:

In [26]:

execution.statusLocation

Out[26]:

'http://ec-better-apps-deployer.terradue.com/zoo-bin/zoo_loader.cgi?request=Execute&service=WPS&version=1.0.0&Identifier=GetStatus&DataInputs=sid=934a9a62-35dc-11e9-93fc-0242ac11000f&RawDataOutput=Result'

In [ ]:

monitorExecution(execution)

Check the outcome of the processing request

In [ ]:

if not execution.isSucceded():
    raise Exception('Processing failed')

Search for the results produced

In [ ]:

results_osd = execution.processOutputs[0].reference

print results_osd

In [ ]:

recast_process_id = 'dataPublication'
recast_wps_url = 'https://recast.terradue.com/t2api/ows'

wps = WebProcessingService(recast_wps_url,
                           verbose=False,
                           skip_caps=False)

recast_inputs = [('items', results_osd),
                  ('index', data_pipeline),
                  ('_T2ApiKey', data_pipeline),
                  ('_T2Username', datapipeline_api_key)]

recast_execution = wps.execute(recast_process_id,
                               recast_inputs,
                               output = [('result_osd', True)])


monitorExecution(recast_execution, sleepSecs=60)

etree.fromstring(recast_execution.processOutputs[0].data[0]).xpath('./@href')[0]