Step 9 - Run the application on the Production CentreΒΆ
Step 9 of the tutorial uses a Jupyter Notebook to query the catalog for a Sentinel-1 GRD product, creates a Web Processing Service (WPS) request invoking the data transformation application that was deployed in Step 8, monitors the WPS request execution and finally retrieves the data transformation execution results
- First do the imports of the Python libraries required
In [1]:
import owslib
from owslib.wps import monitorExecution
import uuid
from owslib.wps import WebProcessingService
import sys
import os
sys.path.append('/opt/anaconda/bin/')
import lxml.etree as etree
import requests
import cioppy
ciop = cioppy.Cioppy()
import shapely
import dateutil.parser
from shapely.wkt import loads
import pandas as pd
import geopandas as gp
from datetime import datetime, timedelta
from io import BytesIO
from zipfile import ZipFile
import matplotlib.pyplot as plt
import gdal
import numpy as np
%matplotlib inline
- Define the search parameters: the catalog series OpenSearch endpoint, the time of interest and the area of interest
In [2]:
series = 'https://catalog.terradue.com/sentinel1/search'
start_date = '2017-09-01T00:00:00'
stop_date = '2017-12-10T23:59:59'
geom = 'MULTIPOLYGON (((26.832 9.5136, 28.6843 9.5136, 28.6843 7.8009, 26.832 7.8009, 26.832 9.5136)), ((32.0572 12.4549, 33.9087 12.4549, 33.9087 10.7344, 32.0572 10.7344, 32.0572 12.4549)), ((-5.5 17.26, -1.08 17.26, -1.08 13.5, -5.5 13.5, -5.5 17.26)), ((12.9415 13.7579, 14.6731 13.7579, 14.6731 12.0093, 12.9415 12.0093, 12.9415 13.7579)))'
- Search for Sentinel-1 GRD products in one of the polygons of the area of interest
In [3]:
wkt = loads(geom)[0].wkt
In [4]:
search_params = dict([('geom', wkt),
('start', start_date),
('stop', stop_date),
('pt', 'GRD')])
In [5]:
search = ciop.search(end_point = series,
params = search_params,
output_fields='self,enclosure,identifier',
model='GeoTime')
- List the Sentinel-1 GRD products found
In [6]:
for index, elem in enumerate(search):
print(index, elem['identifier'])
(0, 'S1A_IW_GRDH_1SDV_20171207T035045_20171207T035110_019591_021463_A5A3')
(1, 'S1A_IW_GRDH_1SDV_20171207T035016_20171207T035045_019591_021463_0026')
(2, 'S1B_IW_GRDH_1SDV_20171206T035811_20171206T035836_008593_00F41C_389F')
(3, 'S1B_IW_GRDH_1SDV_20171206T035746_20171206T035811_008593_00F41C_DC3A')
(4, 'S1B_IW_GRDH_1SDV_20171201T034954_20171201T035019_008520_00F1CA_204A')
(5, 'S1B_IW_GRDH_1SDV_20171201T034929_20171201T034954_008520_00F1CA_DB5D')
(6, 'S1A_IW_GRDH_1SDV_20171125T035046_20171125T035111_019416_020EED_F4FC')
(7, 'S1A_IW_GRDH_1SDV_20171125T035017_20171125T035046_019416_020EED_2508')
(8, 'S1B_IW_GRDH_1SDV_20171124T035811_20171124T035836_008418_00EE8B_68B5')
(9, 'S1B_IW_GRDH_1SDV_20171124T035746_20171124T035811_008418_00EE8B_5E11')
(10, 'S1A_IW_GRDH_1SDV_20171113T035046_20171113T035111_019241_020964_DCB1')
(11, 'S1A_IW_GRDH_1SDV_20171113T035017_20171113T035046_019241_020964_7E05')
(12, 'S1B_IW_GRDH_1SDV_20171112T035811_20171112T035836_008243_00E941_9FD2')
(13, 'S1B_IW_GRDH_1SDV_20171112T035746_20171112T035811_008243_00E941_408D')
(14, 'S1B_IW_GRDH_1SDV_20171107T034954_20171107T035019_008170_00E70C_BA8C')
(15, 'S1B_IW_GRDH_1SDV_20171107T034929_20171107T034954_008170_00E70C_2159')
(16, 'S1A_IW_GRDH_1SDV_20171101T035046_20171101T035111_019066_0203FC_6892')
(17, 'S1A_IW_GRDH_1SDV_20171101T035017_20171101T035046_019066_0203FC_D306')
(18, 'S1B_IW_GRDH_1SDV_20171031T035812_20171031T035837_008068_00E41A_1829')
(19, 'S1B_IW_GRDH_1SDV_20171031T035747_20171031T035812_008068_00E41A_3D90')
- Select the Sentinel-1 GRD product to process
In [7]:
s1_index = 2
- Connect to the WPS server
In [8]:
wps_url = 'https://ec-better-apps-deployer.terradue.com/zoo-bin/zoo_loader.cgi'
wps = WebProcessingService(wps_url, verbose=False, skip_caps=True)
- Do a GetCapabilities WPS request and list the process:
In [9]:
wps.getcapabilities()
In [10]:
for index, elem in enumerate(wps.processes):
print(index, elem.identifier)
(0, 'ec_better_wfp_01_01_01_wfp_01_01_01_0_4')
(1, 'TerradueUnDeployProcess')
(2, 'ec_better_wfp_01_01_01_wfp_01_01_01_0_1')
(3, 'GetStatus')
(4, 'TerradueDeployProcess')
(5, 'ec_better_wfp_01_01_01_a_wfp_01_01_01_0_1')
- Select the ec_better_wfp_01_01_01_wfp_01_01_01_0_1 process and print the title and abstract after having submited a WPS DescribeProcess request
In [11]:
process_id = wps.processes[0].identifier
In [12]:
process = wps.describeprocess(process_id)
In [13]:
process.title
Out[13]:
'WFP-01-01-01 Sentinel-1 backscatter timeseries'
In [14]:
process.abstract
Out[14]:
'WFP-01-01-01 Data transformation application - Sentinel-1 backscatter timeseries'
- List the WPS process inputs:
In [15]:
for input in process.dataInputs:
print(input.identifier)
source
filterSizeX
filterSizeY
polarisation
wkt
quotation
_T2Username
- Create a Python dictionary with the inputs:
In [16]:
filterSizeX = '5'
filterSizeY = '5'
polarisation = 'VV'
In [17]:
inputs = [('source', search[s1_index]['self']),
('filterSizeX', filterSizeX),
('filterSizeY', filterSizeY),
('polarisation', polarisation),
('wkt', wkt),
('quotation', 'No'),
('_T2Username', 'fbrito')]
- Submit the Execute WPS request:
In [18]:
execution = owslib.wps.WPSExecution(url=wps.url)
execution_request = execution.buildRequest(process_id,
inputs,
output = [('result_osd', False)])
execution_response = execution.submitRequest(etree.tostring(execution_request))
execution.parseResponse(execution_response)
- Monitor the request:
In [19]:
execution.statusLocation
Out[19]:
'http://ec-better-apps-deployer.terradue.com/zoo-bin/zoo_loader.cgi?request=Execute&service=WPS&version=1.0.0&Identifier=GetStatus&DataInputs=sid=5f9d2112-84f7-11e8-a82d-0242ac110007&RawDataOutput=Result'
In [20]:
monitorExecution(execution)
owslib.wps.WPSException : {'locator': 'https://recast.terradue.com/t2api/searchfile/fbrito/_results/workflows/ec_better_wfp_01_01_01_wfp_01_01_01_0_4/run/5f9d2112-84f7-11e8-a82d-0242ac110007/0003841-180330140554685-oozie-oozi-W/.t2error.json', 'code': 'NoApplicableCode', 'text': '2018-07-11T10:52:02.050796 [ERROR ] [user process] Error executing the notebook "/application/notebook/libexec/input.ipynb".\nreporter:status:2018-07-11T10:52:02.050796 [ERROR ] [user process] Error executing the notebook "/application/notebook/libexec/input.ipynb".\n\n'}
- Check the outcome of the processing request
In [21]:
execution.isSucceded()
Out[21]:
False
- Search for the results produced
In [22]:
for output in execution.processOutputs:
print(output.identifier)
result_osd
In [23]:
results_osd = execution.processOutputs[0].reference
In [24]:
results_osd
In [25]:
search_results = ciop.search(end_point=results_osd,
params=[],
output_fields='identifier,enclosure',
model='GeoTime')
TypeErrorTraceback (most recent call last)
<ipython-input-25-0d3ebd351e5d> in <module>()
2 params=[],
3 output_fields='identifier,enclosure',
----> 4 model='GeoTime')
/opt/anaconda/envs/python2/lib/python2.7/site-packages/cioppy/cioppy.pyc in search(self, end_point, params, output_fields, model, pagination, timeout)
289 stdout=subprocess.PIPE,
290 stdin=subprocess.PIPE,
--> 291 stderr=subprocess.PIPE)
292
293 res, err = p.communicate()
/opt/anaconda/envs/python2/lib/python2.7/subprocess.pyc in __init__(self, args, bufsize, executable, stdin, stdout, stderr, preexec_fn, close_fds, shell, cwd, env, universal_newlines, startupinfo, creationflags)
709 p2cread, p2cwrite,
710 c2pread, c2pwrite,
--> 711 errread, errwrite)
712 except Exception:
713 # Preserve original exception in case os.close raises.
/opt/anaconda/envs/python2/lib/python2.7/subprocess.pyc in _execute_child(self, args, executable, preexec_fn, close_fds, cwd, env, universal_newlines, startupinfo, creationflags, shell, to_close, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite)
1341 raise
1342 child_exception = pickle.loads(data)
-> 1343 raise child_exception
1344
1345
TypeError: execv() arg 2 must contain only strings
In [ ]:
for index, elem in enumerate(search_results):
print(elem['enclosure'])