Test the trigger

This Notebook tests:

  • the process for feeding the data pipeline queue with data items
  • the execution of a single data item
  • the piping of all the queued data items
  • First do the imports of the Python libraries required
In [1]:

import sys
import os

import owslib
from owslib.wps import monitorExecution
from owslib.wps import WebProcessingService

import ellip_triggers
import pandas as pd

import lxml.etree as etree
import requests

import cioppy

from shapely.wkt import loads
import getpass

import folium

from datetime import datetime, timedelta
import dateutil.parser

from nbconvert.preprocessors import ExecutePreprocessor, CellExecutionError
import nbformat as nbf

import pandas as pd
from geopandas import GeoDataFrame

import time
  • Read the data pipeline configuration information:
In [2]:

%store -r

nb_config = os.path.join('..', 'configuration.ipynb')

nb = nbf.read(nb_config, 4)

exec(nb['cells'][2]['source']) in globals(), locals()

app = dict([('artifact_id', app_artifact_id),
            ('version', app_version),
            ('repository', repository),
            ('community', community)])

app_process_id = '%s_%s_%s_%s' % (app['community'].replace('-', '_'),
                                  app['artifact_id'].replace('-', '_'),
                                  app['artifact_id'].replace('-', '_'),
                                  app['version'].replace('.', '_'))

trigger_queue = dict([('artifact_id', trigger_queue_artifact_id),
                      ('version', trigger_queue_version),
                      ('repository', repository),
                      ('folder', folder),
                      ('community', community)])

trigger_queue_process_id = '%s_%s_%s_%s' % (trigger_queue['community'].replace('-', '_'),
                                            trigger_queue['artifact_id'].replace('-', '_'),
                                            trigger_queue['artifact_id'].replace('-', '_'),
                                            trigger_queue['version'].replace('.', '_'))

print 'This notebook will create a queue to invoke the application %s with the trigger %s' % (app_process_id,
                                                                                              trigger_queue_process_id)

This notebook will create a queue to invoke the application ec_better_ewf_ethz_01_02_01_ewf_ethz_01_02_01_0_7 with the trigger ec_better_tg_ethz_01_02_01_queue_tg_ethz_01_02_01_queue_0_5

Set the data discovery parameters

In [3]:

bbox = (-180, -90, 180, 90)
min_mag = 5
start_date = '2018-01-09'
end_date= '2018-01-20'
buffer_size = '0.9'
  • Establish the connection with the WPS server hosting the triggers:
In [4]:

wps_url = '%s/zoo-bin/zoo_loader.cgi' % trigger_deployer

wps = WebProcessingService(wps_url,
                           verbose=False,
                           skip_caps=True)

wps.getcapabilities()

process = wps.describeprocess(trigger_queue_process_id)

print process.title
print process.abstract

ETHZ-01-02-01 Filtered DInSAR interferograms Trigger - Queue
Trigger for ETHZ-01-02-01 Filtered DInSAR interferograms - queue

In [5]:

trigger_queue_process_id

Out[5]:

'ec_better_tg_ethz_01_02_01_queue_tg_ethz_01_02_01_queue_0_5'
  • List the WPS trigger queue process data inputs:
In [6]:

for data_input in process.dataInputs:

    print data_input.identifier

data_pipeline
wps_url
process_id
api_key
username
tg_quotation
start
end
bbox
min_mag
buffer_size
quotation
_T2Username
  • Create a Python dictionary with the inputs:
In [7]:

tg_quotation = 'No'

inputs = [('data_pipeline', data_pipeline),
          ('wps_url', '%s/zoo-bin/zoo_loader.cgi' % apps_deployer),
          ('process_id', app_process_id),
          ('api_key', datapipeline_api_key),
          ('username', data_pipeline),
          ('tg_quotation', tg_quotation),
          ('start', start_date),
          ('end', end_date),
          ('bbox', ','.join(repr(e) for e in bbox)),
          ('min_mag', str(min_mag)),
          ('buffer_size', buffer_size),
          ('quotation', tg_quotation),
          ('_T2Username', data_pipeline)]
  • Submit the Execute WPS and monitor the request:
In [8]:

execution = owslib.wps.WPSExecution(url=wps_url)

execution_request = execution.buildRequest(trigger_queue_process_id,
                                           inputs,
                                           output=[('result_osd', False)])

execution_response = execution.submitRequest(etree.tostring(execution_request))

execution.parseResponse(execution_response)

print('Queue process status location:')
print(execution.statusLocation)

Queue process status location:
http://ec-better-triggers-deployer.terradue.com/zoo-bin/zoo_loader.cgi?request=Execute&service=WPS&version=1.0.0&Identifier=GetStatus&DataInputs=sid=4a3dda82-feb9-11e8-9b4b-0242ac110012&RawDataOutput=Result

In [9]:

monitorExecution(execution)
  • Check the outcome of the processing request
In [10]:

if not execution.isSucceded():

    raise Exception('Processing failed')
  • Search the queue content
In [11]:

series_queue = 'https://catalog.terradue.com/%s/series/source-queue/description' % data_pipeline

In [12]:

ciop = cioppy.Cioppy()

search_params_queue = dict()

search_params_queue['count'] = '100'

search_queue = GeoDataFrame(ciop.search(end_point=series_queue,
                            params=search_params_queue,
                            output_fields='self,identifier,wkt,startdate',
                            model='GeoTime',
                            timeout=50000))

search_queue['wkt'] = search_queue['wkt'].apply(loads)

In [13]:

search_queue

Out[13]:
identifier self startdate wkt
0 us2000clji https://catalog.terradue.com//better-ethz-0000... 2018-01-19T22:17:55.8800000Z POINT (45.7137 33.7156)
1 us2000cifw https://catalog.terradue.com//better-ethz-0000... 2018-01-11T18:42:59.5100000Z POINT (96.1686 18.4505)
2 us2000cpbb https://catalog.terradue.com//better-ethz-0000... 2018-01-11T18:28:38.4300000Z POINT (95.9558 18.4229)
3 us2000cifa https://catalog.terradue.com//better-ethz-0000... 2018-01-11T18:26:24.2400000Z POINT (96.07170000000001 18.3719)
4 us2000cidd https://catalog.terradue.com//better-ethz-0000... 2018-01-11T17:03:18.4700000Z POINT (-84.9196 10.6577)
5 us2000ci4h https://catalog.terradue.com//better-ethz-0000... 2018-01-11T08:00:39.8600000Z POINT (45.7027 33.7205)
6 us2000ci4f https://catalog.terradue.com//better-ethz-0000... 2018-01-11T07:55:00.4300000Z POINT (45.7897 33.711)
7 us2000ci3z https://catalog.terradue.com//better-ethz-0000... 2018-01-11T07:21:43.6000000Z POINT (46.42 33.2164)
8 us2000ci3y https://catalog.terradue.com//better-ethz-0000... 2018-01-11T07:14:15.9900000Z POINT (45.7299 33.8017)
9 us2000ci3x https://catalog.terradue.com//better-ethz-0000... 2018-01-11T07:00:52.3900000Z POINT (45.7935 33.8578)
10 us2000ci3s https://catalog.terradue.com//better-ethz-0000... 2018-01-11T06:59:30.4700000Z POINT (45.7239 33.7131)
  • plot the queued items
In [14]:

m = folium.Map(
    location=[45, 90],
    zoom_start=2,
    tiles='Stamen Terrain'
)

for index, row in search_queue.iterrows():

    folium.Marker([search_queue.iloc[index]['wkt'].y,
                   search_queue.iloc[index]['wkt'].x],
                   popup='<i>%s</i>' % search_queue.iloc[index]['identifier']).add_to(m)

m

Out[14]:

Test single queued date item submission

Get the first queue data item and submit the WPS Execution

In [15]:

data_item = search_queue.iloc[0]['self']
print data_item
root = etree.fromstring(requests.get(data_item).content)
print root

https://catalog.terradue.com//better-ethz-00002/series/source-queue/search?format=atom&uid=us2000clji
<Element {http://www.w3.org/2005/Atom}feed at 0x7fed6c219908>

In [16]:

ns = {'a':'http://www.w3.org/2005/Atom',
      'b':'http://www.opengis.net/owc/1.0',
      'c':'http://www.opengis.net/wps/1.0.0',
      'd':'http://www.opengis.net/ows/1.1'}

In [17]:

wps_url = root.xpath('/a:feed/a:entry/b:offering/b:operation[@code="Execute"]',
                                 namespaces=ns)[0].attrib['href']

wps_url

Out[17]:

'https://ec-better-apps-deployer.terradue.com/zoo-bin/zoo_loader.cgi'

In [18]:

process_id = root.xpath('/a:feed/a:entry/b:offering/b:operation[@code="Execute"]/b:request/c:Execute/d:Identifier',
                                 namespaces=ns)[0].text

process_id

Out[18]:

'ec_better_ewf_ethz_01_02_01_ewf_ethz_01_02_01_0_7'

In [19]:

identifiers = root.xpath('/a:feed/a:entry/b:offering/b:operation[@code="Execute"]/b:request/c:Execute/c:DataInputs/c:Input/d:Identifier',
                                 namespaces=ns)

values = root.xpath('/a:feed/a:entry/b:offering/b:operation[@code="Execute"]/b:request/c:Execute/c:DataInputs/c:Input/c:Data/c:LiteralData',
                                 namespaces=ns)

params = dict()

for index, elem in enumerate(identifiers):

    params[elem.text] = values[index].text


params

Out[19]:

{'_T2Username': 'better-ethz-00002',
 'buffer_size': '0.9',
 'source': 'us2000clji'}
  • Submit the request
In [20]:

wps = WebProcessingService(wps_url, verbose=False, skip_caps=True)

execution = owslib.wps.WPSExecution(url=wps.url)

execution_request = root.xpath('/a:feed/a:entry/b:offering/b:operation[@code="Execute"]/b:request/c:Execute',
                                 namespaces=ns)[0]


execution_response = execution.submitRequest(etree.tostring(execution_request))

execution.parseResponse(execution_response)

print execution.statusLocation

http://ec-better-apps-deployer.terradue.com/zoo-bin/zoo_loader.cgi?request=Execute&service=WPS&version=1.0.0&Identifier=GetStatus&DataInputs=sid=fad4e8b8-feb9-11e8-b646-0242ac11000f&RawDataOutput=Result

In [21]:

monitorExecution(execution)

In [22]:

if not execution.isSucceded():
    raise Exception('Single data item submission failed')

In [23]:

for output in execution.processOutputs:
    print(output.identifier)

results_osd = execution.processOutputs[0].reference

print results_osd

result_osd
QUOTATION
https://recast.terradue.com/t2api/describe/better-ethz-00002/_results/workflows/ec_better_ewf_ethz_01_02_01_ewf_ethz_01_02_01_0_7/run/fad4e8b8-feb9-11e8-b646-0242ac11000f/0028515-180330140554685-oozie-oozi-W

In [25]:

search_results = GeoDataFrame(ciop.search(end_point=results_osd,
                         params=[],
                         output_fields='title,enclosure',
                         model='GeoTime',
                            timeout=50000))

search_results

Out[25]:
enclosure title
0 Iraq (us2000clji),https://store.terradue.com/... M 5.0 - 15km ESE of Mandali
1 https://store.terradue.com/better-ethz-00002/_... Pickle file for search (us2000clji)
2 https://store.terradue.com/better-ethz-00002/_... Pickle file for selected (us2000clji)
3 https://store.terradue.com/better-ethz-00002/_... Reproducibility notebook for event us2000clji
4 https://store.terradue.com/better-ethz-00002/_... us2000clji - GEO_AMPL_CHANGE
5 https://store.terradue.com/better-ethz-00002/_... us2000clji - GEO_COH_AMPL
6 https://store.terradue.com/better-ethz-00002/_... us2000clji - GEO_DCOHER
7 https://store.terradue.com/better-ethz-00002/_... us2000clji - GEO_DPHASE

Pipe the queue

Simulate the pipe execution and process all data items queued previously

In [29]:

trigger_pipe = ellip_triggers.Trigger(data_pipeline,
                                      data_pipeline,
                                      datapipeline_api_key,
                                      '',
                                      app_process_id,
                                      'via',
                                      False)

reporter:status:2018-12-13T12:03:08.494016 [WARNING] [user process] No data pipeline configuration found (expected string or buffer), using default config
2018-12-13T12:03:08.494016 [WARNING] [user process] No data pipeline configuration found (expected string or buffer), using default config
reporter:status:2018-12-13T12:03:08.910627 [INFO   ] [user process] {
  "max_err_recovery_in": 2,
  "total_queue": 1,
  "max_err_recovery_retry": 2,
  "total_in": 1,
  "max_in": 10,
  "total_err": 1
}
2018-12-13T12:03:08.910627 [INFO   ] [user process] {
  "max_err_recovery_in": 2,
  "total_queue": 1,
  "max_err_recovery_retry": 2,
  "total_in": 1,
  "max_in": 10,
  "total_err": 1
}

In [30]:

search_params = dict()
search_params['cat'] = '{in,queue}'
search_params['count'] = '100'

end_point = "https://catalog.terradue.com/{0}/search".format(data_pipeline)
  • loop as many times as required until queue is empty
In [32]:

exit = False

while not exit:
    try:
        queue_search = ciop.search(end_point=end_point,
                                   params=search_params,
                                   output_fields='self,title',
                                   model='GeoTime',
                                  timeout=50000)


        for index, elem in enumerate(queue_search):

            data_input_queue_ref = trigger_pipe.create_data_item_from_single_reference(elem['self'])

            trigger_pipe.pipe(data_input_queue_ref)

        print '------------- sleeping ------------------'
        print '------------- sleeping ------------------'
        print '------------- sleeping ------------------'
        print '------------- sleeping ------------------'
        print '------------- sleeping ------------------'

        time.sleep(60)

    except IndexError:

        exit = True

reporter:status:2018-12-13T14:25:25.984928 [INFO   ] [user process] Processing source-queue data item : https://catalog.terradue.com/better-ethz-00002/search?uid=us2000ci3s
2018-12-13T14:25:25.984928 [INFO   ] [user process] Processing source-queue data item : https://catalog.terradue.com/better-ethz-00002/search?uid=us2000ci3s
reporter:status:2018-12-13T14:25:25.985566 [INFO   ] [user process] Job Submission...
2018-12-13T14:25:25.985566 [INFO   ] [user process] Job Submission...
reporter:status:2018-12-13T14:25:26.162977 [INFO   ] [user process] WPS submission OK. status location: http://ec-better-apps-deployer.terradue.com/zoo-bin/zoo_loader.cgi?request=Execute&service=WPS&version=1.0.0&Identifier=GetStatus&DataInputs=sid=8d41851a-feda-11e8-abce-0242ac11000f&RawDataOutput=Result
2018-12-13T14:25:26.162977 [INFO   ] [user process] WPS submission OK. status location: http://ec-better-apps-deployer.terradue.com/zoo-bin/zoo_loader.cgi?request=Execute&service=WPS&version=1.0.0&Identifier=GetStatus&DataInputs=sid=8d41851a-feda-11e8-abce-0242ac11000f&RawDataOutput=Result
reporter:status:2018-12-13T14:25:27.409304 [INFO   ] [user process] Data item updated https://catalog.terradue.com/better-ethz-00002/search?uid=us2000ci3s (200)
2018-12-13T14:25:27.409304 [INFO   ] [user process] Data item updated https://catalog.terradue.com/better-ethz-00002/search?uid=us2000ci3s (200)
reporter:status:2018-12-13T14:25:27.410072 [INFO   ] [user process] Metrics
2018-12-13T14:25:27.410072 [INFO   ] [user process] Metrics

------------- sleeping ------------------
------------- sleeping ------------------
------------- sleeping ------------------
------------- sleeping ------------------
------------- sleeping ------------------

reporter:status:2018-12-13T14:26:34.112353 [INFO   ] [user process] Processing source-in data item : https://catalog.terradue.com/better-ethz-00002/search?uid=us2000ci3s
2018-12-13T14:26:34.112353 [INFO   ] [user process] Processing source-in data item : https://catalog.terradue.com/better-ethz-00002/search?uid=us2000ci3s
reporter:status:2018-12-13T14:26:34.113033 [INFO   ] [user process] Checking running processing for DI https://catalog.terradue.com/better-ethz-00002/search?uid=us2000ci3s (http://ec-better-apps-deployer.terradue.com/zoo-bin/zoo_loader.cgi?request=Execute&service=WPS&version=1.0.0&Identifier=GetStatus&DataInputs=sid=8d41851a-feda-11e8-abce-0242ac11000f&RawDataOutput=Result)
2018-12-13T14:26:34.113033 [INFO   ] [user process] Checking running processing for DI https://catalog.terradue.com/better-ethz-00002/search?uid=us2000ci3s (http://ec-better-apps-deployer.terradue.com/zoo-bin/zoo_loader.cgi?request=Execute&service=WPS&version=1.0.0&Identifier=GetStatus&DataInputs=sid=8d41851a-feda-11e8-abce-0242ac11000f&RawDataOutput=Result)
reporter:status:2018-12-13T14:26:34.113284 [INFO   ] [user process] Metrics
2018-12-13T14:26:34.113284 [INFO   ] [user process] Metrics
reporter:status:2018-12-13T14:26:35.264701 [INFO   ] [user process] ProcessStarted for DI https://catalog.terradue.com/better-ethz-00002/search?uid=us2000ci3s
2018-12-13T14:26:35.264701 [INFO   ] [user process] ProcessStarted for DI https://catalog.terradue.com/better-ethz-00002/search?uid=us2000ci3s
reporter:status:2018-12-13T14:26:35.264928 [INFO   ] [user process] 25% : Job id: 0028584-180330140554685-oozie-oozi-W
 Workflow ID: ec-better_ewf-ethz-01-02-01_ewf-ethz-01-02-01 ID: 0028584-180330140554685-oozie-oozi-W status: RUNNING start: Thu, 13 Dec 2018 13:25:26 UTC end:
Action size: 4
        ID: 0028584-180330140554685-oozie-oozi-W@:start: Name: :start: Status: OK Start: Thu, 13 Dec 2018 13:25:26 UTC Date:  End: Thu, 13 Dec 2018 13:25:26 UTC
        ID: 0028584-180330140554685-oozie-oozi-W@hdfsPrepare Name: hdfsPrepare Status: OK Start: Thu, 13 Dec 2018 13:25:26 UTC Date:  End: Thu, 13 Dec 2018 13:25:26 UTC
        ID: 0028584-180330140554685-oozie-oozi-W@source Name: source Status: OK Start: Thu, 13 Dec 2018 13:25:27 UTC Date:  End: Thu, 13 Dec 2018 13:25:51 UTC
        ID: 0028584-180330140554685-oozie-oozi-W@notebook Name: notebook Status: RUNNING Start: Thu, 13 Dec 2018 13:25:51 UTC Date:  End:

2018-12-13T14:26:35.264928 [INFO   ] [user process] 25% : Job id: 0028584-180330140554685-oozie-oozi-W
 Workflow ID: ec-better_ewf-ethz-01-02-01_ewf-ethz-01-02-01 ID: 0028584-180330140554685-oozie-oozi-W status: RUNNING start: Thu, 13 Dec 2018 13:25:26 UTC end:
Action size: 4
        ID: 0028584-180330140554685-oozie-oozi-W@:start: Name: :start: Status: OK Start: Thu, 13 Dec 2018 13:25:26 UTC Date:  End: Thu, 13 Dec 2018 13:25:26 UTC
        ID: 0028584-180330140554685-oozie-oozi-W@hdfsPrepare Name: hdfsPrepare Status: OK Start: Thu, 13 Dec 2018 13:25:26 UTC Date:  End: Thu, 13 Dec 2018 13:25:26 UTC
        ID: 0028584-180330140554685-oozie-oozi-W@source Name: source Status: OK Start: Thu, 13 Dec 2018 13:25:27 UTC Date:  End: Thu, 13 Dec 2018 13:25:51 UTC
        ID: 0028584-180330140554685-oozie-oozi-W@notebook Name: notebook Status: RUNNING Start: Thu, 13 Dec 2018 13:25:51 UTC Date:  End:

reporter:status:2018-12-13T14:26:36.454223 [INFO   ] [user process] Data item updated https://catalog.terradue.com/better-ethz-00002/search?uid=us2000ci3s (200)
2018-12-13T14:26:36.454223 [INFO   ] [user process] Data item updated https://catalog.terradue.com/better-ethz-00002/search?uid=us2000ci3s (200)

------------- sleeping ------------------
------------- sleeping ------------------
------------- sleeping ------------------
------------- sleeping ------------------
------------- sleeping ------------------

reporter:status:2018-12-13T14:27:43.615100 [INFO   ] [user process] Processing source-in data item : https://catalog.terradue.com/better-ethz-00002/search?uid=us2000ci3s
2018-12-13T14:27:43.615100 [INFO   ] [user process] Processing source-in data item : https://catalog.terradue.com/better-ethz-00002/search?uid=us2000ci3s
reporter:status:2018-12-13T14:27:43.615722 [INFO   ] [user process] Checking running processing for DI https://catalog.terradue.com/better-ethz-00002/search?uid=us2000ci3s (http://ec-better-apps-deployer.terradue.com/zoo-bin/zoo_loader.cgi?request=Execute&service=WPS&version=1.0.0&Identifier=GetStatus&DataInputs=sid=8d41851a-feda-11e8-abce-0242ac11000f&RawDataOutput=Result)
2018-12-13T14:27:43.615722 [INFO   ] [user process] Checking running processing for DI https://catalog.terradue.com/better-ethz-00002/search?uid=us2000ci3s (http://ec-better-apps-deployer.terradue.com/zoo-bin/zoo_loader.cgi?request=Execute&service=WPS&version=1.0.0&Identifier=GetStatus&DataInputs=sid=8d41851a-feda-11e8-abce-0242ac11000f&RawDataOutput=Result)
reporter:status:2018-12-13T14:27:43.615945 [INFO   ] [user process] Metrics
2018-12-13T14:27:43.615945 [INFO   ] [user process] Metrics
reporter:status:2018-12-13T14:27:44.767524 [INFO   ] [user process] ProcessStarted for DI https://catalog.terradue.com/better-ethz-00002/search?uid=us2000ci3s
2018-12-13T14:27:44.767524 [INFO   ] [user process] ProcessStarted for DI https://catalog.terradue.com/better-ethz-00002/search?uid=us2000ci3s
reporter:status:2018-12-13T14:27:44.768261 [INFO   ] [user process] 25% : Job id: 0028584-180330140554685-oozie-oozi-W
 Workflow ID: ec-better_ewf-ethz-01-02-01_ewf-ethz-01-02-01 ID: 0028584-180330140554685-oozie-oozi-W status: RUNNING start: Thu, 13 Dec 2018 13:25:26 UTC end:
Action size: 4
        ID: 0028584-180330140554685-oozie-oozi-W@:start: Name: :start: Status: OK Start: Thu, 13 Dec 2018 13:25:26 UTC Date:  End: Thu, 13 Dec 2018 13:25:26 UTC
        ID: 0028584-180330140554685-oozie-oozi-W@hdfsPrepare Name: hdfsPrepare Status: OK Start: Thu, 13 Dec 2018 13:25:26 UTC Date:  End: Thu, 13 Dec 2018 13:25:26 UTC
        ID: 0028584-180330140554685-oozie-oozi-W@source Name: source Status: OK Start: Thu, 13 Dec 2018 13:25:27 UTC Date:  End: Thu, 13 Dec 2018 13:25:51 UTC
        ID: 0028584-180330140554685-oozie-oozi-W@notebook Name: notebook Status: RUNNING Start: Thu, 13 Dec 2018 13:25:51 UTC Date:  End:

2018-12-13T14:27:44.768261 [INFO   ] [user process] 25% : Job id: 0028584-180330140554685-oozie-oozi-W
 Workflow ID: ec-better_ewf-ethz-01-02-01_ewf-ethz-01-02-01 ID: 0028584-180330140554685-oozie-oozi-W status: RUNNING start: Thu, 13 Dec 2018 13:25:26 UTC end:
Action size: 4
        ID: 0028584-180330140554685-oozie-oozi-W@:start: Name: :start: Status: OK Start: Thu, 13 Dec 2018 13:25:26 UTC Date:  End: Thu, 13 Dec 2018 13:25:26 UTC
        ID: 0028584-180330140554685-oozie-oozi-W@hdfsPrepare Name: hdfsPrepare Status: OK Start: Thu, 13 Dec 2018 13:25:26 UTC Date:  End: Thu, 13 Dec 2018 13:25:26 UTC
        ID: 0028584-180330140554685-oozie-oozi-W@source Name: source Status: OK Start: Thu, 13 Dec 2018 13:25:27 UTC Date:  End: Thu, 13 Dec 2018 13:25:51 UTC
        ID: 0028584-180330140554685-oozie-oozi-W@notebook Name: notebook Status: RUNNING Start: Thu, 13 Dec 2018 13:25:51 UTC Date:  End:

reporter:status:2018-12-13T14:27:46.072345 [INFO   ] [user process] Data item updated https://catalog.terradue.com/better-ethz-00002/search?uid=us2000ci3s (200)
2018-12-13T14:27:46.072345 [INFO   ] [user process] Data item updated https://catalog.terradue.com/better-ethz-00002/search?uid=us2000ci3s (200)

------------- sleeping ------------------
------------- sleeping ------------------
------------- sleeping ------------------
------------- sleeping ------------------
------------- sleeping ------------------

reporter:status:2018-12-13T14:28:53.683992 [INFO   ] [user process] Processing source-in data item : https://catalog.terradue.com/better-ethz-00002/search?uid=us2000ci3s
2018-12-13T14:28:53.683992 [INFO   ] [user process] Processing source-in data item : https://catalog.terradue.com/better-ethz-00002/search?uid=us2000ci3s
reporter:status:2018-12-13T14:28:53.684599 [INFO   ] [user process] Checking running processing for DI https://catalog.terradue.com/better-ethz-00002/search?uid=us2000ci3s (http://ec-better-apps-deployer.terradue.com/zoo-bin/zoo_loader.cgi?request=Execute&service=WPS&version=1.0.0&Identifier=GetStatus&DataInputs=sid=8d41851a-feda-11e8-abce-0242ac11000f&RawDataOutput=Result)
2018-12-13T14:28:53.684599 [INFO   ] [user process] Checking running processing for DI https://catalog.terradue.com/better-ethz-00002/search?uid=us2000ci3s (http://ec-better-apps-deployer.terradue.com/zoo-bin/zoo_loader.cgi?request=Execute&service=WPS&version=1.0.0&Identifier=GetStatus&DataInputs=sid=8d41851a-feda-11e8-abce-0242ac11000f&RawDataOutput=Result)
reporter:status:2018-12-13T14:28:53.684826 [INFO   ] [user process] Metrics
2018-12-13T14:28:53.684826 [INFO   ] [user process] Metrics
reporter:status:2018-12-13T14:28:54.877820 [INFO   ] [user process] ProcessStarted for DI https://catalog.terradue.com/better-ethz-00002/search?uid=us2000ci3s
2018-12-13T14:28:54.877820 [INFO   ] [user process] ProcessStarted for DI https://catalog.terradue.com/better-ethz-00002/search?uid=us2000ci3s
reporter:status:2018-12-13T14:28:54.878030 [INFO   ] [user process] 41% : Job id: 0028584-180330140554685-oozie-oozi-W
 Workflow ID: ec-better_ewf-ethz-01-02-01_ewf-ethz-01-02-01 ID: 0028584-180330140554685-oozie-oozi-W status: RUNNING start: Thu, 13 Dec 2018 13:25:26 UTC end:
Action size: 6
        ID: 0028584-180330140554685-oozie-oozi-W@:start: Name: :start: Status: OK Start: Thu, 13 Dec 2018 13:25:26 UTC Date:  End: Thu, 13 Dec 2018 13:25:26 UTC
        ID: 0028584-180330140554685-oozie-oozi-W@hdfsPrepare Name: hdfsPrepare Status: OK Start: Thu, 13 Dec 2018 13:25:26 UTC Date:  End: Thu, 13 Dec 2018 13:25:26 UTC
        ID: 0028584-180330140554685-oozie-oozi-W@source Name: source Status: OK Start: Thu, 13 Dec 2018 13:25:27 UTC Date:  End: Thu, 13 Dec 2018 13:25:51 UTC
        ID: 0028584-180330140554685-oozie-oozi-W@notebook Name: notebook Status: OK Start: Thu, 13 Dec 2018 13:25:51 UTC Date:  End: Thu, 13 Dec 2018 13:27:41 UTC
        ID: 0028584-180330140554685-oozie-oozi-W@publish-results Name: publish-results Status: OK Start: Thu, 13 Dec 2018 13:27:41 UTC Date:  End: Thu, 13 Dec 2018 13:28:12 UTC
        ID: 0028584-180330140554685-oozie-oozi-W@T2PUSHRESULTS Name: T2PUSHRESULTS Status: RUNNING Start: Thu, 13 Dec 2018 13:28:12 UTC Date:  End:

2018-12-13T14:28:54.878030 [INFO   ] [user process] 41% : Job id: 0028584-180330140554685-oozie-oozi-W
 Workflow ID: ec-better_ewf-ethz-01-02-01_ewf-ethz-01-02-01 ID: 0028584-180330140554685-oozie-oozi-W status: RUNNING start: Thu, 13 Dec 2018 13:25:26 UTC end:
Action size: 6
        ID: 0028584-180330140554685-oozie-oozi-W@:start: Name: :start: Status: OK Start: Thu, 13 Dec 2018 13:25:26 UTC Date:  End: Thu, 13 Dec 2018 13:25:26 UTC
        ID: 0028584-180330140554685-oozie-oozi-W@hdfsPrepare Name: hdfsPrepare Status: OK Start: Thu, 13 Dec 2018 13:25:26 UTC Date:  End: Thu, 13 Dec 2018 13:25:26 UTC
        ID: 0028584-180330140554685-oozie-oozi-W@source Name: source Status: OK Start: Thu, 13 Dec 2018 13:25:27 UTC Date:  End: Thu, 13 Dec 2018 13:25:51 UTC
        ID: 0028584-180330140554685-oozie-oozi-W@notebook Name: notebook Status: OK Start: Thu, 13 Dec 2018 13:25:51 UTC Date:  End: Thu, 13 Dec 2018 13:27:41 UTC
        ID: 0028584-180330140554685-oozie-oozi-W@publish-results Name: publish-results Status: OK Start: Thu, 13 Dec 2018 13:27:41 UTC Date:  End: Thu, 13 Dec 2018 13:28:12 UTC
        ID: 0028584-180330140554685-oozie-oozi-W@T2PUSHRESULTS Name: T2PUSHRESULTS Status: RUNNING Start: Thu, 13 Dec 2018 13:28:12 UTC Date:  End:

reporter:status:2018-12-13T14:28:56.114190 [INFO   ] [user process] Data item updated https://catalog.terradue.com/better-ethz-00002/search?uid=us2000ci3s (200)
2018-12-13T14:28:56.114190 [INFO   ] [user process] Data item updated https://catalog.terradue.com/better-ethz-00002/search?uid=us2000ci3s (200)

------------- sleeping ------------------
------------- sleeping ------------------
------------- sleeping ------------------
------------- sleeping ------------------
------------- sleeping ------------------

reporter:status:2018-12-13T14:30:03.179752 [INFO   ] [user process] Processing source-in data item : https://catalog.terradue.com/better-ethz-00002/search?uid=us2000ci3s
2018-12-13T14:30:03.179752 [INFO   ] [user process] Processing source-in data item : https://catalog.terradue.com/better-ethz-00002/search?uid=us2000ci3s
reporter:status:2018-12-13T14:30:03.180387 [INFO   ] [user process] Checking running processing for DI https://catalog.terradue.com/better-ethz-00002/search?uid=us2000ci3s (http://ec-better-apps-deployer.terradue.com/zoo-bin/zoo_loader.cgi?request=Execute&service=WPS&version=1.0.0&Identifier=GetStatus&DataInputs=sid=8d41851a-feda-11e8-abce-0242ac11000f&RawDataOutput=Result)
2018-12-13T14:30:03.180387 [INFO   ] [user process] Checking running processing for DI https://catalog.terradue.com/better-ethz-00002/search?uid=us2000ci3s (http://ec-better-apps-deployer.terradue.com/zoo-bin/zoo_loader.cgi?request=Execute&service=WPS&version=1.0.0&Identifier=GetStatus&DataInputs=sid=8d41851a-feda-11e8-abce-0242ac11000f&RawDataOutput=Result)
reporter:status:2018-12-13T14:30:03.180607 [INFO   ] [user process] Metrics
2018-12-13T14:30:03.180607 [INFO   ] [user process] Metrics
reporter:status:2018-12-13T14:30:04.337745 [INFO   ] [user process] ProcessStarted for DI https://catalog.terradue.com/better-ethz-00002/search?uid=us2000ci3s
2018-12-13T14:30:04.337745 [INFO   ] [user process] ProcessStarted for DI https://catalog.terradue.com/better-ethz-00002/search?uid=us2000ci3s
reporter:status:2018-12-13T14:30:04.337964 [INFO   ] [user process] 75% : Job id: 0028584-180330140554685-oozie-oozi-W
 Workflow ID: ec-better_ewf-ethz-01-02-01_ewf-ethz-01-02-01 ID: 0028584-180330140554685-oozie-oozi-W status: RUNNING start: Thu, 13 Dec 2018 13:25:26 UTC end:
Action size: 10
        ID: 0028584-180330140554685-oozie-oozi-W@:start: Name: :start: Status: OK Start: Thu, 13 Dec 2018 13:25:26 UTC Date:  End: Thu, 13 Dec 2018 13:25:26 UTC
        ID: 0028584-180330140554685-oozie-oozi-W@hdfsPrepare Name: hdfsPrepare Status: OK Start: Thu, 13 Dec 2018 13:25:26 UTC Date:  End: Thu, 13 Dec 2018 13:25:26 UTC
        ID: 0028584-180330140554685-oozie-oozi-W@source Name: source Status: OK Start: Thu, 13 Dec 2018 13:25:27 UTC Date:  End: Thu, 13 Dec 2018 13:25:51 UTC
        ID: 0028584-180330140554685-oozie-oozi-W@notebook Name: notebook Status: OK Start: Thu, 13 Dec 2018 13:25:51 UTC Date:  End: Thu, 13 Dec 2018 13:27:41 UTC
        ID: 0028584-180330140554685-oozie-oozi-W@publish-results Name: publish-results Status: OK Start: Thu, 13 Dec 2018 13:27:41 UTC Date:  End: Thu, 13 Dec 2018 13:28:12 UTC
        ID: 0028584-180330140554685-oozie-oozi-W@T2PUSHRESULTS Name: T2PUSHRESULTS Status: OK Start: Thu, 13 Dec 2018 13:28:12 UTC Date: #
#Thu Dec 13 14:28:58 CET 2018
t2pushresults=2
 End: Thu, 13 Dec 2018 13:28:58 UTC
        ID: 0028584-180330140554685-oozie-oozi-W@nodeDecisionPush Name: nodeDecisionPush Status: OK Start: Thu, 13 Dec 2018 13:28:58 UTC Date:  End: Thu, 13 Dec 2018 13:28:58 UTC
        ID: 0028584-180330140554685-oozie-oozi-W@EndNode Name: EndNode Status: OK Start: Thu, 13 Dec 2018 13:28:58 UTC Date: #
#Thu Dec 13 14:29:16 CET 2018
endValue=2
 End: Thu, 13 Dec 2018 13:29:16 UTC
        ID: 0028584-180330140554685-oozie-oozi-W@t2CleanNode Name: t2CleanNode Status: OK Start: Thu, 13 Dec 2018 13:29:16 UTC Date: #
#Thu Dec 13 14:29:37 CET 2018
startClean=1
 End: Thu, 13 Dec 2018 13:29:37 UTC
        ID: 0028584-180330140554685-oozie-oozi-W@t2CleanHdfsDataNode Name: t2CleanHdfsDataNode Status: RUNNING Start: Thu, 13 Dec 2018 13:29:37 UTC Date:  End:

2018-12-13T14:30:04.337964 [INFO   ] [user process] 75% : Job id: 0028584-180330140554685-oozie-oozi-W
 Workflow ID: ec-better_ewf-ethz-01-02-01_ewf-ethz-01-02-01 ID: 0028584-180330140554685-oozie-oozi-W status: RUNNING start: Thu, 13 Dec 2018 13:25:26 UTC end:
Action size: 10
        ID: 0028584-180330140554685-oozie-oozi-W@:start: Name: :start: Status: OK Start: Thu, 13 Dec 2018 13:25:26 UTC Date:  End: Thu, 13 Dec 2018 13:25:26 UTC
        ID: 0028584-180330140554685-oozie-oozi-W@hdfsPrepare Name: hdfsPrepare Status: OK Start: Thu, 13 Dec 2018 13:25:26 UTC Date:  End: Thu, 13 Dec 2018 13:25:26 UTC
        ID: 0028584-180330140554685-oozie-oozi-W@source Name: source Status: OK Start: Thu, 13 Dec 2018 13:25:27 UTC Date:  End: Thu, 13 Dec 2018 13:25:51 UTC
        ID: 0028584-180330140554685-oozie-oozi-W@notebook Name: notebook Status: OK Start: Thu, 13 Dec 2018 13:25:51 UTC Date:  End: Thu, 13 Dec 2018 13:27:41 UTC
        ID: 0028584-180330140554685-oozie-oozi-W@publish-results Name: publish-results Status: OK Start: Thu, 13 Dec 2018 13:27:41 UTC Date:  End: Thu, 13 Dec 2018 13:28:12 UTC
        ID: 0028584-180330140554685-oozie-oozi-W@T2PUSHRESULTS Name: T2PUSHRESULTS Status: OK Start: Thu, 13 Dec 2018 13:28:12 UTC Date: #
#Thu Dec 13 14:28:58 CET 2018
t2pushresults=2
 End: Thu, 13 Dec 2018 13:28:58 UTC
        ID: 0028584-180330140554685-oozie-oozi-W@nodeDecisionPush Name: nodeDecisionPush Status: OK Start: Thu, 13 Dec 2018 13:28:58 UTC Date:  End: Thu, 13 Dec 2018 13:28:58 UTC
        ID: 0028584-180330140554685-oozie-oozi-W@EndNode Name: EndNode Status: OK Start: Thu, 13 Dec 2018 13:28:58 UTC Date: #
#Thu Dec 13 14:29:16 CET 2018
endValue=2
 End: Thu, 13 Dec 2018 13:29:16 UTC
        ID: 0028584-180330140554685-oozie-oozi-W@t2CleanNode Name: t2CleanNode Status: OK Start: Thu, 13 Dec 2018 13:29:16 UTC Date: #
#Thu Dec 13 14:29:37 CET 2018
startClean=1
 End: Thu, 13 Dec 2018 13:29:37 UTC
        ID: 0028584-180330140554685-oozie-oozi-W@t2CleanHdfsDataNode Name: t2CleanHdfsDataNode Status: RUNNING Start: Thu, 13 Dec 2018 13:29:37 UTC Date:  End:

reporter:status:2018-12-13T14:30:05.692325 [INFO   ] [user process] Data item updated https://catalog.terradue.com/better-ethz-00002/search?uid=us2000ci3s (200)
2018-12-13T14:30:05.692325 [INFO   ] [user process] Data item updated https://catalog.terradue.com/better-ethz-00002/search?uid=us2000ci3s (200)

------------- sleeping ------------------
------------- sleeping ------------------
------------- sleeping ------------------
------------- sleeping ------------------
------------- sleeping ------------------

reporter:status:2018-12-13T14:31:12.263991 [INFO   ] [user process] Processing source-in data item : https://catalog.terradue.com/better-ethz-00002/search?uid=us2000ci3s
2018-12-13T14:31:12.263991 [INFO   ] [user process] Processing source-in data item : https://catalog.terradue.com/better-ethz-00002/search?uid=us2000ci3s
reporter:status:2018-12-13T14:31:12.264618 [INFO   ] [user process] Checking running processing for DI https://catalog.terradue.com/better-ethz-00002/search?uid=us2000ci3s (http://ec-better-apps-deployer.terradue.com/zoo-bin/zoo_loader.cgi?request=Execute&service=WPS&version=1.0.0&Identifier=GetStatus&DataInputs=sid=8d41851a-feda-11e8-abce-0242ac11000f&RawDataOutput=Result)
2018-12-13T14:31:12.264618 [INFO   ] [user process] Checking running processing for DI https://catalog.terradue.com/better-ethz-00002/search?uid=us2000ci3s (http://ec-better-apps-deployer.terradue.com/zoo-bin/zoo_loader.cgi?request=Execute&service=WPS&version=1.0.0&Identifier=GetStatus&DataInputs=sid=8d41851a-feda-11e8-abce-0242ac11000f&RawDataOutput=Result)
reporter:status:2018-12-13T14:31:12.264873 [INFO   ] [user process] Metrics
2018-12-13T14:31:12.264873 [INFO   ] [user process] Metrics
reporter:status:2018-12-13T14:31:12.459461 [INFO   ] [user process] ProcessSucceeded for DI https://catalog.terradue.com/better-ethz-00002/search?uid=us2000ci3s
2018-12-13T14:31:12.459461 [INFO   ] [user process] ProcessSucceeded for DI https://catalog.terradue.com/better-ethz-00002/search?uid=us2000ci3s
reporter:status:2018-12-13T14:31:12.460276 [INFO   ] [user process] Metrics
2018-12-13T14:31:12.460276 [INFO   ] [user process] Metrics
reporter:status:2018-12-13T14:31:12.460580 [INFO   ] [user process] Process succeeded for DI https://catalog.terradue.com/better-ethz-00002/search?uid=us2000ci3s
2018-12-13T14:31:12.460580 [INFO   ] [user process] Process succeeded for DI https://catalog.terradue.com/better-ethz-00002/search?uid=us2000ci3s
reporter:status:2018-12-13T14:31:12.461003 [INFO   ] [user process] Recast osd: https://recast.terradue.com/t2api/describe/better-ethz-00002/_results/workflows/ec_better_ewf_ethz_01_02_01_ewf_ethz_01_02_01_0_7/run/8d41851a-feda-11e8-abce-0242ac11000f/0028584-180330140554685-oozie-oozi-W
2018-12-13T14:31:12.461003 [INFO   ] [user process] Recast osd: https://recast.terradue.com/t2api/describe/better-ethz-00002/_results/workflows/ec_better_ewf_ethz_01_02_01_ewf_ethz_01_02_01_0_7/run/8d41851a-feda-11e8-abce-0242ac11000f/0028584-180330140554685-oozie-oozi-W
reporter:status:2018-12-13T14:31:12.461413 [INFO   ] [user process] Recast WPS process submission
2018-12-13T14:31:12.461413 [INFO   ] [user process] Recast WPS process submission
reporter:status:2018-12-13T14:31:12.627503 [INFO   ] [user process] Recast WPS connected
2018-12-13T14:31:12.627503 [INFO   ] [user process] Recast WPS connected
reporter:status:2018-12-13T14:31:12.662406 [INFO   ] [user process] Recast WPS execution submitted, monitoring...
2018-12-13T14:31:12.662406 [INFO   ] [user process] Recast WPS execution submitted, monitoring...
reporter:status:2018-12-13T14:37:13.077649 [INFO   ] [user process] Recast WPS execution is succeeded
2018-12-13T14:37:13.077649 [INFO   ] [user process] Recast WPS execution is succeeded
reporter:status:2018-12-13T14:37:13.078293 [ERROR  ] [user process] Recast execution failed, no results
2018-12-13T14:37:13.078293 [ERROR  ] [user process] Recast execution failed, no results
reporter:status:2018-12-13T14:37:13.078726 [INFO   ] [user process] Metrics
2018-12-13T14:37:13.078726 [INFO   ] [user process] Metrics
reporter:status:2018-12-13T14:37:13.079064 [INFO   ] [user process] Recast step is succeeded: False
2018-12-13T14:37:13.079064 [INFO   ] [user process] Recast step is succeeded: False
reporter:status:2018-12-13T14:37:13.080084 [ERROR  ] [user process] Unexpected error: 'bool' object has no attribute 'replace'
2018-12-13T14:37:13.080084 [ERROR  ] [user process] Unexpected error: 'bool' object has no attribute 'replace'
reporter:status:2018-12-13T14:37:13.081203 [ERROR  ] [user process]   File "/opt/anaconda/lib/python2.7/runpy.py", line 174, in _run_module_as_main
    "__main__", fname, loader, pkg_name)
  File "/opt/anaconda/lib/python2.7/runpy.py", line 72, in _run_code
    exec code in run_globals
  File "/opt/anaconda/lib/python2.7/site-packages/ipykernel/__main__.py", line 3, in <module>
    app.launch_new_instance()
  File "/opt/anaconda/lib/python2.7/site-packages/traitlets/config/application.py", line 658, in launch_instance
    app.start()
  File "/opt/anaconda/lib/python2.7/site-packages/ipykernel/kernelapp.py", line 499, in start
    self.io_loop.start()
  File "/opt/anaconda/lib/python2.7/site-packages/zmq/eventloop/ioloop.py", line 177, in start
    super(ZMQIOLoop, self).start()
  File "/opt/anaconda/lib/python2.7/site-packages/tornado/ioloop.py", line 887, in start
    handler_func(fd_obj, events)
  File "/opt/anaconda/lib/python2.7/site-packages/tornado/stack_context.py", line 275, in null_wrapper
    return fn(*args, **kwargs)
  File "/opt/anaconda/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 440, in _handle_events
    self._handle_recv()
  File "/opt/anaconda/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 472, in _handle_recv
    self._run_callback(callback, msg)
  File "/opt/anaconda/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 414, in _run_callback
    callback(*args, **kwargs)
  File "/opt/anaconda/lib/python2.7/site-packages/tornado/stack_context.py", line 275, in null_wrapper
    return fn(*args, **kwargs)
  File "/opt/anaconda/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 283, in dispatcher
    return self.dispatch_shell(stream, msg)
  File "/opt/anaconda/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 233, in dispatch_shell
    handler(stream, idents, msg)
  File "/opt/anaconda/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 399, in execute_request
    user_expressions, allow_stdin)
  File "/opt/anaconda/lib/python2.7/site-packages/ipykernel/ipkernel.py", line 208, in do_execute
    res = shell.run_cell(code, store_history=store_history, silent=silent)
  File "/opt/anaconda/lib/python2.7/site-packages/ipykernel/zmqshell.py", line 537, in run_cell
    return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)
  File "/opt/anaconda/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2714, in run_cell
    interactivity=interactivity, compiler=compiler, result=result)
  File "/opt/anaconda/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2818, in run_ast_nodes
    if self.run_code(code, result):
  File "/opt/anaconda/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2878, in run_code
    exec(code_obj, self.user_global_ns, self.user_ns)
  File "<ipython-input-32-b955d70c1136>", line 16, in <module>
    trigger_pipe.pipe(data_input_queue_ref)
  File "/opt/anaconda/lib/python2.7/site-packages/ellip_triggers/trigger.py", line 254, in pipe
    ciop.log('ERROR', ''.join(traceback.format_stack()))

2018-12-13T14:37:13.081203 [ERROR  ] [user process]   File "/opt/anaconda/lib/python2.7/runpy.py", line 174, in _run_module_as_main
    "__main__", fname, loader, pkg_name)
  File "/opt/anaconda/lib/python2.7/runpy.py", line 72, in _run_code
    exec code in run_globals
  File "/opt/anaconda/lib/python2.7/site-packages/ipykernel/__main__.py", line 3, in <module>
    app.launch_new_instance()
  File "/opt/anaconda/lib/python2.7/site-packages/traitlets/config/application.py", line 658, in launch_instance
    app.start()
  File "/opt/anaconda/lib/python2.7/site-packages/ipykernel/kernelapp.py", line 499, in start
    self.io_loop.start()
  File "/opt/anaconda/lib/python2.7/site-packages/zmq/eventloop/ioloop.py", line 177, in start
    super(ZMQIOLoop, self).start()
  File "/opt/anaconda/lib/python2.7/site-packages/tornado/ioloop.py", line 887, in start
    handler_func(fd_obj, events)
  File "/opt/anaconda/lib/python2.7/site-packages/tornado/stack_context.py", line 275, in null_wrapper
    return fn(*args, **kwargs)
  File "/opt/anaconda/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 440, in _handle_events
    self._handle_recv()
  File "/opt/anaconda/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 472, in _handle_recv
    self._run_callback(callback, msg)
  File "/opt/anaconda/lib/python2.7/site-packages/zmq/eventloop/zmqstream.py", line 414, in _run_callback
    callback(*args, **kwargs)
  File "/opt/anaconda/lib/python2.7/site-packages/tornado/stack_context.py", line 275, in null_wrapper
    return fn(*args, **kwargs)
  File "/opt/anaconda/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 283, in dispatcher
    return self.dispatch_shell(stream, msg)
  File "/opt/anaconda/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 233, in dispatch_shell
    handler(stream, idents, msg)
  File "/opt/anaconda/lib/python2.7/site-packages/ipykernel/kernelbase.py", line 399, in execute_request
    user_expressions, allow_stdin)
  File "/opt/anaconda/lib/python2.7/site-packages/ipykernel/ipkernel.py", line 208, in do_execute
    res = shell.run_cell(code, store_history=store_history, silent=silent)
  File "/opt/anaconda/lib/python2.7/site-packages/ipykernel/zmqshell.py", line 537, in run_cell
    return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)
  File "/opt/anaconda/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2714, in run_cell
    interactivity=interactivity, compiler=compiler, result=result)
  File "/opt/anaconda/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2818, in run_ast_nodes
    if self.run_code(code, result):
  File "/opt/anaconda/lib/python2.7/site-packages/IPython/core/interactiveshell.py", line 2878, in run_code
    exec(code_obj, self.user_global_ns, self.user_ns)
  File "<ipython-input-32-b955d70c1136>", line 16, in <module>
    trigger_pipe.pipe(data_input_queue_ref)
  File "/opt/anaconda/lib/python2.7/site-packages/ellip_triggers/trigger.py", line 254, in pipe
    ciop.log('ERROR', ''.join(traceback.format_stack()))

reporter:status:2018-12-13T14:37:14.365525 [INFO   ] [user process] Data item updated https://catalog.terradue.com/better-ethz-00002/search?uid=us2000ci3s (200)
2018-12-13T14:37:14.365525 [INFO   ] [user process] Data item updated https://catalog.terradue.com/better-ethz-00002/search?uid=us2000ci3s (200)
reporter:status:2018-12-13T14:37:14.366336 [INFO   ] [user process] Metrics
2018-12-13T14:37:14.366336 [INFO   ] [user process] Metrics


AttributeErrorTraceback (most recent call last)
<ipython-input-32-b955d70c1136> in <module>()
     14             data_input_queue_ref = trigger_pipe.create_data_item_from_single_reference(elem['self'])
     15
---> 16             trigger_pipe.pipe(data_input_queue_ref)
     17
     18         print '------------- sleeping ------------------'

/opt/anaconda/lib/python2.7/site-packages/ellip_triggers/trigger.pyc in pipe(self, data_item, waitForCompletion)
    230
    231                 # !YEAH! lets save the results!
--> 232                 self.datapipeline.success(data_item, final_result_osd)
    233
    234                 self.metrics.send_metrics(data_item, 'source-out')

/opt/anaconda/lib/python2.7/site-packages/ellip_triggers/datapipeline.pyc in success(self, data_item, result_osd)
    128         data_item.atom.set_title_text(title)
    129         data_item.atom.append_summary_html('Processing complete at %s' % success)
--> 130         data_item.add_results(result_osd.replace("description", "search"), "View results")
    131
    132         # Success!

AttributeError: 'bool' object has no attribute 'replace'
  • Check the outcome
In [ ]:

search_params = dict()
search_params['cat'] = '{out,err}'
search_params['count'] = '100'

end_point = "https://catalog.terradue.com/{0}/search".format(data_pipeline)


In [ ]:

search_piped = GeoDataFrame(ciop.search(end_point=end_point,
                           params=search_params,
                           output_fields='self,wkt,identifier',
                           model='GeoTime',
                         timeout=50000))

search_piped['wkt'] = search_piped['wkt'].apply(loads)

In [ ]:

m = folium.Map(
    location=[45, 90],
    zoom_start=2,
    tiles='Stamen Terrain'
)

for index, row in search_queue.iterrows():

    folium.Marker([search_piped.iloc[index]['wkt'].y,
                   search_piped.iloc[index]['wkt'].x],
                   popup='<i>%s</i>' % search_piped.iloc[index]['identifier']).add_to(m)

m

In [ ]:

search_piped.head()