{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Trigger Queue example for Sentinel-2 L2A BOA ARD\n",
    "\n",
    "This example shows an implementation for a Trigger Queue *input.ipynb* file, as a part of a Trigger Queue Application."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prerequisites\n",
    "\n",
    "In order to run this example you will need to edit the value for the key 'value' for the following dictionaries under the **Parameters** section:\n",
    "\n",
    "* data_pipeline\n",
    "* api_key\n",
    "\n",
    "For the *data_pipeline* value, contact support@terradue.com.\n",
    "The *api_key* value can be obtained from your [Profile page](https://docs.terradue.com/ellip/accounts/myaccount/profile.html#api-key).\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Modules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from owslib.etree import etree\n",
    "import json\n",
    "import geopandas as gp\n",
    "import requests\n",
    "from shapely.geometry import box\n",
    "from shapely.wkt import loads\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import owslib\n",
    "from owslib.wps import monitorExecution\n",
    "from owslib.wps import WebProcessingService\n",
    "import cioppy\n",
    "import ellip_triggers\n",
    "import sys\n",
    "import os\n",
    "import dateutil.parser"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### <a name=\"service\">Service definition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "service = dict([('title', 'Trigger queue for Sentinel-2 L2A BOA ARD'),\n",
    "                ('abstract', 'Trigger queue for Sentinel-2 L2A BOA ARD'),\n",
    "                ('id', 'tg-common-s2-l2a-boa-ard')])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_pipeline = dict([('id', 'data_pipeline', ),\n",
    "              ('title', 'Ellip data pipeline'),\n",
    "              ('abstract', 'Ellip data pipeline'),\n",
    "              ('value', '<set value here>')])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "wps_url = dict([('id', 'wps_url'),\n",
    "              ('title', 'Application WPS end point URL'),\n",
    "              ('abstract', 'Application WPS end point URL'),\n",
    "              ('value', 'https://ec-better-apps-deployer.terradue.com/zoo/?')])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "process_id = dict([('id', 'process_id'),\n",
    "              ('title', 'Application process id'),\n",
    "              ('abstract', 'Application process id'),\n",
    "              ('value', 'ec_better_ewf_s2_l2a_boa_ard_ewf_s2_l2a_boa_ard_1_2')])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "api_key = dict([('id', 'api_key'),\n",
    "              ('title', 'Ellip API key for data pipeline'),\n",
    "              ('abstract', 'Ellip API key for data pipeline'),\n",
    "              ('value', '<set value here>')])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "aoi = dict([('id', 'aoi'),\n",
    "              ('title', 'Area of interest (bbox or WKT)'),\n",
    "              ('abstract', 'Area of interest (bbox or WKT)'),\n",
    "              ('value', '36.115823746000046,34.34882545500005,37.91779518100003,35.104433060000076')])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "start_date = dict([('id', 'start_date'),\n",
    "              ('title', 'Start date in ISO8601'),\n",
    "              ('abstract', 'Start date in ISO8601'),\n",
    "              ('value', '2019-04-10T00:00:00Z')])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "end_date = dict([('id', 'end_date'),\n",
    "              ('title', 'End date in ISO8601'),\n",
    "              ('abstract', 'Start date in ISO8601'),\n",
    "              ('value', '2019-04-25T23:59:59Z')])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "update = dict([('id', 'update'),\n",
    "              ('title', 'End date in ISO8601'),\n",
    "              ('abstract', 'Start date in ISO8601'),\n",
    "              ('value', 'void')])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### <a name=\"runtime\">Runtime parameter definition"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Input references**\n",
    "\n",
    "This is the Sentinel-1 stack catalogue references"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [],
   "source": [
    "input_references = ('dummy') "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data pipeline information"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_pipeline_parameters=dict()\n",
    "\n",
    "data_pipeline_parameters['data_pipeline'] = data_pipeline['value']\n",
    "data_pipeline_parameters['username'] = data_pipeline_parameters['data_pipeline']\n",
    "data_pipeline_parameters['wps_url'] = wps_url['value']\n",
    "data_pipeline_parameters['process_id'] = process_id['value'] \n",
    "data_pipeline_parameters['api_key'] = api_key['value']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Get information about the processing service and its inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "wps = WebProcessingService(data_pipeline_parameters['wps_url'], verbose=False, skip_caps=True)\n",
    "\n",
    "wps.getcapabilities()\n",
    "\n",
    "deployed = False\n",
    "\n",
    "for index, elem in enumerate(wps.processes):\n",
    "    \n",
    "    if data_pipeline_parameters['process_id'] in elem.identifier:\n",
    "        deployed = True\n",
    "        print 'Process {0} is deployed'.format(data_pipeline_parameters['process_id'])\n",
    "    \n",
    "if not deployed:\n",
    "    \n",
    "    raise ValueError('Process not deployed')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Describe process information"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "process = wps.describeprocess(data_pipeline_parameters['process_id'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print ('Title: {0}'.format(process.title))\n",
    "print ('Abstract: {0}'.format(process.abstract))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for data_input in process.dataInputs:\n",
    "    print ('Parameter identifier: {0}\\n'.format(data_input.identifier))\n",
    "    print ('Title/abstract: {0} - {1}\\n'.format(data_input.title, data_input.abstract)) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Area and time of interest analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "interest = dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "\n",
    "    interest['aoi'] = box(*[float(i) for i in aoi['value'].split(',')]).wkt\n",
    "\n",
    "except ValueError:\n",
    "\n",
    "    interest['aoi'] = loads(aoi['value']).wkt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "interest['start'] = start_date['value']\n",
    "interest['stop'] = end_date['value']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Search for Sentinel-2 acquisitions over the AOI"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "search_params = dict()\n",
    "\n",
    "search_params['geom'] = interest['aoi']\n",
    "search_params['pt'] = 'S2MSI2A'\n",
    "search_params['count'] = '150'\n",
    "search_params['cc'] = '100]'\n",
    "\n",
    "if update['value'] == 'void':\n",
    "    \n",
    "    search_params['start'] = interest['start']\n",
    "    search_params['stop'] = interest['stop']\n",
    "    \n",
    "else:\n",
    "    \n",
    "    search_params['update'] = update['value']\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ciop = cioppy.Cioppy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "end_point = 'https://catalog.terradue.com/sentinel2/description'\n",
    "\n",
    "\n",
    "\n",
    "search = gp.GeoDataFrame(ciop.search(end_point=end_point,\n",
    "                                      params=search_params,\n",
    "                                       output_fields='self,track,enclosure,identifier,' \\\n",
    "                                      'wkt,startdate,enddate,updated,platform,cc', \n",
    "                                       model='EOP'))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def analyse_search(row):\n",
    "    \n",
    "    series = dict()\n",
    "   \n",
    "    series['utm_zone'] = row['identifier'][39:41]\n",
    "    series['latitude_band'] = row['identifier'][41]\n",
    "    series['grid_square']  = row['identifier'][42:44]\n",
    "\n",
    "    \n",
    "    return pd.Series(series)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "search = search.merge(search.apply(lambda row: analyse_search(row), axis=1), \n",
    "              left_index=True,\n",
    "              right_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "search['startdate'] = pd.to_datetime(search['startdate'].apply(lambda x: dateutil.parser.parse(x)))\n",
    "search['enddate'] = pd.to_datetime(search['enddate'].apply(lambda x: dateutil.parser.parse(x)))\n",
    "\n",
    "search = search.rename(index=str, columns={'wkt': 'geometry'})\n",
    "search['geometry'] = search['geometry'].apply(loads)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "search = gp.GeoDataFrame(search)\n",
    "\n",
    "search.crs = {'init':'epsg:4326'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "search.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Trigger "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Instantiate a trigger:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "trigger = ellip_triggers.Trigger(data_pipeline_parameters['data_pipeline'],\n",
    "                                 data_pipeline_parameters['username'],\n",
    "                                 data_pipeline_parameters['api_key'], '', '')\n",
    "\n",
    "trigger.wps_url = data_pipeline_parameters['wps_url']\n",
    "trigger.process_id = data_pipeline_parameters['process_id']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create a function to create and queue data items:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def queue(row, trigger, username):\n",
    "    \n",
    "    series = dict()\n",
    "    \n",
    "    data_item = trigger.create_data_item_from_single_reference(row.self)\n",
    "    \n",
    "    title = 'Sentinel-2 BOA ARD ({0}-{1})'.format(row.startdate.strftime('%Y-%m-%dT%H:%m:%S'), \n",
    "                                                                                     row.enddate.strftime('%Y-%m-%dT%H:%m:%S'))\n",
    "                                                         \n",
    "    \n",
    "    data_item.set_title(title) \n",
    "    \n",
    "    data_item.set_description(title)\n",
    "    \n",
    "    data_item.set_category('validation', 'Validation data item')\n",
    "    \n",
    "    data_item.processing_parameters.append(('source', row.self))\n",
    "    data_item.processing_parameters.append(('_T2Username', username))\n",
    "    \n",
    "    trigger.queue(data_item)\n",
    "    \n",
    "    series['data_item_self'] = 'https://catalog.terradue.com/{0}/search?uid={1}'.format(data_pipeline_parameters['username'],\n",
    "                                                                                      data_item.get_identifier())\n",
    "    \n",
    "    return pd.Series(series)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "search = search.merge(search.apply(lambda row: queue(row, trigger, data_pipeline_parameters['username']),\n",
    "                                   axis=1), \n",
    "                      left_index=True,\n",
    "                      right_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "search.head(5)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}