{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Generate an ATOM entry for the catalog and perform its ingestion\n",
"\n",
"In this scenario, we generate an ATOM entry containing the metadata of a product and feed it into the catalog (we use a sample product downloaded from a public repository).\n",
"\n",
"## 1. Set the necessary variables\n",
"\n",
"The following section defines all the necessary information as variables so the code below can be easily reused."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import getpass\n",
"\n",
"# Set the credentials (Ellip username and API key)\n",
"username = raw_input(\"What is your Ellip username? \")\n",
"api_key = getpass.getpass(\"What is your Ellip API key? \")\n",
"\n",
"# Set the name of the destination index on the Terradue catalog\n",
"index_name = raw_input(\"What is the destination index name? (press Enter to confirm default [{0}]) \".format(username))\n",
"\n",
"if not index_name:\n",
" index_name = username\n",
"\n",
"# Set the catalog endpoint URL\n",
"endpoint = \"https://catalog.terradue.com/{0}\".format(index_name)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## 2. Define a function to generate an *EarthObservation* extension element"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import lxml.etree as etree\n",
"import numpy as np\n",
"from shapely.wkt import loads\n",
"\n",
"def eop_metadata(metadata):\n",
" \n",
" # Define namespace URIs\n",
" opt = 'http://www.opengis.net/opt/2.1'\n",
" om = 'http://www.opengis.net/om/2.0'\n",
" gml = 'http://www.opengis.net/gml/3.2'\n",
" eop = 'http://www.opengis.net/eop/2.1'\n",
" sar = 'http://www.opengis.net/sar/2.1'\n",
" \n",
" # Define the element structure\n",
" # There are several levels for much of the content and many elements are only containers for other elements;\n",
" # elements that hold actual values are marked as 'content element' below.\n",
" root = etree.Element('{%s}EarthObservation' % opt)\n",
"\n",
" phenomenon_time = etree.SubElement(root, '{%s}phenomenonTime' % om)\n",
" time_period = etree.SubElement(phenomenon_time, '{%s}TimePeriod' % gml)\n",
" # Content element:\n",
" begin_position = etree.SubElement(time_period, '{%s}beginPosition' % gml)\n",
" # Content element:\n",
" end_position = etree.SubElement(time_period, '{%s}endPosition' % gml)\n",
"\n",
" procedure = etree.SubElement(root, '{%s}procedure' % om)\n",
" earth_observation_equipment = etree.SubElement(procedure, '{%s}EarthObservationEquipment' % eop)\n",
" acquisition = etree.SubElement(earth_observation_equipment, '{%s}acquisitionParameters' % eop)\n",
" # Content element:\n",
" orbit_number = etree.SubElement(acquisition, '{%s}orbitNumber' % eop)\n",
" # Content element:\n",
" wrs_longitude_grid = etree.SubElement(acquisition, '{%s}wrsLongitudeGrid' % eop)\n",
" # Content element:\n",
" orbit_direction = etree.SubElement(acquisition, '{%s}orbitDirection' % eop)\n",
"\n",
" feature_of_interest = etree.SubElement(root, '{%s}featureOfInterest' % om)\n",
" footprint = etree.SubElement(feature_of_interest, '{%s}Footprint' % eop)\n",
" multi_extentOf = etree.SubElement(footprint, '{%s}multiExtentOf' % eop)\n",
" multi_surface = etree.SubElement(multi_extentOf, '{%s}MultiSurface' % gml)\n",
" surface_members = etree.SubElement(multi_surface, '{%s}surfaceMembers' % gml)\n",
" polygon = etree.SubElement(surface_members, '{%s}Polygon' % gml) \n",
" exterior = etree.SubElement(polygon, '{%s}exterior' % gml) \n",
" linear_ring = etree.SubElement(exterior, '{%s}LinearRing' % gml) \n",
" # Content element:\n",
" poslist = etree.SubElement(linear_ring, '{%s}posList' % gml) \n",
"\n",
" result = etree.SubElement(root, '{%s}result' % om)\n",
" earth_observation_result = etree.SubElement(result, '{%s}EarthObservationResult' % opt)\n",
" # Content element:\n",
" cloud_cover_percentage = etree.SubElement(earth_observation_result, '{%s}cloudCoverPercentage' % opt)\n",
" \n",
" metadata_property = etree.SubElement(root, '{%s}metaDataProperty' % eop)\n",
" earth_observation_metadata = etree.SubElement(metadata_property, '{%s}EarthObservationMetaData' % eop)\n",
" # Content element:\n",
" identifier = etree.SubElement(earth_observation_metadata, '{%s}identifier' % eop)\n",
" # Content element:\n",
" product_type = etree.SubElement(earth_observation_metadata, '{%s}productType' % eop)\n",
" \n",
" vendor_specific = etree.SubElement(earth_observation_metadata, '{%s}vendorSpecific' % eop)\n",
" specific_information = etree.SubElement(vendor_specific, '{%s}SpecificInformation' % eop)\n",
" # Content element:\n",
" local_attribute = etree.SubElement(specific_information, '{%s}localAttribute' % eop)\n",
" # Content element:\n",
" local_value = etree.SubElement(specific_information, '{%s}localValue' % eop)\n",
" \n",
" # Set values for content elements\n",
" begin_position.text = metadata['startdate']\n",
" end_position.text = metadata['enddate']\n",
" orbit_number.text = metadata['orbitNumber']\n",
" wrs_longitude_grid.text = metadata['wrsLongitudeGrid']\n",
" orbit_direction.text = metadata['orbitDirection']\n",
" \n",
" coords = np.asarray([t[::-1] for t in list(loads(metadata['wkt']).exterior.coords)]).tolist()\n",
" pos_list = ''\n",
" for elem in coords:\n",
" pos_list += ' '.join(str(e) for e in elem) + ' ' \n",
"\n",
" poslist.attrib['count'] = str(len(coords))\n",
" poslist.text = pos_list\n",
" \n",
" cloud_cover_percentage.text = metadata['cc']\n",
" \n",
" identifier.text = metadata['identifier']\n",
" product_type.text = metadata['productType']\n",
" \n",
" local_attribute.text = 'MY_ATTRIBUTE'\n",
" local_value.text = metadata['MY_ATTRIBUTE']\n",
" \n",
" return root"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## 3. Define a class for ATOM manipulation\n",
"\n",
"This class allows us to add basic elements such as an identifier, a title, an enclosure link and a product date; and to append the ``EarthObservation`` extension created above."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import lxml.etree as etree\n",
"import sys\n",
"import os\n",
"import string\n",
"import hashlib\n",
"import urllib2\n",
"import base64\n",
"import time\n",
"\n",
"class Atom:\n",
" tree = None\n",
" root = None\n",
" entry = None\n",
" \n",
" def __init__(self, root):\n",
" self.root = root\n",
" self.tree = root\n",
" self.links = self.root.xpath('/a:feed/a:entry/a:link', namespaces={'a':'http://www.w3.org/2005/Atom'})\n",
" entries = self.root.xpath('/a:feed/a:entry', namespaces={'a':'http://www.w3.org/2005/Atom'})\n",
" if len(entries) > 0:\n",
" self.entry = entries[0]\n",
" \n",
" @staticmethod\n",
" def from_template():\n",
" template = \"\"\"\n",
"
%s
\" % text\n", "\n", " self.set_summary_text(html_summary)\n", "\n", "\n", " def to_string(self, pretty_print = True):\n", " \n", " return etree.tostring(self.tree, pretty_print=pretty_print)\n", " \n", " def clear_enclosures(self):\n", " \n", " links = self.get_links(\"enclosure\")\n", " for link in links:\n", " link.getparent().remove(link) \n", " \n", " def get_extensions(self, name, namespace):\n", " \n", " return self.root.xpath('/a:feed/a:entry/e:{0}'.format(name), \n", " namespaces={'a':'http://www.w3.org/2005/Atom',\n", " 'e':namespace})" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## 4. Build the *EarthObservation* extension element\n", "\n", "We define a dictionary containing the metadata and use it as argument for the function defined above." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "metadata = {'startdate': '2019-01-02T03:04:05.678Z',\n", " 'enddate': '2019-01-02T03:05:06.789Z',\n", " 'orbitNumber': '99',\n", " 'wrsLongitudeGrid':'123',\n", " 'orbitDirection': 'DESCENDING',\n", " 'wkt': 'POLYGON((10.1 10.2,20.3 10.4,20.5 20.6,10.7 20.8,10.1 10.2))',\n", " 'cc': '55',\n", " 'identifier' : 'MY_PRODUCT',\n", " 'productType': 'MY_TYPE',\n", " 'MY_ATTRIBUTE': 'MY_VALUE'\n", "}\n", "\n", "# Build the element\n", "eo = eop_metadata(metadata)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Show the `EarthObservation` element just created:" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "print(etree.tostring(eo, pretty_print=True))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## 5. Build an ATOM feed\n", "\n", "We create an ATOM feed with one entry to which we append the extension created above." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import datetime\n", "\n", "atom = Atom.from_template()\n", "atom.set_identifier(metadata['identifier'])\n", "atom.set_title_text(\"Title for MY_PRODUCT\")\n", "atom.set_summary_text(\"This is the summary for MY_PRODUCT\")\n", "atom.set_dcdate(\"{0}/{1}\".format(metadata['startdate'], metadata['enddate']))\n", "atom.set_published(\"{0}Z\".format(datetime.datetime.now().isoformat()))\n", "\n", "atom.add_extension(eo)\n", "\n", "atom.set_enclosure_link(\"https://store.terradue.com/myindex/MY_PRODUCT.tif\", \"Location on storage\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Show the resulting ATOM feed:" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "print(etree.tostring(atom.root, pretty_print=True))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## 6. Post the ATOM feed\n", "\n", "We post the ATOM feed to an index on the catalog (variables are defined in the first step)." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import requests\n", "\n", "request = requests.post(endpoint,\n", " headers={\"Content-Type\": \"application/atom+xml\", \"Accept\": \"application/xml\"},\n", " auth=(username, api_key),\n", " data=atom.to_string()\n", ")\n", "\n", "if request.status_code == 200:\n", " print('Data item updated at {0}/search?uid={1}&apikey={2} ({3})'.format(endpoint, atom.get_identifier(), api_key, str(request.status_code)))\n", "else:\n", " print('Data item NOT updated at {0}/search?uid={1}&apikey={2} ({3})'.format(endpoint, atom.get_identifier(), api_key, str(request.status_code)))\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "If a product URL with status 200 is displayed, the ATOM feed has been successfully uploaded and the product information is available on the Terradue catalog.\n", "\n", "**END**" ] } ], "metadata": { "kernelspec": { "display_name": "Python 2", "language": "python", "name": "python2" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 2 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython2", "version": "2.7.12" } }, "nbformat": 4, "nbformat_minor": 2 }