{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Upload data to the Terradue storage\n",
    "\n",
    "In this scenario, we generate a *.properties* file that accompanies the result files and upload all files that form the result to the Terradue storage.\n",
    "\n",
    "## 1. Set the necessary variables\n",
    "\n",
    "The following section defines all the necessary information as variables so the code below can be easily reused."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import getpass\n",
    "\n",
    "# Set the credentials (Ellip username and API key)\n",
    "username = raw_input(\"What is your Ellip username? \")\n",
    "api_key = getpass.getpass(\"What is your Ellip API key? \")\n",
    "\n",
    "# Set the name of the destination repository on the Terradue storage\n",
    "repo_name = raw_input(\"What is the destination respository name? (press Enter to confirm default [{0}]) \".format(username))\n",
    "if not repo_name:\n",
    "    repo_name = username\n",
    "\n",
    "# Set the destination URL on store (directory)\n",
    "store_url = \"https://store.terradue.com/{0}/data-publication-sample/\".format(repo_name)\n",
    "\n",
    "# Set dictionary with file information\n",
    "files = [\n",
    "    {'name':'data-publication-sample.tif', 'content_type':'image/tiff'},\n",
    "    {'name':'data-publication-sample.kmz', 'content_type':'application/octet-stream'},\n",
    "    {'name':'data-publication-sample.properties', 'content_type':'text/plain'}\n",
    "]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Download sample files\n",
    "\n",
    "This step is necessary to have some local files to work with. In a normal processing scenario, the files to be uploaded are already available locally."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import requests\n",
    "\n",
    "def retrieve_file(url, local_file):\n",
    "    res = requests.get(url, allow_redirects=True)\n",
    "    open(local_file, 'wb').write(res.content)\n",
    "    \n",
    "    \n",
    "sample_dir = \"data-publication-sample\"\n",
    "    \n",
    "if not os.path.exists(sample_dir):\n",
    "    os.mkdir(sample_dir)\n",
    "    \n",
    "for ext in ['tif', 'xml', 'kmz']:\n",
    "    retrieve_file(\n",
    "        \"https://store.terradue.com/eo-samples/data-publication-sample/data-publication-sample.{0}\".format(ext),\n",
    "        \"{0}/data-publication-sample.{1}\".format(sample_dir, ext)\n",
    "    )\n",
    " "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Generate the metadata file\n",
    "\n",
    "We create a dictionary containing the values and then write its content to a file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import OrderedDict\n",
    "\n",
    "metadata = OrderedDict()\n",
    "metadata['Product_ID'] = 'data-publication-sample'\n",
    "metadata['Product_format'] = 'GEOTIFF'\n",
    "metadata['Product_size'] = '31624950 '\n",
    "metadata['Product_url'] = '{0}/data-publication-sample.tif'.format(store_url)\n",
    "metadata['Bounding_box'] = '40.448583 13.587901 41.234180 14.856833'\n",
    "metadata['Bounding_box_wkt'] = 'POLYGON((13.587901 40.604756,13.737746 41.234180,14.856833 41.077912,14.695978 40.448583,13.587901 40.604756))'\n",
    "metadata['License'] = 'https://creativecommons.org/licenses/by/4.0'\n",
    "metadata['User_ID'] = 'me'\n",
    "metadata['Software_version'] = 'Sample processor v2.5'\n",
    "metadata['Date'] = '2017-02-11T05:10:43.490699Z/2018-01-01T05:10:51.413166Z'\n",
    "metadata['Date_of_measurement_start'] = '2017-02-11T05:10:43.490699Z'\n",
    "metadata['Date_of_measurement_end'] = '2018-01-01T05:10:51.413166Z'\n",
    "metadata['Date_of_production'] = '2018-10-05T23:34:01Z'\n",
    "metadata['Super_master_SAR_image_ID'] = 'S1A_IW_SLC__1SDV_20160902T051142_20160902T051210_012869_014526_541D.SAFE, S1A_IW_SLC__1SDV_20160902T051207_20160902T051234_012869_014526_B4AA.SAFE'\n",
    "metadata['Master_SAR_image_ID'] = 'S1B_IW_SLC__1SDV_20170211T051043_20170211T051110_004248_0075D9_BCAE.SAFE, S1B_IW_SLC__1SDV_20170211T051108_20170211T051135_004248_0075D9_6A83.SAFE'\n",
    "metadata['Slave_SAR_image_ID'] = 'S1B_IW_SLC__1SDV_20180101T051051_20180101T051118_008973_010039_B427.SAFE, S1B_IW_SLC__1SDV_20180101T051116_20180101T051143_008973_010039_CF15.SAFE'\n",
    "metadata['Perpendicular_baseline'] = '10.1852'\n",
    "metadata['Parallel_baseline'] = '18.1659'\n",
    "metadata['Along_track_baseline'] = '0.877837'\n",
    "metadata['Reference_point'] = '14.250889 40.833728'\n",
    "metadata['Spatial_resolution'] = '37, 37'\n",
    "metadata['Sensor'] = 'S1'\n",
    "metadata['Mode'] = 'IW'\n",
    "metadata['Antenna_side'] = 'Right'\n",
    "metadata['Relative_orbit_number'] = '22'\n",
    "metadata['Orbit_direction'] = 'DESCENDING'\n",
    "metadata['Wavelenght'] = '0.055465760'\n",
    "metadata['Value_unit'] = 'cm'\n",
    "metadata['Number_of_looks_azimuth'] = '2'\n",
    "metadata['Number_of_looks_range'] = '10'\n",
    "metadata['Applied_filter'] = 'No_Filter'\n",
    "\n",
    "properties_file = open(\"data-publication-sample/data-publication-sample.properties\", \"w\")\n",
    "for key in metadata:\n",
    "    properties_file.write(\"{0}={1}\\n\".format(key, metadata[key]))\n",
    "properties_file.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. Define a function for the file upload\n",
    "\n",
    "To upload the file, we have to use the HTTP PUT method and basic HTTP authentication. This function verifies also the file integrity by comparing the hash codes of the local and remote files."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import json\n",
    "import hashlib\n",
    "import os\n",
    "\n",
    "\n",
    "def send_file(local_file, content_type, store_url, username, api_key):\n",
    "\n",
    "    # Open file to send\n",
    "    content = open(local_file, 'rb').read()\n",
    "\n",
    "    # Send file (using the HTTP PUT method)\n",
    "    res = requests.put(url=\"{0}/{1}\".format(store_url, os.path.basename(local_file)),\n",
    "                       headers={\"Content-Type\": content_type},\n",
    "                       auth=(username, api_key),\n",
    "                       data=content\n",
    "    )\n",
    "    \n",
    "    print(\"Response status code: {0}\".format(res.status_code))\n",
    "\n",
    "    assert res.status_code == 201\n",
    "\n",
    "    print('File {0} uploaded'.format(local_file))\n",
    "    \n",
    "    # Verify the content (hash check)\n",
    "    \n",
    "    local_checksum = hashlib.sha256(content).hexdigest()\n",
    "    remote_checksum = res.json()['checksums']['sha256']\n",
    "\n",
    "    print(\"- Local checksum:  {0}\".format(local_checksum))\n",
    "    print(\"- Remote checksum: {0}\".format(remote_checksum))\n",
    "\n",
    "    assert local_checksum == remote_checksum"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5. Upload the files\n",
    "\n",
    "The output of this cell shows the upload success (HTTP status code and checksum)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for file in files:\n",
    "    try:\n",
    "        send_file(\"data-publication-sample/{0}\".format(file['name']), file['content_type'], store_url, username, api_key)\n",
    "    except Exception as e:\n",
    "        print('ERROR: File {0} NOT uploaded: {1}'.format(file['name'], e))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**END**"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}