{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import requests\n",
    "import xml.etree.ElementTree as dom\n",
    "from xml.dom import minidom"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# caom2 TAP endpoints at cadc and stsci\n",
    "cadc='http://www.cadc-ccda.hia-iha.nrc-cnrc.gc.ca/tap/tables'\n",
    "stsci='http://vao.stsci.edu/CAOMTAP/tapservice.aspx/tables'\n",
    "\n",
    "# VO-DML model urls\n",
    "ivoa_url=\"http://volute.g-vo.org/svn/trunk/projects/dm/vo-dml/models/ivoa/vo-dml/IVOA-v1.0.vo-dml.xml\"\n",
    "caom2_url=\"https://github.com/opencadc/caom2/blob/master/caom2-dm/src/main/resources/CAOM-2.3-vodml.xml\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "def writePretty(elem, file):\n",
    "    \"\"\"\n",
    "    Return a pretty-printed XML string for the Element.\n",
    "    \"\"\"\n",
    "    rough_string = dom.tostring(elem, 'utf-8')\n",
    "    reparsed = minidom.parseString(rough_string)\n",
    "    p= reparsed.toprettyxml(indent=\"  \")\n",
    "    with open(file, \"w\") as text_file:\n",
    "        text_file.write(p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\n",
    "def datatype(_dataType):\n",
    "    '''\n",
    "    Translate from TAP type to votable datatype and xtype and to ivoa primitive type. \n",
    "    '''\n",
    "    t=_dataType.text.lower()\n",
    "    if 'size' in _dataType.attrib:\n",
    "        arrsize=_dataType.attrib['size']\n",
    "    elif 'arraysize' in _dataType.attrib:\n",
    "        arrsize=_dataType.attrib['arraysize']\n",
    "    else:\n",
    "        arrsize=1\n",
    "    if t=='integer':\n",
    "        dt ='int'\n",
    "    if t=='long':\n",
    "        dt ='long'\n",
    "    elif t=='real':\n",
    "        dt ='float'\n",
    "    elif t=='double':\n",
    "        dt ='double'\n",
    "    elif t=='timestamp':\n",
    "        dt='char'\n",
    "        arrsize=28\n",
    "    else:\n",
    "        dt='char'\n",
    "    return dt,str(arrsize),'adql:'+_dataType.text.upper(),ivoatype(_dataType)\n",
    "\n",
    "def ivoatype(_dataType):\n",
    "    '''\n",
    "    Translate from TAP type to ivoa primitive type. \n",
    "    '''\n",
    "    t=_dataType.text.lower()\n",
    "    if t=='integer':\n",
    "        dt ='ivoa:integer'\n",
    "    if t=='long':\n",
    "        dt ='ivoa:integer'\n",
    "    elif t=='real':\n",
    "        dt ='ivoa:real'\n",
    "    elif t=='double':\n",
    "        dt ='ivoa:real'\n",
    "    elif t=='timestamp':\n",
    "        dt='ivoa:datetime'\n",
    "    else:\n",
    "        dt='ivoa:string'\n",
    "    return dt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def newmodel(name,uri):\n",
    "    '''\n",
    "    Create a VOTABLE:VO-DML:MODEL element\n",
    "    '''\n",
    "    model=dom.Element(\"MODEL\")\n",
    "    _name=dom.Element(\"NAME\")\n",
    "    model.append(_name)\n",
    "    _name.text=name\n",
    "    _uri=dom.Element(\"URI\")\n",
    "    model.append(_uri)\n",
    "    _uri.text=uri\n",
    "    return model\n",
    "\n",
    "def newattribute(_column,ns,vd,fieldid_prefix):\n",
    "    '''\n",
    "    create a VOTABLE:VODML:ATTRIBUTE and a VOTABLE:FIELD for a VOSI:table:column\n",
    "    '''\n",
    "    dt,arrsize,xt,ivoatype=datatype(_column.find(vd+'dataType'))\n",
    "    name=_column.find(ns+'name').text\n",
    "    field=dom.Element(\"FIELD\")\n",
    "    field.set(\"name\",name)\n",
    "    field.set(\"arraysize\",arrsize)\n",
    "    field.set(\"datatype\",dt)\n",
    "    field.set(\"xtype\",xt)\n",
    "\n",
    "    if _column.find(ns+'unit') is not None:\n",
    "        field.set('unit',_column.find(ns+'unit').text)\n",
    "    if _column.find(ns+'ucd') is not None:\n",
    "        field.set('ucd',_column.find(ns+'ucd').text)\n",
    "\n",
    "    description=dom.Element(\"DESCRIPTION\")\n",
    "    description.text=_column.find(ns+'description').text\n",
    "    field.append(description)\n",
    "    ID=fieldid_prefix+'.'+field.get(\"name\")\n",
    "    field.set(\"ID\",ID)\n",
    "\n",
    "    attribute=dom.Element(\"ATTRIBUTE\")\n",
    "    column=dom.Element(\"COLUMN\")\n",
    "    attribute.append(column)\n",
    "    if _column.find(ns+'utype') is not None:\n",
    "        field.set('utype',_column.find(ns+'utype').text)\n",
    "        attribute.set(\"dmrole\",_column.find(ns+'utype').text)\n",
    "    else:\n",
    "        attribute.set(\"dmrole\",\"\")\n",
    "    column.set(\"dmtype\",ivoatype)\n",
    "    column.set(\"ref\",ID)\n",
    "    return attribute, field"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def toVotable(url,schema=None,ns='{http://www.ivoa.net/xml/VOSITables/v1.0}',\n",
    "              vd='{http://www.ivoa.net/xml/VODataService/v1.1}'):\n",
    "    file = requests.get(url)\n",
    "    data = file.content.decode()\n",
    "#    print(data)\n",
    "    root=dom.fromstring(data)\n",
    "\n",
    "    if schema is None:\n",
    "        prefix=''\n",
    "    else:\n",
    "        prefix=schema+'.'\n",
    "\n",
    "#    votable = dom.Element(\"{http://www.ivoa.net/xml/VOTable/v1.4_vodml}VOTABLE\")\n",
    "    votable = dom.Element(\"VOTABLE\")\n",
    "    votable.set(\"xmlns\",\"http://www.ivoa.net/xml/VOTable/v1.4_vodml\")\n",
    "    vodml=dom.Element(\"VODML\")\n",
    "    votable.append(vodml)\n",
    "\n",
    "    resource=dom.Element(\"RESOURCE\")\n",
    "    votable.append(resource)\n",
    "    \n",
    "    vodml.append(newmodel(\"ivoa\",ivoa_url))\n",
    "    vodml.append(newmodel(\"caom2\",caom2_url))\n",
    "\n",
    "    \n",
    "    for _table in root.iter(ns+'table'):\n",
    "       \n",
    "        tname=_table.find(ns+'name').text\n",
    "        if not(tname.startswith(prefix)):\n",
    "            continue\n",
    "        print(tname)\n",
    "        \n",
    "        templates=dom.Element(\"TEMPLATES\")\n",
    "        templates.set(\"tableref\",tname)\n",
    "        vodml.append(templates)\n",
    "        instance=dom.Element(\"INSTANCE\")\n",
    "        templates.append(instance)\n",
    "        instance.set(\"dmtype\",\"\")\n",
    "        \n",
    "        table=dom.Element(\"TABLE\")\n",
    "        resource.append(table)\n",
    "        table.set(\"ID\",tname)\n",
    "        table.set(\"name\",tname)\n",
    "        for _column in _table.findall(ns+'column'):\n",
    "            attribute,field=newattribute(_column,ns,vd,tname)\n",
    "            instance.append(attribute)\n",
    "            table.append(field)\n",
    "         \n",
    "    return votable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "caom2.Observation\n",
      "caom2.Plane\n",
      "caom2.Artifact\n",
      "caom2.Part\n",
      "caom2.Chunk\n",
      "caom2.EnumField\n",
      "caom2.ObsCoreEnumField\n",
      "caom2.distinct_proposal_id\n",
      "caom2.distinct_proposal_pi\n",
      "caom2.distinct_proposal_title\n",
      "caom2.SIAv1\n"
     ]
    }
   ],
   "source": [
    "votable=toVotable(cadc,'caom2',ns='',vd='')\n",
    "writePretty(votable,'cadc_caom_tap.vodml.xml')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dbo.CaomObservation\n",
      "dbo.CaomPlane\n",
      "dbo.CaomArtifact\n",
      "dbo.CaomPart\n",
      "dbo.CaomChunk\n",
      "dbo.CaomEnumFields\n",
      "dbo.CaomMembers\n",
      "dbo.CaomProductDescription\n"
     ]
    }
   ],
   "source": [
    "votable=toVotable(stsci,'dbo')\n",
    "writePretty(votable,'stsci_caom_tap.vodml.xml')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}