more nat language stuff

llama_test.ipynb

@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 13,
    "metadata": {},
    "outputs": [
     {
@@ -11,7 +11,7 @@
        "True"
       ]
      },
-     "execution_count": 1,
+     "execution_count": 13,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -32,7 +32,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 14,
    "metadata": {},
    "outputs": [
     {
@@ -47,7 +47,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "fc0f4312aa7c4009a912c66dd1443763",
+       "model_id": "ca1fb983d9884b91a3c0feed1e207d0e",
        "version_major": 2,
        "version_minor": 0
       },
@@ -83,7 +83,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 15,
    "metadata": {},
    "outputs": [
     {
@@ -132,7 +132,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 16,
    "metadata": {},
    "outputs": [
     {
@@ -158,9 +158,17 @@
     "! tar xvjf WikiSQL/data.tar.bz2"
    ]
   },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Figure out what the actual data set has in it."
+   ]
+  },
   {
    "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": 17,
    "metadata": {},
    "outputs": [
     {
@@ -213,6 +221,135 @@
     "qs = replace_cols(str(q),cm)\n",
     "print(qs)"
    ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Ok, their query class deals poorly with stringifying the constraints.  Let's mock up natural language prompt and SQL response."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 56,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "Respond to the following data request with a SQL query.\n",
+      "Q: Table 2-16763320-1 has columns Tournament (text),Surface (text),Week (text),Winner (text),Finalist (text),Semifinalists (text).  Which finalist has Semifinalists of andre agassi (1) lleyton hewitt (14)?\n",
+      "A: SELECT  Finalist FROM 2-16763320-1 WHERE Semifinalists = 'andre agassi (1) lleyton hewitt (14)'\n",
+      "\n",
+      "Respond to the following data request with a SQL query.\n",
+      "Q: Table 1-27755784-10 has columns Game (real),Date (text),Team (text),Score (text),High points (text),High rebounds (text),High assists (text),Location Attendance (text),Record (text).  What is the highest game number?\n",
+      "A: SELECT MAX Game FROM 1-27755784-10\n",
+      "\n",
+      "Respond to the following data request with a SQL query.\n",
+      "Q: Table 2-17231086-5 has columns Place (text),Player (text),Country (text),Score (text),To par (text).  What place is the United States in that has a score of 68-73-68=209?\n",
+      "A: SELECT  Place FROM 2-17231086-5 WHERE Country = 'united states' AND Score = '68-73-68=209'\n",
+      "\n",
+      "Respond to the following data request with a SQL query.\n",
+      "Q: Table 2-1302729-1 has columns Season (real),Overall (text),Slalom (text),Giant Slalom (text),Super G (text),Downhill (text),Combined (text).  What is the combined of 2 overalls and 5 slaloms?\n",
+      "A: SELECT  Combined FROM 2-1302729-1 WHERE Overall = '2' AND Slalom = '5'\n",
+      "\n",
+      "Respond to the following data request with a SQL query.\n",
+      "Q: Table 2-15295737-56 has columns Nation (text),Skip (text),Third (text),Second (text),Lead (text),Alternate (text).  Who is the alternate for the team for which Monika Wagner is the third?\n",
+      "A: SELECT  Alternate FROM 2-15295737-56 WHERE Third = 'monika wagner'\n"
+     ]
+    }
+   ],
+   "source": [
+    "import random\n",
+    "\n",
+    "# defined by WikiSQL\n",
+    "\n",
+    "agg_ops = ['', 'MAX', 'MIN', 'COUNT', 'SUM', 'AVG']\n",
+    "cond_ops = ['=', '>', '<', 'OP']\n",
+    "syms = ['SELECT', 'WHERE', 'AND', 'COL', 'TABLE', 'CAPTION', 'PAGE', 'SECTION', 'OP', 'COND', 'QUESTION', 'AGG', 'AGGOPS', 'CONDOPS']\n",
+    "\n",
+    "def fix_repr(d,cols,types,tid):\n",
+    "    sel_index=d['sel'] \n",
+    "    agg_index=d['agg']\n",
+    "    conditions=d['conds']\n",
+    "    col = cols[sel_index]\n",
+    "    rep = 'SELECT {agg} {sel} FROM {tid}'.format(\n",
+    "            agg=agg_ops[agg_index],\n",
+    "            sel=col,\n",
+    "            tid=tid\n",
+    "            )\n",
+    "    if conditions:\n",
+    "        cs = []\n",
+    "        for i, o, v in conditions:\n",
+    "            #print(i,cols)\n",
+    "            nm = cols[i]\n",
+    "            op = cond_ops[o]\n",
+    "            \n",
+    "            if types[i] in ['text']:\n",
+    "                val = f\"\\'{v}\\'\"\n",
+    "            else:\n",
+    "                val = v\n",
+    "            cs.append(f'{nm} {op} {val}')\n",
+    "        #print(cs)\n",
+    "\n",
+    "        rep +=  ' WHERE ' + ' AND '.join(cs)\n",
+    "    \n",
+    "    return rep\n",
+    "\n",
+    "tbl_cols = {}\n",
+    "tbl_types = {}\n",
+    "tbl_str = {}\n",
+    "\n",
+    "prefix = 'Respond to the following data request with a SQL query.\\n'\n",
+    "\n",
+    "def tbl_def_to_string(id, header, types):\n",
+    "    ht = [f'{header[i]} ({types[i]})' for i in range(len(header))]\n",
+    "    s = f'Q: Table {id} has columns ' + ','.join(ht) + '.  '\n",
+    "    return s\n",
+    "\n",
+    "with open('data/train.tables.jsonl') as f:\n",
+    "    for line in f:\n",
+    "        js = json.loads(line)\n",
+    "        id = js['id']\n",
+    "        hdr = js['header']\n",
+    "        ts = js['types']\n",
+    "        tbl_str[id] = tbl_def_to_string(id,hdr,ts)\n",
+    "        tbl_cols[id] = hdr\n",
+    "        tbl_types[id] = ts\n",
+    "\n",
+    "\n",
+    "nl_q = []\n",
+    "sql_a = []\n",
+    "\n",
+    "with open('data/train.jsonl') as f:\n",
+    "    for line in f:\n",
+    "        js = json.loads(line)\n",
+    "        id = js['table_id']\n",
+    "        s = tbl_str[id]\n",
+    "        qst = js['question']\n",
+    "        nl = prefix + s + qst\n",
+    "        nl_q.append(nl)\n",
+    "\n",
+    "        sql = js['sql']\n",
+    "        a = fix_repr(sql,tbl_cols[id],tbl_types[id],id)\n",
+    "        a = 'A: ' + a\n",
+    "        sql_a.append(a)\n",
+    "\n",
+    "\n",
+    "M = len(nl_q)\n",
+    "\n",
+    "\n",
+    "for i in range(5):\n",
+    "    j = random.randint(0,M-1)\n",
+    "    print()\n",
+    "    print(nl_q[j])\n",
+    "    print(sql_a[j])    \n",
+    "        \n",
+    "   "
+   ]
   }
  ],
  "metadata": {