qasper_generate.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "qas = json.load(open(\"./data/qasper_reranked_t53b.json\",'r'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "data = json.load(open(\"data/qasper/qasper-test-v0.3.json\",\"r\"))\n",
    "questions = {}\n",
    "for k in data:\n",
    "    for qa in data[k]['qas']:\n",
    "        questions[qa['question_id']] = qa\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## use this to select a stratified subset of the datase.\n",
    "extractives = []\n",
    "abstractives = []\n",
    "none_ = []\n",
    "yes_no = []\n",
    "stratfy_by = []\n",
    "for item in qas:\n",
    "    annotation_info = questions[item['question_id']][\"answers\"][0]\n",
    "    # annotation_info = item['answers'][0]\n",
    "    answer_info = annotation_info[\"answer\"]\n",
    "    if answer_info[\"unanswerable\"]:\n",
    "        stratfy_by.append(0)\n",
    "        none_.append(item)\n",
    "    else:\n",
    "        if answer_info[\"extractive_spans\"]:\n",
    "            extractives.append(item)\n",
    "            stratfy_by.append(1)\n",
    "        elif answer_info[\"free_form_answer\"]:\n",
    "            # answer = answer_info[\"free_form_answer\"]\n",
    "            abstractives.append(item)\n",
    "            stratfy_by.append(2)\n",
    "            # answer_type = \"abstractive\"\n",
    "        elif answer_info[\"yes_no\"]:\n",
    "            yes_no.append(item)\n",
    "            stratfy_by.append(3)\n",
    "        elif answer_info[\"yes_no\"] is not None:\n",
    "            yes_no.append(item)\n",
    "            stratfy_by.append(3)\n",
    "\n",
    "len(extractives), len(abstractives),len(none_), len(yes_no)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X_train, test, y_train, y_test = train_test_split(qas, stratfy_by, test_size=0.15, random_state=42, stratify=stratfy_by)\n",
    "len(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sentence_transformers import SentenceTransformer, util\n",
    "model = SentenceTransformer('sentence-transformers/msmarco-bert-base-dot-v5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = json.load(open(\"data/qasper/explained_train.json\",'r'))\n",
    "train_embeddings = model.encode([a['question'] for a in train])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import openai\n",
    "\n",
    "from transformers import GPT2Tokenizer\n",
    "tokenizer = GPT2Tokenizer.from_pretrained('gpt2')\n",
    "\n",
    "openai.api_key = os.getenv(\"OPENAI_API_KEY\")\n",
    "\n",
    "def generate(prompt,max_tokens=1000, temperature=0):\n",
    "    # tokens = tokenizer.tokenize(prompt)\n",
    "    response = openai.Completion.create(\n",
    "        engine=\"text-davinci-002\",\n",
    "        prompt=prompt,\n",
    "        temperature=temperature,\n",
    "        max_tokens=256,\n",
    "        top_p=0,\n",
    "        frequency_penalty=0,\n",
    "        presence_penalty=0\n",
    "    )\n",
    "\n",
    "    return response[\"choices\"][0]['text']\n",
    "# prompt=\"Classify the documents as relevant or irrelevant to answer the question. Explain the decisions. Then, answer the question using the relevant documents.\\n\\nExample 1:\\n\\n[Document 1]: In this section we describe a number of experiments targeted to compare the performance of popular named entity recognition algorithms on our data. We trained and evaluated Stanford NER, spaCy 2.0, and a recurrent model similar to BIBREF13 , BIBREF14 that uses bidirectional LSTM cells for character-based feature extraction and CRF, described in Guillaume Genthial's Sequence Tagging with Tensorflow blog post BIBREF15 .\\n[Document 2]: Stanford NER is conditional random fields (CRF) classifier based on lexical and contextual features such as the current word, character-level n-grams of up to length 6 at its beginning and the end, previous and next words, word shape and sequence features BIBREF16 .\\n[Document 3]: spaCy 2.0 uses a CNN-based transition system for named entity recognition. For each token, a Bloom embedding is calculated based on its lowercase form, prefix, suffix and shape, then using residual CNNs, a contextual representation of that token is extracted that potentially draws information from up to 4 tokens from each side BIBREF17 . Each update of the transition system's configuration is a classification task that uses the contextual representation of the top token on the stack, preceding and succeeding tokens, first two tokens of the buffer, and their leftmost, second leftmost, rightmost, second rightmost children. The valid transition with the highest score is applied to the system. This approach reportedly performs within 1% of the current state-of-the-art for English . In our experiments, we tried out 50-, 100-, 200- and 300-dimensional pre-trained GloVe embeddings. Due to time constraints, we did not tune the rest of hyperparameters and used their default values.\\n[Document 4]: In order to evaluate the models trained on generated data, we manually annotated a named entities dataset comprising 53453 tokens and 2566 sentences selected from over 250 news texts from ilur.am. This dataset is comparable in size with the test sets of other languages (Table TABREF10 ). Included sentences are from political, sports, local and world news (Figures FIGREF8 , FIGREF9 ), covering the period between August 2012 and July 2018. The dataset provides annotations for 3 popular named entity classes: people (PER), organizations (ORG), and locations (LOC), and is released in CoNLL03 format with IOB tagging scheme. Tokens and sentences were segmented according to the UD standards for the Armenian language BIBREF11 .\\n[Document 5]: The main model that we focused on was the recurrent model with a CRF top layer, and the above-mentioned methods served mostly as baselines. The distinctive feature of this approach is the way contextual word embeddings are formed. For each token separately, to capture its word shape features, character-based representation is extracted using a bidirectional LSTM BIBREF18 . This representation gets concatenated with a distributional word vector such as GloVe, forming an intermediate word embedding. Using another bidirectional LSTM cell on these intermediate word embeddings, the contextual representation of tokens is obtained (Figure FIGREF17 ). Finally, a CRF layer labels the sequence of these contextual representations. In our experiments, we used Guillaume Genthial's implementation of the algorithm. We set the size of character-based biLSTM to 100 and the size of second biLSTM network to 300\\n\\nQuestion: what ner models were evaluated?\\n\\nRelevant documents:\\n[Document 1] is relevant to answer the question because it describes a number of experiments that compare the performance of popular named entity recognition algorithms.\\n[Document 2] is relevant to answer the question because it describes the Stanford NER algorithm, which is one of the models that was evaluated.\\n[Document 3] is relevant to answer the question because it describes the spaCy 2.0 algorithm, which is one of the models that was evaluated.\\n[Document 5] is relevant to answer the question because it describes the recurrent model with a CRF top layer, which is one of the models that was evaluated.\\n\\nIrrelevant documents:\\n[Document 4] is not relevant to answer the question because it describes a named entities dataset that was used to evaluate the models, but it does not describe any of the models themselves.\\n\\nAnswer: Stanford NER algorithm, the spaCy 2.0 algorithm, recurrent model with a CRF top layer.\\n\\nExample 2:\\n\\n[Document 1]: Social media are becoming an increasingly important source of information about the public mood regarding issues such as elections, Brexit, stock market, etc. In this paper we focus on sentiment classification of Twitter data. Construction of sentiment classifiers is a standard text mining task, but here we address the question of how to properly evaluate them as there is no settled way to do so. Sentiment classes are ordered and unbalanced, and Twitter produces a stream of time-ordered data. The problem we address concerns the procedures used to obtain reliable estimates of performance measures, and whether the temporal ordering of the training and test data matters. We collected a large set of 1.5 million tweets in 13 European languages. We created 138 sentiment models and out-of-sample datasets, which are used as a gold standard for evaluations. The corresponding 138 in-sample datasets are used to empirically compare six different estimation procedures: three variants of cross-validation, and three variants of sequential validation (where test set always follows the training set). We find no significant difference between the best cross-validation and sequential validation. However, we observe that all cross-validation variants tend to overestimate the performance, while the sequential methods tend to underestimate it. Standard cross-validation with random selection of examples is significantly worse than the blocked cross-validation, and should not be used to evaluate classifiers in time-ordered data scenarios.\\n[Document 2]: The complexity of Twitter data raises some challenges on how to perform such estimations, as, to the best of our knowledge, there is currently no settled approach to this. Sentiment classes are typically ordered and unbalanced, and the data itself is time-ordered. Taking these properties into account is important for the selection of appropriate estimation procedures.\\n[Document 3]: We collected a large corpus of nearly 1.5 million Twitter posts written in 13 European languages. This is, to the best of our knowledge, by far the largest set of sentiment labeled tweets publicly available. We engaged native speakers to label the tweets based on the sentiment expressed in them. The sentiment label has three possible values: negative, neutral or positive. It turned out that the human annotators perceived the values as ordered. The quality of annotations varies though, and is estimated from the self- and inter-annotator agreements. All the details about the datasets, the annotator agreements, and the ordering of sentiment values are in our previous study BIBREF22 . The sentiment distribution and quality of individual language datasets is in Table TABREF2 . The tweets in the datasets are ordered by tweet ids, which corresponds to ordering by the time of posting.\\n\\nQuestion: In what way are sentiment classes ordered?\\n\\nRelevant documents:\"\n",
    "\n",
    "# generate(prompt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "\n",
    "kshot = 3\n",
    "RANDOM = True\n",
    "STATIC = False\n",
    "\n",
    "if STATIC:\n",
    "    static_sample = random.sample(train,k=kshot)\n",
    "def generate_explanation_answer(question, documents):\n",
    "    prompt=\"For each example, use the documents to create an \\\"Answer\\\" and an \\\"Explanation\\\" to the \\\"Question\\\". Answer \\\"Unanswerable\\\" when not enough information is provided in the documents. Pay attention to answer only \\\"yes\\\" or \\\"no\\\" in boolean questions.\\n\\n\"\n",
    "    if RANDOM:\n",
    "        hits = random.sample(train,k=kshot)\n",
    "    elif STATIC:\n",
    "        hits = static_sample\n",
    "    else:\n",
    "        item_embeddings = model.encode(question)\n",
    "\n",
    "        all_top = util.dot_score(item_embeddings, train_embeddings)[0].topk(kshot)\n",
    "\n",
    "        hits = np.array(train)[all_top.indices].tolist()\n",
    "\n",
    "        hits.reverse()\n",
    "\n",
    "    for i, hit in enumerate(hits):\n",
    "        prompt += \"Example {0}:\\n\\n\".format(i+1)\n",
    "\n",
    "        for j,d in enumerate(hit['evidence']):\n",
    "            prompt += \"[Document {0}]: {1}\\n\\n\".format(j+1, d)    \n",
    "\n",
    "        prompt += \"Question: {0}\\n\\nExplanation: {1}\\n\\nAnswer: {2}\\n\\n\".format(hit['question'],hit['explanation'],hit['answer'])\n",
    "\n",
    "    prompt += \"Example {0}:\\n\\n\".format(i+2)\n",
    "\n",
    "\n",
    "    for k, document in enumerate(documents):\n",
    "        prompt += \"[Document {0}]: {1}\\n\\n\".format(k+1, document['title'])\n",
    "    \n",
    "    prompt+= \"Question: {0}\\n\\nExplanation:\".format(question)\n",
    "\n",
    "    tokens = tokenizer.tokenize(prompt)\n",
    "    if len(tokens) >= 3800:\n",
    "        return generate_explanation_answer(question, documents)\n",
    "    \n",
    "    res = generate(prompt)\n",
    "\n",
    "    if \"answer:\" not in res.lower():\n",
    "        prompt = prompt+res+\"\\n\\nAnswer:\"\n",
    "        res = res+\"\\n\\nAnswer:\"+ generate(prompt)\n",
    "    \n",
    "    explanation = res.lower().split(\"answer:\")[0]\n",
    "    \n",
    "    answer = res.lower().split(\"answer:\")[1]\n",
    "\n",
    "    return explanation, answer, prompt, res\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "from tqdm import tqdm\n",
    "import numpy as np\n",
    "from datetime import date\n",
    "import random \n",
    "\n",
    "RANDOM = True\n",
    "\n",
    "kshot = 3\n",
    "docs = 3\n",
    "\n",
    "regex = r\"\\[document \\d+\\]\"\n",
    "\n",
    "\n",
    "f = open(\"./results/qasper.jsonl\",'w')\n",
    "# results =\n",
    "logs = []\n",
    "tokens= []\n",
    "for item in tqdm(test):\n",
    "\n",
    "    documents = item['documents'][:docs]\n",
    "    explanation, answer, prompt, res = generate_explanation_answer(item['question'], documents)\n",
    "\n",
    "    matches = re.finditer(regex, explanation, re.MULTILINE)\n",
    "    relevant_documents = []\n",
    "    for match in matches:\n",
    "        nums = re.findall(r'\\b\\d+\\b', match.group())\n",
    "        if len(nums) > 0:\n",
    "            try:\n",
    "                relevant_documents.append(documents[int(nums[0])-1]['title'])\n",
    "            except:\n",
    "                pass\n",
    "\n",
    "    f.write(json.dumps({\n",
    "        'question_id': item['question_id'], \n",
    "        'predicted_answer': answer.rstrip(), \n",
    "        'predicted_evidence': relevant_documents\n",
    "    })+\"\\n\")\n",
    "\n",
    "    logs.append({\n",
    "        'question_id': item['question_id'],\n",
    "        'predicted_answer': answer.rstrip(),\n",
    "        'predicted_evidence': relevant_documents,\n",
    "        'prompt': prompt,\n",
    "        \"res\": res\n",
    "    })  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!python3 qasper_evaluator.py --predictions results/qasper.jsonl --gold data/qasper-test-v0.3.json --text_evidence_only"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6"
  },
  "kernelspec": {
   "display_name": "Python 3.8.10 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}