{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "31713587-6ba2-4a51-81eb-f3d63206ed64",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import time\n",
    "import yaml\n",
    "import openai\n",
    "import tiktoken\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1f63d107-67f1-4e1d-9bd1-e16ccdad32e1",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "with open(os.path.expanduser(\"~/secrets.yaml\")) as f:\n",
    "    secrets = yaml.load(f, Loader=yaml.CLoader)\n",
    "\n",
    "openai.api_key = secrets['OPENAI_API_KEY']\n",
    "enc = tiktoken.get_encoding(\"cl100k_base\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "557efb6b-ef7c-45dc-bf0f-74b00d0e0afd",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "doc_df = pd.read_pickle('corpus/documents.pkl')\n",
    "corpus_df = pd.read_csv('corpus/GenQ-5000.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1aaf9e33-91f1-4ac9-9a49-4e81a318f004",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def gen_q(doc):\n",
    "    message = f'''Please generate a question for a given context:\n",
    "- the question should be answerable by that context\n",
    "- the question must be English regardless of what language the context is\n",
    "\n",
    "context: Similar to [the TiUP playground component](/tiup/tiup-playground.md) used for a local test deployment, the TiUP cluster component quickly deploys TiDB for production environment. Compared with playground, the cluster component provides more powerful production cluster management features, including upgrading, scaling, and even operation and auditing.\n",
    "question: What is TiUP Cluster used to?\n",
    "\n",
    "context: Clinic Server is a cloud service deployed in the cloud. By providing diagnostic services in the SaaS model, the Clinic Server can not only receive uploaded diagnostic data but also work as an online diagnostic environment to store data, view data, and provide cluster diagnostic reports.\n",
    "question: Can clinic server receive uploaded diagnostic data?\n",
    "\n",
    "context: 所有通过 scale-in 下线的服务组件，tiup 都需要进行探活验证的，主要对集群稳定性的保护（未知且非预期状态节点下线）。Scale-in 是一个 graceful shutdown + scale in 的缩容方法。如果是对应组件已经是 crash 和关机状态，且为当前情况下预期的情况，需要使用 --force 强制下线操作。\n",
    "question: Why tiup check if the component is alive when scale-in?\n",
    "\n",
    "context: {doc}\n",
    "question: '''\n",
    "\n",
    "    completion = openai.ChatCompletion.create(\n",
    "        model=\"gpt-3.5-turbo\",\n",
    "        messages=[{\n",
    "            \"role\": \"user\",\n",
    "            \"content\": message,\n",
    "        }],\n",
    "        temperature=1.0,\n",
    "        max_tokens=1000,\n",
    "    )\n",
    "    return completion.choices[0].message.content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9c1a3575-68ba-4a8b-83e8-76dc713448a2",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def has_query(context):\n",
    "    return corpus_df.context.eq(context.strip()).any()\n",
    "\n",
    "def add_query(context, query):\n",
    "    global corpus_df\n",
    "    corpus_df = pd.concat([corpus_df, pd.DataFrame.from_records({\n",
    "        'context': [context],\n",
    "        'query': [query],\n",
    "    })])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ffec9cfb-e147-4ac6-9e6d-48bd191636bd",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "for context in doc_df.content.sample(150):\n",
    "    if has_query(context):\n",
    "        continue\n",
    "    if len(enc.encode(context)) > 512:\n",
    "        continue\n",
    "    q = gen_q(context)\n",
    "    add_query(context.strip(), q)\n",
    "    print(q)\n",
    "    time.sleep(0.5)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}