{ "cells": [ { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# this notebook is used to generate corpus/community.csv\n", "# to run this notebook, you should clone these repos into repos directory:\n", "# - git@github.com:pingcap/book.tidb.net.git\n", "\n", "import os\n", "import re\n", "import yaml\n", "import requests\n", "import slugify\n", "import pandas as pd\n", "from urllib.parse import urljoin" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "with open(os.path.expanduser(\"~/secrets.yaml\")) as f:\n", " secrets = yaml.load(f, Loader=yaml.CLoader)\n", "\n", "CN_CMS_URL_PREFIX = 'https://cms-cn.pingcap.com'\n", "CN_CMS_EMAIL = 'pingcap.www.ci@gmail.com'\n", "CN_CMS_PASSWORD = secrets['CN_CMS_PASSWORD']\n", "\n", "COMMUNITY_BLOG_URL_PREFIX = 'https://tidb.net/blog'\n", "COMMUNITY_BLOG_TOKEN = secrets['COMMUNITY_BLOG_TOKEN']" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "def download_community_blogs():\n", " headers = {\n", " 'Authorization': f'Bearer {COMMUNITY_BLOG_TOKEN}'\n", " }\n", " _page = 1\n", " _size = 50\n", " has_data = True\n", " blogs = []\n", " while has_data:\n", " resp = requests.get(\n", " f'{COMMUNITY_BLOG_URL_PREFIX}/api/posts/latest/view/markdown'\\\n", " '?page={}&size={}'.format(_page, _size),\n", " headers=headers,\n", " timeout=10\n", " ).json()\n", " for c in resp['content']:\n", " if not c['markdownText']:\n", " continue\n", " blogs.append({\n", " 'title': c['title'],\n", " 'slug': c['slug'],\n", " 'link': f'{COMMUNITY_BLOG_URL_PREFIX}/{c[\"slug\"]}',\n", " 'content': c['markdownText']\n", " })\n", " has_data = bool(resp['content'])\n", " _page += 1\n", " return blogs" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "def download_official_blogs():\n", " resp = requests.post(\n", " f'{CN_CMS_URL_PREFIX}/admin/login',\n", " data={\n", " 'email': CN_CMS_EMAIL,\n", " 'password': CN_CMS_PASSWORD\n", " },\n", " timeout=10,\n", " ).json()\n", " headers = {\n", " 'Authorization': 'Bearer {}'.format(resp['data']['token'])\n", " }\n", " _start = 0\n", " _limit = 100\n", " has_data = True\n", " blogs = []\n", " while has_data:\n", " _url = f'{CN_CMS_URL_PREFIX}/blogs'\\\n", " '?_publicationState=live&_start={}&_limit={}'.format(_start, _limit)\n", " _blogs = requests.get(\n", " _url,\n", " headers=headers,\n", " timeout=10\n", " ).json()\n", " has_data = bool(_blogs)\n", " _start += _limit\n", " for _b in _blogs:\n", " blogs.append({\n", " 'id': _b['id'],\n", " 'title': _b['title'],\n", " 'content': _b['body'],\n", " 'link': f'https://pingcap.com/zh/blog/{_b[\"customUrl\"]}'\n", " })\n", " return blogs\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "NAME_PATTERN = re.compile(r'^\\d+-(.*)')\n", "\n", "def format_v6_action_file_name(file_name):\n", " names = file_name.split('/')\n", " names[-1] = names[-1].replace('.md', '')\n", " new_names = [NAME_PATTERN.sub(r'\\1', n) for n in names]\n", " return '/'.join(new_names)\n", "\n", "\n", "def load_community_tidb_book(doc_path, url_prefix):\n", " # url: https://tidb.net/book\n", " # repo: https://github.com/pingcap/book.tidb.net\n", " xs = []\n", " for root, _, files in os.walk(doc_path):\n", " for f in files:\n", " if f.endswith('.md') and f not in ['index.md', 'README.md', 'catalog.md']:\n", " relative_path = os.path.relpath(os.path.join(root, f), doc_path)\n", " url = urljoin(url_prefix, format_v6_action_file_name(relative_path))\n", " with open(os.path.join(root, f), 'r') as fp:\n", " # skip the first 4 lines, which are the document metadata\n", " content = ''.join(fp.readlines()[4:])\n", " xs.append({\n", " 'title': f,\n", " 'link': url,\n", " 'content': content\n", " })\n", " return xs" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "def split_docs(original_docs):\n", " xs = []\n", " for od in original_docs:\n", " docs = []\n", " for doc in od['content'].split('\\n'):\n", " if not doc.strip():\n", " continue\n", " docs.append(doc)\n", " for i, doc in enumerate(docs):\n", " xs.append({\n", " 'title': od['title'].replace('\\r', ''),\n", " 'link': od['link'],\n", " 'idx': i,\n", " 'content': doc.replace('\\r', '')\n", " })\n", " return xs" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "community_blogs = download_community_blogs()\n", "official_blogs = download_official_blogs()\n", "tidb_v6_action = load_community_tidb_book('repos/book.tidb.net/website/book-rush', 'https://tidb.net/book/book-rush/')\n", "db_selection = load_community_tidb_book('repos/book.tidb.net/website/db-selection', 'https://tidb.net/book/db-selection/')" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "community_docs = split_docs(tidb_v6_action + db_selection + community_blogs + official_blogs)\n", "\n", "community_df = pd.DataFrame.from_records(community_docs)\n", "community_df['source'] = 'community'\n", "community_df.to_csv('corpus/community.csv', index=False)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "community_df" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.10" } }, "nbformat": 4, "nbformat_minor": 4 }