import re import logging import requests from io import BytesIO from itertools import chain from datetime import datetime from django.core.management import BaseCommand from pingcap_sso.discourse.models import ( DiscourseTopic, DiscoursePost ) from pingcap_sso.discourse.consts import QA_FEEDBACK_CATEGORIES, BLOG_CATEGORIES from pingcap_sso.lib.upload import upload_file_to_cdn from pingcap_sso.models import TiDBContributor from pingcap_sso.utils import time as ptime logger = logging.getLogger(__name__) class Command(BaseCommand): topic_re = re.compile(r'^https://asktug\.com/t/[\w|-]+/(?P\d+)\/{0,1}') def add_arguments(self, parser): parser.add_argument( '--cms_url', required=True, ) parser.add_argument( '--cms_email', required=True, ) parser.add_argument( '--cms_password', required=True, ) parser.add_argument( '--github_token', required=True, ) def handle(self, *args, **options): cms_url = options['cms_url'] cms_email = options['cms_email'] cms_password = options['cms_password'] github_token = options['github_token'] cms_headers = self._get_cms_auth_headers(cms_url, cms_email, cms_password) self._update_cms_object( cms_url, cms_headers, 'tidbio-github-info', self.github_info(github_token)) self._update_cms_object( cms_url, cms_headers, 'tidbio-asktug-blogs', self.asktug_blogs(cms_url, cms_headers)) self._update_cms_object( cms_url, cms_headers, 'tidbio-asktug-qa-topics', self.asktug_qa_topics(cms_url, cms_headers)) # Bilibili 风控太严,无法获取到数据 # self._update_cms_object( # cms_url, cms_headers, 'tidbio-blibli-recent-videos', self.bilibili_recent_videos()) def github_info(self, github_token): resp = requests.get( 'https://api.github.com/repos/pingcap/tidb', headers={ 'Content-Type': 'application/json; charset=utf-8', 'Authorization': 'token {}'.format(github_token) }, timeout=3 ) if resp.status_code != 200: raise Exception('Get pingcap/tidb detail failed, status_code: {}'.format( resp.status_code)) result = resp.json() star_num = result['stargazers_count'] # resp = requests.get( # 'https://api-devstats.tidb.io/projects/all/', # timeout=10 # ).json() # pr_num = resp['stats']['pull_requests'] # contributor_num = resp['stats']['contributors'] pr_num = 96383 contributor_num = TiDBContributor.objects.all().count() topic_num = DiscourseTopic.objects.filter( deleted_at__isnull=True, archetype='regular' ).count() post_num = DiscoursePost.objects.filter( deleted_at__isnull=True ).count() return { "star_num": star_num, "pr_num": pr_num, "topic_num": topic_num, "post_num": post_num, "contributor_num": contributor_num } def bilibili_recent_videos(self): resp = requests.get( 'https://api.bilibili.com/x/space/arc/search?mid=86485707&pn=1&ps=6&order=pubdate&index=1', headers={ 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.55 Safari/537.36', 'authority': 'api.bilibili.com', 'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9' }, timeout=10 ) if resp.status_code != 200: raise Exception('Get bilibili recent videos failed, status_code: {}'.format( resp.status_code)) videos = resp.json()['data']['list']['vlist'][:6] data = [] for v in videos: data.append({ 'title': v['title'], 'cover_image': self._download_image_upload_to_cdn(v['pic']), 'created': datetime.fromtimestamp(v['created']).strftime('%Y-%m-%d'), 'length': v['length'], 'link': 'https://www.bilibili.com/video/{}'.format(v['bvid']), 'play_count': v['play'] }) return data def asktug_qa_topics(self, cms_url, cms_headers): max_top_num = 3 total_num = 6 discourse_topics = DiscourseTopic.objects.filter( deleted_at__isnull=True, visible=True, category_id__in=QA_FEEDBACK_CATEGORIES ) top_topics = requests.get( '{}/tidbio-homepage-qa-topics'.format(cms_url), headers=cms_headers, timeout=3).json() top_topic_ids = [] data = [] for topic in top_topics: tr = self.topic_re.match(topic['topic_url']) if tr: top_topic_ids.append(tr.group('topic_id')) top_topic_qs = discourse_topics.filter(id__in=top_topic_ids)[:max_top_num] other_topic_qs = discourse_topics.exclude( id__in=top_topic_ids ).order_by('-created_at')[:(total_num - top_topic_qs.count())] data = [] for topic in chain(top_topic_qs, other_topic_qs): sso_user = topic.user.get_sso_user() categories = [] if topic.category: if topic.category.parent_category: categories.append({ 'id': topic.category.parent_category_id, 'name': topic.category.parent_category.name, 'color': topic.category.parent_category.color }) categories.append({ 'id': topic.category_id, 'name': topic.category.name, 'color': topic.category.color }) data.append({ 'title': topic.title, 'link': 'https://asktug.com/t/topic/{}'.format(topic.id), 'post_number': topic.posts_count - 1, 'updated_at': ptime.local_time_format(topic.updated_at), 'created_at': ptime.local_time_format(topic.created_at), 'creator': { 'username': topic.user.username, 'profile_link': 'https://asktug.com/u/{}/summary'.format(topic.user.username_lower), 'avatar_url': sso_user.avatar_url }, 'categories': categories }) return data def asktug_blogs(self, cms_url, cms_headers): max_top_num = 6 total_num = 6 discourse_topics = DiscourseTopic.objects.filter( deleted_at__isnull=True, visible=True ) top_topics_list = requests.get( '{}/tidbio-homepage-blogs'.format(cms_url), headers=cms_headers, timeout=3).json() top_topics = {} for topic in top_topics_list: tr = self.topic_re.match(topic['topic_url']) if tr: top_topics[int(tr.group('topic_id'))] = topic['title'] top_topic_ids = list(top_topics.keys()) top_topic_qs = discourse_topics.filter( id__in=top_topic_ids)[:max_top_num] other_topic_qs = discourse_topics.filter( category_id__in=BLOG_CATEGORIES ).exclude( id__in=top_topic_ids ).order_by('-created_at')[:(total_num - top_topic_qs.count())] data = [] for topic in chain(top_topic_qs, other_topic_qs): sso_user = topic.user.get_sso_user() categories = [] topic_title = top_topics.get(topic.id) or topic.title if topic.category: if topic.category.parent_category: categories.append({ 'id': topic.category.parent_category_id, 'name': topic.category.parent_category.name, 'color': topic.category.parent_category.color }) categories.append({ 'id': topic.category_id, 'name': topic.category.name, 'color': topic.category.color }) data.append({ 'title': topic_title, 'link': 'https://asktug.com/t/topic/{}'.format(topic.id), 'post_number': topic.posts_count - 1, 'updated_at': ptime.local_time_format(topic.updated_at), 'created_at': ptime.local_time_format(topic.created_at), 'creator': { 'username': topic.user.username, 'profile_link': 'https://asktug.com/u/{}/summary'.format(topic.user.username_lower), 'avatar_url': sso_user.avatar_url }, 'categories': categories }) return data def _get_cms_auth_headers(self, cms_url, cms_email, cms_password): resp = requests.post( '{}/admin/login'.format(cms_url), data={ 'email': cms_email, 'password': cms_password }, timeout=10 ).json() return { 'Authorization': 'Bearer {}'.format(resp['data']['token']) } def _update_cms_object(self, cms_url, cms_headers, object_id, data): resp = requests.put( '{}/{}'.format(cms_url, object_id), headers=cms_headers, json={ 'data': data } ) if resp.status_code != 200: raise Exception('Update CMS.{} failed, status_code: {}, content: {}'.format( object_id, resp.status_code, resp.content)) def _download_image_upload_to_cdn(self, image_url): response = requests.get(image_url, stream=True) if response.status_code != 200: logger.error('Dowload image failed, url: {}, status_code: {}'.format( image_url, response.status_code)) return file = BytesIO() file.name = image_url for block in response.iter_content(1024): if not block: break file.write(block) file.seek(0) path, cdn_url = upload_file_to_cdn(file) return cdn_url