Skip to content

Commit

Permalink
Finish generate report with time range
Browse files Browse the repository at this point in the history
  • Loading branch information
liu599 committed Sep 9, 2024
1 parent 8c4449b commit 3abbba8
Show file tree
Hide file tree
Showing 7 changed files with 176 additions and 78 deletions.
27 changes: 27 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,33 @@
- `python main.py`
- 启动后访问localhost:5010即可

## 生成报告API

```
curl -X 'POST' \
'http://localhost:5010/fischl_api/v1/agent_chat' \
-H 'accept: application/json' \
-H 'Content-Type: application/json' \
-d '{
"repo": "repo:langchain-ai/langchain",
"days": 3
}'
```

返回

```
{
"code": 20000,
"data": {
"repo": "repo:langchain-ai/langchain"
},
"message": "操作成功"
}
```

## 大模型API实验

```
Expand Down
2 changes: 1 addition & 1 deletion api/conf/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,4 @@
from framework.inori_utils.utils import ConfigLoader

api_conf = ConfigLoader()
api_conf.read('api.conf', '请拷贝api.default.conf')
api_conf.read('api.conf', '请拷贝api.default.conf为api.conf并按照需求配置')
79 changes: 12 additions & 67 deletions api/plugins/agent/GithubDataFetcher.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,7 @@
# @Description :
import markdown
import imgkit
from typing import Dict, Optional

import os
from typing import Optional

from api.plugins.agent.github_client import GitHubClient
from framework.inori_plugin_manager.base_plugin import BasePlugin
Expand All @@ -21,16 +19,18 @@ def __init__(self) -> None:
self.query_params: Optional[str] = None
self.token: Optional[str] = None
self.output_dir = None
self.github_client = None
self.github_client: GitHubClient = None
self.days = 0

def configure(self, token: str, output_dir=None) -> None:
def configure(self, token: str, output_dir=None, days=0) -> None:
"""
配置函数,用于设置GitHub用户token和其他必要的配置
:param token: GitHub个人访问令牌
"""
self.token = token
self.days = days
self.headers["Authorization"] = f"token {self.token}"
self.github_client = GitHubClient(api_url=self.api_url, headers=self.headers)
self.github_client: GitHubClient = GitHubClient(api_url=self.api_url, headers=self.headers)
if output_dir:
self.output_dir = output_dir

Expand All @@ -51,79 +51,24 @@ def run(self, *args, **kwargs) -> Optional[str]:
if not self.query_params:
raise ValueError("查询参数未设置,请使用 set_query_params() 方法设置参数")

report_file_path = self.generate_report(self.query_params, self.output_dir)
report_file_path = self.generate_report(self.query_params, self.output_dir, self.days)
return report_file_path
# 将报告转换为图片
# self.convert_markdown_to_image(report, "report.png")
# print("报告图片已生成并保存为report.png")

def generate_report(self, query: str, output_dir: Optional[str] = None) -> str:
def generate_report(self, query: str, output_dir: Optional[str] = None, days: Optional[int] = 0) -> str:
"""
根据GitHub项目数据生成markdown格式的报告,并保存为文件
:param repo_data: GitHub仓库的数据
:param output_dir: 保存报告的目录路径,如果未指定,则保存在当前目录
:return: 返回markdown格式的报告
"""
from datetime import datetime
print(query)
repos = self.github_client.search_repositories(query)
repo_data = None
if "items" in repos and len(repos["items"]) > 0:
repo_data = repos["items"][0]
if repo_data is None:
raise Exception("Github API Failure")
repo_name = repo_data.get("name", "UnknownRepo")
owner = repo_data.get("owner", {}).get("login", "N/A")
stars = repo_data.get("stargazers_count", "N/A")
forks = repo_data.get("forks_count", "N/A")
description = repo_data.get("description", "N/A")
url = repo_data.get("html_url", "N/A")
updated_at = repo_data.get("updated_at", "N/A")
print(repo_name)
print(owner)
# 获取 issues 列表
issues = self.github_client.get_issues(repo_name, owner)
# 获取 commits 列表
commits = self.github_client.get_commits(repo_name, owner)
# 获取 pull requests 列表
pull_requests = self.github_client.get_pull_requests(repo_name, owner)

report = f"""
# GitHub Repository Report: {repo_name}
**Owner:** {owner}
**Stars:** {stars}
**Forks:** {forks}
**Description:** {description}
**URL:** [Link to repository]({url})
**Last Updated:** {updated_at}
## Issues
{issues}
## Commits
{commits}
## Pull Requests
{pull_requests}
Report generated on {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}
"""

# 生成Markdown文件
timestamp = datetime.now().strftime('%Y%m%d%H%M%S')
filename = f"{repo_name}-{timestamp}.md"
print(report)
# 如果指定了路径,则将文件保存在该路径下
if output_dir is not None:
output_path = f"{output_dir}/{filename}"
print(days)
if days != 0:
report = self.github_client.export_progress_by_date_range(query, days, output_dir=output_dir)
else:
output_path = filename
print(output_path)
with open(output_path, 'w', encoding='utf-8') as file:
file.write(report)
print(f"Markdown报告已保存为 {output_path}")

report = self.github_client.export_daily_progress(query, output_dir=output_dir)
return report

def convert_markdown_to_image(self, markdown_text: str, output_file: str) -> None:
Expand Down
136 changes: 129 additions & 7 deletions api/plugins/agent/github_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,11 @@
# @Time : 2024/9/8
# @Author : liuboyuan
# @Description :
import textwrap

import requests
from typing import Dict, List
from datetime import datetime, date, timedelta


class GitHubClient:
Expand All @@ -24,19 +27,30 @@ def search_repositories(self, query: str) -> Dict:
else:
raise Exception(f"Unable to fetch repositories: {response.status_code} {response.text}")


def get_issues(self, repo_name: str, owner: str) -> str:
def get_issues(self, repo_name: str, owner: str, since=None, until=None) -> str:
url = f"{self.api_url}/repos/{owner}/{repo_name}/issues"
response = requests.get(url, headers=self.headers)
params = {
'state': 'closed', # 仅获取已关闭的问题
'since': since,
'until': until
}
response = requests.get(url, headers=self.headers, params=params)
response.raise_for_status()
if response.status_code == 200:
issues = response.json()
return "\n".join(f"- {issue.get('title')} (#{issue.get('number')})" for issue in issues)
else:
return "Unable to fetch issues."

def get_commits(self, repo_name: str, owner: str) -> str:
def get_commits(self, repo_name: str, owner: str, since=None, until=None) -> str:
url = f"{self.api_url}/repos/{owner}/{repo_name}/commits"
response = requests.get(url, headers=self.headers)
params = {}
if since:
params['since'] = since # 如果指定了开始日期,添加到参数中
if until:
params['until'] = until # 如果指定了结束日期,添加到参数中
response = requests.get(url, headers=self.headers, params=params)
response.raise_for_status()
if response.status_code == 200:
commits = response.json()
return "\n".join(
Expand All @@ -45,11 +59,119 @@ def get_commits(self, repo_name: str, owner: str) -> str:
else:
return "Unable to fetch commits."

def get_pull_requests(self, repo_name: str, owner: str) -> str:
def get_pull_requests(self, repo_name: str, owner: str, since=None, until=None) -> str:
url = f"{self.api_url}/repos/{owner}/{repo_name}/pulls"
response = requests.get(url, headers=self.headers)
params = {
'state': 'closed', # 仅获取已合并的拉取请求
'since': since,
'until': until
}
response = requests.get(url, headers=self.headers, params=params)
response.raise_for_status()
if response.status_code == 200:
pulls = response.json()
return "\n".join(f"- {pull.get('title')} (#{pull.get('number')})" for pull in pulls)
else:
return "Unable to fetch pull requests."

def fetch_updates(self, query: str, since=None, until=None) -> Dict:
# 获取指定仓库的更新,可以指定开始和结束日期
repos = self.search_repositories(query)
repo_data = None
if "items" in repos and len(repos["items"]) > 0:
repo_data = repos["items"][0]
if repo_data is None:
raise Exception("Github API Failure")
repo_name = repo_data.get("name", "UnknownRepo")
owner = repo_data.get("owner", {}).get("login", "N/A")
updates = {
'info': {
"owner": owner,
"repo": repo_name,
"stars": repo_data.get("stargazers_count", "NA"),
"forks": repo_data.get("forks_count", "NA"),
"description": repo_data.get("description", "NA"),
"url": repo_data.get("html_url", "NA"),
"updated_at": repo_data.get("updated_at", "NA"),
"from": since if since is not None else None,
"to": until if until is not None else None,
"days": 0,
},
'commits': self.get_commits(repo_name, owner, since, until), # 获取提交记录
'issues': self.get_issues(repo_name, owner, since, until), # 获取问题
'pull_requests': self.get_pull_requests(repo_name, owner, since, until) # 获取拉取请求
}
return updates

def format_report(self, updates: Dict) -> str:
if updates['info']['from'] is None:
tr = "No special range"
tr_type = "Custom"
elif updates['info']['to'] is None:
tr = f"{updates['info']['from']}"
tr_type = "Daily Report"
else:
tr = f"{updates['info']['from']} - {updates['info']['to']}"
tr_type = f"{updates['info']['days']} days report"
report = f"""\
# GitHub Repository Report
- **Repo Name:** {updates["info"]["repo"]}
- **Type:** {tr_type}
- **Time Range:** {tr}
- **Owner:** {updates["info"]["owner"]}
- **Stars:** {updates["info"]["stars"]}
- **Forks:** {updates["info"]["forks"]}
- **Description:** {updates["info"]["description"]}
- **URL:** [Link to repository]({updates["info"]["url"]})
- **Last Updated:** {updates["info"]["updated_at"]}
## Issues
{updates["issues"]}
## Commits
{updates["commits"]}
## Pull Requests
{updates["pull_requests"]}
------
Created by Github Agent
Report generated on {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\
"""
return textwrap.dedent(report)

def export_report(self, repo_name, report, output_dir=None) -> str:
# 生成Markdown文件
timestamp = datetime.now().strftime('%Y%m%d%H%M%S')
filename = f"{repo_name}-{timestamp}.md"
# 如果指定了路径,则将文件保存在该路径下
if output_dir is not None:
output_path = f"{output_dir}/{filename}"
else:
output_path = filename
with open(output_path, 'w', encoding='utf-8') as file:
file.write(report)

print(f"Markdown报告已保存为 {output_path}")
return output_path

def export_daily_progress(self, query, output_dir=None) -> str:
today = datetime.now().date().isoformat() # 获取今天的日期
# today = date.today()
# since = today - timedelta(days=1)
# since = since.isoformat()
updates = self.fetch_updates(query, since=today) # 获取今天的更新数据
repo_name = updates["info"]["repo"]
report = self.format_report(updates)
return self.export_report(repo_name, report, output_dir)

def export_progress_by_date_range(self, query, days, output_dir=None) -> str:
today = date.today() # 获取当前日期
since = today - timedelta(days=days) # 计算开始日期
updates = self.fetch_updates(query, since=since.isoformat(), until=today.isoformat()) # 获取指定日期范围内的更新
repo_name = updates["info"]["repo"]
updates["info"]["days"] = days
report = self.format_report(updates)
return self.export_report(repo_name, report, output_dir)


if __name__ == "__main__":
today = datetime.now().date().isoformat()
print(today)
Binary file modified api/view/base/__pycache__/__init__.cpython-311.pyc
Binary file not shown.
Binary file modified api/view/base/__pycache__/view.cpython-311.pyc
Binary file not shown.
10 changes: 7 additions & 3 deletions api/view/trial_sentinel.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
# @Time : 2024/9/8
# @Author : liuboyuan
# @Description :
import os

from api.view.base.view import BaseView
from api.constants.status_code import Codes
from api_entry import rest_api_description as api
Expand Down Expand Up @@ -30,6 +32,7 @@ def get(self):

agent_payload = api.model('AgentPayload', {
'repo': fields.String(description='repo name', example='repo:langchain-ai/langchain', required=True),
'days': fields.Integer(description='days', example=5),
})

@agent_api_namespace.doc(body=agent_payload)
Expand All @@ -42,18 +45,19 @@ def post(self):
"""
params = self.request.json
repo = params['repo']
days = params['days']
local_folder = self.api_config.get('data', 'local_folder')
token = self.api_config.get('github', 'token')
pm = PluginManager("./api/plugins/agent")
print(pm.get_plugin_list())
plugin = pm.get_plugin("api.plugins.agent.githubdatafetcher")
plugin.configure(token=token, output_dir=local_folder)
plugin.configure(token=token, output_dir=local_folder, days=int(days))
plugin.set_query_params(repo)
plugin.run()
file_path = plugin.run()
return self.response_raw(
code=Codes.SUCCESS.code,
msg=Codes.SUCCESS.desc,
data={
"repo": repo,
"file_path": f"http://localhost:5010/static/{os.path.basename(file_path)}",
}
)

0 comments on commit 3abbba8

Please sign in to comment.