From d989392429f80e3b86cddb9294c5a7f7319bce4d Mon Sep 17 00:00:00 2001 From: "Ronaldo S.A. Batista" Date: Tue, 31 May 2022 18:42:56 -0300 Subject: [PATCH] =?UTF-8?q?:sparkles:=20Extra=C3=A7=C3=A3o=20e=20Download?= =?UTF-8?q?=20de=20Anexos?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adicionada funções principais e auxiliares para extração e parseamento dos anexos na àrvore de Issues. Caso passado uma Inspeção, os anexos são processados e baixados. Caso uma inspeção ou solicitação de inspeção, a árvore é varrida. --- docs/constants.html | 33 -------------------------- docs/info.html | 36 +++++++++++++++++++++++----- fiscaliza/_nbdev.py | 1 + fiscaliza/constants.py | 1 + fiscaliza/info.py | 53 +++++++++++++++++++++++++++++++++--------- fiscaliza/update.py | 4 ++-- nbs/constants.ipynb | 27 ++++----------------- nbs/info.ipynb | 52 ++++++++++++++++++++++++++++++++--------- nbs/update.ipynb | 4 ++-- 9 files changed, 123 insertions(+), 88 deletions(-) diff --git a/docs/constants.html b/docs/constants.html index 90bce69..e023fb8 100644 --- a/docs/constants.html +++ b/docs/constants.html @@ -46,39 +46,6 @@

IDs e Constantes - - {% endraw %} - - {% raw %} - -
-
- -
-
-
ID2FIELD.items()
-
- -
-
-
- -
-
- -
- - - -
-
dict_items([(2, 'Tipo_de_Inspecao'), (5, 'Ano'), (25, 'Fiscal_Responsavel'), (26, 'Fiscais'), (30, 'Entidade_da_Inspecao'), (31, 'UF_Municipio'), (57, 'Servicos_da_Inspecao'), (69, 'Qnt_de_emissoes_na_faixa'), (70, 'Emissoes_nao_autorizadas'), (71, 'Procedimentos'), (89, 'Classe_da_Inspecao'), (91, 'Horas_de_Preparacao'), (92, 'Horas_de_Deslocamento'), (93, 'Horas_de_Execucao'), (94, 'Horas_de_Conclusao'), (111, 'SAV'), (112, 'PCDP'), (151, 'Uso_de_PF'), (154, 'Acao_de_risco_a_vida_criada'), (156, 'Frequencia_Inicial'), (157, 'Unidade_da_Frequencia_Inicial'), (158, 'Frequencia_Final'), (159, 'Unidade_da_Frequencia_Final'), (178, 'Coordenacao'), (213, 'Agrupamento'), (280, 'Utilizou_algum_instrumento'), (422, 'Numero_Sei_do_Processo'), (450, 'Impossibilidade_acesso_online'), (463, 'AppFiscaliza'), (541, 'Gerar_Relatorio'), (544, 'Relatorio_de_Monitoramento'), (543, 'Html'), (596, 'Reservar_Instrumentos'), (597, 'Reserva_de_Instrumentos'), (598, 'Utilizou_algum_instrumento'), (717, 'Coordenadas_Geograficas'), (726, 'Qtd_Licenciadas'), (727, 'Qtd_Identificadas')])
-
- -
- -
-
-
{% endraw %} diff --git a/docs/info.html b/docs/info.html index 86da3ae..bbc07f5 100644 --- a/docs/info.html +++ b/docs/info.html @@ -81,7 +81,7 @@
-

insp2acao[source]

insp2acao(insp:str, fiscaliza:Redmine)

+

insp2acao[source]

insp2acao(insp:str, fiscaliza:Redmine)

Recebe o objeto fiscaliza e a string referente à inspeção insp e retorna um dicionário resumo da Ação atrelada à inspeção

Args: @@ -118,7 +118,7 @@

insp2acao -

issue_type[source]

issue_type(insp, fiscaliza)

+

issue_type[source]

issue_type(insp, fiscaliza)

Checa se a Issue de nº insp do Redmine é de um dos tipos válidos: Inspeção | Ação

@@ -150,7 +150,7 @@

issue_type -

utf2ascii[source]

utf2ascii(s)

+

utf2ascii[source]

utf2ascii(s)

Recebe uma string e retorna a mesma em formato ASCII

@@ -175,7 +175,7 @@

utf2ascii -

detalhar_issue[source]

detalhar_issue(issue:str, login:str=None, senha:str=None, api:str=None, fiscaliza:Redmine=None, teste:bool=True)

+

detalhar_issue[source]

detalhar_issue(issue:str, login:str=None, senha:str=None, api:str=None, fiscaliza:Redmine=None, teste:bool=True)

Recebe número da inspeção inspecao, o login e senha ou opcionalmente objeto Redmine logado fiscaliza inspecao: str - Número da Inspeção a ser relatada @@ -254,7 +254,7 @@

detalhar_issue -

download_attachments[source]

download_attachments(issue_obj:Union[str, int], savepath:Union[str, Path], login:str=None, senha:str=None, api:str=None, teste:bool=True)

+

download_attachments[source]

download_attachments(issue_obj:Union[str, int], savepath:Union[str, Path], login:str=None, senha:str=None, api:str=None, teste:bool=True)

@@ -278,7 +278,31 @@

download_attachments -

extract_attachments[source]

extract_attachments(issue_obj:Union[str, int, Issue], login:str=None, senha:str=None, api:str=None, fiscaliza:Redmine=None, teste:bool=True, filter:Iterable[str]=('relatada', 'conferida', 'aprovada'))

+

extract_attachment[source]

extract_attachment(issue_obj, filename:str='Info.json', only_last:bool=True)

+
+ + + + + + + + + + {% endraw %} + + {% raw %} + +
+ +
+
+ +
+ + +
+

extract_attachments[source]

extract_attachments(issue_obj:Union[str, int, Issue], login:str=None, senha:str=None, api:str=None, fiscaliza:Redmine=None, teste:bool=True, filter:Iterable[str]=('relatada', 'conferida', 'aprovada'), filename:str='Info.json', only_last:bool=True)

diff --git a/fiscaliza/_nbdev.py b/fiscaliza/_nbdev.py index 5f73c7a..70c155e 100644 --- a/fiscaliza/_nbdev.py +++ b/fiscaliza/_nbdev.py @@ -33,6 +33,7 @@ "utf2ascii": "info.ipynb", "detalhar_issue": "info.ipynb", "download_attachments": "info.ipynb", + "extract_attachment": "info.ipynb", "extract_attachments": "info.ipynb", "atualiza_fiscaliza": "update.ipynb", "del_attach": "update.ipynb", diff --git a/fiscaliza/constants.py b/fiscaliza/constants.py index 77a0cd5..c40b9ec 100644 --- a/fiscaliza/constants.py +++ b/fiscaliza/constants.py @@ -285,6 +285,7 @@ "Aguardando Execução": 11, "Em andamento": 13, "Relatando": 14, + "Relatada": 15, } CANCELADA = 19 diff --git a/fiscaliza/info.py b/fiscaliza/info.py index 1e47d64..c30d255 100644 --- a/fiscaliza/info.py +++ b/fiscaliza/info.py @@ -1,13 +1,14 @@ # AUTOGENERATED! DO NOT EDIT! File to edit: nbs/info.ipynb (unless otherwise specified). -__all__ = ['insp2acao', 'issue_type', 'utf2ascii', 'detalhar_issue', 'download_attachments', 'extract_attachments'] +__all__ = ['insp2acao', 'issue_type', 'utf2ascii', 'detalhar_issue', 'download_attachments', 'extract_attachment', + 'extract_attachments'] # Cell import re +import json from unidecode import unidecode from datetime import datetime, timedelta import contextlib -from pathlib import Path from typing import Union, Iterable from rich import print @@ -17,8 +18,10 @@ from redminelib.exceptions import ResourceAttrError from fastcore.foundation import L -from fastcore.xtras import is_listy, listify +from fastcore.xtras import is_listy, listify, Path from fastcore.script import Param, call_parse, bool_arg +from fastcore.parallel import parallel +from fastcore.basics import partialler from .constants import ACAO_DESCRIPTION, KWARGS, IDS, FIELDS, JSON_FIELDS @@ -186,10 +189,17 @@ def _download_attachment( url = getattr(attach, "content_url").replace("http://", "https://") setattr(attach, "content_url", url) + d = dict(attach) + filename = f"{attach.Inspecao}_{attach.id}_{attach.filename}" attach.download( savepath=savepath, - filename=f"{attach.id}_{attach.filename}", + filename=filename, ) + d['author'] = d['author']['name'] + filename = Path(savepath / filename) + js = filename.read_json() + js.update(d) + json.dump(js, filename.open('w')) @call_parse # Workaround the bug in recursion call in extract_attachments when using as a console script @@ -215,8 +225,19 @@ def download_attachments( attachments = extract_attachments(issue_obj, login, senha, api, teste=teste) savepath = Path(f"{savepath}/{issue_obj}") savepath.mkdir(exist_ok=True, parents=True) - attachments.map(lambda attach: _download_attachment(attach, savepath)) - + func = partialler(_download_attachment, savepath=savepath) + parallel(func, attachments, threadpool=True, n_workers=min(len(attachments), 16)) + +def extract_attachment(issue_obj, filename: str = 'Info.json', only_last: bool = True): + if attach := L(getattr(issue_obj, "attachments", [])): + if filename: + attach = attach.filter(lambda x: x.filename == filename) + if only_last: + ids = attach.attrgot('id') + last_id = ids.argwhere(lambda x: x == max(ids)) + attach = attach[last_id] + attach.setattrs('Inspecao', issue_obj.id) + return attach def extract_attachments( issue_obj: Union[str, int, Issue], @@ -226,32 +247,42 @@ def extract_attachments( fiscaliza: Redmine = None, teste: bool = True, filter: Iterable[str] = ('relatada', 'conferida', 'aprovada'), + filename: str = 'Info.json', + only_last: bool = True, ): # sourcery skip: remove-unnecessary-cast attachments = L() fiscaliza = valida_fiscaliza(login, senha, api, fiscaliza, teste) if not isinstance(issue_obj, Issue) and isinstance(issue_obj, (str, int)): issue_obj = fiscaliza.issue.get(issue_obj, include=["relations", "attachments"]) + prefix = issue_obj.id + if ( getattr(getattr(issue_obj, "tracker", None), "name", None) == "Atividade de Inspeção" ) and str(getattr(issue_obj, "status", '')).lower() in filter: - attachments.extend(getattr(issue_obj, "attachments", [])) + attachments.extend(extract_attachment(issue_obj, filename, only_last)) return attachments + if relations := getattr(issue_obj, "relations", []): relations = ( L(relations).map(dict).filter(lambda r: r["issue_to_id"] == issue_obj.id) ) - for r in tqdm(relations, total=len(relations)): + + if getattr(getattr(issue_obj, "tracker", None), "name", None) == 'Solicitação de Inspeção': + relations = tqdm(relations, total=len(relations)) + + for r in relations: issue_obj = fiscaliza.issue.get( r["issue_id"], include=["relations", "attachments"] ) - if ( tipo := getattr(getattr(issue_obj, "tracker", None), "name", None) - ) == "Ação de Inspeção": + ) in {"Solicitação de Inspeção", "Ação de Inspeção"}: attachments.extend(extract_attachments(issue_obj, fiscaliza=fiscaliza)) elif tipo == "Atividade de Inspeção" and str(getattr(issue_obj, "status", '')).lower() in filter: - attachments.extend(getattr(issue_obj, "attachments", [])) + attach = extract_attachment(issue_obj, filename, only_last) + attach.setattrs('Acao', prefix) + attachments.extend(attach) return attachments \ No newline at end of file diff --git a/fiscaliza/update.py b/fiscaliza/update.py index 1402205..583ff87 100644 --- a/fiscaliza/update.py +++ b/fiscaliza/update.py @@ -90,7 +90,7 @@ def excluir_relatorio( } temp["Html"] = {"id": FIELD2ID["Html"], "value": ""} atualiza_fiscaliza( - inspecao, temp, fiscaliza, status=status_atual["status"].name + inspecao, temp, fiscaliza, status=status_atual["status"] ) status_atual = detalhar_issue(inspecao, fiscaliza=fiscaliza, teste=teste) if status_atual.get("Relatorio_de_Monitoramento"): @@ -177,7 +177,7 @@ def relatar_inspecao( with console.status("Resgatando Situação Atual da Inspeção...", spinner="pong"): status_atual = detalhar_issue(inspecao, fiscaliza=fiscaliza, teste=teste) - atual = getattr(status_atual["status"], "name") + atual = status_atual["status"] console.print(f":white_check_mark: [cyan]Estado Atual: [bold green]{atual}") diff --git a/nbs/constants.ipynb b/nbs/constants.ipynb index 3e527b5..82dce29 100644 --- a/nbs/constants.ipynb +++ b/nbs/constants.ipynb @@ -318,27 +318,6 @@ "}" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "2ad67bfd", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "dict_items([(2, 'Tipo_de_Inspecao'), (5, 'Ano'), (25, 'Fiscal_Responsavel'), (26, 'Fiscais'), (30, 'Entidade_da_Inspecao'), (31, 'UF_Municipio'), (57, 'Servicos_da_Inspecao'), (69, 'Qnt_de_emissoes_na_faixa'), (70, 'Emissoes_nao_autorizadas'), (71, 'Procedimentos'), (89, 'Classe_da_Inspecao'), (91, 'Horas_de_Preparacao'), (92, 'Horas_de_Deslocamento'), (93, 'Horas_de_Execucao'), (94, 'Horas_de_Conclusao'), (111, 'SAV'), (112, 'PCDP'), (151, 'Uso_de_PF'), (154, 'Acao_de_risco_a_vida_criada'), (156, 'Frequencia_Inicial'), (157, 'Unidade_da_Frequencia_Inicial'), (158, 'Frequencia_Final'), (159, 'Unidade_da_Frequencia_Final'), (178, 'Coordenacao'), (213, 'Agrupamento'), (280, 'Utilizou_algum_instrumento'), (422, 'Numero_Sei_do_Processo'), (450, 'Impossibilidade_acesso_online'), (463, 'AppFiscaliza'), (541, 'Gerar_Relatorio'), (544, 'Relatorio_de_Monitoramento'), (543, 'Html'), (596, 'Reservar_Instrumentos'), (597, 'Reserva_de_Instrumentos'), (598, 'Utilizou_algum_instrumento'), (717, 'Coordenadas_Geograficas'), (726, 'Qtd_Licenciadas'), (727, 'Qtd_Identificadas')])" - ] - }, - "execution_count": null, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ID2FIELD.items()" - ] - }, { "cell_type": "markdown", "id": "fd139cae", @@ -360,6 +339,7 @@ " \"Aguardando Execução\": 11,\n", " \"Em andamento\": 13,\n", " \"Relatando\": 14,\n", + " \"Relatada\": 15,\n", "}\n", "\n", "CANCELADA = 19\n", @@ -6168,7 +6148,8 @@ "Converted constants.ipynb.\n", "Converted format.ipynb.\n", "Converted index.ipynb.\n", - "Converted redmine.ipynb.\n", + "Converted info.ipynb.\n", + "Converted update.ipynb.\n", "Converted validation.ipynb.\n" ] } @@ -6182,7 +6163,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python [conda env:fiscaliza]", + "display_name": "Python 3.9.12 ('teste')", "language": "python", "name": "python3" } diff --git a/nbs/info.ipynb b/nbs/info.ipynb index 5d79513..a4b9240 100644 --- a/nbs/info.ipynb +++ b/nbs/info.ipynb @@ -45,10 +45,10 @@ "source": [ "#export\n", "import re\n", + "import json\n", "from unidecode import unidecode\n", "from datetime import datetime, timedelta\n", "import contextlib\n", - "from pathlib import Path\n", "from typing import Union, Iterable\n", "\n", "from rich import print\n", @@ -58,8 +58,10 @@ "from redminelib.exceptions import ResourceAttrError\n", "\n", "from fastcore.foundation import L\n", - "from fastcore.xtras import is_listy, listify\n", + "from fastcore.xtras import is_listy, listify, Path\n", "from fastcore.script import Param, call_parse, bool_arg\n", + "from fastcore.parallel import parallel\n", + "from fastcore.basics import partialler\n", "\n", "\n", "from fiscaliza.constants import ACAO_DESCRIPTION, KWARGS, IDS, FIELDS, JSON_FIELDS\n", @@ -248,10 +250,17 @@ "\n", " url = getattr(attach, \"content_url\").replace(\"http://\", \"https://\")\n", " setattr(attach, \"content_url\", url)\n", + " d = dict(attach)\n", + " filename = f\"{attach.Inspecao}_{attach.id}_{attach.filename}\"\n", " attach.download(\n", " savepath=savepath,\n", - " filename=f\"{attach.id}_{attach.filename}\",\n", + " filename=filename,\n", " )\n", + " d['author'] = d['author']['name']\n", + " filename = Path(savepath / filename)\n", + " js = filename.read_json()\n", + " js.update(d)\n", + " json.dump(js, filename.open('w'))\n", "\n", "\n", "@call_parse # Workaround the bug in recursion call in extract_attachments when using as a console script\n", @@ -277,9 +286,20 @@ " attachments = extract_attachments(issue_obj, login, senha, api, teste=teste)\n", " savepath = Path(f\"{savepath}/{issue_obj}\")\n", " savepath.mkdir(exist_ok=True, parents=True)\n", - " attachments.map(lambda attach: _download_attachment(attach, savepath))\n", - "\n", - "\n", + " func = partialler(_download_attachment, savepath=savepath)\n", + " parallel(func, attachments, threadpool=True, n_workers=min(len(attachments), 16))\n", + "\n", + "def extract_attachment(issue_obj, filename: str = 'Info.json', only_last: bool = True):\n", + " if attach := L(getattr(issue_obj, \"attachments\", [])):\n", + " if filename:\n", + " attach = attach.filter(lambda x: x.filename == filename)\n", + " if only_last:\n", + " ids = attach.attrgot('id')\n", + " last_id = ids.argwhere(lambda x: x == max(ids))\n", + " attach = attach[last_id]\n", + " attach.setattrs('Inspecao', issue_obj.id)\n", + " return attach\n", + " \n", "def extract_attachments(\n", " issue_obj: Union[str, int, Issue],\n", " login: str = None,\n", @@ -288,34 +308,44 @@ " fiscaliza: Redmine = None,\n", " teste: bool = True,\n", " filter: Iterable[str] = ('relatada', 'conferida', 'aprovada'),\n", + " filename: str = 'Info.json',\n", + " only_last: bool = True,\n", "): # sourcery skip: remove-unnecessary-cast\n", " attachments = L()\n", " fiscaliza = valida_fiscaliza(login, senha, api, fiscaliza, teste)\n", " if not isinstance(issue_obj, Issue) and isinstance(issue_obj, (str, int)):\n", " issue_obj = fiscaliza.issue.get(issue_obj, include=[\"relations\", \"attachments\"])\n", "\n", + " prefix = issue_obj.id\n", + "\n", " if (\n", " getattr(getattr(issue_obj, \"tracker\", None), \"name\", None)\n", " == \"Atividade de Inspeção\"\n", " ) and str(getattr(issue_obj, \"status\", '')).lower() in filter:\n", - " attachments.extend(getattr(issue_obj, \"attachments\", []))\n", + " attachments.extend(extract_attachment(issue_obj, filename, only_last))\n", " return attachments\n", "\n", + "\n", " if relations := getattr(issue_obj, \"relations\", []):\n", " relations = (\n", " L(relations).map(dict).filter(lambda r: r[\"issue_to_id\"] == issue_obj.id)\n", " )\n", - " for r in tqdm(relations, total=len(relations)):\n", + " \n", + " if getattr(getattr(issue_obj, \"tracker\", None), \"name\", None) == 'Solicitação de Inspeção':\n", + " relations = tqdm(relations, total=len(relations))\n", + "\n", + " for r in relations:\n", " issue_obj = fiscaliza.issue.get(\n", " r[\"issue_id\"], include=[\"relations\", \"attachments\"]\n", " )\n", - "\n", " if (\n", " tipo := getattr(getattr(issue_obj, \"tracker\", None), \"name\", None)\n", - " ) == \"Ação de Inspeção\":\n", + " ) in {\"Solicitação de Inspeção\", \"Ação de Inspeção\"}:\n", " attachments.extend(extract_attachments(issue_obj, fiscaliza=fiscaliza))\n", " elif tipo == \"Atividade de Inspeção\" and str(getattr(issue_obj, \"status\", '')).lower() in filter:\n", - " attachments.extend(getattr(issue_obj, \"attachments\", []))\n", + " attach = extract_attachment(issue_obj, filename, only_last)\n", + " attach.setattrs('Acao', prefix)\n", + " attachments.extend(attach)\n", " return attachments" ] }, diff --git a/nbs/update.ipynb b/nbs/update.ipynb index 3fe4deb..9003038 100644 --- a/nbs/update.ipynb +++ b/nbs/update.ipynb @@ -159,7 +159,7 @@ " }\n", " temp[\"Html\"] = {\"id\": FIELD2ID[\"Html\"], \"value\": \"\"}\n", " atualiza_fiscaliza(\n", - " inspecao, temp, fiscaliza, status=status_atual[\"status\"].name\n", + " inspecao, temp, fiscaliza, status=status_atual[\"status\"]\n", " )\n", " status_atual = detalhar_issue(inspecao, fiscaliza=fiscaliza, teste=teste)\n", " if status_atual.get(\"Relatorio_de_Monitoramento\"):\n", @@ -262,7 +262,7 @@ " with console.status(\"Resgatando Situação Atual da Inspeção...\", spinner=\"pong\"):\n", " status_atual = detalhar_issue(inspecao, fiscaliza=fiscaliza, teste=teste)\n", "\n", - " atual = getattr(status_atual[\"status\"], \"name\")\n", + " atual = status_atual[\"status\"]\n", "\n", " console.print(f\":white_check_mark: [cyan]Estado Atual: [bold green]{atual}\")\n", "\n",