From 4315f34c9d3aad89303ce171761dd3aae43aece8 Mon Sep 17 00:00:00 2001 From: haonan-li Date: Tue, 14 May 2024 01:08:00 +0400 Subject: [PATCH 01/13] map to original doc --- .gitignore | 2 + factcheck/__init__.py | 35 +++++++---- factcheck/__main__.py | 10 ++- factcheck/core/ClaimVerify.py | 29 ++++----- factcheck/core/Decompose.py | 52 ++++++++++++++++ factcheck/core/Retriever/serper_retriever.py | 52 ++++++++-------- factcheck/utils/prompt/chatgpt_prompt.py | 64 +++++++++++--------- 7 files changed, 165 insertions(+), 79 deletions(-) diff --git a/.gitignore b/.gitignore index f94d284..0e1ebdc 100644 --- a/.gitignore +++ b/.gitignore @@ -169,6 +169,8 @@ log/ # Mac *.DS_Store +# lark key and utils +factcheck/utils/lark.py # openai_key_file factcheck/utils/openai_key.py factcheck/config/secret_dict.py diff --git a/factcheck/__init__.py b/factcheck/__init__.py index fece7ec..80627ba 100644 --- a/factcheck/__init__.py +++ b/factcheck/__init__.py @@ -19,7 +19,7 @@ class FactCheck: def __init__( self, - default_model: str = "gpt-4-0125-preview", + default_model: str = "gpt-4-turbo", client: str = None, prompt: str = "chatgpt_prompt", retriever: str = "serper", @@ -27,7 +27,7 @@ def __init__( checkworthy_model: str = None, query_generator_model: str = None, evidence_retrieval_model: str = None, - claim_verify_model: str = None, + claim_verify_model: str = "gpt-3.5-turbo", api_config: dict = None, num_seed_retries: int = 3, ): @@ -76,8 +76,11 @@ def check_response(self, response: str): st_time = time.time() # step 1 claims = self.decomposer.getclaims(doc=response, num_retries=self.num_seed_retries) + claim2doc = self.decomposer.restore_claims(doc=response, claims=claims, num_retries=self.num_seed_retries) for i, claim in enumerate(claims): logger.info(f"== response claims {i}: {claim}") + for claim, origin in claim2doc.items(): + logger.info(f"== restore claims --- {claim} --- {origin}") # step 2 ( @@ -100,6 +103,7 @@ def check_response(self, response: str): "step_info": { "0_response": response, "1_decompose": claims, + "1_restore": claim2doc, "2_checkworthy": checkworthy_claims, "2_checkworthy_pairwise": pairwise_checkworthy, "3_query_generator": {}, @@ -129,6 +133,8 @@ def check_response(self, response: str): # step 5 claim_verify_dict = self.claimverify.verify_claims(claims_evidences_dict=claim_evidence_dict) + for k, v in claim_verify_dict.items(): + logger.info(f"== Claim: {k} --- Verify: {v}") step5_time = time.time() logger.info( f"== State: Done! \n Total time: {step5_time-st_time:.2f}s. (create claims:{step123_time-st_time:.2f}s ||| retrieve:{step4_time-step123_time:.2f}s ||| verify:{step5_time-step4_time:.2f}s)" @@ -147,18 +153,21 @@ def check_response(self, response: str): return api_data_dict def _post_process(self, api_data_dict, claim_verify_dict: dict): - label_list = list() - api_claim_data_list = list() - for claim in api_data_dict["step_info"]["2_checkworthy"]: + api_claim_data_list = [] + for claim, origin in api_data_dict["step_info"]["1_restore"].items(): api_claim_data = {} - claim_detail = claim_verify_dict.get(claim, {}) - curr_claim_label = claim_detail.get("factuality", False) - label_list.append(curr_claim_label) api_claim_data["claim"] = claim - api_claim_data["factuality"] = curr_claim_label - api_claim_data["correction"] = claim_detail.get("correction", "") - api_claim_data["reference_url"] = claim_detail.get("url", "") + api_claim_data["origin"] = origin + if claim not in claim_verify_dict: + api_claim_data["factuality"] = "Nothing to check." + else: + api_claim_data["evidence"] = claim_verify_dict.get(claim, {}) + labels = list(map(lambda x: x["relationship"], api_claim_data["evidence"])) + api_claim_data["factuality"] = labels.count("SUPPORTS") / (labels.count("REFUTES") + labels.count("SUPPORTS")) api_claim_data_list.append(api_claim_data) - api_data_dict["factuality"] = all(label_list) if label_list else True - api_data_dict["claims_details"] = api_claim_data_list + api_data_dict["claim_details"] = api_claim_data_list + api_data_dict["summary"] = { + "num_claims": len(api_data_dict["step_info"]["1_restore"]), + "num_supported_claims": len(list(filter(lambda x: x["factuality"] > 0.5, api_claim_data_list))), + } return api_data_dict diff --git a/factcheck/__main__.py b/factcheck/__main__.py index 4239281..eadf25c 100644 --- a/factcheck/__main__.py +++ b/factcheck/__main__.py @@ -28,7 +28,15 @@ def check(args): content = modal_normalization(args.modal, args.input) res = factcheck.check_response(content) - print(json.dumps(res["step_info"], indent=4)) + print(json.dumps(res, indent=4)) + + # Save the results to lark (only for local testing) + try: + from factcheck.utils import lark + + lark.save_json_to_lark_by_level(res) + except: # noqa + pass if __name__ == "__main__": diff --git a/factcheck/core/ClaimVerify.py b/factcheck/core/ClaimVerify.py index b92e138..5562037 100644 --- a/factcheck/core/ClaimVerify.py +++ b/factcheck/core/ClaimVerify.py @@ -32,10 +32,13 @@ def verify_claims(self, claims_evidences_dict, prompt: str = None) -> dict[str, evidence_lists = list(claims_evidences_dict.values()) results = self._verify_all_claims(claims, evidence_lists, prompt=prompt) - for claim, evidence_list, result in zip(claims, evidence_lists, results): - result["claim"] = claim - result["evidence"] = evidence_list + i = 0 + for claim, evidence_list in claims_evidences_dict.items(): + result = results[i : i + len(evidence_list)] + for j in range(len(result)): + result[j]["evidence"] = evidence_list[j] claim_detail_dict[claim] = result + i += len(evidence_list) return claim_detail_dict def _verify_all_claims( @@ -55,17 +58,17 @@ def _verify_all_claims( Returns: list[dict[str, any]]: a list of factuality results, including reasoning, error, correction, and factuality. """ - factual_results = [None] * len(claims) attempts = 0 # construct user inputs with respect to each claim and its evidences messages_list = [] for claim, evidences in zip(claims, evidence_lists): - if prompt is None: - user_input = self.prompt.verify_prompt.format(claim=claim, evidence=evidences) - else: - user_input = prompt.format(claim=claim, evidence=evidences) - - messages_list.append(user_input) + for e in evidences: + if prompt is None: + user_input = self.prompt.verify_prompt.format(claim=claim, evidence=e) + else: + user_input = prompt.format(claim=claim, evidence=e) + messages_list.append(user_input) + factual_results = [None] * len(messages_list) while (attempts < num_retries) and (None in factual_results): _messages = [_message for _i, _message in enumerate(messages_list) if factual_results[_i] is None] @@ -76,7 +79,7 @@ def _verify_all_claims( for _response, _index in zip(_response_list, _indices): try: _response_json = json.loads(_response) - assert all(k in _response_json for k in ["reasoning", "error", "correction", "factuality"]) + assert all(k in _response_json for k in ["reasoning", "relationship"]) factual_results[_index] = _response_json except: # noqa: E722 logger.info(f"Warning: LLM response parse fail, retry {attempts}.") @@ -84,9 +87,7 @@ def _verify_all_claims( _template_results = { "reasoning": "[System Warning] Can not identify the factuality of the claim.", - "error": "", - "correction": "", - "factuality": False, + "relationship": "IRRELEVANT", } # if cannot get correct response within num_retries times. factual_results = [_item if _item is not None else _template_results for _item in factual_results] diff --git a/factcheck/core/Decompose.py b/factcheck/core/Decompose.py index b0dca2b..e0de2f7 100644 --- a/factcheck/core/Decompose.py +++ b/factcheck/core/Decompose.py @@ -63,3 +63,55 @@ def getclaims(self, doc: str, num_retries: int = 3, prompt: str = None): logger.info("It does not output a list of sentences correctly, return self.doc2sent_tool split results.") claims = self.doc2sent(doc) return claims + + def restore_claims(self, doc: str, claims: str, num_retries: int = 3, prompt: str = None): + """Use GPT to map claims back to the document + + Args: + doc (str): the document to be decomposed into claims + claims (str): a list of claims to be mapped back to the document + num_retries (int, optional): maximum attempts for GPT to decompose the document into claims. Defaults to 3. + + Returns: + dict: a dictionary of claims and their corresponding text spans and start/end indices. + """ + + def restore(claim2doc): + claim2doc_detail = {} + flag = True + for claim, sent in claim2doc.items(): + st = doc.find(sent) + if st != -1: + claim2doc_detail[claim] = {"text": sent, "start": st, "end": st + len(sent)} + else: + flag = False + return claim2doc_detail, flag + + if prompt is None: + user_input = self.prompt.restore_prompt.format(doc=doc, claims=claims).strip() + else: + user_input = prompt.format(doc=doc, claims=claims).strip() + + messages = self.llm_client.construct_message_list([user_input]) + + tmp_restore = {} + for i in range(num_retries): + response = self.llm_client.call( + messages=messages, + num_retries=1, + seed=42 + i, + ) + try: + claim2doc = eval(response) + assert len(claim2doc) == len(claims) + claim2doc_detail, flag = restore(claim2doc) + if flag: + return claim2doc_detail + else: + tmp_restore = claim2doc_detail + raise Exception("Restore claims not satisfied.") + except Exception as e: + logger.error(f"Parse LLM response error {e}, response is: {response}") + logger.error(f"Parse LLM response error, prompt is: {messages}") + + return tmp_restore diff --git a/factcheck/core/Retriever/serper_retriever.py b/factcheck/core/Retriever/serper_retriever.py index 74e3c4d..f29c5fa 100644 --- a/factcheck/core/Retriever/serper_retriever.py +++ b/factcheck/core/Retriever/serper_retriever.py @@ -37,13 +37,16 @@ def retrieve_evidence(self, claim_query_dict, top_k: int = 5, snippet_extend_fla i = 0 claim_evidence_dict = {} for claim, queries in claim_query_dict_L: - claim_evidence_dict[claim] = evidence_list[i : i + len(queries)] + evidences_per_query_L = evidence_list[i : i + len(queries)] + claim_evidence_dict[claim] = [e for evidences in evidences_per_query_L for e in evidences] i += len(queries) assert i == len(evidence_list) logger.info("Collect evidences done!") return claim_evidence_dict - def _retrieve_evidence_4_all_claim(self, query_list: list[str], top_k: int = 5, snippet_extend_flag: bool = True): + def _retrieve_evidence_4_all_claim( + self, query_list: list[str], top_k: int = 5, snippet_extend_flag: bool = True + ) -> list[list[str]]: """Retrieve evidences for the given queries Args: @@ -52,11 +55,11 @@ def _retrieve_evidence_4_all_claim(self, query_list: list[str], top_k: int = 5, snippet_extend_flag (bool, optional): whether to extend the snippet. Defaults to True. Returns: - _type_: _description_ + list[list[]]: a list of [a list of evidences for each given query]. """ # init the evidence list with None - evidences = [None] * len(query_list) + evidences = [[]] * len(query_list) # get the response from serper serper_response = self._request_serper_api(query_list) @@ -75,23 +78,27 @@ def _retrieve_evidence_4_all_claim(self, query_list: list[str], top_k: int = 5, if "answerBox" in result: if "answer" in result["answerBox"]: - evidences[i] = { - "text": f"{query}\nAnswer: {result['answerBox']['answer']}", - "url": "Google Answer Box", - } + evidences[i] = [ + { + "text": f"{query}\nAnswer: {result['answerBox']['answer']}", + "url": "Google Answer Box", + } + ] else: - evidences[i] = { - "text": f"{query}\nAnswer: {result['answerBox']['snippet']}", - "url": "Google Answer Box", - } + evidences[i] = [ + { + "text": f"{query}\nAnswer: {result['answerBox']['snippet']}", + "url": "Google Answer Box", + } + ] + # TODO: currently --- if there is google answer box, we only got 1 evidence, otherwise, we got multiple, this will deminish the value of the google answer. else: results = result.get("organic", [])[:top_k] # Choose top 5 result - merge_evidence_text = [f"Text: {_result['snippet']} \n Source: {_result['link']}" for _result in results] - merge_evidence_text = [re.sub(r"\n+", "\n", evidence) for evidence in merge_evidence_text] - evidences[i] = { - "text": "\n\n".join(merge_evidence_text), - "url": "Multiple", - } + + if (len(_snippet_to_check) == 0) or (not snippet_extend_flag): + evidences[i] += [ + {"text": re.sub(r"\n+", "\n", _result["snippet"]), "url": _result["link"]} for _result in results + ] # Save date for each url url_to_date.update({result.get("link"): result.get("date") for result in results}) @@ -160,12 +167,9 @@ def bs4_parse_text(response, snippet, flag): for _query in query_snippet_dict.keys(): _query_index = query_list.index(_query) _snippet_list = query_snippet_dict[_query] - merge_evidence_text = [f"Text: {snippet} \n Source: {_url}" for snippet, _url in zip(_snippet_list, url_to_check)] - merge_evidence_text = [re.sub(r"\n+", "\n", evidence) for evidence in merge_evidence_text] - evidences[_query_index] = { - "text": "\n\n".join(merge_evidence_text), - "url": "Multiple", - } + evidences[_query_index] += [ + {"text": re.sub(r"\n+", "\n", snippet), "url": _url} for snippet, _url in zip(_snippet_list, url_to_check) + ] return evidences diff --git a/factcheck/utils/prompt/chatgpt_prompt.py b/factcheck/utils/prompt/chatgpt_prompt.py index 5213021..8ef8c39 100644 --- a/factcheck/utils/prompt/chatgpt_prompt.py +++ b/factcheck/utils/prompt/chatgpt_prompt.py @@ -15,6 +15,24 @@ Output: """ +restore_prompt = """Given a text and a list of facts derived from the text, your task is to identify the corresponding words in the text that derive each fact. +For each fact, please find the minimal span of text that contains the information necessary to derive the fact. The answer should be a JSON dict where the keys are the facts and the values are the corresponding text spans. + +For example, +Text: Mary is a five-year old girl, she likes playing piano and she doesn't like cookies. +Facts: ["Mary is a five-year old girl.", "Mary likes playing piano.", "Mary doesn't like cookies."] + +Output: +{{"Mary is a five-year old girl.":"Mary is a five-year old girl", +"Mary likes playing piano.":"she likes playing piano", +"Mary doesn't like cookies.":"she doesn't like cookies."] + +Text: {doc} +Facts: {claims} +Output: +""" + + checkworthy_prompt = """ Your task is to evaluate each provided statement to determine if it presents information whose factuality can be objectively verified by humans, irrespective of the statement's current accuracy. Consider the following guidelines: 1. Opinions versus Facts: Distinguish between opinions, which are subjective and not verifiable, and statements that assert factual information, even if broad or general. Focus on whether there's a factual claim that can be investigated. @@ -62,53 +80,45 @@ """ verify_prompt = """ -Your task is to evaluate the accuracy of a provided statement using the accompanying evidence. Carefully review the evidence, noting that it may vary in detail and sometimes present conflicting information. Your judgment should be informed by this evidence, taking into account its relevance and reliability. - -Keep in mind that a lack of detail in the evidence does not necessarily indicate that the statement is inaccurate. When assessing the statement's factuality, distinguish between errors and areas where the evidence supports the statement. - +Your task is to decide whether the evidence supports, refutes, or is irrelevant to the claim. Carefully review the evidence, noting that it may vary in detail and sometimes present conflicting information. Your judgment should be informed by this evidence, taking into account its relevance and reliability. Please structure your response in JSON format, including the following four keys: - "reasoning": explain the thought process behind your judgment. -- "error": none if the text is factual; otherwise, identify any specific inaccuracies in the statement. -- "correction": none if the text is factual; otherwise, provide corrections to any identified inaccuracies, using the evidence to support your corrections. -- "factuality": true if the given text is factual, false otherwise, indicating whether the statement is factual, or non-factual based on the evidence. - -For example: +- "relationship": the stance label, which can be one of "SUPPORTS", "REFUTES", or "IRRELEVANT". +For example, Input: -[text]: MBZUAI is located in Abu Dhabi, United Arab Emirates. +[claim]: MBZUAI is located in Abu Dhabi, United Arab Emirates. [evidence]: Where is MBZUAI located?\nAnswer: Masdar City - Abu Dhabi - United Arab Emirates - Output: {{ - "reasoning": "The evidence confirms that MBZUAI is located in Masdar City, Abu Dhabi, United Arab Emirates, so the statement is factually correct", - "error": none, - "correction": none, - "factuality": true + "reasoning": "The evidence confirms that MBZUAI is located in Masdar City, Abu Dhabi, United Arab Emirates, so the relationship is SUPPORTS.", + "relationship": "SUPPORTS" }} - - Input: -[text]: Copper reacts with ferrous sulfate (FeSO4). +[claim]: Copper reacts with ferrous sulfate (FeSO4). [evidence]: Copper is less reactive metal. It has positive value of standard reduction potential. Metal with high standard reduction potential can not displace other metal with low standard reduction potential values. Hence copper can not displace iron from ferrous sulphate solution. So no change will take place. - Output: {{ - "reasoning": "The evidence provided confirms that copper cannot displace iron from ferrous sulphate solution, and no change will take place.", - "error": "Copper does not react with ferrous sulfate as stated in the text.", - "correction": "Copper does not react with ferrous sulfate as it cannot displace iron from ferrous sulfate solution.", - "factuality": false + "reasoning": "The evidence provided confirms that copper cannot displace iron from ferrous sulphate solution, and no change will take place. Therefore, the evidence refutes the claim", + "relationship": "REFUTES" +}} +Input: +[claim]: Apple is a leading technology company in UK. +[evidence]: International Business Machines Corporation, nicknamed Big Blue, is an American multinational technology company headquartered in Armonk, New York and present in over 175 countries. +Output: +{{ + "reasoning": "The evidence is about IBM, while the claim is about Apple. Therefore, the evidence is irrelevant to the claim", + "relationship": "IRRELEVANT" }} - - Input -[text]: {claim} +[claim]: {claim} [evidences]: {evidence} - Output: """ class ChatGPTPrompt: decompose_prompt = decompose_prompt + restore_prompt = restore_prompt checkworthy_prompt = checkworthy_prompt qgen_prompt = qgen_prompt verify_prompt = verify_prompt From 5f31195450b4cdd4d1c5151b4d2e69cbadf3c8c7 Mon Sep 17 00:00:00 2001 From: haonan-li Date: Tue, 21 May 2024 00:23:21 +0400 Subject: [PATCH 02/13] optimize speed, debug evidence --- factcheck/__init__.py | 38 ++++++++++++-------- factcheck/__main__.py | 2 +- factcheck/core/Retriever/serper_retriever.py | 18 +++++----- 3 files changed, 34 insertions(+), 24 deletions(-) diff --git a/factcheck/__init__.py b/factcheck/__init__.py index 80627ba..e6ac057 100644 --- a/factcheck/__init__.py +++ b/factcheck/__init__.py @@ -1,3 +1,4 @@ +import concurrent.futures import time import tiktoken @@ -19,7 +20,7 @@ class FactCheck: def __init__( self, - default_model: str = "gpt-4-turbo", + default_model: str = "gpt-4o", client: str = None, prompt: str = "chatgpt_prompt", retriever: str = "serper", @@ -76,17 +77,28 @@ def check_response(self, response: str): st_time = time.time() # step 1 claims = self.decomposer.getclaims(doc=response, num_retries=self.num_seed_retries) - claim2doc = self.decomposer.restore_claims(doc=response, claims=claims, num_retries=self.num_seed_retries) - for i, claim in enumerate(claims): - logger.info(f"== response claims {i}: {claim}") - for claim, origin in claim2doc.items(): - logger.info(f"== restore claims --- {claim} --- {origin}") - - # step 2 - ( - checkworthy_claims, - pairwise_checkworthy, - ) = self.checkworthy.identify_checkworthiness(claims, num_retries=self.num_seed_retries) + # Parallel run restore claims and checkworthy + with concurrent.futures.ThreadPoolExecutor() as executor: + future_claim2doc = executor.submit( + self.decomposer.restore_claims, doc=response, claims=claims, num_retries=self.num_seed_retries + ) + # step 2 + future_checkworthy_claims = executor.submit( + self.checkworthy.identify_checkworthiness, claims, num_retries=self.num_seed_retries + ) + # step 3 + future_claim_query_dict = executor.submit(self.query_generator.generate_query, claims=claims) + + # Wait for all futures to complete + claim2doc = future_claim2doc.result() + checkworthy_claims, pairwise_checkworthy = future_checkworthy_claims.result() + claim_query_dict = future_claim_query_dict.result() + + checkworthy_claims_S = set(checkworthy_claims) + claim_query_dict = {k: v for k, v in claim_query_dict.items() if k in checkworthy_claims_S} + + for i, (claim, origin) in enumerate(claim2doc.items()): + logger.info(f"== response claims {i} --- {claim} --- {origin}") for i, claim in enumerate(checkworthy_claims): logger.info(f"== Check-worthy claims {i}: {claim}") @@ -117,8 +129,6 @@ def check_response(self, response: str): logger.info("== State: Done! (Nothing to check.)") return api_data_dict - # step 3 - claim_query_dict = self.query_generator.generate_query(claims=checkworthy_claims) for k, v in claim_query_dict.items(): logger.info(f"== Claim: {k} --- Queries: {v}") diff --git a/factcheck/__main__.py b/factcheck/__main__.py index eadf25c..ebafdd2 100644 --- a/factcheck/__main__.py +++ b/factcheck/__main__.py @@ -41,7 +41,7 @@ def check(args): if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument("--model", type=str, default="gpt-4-0125-preview") + parser.add_argument("--model", type=str, default="gpt-4o") parser.add_argument("--client", type=str, default=None, choices=CLIENTS.keys()) parser.add_argument("--prompt", type=str, default="chatgpt_prompt") parser.add_argument("--retriever", type=str, default="serper") diff --git a/factcheck/core/Retriever/serper_retriever.py b/factcheck/core/Retriever/serper_retriever.py index f29c5fa..fa95ec2 100644 --- a/factcheck/core/Retriever/serper_retriever.py +++ b/factcheck/core/Retriever/serper_retriever.py @@ -59,7 +59,7 @@ def _retrieve_evidence_4_all_claim( """ # init the evidence list with None - evidences = [[]] * len(query_list) + evidences = [[] for _ in query_list] # get the response from serper serper_response = self._request_serper_api(query_list) @@ -157,18 +157,18 @@ def bs4_parse_text(response, snippet, flag): ) # merge the snippets by query - query_snippet_dict = {} - for _query, _snippet in zip(query_to_check, _extended_snippet): - _snippet_list = query_snippet_dict.get(_query, []) - _snippet_list.append(_snippet) - query_snippet_dict[_query] = _snippet_list + query_snippet_url_dict = {} + for _query, _url, _snippet in zip(query_to_check, url_to_check, _extended_snippet): + _snippet_url_list = query_snippet_url_dict.get(_query, []) + _snippet_url_list.append((_snippet, _url)) + query_snippet_url_dict[_query] = _snippet_url_list # extend the evidence list for each query - for _query in query_snippet_dict.keys(): + for _query in query_snippet_url_dict.keys(): _query_index = query_list.index(_query) - _snippet_list = query_snippet_dict[_query] + _snippet_url_list = query_snippet_url_dict[_query] evidences[_query_index] += [ - {"text": re.sub(r"\n+", "\n", snippet), "url": _url} for snippet, _url in zip(_snippet_list, url_to_check) + {"text": re.sub(r"\n+", "\n", snippet), "url": _url} for snippet, _url in _snippet_url_list ] return evidences From d99cbb553c0f0e124f1a9857e376158f387871fe Mon Sep 17 00:00:00 2001 From: haonan-li Date: Tue, 21 May 2024 15:23:15 +0400 Subject: [PATCH 03/13] debug to make sure restore the text --- factcheck/__init__.py | 9 ++++-- factcheck/core/Decompose.py | 18 ++++++++++++ factcheck/core/Retriever/serper_retriever.py | 2 +- factcheck/utils/prompt/chatgpt_prompt.py | 29 ++++++++++++++++---- 4 files changed, 50 insertions(+), 8 deletions(-) diff --git a/factcheck/__init__.py b/factcheck/__init__.py index e6ac057..f71ec60 100644 --- a/factcheck/__init__.py +++ b/factcheck/__init__.py @@ -173,11 +173,16 @@ def _post_process(self, api_data_dict, claim_verify_dict: dict): else: api_claim_data["evidence"] = claim_verify_dict.get(claim, {}) labels = list(map(lambda x: x["relationship"], api_claim_data["evidence"])) - api_claim_data["factuality"] = labels.count("SUPPORTS") / (labels.count("REFUTES") + labels.count("SUPPORTS")) + if labels.count("SUPPORTS") + labels.count("REFUTES") == 0: + api_claim_data["factuality"] = "No evidence found." + else: + api_claim_data["factuality"] = labels.count("SUPPORTS") / ( + labels.count("REFUTES") + labels.count("SUPPORTS") + ) api_claim_data_list.append(api_claim_data) api_data_dict["claim_details"] = api_claim_data_list api_data_dict["summary"] = { "num_claims": len(api_data_dict["step_info"]["1_restore"]), - "num_supported_claims": len(list(filter(lambda x: x["factuality"] > 0.5, api_claim_data_list))), + # "num_supported_claims": len(list(filter(lambda x: x["factuality"] > 0.5, api_claim_data_list))), } return api_data_dict diff --git a/factcheck/core/Decompose.py b/factcheck/core/Decompose.py index e0de2f7..992fdce 100644 --- a/factcheck/core/Decompose.py +++ b/factcheck/core/Decompose.py @@ -85,6 +85,24 @@ def restore(claim2doc): claim2doc_detail[claim] = {"text": sent, "start": st, "end": st + len(sent)} else: flag = False + + cur_pos = -1 + texts = [] + for k, v in claim2doc_detail.items(): + if v["start"] < cur_pos + 1 and v["end"] > cur_pos: + v["start"] = cur_pos + 1 + flag = False + elif v["start"] < cur_pos + 1 and v["end"] <= cur_pos: + v["start"] = v["end"] # temporarily ignore this span + flag = False + elif v["start"] > cur_pos + 1: + v["start"] = cur_pos + 1 + flag = False + v["text"] = doc[v["start"] : v["end"]] + texts.append(v["text"]) + claim2doc_detail[k] = v + cur_pos = v["end"] + return claim2doc_detail, flag if prompt is None: diff --git a/factcheck/core/Retriever/serper_retriever.py b/factcheck/core/Retriever/serper_retriever.py index fa95ec2..b24583e 100644 --- a/factcheck/core/Retriever/serper_retriever.py +++ b/factcheck/core/Retriever/serper_retriever.py @@ -104,7 +104,7 @@ def _retrieve_evidence_4_all_claim( url_to_date.update({result.get("link"): result.get("date") for result in results}) # Save query-url pair, 1 query may have multiple urls query_url_dict.update({query: [result.get("link") for result in results]}) - _snippet_to_check += [result["snippet"] for result in results] + _snippet_to_check += [result["snippet"] if "snippet" in result else "" for result in results] # return if there is no snippet to check or snippet_extend_flag is False if (len(_snippet_to_check) == 0) or (not snippet_extend_flag): diff --git a/factcheck/utils/prompt/chatgpt_prompt.py b/factcheck/utils/prompt/chatgpt_prompt.py index 8ef8c39..5f17e6e 100644 --- a/factcheck/utils/prompt/chatgpt_prompt.py +++ b/factcheck/utils/prompt/chatgpt_prompt.py @@ -15,23 +15,42 @@ Output: """ -restore_prompt = """Given a text and a list of facts derived from the text, your task is to identify the corresponding words in the text that derive each fact. -For each fact, please find the minimal span of text that contains the information necessary to derive the fact. The answer should be a JSON dict where the keys are the facts and the values are the corresponding text spans. +# restore_prompt = """Given a text and a list of facts derived from the text, your task is to identify the corresponding words in the text that derive each fact. +# For each fact, please find the minimal continues span in the original text that contains the information to derive the fact. The answer should be a JSON dict where the keys are the facts and the values are the corresponding spans copied from the original text. +# +# For example, +# Text: Mary is a five-year old girl, she likes playing piano and she doesn't like cookies. +# Facts: ["Mary is a five-year old girl.", "Mary likes playing piano.", "Mary doesn't like cookies."] +# +# Output: +# {{"Mary is a five-year old girl.":"Mary is a five-year old girl", +# "Mary likes playing piano.":"she likes playing piano", +# "Mary doesn't like cookies.":"she doesn't like cookies."] +# +# Text: {doc} +# Facts: {claims} +# Output: +# """ + +# use this for demo +restore_prompt = """Given a text and a list of facts derived from the text, your task is to split the text into chunks that derive each fact. +For each fact, please find the corresponding continues span in the original text that contains the information to derive the fact. The answer should be a JSON dict where the keys are the facts and the values are the corresponding spans copied from the original text. +Please make sure the returned spans can be concatenated to the full original doc. For example, Text: Mary is a five-year old girl, she likes playing piano and she doesn't like cookies. Facts: ["Mary is a five-year old girl.", "Mary likes playing piano.", "Mary doesn't like cookies."] Output: -{{"Mary is a five-year old girl.":"Mary is a five-year old girl", +{{"Mary is a five-year old girl.":"Mary is a five-year old girl,", "Mary likes playing piano.":"she likes playing piano", -"Mary doesn't like cookies.":"she doesn't like cookies."] +"Mary doesn't like cookies.":"and she doesn't like cookies."] Text: {doc} Facts: {claims} Output: -""" +""" checkworthy_prompt = """ Your task is to evaluate each provided statement to determine if it presents information whose factuality can be objectively verified by humans, irrespective of the statement's current accuracy. Consider the following guidelines: From 220af64199d9917c8ac7aacf4b820ba0d3b0d04d Mon Sep 17 00:00:00 2001 From: haonan-li Date: Tue, 21 May 2024 16:40:51 +0400 Subject: [PATCH 04/13] minor bug for demo --- factcheck/__init__.py | 2 +- factcheck/core/Retriever/serper_retriever.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/factcheck/__init__.py b/factcheck/__init__.py index f71ec60..f1df319 100644 --- a/factcheck/__init__.py +++ b/factcheck/__init__.py @@ -174,7 +174,7 @@ def _post_process(self, api_data_dict, claim_verify_dict: dict): api_claim_data["evidence"] = claim_verify_dict.get(claim, {}) labels = list(map(lambda x: x["relationship"], api_claim_data["evidence"])) if labels.count("SUPPORTS") + labels.count("REFUTES") == 0: - api_claim_data["factuality"] = "No evidence found." + api_claim_data["factuality"] = 1 # "No evidence found." else: api_claim_data["factuality"] = labels.count("SUPPORTS") / ( labels.count("REFUTES") + labels.count("SUPPORTS") diff --git a/factcheck/core/Retriever/serper_retriever.py b/factcheck/core/Retriever/serper_retriever.py index b24583e..5a873a0 100644 --- a/factcheck/core/Retriever/serper_retriever.py +++ b/factcheck/core/Retriever/serper_retriever.py @@ -62,6 +62,7 @@ def _retrieve_evidence_4_all_claim( evidences = [[] for _ in query_list] # get the response from serper + # TODO: Can send up to 100 queries once serper_response = self._request_serper_api(query_list) if serper_response is None: From 0551ba6d697203a59b00ce5e7ea1897fe107e950 Mon Sep 17 00:00:00 2001 From: haonan-li Date: Tue, 21 May 2024 18:29:28 +0400 Subject: [PATCH 05/13] minor --- factcheck/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/factcheck/__init__.py b/factcheck/__init__.py index f1df319..53c9853 100644 --- a/factcheck/__init__.py +++ b/factcheck/__init__.py @@ -181,8 +181,10 @@ def _post_process(self, api_data_dict, claim_verify_dict: dict): ) api_claim_data_list.append(api_claim_data) api_data_dict["claim_details"] = api_claim_data_list + + valid_claims = list(filter(lambda x: not isinstance(x["factuality"], str), api_claim_data_list)) api_data_dict["summary"] = { - "num_claims": len(api_data_dict["step_info"]["1_restore"]), - # "num_supported_claims": len(list(filter(lambda x: x["factuality"] > 0.5, api_claim_data_list))), + "num_claims": len(valid_claims), + "overall_factuality": sum(map(lambda x: x["factuality"], valid_claims)) / len(valid_claims), } return api_data_dict From 57d3910774f7ee2ff1caed183c05ed27b0e00478 Mon Sep 17 00:00:00 2001 From: haonan-li Date: Tue, 21 May 2024 22:20:28 +0400 Subject: [PATCH 06/13] udpate to 4o --- factcheck/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/factcheck/__init__.py b/factcheck/__init__.py index 53c9853..a31abab 100644 --- a/factcheck/__init__.py +++ b/factcheck/__init__.py @@ -28,7 +28,7 @@ def __init__( checkworthy_model: str = None, query_generator_model: str = None, evidence_retrieval_model: str = None, - claim_verify_model: str = "gpt-3.5-turbo", + claim_verify_model: str = None, # "gpt-3.5-turbo", api_config: dict = None, num_seed_retries: int = 3, ): From ee14509313ed0c14e2ede97cdd811a4f31d08a29 Mon Sep 17 00:00:00 2001 From: Xudong Han Date: Thu, 23 May 2024 01:54:22 +0400 Subject: [PATCH 07/13] Update result.html --- templates/result.html | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/templates/result.html b/templates/result.html index c51f2a4..223b57c 100644 --- a/templates/result.html +++ b/templates/result.html @@ -97,23 +97,23 @@

Results

Response: {{ responses.response }}

-

Factuality: {{ responses.factuality }}

+

Factuality: {{ responses.summary.overall_factuality }}

- {% for key, value in responses['step_info']["5_claim_verify"].items() %} + {% for claim in responses['claim_details'] %}
-

Claim: {{ key }}

-

Factuality: {{ value.factuality }}

-

Error: {{ value.error }}

-

Correction: {{ value.correction }}

-

Reasoning: {{ value.reasoning }}

-

Evidences:

+

Claim: {{ claim.claim }}

+

Factuality: {{ claim.factuality }}

+

Evidences:

\n
',null!=t.firstChild?t.insertBefore(this.el,t.firstChild):t.appendChild(this.el)}return this.el},$.prototype.finish=function(){var t=this.getElement();return t.className=t.className.replace("pace-active","pace-inactive"),document.body.className=document.body.className.replace("pace-running ","pace-done ")},$.prototype.update=function(t){return this.progress=t,y.trigger("progress",t),this.render()},$.prototype.destroy=function(){try{this.getElement().parentNode.removeChild(this.getElement())}catch(t){n=t}return this.el=void 0},$.prototype.render=function(){var t,e,n,r,s,o,i;if(null==document.querySelector(T.target))return!1;for(t=this.getElement(),r="translate3d("+this.progress+"%, 0, 0)",s=0,o=(i=["webkitTransform","msTransform","transform"]).length;sT.eventLag.sampleCount&&o.shift(),e=h(o),++s>=T.eventLag.minSamples&&ethis.progress&&(this.progress+=this.catchup*t),e=1-Math.pow(this.progress/100,T.easeFactor),this.progress+=e*this.rate*t,this.progress=Math.min(this.lastProgress+T.maxProgressPerFrame,this.progress),this.progress=Math.max(0,this.progress),this.progress=Math.min(100,this.progress),this.lastProgress=this.progress,this.progress},v=it,b=e=_=w=E=N=null,y.running=!1,q=function(){if(T.restartOnPushState)return y.restart()},null!=window.history.pushState&&(I=window.history.pushState,window.history.pushState=function(){return q(),I.apply(window.history,arguments)}),null!=window.history.replaceState&&(G=window.history.replaceState,window.history.replaceState=function(){return q(),G.apply(window.history,arguments)}),l={ajax:u,elements:k,document:B,eventLag:Q},(x=function(){var t,e,n,r,s,o,i,a;for(y.sources=N=[],e=0,r=(o=["ajax","elements","document","eventLag"]).length;e+~]|"+ge+")"+ge+"*"),x=new RegExp(ge+"|>"),j=new RegExp(g),A=new RegExp("^"+t+"$"),D={ID:new RegExp("^#("+t+")"),CLASS:new RegExp("^\\.("+t+")"),TAG:new RegExp("^("+t+"|[*])"),ATTR:new RegExp("^"+p),PSEUDO:new RegExp("^"+g),CHILD:new RegExp("^:(only|first|last|nth|nth-last)-(child|of-type)(?:\\("+ge+"*(even|odd|(([+-]|)(\\d*)n|)"+ge+"*(?:([+-]|)"+ge+"*(\\d+)|))"+ge+"*\\)|)","i"),bool:new RegExp("^(?:"+f+")$","i"),needsContext:new RegExp("^"+ge+"*[>+~]|:(even|odd|eq|gt|lt|nth|first|last)(?:\\("+ge+"*((?:-\\d)?\\d*)"+ge+"*\\)|)(?=[^-]|$)","i")},N=/^(?:input|select|textarea|button)$/i,q=/^h\d$/i,L=/^(?:#([\w-]+)|(\w+)|\.([\w-]+))$/,H=/[+~]/,O=new RegExp("\\\\[\\da-fA-F]{1,6}"+ge+"?|\\\\([^\\r\\n\\f])","g"),P=function(e,t){var n="0x"+e.slice(1)-65536;return t||(n<0?String.fromCharCode(n+65536):String.fromCharCode(n>>10|55296,1023&n|56320))},M=function(){V()},R=J(function(e){return!0===e.disabled&&fe(e,"fieldset")},{dir:"parentNode",next:"legend"});try{k.apply(oe=ae.call(ye.childNodes),ye.childNodes),oe[ye.childNodes.length].nodeType}catch(e){k={apply:function(e,t){me.apply(e,ae.call(t))},call:function(e){me.apply(e,ae.call(arguments,1))}}}function I(t,e,n,r){var i,o,a,s,u,l,c,f=e&&e.ownerDocument,p=e?e.nodeType:9;if(n=n||[],"string"!=typeof t||!t||1!==p&&9!==p&&11!==p)return n;if(!r&&(V(e),e=e||T,C)){if(11!==p&&(u=L.exec(t)))if(i=u[1]){if(9===p){if(!(a=e.getElementById(i)))return n;if(a.id===i)return k.call(n,a),n}else if(f&&(a=f.getElementById(i))&&I.contains(e,a)&&a.id===i)return k.call(n,a),n}else{if(u[2])return k.apply(n,e.getElementsByTagName(t)),n;if((i=u[3])&&e.getElementsByClassName)return k.apply(n,e.getElementsByClassName(i)),n}if(!(h[t+" "]||d&&d.test(t))){if(c=t,f=e,1===p&&(x.test(t)||m.test(t))){(f=H.test(t)&&U(e.parentNode)||e)==e&&le.scope||((s=e.getAttribute("id"))?s=ce.escapeSelector(s):e.setAttribute("id",s=S)),o=(l=Y(t)).length;while(o--)l[o]=(s?"#"+s:":scope")+" "+Q(l[o]);c=l.join(",")}try{return k.apply(n,f.querySelectorAll(c)),n}catch(e){h(t,!0)}finally{s===S&&e.removeAttribute("id")}}}return re(t.replace(ve,"$1"),e,n,r)}function W(){var r=[];return function e(t,n){return r.push(t+" ")>b.cacheLength&&delete e[r.shift()],e[t+" "]=n}}function F(e){return e[S]=!0,e}function $(e){var t=T.createElement("fieldset");try{return!!e(t)}catch(e){return!1}finally{t.parentNode&&t.parentNode.removeChild(t),t=null}}function B(t){return function(e){return fe(e,"input")&&e.type===t}}function _(t){return function(e){return(fe(e,"input")||fe(e,"button"))&&e.type===t}}function z(t){return function(e){return"form"in e?e.parentNode&&!1===e.disabled?"label"in e?"label"in e.parentNode?e.parentNode.disabled===t:e.disabled===t:e.isDisabled===t||e.isDisabled!==!t&&R(e)===t:e.disabled===t:"label"in e&&e.disabled===t}}function X(a){return F(function(o){return o=+o,F(function(e,t){var n,r=a([],e.length,o),i=r.length;while(i--)e[n=r[i]]&&(e[n]=!(t[n]=e[n]))})})}function U(e){return e&&"undefined"!=typeof e.getElementsByTagName&&e}function V(e){var t,n=e?e.ownerDocument||e:ye;return n!=T&&9===n.nodeType&&n.documentElement&&(r=(T=n).documentElement,C=!ce.isXMLDoc(T),i=r.matches||r.webkitMatchesSelector||r.msMatchesSelector,r.msMatchesSelector&&ye!=T&&(t=T.defaultView)&&t.top!==t&&t.addEventListener("unload",M),le.getById=$(function(e){return r.appendChild(e).id=ce.expando,!T.getElementsByName||!T.getElementsByName(ce.expando).length}),le.disconnectedMatch=$(function(e){return i.call(e,"*")}),le.scope=$(function(){return T.querySelectorAll(":scope")}),le.cssHas=$(function(){try{return T.querySelector(":has(*,:jqfake)"),!1}catch(e){return!0}}),le.getById?(b.filter.ID=function(e){var t=e.replace(O,P);return function(e){return e.getAttribute("id")===t}},b.find.ID=function(e,t){if("undefined"!=typeof t.getElementById&&C){var n=t.getElementById(e);return n?[n]:[]}}):(b.filter.ID=function(e){var n=e.replace(O,P);return function(e){var t="undefined"!=typeof e.getAttributeNode&&e.getAttributeNode("id");return t&&t.value===n}},b.find.ID=function(e,t){if("undefined"!=typeof t.getElementById&&C){var n,r,i,o=t.getElementById(e);if(o){if((n=o.getAttributeNode("id"))&&n.value===e)return[o];i=t.getElementsByName(e),r=0;while(o=i[r++])if((n=o.getAttributeNode("id"))&&n.value===e)return[o]}return[]}}),b.find.TAG=function(e,t){return"undefined"!=typeof t.getElementsByTagName?t.getElementsByTagName(e):t.querySelectorAll(e)},b.find.CLASS=function(e,t){if("undefined"!=typeof t.getElementsByClassName&&C)return t.getElementsByClassName(e)},d=[],$(function(e){var t;r.appendChild(e).innerHTML="",e.querySelectorAll("[selected]").length||d.push("\\["+ge+"*(?:value|"+f+")"),e.querySelectorAll("[id~="+S+"-]").length||d.push("~="),e.querySelectorAll("a#"+S+"+*").length||d.push(".#.+[+~]"),e.querySelectorAll(":checked").length||d.push(":checked"),(t=T.createElement("input")).setAttribute("type","hidden"),e.appendChild(t).setAttribute("name","D"),r.appendChild(e).disabled=!0,2!==e.querySelectorAll(":disabled").length&&d.push(":enabled",":disabled"),(t=T.createElement("input")).setAttribute("name",""),e.appendChild(t),e.querySelectorAll("[name='']").length||d.push("\\["+ge+"*name"+ge+"*="+ge+"*(?:''|\"\")")}),le.cssHas||d.push(":has"),d=d.length&&new RegExp(d.join("|")),l=function(e,t){if(e===t)return a=!0,0;var n=!e.compareDocumentPosition-!t.compareDocumentPosition;return n||(1&(n=(e.ownerDocument||e)==(t.ownerDocument||t)?e.compareDocumentPosition(t):1)||!le.sortDetached&&t.compareDocumentPosition(e)===n?e===T||e.ownerDocument==ye&&I.contains(ye,e)?-1:t===T||t.ownerDocument==ye&&I.contains(ye,t)?1:o?se.call(o,e)-se.call(o,t):0:4&n?-1:1)}),T}for(e in I.matches=function(e,t){return I(e,null,null,t)},I.matchesSelector=function(e,t){if(V(e),C&&!h[t+" "]&&(!d||!d.test(t)))try{var n=i.call(e,t);if(n||le.disconnectedMatch||e.document&&11!==e.document.nodeType)return n}catch(e){h(t,!0)}return 0":{dir:"parentNode",first:!0}," ":{dir:"parentNode"},"+":{dir:"previousSibling",first:!0},"~":{dir:"previousSibling"}},preFilter:{ATTR:function(e){return e[1]=e[1].replace(O,P),e[3]=(e[3]||e[4]||e[5]||"").replace(O,P),"~="===e[2]&&(e[3]=" "+e[3]+" "),e.slice(0,4)},CHILD:function(e){return e[1]=e[1].toLowerCase(),"nth"===e[1].slice(0,3)?(e[3]||I.error(e[0]),e[4]=+(e[4]?e[5]+(e[6]||1):2*("even"===e[3]||"odd"===e[3])),e[5]=+(e[7]+e[8]||"odd"===e[3])):e[3]&&I.error(e[0]),e},PSEUDO:function(e){var t,n=!e[6]&&e[2];return D.CHILD.test(e[0])?null:(e[3]?e[2]=e[4]||e[5]||"":n&&j.test(n)&&(t=Y(n,!0))&&(t=n.indexOf(")",n.length-t)-n.length)&&(e[0]=e[0].slice(0,t),e[2]=n.slice(0,t)),e.slice(0,3))}},filter:{TAG:function(e){var t=e.replace(O,P).toLowerCase();return"*"===e?function(){return!0}:function(e){return fe(e,t)}},CLASS:function(e){var t=s[e+" "];return t||(t=new RegExp("(^|"+ge+")"+e+"("+ge+"|$)"))&&s(e,function(e){return t.test("string"==typeof e.className&&e.className||"undefined"!=typeof e.getAttribute&&e.getAttribute("class")||"")})},ATTR:function(n,r,i){return function(e){var t=I.attr(e,n);return null==t?"!="===r:!r||(t+="","="===r?t===i:"!="===r?t!==i:"^="===r?i&&0===t.indexOf(i):"*="===r?i&&-1:\x20\t\r\n\f]*)[\x20\t\r\n\f]*\/?>(?:<\/\1>|)$/i;function T(e,n,r){return v(n)?ce.grep(e,function(e,t){return!!n.call(e,t,e)!==r}):n.nodeType?ce.grep(e,function(e){return e===n!==r}):"string"!=typeof n?ce.grep(e,function(e){return-1)[^>]*|#([\w-]+))$/;(ce.fn.init=function(e,t,n){var r,i;if(!e)return this;if(n=n||k,"string"==typeof e){if(!(r="<"===e[0]&&">"===e[e.length-1]&&3<=e.length?[null,e,null]:S.exec(e))||!r[1]&&t)return!t||t.jquery?(t||n).find(e):this.constructor(t).find(e);if(r[1]){if(t=t instanceof ce?t[0]:t,ce.merge(this,ce.parseHTML(r[1],t&&t.nodeType?t.ownerDocument||t:C,!0)),w.test(r[1])&&ce.isPlainObject(t))for(r in t)v(this[r])?this[r](t[r]):this.attr(r,t[r]);return this}return(i=C.getElementById(r[2]))&&(this[0]=i,this.length=1),this}return e.nodeType?(this[0]=e,this.length=1,this):v(e)?void 0!==n.ready?n.ready(e):e(ce):ce.makeArray(e,this)}).prototype=ce.fn,k=ce(C);var E=/^(?:parents|prev(?:Until|All))/,j={children:!0,contents:!0,next:!0,prev:!0};function A(e,t){while((e=e[t])&&1!==e.nodeType);return e}ce.fn.extend({has:function(e){var t=ce(e,this),n=t.length;return this.filter(function(){for(var e=0;e\x20\t\r\n\f]*)/i,Ce=/^$|^module$|\/(?:java|ecma)script/i;xe=C.createDocumentFragment().appendChild(C.createElement("div")),(be=C.createElement("input")).setAttribute("type","radio"),be.setAttribute("checked","checked"),be.setAttribute("name","t"),xe.appendChild(be),le.checkClone=xe.cloneNode(!0).cloneNode(!0).lastChild.checked,xe.innerHTML="",le.noCloneChecked=!!xe.cloneNode(!0).lastChild.defaultValue,xe.innerHTML="",le.option=!!xe.lastChild;var ke={thead:[1,"","
"],col:[2,"","
"],tr:[2,"","
"],td:[3,"","
"],_default:[0,"",""]};function Se(e,t){var n;return n="undefined"!=typeof e.getElementsByTagName?e.getElementsByTagName(t||"*"):"undefined"!=typeof e.querySelectorAll?e.querySelectorAll(t||"*"):[],void 0===t||t&&fe(e,t)?ce.merge([e],n):n}function Ee(e,t){for(var n=0,r=e.length;n",""]);var je=/<|&#?\w+;/;function Ae(e,t,n,r,i){for(var o,a,s,u,l,c,f=t.createDocumentFragment(),p=[],d=0,h=e.length;d\s*$/g;function Re(e,t){return fe(e,"table")&&fe(11!==t.nodeType?t:t.firstChild,"tr")&&ce(e).children("tbody")[0]||e}function Ie(e){return e.type=(null!==e.getAttribute("type"))+"/"+e.type,e}function We(e){return"true/"===(e.type||"").slice(0,5)?e.type=e.type.slice(5):e.removeAttribute("type"),e}function Fe(e,t){var n,r,i,o,a,s;if(1===t.nodeType){if(_.hasData(e)&&(s=_.get(e).events))for(i in _.remove(t,"handle events"),s)for(n=0,r=s[i].length;n").attr(n.scriptAttrs||{}).prop({charset:n.scriptCharset,src:n.url}).on("load error",i=function(e){r.remove(),i=null,e&&t("error"===e.type?404:200,e.type)}),C.head.appendChild(r[0])},abort:function(){i&&i()}}});var Jt,Kt=[],Zt=/(=)\?(?=&|$)|\?\?/;ce.ajaxSetup({jsonp:"callback",jsonpCallback:function(){var e=Kt.pop()||ce.expando+"_"+jt.guid++;return this[e]=!0,e}}),ce.ajaxPrefilter("json jsonp",function(e,t,n){var r,i,o,a=!1!==e.jsonp&&(Zt.test(e.url)?"url":"string"==typeof e.data&&0===(e.contentType||"").indexOf("application/x-www-form-urlencoded")&&Zt.test(e.data)&&"data");if(a||"jsonp"===e.dataTypes[0])return r=e.jsonpCallback=v(e.jsonpCallback)?e.jsonpCallback():e.jsonpCallback,a?e[a]=e[a].replace(Zt,"$1"+r):!1!==e.jsonp&&(e.url+=(At.test(e.url)?"&":"?")+e.jsonp+"="+r),e.converters["script json"]=function(){return o||ce.error(r+" was not called"),o[0]},e.dataTypes[0]="json",i=ie[r],ie[r]=function(){o=arguments},n.always(function(){void 0===i?ce(ie).removeProp(r):ie[r]=i,e[r]&&(e.jsonpCallback=t.jsonpCallback,Kt.push(r)),o&&v(i)&&i(o[0]),o=i=void 0}),"script"}),le.createHTMLDocument=((Jt=C.implementation.createHTMLDocument("").body).innerHTML="
",2===Jt.childNodes.length),ce.parseHTML=function(e,t,n){return"string"!=typeof e?[]:("boolean"==typeof t&&(n=t,t=!1),t||(le.createHTMLDocument?((r=(t=C.implementation.createHTMLDocument("")).createElement("base")).href=C.location.href,t.head.appendChild(r)):t=C),o=!n&&[],(i=w.exec(e))?[t.createElement(i[1])]:(i=Ae([e],t,o),o&&o.length&&ce(o).remove(),ce.merge([],i.childNodes)));var r,i,o},ce.fn.load=function(e,t,n){var r,i,o,a=this,s=e.indexOf(" ");return-1").append(ce.parseHTML(e)).find(r):e)}).always(n&&function(e,t){a.each(function(){n.apply(this,o||[e.responseText,t,e])})}),this},ce.expr.pseudos.animated=function(t){return ce.grep(ce.timers,function(e){return t===e.elem}).length},ce.offset={setOffset:function(e,t,n){var r,i,o,a,s,u,l=ce.css(e,"position"),c=ce(e),f={};"static"===l&&(e.style.position="relative"),s=c.offset(),o=ce.css(e,"top"),u=ce.css(e,"left"),("absolute"===l||"fixed"===l)&&-1<(o+u).indexOf("auto")?(a=(r=c.position()).top,i=r.left):(a=parseFloat(o)||0,i=parseFloat(u)||0),v(t)&&(t=t.call(e,n,ce.extend({},s))),null!=t.top&&(f.top=t.top-s.top+a),null!=t.left&&(f.left=t.left-s.left+i),"using"in t?t.using.call(e,f):c.css(f)}},ce.fn.extend({offset:function(t){if(arguments.length)return void 0===t?this:this.each(function(e){ce.offset.setOffset(this,t,e)});var e,n,r=this[0];return r?r.getClientRects().length?(e=r.getBoundingClientRect(),n=r.ownerDocument.defaultView,{top:e.top+n.pageYOffset,left:e.left+n.pageXOffset}):{top:0,left:0}:void 0},position:function(){if(this[0]){var e,t,n,r=this[0],i={top:0,left:0};if("fixed"===ce.css(r,"position"))t=r.getBoundingClientRect();else{t=this.offset(),n=r.ownerDocument,e=r.offsetParent||n.documentElement;while(e&&(e===n.body||e===n.documentElement)&&"static"===ce.css(e,"position"))e=e.parentNode;e&&e!==r&&1===e.nodeType&&((i=ce(e).offset()).top+=ce.css(e,"borderTopWidth",!0),i.left+=ce.css(e,"borderLeftWidth",!0))}return{top:t.top-i.top-ce.css(r,"marginTop",!0),left:t.left-i.left-ce.css(r,"marginLeft",!0)}}},offsetParent:function(){return this.map(function(){var e=this.offsetParent;while(e&&"static"===ce.css(e,"position"))e=e.offsetParent;return e||J})}}),ce.each({scrollLeft:"pageXOffset",scrollTop:"pageYOffset"},function(t,i){var o="pageYOffset"===i;ce.fn[t]=function(e){return M(this,function(e,t,n){var r;if(y(e)?r=e:9===e.nodeType&&(r=e.defaultView),void 0===n)return r?r[i]:e[t];r?r.scrollTo(o?r.pageXOffset:n,o?n:r.pageYOffset):e[t]=n},t,e,arguments.length)}}),ce.each(["top","left"],function(e,n){ce.cssHooks[n]=Ye(le.pixelPosition,function(e,t){if(t)return t=Ge(e,n),_e.test(t)?ce(e).position()[n]+"px":t})}),ce.each({Height:"height",Width:"width"},function(a,s){ce.each({padding:"inner"+a,content:s,"":"outer"+a},function(r,o){ce.fn[o]=function(e,t){var n=arguments.length&&(r||"boolean"!=typeof e),i=r||(!0===e||!0===t?"margin":"border");return M(this,function(e,t,n){var r;return y(e)?0===o.indexOf("outer")?e["inner"+a]:e.document.documentElement["client"+a]:9===e.nodeType?(r=e.documentElement,Math.max(e.body["scroll"+a],r["scroll"+a],e.body["offset"+a],r["offset"+a],r["client"+a])):void 0===n?ce.css(e,t,i):ce.style(e,t,n,i)},s,n?e:void 0,n)}})}),ce.each(["ajaxStart","ajaxStop","ajaxComplete","ajaxError","ajaxSuccess","ajaxSend"],function(e,t){ce.fn[t]=function(e){return this.on(t,e)}}),ce.fn.extend({bind:function(e,t,n){return this.on(e,null,t,n)},unbind:function(e,t){return this.off(e,null,t)},delegate:function(e,t,n,r){return this.on(t,e,n,r)},undelegate:function(e,t,n){return 1===arguments.length?this.off(e,"**"):this.off(t,e||"**",n)},hover:function(e,t){return this.on("mouseenter",e).on("mouseleave",t||e)}}),ce.each("blur focus focusin focusout resize scroll click dblclick mousedown mouseup mousemove mouseover mouseout mouseenter mouseleave change select submit keydown keypress keyup contextmenu".split(" "),function(e,n){ce.fn[n]=function(e,t){return 0",options:{classes:{},disabled:!1,create:null},_createWidget:function(t,e){e=V(e||this.defaultElement||this)[0],this.element=V(e),this.uuid=i++,this.eventNamespace="."+this.widgetName+this.uuid,this.bindings=V(),this.hoverable=V(),this.focusable=V(),this.classesElementLookup={},e!==this&&(V.data(e,this.widgetFullName,this),this._on(!0,this.element,{remove:function(t){t.target===e&&this.destroy()}}),this.document=V(e.style?e.ownerDocument:e.document||e),this.window=V(this.document[0].defaultView||this.document[0].parentWindow)),this.options=V.widget.extend({},this.options,this._getCreateOptions(),t),this._create(),this.options.disabled&&this._setOptionDisabled(this.options.disabled),this._trigger("create",null,this._getCreateEventData()),this._init()},_getCreateOptions:function(){return{}},_getCreateEventData:V.noop,_create:V.noop,_init:V.noop,destroy:function(){var i=this;this._destroy(),V.each(this.classesElementLookup,function(t,e){i._removeClass(e,t)}),this.element.off(this.eventNamespace).removeData(this.widgetFullName),this.widget().off(this.eventNamespace).removeAttr("aria-disabled"),this.bindings.off(this.eventNamespace)},_destroy:V.noop,widget:function(){return this.element},option:function(t,e){var i,s,n,o=t;if(0===arguments.length)return V.widget.extend({},this.options);if("string"==typeof t)if(o={},t=(i=t.split(".")).shift(),i.length){for(s=o[t]=V.widget.extend({},this.options[t]),n=0;n
"),i=e.children()[0];return V("body").append(e),t=i.offsetWidth,e.css("overflow","scroll"),t===(i=i.offsetWidth)&&(i=e[0].clientWidth),e.remove(),s=t-i},getScrollInfo:function(t){var e=t.isWindow||t.isDocument?"":t.element.css("overflow-x"),i=t.isWindow||t.isDocument?"":t.element.css("overflow-y"),e="scroll"===e||"auto"===e&&t.widthx(k(s),k(n))?o.important="horizontal":o.important="vertical",u.using.call(this,t,o)}),a.offset(V.extend(h,{using:t}))})},V.ui.position={fit:{left:function(t,e){var i=e.within,s=i.isWindow?i.scrollLeft:i.offset.left,n=i.width,o=t.left-e.collisionPosition.marginLeft,a=s-o,r=o+e.collisionWidth-n-s;e.collisionWidth>n?0n?0")[0],w=d.each;function P(t){return null==t?t+"":"object"==typeof t?p[e.call(t)]||"object":typeof t}function M(t,e,i){var s=v[e.type]||{};return null==t?i||!e.def?null:e.def:(t=s.floor?~~t:parseFloat(t),isNaN(t)?e.def:s.mod?(t+s.mod)%s.mod:Math.min(s.max,Math.max(0,t)))}function S(s){var n=m(),o=n._rgba=[];return s=s.toLowerCase(),w(g,function(t,e){var i=e.re.exec(s),i=i&&e.parse(i),e=e.space||"rgba";if(i)return i=n[e](i),n[_[e].cache]=i[_[e].cache],o=n._rgba=i._rgba,!1}),o.length?("0,0,0,0"===o.join()&&d.extend(o,B.transparent),n):B[s]}function H(t,e,i){return 6*(i=(i+1)%1)<1?t+(e-t)*i*6:2*i<1?e:3*i<2?t+(e-t)*(2/3-i)*6:t}y.style.cssText="background-color:rgba(1,1,1,.5)",b.rgba=-1o.mod/2?s+=o.mod:s-n>o.mod/2&&(s-=o.mod)),l[i]=M((n-s)*a+s,e)))}),this[e](l)},blend:function(t){if(1===this._rgba[3])return this;var e=this._rgba.slice(),i=e.pop(),s=m(t)._rgba;return m(d.map(e,function(t,e){return(1-i)*s[e]+i*t}))},toRgbaString:function(){var t="rgba(",e=d.map(this._rgba,function(t,e){return null!=t?t:2").addClass("ui-effects-wrapper").css({fontSize:"100%",background:"transparent",border:"none",margin:0,padding:0}),e={width:i.width(),height:i.height()},n=document.activeElement;try{n.id}catch(t){n=document.body}return i.wrap(t),i[0]!==n&&!V.contains(i[0],n)||V(n).trigger("focus"),t=i.parent(),"static"===i.css("position")?(t.css({position:"relative"}),i.css({position:"relative"})):(V.extend(s,{position:i.css("position"),zIndex:i.css("z-index")}),V.each(["top","left","bottom","right"],function(t,e){s[e]=i.css(e),isNaN(parseInt(s[e],10))&&(s[e]="auto")}),i.css({position:"relative",top:0,left:0,right:"auto",bottom:"auto"})),i.css(e),t.css(s).show()},removeWrapper:function(t){var e=document.activeElement;return t.parent().is(".ui-effects-wrapper")&&(t.parent().replaceWith(t),t[0]!==e&&!V.contains(t[0],e)||V(e).trigger("focus")),t}}),V.extend(V.effects,{version:"1.13.2",define:function(t,e,i){return i||(i=e,e="effect"),V.effects.effect[t]=i,V.effects.effect[t].mode=e,i},scaledDimensions:function(t,e,i){if(0===e)return{height:0,width:0,outerHeight:0,outerWidth:0};var s="horizontal"!==i?(e||100)/100:1,e="vertical"!==i?(e||100)/100:1;return{height:t.height()*e,width:t.width()*s,outerHeight:t.outerHeight()*e,outerWidth:t.outerWidth()*s}},clipToBox:function(t){return{width:t.clip.right-t.clip.left,height:t.clip.bottom-t.clip.top,left:t.clip.left,top:t.clip.top}},unshift:function(t,e,i){var s=t.queue();1").insertAfter(t).css({display:/^(inline|ruby)/.test(t.css("display"))?"inline-block":"block",visibility:"hidden",marginTop:t.css("marginTop"),marginBottom:t.css("marginBottom"),marginLeft:t.css("marginLeft"),marginRight:t.css("marginRight"),float:t.css("float")}).outerWidth(t.outerWidth()).outerHeight(t.outerHeight()).addClass("ui-effects-placeholder"),t.data(j+"placeholder",e)),t.css({position:i,left:s.left,top:s.top}),e},removePlaceholder:function(t){var e=j+"placeholder",i=t.data(e);i&&(i.remove(),t.removeData(e))},cleanUp:function(t){V.effects.restoreStyle(t),V.effects.removePlaceholder(t)},setTransition:function(s,t,n,o){return o=o||{},V.each(t,function(t,e){var i=s.cssUnit(e);0");l.appendTo("body").addClass(t.className).css({top:s.top-a,left:s.left-r,height:i.innerHeight(),width:i.innerWidth(),position:n?"fixed":"absolute"}).animate(o,t.duration,t.easing,function(){l.remove(),"function"==typeof e&&e()})}}),V.fx.step.clip=function(t){t.clipInit||(t.start=V(t.elem).cssClip(),"string"==typeof t.end&&(t.end=G(t.end,t.elem)),t.clipInit=!0),V(t.elem).cssClip({top:t.pos*(t.end.top-t.start.top)+t.start.top,right:t.pos*(t.end.right-t.start.right)+t.start.right,bottom:t.pos*(t.end.bottom-t.start.bottom)+t.start.bottom,left:t.pos*(t.end.left-t.start.left)+t.start.left})},Y={},V.each(["Quad","Cubic","Quart","Quint","Expo"],function(e,t){Y[t]=function(t){return Math.pow(t,e+2)}}),V.extend(Y,{Sine:function(t){return 1-Math.cos(t*Math.PI/2)},Circ:function(t){return 1-Math.sqrt(1-t*t)},Elastic:function(t){return 0===t||1===t?t:-Math.pow(2,8*(t-1))*Math.sin((80*(t-1)-7.5)*Math.PI/15)},Back:function(t){return t*t*(3*t-2)},Bounce:function(t){for(var e,i=4;t<((e=Math.pow(2,--i))-1)/11;);return 1/Math.pow(4,3-i)-7.5625*Math.pow((3*e-2)/22-t,2)}}),V.each(Y,function(t,e){V.easing["easeIn"+t]=e,V.easing["easeOut"+t]=function(t){return 1-e(1-t)},V.easing["easeInOut"+t]=function(t){return t<.5?e(2*t)/2:1-e(-2*t+2)/2}});y=V.effects,V.effects.define("blind","hide",function(t,e){var i={up:["bottom","top"],vertical:["bottom","top"],down:["top","bottom"],left:["right","left"],horizontal:["right","left"],right:["left","right"]},s=V(this),n=t.direction||"up",o=s.cssClip(),a={clip:V.extend({},o)},r=V.effects.createPlaceholder(s);a.clip[i[n][0]]=a.clip[i[n][1]],"show"===t.mode&&(s.cssClip(a.clip),r&&r.css(V.effects.clipToBox(a)),a.clip=o),r&&r.animate(V.effects.clipToBox(a),t.duration,t.easing),s.animate(a,{queue:!1,duration:t.duration,easing:t.easing,complete:e})}),V.effects.define("bounce",function(t,e){var i,s,n=V(this),o=t.mode,a="hide"===o,r="show"===o,l=t.direction||"up",h=t.distance,c=t.times||5,o=2*c+(r||a?1:0),u=t.duration/o,d=t.easing,p="up"===l||"down"===l?"top":"left",f="up"===l||"left"===l,g=0,t=n.queue().length;for(V.effects.createPlaceholder(n),l=n.css(p),h=h||n["top"==p?"outerHeight":"outerWidth"]()/3,r&&((s={opacity:1})[p]=l,n.css("opacity",0).css(p,f?2*-h:2*h).animate(s,u,d)),a&&(h/=Math.pow(2,c-1)),(s={})[p]=l;g").css({position:"absolute",visibility:"visible",left:-s*p,top:-i*f}).parent().addClass("ui-effects-explode").css({position:"absolute",overflow:"hidden",width:p,height:f,left:n+(u?a*p:0),top:o+(u?r*f:0),opacity:u?0:1}).animate({left:n+(u?0:a*p),top:o+(u?0:r*f),opacity:u?1:0},t.duration||500,t.easing,m)}),V.effects.define("fade","toggle",function(t,e){var i="show"===t.mode;V(this).css("opacity",i?0:1).animate({opacity:i?1:0},{queue:!1,duration:t.duration,easing:t.easing,complete:e})}),V.effects.define("fold","hide",function(e,t){var i=V(this),s=e.mode,n="show"===s,o="hide"===s,a=e.size||15,r=/([0-9]+)%/.exec(a),l=!!e.horizFirst?["right","bottom"]:["bottom","right"],h=e.duration/2,c=V.effects.createPlaceholder(i),u=i.cssClip(),d={clip:V.extend({},u)},p={clip:V.extend({},u)},f=[u[l[0]],u[l[1]]],s=i.queue().length;r&&(a=parseInt(r[1],10)/100*f[o?0:1]),d.clip[l[0]]=a,p.clip[l[0]]=a,p.clip[l[1]]=0,n&&(i.cssClip(p.clip),c&&c.css(V.effects.clipToBox(p)),p.clip=u),i.queue(function(t){c&&c.animate(V.effects.clipToBox(d),h,e.easing).animate(V.effects.clipToBox(p),h,e.easing),t()}).animate(d,h,e.easing).animate(p,h,e.easing).queue(t),V.effects.unshift(i,s,4)}),V.effects.define("highlight","show",function(t,e){var i=V(this),s={backgroundColor:i.css("backgroundColor")};"hide"===t.mode&&(s.opacity=0),V.effects.saveStyle(i),i.css({backgroundImage:"none",backgroundColor:t.color||"#ffff99"}).animate(s,{queue:!1,duration:t.duration,easing:t.easing,complete:e})}),V.effects.define("size",function(s,e){var n,i=V(this),t=["fontSize"],o=["borderTopWidth","borderBottomWidth","paddingTop","paddingBottom"],a=["borderLeftWidth","borderRightWidth","paddingLeft","paddingRight"],r=s.mode,l="effect"!==r,h=s.scale||"both",c=s.origin||["middle","center"],u=i.css("position"),d=i.position(),p=V.effects.scaledDimensions(i),f=s.from||p,g=s.to||V.effects.scaledDimensions(i,0);V.effects.createPlaceholder(i),"show"===r&&(r=f,f=g,g=r),n={from:{y:f.height/p.height,x:f.width/p.width},to:{y:g.height/p.height,x:g.width/p.width}},"box"!==h&&"both"!==h||(n.from.y!==n.to.y&&(f=V.effects.setTransition(i,o,n.from.y,f),g=V.effects.setTransition(i,o,n.to.y,g)),n.from.x!==n.to.x&&(f=V.effects.setTransition(i,a,n.from.x,f),g=V.effects.setTransition(i,a,n.to.x,g))),"content"!==h&&"both"!==h||n.from.y!==n.to.y&&(f=V.effects.setTransition(i,t,n.from.y,f),g=V.effects.setTransition(i,t,n.to.y,g)),c&&(c=V.effects.getBaseline(c,p),f.top=(p.outerHeight-f.outerHeight)*c.y+d.top,f.left=(p.outerWidth-f.outerWidth)*c.x+d.left,g.top=(p.outerHeight-g.outerHeight)*c.y+d.top,g.left=(p.outerWidth-g.outerWidth)*c.x+d.left),delete f.outerHeight,delete f.outerWidth,i.css(f),"content"!==h&&"both"!==h||(o=o.concat(["marginTop","marginBottom"]).concat(t),a=a.concat(["marginLeft","marginRight"]),i.find("*[width]").each(function(){var t=V(this),e=V.effects.scaledDimensions(t),i={height:e.height*n.from.y,width:e.width*n.from.x,outerHeight:e.outerHeight*n.from.y,outerWidth:e.outerWidth*n.from.x},e={height:e.height*n.to.y,width:e.width*n.to.x,outerHeight:e.height*n.to.y,outerWidth:e.width*n.to.x};n.from.y!==n.to.y&&(i=V.effects.setTransition(t,o,n.from.y,i),e=V.effects.setTransition(t,o,n.to.y,e)),n.from.x!==n.to.x&&(i=V.effects.setTransition(t,a,n.from.x,i),e=V.effects.setTransition(t,a,n.to.x,e)),l&&V.effects.saveStyle(t),t.css(i),t.animate(e,s.duration,s.easing,function(){l&&V.effects.restoreStyle(t)})})),i.animate(g,{queue:!1,duration:s.duration,easing:s.easing,complete:function(){var t=i.offset();0===g.opacity&&i.css("opacity",f.opacity),l||(i.css("position","static"===u?"relative":u).offset(t),V.effects.saveStyle(i)),e()}})}),V.effects.define("scale",function(t,e){var i=V(this),s=t.mode,s=parseInt(t.percent,10)||(0===parseInt(t.percent,10)||"effect"!==s?0:100),s=V.extend(!0,{from:V.effects.scaledDimensions(i),to:V.effects.scaledDimensions(i,s,t.direction||"both"),origin:t.origin||["middle","center"]},t);t.fade&&(s.from.opacity=1,s.to.opacity=0),V.effects.effect.size.call(this,s,e)}),V.effects.define("puff","hide",function(t,e){t=V.extend(!0,{},t,{fade:!0,percent:parseInt(t.percent,10)||150});V.effects.effect.scale.call(this,t,e)}),V.effects.define("pulsate","show",function(t,e){var i=V(this),s=t.mode,n="show"===s,o=2*(t.times||5)+(n||"hide"===s?1:0),a=t.duration/o,r=0,l=1,s=i.queue().length;for(!n&&i.is(":visible")||(i.css("opacity",0).show(),r=1);l li > :first-child").add(t.find("> :not(li)").even())},heightStyle:"auto",icons:{activeHeader:"ui-icon-triangle-1-s",header:"ui-icon-triangle-1-e"},activate:null,beforeActivate:null},hideProps:{borderTopWidth:"hide",borderBottomWidth:"hide",paddingTop:"hide",paddingBottom:"hide",height:"hide"},showProps:{borderTopWidth:"show",borderBottomWidth:"show",paddingTop:"show",paddingBottom:"show",height:"show"},_create:function(){var t=this.options;this.prevShow=this.prevHide=V(),this._addClass("ui-accordion","ui-widget ui-helper-reset"),this.element.attr("role","tablist"),t.collapsible||!1!==t.active&&null!=t.active||(t.active=0),this._processPanels(),t.active<0&&(t.active+=this.headers.length),this._refresh()},_getCreateEventData:function(){return{header:this.active,panel:this.active.length?this.active.next():V()}},_createIcons:function(){var t,e=this.options.icons;e&&(t=V(""),this._addClass(t,"ui-accordion-header-icon","ui-icon "+e.header),t.prependTo(this.headers),t=this.active.children(".ui-accordion-header-icon"),this._removeClass(t,e.header)._addClass(t,null,e.activeHeader)._addClass(this.headers,"ui-accordion-icons"))},_destroyIcons:function(){this._removeClass(this.headers,"ui-accordion-icons"),this.headers.children(".ui-accordion-header-icon").remove()},_destroy:function(){var t;this.element.removeAttr("role"),this.headers.removeAttr("role aria-expanded aria-selected aria-controls tabIndex").removeUniqueId(),this._destroyIcons(),t=this.headers.next().css("display","").removeAttr("role aria-hidden aria-labelledby").removeUniqueId(),"content"!==this.options.heightStyle&&t.css("height","")},_setOption:function(t,e){"active"!==t?("event"===t&&(this.options.event&&this._off(this.headers,this.options.event),this._setupEvents(e)),this._super(t,e),"collapsible"!==t||e||!1!==this.options.active||this._activate(0),"icons"===t&&(this._destroyIcons(),e&&this._createIcons())):this._activate(e)},_setOptionDisabled:function(t){this._super(t),this.element.attr("aria-disabled",t),this._toggleClass(null,"ui-state-disabled",!!t),this._toggleClass(this.headers.add(this.headers.next()),null,"ui-state-disabled",!!t)},_keydown:function(t){if(!t.altKey&&!t.ctrlKey){var e=V.ui.keyCode,i=this.headers.length,s=this.headers.index(t.target),n=!1;switch(t.keyCode){case e.RIGHT:case e.DOWN:n=this.headers[(s+1)%i];break;case e.LEFT:case e.UP:n=this.headers[(s-1+i)%i];break;case e.SPACE:case e.ENTER:this._eventHandler(t);break;case e.HOME:n=this.headers[0];break;case e.END:n=this.headers[i-1]}n&&(V(t.target).attr("tabIndex",-1),V(n).attr("tabIndex",0),V(n).trigger("focus"),t.preventDefault())}},_panelKeyDown:function(t){t.keyCode===V.ui.keyCode.UP&&t.ctrlKey&&V(t.currentTarget).prev().trigger("focus")},refresh:function(){var t=this.options;this._processPanels(),!1===t.active&&!0===t.collapsible||!this.headers.length?(t.active=!1,this.active=V()):!1===t.active?this._activate(0):this.active.length&&!V.contains(this.element[0],this.active[0])?this.headers.length===this.headers.find(".ui-state-disabled").length?(t.active=!1,this.active=V()):this._activate(Math.max(0,t.active-1)):t.active=this.headers.index(this.active),this._destroyIcons(),this._refresh()},_processPanels:function(){var t=this.headers,e=this.panels;"function"==typeof this.options.header?this.headers=this.options.header(this.element):this.headers=this.element.find(this.options.header),this._addClass(this.headers,"ui-accordion-header ui-accordion-header-collapsed","ui-state-default"),this.panels=this.headers.next().filter(":not(.ui-accordion-content-active)").hide(),this._addClass(this.panels,"ui-accordion-content","ui-helper-reset ui-widget-content"),e&&(this._off(t.not(this.headers)),this._off(e.not(this.panels)))},_refresh:function(){var i,t=this.options,e=t.heightStyle,s=this.element.parent();this.active=this._findActive(t.active),this._addClass(this.active,"ui-accordion-header-active","ui-state-active")._removeClass(this.active,"ui-accordion-header-collapsed"),this._addClass(this.active.next(),"ui-accordion-content-active"),this.active.next().show(),this.headers.attr("role","tab").each(function(){var t=V(this),e=t.uniqueId().attr("id"),i=t.next(),s=i.uniqueId().attr("id");t.attr("aria-controls",s),i.attr("aria-labelledby",e)}).next().attr("role","tabpanel"),this.headers.not(this.active).attr({"aria-selected":"false","aria-expanded":"false",tabIndex:-1}).next().attr({"aria-hidden":"true"}).hide(),this.active.length?this.active.attr({"aria-selected":"true","aria-expanded":"true",tabIndex:0}).next().attr({"aria-hidden":"false"}):this.headers.eq(0).attr("tabIndex",0),this._createIcons(),this._setupEvents(t.event),"fill"===e?(i=s.height(),this.element.siblings(":visible").each(function(){var t=V(this),e=t.css("position");"absolute"!==e&&"fixed"!==e&&(i-=t.outerHeight(!0))}),this.headers.each(function(){i-=V(this).outerHeight(!0)}),this.headers.next().each(function(){V(this).height(Math.max(0,i-V(this).innerHeight()+V(this).height()))}).css("overflow","auto")):"auto"===e&&(i=0,this.headers.next().each(function(){var t=V(this).is(":visible");t||V(this).show(),i=Math.max(i,V(this).css("height","").height()),t||V(this).hide()}).height(i))},_activate:function(t){t=this._findActive(t)[0];t!==this.active[0]&&(t=t||this.active[0],this._eventHandler({target:t,currentTarget:t,preventDefault:V.noop}))},_findActive:function(t){return"number"==typeof t?this.headers.eq(t):V()},_setupEvents:function(t){var i={keydown:"_keydown"};t&&V.each(t.split(" "),function(t,e){i[e]="_eventHandler"}),this._off(this.headers.add(this.headers.next())),this._on(this.headers,i),this._on(this.headers.next(),{keydown:"_panelKeyDown"}),this._hoverable(this.headers),this._focusable(this.headers)},_eventHandler:function(t){var e=this.options,i=this.active,s=V(t.currentTarget),n=s[0]===i[0],o=n&&e.collapsible,a=o?V():s.next(),r=i.next(),a={oldHeader:i,oldPanel:r,newHeader:o?V():s,newPanel:a};t.preventDefault(),n&&!e.collapsible||!1===this._trigger("beforeActivate",t,a)||(e.active=!o&&this.headers.index(s),this.active=n?V():s,this._toggle(a),this._removeClass(i,"ui-accordion-header-active","ui-state-active"),e.icons&&(i=i.children(".ui-accordion-header-icon"),this._removeClass(i,null,e.icons.activeHeader)._addClass(i,null,e.icons.header)),n||(this._removeClass(s,"ui-accordion-header-collapsed")._addClass(s,"ui-accordion-header-active","ui-state-active"),e.icons&&(n=s.children(".ui-accordion-header-icon"),this._removeClass(n,null,e.icons.header)._addClass(n,null,e.icons.activeHeader)),this._addClass(s.next(),"ui-accordion-content-active")))},_toggle:function(t){var e=t.newPanel,i=this.prevShow.length?this.prevShow:t.oldPanel;this.prevShow.add(this.prevHide).stop(!0,!0),this.prevShow=e,this.prevHide=i,this.options.animate?this._animate(e,i,t):(i.hide(),e.show(),this._toggleComplete(t)),i.attr({"aria-hidden":"true"}),i.prev().attr({"aria-selected":"false","aria-expanded":"false"}),e.length&&i.length?i.prev().attr({tabIndex:-1,"aria-expanded":"false"}):e.length&&this.headers.filter(function(){return 0===parseInt(V(this).attr("tabIndex"),10)}).attr("tabIndex",-1),e.attr("aria-hidden","false").prev().attr({"aria-selected":"true","aria-expanded":"true",tabIndex:0})},_animate:function(t,i,e){var s,n,o,a=this,r=0,l=t.css("box-sizing"),h=t.length&&(!i.length||t.index()",delay:300,options:{icons:{submenu:"ui-icon-caret-1-e"},items:"> *",menus:"ul",position:{my:"left top",at:"right top"},role:"menu",blur:null,focus:null,select:null},_create:function(){this.activeMenu=this.element,this.mouseHandled=!1,this.lastMousePosition={x:null,y:null},this.element.uniqueId().attr({role:this.options.role,tabIndex:0}),this._addClass("ui-menu","ui-widget ui-widget-content"),this._on({"mousedown .ui-menu-item":function(t){t.preventDefault(),this._activateItem(t)},"click .ui-menu-item":function(t){var e=V(t.target),i=V(V.ui.safeActiveElement(this.document[0]));!this.mouseHandled&&e.not(".ui-state-disabled").length&&(this.select(t),t.isPropagationStopped()||(this.mouseHandled=!0),e.has(".ui-menu").length?this.expand(t):!this.element.is(":focus")&&i.closest(".ui-menu").length&&(this.element.trigger("focus",[!0]),this.active&&1===this.active.parents(".ui-menu").length&&clearTimeout(this.timer)))},"mouseenter .ui-menu-item":"_activateItem","mousemove .ui-menu-item":"_activateItem",mouseleave:"collapseAll","mouseleave .ui-menu":"collapseAll",focus:function(t,e){var i=this.active||this._menuItems().first();e||this.focus(t,i)},blur:function(t){this._delay(function(){V.contains(this.element[0],V.ui.safeActiveElement(this.document[0]))||this.collapseAll(t)})},keydown:"_keydown"}),this.refresh(),this._on(this.document,{click:function(t){this._closeOnDocumentClick(t)&&this.collapseAll(t,!0),this.mouseHandled=!1}})},_activateItem:function(t){var e,i;this.previousFilter||t.clientX===this.lastMousePosition.x&&t.clientY===this.lastMousePosition.y||(this.lastMousePosition={x:t.clientX,y:t.clientY},e=V(t.target).closest(".ui-menu-item"),i=V(t.currentTarget),e[0]===i[0]&&(i.is(".ui-state-active")||(this._removeClass(i.siblings().children(".ui-state-active"),null,"ui-state-active"),this.focus(t,i))))},_destroy:function(){var t=this.element.find(".ui-menu-item").removeAttr("role aria-disabled").children(".ui-menu-item-wrapper").removeUniqueId().removeAttr("tabIndex role aria-haspopup");this.element.removeAttr("aria-activedescendant").find(".ui-menu").addBack().removeAttr("role aria-labelledby aria-expanded aria-hidden aria-disabled tabIndex").removeUniqueId().show(),t.children().each(function(){var t=V(this);t.data("ui-menu-submenu-caret")&&t.remove()})},_keydown:function(t){var e,i,s,n=!0;switch(t.keyCode){case V.ui.keyCode.PAGE_UP:this.previousPage(t);break;case V.ui.keyCode.PAGE_DOWN:this.nextPage(t);break;case V.ui.keyCode.HOME:this._move("first","first",t);break;case V.ui.keyCode.END:this._move("last","last",t);break;case V.ui.keyCode.UP:this.previous(t);break;case V.ui.keyCode.DOWN:this.next(t);break;case V.ui.keyCode.LEFT:this.collapse(t);break;case V.ui.keyCode.RIGHT:this.active&&!this.active.is(".ui-state-disabled")&&this.expand(t);break;case V.ui.keyCode.ENTER:case V.ui.keyCode.SPACE:this._activate(t);break;case V.ui.keyCode.ESCAPE:this.collapse(t);break;default:e=this.previousFilter||"",s=n=!1,i=96<=t.keyCode&&t.keyCode<=105?(t.keyCode-96).toString():String.fromCharCode(t.keyCode),clearTimeout(this.filterTimer),i===e?s=!0:i=e+i,e=this._filterMenuItems(i),(e=s&&-1!==e.index(this.active.next())?this.active.nextAll(".ui-menu-item"):e).length||(i=String.fromCharCode(t.keyCode),e=this._filterMenuItems(i)),e.length?(this.focus(t,e),this.previousFilter=i,this.filterTimer=this._delay(function(){delete this.previousFilter},1e3)):delete this.previousFilter}n&&t.preventDefault()},_activate:function(t){this.active&&!this.active.is(".ui-state-disabled")&&(this.active.children("[aria-haspopup='true']").length?this.expand(t):this.select(t))},refresh:function(){var t,e,s=this,n=this.options.icons.submenu,i=this.element.find(this.options.menus);this._toggleClass("ui-menu-icons",null,!!this.element.find(".ui-icon").length),e=i.filter(":not(.ui-menu)").hide().attr({role:this.options.role,"aria-hidden":"true","aria-expanded":"false"}).each(function(){var t=V(this),e=t.prev(),i=V("").data("ui-menu-submenu-caret",!0);s._addClass(i,"ui-menu-icon","ui-icon "+n),e.attr("aria-haspopup","true").prepend(i),t.attr("aria-labelledby",e.attr("id"))}),this._addClass(e,"ui-menu","ui-widget ui-widget-content ui-front"),(t=i.add(this.element).find(this.options.items)).not(".ui-menu-item").each(function(){var t=V(this);s._isDivider(t)&&s._addClass(t,"ui-menu-divider","ui-widget-content")}),i=(e=t.not(".ui-menu-item, .ui-menu-divider")).children().not(".ui-menu").uniqueId().attr({tabIndex:-1,role:this._itemRole()}),this._addClass(e,"ui-menu-item")._addClass(i,"ui-menu-item-wrapper"),t.filter(".ui-state-disabled").attr("aria-disabled","true"),this.active&&!V.contains(this.element[0],this.active[0])&&this.blur()},_itemRole:function(){return{menu:"menuitem",listbox:"option"}[this.options.role]},_setOption:function(t,e){var i;"icons"===t&&(i=this.element.find(".ui-menu-icon"),this._removeClass(i,null,this.options.icons.submenu)._addClass(i,null,e.submenu)),this._super(t,e)},_setOptionDisabled:function(t){this._super(t),this.element.attr("aria-disabled",String(t)),this._toggleClass(null,"ui-state-disabled",!!t)},focus:function(t,e){var i;this.blur(t,t&&"focus"===t.type),this._scrollIntoView(e),this.active=e.first(),i=this.active.children(".ui-menu-item-wrapper"),this._addClass(i,null,"ui-state-active"),this.options.role&&this.element.attr("aria-activedescendant",i.attr("id")),i=this.active.parent().closest(".ui-menu-item").children(".ui-menu-item-wrapper"),this._addClass(i,null,"ui-state-active"),t&&"keydown"===t.type?this._close():this.timer=this._delay(function(){this._close()},this.delay),(i=e.children(".ui-menu")).length&&t&&/^mouse/.test(t.type)&&this._startOpening(i),this.activeMenu=e.parent(),this._trigger("focus",t,{item:e})},_scrollIntoView:function(t){var e,i,s;this._hasScroll()&&(i=parseFloat(V.css(this.activeMenu[0],"borderTopWidth"))||0,s=parseFloat(V.css(this.activeMenu[0],"paddingTop"))||0,e=t.offset().top-this.activeMenu.offset().top-i-s,i=this.activeMenu.scrollTop(),s=this.activeMenu.height(),t=t.outerHeight(),e<0?this.activeMenu.scrollTop(i+e):s",options:{appendTo:null,autoFocus:!1,delay:300,minLength:1,position:{my:"left top",at:"left bottom",collision:"none"},source:null,change:null,close:null,focus:null,open:null,response:null,search:null,select:null},requestIndex:0,pending:0,liveRegionTimer:null,_create:function(){var i,s,n,t=this.element[0].nodeName.toLowerCase(),e="textarea"===t,t="input"===t;this.isMultiLine=e||!t&&this._isContentEditable(this.element),this.valueMethod=this.element[e||t?"val":"text"],this.isNewMenu=!0,this._addClass("ui-autocomplete-input"),this.element.attr("autocomplete","off"),this._on(this.element,{keydown:function(t){if(this.element.prop("readOnly"))s=n=i=!0;else{s=n=i=!1;var e=V.ui.keyCode;switch(t.keyCode){case e.PAGE_UP:i=!0,this._move("previousPage",t);break;case e.PAGE_DOWN:i=!0,this._move("nextPage",t);break;case e.UP:i=!0,this._keyEvent("previous",t);break;case e.DOWN:i=!0,this._keyEvent("next",t);break;case e.ENTER:this.menu.active&&(i=!0,t.preventDefault(),this.menu.select(t));break;case e.TAB:this.menu.active&&this.menu.select(t);break;case e.ESCAPE:this.menu.element.is(":visible")&&(this.isMultiLine||this._value(this.term),this.close(t),t.preventDefault());break;default:s=!0,this._searchTimeout(t)}}},keypress:function(t){if(i)return i=!1,void(this.isMultiLine&&!this.menu.element.is(":visible")||t.preventDefault());if(!s){var e=V.ui.keyCode;switch(t.keyCode){case e.PAGE_UP:this._move("previousPage",t);break;case e.PAGE_DOWN:this._move("nextPage",t);break;case e.UP:this._keyEvent("previous",t);break;case e.DOWN:this._keyEvent("next",t)}}},input:function(t){if(n)return n=!1,void t.preventDefault();this._searchTimeout(t)},focus:function(){this.selectedItem=null,this.previous=this._value()},blur:function(t){clearTimeout(this.searching),this.close(t),this._change(t)}}),this._initSource(),this.menu=V("