From 7f258d0cbea2636d15e976c4007a267523f339df Mon Sep 17 00:00:00 2001 From: Made NN Date: Wed, 10 Nov 2021 09:02:39 +0000 Subject: [PATCH 1/9] WIP --- Untitled.ipynb | 2348 ++++++++++++++++++++++ newlm/lm/elmo/lm_builder.py | 2 +- newlm/lm/elmo/modeling_elmo/elmo_head.py | 2 + 3 files changed, 2351 insertions(+), 1 deletion(-) create mode 100644 Untitled.ipynb diff --git a/Untitled.ipynb b/Untitled.ipynb new file mode 100644 index 0000000..ee6b896 --- /dev/null +++ b/Untitled.ipynb @@ -0,0 +1,2348 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "played-perfume", + "metadata": {}, + "outputs": [], + "source": [ + "from transformers import BertModel, BertConfig" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fatty-broadway", + "metadata": {}, + "outputs": [], + "source": [ + "class BertCausalModel(BertModel):\n", + " def __init__(self, config, add_pooling_layer=True):\n", + " super().__init__(config, add_pooling_layer)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "impressed-thompson", + "metadata": {}, + "outputs": [], + "source": [ + "bconfig = BertConfig()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "brown-canberra", + "metadata": {}, + "outputs": [], + "source": [ + "bconfig" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fuzzy-portuguese", + "metadata": {}, + "outputs": [], + "source": [ + "bmodel = BertCausalModel(bconfig)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "decent-healthcare", + "metadata": {}, + "outputs": [], + "source": [ + "bmodel.config.pad_token_id" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "treated-sight", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "coordinated-trailer", + "metadata": {}, + "outputs": [], + "source": [ + "import torch\n", + "\n", + "def get_sequence_lengths(pad_token_id, input_ids=None, inputs_embeds=None, **kwargs):\n", + " if input_ids is not None:\n", + " batch_size, sequence_length = input_ids.shape[:2]\n", + " else:\n", + " batch_size, sequence_length = inputs_embeds.shape[:2]\n", + "\n", + " assert (\n", + " pad_token_id is not None or batch_size == 1\n", + " ), \"Cannot handle batch sizes > 1 if no padding token is defined.\"\n", + " if pad_token_id is None:\n", + " sequence_lengths = -1\n", + " else:\n", + " if input_ids is not None:\n", + " sequence_lengths = torch.ne(input_ids, pad_token_id).sum(-1) - 1\n", + " else:\n", + " sequence_lengths = -1\n", + " logger.warning(\n", + " f\"{self.__class__.__name__} will not detect padding tokens in `inputs_embeds`. Results may be \"\n", + " f\"unexpected if using padding tokens in conjunction with `inputs_embeds.`\"\n", + " )\n", + "\n", + " return (batch_size, sequence_lengths)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "desperate-fellow", + "metadata": {}, + "outputs": [], + "source": [ + "bs, seqlen = get_sequence_lengths(0, batch['input_ids'])\n", + "bs, seqlen" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "decreased-logistics", + "metadata": {}, + "outputs": [], + "source": [ + "embed = batch['input_ids']\n", + "embed" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "forty-situation", + "metadata": {}, + "outputs": [], + "source": [ + "embed[range(bs), seqlen]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "adjusted-board", + "metadata": {}, + "outputs": [], + "source": [ + "from transformers import BertModel\n", + "from newlm.lm.bert.modeling_bert.bert_model import BertCausalModel\n", + "\n", + "bm = BertModel.from_pretrained(\"indobenchmark/indobert-base-p1\")\n", + "bcm = BertCausalModel.from_pretrained(\"indobenchmark/indobert-base-p1\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "oriented-truth", + "metadata": {}, + "outputs": [], + "source": [ + "bm" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "sticky-cambodia", + "metadata": {}, + "outputs": [], + "source": [ + "bcm" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "destroyed-particle", + "metadata": {}, + "outputs": [], + "source": [ + "from transformers import BertTokenizerFast\n", + "\n", + "tokenizer = BertTokenizerFast.from_pretrained(\"indobenchmark/indobert-base-p1\")\n", + "\n", + "batch = tokenizer(\n", + " [\"saya pergi ke pasar\", \"saya makan di warung\", \"pada hari senin ku turut ayah ke kota\"],\n", + " padding=True,\n", + " return_tensors=\"pt\"\n", + ")\n", + "batch" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "blond-shepherd", + "metadata": {}, + "outputs": [], + "source": [ + "out = bm(batch)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "extreme-russia", + "metadata": {}, + "outputs": [], + "source": [ + "out2 = bcm(**batch)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "worth-landing", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "killing-weekly", + "metadata": {}, + "outputs": [], + "source": [ + "from transformers import BertTokenizerFast\n", + "\n", + "tokenizer = BertTokenizerFast.from_pretrained(\"indobenchmark/indobert-base-p1\")\n", + "\n", + "batch = tokenizer(\n", + " [\"saya pergi ke pasar\", \"saya makan di warung\", \"pada hari senin ku turut ayah ke kota\"],\n", + " padding=True,\n", + " return_tensors=\"pt\"\n", + ")\n", + "batch" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "spread-debut", + "metadata": {}, + "outputs": [], + "source": [ + "from transformers import BertForSequenceClassification, BertTokenizerFast\n", + "from newlm.lm.bert.modeling_bert.bert_model import BertCausalModel, BertModelCausalForSequenceClassification\n", + "\n", + "model = BertForSequenceClassification.from_pretrained(\n", + " \"indobenchmark/indobert-base-p1\",\n", + " num_labels=3\n", + ")\n", + "model.bert = BertCausalModel(model.config)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "hazardous-plant", + "metadata": {}, + "outputs": [], + "source": [ + "model(**batch)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "incorporate-listing", + "metadata": {}, + "outputs": [], + "source": [ + "from newlm.lm.bert.modeling_bert.bert_model import BertModelCausalForSequenceClassification\n", + "\n", + "model2 = BertModelCausalForSequenceClassification.from_pretrained(\"indobenchmark/indobert-base-p1\", num_labels=3)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "august-horizontal", + "metadata": {}, + "outputs": [], + "source": [ + "model2(**batch)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "handed-correction", + "metadata": {}, + "outputs": [], + "source": [ + "model2(**batch)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dominican-heath", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "liquid-wellington", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "clean-sheriff", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "id": "vietnamese-trace", + "metadata": {}, + "source": [ + "## Sanity Check" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "grand-richards", + "metadata": {}, + "outputs": [], + "source": [ + "from newlm.lm.elmo.modeling_elmo.elmo_head import ELMOBertLMHeadModel\n", + "from newlm.lm.elmo.lm_builder import ELMOLMBuilder\n", + "from transformers import BertConfig" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "pleased-termination", + "metadata": {}, + "outputs": [], + "source": [ + "from newlm.utils.file_util import read_from_yaml\n", + "config_file = read_from_yaml('examples/configs/run.1-percent-bert-causal.yaml')" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "radical-cooperation", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'vocab_size': 30000,\n", + " 'hidden_size': 768,\n", + " 'num_attention_heads': 12,\n", + " 'num_hidden_layers': 12,\n", + " 'intermediate_size': 3072,\n", + " 'max_position_embeddings': 1024,\n", + " 'is_decoder': True}" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "config_file['lm']['model']['config']" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "compact-confirmation", + "metadata": {}, + "outputs": [], + "source": [ + "elmo_lm_builder = ELMOLMBuilder(\n", + " model_config = config_file['lm']['model']['config'],\n", + " tokenizer=\"bert-base-cased\",\n", + " model_type=\"bert-causal-elmo\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "greater-priest", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['Jews',\n", + " 'were',\n", + " 'indeed',\n", + " 'infected',\n", + " 'in',\n", + " 'numbers',\n", + " 'similar',\n", + " 'to',\n", + " 'their',\n", + " 'non',\n", + " '-',\n", + " 'Jewish',\n", + " 'neighbors',\n", + " 'Yet',\n", + " 'they',\n", + " 'were',\n", + " 'still',\n", + " 'made',\n", + " 's',\n", + " '##cape',\n", + " '##go',\n", + " '##ats',\n", + " '.']" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "elmo_lm_builder.tokenizer.tokenize(\"Jews were indeed infected in numbers similar to their non-Jewish neighbors Yet they were still made scapegoats.\")" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "twenty-likelihood", + "metadata": {}, + "outputs": [], + "source": [ + "with open('/mnt/data4/made_workspace/newlm-data/en.1-percent/text.txt', 'r+') as fr:\n", + " lines = fr.readlines()" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "fresh-trainer", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['\\n',\n", + " 'Typha albida is a plant species endemic to Afghanistan.\\n',\n", + " 'It grows in freshwater marshes.\\n',\n", + " '\\n',\n", + " 'Talczyn-Kolonia [ˈtalt͡ʂɨn\\xa0kɔˈlɔɲa] is a village in the administrative district of Gmina Kock, within Lubartów County, Lublin Voivodeship, in eastern Poland.\\n',\n", + " '\\n',\n", + " 'Niveocatharylla romieuxi is a moth in the Crambidae family.\\n',\n", + " 'It was described by Graziano Bassi in 1999. It is found in the Democratic Republic of the Congo.\\n',\n", + " '\\n',\n", + " 'Floyd Burton Jones (November 22, 1910, Cisco, Texas – April 15, 1999, Santa Barbara, California) was an American mathematician, active mainly in topology.\\n',\n", + " \"Jones's father was a pharmacist and local politician in Shackelford County, Texas.\\n\",\n", + " \"As the valedictorian of his high school class, Jones earned a Regents' Scholarship to The University of Texas, intending to study law eventually.\\n\",\n", + " 'Jones soon discovered that he had a poor memory for dates and history, and thus changed his major to chemistry.\\n',\n", + " 'Jones had the extraordinary good fortune to be taught freshman calculus by Robert Lee Moore, a founder of topology in the USA, a legendary mathematics teacher, and the inventor of the Moore method.\\n',\n", + " 'Jones went on to take more mathematics courses than required to be a chemist.\\n',\n", + " 'He displayed sufficient ability in those courses that when he graduated in 1932, Moore invited him to do a Ph.D.\\n',\n", + " 'in mathematics and offered him a part-time job as a math instructor.\\n',\n", + " \"Moore later supervised Jones's Ph.D.\\n\",\n", + " 'dissertation, completed in 1935. Jones then taught at the University of Texas for the next 15 years except during 1942-44, when he was a Research Associate at the Harvard Underwater Sound Laboratory, helping develop scanning sonar for the Navy.\\n',\n", + " 'In 1950, Jones moved to the University of North Carolina, where he eventually headed the Department of Mathematics.\\n',\n", + " 'From 1962 until his 1978 retirement, he was at the University of California at Riverside, where he helped launch the doctoral program in mathematics.\\n',\n", + " 'Over the course of his career, Jones published 67 articles and\\n',\n", + " 'dissertations. In 1987, he endowed a Chair in Topology at the University of California at Riverside.\\n',\n", + " 'Jones taught using a modified version of the Moore method.\\n',\n", + " 'He believed in \"learning by doing\" but unlike Moore, he incorporated textbooks into his courses.\\n',\n", + " 'In 1969, Louis McAuley wrote that \"the magical powers of Jones in the classroom-a master-who breathes the very life of mathematics into his students.\"\\n',\n", + " '\\n',\n", + " 'Paul Cameron (12 June 1904 – 31 December 1978) was a former Australian rules footballer who played with Carlton and South Melbourne in the Victorian Football League (VFL).\\n',\n", + " '\\n',\n", + " \"Abraham F. Sarmiento, Sr. (October 8, 1921 – October 3, 2010) was a Filipino jurist who served as an Associate Justice of the Supreme Court of the Philippines from 1987 to 1991. An active figure in the political opposition against the martial law government of President Ferdinand Marcos, he was appointed to the Court by Marcos' successor, President Corazon Aquino.\\n\",\n", + " 'Sarmiento was born in Santa Cruz, Ilocos Sur.\\n',\n", + " 'He completed his primary and secondary education in Laoag City, Ilocos Norte, graduating as the valedictorian of his high school class.\\n',\n", + " 'Upon the Japanese invasion of the Philippines during World War II, Sarmiento joined the USAFFE and the underground guerilla resistance against the Imperial Japanese Army.\\n',\n", + " 'After the war, Sarmiento completed his law studies at the University of the Philippines College of Law.\\n',\n", + " 'He was a member of the Alpha Phi Beta fraternity.\\n',\n", + " 'Sarmiento earned his Bachelor of Laws degree in 1949. In that year, he authored a biography on the murdered Chief Justice of the Supreme Court of the Philippines, José Abad Santos, entitled Jose Abad Santos: An Apotheosis.\\n',\n", + " 'Upon his admission to the Philippine Bar, Sarmiento entered into private practice.\\n',\n", + " 'In the 1950s, he formed a law partnership with Senators Gerardo Roxas and Justiniano Montano, maintaining his partnership with Roxas until 1967, when he established the Abraham F. Sarmiento Law Office.\\n',\n", + " 'Sarmiento successfully ran for a seat to the 1971 Constitutional Convention as a delegate from Cavite.\\n',\n", + " 'He was elected Vice-President of the Convention, which was tasked with the drafting of a new Constitution.\\n',\n", + " \"President Ferdinand Marcos declared martial law in September 1972. During this time, Sarmiento's eldest son, Ditto, was a student at the University of the Philippines.\\n\",\n", + " 'Ditto became the editor-in-chief of the official university newspaper, the Philippine Collegian, and under his leadership, the paper began publishing editorials critical of Marcos and martial law.\\n',\n", + " \"When Ditto was arrested in 1976, Sarmiento spent months negotiating with government officials to obtain his son's release.\\n\",\n", + " 'Ditto was released after seven months, but died within a year after his health was aggravated by the conditions of his imprisonment.\\n',\n", + " 'After the death of his son, Sarmiento was visibly active in human rights and anti-Marcos groups.\\n',\n", + " 'He co-founded the National Union for Democracy and Freedom, the Philippine Organization for Human Rights, and the National Union for Liberation.\\n',\n", + " 'He was among the founders of the United Nationalists Democratic Organizations (UNIDO), and served as its secretary-general from 1981 to 1983. From 1985 to 1987, Sarmiento served as the Chief Legal Counsel and Member of the Governing Council of the Lakas ng Bayan (LABAN).\\n',\n", + " 'It would be under the auspices of UNIDO and LABAN that Corazon Aquino and Salvador Laurel would form their presidential ticket that challenged Marcos in the 1986 presidential elections.\\n',\n", + " 'Sarmiento was also a member of the Executive Committee and National Council of the Bagong Alyansang Makabayan (BAYAN) from 1985 to 1987. In 1979, Sarmiento co-authored a book, The Road Back to Democracy, with former Philippine President Diosdado Macapagal and three others.\\n',\n", + " 'Later that year, he and Manuel Concordia published a book, Ang Demokrasya sa Pilipinas, which led to their arrest on charges of subversion and inciting to sedition.\\n',\n", + " 'They were later placed under house arrest.\\n',\n", + " 'Upon the assumption to the presidency of Corazon Aquino, following the 1986 People Power Revolution, Sarmiento was appointed to the Board of Directors of San Miguel Corporation, which was then under government sequestration.\\n',\n", + " 'In January 1987, President Aquino appointed Sarmiento as an Associate Justice of the Philippine Supreme Court.\\n',\n", + " 'He would serve on the High Court until he reached the compulsory retirement age of 70 in 1991. During his stint on the Court, Sarmiento held staunch civil libertarian views.\\n',\n", + " 'In People v. Nazario, 165 SCRA 186, the Court through Sarmiento acknowledged the void for vagueness rule as able to invalidate criminal statutes.\\n',\n", + " 'In Pita v. Court of Appeals, G.R.\\n',\n", + " 'No. 80806, 5 October 1989, 163 SCRA 386, he wrote for the Court that any restraint on the publication of purportedly obscene materials must satisfy the clear and present danger test.\\n',\n", + " 'In Salaw v. NLRC, G.R.\\n',\n", + " \"No. 90786, 27 September 1991, 202 SCRA 7, Sarmiento's opinion for the Court held that the dismissal of an employee in the private sector must be attended with procedural due process, a ruling which has since been reversed by the Court.\\n\",\n", + " 'At the same time, in PASE v. Drilon, G.R.\\n',\n", + " \"No. L-81958, 30 June 1988, 178 SCRA 362, Sarmiento's opinion for the Court upheld as a valid police power measure, the Philippine government's right to temporarily ban the deployment abroad of Filipino domestics and household workers. At the same time, Sarmiento dissented from the majority in some high-profile cases.\\n\",\n", + " \"In Marcos v. Manglapus, 178 SCRA 760, Sarmiento dissented from the majority which affirmed President Aquino's ban on the re-entry to the Philippines of Ferdinand Marcos.\\n\",\n", + " 'In Umil v. Ramos, 187 SCRA 311, Sarmiento published a strongly worded dissenting opinion to the majority opinion, which had held that there was no need to procure an arrest warrant to detain persons charged with the crimes of rebellion or subversion.\\n',\n", + " 'Sarmiento invoked the diminution of civil liberties during the Marcos administration, writing: The apprehensions in question chronicle in my mind the increasing pattern of arrests and detention in the country without the sanction of a judicial decree.\\n',\n", + " 'Four years ago at \"EDSA\", and many years before it, although with much fewer of us, we valiantly challenged a dictator and all the evils his regime had stood for: repression of civil liberties and trampling on of human rights.\\n',\n", + " 'We set up a popular government, restored its honored institutions, and crafted a democratic constitution that rests on the guideposts of peace and freedom.\\n',\n", + " \"I feel that with this Court's ruling, we have frittered away, by a stroke of the pen, what we had so painstakingly built in four years of democracy, and almost twenty years of struggle against tyranny.\\n\",\n", + " 'Since 2002, Sarmiento had served as a member of the Board of Regents of the University of the Philippines.\\n',\n", + " 'In 2004, he was among the candidates for appointment as President of the University of the Philippines.\\n',\n", + " 'However, his membership in the Board of Regents of the University of the Philippines was marked with controversy, when he protested the election of UP-Philippine General Hospital (UP-PGH) Director Jose Gonzalez, by questioning the legality of the vote of Student Regent Charisse Banez because of her pending application for residency, thus questioning her representation in the Board of Regents and finally replacing the PGH Director and ouster of the Student Regent.\\n',\n", + " 'But according to the records of his appointment by the President of the Republic of the Philippines, his term as Regent is already expired along with the 2 Malacanang Appointees.\\n',\n", + " 'It is speculated, that his decision to question the status of the Student Regent is politically motivated because the other nominees are close to President Gloria-Macapagal Arroyo and they will continue the private leasing of the Faculty Medical Arts building in the UP-Philippine General Hospital to the Daniel Mercado Medical Center, a private hospital owned by a UP alumnus in Tanauan, Batangas, while Dr. Jose Gonzales wants to revoke the agreement because, it will not benefit the poor patients of the hospital but it will jumpstart commercialization of the UP-PGH.\\n',\n", + " 'Now, his actions hound his reputation as a Martial Law activist.\\n',\n", + " 'Sarmiento was in fact appointed by former President Gloria Macapagal-Arroyo as a Regent until September 29, 2010. However, U.P.\\n',\n", + " 'President Emerlinda R. Roman renominated him as a U.P.\\n',\n", + " 'Regent to President Benigno S. Aquino III and this renomination remained pending even as Sarmiento died in Prague, Czech Republic on October 3, 2010. As of the time of his death, therefore, Sarmiento was still a sitting U.P.\\n',\n", + " 'Regent in a holdover capacity.\\n',\n", + " 'Sarmiento died due to apparent organ failure, while visiting Prague, Czech Republic, was reported in the Philippine media on October 4, 2010.\\n',\n", + " '\\n',\n", + " \"The 1937 World Table Tennis Championships – Women's Singles was the 11th edition of the women's singles championship. Ruth Aarons and Gertrude Pritzi competed in the final which ended in an anticlimax because both players were stopped from finishing the final under the new time limit rules and declared joint champions.\\n\",\n", + " '+ both players disqualified under time limit rules\\n',\n", + " '\\n',\n", + " 'Plaza de Armas of Trujillo is the main square where the Spanish foundation of Trujillo was made, in northern Peru.\\n',\n", + " 'It has been the principal locus of history in this city in the republic era.\\n',\n", + " 'It is located in the central zone of the Historic Centre of Trujillo.\\n',\n", + " 'In the streets that form this main square are located the buildings of Municipality Palace, the Cathedral, among others.\\n',\n", + " 'The Plaza de Armas of Trujillo is formed by the Pizarro, Independencia, Orbegoso and Almagro streets.\\n',\n", + " 'In this square, the proclamation of the independence of Trujillo\\n',\n", + " 'Martin de Estete began the design of the streets of Trujillo by order of Diego de Almagro in the end of the year of 1534, so he designed the structure and dimensions of the Plaza de Armas.\\n',\n", + " 'In 1841, Pedro de Mandalengoitia Sanz de Zarate donated to the square a fountain made of white stone that was built on his farm and mill in Santa Cruz de Carabamba.\\n',\n", + " 'On December 29, 1820, the Plaza de Armas was the scene of the proclamation of Independence of Trujillo by the Marquis of Torre Tagle.\\n',\n", + " 'In this square are located the cathedral of the city and mansions dating back to the Hispanic and republican periods.\\n',\n", + " 'On July 4, 1929, in the middle of this square, the Freedom Monument was opened, which represents the country\\'s independence process and according to \"commentators it represents the most precious of human beings, the love of freedom, the memory and recognition of the distinguished men who gave us independence\".\\n',\n", + " 'The statue was a work in Germany, the materials used are marble and copper, the sculptor was Edmund Moeller and it was armed by Henry Albrecht.\\n',\n", + " 'The Freedom Monument, located in the center of the Plaza de Armas of Trujillo (Peru) and work of sculptor Edmund Möeller, consists of three sections: the first is on a circular platform with pedestals, resting on a granite base, supporting the sculptures represent the art, science, trade and health.\\n',\n", + " 'The second consists of three statues robust.\\n',\n", + " 'A statue of a man who snorts, it is bent, symbolizing oppression or slavery.\\n',\n", + " \"A second statue's arms back, symbolizing the struggle for freedom.\\n\",\n", + " 'The third statue is a man with arms raised and hands making fist, symbolizing liberation.\\n',\n", + " 'Also in this body are located the next plates: the first recalling the proclamation of the independence\\n',\n", + " 'Francisco Pizarro Diego de Almagro Independencia Mariscal Orbegoso In the Plaza de Armas of Trujillo are realized important celebrations y ceremonies such as Corpus Christi in June, parade for Trujillo Marinera Festival in January, independence day of Trujillo in December 29 of every year, etc.\\n',\n", + " '\\n',\n", + " 'In mathematics, specifically transcendental number theory, the six exponentials theorem is a result that, given the right conditions on the exponents, guarantees the transcendence of at least one of a set of exponentials.\\n',\n", + " 'If x₁, x₂,..., xd are d complex numbers that are linearly independent over the rational numbers, and y₁, y₂,...,yₗ are l complex numbers that are also linearly independent over the rational numbers, and if dl\\xa0>\\xa0d\\xa0+\\xa0l, then at least one of the following dl numbers is transcendental: The most interesting case is when d\\xa0=\\xa03 and l\\xa0=\\xa02, in which case there are six exponentials, hence the name of the result.\\n',\n", + " 'They claimed that Carl Ludwig Siegel knew of a proof of this special case, but it is not recorded.\\n',\n", + " 'Using the special case they manage to prove that the ratio of consecutive colossally abundant numbers is always either a prime or a semiprime.\\n',\n", + " 'The theorem was first explicitly stated and proved in its complete form independently by Serge Lang and Kanakanahalli Ramachandra in the 1960s.\\n',\n", + " 'A stronger, related result is the five exponentials theorem, which is as follows.\\n',\n", + " 'Let x₁, x₂ and y₁, y₂ be two pairs of complex numbers, with each pair being linearly independent over the rational numbers, and let γ be a non-zero algebraic number.\\n',\n", + " 'Then at least one of the following five numbers is transcendental: This theorem implies the six exponentials theorem and in turn is implied by the as yet unproven four exponentials conjecture, which says that in fact one of the first four numbers on this list must be transcendental.\\n',\n", + " 'Another related result that implies both the six exponentials theorem and the five exponentials theorem is the sharp six exponentials theorem.\\n',\n", + " 'This theorem is as follows.\\n',\n", + " \"Let x₁, x₂, and x₃ be complex numbers that are linearly independent over the rational numbers, and let y₁ and y₂ be a pair of complex numbers that are linearly independent over the rational numbers, and suppose that βᵢⱼ are six algebraic numbers for 1\\xa0≤\\xa0i\\xa0≤\\xa03 and 1\\xa0≤\\xa0j\\xa0≤\\xa02 such that the following six numbers are algebraic: Then xᵢ\\xa0yⱼ\\xa0=\\xa0βᵢⱼ for 1\\xa0≤\\xa0i\\xa0≤\\xa03 and 1\\xa0≤\\xa0j\\xa0≤\\xa02. The six exponentials theorem then follows by setting βᵢⱼ\\xa0=\\xa00 for every i and j, while the five exponentials theorem follows by setting x₃\\xa0=\\xa0γ/x₁ and using Baker's theorem to ensure that the xᵢ are linearly independent.\\n\",\n", + " 'There is a sharp version of the five exponentials theorem as well, although it as yet unproven so is known as the sharp five exponentials conjecture.\\n',\n", + " 'This conjecture implies both the sharp six exponentials theorem and the five exponentials theorem, and is stated as follows.\\n',\n", + " 'Let x₁, x₂ and y₁, y₂ be two pairs of complex numbers, with each pair being linearly independent over the rational numbers, and let α, β₁₁, β₁₂, β₂₁, β₂₂, and γ be six algebraic numbers with γ\\xa0≠\\xa00 such that the following five numbers are algebraic: Then xᵢ\\xa0yⱼ\\xa0=\\xa0βᵢⱼ for 1\\xa0≤\\xa0i, j\\xa0≤\\xa02 and γx₂\\xa0=\\xa0αx₁.\\n',\n", + " \"A consequence of this conjecture that isn't currently known would be the transcendence of eπ², by setting x₁\\xa0=\\xa0y₁\\xa0=\\xa0β₁₁\\xa0=\\xa01, x₂\\xa0=\\xa0y₂\\xa0=\\xa0iπ, and all the other values in the statement to be zero.\\n\",\n", + " 'A further strengthening of the theorems and conjectures in this area are the strong versions.\\n',\n", + " 'The strong six exponentials theorem is a result proved by Damien Roy that implies the sharp six exponentials theorem.\\n',\n", + " 'This result concerns the vector space over the algebraic numbers generated by 1 and all logarithms of algebraic numbers, denoted here as L∗.\\n',\n", + " 'So L∗ is the set of all complex numbers of the form for some n\\xa0≥\\xa00, where all the βᵢ and αᵢ are algebraic and every branch of the logarithm is considered.\\n',\n", + " 'The strong six exponentials theorem then says that if x₁, x₂, and x₃ are complex numbers that are linearly independent over the algebraic numbers, and if y₁ and y₂ are a pair of complex numbers that are also linearly independent over the algebraic numbers then at least one of the six numbers xᵢ\\xa0yⱼ for 1\\xa0≤\\xa0i\\xa0≤\\xa03 and 1\\xa0≤\\xa0j\\xa0≤\\xa02 is not in L∗.\\n',\n", + " 'This is stronger than the standard six exponentials theorem which says that one of these six numbers is not simply the logarithm of\\n',\n", + " 'There is also a strong five exponentials conjecture formulated by Michel Waldschmidt It would imply both, the strong six exponentials theorem and the sharp five exponentials conjecture.\\n',\n", + " \"This conjecture claims that if x₁, x₂ and y₁, y₂ are two pairs of complex numbers, with each pair being linearly independent over the algebraic numbers, then at least one of the following five numbers is not in L∗: All the above conjectures and theorems are consequences of the unproven extension of Baker's theorem, that logarithms of algebraic numbers that are linearly independent over the rational numbers are automatically algebraically independent too.\\n\",\n", + " 'The diagram on the right shows the logical implications between all these results.\\n',\n", + " 'The exponential function eᶻ uniformizes the exponential map of the multiplicative group Gₘ.\\n',\n", + " 'Therefore, we can reformulate the six exponential theorem more abstractly as follows: Let G = Gₘ × Gₘ and take u\\xa0: C → G(C) to be a non-zero complex-analytic group homomorphism.\\n',\n", + " 'Define L to be the set of complex numbers l for which u(l) is an algebraic point of G. If a minimal generating set of L over Q has more than two elements then the image u(C) is an algebraic subgroup of G(C).\\n',\n", + " '(In order to derive the classical statement, set u(z) =(e ʸ¹ ᶻ; e ʸ² ᶻ) and note that Qx₁ + Qx₂ + Qx₃ is a subset of L).\\n',\n", + " 'In this way, the statement of the six exponentials theorem can be generalized to an arbitrary commutative group variety G over the field of algebraic numbers.\\n',\n", + " 'This generalized six exponential conjecture, however, seems out of scope at the current state of transcendental number theory.\\n',\n", + " 'For the special, but interesting cases G = Gₘ × E and G = E × E′, where E, E′ are elliptic curves over the field of algebraic numbers, results towards the generalized six exponential conjecture were proven by Aleksander Momot.\\n',\n", + " 'These results involve the exponential function eᶻ and a Weierstrass function resp.\\n',\n", + " 'two Weierstrass functions with algebraic invariants , instead of the two exponential functions in the classical statement.\\n',\n", + " 'Let G = Gₘ × E and suppose E is not isogenous to a curve over a real field and that u(C) is not an algebraic subgroup of G(C).\\n',\n", + " 'Then L is generated over Q either by two elements x₁, x₂, or three elements x₁, x₂, x₃ which are not all contained in a real line Rc, where c is a non-zero complex number.\\n',\n", + " 'A similar result is shown for G = E × E′.\\n',\n", + " '\\n',\n", + " \"The 1974 October Revolution Parade was a parade on Red Square dedicated to the 57th anniversary of the October Revolution on November 7th 1974. It was inspected by the Minister of Defense and Marshal of the Soviet Union Andrei Grechko who also made a speech to the Soviet People on Red Square on the grandstand of Lenin's Mausoleum.\\n\",\n", + " 'The Parade Commander is the Commander of the Moscow Garrison Colonel General Vladimir Govorov.\\n',\n", + " \"Also on Lenin's Mausoleum is the General Secretary of the Communist Party of the Soviet Union Leonid Brezhnev and the President Nikolai Podgorny.\\n\",\n", + " 'This Parade is the last to feature heavy nuclear missiles such as Ballistic Missiles.\\n',\n", + " 'Due to the rain that took place that day the parade of workers and athletes after the parade finale was cancelled.Although the demonstrations on Red Square was Cancelled the demonstrations on Palace Square in Leningrad continued following the parade there.\\n',\n", + " '\\n',\n", + " 'East Lake Weir is an unincorporated community in eastern Marion County, Florida, United States, on the east shore of Lake Weir.\\n',\n", + " 'The community is part of the Ocala Metropolitan Statistical Area.\\n',\n", + " 'East Lake Weir is located at 29°01′12″N 81°54′30″W.\\n',\n", + " '\\n',\n", + " 'Point Possession is an unincorporated community on the Kenai Peninsula in Alaska.\\n',\n", + " \"As of April 1, 2010, the village's population was 3. Since 2009 Point Possession has been owned by the federal government and has been made a part of the Kenai National Wildlife Refuge.\\n\",\n", + " '\\n',\n", + " 'Armin van Buuren: In The Mix is a music video game for the Wii console.\\n',\n", + " 'It was announced in May 2009 and was developed by TransGaming Technologies of Canada and published by Foreign Media Games (now Easy Interactive).\\n',\n", + " 'The game was released in the United Kingdom on 12 November 2010.\\n',\n", + " '\\n',\n", + " 'Adenylyl cyclase type 6 is an enzyme that in humans is encoded by the ADCY6 gene.\\n',\n", + " 'This gene encodes adenylyl cyclase 6, which is a membrane-associated enzyme and catalyzes the formation of the secondary messenger cyclic adenosine monophosphate (cAMP).\\n',\n", + " 'The expression of this gene is found in normal thyroid and brain tissues, as well as some tumors;\\n',\n", + " 'and its expression is significantly higher in one hyperfunctioning thyroid tumor than in normal thyroid tissue.\\n',\n", + " 'Alternative splicing generates 2 transcript variants.\\n',\n", + " '\\n',\n", + " 'Millennium Park, located at the historic intersection of Park Row and Broadway is a Greenstreet site designed to replicate a forest amid a busy intersection in Downtown Manhattan. Located to the south of City Hall Park, it is used as a bus turnaround loop, formerly a paved median transformed into a green space at the turn of the 21st century.\\n',\n", + " 'The park was dedicated in November 2000, described by Mayor Rudolph W. Giuliani as “a final gift from the 20th century to New Yorkers of the 21st.” With its completion date in mind, it was named Millennium Park.\\n',\n", + " 'At its southern tip is the David Rockefeller Clock, dedicated by the Downtown Alliance in honor of longtime Chase Manhattan Bank executive David Rockefeller.\\n',\n", + " 'The youngest grandson of John D. Rockefeller, he was instrumental in maintaining the vitality of downtown Manhattan as a business district at a time when many businesses were leaving urban centers for suburban settings.\\n',\n", + " 'Rockefeller’s contributions to the neighborhood includes the preservation of South Street Seaport, and the construction of World Trade Center and Battery Park City.\\n',\n", + " 'Keeping in theme with City Hall Park, the 3,500-pound clock was designed in the late 19th-century style to match the fountain and light fixtures of the Victorian period.\\n',\n", + " '\\n',\n", + " 'A total solar eclipse occurred on August 1, 2008. A solar eclipse occurs when the Moon passes between Earth and the Sun, thereby totally or partly obscuring the image of the Sun for a viewer on Earth.\\n',\n", + " \"A total solar eclipse occurs when the Moon's apparent diameter is larger than the Sun's, blocking all direct sunlight, turning day into darkness.\\n\",\n", + " \"Totality occurs in a narrow path across Earth's surface, with the partial solar eclipse visible over a surrounding region thousands of kilometres wide.\\n\",\n", + " 'It had a magnitude of 1.0394 that was visible from a narrow corridor through northern Canada (Nunavut), Greenland, central Russia, eastern Kazakhstan, western Mongolia and China.\\n',\n", + " 'Occurring north of the arctic circle, it belonged to the so-called midnight sun eclipses.\\n',\n", + " 'The largest city in the path of the eclipse was Novosibirsk in Russia.\\n',\n", + " \"The total eclipse lasted for 2 minutes, and covered 0.4% of the Earth's surface in a 10,200\\xa0km long path.\\n\",\n", + " \"It was the 47th eclipse of the 126th Saros cycle, which began with a partial eclipse on March 10, 1179 and will conclude with a partial eclipse on May 3, 2459. A partial eclipse could be seen from the much broader path of the Moon's penumbra, including northeastern North America and most of Europe and Asia.\\n\",\n", + " 'It was described by observers as \"special for its colours around the horizon. There were wonderful oranges and reds all around, the clouds lit up, some dark in silhouette, some golden, glowing yellowy-orange in the distance. You could see the shadow approaching against the clouds and then rushing away as it left.\"\\n',\n", + " 'Animated path The eclipse began in the far north of Canada in Nunavut at 09:21 UT, the zone of totality being 206\\xa0km wide, and lasting for 1 minute 30 seconds.\\n',\n", + " 'The path of the eclipse then headed north-east, crossing over northern Greenland and reaching the northernmost latitude of 83° 47′ at 09:38 UT before dipping down into Russia.\\n',\n", + " 'The path of totality touched the northeast corner of Kvitøya, an uninhabited Norwegian island in the Svalbard archipelago, at 09:47 UT.\\n',\n", + " 'The eclipse reached the Russian mainland at 10:10 UT, with a path 232\\xa0km wide and a duration of 2 minutes 26 seconds.\\n',\n", + " 'The greatest eclipse occurred shortly after, at 10:21:07 UT at coordinates 65°39′N 72°18′E (close to Nadym), when the path was 237\\xa0km wide, and the duration was 2 minutes 27 seconds. Cities in the path of the total eclipse included Megion, Nizhnevartovsk, Strezhevoy, Novosibirsk and Barnaul.\\n',\n", + " 'Around 10,000 tourists were present in Novosibirsk, the largest city to experience the eclipse.\\n',\n", + " 'The path of the eclipse then moved south-east, crossing into Mongolia and just clipping Kazakhstan at around 10:58 UT.\\n',\n", + " 'The path here was 252\\xa0km wide, but the duration was decreased to 2 minutes 10 seconds.\\n',\n", + " 'The path then ran down the China-Mongolia border, ending in China at 11:18 UT, with an eclipse lasting 1 minute 27 seconds at sunset.\\n',\n", + " 'The total eclipse finished at 11:21 UT.\\n',\n", + " 'The total eclipse passed over Yiwu County, Jiuquan and Xi’an.\\n',\n", + " 'Around 10,000 people were gathered to watch the eclipse in Yiwu.\\n',\n", + " \"A partial eclipse was seen from the much broader path of the Moon's penumbra, including the north east coast of North America and most of Europe and Asia.\\n\",\n", + " 'In London, England, the partial eclipse began at 09:33 BST, with a maximum eclipse of 12% at 10:18 BST, before concluding at 11:05 BST. At Edinburgh the partial eclipse was 23.5% of the sun, whilst it was 36% in Lerwick in the Shetland Isles.\\n',\n", + " 'German charter airline LTU, now trading as Air Berlin, operated a special flight from Düsseldorf to the North Pole to observe the eclipse.\\n',\n", + " 'Flight number LT 1111 spent over 11 hours in the air, returning to base at 6pm after flying a planeload of eclipse chasers, scientists, journalists and TV crews to watch the celestial event.\\n',\n", + " 'The route also included a low-level sightseeing tour of Svalbard before the eclipse and the magnetic pole afterwards.\\n',\n", + " \"Each member in a semester series of solar eclipses repeats approximately every 177 days and 4 hours (a semester) at alternating nodes of the Moon's orbit.\\n\",\n", + " 'It is a part of Saros cycle 126, repeating every 18 years, 11 days, containing 71 events.\\n',\n", + " 'The series started with partial solar eclipse on March 10, 1179. It contains annular eclipses from June 4, 1323 through April 4, 1810 and hybrid eclipses from April 14, 1828 through May 6, 1864. It contains total eclipses from May 17, 1882 through August 23, 2044. The series ends at member 72 as a partial eclipse on May 3, 2459. The longest duration of central eclipse (annular or total) was 5 minutes, 46 seconds of annularity on November 22, 1593. The longest duration of totality was 2 minutes, 36 seconds on July 10, 1972.\\n',\n", + " '\\n',\n", + " 'The Western Canada Soccer League was a soccer league in Canada.\\n',\n", + " 'The league operated for eight seasons over a nine-year period from 1963 to 1971 and eventually operated across nine cities in four Canadian provinces and one American state.\\n',\n", + " 'In the 1960s, the Western Canada Soccer League was one of four major leagues in Canadian soccer alongside the Pacific Coast League, the National Soccer League of Ontario/Quebec, and the Eastern Canada Professional Soccer League.\\n',\n", + " 'In 1963, the league featured 10 clubs across five cities in Alberta and Saskatchewan. In 1967, the league expanded to Manitoba, thus becoming the first Canadian soccer league to operate a league schedule across three provinces.\\n',\n", + " 'The league later added teams in British Columbia and, for one season only, a team in Washington, United States.\\n',\n", + " 'Along with a regular season schedule that usually operated from May to August, the Molson Cup playoffs took place in August or September.\\n',\n", + " \"Teams also participated in other competitions including their provincial playoffs and Canada Soccer's national The Challenge Trophy competition. In 1967, in the weeks after winning the league's playoff Molson Cup, the Calgary Buffalo Kickers became the first and only Western Canada Soccer League team to reach the national final (a 2-1 loss to Toronto Ballymena).\\n\",\n", + " 'The Western Canada Soccer League was co-founded by Alec Castleton and Sam Donaghey, both of whom were involved with the Alberta Soccer Association.\\n',\n", + " \"The league's inaugural match was played on May 4, 1963 in Saskatoon, with opening weekend matches in Calgary and Edmonton postponed because of inclement weather. The price of admission to that opening match at Griffiths Stadium was 50 cents per ticket.\\n\",\n", + " 'Saskatoon Concordia won that first match 7-1 over Saskatoon Hollandia, with Adolf Becker the four-goal hero.\\n',\n", + " \"The most famous player to play for a Western Canada League team, albeit not in league action, was Stanley Matthews who dressed for Calgary ATCO in two exhibition matches in July 1964. Matthews joined the Calgary squad under player-manager Les Medley, who was one of Matthews' former England teammates.\\n\",\n", + " 'The league featured a single table from 1963 to 1969 and then separate Coast and Prairie divisions in 1970 and 1971. There was also a separate playoff (Cup) competition from 1964 to 1970. Regina Concordia SC were the first champions in 1963, but they were upset in the Molson Cup by 10th place Lethbridge Hungária on 24 August 1963. 1963 Regina Concordia SC 1964 Regina Concordia SC 1966 Edmonton Canadians 1967 Edmonton Victoria Canadians 1968 Calgary Buffalo Kickers 1969 Vancouver Spartans 1963 Lethbridge Hungária SC def. Regina Concordia SC 1964 Regina Concordia SC def.\\n',\n", + " 'Lethbridge Hungária SC 1966 Calgary Buffalo Kickers def.\\n',\n", + " 'Regina Concordia SC 1967 Calgary Buffalo Kickers def.\\n',\n", + " 'Saskatoon City SC 1968 (finalists Calgary Buffalo Kickers v Edmonton Victoria Canadians) 1969 Vancouver Spartans def.\\n',\n", + " 'Regina Concordia SC 1970 Victoria Royals def.\\n',\n", + " 'Regina Concordia SC Calgary ATCO (1964) Calgary Kickers (1966-1969) Calgary Croatia (1963-64) Edmonton Canadians / Edmonton Victoria Canadians (1966- 1969) Edmonton City (1970) / Edmonton Eagles (1971) Edmonton Edelweiss (1963-1964) Edmonton Excelsior (1963-1964) Edmonton Hungária (1963-1964) Edmonton Rangers (1963-1964) Lethbridge Hungária SC (1963-1964) Regina Concordia SC (1963-1964, 1966-1971) Regina Internationals (1963) Saskatoon City SC (1966-1969) Saskatoon Concordia (1963-1964) Saskatoon Hollandia (1963) Saskatoon Internationals (1964) Seattle Sea-Tac (1970) Vancouver Cougars (1970) Vancouver Spartans (1969-1971) Victoria Royals (1970-1971) Winnipeg International FC (1967-1968) Winnipeg Blues (1969-1971) Eleven players from the Western Canada Soccer League have since been inducted in the Canada Soccer Hall of Fame as honoured players.\\n',\n", + " 'Bob Bolitho (1970) Tony Chursky (1971) Neil Ellett (1970) Glen Johnson (1969 to 1971) Victor Kodelja (1971) Bob Lenarduzzi (1970 and 1971) Sam Lenarduzzi (1970) Buzz Parsons (1969 and 1970) Ken Pears (1969) Brian Robinson (1970) Bruce Wilson (1970)\\n',\n", + " '\\n',\n", + " 'iRadio is a regional radio station in the Republic of Ireland that broadcasts to the north-east, midlands, north-west and west of the state.\\n',\n", + " 'The station is one of four regional youth oriented stations that were licensed by the Broadcasting Authority of Ireland to challenge the current duopoly in the 15 to 34 age bracket for those outside Dublin by national stations RTÉ 2fm and Today FM.\\n',\n", + " 'It was created by the merger of i102-104FM and i105-107FM in 2011. The station is based in Athlone and broadcasts to 15 counties between the frequencies of 102.1 MHz and 104.4 MHz (96.9 Donegal) in the north-west and 104.7 MHz and 106.7 MHz in the north-east, as well as offering an online stream.\\n',\n", + " '\\n',\n", + " 'Yves Benoist is a French mathematician, known for his work on group dynamics on homogeneous spaces.\\n',\n", + " 'He is currently a Director of Research (Directeur de Recherche) of CNRS at the University of Paris-Sud.\\n',\n", + " 'In 1990 Benoist proved a long open conjecture with Patrick Foulon and François Labourie about Anosov flows on compact, negatively curved manifolds.\\n',\n", + " 'In the 2000s he wrote a series of papers on the divisible convex sets in projective space and periodic tilings by such sets.\\n',\n", + " 'In 2011 he was awarded the Clay Research Award along with his former doctoral student, Jean-François Quint.\\n',\n", + " 'The prize citation highlighted their work on stationary measures and closed orbits for non-abelian group actions on homogeneous spaces, solving a long-standing conjecture of Hillel Furstenberg.\\n',\n", + " 'They showed that in homogeneous spaces of finite volume, orbits of a Zariski dense subgroup of a semisimple group equidistribute towards algebraic measures.\\n',\n", + " 'He gave the 2012 Takagi Lectures in Kyoto at the Research Institute for Mathematical Sciences (RIMS).\\n',\n", + " 'In 2014 he was an Invited speaker at the International Congress of Mathematicians in Seoul.\\n',\n", + " 'With J.-F.\\n',\n", + " 'Quint: Mesures stationaires et fermés invariants of homogeneous espaces.\\n',\n", + " 'Annals of Mathematics, vol 174, 2011, pp 1111–1162. With J.-F.\\n',\n", + " 'Quint: Random walks on finite volume homogeneous spaces.\\n',\n", + " 'Inventiones Mathematicae, vol 187, 2012, pp 37–59.\\n',\n", + " 'Convexes divisibles I: Algebraic groups and arithmetic, 339–374, Tata Inst Fund.. Res., Mumbai in 2004. Convexes divisibles II.\\n',\n", + " 'Duke Math J. 120 (2003), No 1, 97–120.\\n',\n", + " 'Convexes divisibles III: Ann.\\n',\n", + " 'Sci.\\n',\n", + " 'École Norm. Sup.\\n',\n", + " '(4) 38 (2005), no.\\n',\n", + " '5, 793–832.\\n',\n", + " 'Convexes divisibles IV: Invent.\\n',\n", + " 'Math.\\n',\n", + " '164 (2006), no.\\n',\n", + " '2, 249–278.\\n',\n", + " 'Propriétés\\n',\n", + " 'Geom.\\n',\n", + " 'Funct.\\n',\n", + " 'Anal.\\n',\n", + " '7 (1997), no.\\n',\n", + " '1, 1–47.\\n',\n", + " 'Actions propres sur les espaces homogeneous réductifs.\\n',\n", + " 'Ann.\\n',\n", + " 'of Math.\\n',\n", + " '(2) 144 (1996), no. 2, 315–347.\\n',\n", + " \"With P. Foulon, F. Labourie: Flots d'Anosov à distributions stable et instable différentiables.\\n\",\n", + " 'J. Amer.\\n',\n", + " 'Math.\\n',\n", + " 'Soc.\\n',\n", + " '5 (1992), no.\\n',\n", + " '1, 33–74.\\n',\n", + " \"Jean-François Quint: Convexes divisibles, d'après Yves Benoist.\\n\",\n", + " \"Séminaire Bourbaki, June 2008. François Ledrappier: Mesures sur les espaces station aires homogeneous, d'après Yves Benoist et Jean-François Quint Séminaire Bourbaki, 2012.\\n\",\n", + " '\\n',\n", + " 'Doris Baaten (born April 12, 1956 in Maastricht, Limburg) is a Dutch voice actress.\\n',\n", + " 'Baaten performed various female voices on Sesamstraat, the Dutch co-production of Sesame Street.\\n',\n", + " 'She also provided the voice for female roles, such as Countess Dahling Von Dahling and Deena, and was featured in musical numbers during the 1980s. She also performed in a Dutch version of the musical Chicago.\\n',\n", + " '\\n',\n", + " 'Mahonia is a genus of about 70 species of evergreen shrubs and rarely small trees in the family Berberidaceae, native to eastern Asia, the Himalaya, North and Central America.\\n',\n", + " 'They are closely related to the genus Berberis and botanists disagree on whether to recognize a separate Mahonia.\\n',\n", + " 'Some authorities argue Mahonia should be included in Berberis because several species in both genera are able to hybridize, and because when the two genera are looked at as a whole, no consistent morphological separation exists except simple versus compound leaves.\\n',\n", + " 'However, recent DNA-based phylogenetic studies support recognition of Mahonia, though after the removal of several species into the newly-described genera Alloberberis (formerly Mahonia section Horridae) and Moranothamnus (formerly Mahonia claireae).\\n',\n", + " 'Mahonia species typically have large, pinnate leaves 10–50\\xa0cm (3.9–19.7\\xa0in) long with five to 15 leaflets, and flowers in racemes which are 5–20\\xa0cm (2.0–7.9\\xa0in) long.\\n',\n", + " 'The genus name Mahonia honors the Philadelphia horticulturist Bernard McMahon, who introduced the plant from materials collected by the Lewis and Clark expedition.\\n',\n", + " 'The type species of the genus is Mahonia aquifolium, (Oregon-grape) from the Pacific coast of North America.\\n',\n", + " 'Several species are popular garden shrubs, grown for their ornamental, often spiny, evergreen foliage, yellow (or rarely red) flowers in autumn, winter and early spring, and blue-black berries.\\n',\n", + " 'The flowers are borne in terminal clusters or spreading racemes, and may be among the earliest flowers to appear in the growing season.\\n',\n", + " 'The berries are edible, and rich in vitamin C, though with a very sharp flavor.\\n',\n", + " 'Although edible, the plants contain berberine, a compound found in many Berberis and Mahonia species, which can cause vomiting, lowered blood pressure, reduced heart rate, lethargy, and other ill effects when consumed in large quantities.\\n',\n", + " 'The following list includes all currently recognized species of the genus Mahonia as accepted by Tropicos, Missouri Botanical Garden as of February 2016, sorted alphabetically.\\n',\n", + " 'For each, binomial name is followed by author citation. Mahonia aquifolium Mahonia bealei (Fortune) Carrière Mahonia bodinieri Gagnep.\\n',\n", + " 'Mahonia bracteolata Takeda Mahonia breviracema Y.S.\\n',\n", + " 'Wang & P.G.\\n',\n", + " 'Xiao Mahonia cardiophylla T.S.\\n',\n", + " 'Ying & Boufford Mahonia decipiens C.K.\\n',\n", + " 'Schneid.\\n',\n", + " 'Mahonia duclouxiana Gagnep.\\n',\n", + " 'Mahonia eurybracteata Fedde Mahonia fordii C.K.\\n',\n", + " 'Schneid.\\n',\n", + " 'Mahonia fortunei (Lindl.) Fedde Mahonia fremontii (Torr.) Fedde Mahonia gracilipes (Oliv.) Fedde Mahonia hancockiana Takeda Mahonia imbricata T.S.\\n',\n", + " 'Ying & Boufford Mahonia japonica (Thunb.) DC.\\n',\n", + " 'Mahonia leptodonta Gagnep.\\n',\n", + " 'Mahonia longibracteata Takeda Mahonia miccia Buch.-Ham.\\n',\n", + " 'ex D. Don Mahonia microphylla T.S.\\n',\n", + " 'Ying & G.R.\\n',\n", + " 'Long Mahonia monyulensis Ahrendt Mahonia moranensis (Schult. & Schult. f.) I.M. Johnstone Mahonia napaulensis DC.\\n',\n", + " 'Mahonia nitens C.K.\\n',\n", + " 'Schneid.\\n',\n", + " 'Mahonia oiwakensis Hayata Mahonia\\n',\n", + " 'Wu ex S.Y.\\n',\n", + " 'Bao Mahonia polyodonta Fedde Mahonia retinervis P.G. Xiao & Y.S.\\n',\n", + " 'Wang Mahonia setosa Gagnep.\\n',\n", + " 'Mahonia shenii Chun Mahonia sheridaniana C.K.\\n',\n", + " 'Schneid.\\n',\n", + " 'Mahonia subimbricata Chun & F. Chun Mahonia taronensis Hand.-Mazz.\\n',\n", + " 'Mahonia tenuifolia (Lindl.) Loudon ex Fedde Mahonia tinctoria (Terán & Berland.) I.M. Johnst.\\n',\n", + " 'Mahonia volcanica Standl.\\n',\n", + " '& Steyerm.\\n',\n", + " '\\n',\n", + " 'XO Skeletons is an electronic/hardcore/punk group from Waterbury, Connecticut, featuring well-known Boston hardcore legend Wesley Eisold from bands such as American Nightmare and Cold Cave.\\n',\n", + " 'Other members include Chauncey Johnson, F. Sean Martin, and Nicky Money.\\n',\n", + " 'XO Skeletons was notorious for their do-it-yourself mentality.\\n',\n", + " 'The band released their debut album Bored By Heaven for free, stating that \"It\\'s 2007.. Who needs a record label?\"\\n',\n", + " 'Asthmagasm No Muzzles Asthmagasm Suburban Dust XO Skeletons Bored By Heaven Town Crier Spider Earth Between Spider Heaven And Hell Better Babes Stranger Tongues Live From Planet Death Birthdays Under The Ice Of Hell Freezing Over Lascivious Facts (feat. Cage) California Needs You And Me\\n',\n", + " '\\n',\n", + " 'Bishop Solomon Doraiswamy was the second Bishop-in-Trichy-Tanjore Diocese of the Church of South India whose bishopric lasted from 1964 to 1982. Doraiswamy elected as the second Bishop - in - Trichy-Tanjore Diocese of the Church of South India who was principally consecrated in 1964 by Moderator, A. H. Legg and co-consecrated by P. Solomon, the Deputy Moderator.\\n',\n", + " \"During the fourteenth Church of South India Synod held from 10–14 January 1974 at the Women's Christian College, Madras, Doraiswamy was elected as the Deputy Moderator and held the office from 1974 to 1980 for over three terms (1974-1976; 1976-1978 and 1978-1980).\\n\",\n", + " 'Again during the seventeenth Church of South India Synod held from 10–14 January 1980 at Madras Christian College, Tambaram, Doraiswamy became the Moderator and held the office for a term up to 1982. Doraiswamy retired from the bishopric in 1982 on attaining superannuation.\\n',\n", + " 'The Senate of Serampore College (University) awarded an honorary doctorate degree upon Doraiswamy in 1981.\\n',\n", + " '\\n',\n", + " 'The Antilles Episcopal Conference (AEC) is a Roman Catholic episcopal conference.\\n',\n", + " 'Its members are bishops and archbishops from current and former British, Dutch, and French colonies and dependencies in the Caribbean (excluding Haiti), Central America, and northern South America.\\n',\n", + " \"The conference's membership includes five archdioceses, fourteen dioceses, and two missions sui iuris.\\n\",\n", + " 'These particular Churches minister to Catholics in thirteen independent nations, six British Overseas Territories, three departments of France, three countries of the Kingdom of the Netherlands and 3 municipalities of the Netherlands proper.\\n',\n", + " 'The bishop from an American insular area has been granted observer status.\\n',\n", + " 'The episcopal conference is led by a president, who must be a diocesan ordinary and is elected by the membership of the conference for a three-year term.\\n',\n", + " 'The conference also elects a vice president, who has the same qualifications as the president, and a treasurer, who can be a diocesan ordinary, a coadjutor bishop, or an auxiliary bishop.\\n',\n", + " 'Additionally, a permanent board — consisting of the president, vice president, treasurer, the metropolitan archbishops and two other elected members — handles administrative issues between plenary meetings of the conference.\\n',\n", + " 'The president of the conference is currently Gabriel Malzaire, Bishop of Roseau, while the vice president is Charles Jason Gordon, Archbishop of Port of Spain.\\n',\n", + " 'The Holy See appoints an apostolic delegate to the Antilles Episcopal Conference, who also serves as the Apostolic nuncio (papal ambassador) to the independent nations of the conference, except Belize.\\n',\n", + " 'The nunciature is located in Port of Spain, Trinidad and Tobago.\\n',\n", + " 'The current apostolic delegate is Archbishop Fortunatus Nwachukwu, who replaced Archbishop Nicola Girasoli after he was appointed as Apostolic Nuncio to Peru.\\n',\n", + " 'Antilles Episcopal Conference GCatholic\\n',\n", + " '\\n',\n", + " 'Richard \"Dick\" Briefer (January 9, 1915 – December 1980) was an American comic-book artist best known for his various\\n',\n", + " 'Under the pseudonym Dick Hamilton, he also created the superhero team the Target and the Targeteers for Novelty Press.\\n',\n", + " 'Dick Briefer studied at the Art Students League in Manhattan, New York City, and debuted in comic books in 1936 with work in Wow, What A Magazine!, one of the era\\'s proto-comics \"Comic books\": tabloid-sized collections of comic strip reprints in color, which would later include occasional new comic strip-like material.\\n',\n", + " 'Wow was edited by Jerry Iger, and when the comic ceased publication with issue #4 (cover-dated Nov. 1936), Briefer freelanced for the newly formed Eisner & Iger, one of the earliest \"packagers\" that produced complete comics on demand for publishers entering the fledgling medium.\\n',\n", + " \"Briefer's earliest recorded credit is as writer and artist of a five-page story beginning an adaptation of the Victor Hugo novel The Hunchback of Notre Dame, in Jumbo Comics #1-8 & 10 (Sept. 1938 - July 1939 & Nov. 1939), for the Eisner-Iger client Fiction House.\\n\",\n", + " 'Other seminal work includes drawing and possibly writing the science-fiction adventure feature \"Rex Dexter of Mars\", which ran in several issues of Fox Comics\\' Mystery Men Comics;\\n',\n", + " '\"Dynamo\" in Fox\\'s Science Comics;\\n',\n", + " '\"Biff Bannon\" in Harvey Comics\\' Speed Comics;\\n',\n", + " '\"Storm Curtis\" in Prize Comics\\' Prize Comics;\\n',\n", + " 'and \"Crash Parker\" in Fiction House\\'s Planet Comics.\\n',\n", + " 'For Timely Comics, the precursor of Marvel Comics during the 1930s to 1940s period fans and historians call the Golden Age of Comic Books, Briefer created or co-created (writer credit unknown) the single-appearance superhero the Human Top in Red Raven #1 (Aug. 1940).\\n',\n", + " 'Also during this time he also drew the comic strip Pinky Rankin, about a Nazi-fighter, for the American Communist Party newspaper The Daily Worker.\\n',\n", + " 'Briefer, using the pen name Dick Hamilton, created the superhero team the Target and the Targeteers for Novelty Press in 1940. The Target first appeared in Target Comics #10 (Nov. 1940), and the Targeteers the following issue.\\n',\n", + " 'The team starred in Target Comics through issue #95 / vol.\\n',\n", + " '9, #5 (July 1948).\\n',\n", + " 'Target itself ran 10 more issues. In Prize Comics #7 (Dec. 1940), writer-artist Briefer (using the pseudonym \"Frank N. Stein\" in the latter role) introduced the eight-page feature \"New Adventures of Frankenstein\", an updated version of the much-adapted Frankenstein monster created by Mary Shelley in her 1818 novel Frankenstein.\\n',\n", + " \"It's nice.\\n\",\n", + " \"[...] And they're funny as comics.\\n\",\n", + " 'They read well and are beautifully drawn;\\n',\n", + " \"they're full of unforgettable images, like the wizard eating Frankenstein on a hot dog.\\n\",\n", + " \"You'll never forget it, for better or for worse.\\n\",\n", + " \"Briefer's humorous Frankenstein ran through Prize Comics #68 (March 1948), and his humorous Frankenstein ran through issue #17 (Feb. 1949).\\n\",\n", + " 'Three years later, Briefer revived the series with his original, horrific Frankenstein from #18-33 (March 1952 - Nov. 1954).\\n',\n", + " 'Following the cancellation of Frankenstein during an era that put much pressure on horror comics and other violent comic books, leading to the creation of the Comics Code, Briefer left the comic industry for commercial advertising art. At the time of his death, Briefer was living in the Hollywood / Pembroke Pines area of Broward County, Florida.\\n',\n", + " 'Briefer, Dick.\\n',\n", + " 'The Monster of Frankenstein (Idea Men Productions, 2006) ISBN\\xa01-4196-4017-8, ISBN\\xa0978-1-4196-4017-9 Briefer, Dick.\\n',\n", + " \"Dick Briefer's Frankenstein (Library of Horror Comics' Masters, IDW/Yoe Books, 2010) ISBN\\xa01-60010-722-2\\n\",\n", + " '\\n',\n", + " \"Brown Estate Vineyards is the first and only Black-owned estate winery in California's Napa Valley (USA), and is best known as one of that region's most well-regarded zinfandel producers.\\n\",\n", + " 'In addition to zinfandel, Brown Estate produces cabernet sauvignon, chardonnay, and petite sirah.\\n',\n", + " 'Founded in 1995 by siblings Deneen,\\n',\n", + " 'Abandoned for some ten years prior, the land and its two structures - an 1859 stone and redwood barn and an 1885 Queen Anne Victorian home - were derelict. The senior Browns cut roads and brought in plumbing and electricity, ultimately earning an award from the Napa County Historical Society for their restoration of the residence structure.\\n',\n", + " 'In 1985 they planted their first vineyard on the property, approximately nine acres of zinfandel.\\n',\n", + " 'The Chiles Valley microclimate, characterized by extreme temperature\\n',\n", + " 'As well, the Browns sold cabernet sauvignon to the famed Mike Grgich.\\n',\n", + " \"It was through a series of apprenticeships with winemakers who were working with his fruit that David Brown - who by then had been farming his family's vineyards for five years - began learning the art and science of winemaking.\\n\",\n", + " 'In the mid-1990s, as a result of rising demand for their fruit, Deneen and David, then both residing on the vineyard property, made the decision to produce wine under their own label.\\n',\n", + " 'Because they had no winery facility on site, they secured a custom crush contract with Rombauer Vineyards in Saint Helena, and for six years they produced their wines at the Rombauer facility.\\n',\n", + " 'On January 29, 2000, Brown Estate debuted their first two vintages of Napa Valley zinfandel, 1996 and 1997, at the annual Zinfandel Advocates and Producers (ZAP) tasting at Fort Mason in San Francisco.\\n',\n", + " 'ZAP is the largest single varietal wine tasting event in the world, and the Brown offerings were very well received by the trade and public alike, thereby establishing Brown Estate as a winery to watch in the arena of Napa Valley zinfandel.\\n',\n", + " 'Simultaneously, in the January 31, 2000 issue of the Wine Spectator, the 1997 Brown Estate zinfandel received a score of 91 points.\\n',\n", + " \"Then, in June 2000, tragedy struck when a fire broke out at the warehouse facility where Brown's wine library and recently bottled 1998 vintage were stored.\\n\",\n", + " \"With the exception of sixteen bottles - two that went to Robert M. Parker, Jr. for review, two that went to Christie's for auction, and twelve that the Browns had stored at home - all of their 1998 zinfandel was destroyed, as were the remaining cases of their first two vintages.\\n\",\n", + " 'The lost 1998 vintage left the Browns out of the zinfandel market for one whole year, a devastating setback that was mitigated only by the kindness of restaurant and retail accounts who held space on their wine lists and shelves for the next vintage of Brown zin.\\n',\n", + " 'In order to bridge that gap, the Browns accelerated the release of their 1999 Napa Valley zinfandel, which they showed at ZAP in 2001. The previous month, in December 2000, Parker, the most influential wine critic in the world, gave the Brown 1998 zinfandel a score of 90, noting that his review was \"of academic interest only\" since all of the wine had perished in the fire.\\n',\n", + " \"The loss of the 1998 vintage has kept Brown Estate's Napa Valley zinfandel bottlings on a perennially early release schedule.\\n\",\n", + " 'Meanwhile, the Browns and their small production of a few thousand cases slowly were being squeezed out of the Rombauer facility. In order to continue their now quite promising winemaking endeavor, they were compelled to establish a winery facility on their own property.\\n',\n", + " 'The location of the old stone and redwood barn, still in its derelict state - doorless, windowless, and with a dirt floor - all but demanded development of an adjacent subterranean wine cave.\\n',\n", + " \"Winery construction began in Summer of 2002 and by September of that year, the old barn had a new lease on life as Brown Estate's on-site wine production facility.\\n\",\n", + " 'Brown then became a full-fledged estate winery.\\n',\n", + " 'Soon production began increasing, and eventually the barrel room in the lower level of the barn was filled to capacity, transforming the wine cave from notion to necessity.\\n',\n", + " 'Excavation commenced in April 2004, and because the host hillside was solid granite, drilling alone was inadequate.\\n',\n", + " 'For one solid year, muffled explosions rocked the Brown Estate property as one of the most unusual wine caves in the Napa Valley literally was designed by dynamite.\\n',\n", + " \"In Fall 2010, the Brown family celebrated their 30th year in the Napa Valley, and Brown Estate's 15th crush.\\n\",\n", + " 'The family posted the following message on their Website to commemorate these occasions: \"This year we celebrate our thirtieth year in the Napa Valley, and Brown Estate\\'s fifteenth crush. We thank all of you, with all of our hearts, for your continued enjoyment of our wines and your truly stupendous support of our once little endeavor. Each time you open and share a bottle of Brown Estate, you wrap your arms around us. It is an embrace that we cherish, a gesture that gives meaning to all of the passion and commitment we put into what we do. Thank you for making us a part of your lives, and for being a part of ours. Now onward & upward! The sky\\'s the limit!\"\\n',\n", + " \"In November 2010, the Browns released the world's first hashtagged wine, their 2009 Napa Valley Zinfandel. Subsequent vintages of this wine continue to bear the #brownzin hashtag, an homage to Twitter, which has been a source of inspiration and support as Brown Estate has built its reputation as a wine industry leader in the realm of social media.\\n\",\n", + " \"In April 2017, Brown Downtown Napa opened on downtown Napa's re-emerging First Street corridor.\\n\",\n", + " '\\n',\n", + " 'The 2015 Gedling Borough Council election was held on 7 May 2015 to elect all members of Gedling Borough Council in Nottinghamshire, England as part of the English local elections coinciding with the 2015 General Election.\\n',\n", + " 'Each councillor serves a four-year term of office.\\n',\n", + " 'Following Boundary Commission recommendations boundary change took place for this election to provide for 19 wards, many of which are two or three-member, thus electing 41 councillors.\\n',\n", + " 'Labour Party candidates won a majority, exceeding the 21 member threshold needed to reach majority party status, formerly having 64% of councillors.\\n',\n", + " \"The Conservative group was formerly the opposition party on the council, holding 15 seats out of 50 on the larger council between 2011 and 2015. The remainder of councillors comprised Liberal Democrat members, formerly holding 3 seats on the council from 2011 to 2015, returned 1 of the council's 41 councillors.\\n\",\n", + " '\\n',\n", + " 'Superpedestrian Inc., based in Cambridge, MA, develops technological solutions to improve urban mobility worldwide.\\n',\n", + " 'Superpedestrian specializes in micro-mobility technologies which enable the development of small-scale electric vehicles that can sense and adapt to the road and the driver.\\n',\n", + " 'The company works with ride-hailing and electric bike sharing companies to create electric-bike fleets optimized for safety, reliability, and enjoyable riding experiences.\\n',\n", + " 'Their consumer-facing product, the Copenhagen Wheel, is the first human-enhancing transportation technology available to the mass market.\\n',\n", + " \"Superpedestrian's first product, The Copenhagen Wheel, was developed at MIT’s Senseable City Lab in 2009 in partnership with the city of Copenhagen, and unveiled at the 2009 United Nations Climate Change Conference.\\n\",\n", + " 'In December 2012, Assaf Biderman, a co-inventor of the Wheel and Associate Director of the MIT Senseable City Lab, founded Superpedestrian to commercialize the Wheel.\\n',\n", + " 'After several years of engineering, testing, and validation, the Copenhagen Wheel officially launched in April 2017. The world is currently experiencing the fastest wave of urbanization in human history, leading to unprecedented levels of traffic congestion. Superpedestrian’s vehicle technology was developed in direct response to rising urban populations and urban transportation demand.\\n',\n", + " 'The current average car in the U.S. carries only 1.55 people per vehicle, but occupies enough road space to fit five. Many people assume that self-driving cars in the future will be used as shared vehicles, but research shows it is unlikely that autonomous vehicles will be shared for the majority of the time.\\n',\n", + " 'Even with a steep discount in the price of a ride, surveys show only 52% of riders will be willing to share an autonomous vehicle.\\n',\n", + " 'As a result, we are unlikely to meet future transportation needs without turning to micro-mobility - one-to-two person vehicles that enable individualized trips while taking up less road space.\\n',\n", + " 'Superpedestrian’s first foray into micro-mobility was through the bicycle.\\n',\n", + " 'Bicycles have always been the most popular form of single-occupancy vehicle.\\n',\n", + " 'However, since their introduction, the average city has grown 20x in land area, while the bicycle has evolved very little.\\n',\n", + " 'As a result, the bicycle is no longer a viable transportation solution for the many people who commute through these expanded cities.\\n',\n", + " 'Electric bicycles offer the opportunity to transform the humble bicycle into a compelling alternative to the car.\\n',\n", + " 'The Copenhagen Wheel is a rear wheel electric bicycle system that fits most existing bicycle frames - requiring a metal frame and V-brakes.\\n',\n", + " 'It replaces an existing rear wheel, transforming the bicycle into a smart electric hybrid/pedelec.\\n',\n", + " 'The Copenhagen Wheel contains a custom brushless motor, advanced sensors, control systems, and a lithium-ion battery, all contained within the hub of a single rear bicycle wheel.\\n',\n", + " 'Combining actual torque, power, cadence, pedal position, and acceleration sensing with high-speed controllers and actuators, the Wheel generates power that is seamlessly synchronized with your pedaling.\\n',\n", + " 'It’s just like riding a bike, but with a superhuman boost.\\n',\n", + " 'Bluetooth connectivity enables riders to personalize their cycling experience from their smartphone with ride modes such as Turbo, Eco, and Exercise.\\n',\n", + " 'A smartphone also acts as a digital key, activating the Wheel automatically when ready to ride, and communicating with the cloud in real time.\\n',\n", + " 'A self-diagnostic safety system monitors components within the Wheel and proactively responds to events within milliseconds, protecting both rider and Wheel.\\n',\n", + " 'Electronic Braking Assistance allows riders to brake and recharge the battery simultaneously, simply by backpedaling.\\n',\n", + " 'Finally, with over-the-air updates, the Wheel improves with new features over time.\\n',\n", + " 'The Copenhagen Wheel is available throughout the U.S., Canada, and Europe.\\n',\n", + " 'Superpedestrian has manufacturing facilities in both Massachusetts and the Netherlands.\\n',\n", + " 'Superpedestrian offers a complete bike product as well, called Wheel + Bike.\\n',\n", + " 'Customers can choose from a wide range of bicycle frames, from OEM’s like Marin, Public, Montague, Fyxation, Tern, Cinelli, and Fortified to pair with the Copenhagen Wheel.\\n',\n", + " 'Superpedestrian also works with over 260 independent bicycle retailers across the United States and Europe to provide demos, maintenance, and installation services.\\n',\n", + " 'Superpedestrian works with ride-hailing and e-bike-sharing companies worldwide to create fleets of small electric vehicles optimized for safety, reliability, and seamless riding experiences.\\n',\n", + " 'As the industry expands to include e-bikes, scooters, and mopeds, fleet operators contend with the challenges of managing and maintaining large numbers of motorized vehicles.\\n',\n", + " 'Building on Superpedestrian’s robust sensing, monitoring, and control systems developed for the Copenhagen Wheel, Superpedestrian offers solutions to common fleet operation problems.\\n',\n", + " 'Their platform allows operators to remotely monitor fleet health, boost maintenance efficiency, manage user accounts, and improve customer experiences, leading to lower operating costs and increased revenue.\\n',\n", + " 'Superpedestrian is currently working on future versions of the Copenhagen Wheel with updated features and wider compatibility requirements.\\n',\n", + " 'In the meantime, they offer over the air firmware updates to current riders.\\n',\n", + " 'Superpedestrian has raised more than $43 million in venture capital investment from firms including Cambridge-based General Catalyst and Spark Capital.\\n',\n", + " 'Additional investors include Tony Fadell, Charles Kim, Jared Leto, Xavier Niel, Paul Sagan, and Christian Klingler.\\n',\n", + " '\\n',\n", + " \"Alice Doesn't Live Here Anymore is a 1974 American romantic comedy-drama film directed by Martin Scorsese and written by Robert Getchell.\\n\",\n", + " 'It stars Ellen Burstyn as a widow who travels with her preteen son across the Southwestern United States in search of a better life.\\n',\n", + " 'Kris Kristofferson, Billy \"Green\" Bush, Diane Ladd, Valerie Curtin, Lelia Goldoni, Vic Tayback, Jodie Foster, Alfred Lutter and Harvey Keitel are featured in supporting roles.\\n',\n", + " \"Alice Doesn't Live Here Anymore premiered at the 27th Cannes Film Festival where it competed for the Palme d'Or and was released theatrically on December 9, 1974, by Warner Bros. The film was a critical and commercial success, grossing $21 million on a $1.8 million budget.\\n\",\n", + " 'At the 47th Academy Awards, Burstyn won Best Actress, while Ladd and Getchell received nominations for Best Supporting Actress and Best Original Screenplay.\\n',\n", + " \"When Socorro, New Mexico, housewife Alice Hyatt's uncaring husband Donald is killed in an auto accident, she decides to have a garage sale, pack what's left of her meager belongings, and take her precocious son Tommy to her childhood hometown of Monterey, California, where she hopes to pursue the singing career she abandoned when she married.\\n\",\n", + " 'Their financial situation forces them to take temporary lodgings in Phoenix, Arizona, where she finds work as a lounge singer in a seedy bar.\\n',\n", + " 'There she meets Ben, who uses his charm to lure her into a sexual relationship that comes to a sudden end when his wife Rita confronts Alice.\\n',\n", + " \"Ben breaks into Alice's apartment while Alice is there and physically assaults Rita for interfering with his extramarital affair.\\n\",\n", + " 'When Alice tells Ben to calm down, he threatens her also and further smashes up the apartment.\\n',\n", + " 'Fearing for their safety, Alice and Tommy quickly leave town.\\n',\n", + " 'Having spent most of the little money she earned on a new wardrobe, Alice is forced to delay her journey to the West Coast and accept a job as a waitress in Tucson so she can accumulate more cash.\\n',\n", + " \"At the local diner owned by Mel, she eventually bonds with her fellow servers - independent, no-nonsense, outspoken Flo and quiet, timid, incompetent Vera — and meets divorced local rancher David, who soon realizes the way to Alice's heart is through Tommy.\\n\",\n", + " 'Still emotionally wounded from the difficult relationship she had with her uncommunicative husband and the frightening encounter she had with Ben, Alice is hesitant to get involved with another man so quickly.\\n',\n", + " 'However, she finds out that David is a good influence on Tommy, who has befriended wisecracking, shoplifting, Ripple-guzzling Audrey, a slightly older girl forced to fend for herself while her mother makes a living as a prostitute.\\n',\n", + " 'Alice and David warily fall in love, but their relationship is threatened when Alice objects to his discipline of the perpetually bratty Tommy.\\n',\n", + " 'The two reconcile, and David offers to sell his ranch and move to Monterey so Alice can try to fulfill her childhood dream of becoming another Alice Faye.\\n',\n", + " 'In the end, Alice decides to stay in Tucson, coming to the conclusion that she can become a singer anywhere.\\n',\n", + " 'The part of Alice was originally offered to Shirley MacLaine.\\n',\n", + " 'However, MacLaine turned down the role.\\n',\n", + " 'MacLaine admitted in a 2005 interview that she regretted this decision.\\n',\n", + " 'Ellen Burstyn was still in the midst of filming The Exorcist when Warner Bros. executives expressed interest in working with her on another project.\\n',\n", + " 'It hadn\\'t been released yet, so I booked a screening to look at it and I felt that it was exactly what…Alice needed, because [it] was a wonderful script and well written, but for my taste it was a little slick. You know – in a good way, in a kind of Doris Day–Rock Hudson kind of way. I wanted something a bit more gritty.\"\\n',\n", + " 'Burstyn described her collaboration with director Martin Scorsese, making his first Hollywood studio production, as \"one of the best experiences I\\'ve ever had\".\\n',\n", + " 'The director agreed with his star that the film should have a message. \"It\\'s a picture about emotions and feelings and relationships and people in chaos,\" he said.\\n',\n", + " '\"We felt like charting all that and showing the differences and showing people making terrible mistakes ruining their lives and then realizing it and trying to push back when everything is crumbling – without getting into soap opera. We opened ourselves up to a lot of experimentation.\"\\n',\n", + " \"Scorsese's casting director auditioned 300 boys for the role of Tommy before they discovered Alfred Lutter.\\n\",\n", + " '\"I met the kid in my hotel room and he was kind of quiet and shy,\" Scorsese said. But when he paired him with Burstyn and suggested she deviate from the script, he held his own.\\n',\n", + " '\"Usually, when we were improvising with the kids, they would either freeze and look down or go right back to the script. But this kid, you couldn\\'t shut him up.\"\\n',\n", + " 'The film was shot on location predominantly in and around Tucson, but some scenes were shot in Amado, and Phoenix.\\n',\n", + " \"A Mel's Diner still exists in Phoenix.\\n\",\n", + " 'The soundtrack includes \"All the Way from Memphis\" by Mott the Hoople;\\n',\n", + " '\"Roll Away the Stone\" by Leon Russell;\\n',\n", + " '\"Daniel\" by Elton John;\\n',\n", + " '\"Jeepster\" by T-Rex;\\n',\n", + " 'and \"I Will Always Love You\" by Dolly Parton.\\n',\n", + " 'During her lounge act, Alice sings \"Where or When\" by Richard Rodgers and Lorenz Hart;\\n',\n", + " '\"When Your Lover Has Gone\" by Einar Aaron Swan;\\n',\n", + " '\"Gone with the Wind\" by Allie Wrubel and Herb Magidson;\\n',\n", + " 'and \"I\\'ve Got a Crush on You\" by George and Ira Gershwin.\\n',\n", + " 'In a film clip from Coney Island, Betty Grable is heard singing \"Cuddle Up a\\n',\n", + " 'and in a film clip from Hello Frisco, Hello, Alice Faye performs \"You\\'ll Never Know\" by Harry Warren and Mack Gordon.\\n',\n", + " 'Upon its release, the film received critical acclaim and grossed $21 million worldwide.\\n',\n", + " 'Roger Ebert of the Chicago Sun-Times called the film \"one of the most perceptive, funny, occasionally painful portraits of an American woman I\\'ve seen\" and commented, \"The movie has been both attacked and defended on feminist grounds, but I think it belongs somewhere outside ideology, maybe in the area of contemporary myth and romance.\"\\n',\n", + " \"Ebert put the film at #3 of his list of the best films of 1975 (even though the film came out in '74).\\n\",\n", + " 'The film did not go without its detractors, however.\\n',\n", + " 'Variety thought the film was \"a distended bore,\" saying it \"takes a group of well-cast film players and largely wastes them on a smaller-than-life film — one of those \\'little people\\' dramas that makes one despise little people.\"\\n',\n", + " 'TV Guide rated the film three out of four stars, calling it an \"effective but uneven work\" with performances that \"cannot conceal the storyline\\'s shortcomings.\"\\n',\n", + " 'Ellen Burstyn won the Academy Award for Best Actress.\\n',\n", + " 'Diane Ladd was nominated for the Academy Award for Best Supporting Actress but lost to Ingrid Bergman in Murder on the Orient Express, and Robert Getchell was nominated for the Academy Award for Best Original Screenplay but lost to Robert Towne for Chinatown.\\n',\n", + " 'The film won the BAFTA Award for Best Film, and BAFTA Awards went to Burstyn for Best Actress in a Leading Role, to Diane Ladd for Best Actress in a Supporting Role, and to Getchell for Best Screenplay.\\n',\n", + " 'Martin Scorsese was nominated for Best Direction but lost to Stanley Kubrick for Barry Lyndon.\\n',\n", + " \"Getchell was nominated for the Writers Guild of America Award for Best Original Screenplay, Burstyn and Ladd were nominated for Golden Globe Awards for Best Actress in a Motion Picture Drama and Best Supporting Actress in a Motion Picture, respectively, and Scorsese was nominated for the Palme D'Or at the 1975 Cannes Film Festival.\\n\",\n", + " 'The film inspired the sitcom Alice, which was broadcast by CBS from August 1976 through July 1985. The only member of the film cast to reprise his role was Vic Tayback as Mel (though his diner was moved to Phoenix).\\n',\n", + " 'Alfred Lutter portrayed Tommy in the pilot episode but was replaced by Philip McKeon for the series.\\n',\n", + " 'Diane Ladd joined the show later in its run, but in a role different from that she had played in the film.\\n',\n", + " 'Warner Home Video released the film on Region 1 DVD on August 17, 2004. It is in anamorphic widescreen format with audio tracks in English and French and subtitles in English, French, and Spanish.\\n',\n", + " 'Bonus features include commentary by Martin Scorsese, Ellen Burstyn, and Kris Kristofferson and Second Chances, a background look at the making of the film.\\n',\n", + " 'The movie was parodied in The Simpsons by Troy McClure in \"Alice Doesn\\'t Live Anymore\".\\n',\n", + " '\\n',\n", + " 'The Quantic Soul Orchestra is a live band project of musician and DJ, Will Holland.\\n']" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "lines[1000:1500]" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "advance-daniel", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Using custom data configuration default-e9c2b1c84264b5e7\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Downloading and preparing dataset text/default (download: Unknown size, generated: Unknown size, post-processed: Unknown size, total: Unknown size) to /mnt/data1/hf_dataset_cache/text/default-e9c2b1c84264b5e7/0.0.0/e16f44aa1b321ece1f87b07977cc5d70be93d69b20486d6dacd62e12cf25c9a5...\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=1.0, bar_style='info', layout=Layout(width='20px'), max=1.0), HTML(value=''…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Dataset text downloaded and prepared to /mnt/data1/hf_dataset_cache/text/default-e9c2b1c84264b5e7/0.0.0/e16f44aa1b321ece1f87b07977cc5d70be93d69b20486d6dacd62e12cf25c9a5. Subsequent calls will reuse this data.\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "c3509c52bbbe44e9aa5a3876b42c3ad8", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, max=1.0), HTML(value='')))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2021-11-10 07:52:58.791 | INFO | newlm.lm.elmo.lm_builder:_get_dataset:142 - Constructing roBERTa style dataset\n", + "100%|██████████| 12/12 [00:00<00:00, 39787.86it/s]" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "13\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" + ] + } + ], + "source": [ + "# dataset\n", + "train_path = \"/mnt/data4/made_workspace/newlm-data/en.1-percent/text_small.txt\"\n", + "dataset = elmo_lm_builder._get_dataset(train_path)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "undefined-butler", + "metadata": {}, + "outputs": [], + "source": [ + "# model\n", + "config = BertConfig(**elmo_lm_builder.model_config)\n", + "model = ELMOBertLMHeadModel(config=config)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "social-refund", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "max_steps is given, it will override any value given in num_train_epochs\n" + ] + } + ], + "source": [ + "# trainer\n", + "from transformers import TrainingArguments, Trainer\n", + "\n", + "output_dir = \"tmp-outputs\"\n", + "\n", + "args = TrainingArguments(\n", + " output_dir=output_dir,\n", + " overwrite_output_dir=True,\n", + " **{\n", + " \"per_device_train_batch_size\": 1,\n", + " \"max_steps\": 50, # scale this based on data size\n", + " \"save_total_limit\": 1,\n", + " \"prediction_loss_only\": True,\n", + " # optimizer, according to the paper\n", + " \"learning_rate\": 0.0001,\n", + " \"warmup_steps\": 10000, # do we scale this as well?\n", + " \"weight_decay\": 0.01,\n", + " },\n", + ")\n", + "trainer = Trainer(\n", + " model=model,\n", + " args=args,\n", + " train_dataset=dataset,\n", + " data_collator=elmo_lm_builder.data_collator,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "traditional-whole", + "metadata": {}, + "outputs": [], + "source": [ + "data_loader = trainer.get_train_dataloader()" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "considered-cotton", + "metadata": {}, + "outputs": [], + "source": [ + "b1 = next(iter(data_loader))" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "above-entrance", + "metadata": {}, + "outputs": [], + "source": [ + "# elmo_lm_builder.tokenizer.__dict__" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "foreign-methodology", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "input_ids\n", + "attention_mask\n", + "labels\n" + ] + } + ], + "source": [ + "for k in b1:\n", + " print(k)\n", + " b1[k].to('cuda:0')" + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "id": "sublime-pontiac", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['[CLS]',\n", + " 'National',\n", + " 'Route',\n", + " 'A',\n", + " '##00',\n", + " '##3',\n", + " 'also',\n", + " 'known',\n", + " 'as',\n", + " 'T',\n", + " '##ig',\n", + " '##re',\n", + " 'Access',\n", + " 'is',\n", + " 'an',\n", + " '8',\n", + " '.',\n", + " '8',\n", + " 'km',\n", + " '(',\n", + " '5',\n", + " '.',\n", + " '5',\n", + " 'mi',\n", + " ')',\n", + " '-',\n", + " 'long',\n", + " 'four',\n", + " '-',\n", + " 'lane',\n", + " 'highway',\n", + " '.',\n", + " 'It',\n", + " 'goes',\n", + " 'from',\n", + " 'the',\n", + " 'junction',\n", + " 'with',\n", + " 'National',\n", + " 'Route',\n", + " '9',\n", + " 'and',\n", + " 'Cam',\n", + " '##ino',\n", + " 'de',\n", + " 'C',\n", + " '##int',\n", + " '##ura',\n", + " '(',\n", + " 'Provincial',\n", + " 'Route',\n", + " '4',\n", + " ')',\n", + " 'to',\n", + " 'the',\n", + " 'town',\n", + " 'of',\n", + " 'T',\n", + " '##ig',\n", + " '##re',\n", + " ',',\n", + " 'passing',\n", + " 'the',\n", + " 'towns',\n", + " 'of',\n", + " ':',\n", + " 'Bo',\n", + " '##ulo',\n", + " '##gne',\n", + " 'San',\n", + " 'Is',\n", + " '##id',\n", + " '##ro',\n", + " 'B',\n", + " '##é',\n", + " '##cca',\n", + " '##r',\n", + " 'Victoria',\n", + " 'V',\n", + " '##ir',\n", + " '##rey',\n", + " '##es',\n", + " 'San',\n", + " 'Fernando',\n", + " 'In',\n", + " '1993',\n", + " 'the',\n", + " 'Federal',\n", + " 'Government',\n", + " 'opened',\n", + " 'a',\n", + " 'bid',\n", + " 'for',\n", + " 'the',\n", + " 'Buenos',\n", + " 'Aires',\n", + " 'road',\n", + " 'access',\n", + " 'network',\n", + " '.',\n", + " 'The',\n", + " 'winner',\n", + " 'for',\n", + " 'the',\n", + " 'maintenance',\n", + " 'contract',\n", + " 'for',\n", + " 'the',\n", + " 'North',\n", + " 'Access',\n", + " 'roads',\n", + " 'was',\n", + " 'Auto',\n", + " '##pis',\n", + " '##tas',\n", + " 'del',\n", + " 'Sol',\n", + " '.',\n", + " 'North',\n", + " 'Access',\n", + " 'includes',\n", + " 'Ave',\n", + " '##ni',\n", + " '##da',\n", + " 'General',\n", + " 'Paz',\n", + " ',',\n", + " 'National',\n", + " 'Route',\n", + " 'A',\n", + " '##00',\n", + " '##3',\n", + " ',',\n", + " 'and',\n", + " 'a',\n", + " 'portion',\n", + " 'of',\n", + " 'National',\n", + " 'Route',\n", + " '8',\n", + " ',',\n", + " 'National',\n", + " 'Route',\n", + " '9',\n", + " '.',\n", + " 'Auto',\n", + " '##pis',\n", + " '##tas',\n", + " 'del',\n", + " 'Sol',\n", + " 'signed',\n", + " 'the',\n", + " 'Con',\n", + " '##cession',\n", + " 'Con',\n", + " '##tract',\n", + " 'on',\n", + " '26',\n", + " 'May',\n", + " '1994',\n", + " '.',\n", + " 'To',\n", + " '##lls',\n", + " 'started',\n", + " 'being',\n", + " 'collected',\n", + " 'in',\n", + " 'August',\n", + " '1996',\n", + " ',',\n", + " 'after',\n", + " 'the',\n", + " 'company',\n", + " 'finished',\n", + " 'the',\n", + " 'road',\n", + " 'improvements',\n", + " 'according',\n", + " 'to',\n", + " 'the',\n", + " 'contract',\n", + " '.',\n", + " 'The',\n", + " 'concession',\n", + " 'ends',\n", + " 'on',\n", + " '31',\n", + " 'December',\n", + " '2020',\n", + " '.',\n", + " '[SEP]',\n", + " 'R',\n", + " '##ø',\n", + " '##d',\n", + " '##berg',\n", + " 'is',\n", + " 'the',\n", + " 'administrative',\n", + " 'centre',\n", + " 'of',\n", + " 'Nor',\n", + " '##e',\n", + " 'og',\n", + " 'U',\n", + " '##v',\n", + " '##dal',\n", + " 'municipality',\n", + " 'in',\n", + " 'Bus',\n", + " '##ker',\n", + " '##ud',\n", + " ',',\n", + " 'Norway',\n", + " '.',\n", + " 'R',\n", + " '##ø',\n", + " '##d',\n", + " '##berg',\n", + " 'is',\n", + " 'located',\n", + " 'in',\n", + " 'the',\n", + " 'Norwegian',\n", + " 'traditional',\n", + " 'district',\n", + " 'and',\n", + " 'valley',\n", + " 'of',\n", + " 'N',\n", + " '##ume',\n", + " '##dal',\n", + " '.',\n", + " 'Its',\n", + " 'population',\n", + " '(',\n", + " '2005',\n", + " ')',\n", + " 'is',\n", + " '46',\n", + " '##8',\n", + " ',',\n", + " 'and',\n", + " 'it',\n", + " 'is',\n", + " 'located',\n", + " 'on',\n", + " 'the',\n", + " 'R',\n", + " '##ø',\n", + " '##d',\n", + " '##berg',\n", + " 'dam',\n", + " '(',\n", + " 'Norwegian',\n", + " ':',\n", + " 'R',\n", + " '##ø',\n", + " '##d',\n", + " '##berg',\n", + " '##dam',\n", + " '##men',\n", + " ')',\n", + " 'on',\n", + " 'the',\n", + " 'N',\n", + " '##ume',\n", + " '##dal',\n", + " '##s',\n", + " '##l',\n", + " '##å',\n", + " '##gen',\n", + " 'River',\n", + " '.',\n", + " 'The',\n", + " 'local',\n", + " 'power',\n", + " 'stations',\n", + " ',',\n", + " 'R',\n", + " '##ø',\n", + " '##d',\n", + " '##berg',\n", + " 'Nor',\n", + " '##e',\n", + " '1',\n", + " 'in',\n", + " 'operation',\n", + " '1928',\n", + " 'and',\n", + " 'Nor',\n", + " '##e',\n", + " '2',\n", + " 'in',\n", + " 'operation',\n", + " 'since',\n", + " '1946',\n", + " ',',\n", + " 'utilize',\n", + " 'water',\n", + " '##flow',\n", + " 'from',\n", + " 'the',\n", + " 'R',\n", + " '##ø',\n", + " '##d',\n", + " '##berg',\n", + " 'Dam',\n", + " '.',\n", + " 'The',\n", + " 'plants',\n", + " 'are',\n", + " 'affiliated',\n", + " 'with',\n", + " 'St',\n", + " '##at',\n", + " '##kra',\n", + " '##ft',\n", + " ',',\n", + " 'the',\n", + " 'Norwegian',\n", + " 'state',\n", + " 'owned',\n", + " 'electricity',\n", + " 'company',\n", + " '.',\n", + " 'R',\n", + " '##ø',\n", + " '##d',\n", + " '##berg',\n", + " 'was',\n", + " 'once',\n", + " 'a',\n", + " 'railway',\n", + " 'station',\n", + " ',',\n", + " 'being',\n", + " 'the',\n", + " 'terminal',\n", + " 'station',\n", + " 'of',\n", + " 'the',\n", + " 'now',\n", + " '-',\n", + " 'defunct',\n", + " 'N',\n", + " '##ume',\n", + " '##dal',\n", + " 'railway',\n", + " 'line',\n", + " '(',\n", + " 'N',\n", + " '##ume',\n", + " '##dal',\n", + " '##s',\n", + " '##ban',\n", + " '##en',\n", + " ')',\n", + " 'which',\n", + " 'ran',\n", + " 'up',\n", + " 'the',\n", + " 'N',\n", + " '##ume',\n", + " '##dal',\n", + " 'valley',\n", + " 'between',\n", + " 'Kong',\n", + " '##sberg',\n", + " 'and',\n", + " 'R',\n", + " '##ø',\n", + " '##d',\n", + " '##berg',\n", + " '.',\n", + " 'The',\n", + " 'final',\n", + " 'passenger',\n", + " 'service',\n", + " 'ended',\n", + " 'in',\n", + " '1988',\n", + " '.',\n", + " 'The',\n", + " 'rail',\n", + " 'line',\n", + " 'north',\n", + " 'of',\n", + " 'Roll',\n", + " '##ag',\n", + " 'was',\n", + " 'closed',\n", + " 'in',\n", + " '1989',\n", + " '.',\n", + " '[SEP]']" + ] + }, + "execution_count": 42, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# elmo_lm_builder.tokenizer.convert_ids_to_tokens(b1['input_ids'][0])" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "broadband-chancellor", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'input_ids': tensor([[ 101, 1305, 3320, 138, 7629, 1495, 1145, 1227, 1112, 157,\n", + " 6512, 1874, 11737, 1110, 1126, 129, 119, 129, 1557, 113,\n", + " 126, 119, 126, 1940, 114, 118, 1263, 1300, 118, 7576,\n", + " 4083, 119, 1135, 2947, 1121, 1103, 6698, 1114, 1305, 3320,\n", + " 130, 1105, 14805, 4559, 1260, 140, 10879, 4084, 113, 9087,\n", + " 3320, 125, 114, 1106, 1103, 1411, 1104, 157, 6512, 1874,\n", + " 117, 3744, 1103, 4281, 1104, 131, 9326, 22806, 8376, 1727,\n", + " 2181, 2386, 2180, 139, 2744, 19495, 1197, 3006, 159, 3161,\n", + " 12210, 1279, 1727, 8834, 1130, 1949, 1103, 3467, 2384, 1533,\n", + " 170, 6875, 1111, 1103, 8883, 8837, 1812, 2469, 2443, 119,\n", + " 1109, 2981, 1111, 1103, 5972, 2329, 1111, 1103, 1456, 11737,\n", + " 4744, 1108, 12983, 19093, 10401, 3687, 17135, 119, 1456, 11737,\n", + " 2075, 15057, 2605, 1810, 1615, 20136, 117, 1305, 3320, 138,\n", + " 7629, 1495, 117, 1105, 170, 3849, 1104, 1305, 3320, 129,\n", + " 117, 1305, 3320, 130, 119, 12983, 19093, 10401, 3687, 17135,\n", + " 1878, 1103, 16752, 16122, 16752, 15017, 1113, 1744, 1318, 1898,\n", + " 119, 1706, 9872, 1408, 1217, 4465, 1107, 1360, 1820, 117,\n", + " 1170, 1103, 1419, 1845, 1103, 1812, 8313, 2452, 1106, 1103,\n", + " 2329, 119, 1109, 18750, 3769, 1113, 1955, 1382, 12795, 119,\n", + " 102, 155, 17945, 1181, 2953, 1110, 1103, 3207, 2642, 1104,\n", + " 16162, 1162, 18631, 158, 1964, 6919, 2667, 1107, 8947, 4188,\n", + " 4867, 117, 4323, 119, 155, 17945, 1181, 2953, 1110, 1388,\n", + " 1107, 1103, 4236, 2361, 1629, 1105, 4524, 1104, 151, 15447,\n", + " 6919, 119, 2098, 1416, 113, 1478, 114, 1110, 3993, 1604,\n", + " 117, 1105, 1122, 1110, 1388, 1113, 1103, 155, 17945, 1181,\n", + " 2953, 6961, 113, 4236, 131, 155, 17945, 1181, 2953, 10775,\n", + " 2354, 114, 1113, 1103, 151, 15447, 6919, 1116, 1233, 16835,\n", + " 4915, 1595, 119, 1109, 1469, 1540, 2930, 117, 155, 17945,\n", + " 1181, 2953, 16162, 1162, 122, 1107, 2805, 3825, 1105, 16162,\n", + " 1162, 123, 1107, 2805, 1290, 3064, 117, 17573, 1447, 12712,\n", + " 1121, 1103, 155, 17945, 1181, 2953, 8732, 119, 1109, 3546,\n", + " 1132, 6559, 1114, 1457, 2980, 27311, 4964, 117, 1103, 4236,\n", + " 1352, 2205, 6495, 1419, 119, 155, 17945, 1181, 2953, 1108,\n", + " 1517, 170, 2529, 1466, 117, 1217, 1103, 6020, 1466, 1104,\n", + " 1103, 1208, 118, 12273, 151, 15447, 6919, 2529, 1413, 113,\n", + " 151, 15447, 6919, 1116, 7167, 1424, 114, 1134, 1868, 1146,\n", + " 1103, 151, 15447, 6919, 4524, 1206, 3462, 19945, 1105, 155,\n", + " 17945, 1181, 2953, 119, 1109, 1509, 4059, 1555, 2207, 1107,\n", + " 2115, 119, 1109, 4356, 1413, 1564, 1104, 9916, 8517, 1108,\n", + " 1804, 1107, 2056, 119, 102]]),\n", + " 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n", + " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n", + " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n", + " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n", + " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n", + " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n", + " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n", + " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n", + " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n", + " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n", + " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n", + " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n", + " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n", + " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n", + " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n", + " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n", + " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]),\n", + " 'labels': tensor([[ 101, 1305, 3320, 138, 7629, 1495, 1145, 1227, 1112, 157,\n", + " 6512, 1874, 11737, 1110, 1126, 129, 119, 129, 1557, 113,\n", + " 126, 119, 126, 1940, 114, 118, 1263, 1300, 118, 7576,\n", + " 4083, 119, 1135, 2947, 1121, 1103, 6698, 1114, 1305, 3320,\n", + " 130, 1105, 14805, 4559, 1260, 140, 10879, 4084, 113, 9087,\n", + " 3320, 125, 114, 1106, 1103, 1411, 1104, 157, 6512, 1874,\n", + " 117, 3744, 1103, 4281, 1104, 131, 9326, 22806, 8376, 1727,\n", + " 2181, 2386, 2180, 139, 2744, 19495, 1197, 3006, 159, 3161,\n", + " 12210, 1279, 1727, 8834, 1130, 1949, 1103, 3467, 2384, 1533,\n", + " 170, 6875, 1111, 1103, 8883, 8837, 1812, 2469, 2443, 119,\n", + " 1109, 2981, 1111, 1103, 5972, 2329, 1111, 1103, 1456, 11737,\n", + " 4744, 1108, 12983, 19093, 10401, 3687, 17135, 119, 1456, 11737,\n", + " 2075, 15057, 2605, 1810, 1615, 20136, 117, 1305, 3320, 138,\n", + " 7629, 1495, 117, 1105, 170, 3849, 1104, 1305, 3320, 129,\n", + " 117, 1305, 3320, 130, 119, 12983, 19093, 10401, 3687, 17135,\n", + " 1878, 1103, 16752, 16122, 16752, 15017, 1113, 1744, 1318, 1898,\n", + " 119, 1706, 9872, 1408, 1217, 4465, 1107, 1360, 1820, 117,\n", + " 1170, 1103, 1419, 1845, 1103, 1812, 8313, 2452, 1106, 1103,\n", + " 2329, 119, 1109, 18750, 3769, 1113, 1955, 1382, 12795, 119,\n", + " 102, 155, 17945, 1181, 2953, 1110, 1103, 3207, 2642, 1104,\n", + " 16162, 1162, 18631, 158, 1964, 6919, 2667, 1107, 8947, 4188,\n", + " 4867, 117, 4323, 119, 155, 17945, 1181, 2953, 1110, 1388,\n", + " 1107, 1103, 4236, 2361, 1629, 1105, 4524, 1104, 151, 15447,\n", + " 6919, 119, 2098, 1416, 113, 1478, 114, 1110, 3993, 1604,\n", + " 117, 1105, 1122, 1110, 1388, 1113, 1103, 155, 17945, 1181,\n", + " 2953, 6961, 113, 4236, 131, 155, 17945, 1181, 2953, 10775,\n", + " 2354, 114, 1113, 1103, 151, 15447, 6919, 1116, 1233, 16835,\n", + " 4915, 1595, 119, 1109, 1469, 1540, 2930, 117, 155, 17945,\n", + " 1181, 2953, 16162, 1162, 122, 1107, 2805, 3825, 1105, 16162,\n", + " 1162, 123, 1107, 2805, 1290, 3064, 117, 17573, 1447, 12712,\n", + " 1121, 1103, 155, 17945, 1181, 2953, 8732, 119, 1109, 3546,\n", + " 1132, 6559, 1114, 1457, 2980, 27311, 4964, 117, 1103, 4236,\n", + " 1352, 2205, 6495, 1419, 119, 155, 17945, 1181, 2953, 1108,\n", + " 1517, 170, 2529, 1466, 117, 1217, 1103, 6020, 1466, 1104,\n", + " 1103, 1208, 118, 12273, 151, 15447, 6919, 2529, 1413, 113,\n", + " 151, 15447, 6919, 1116, 7167, 1424, 114, 1134, 1868, 1146,\n", + " 1103, 151, 15447, 6919, 4524, 1206, 3462, 19945, 1105, 155,\n", + " 17945, 1181, 2953, 119, 1109, 1509, 4059, 1555, 2207, 1107,\n", + " 2115, 119, 1109, 4356, 1413, 1564, 1104, 9916, 8517, 1108,\n", + " 1804, 1107, 2056, 119, 102]])}" + ] + }, + "execution_count": 32, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "b1" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "knowing-stone", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "ELMOBertLMHeadModel(\n", + " (transformer): ELMOBertModel(\n", + " (l2r_gpt): BertModel(\n", + " (embeddings): BertEmbeddings(\n", + " (word_embeddings): Embedding(30000, 768, padding_idx=0)\n", + " (position_embeddings): Embedding(1024, 768)\n", + " (token_type_embeddings): Embedding(2, 768)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (encoder): BertEncoder(\n", + " (layer): ModuleList(\n", + " (0): BertLayer(\n", + " (attention): BertAttention(\n", + " (self): BertSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): BertSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): BertIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): BertOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (1): BertLayer(\n", + " (attention): BertAttention(\n", + " (self): BertSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): BertSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): BertIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): BertOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (2): BertLayer(\n", + " (attention): BertAttention(\n", + " (self): BertSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): BertSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): BertIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): BertOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (3): BertLayer(\n", + " (attention): BertAttention(\n", + " (self): BertSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): BertSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): BertIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): BertOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (4): BertLayer(\n", + " (attention): BertAttention(\n", + " (self): BertSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): BertSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): BertIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): BertOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (5): BertLayer(\n", + " (attention): BertAttention(\n", + " (self): BertSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): BertSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): BertIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): BertOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (6): BertLayer(\n", + " (attention): BertAttention(\n", + " (self): BertSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): BertSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): BertIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): BertOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (7): BertLayer(\n", + " (attention): BertAttention(\n", + " (self): BertSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): BertSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): BertIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): BertOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (8): BertLayer(\n", + " (attention): BertAttention(\n", + " (self): BertSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): BertSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): BertIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): BertOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (9): BertLayer(\n", + " (attention): BertAttention(\n", + " (self): BertSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): BertSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): BertIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): BertOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (10): BertLayer(\n", + " (attention): BertAttention(\n", + " (self): BertSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): BertSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): BertIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): BertOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (11): BertLayer(\n", + " (attention): BertAttention(\n", + " (self): BertSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): BertSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): BertIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): BertOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " )\n", + " )\n", + " (pooler): BertPooler(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (activation): Tanh()\n", + " )\n", + " )\n", + " (r2l_gpt): BertModel(\n", + " (embeddings): BertEmbeddings(\n", + " (word_embeddings): Embedding(30000, 768, padding_idx=0)\n", + " (position_embeddings): Embedding(1024, 768)\n", + " (token_type_embeddings): Embedding(2, 768)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (encoder): BertEncoder(\n", + " (layer): ModuleList(\n", + " (0): BertLayer(\n", + " (attention): BertAttention(\n", + " (self): BertSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): BertSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): BertIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): BertOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (1): BertLayer(\n", + " (attention): BertAttention(\n", + " (self): BertSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): BertSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): BertIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): BertOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (2): BertLayer(\n", + " (attention): BertAttention(\n", + " (self): BertSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): BertSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): BertIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): BertOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (3): BertLayer(\n", + " (attention): BertAttention(\n", + " (self): BertSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): BertSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): BertIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): BertOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (4): BertLayer(\n", + " (attention): BertAttention(\n", + " (self): BertSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): BertSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): BertIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): BertOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (5): BertLayer(\n", + " (attention): BertAttention(\n", + " (self): BertSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): BertSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): BertIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): BertOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (6): BertLayer(\n", + " (attention): BertAttention(\n", + " (self): BertSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): BertSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): BertIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): BertOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (7): BertLayer(\n", + " (attention): BertAttention(\n", + " (self): BertSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): BertSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): BertIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): BertOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (8): BertLayer(\n", + " (attention): BertAttention(\n", + " (self): BertSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): BertSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): BertIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): BertOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (9): BertLayer(\n", + " (attention): BertAttention(\n", + " (self): BertSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): BertSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): BertIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): BertOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (10): BertLayer(\n", + " (attention): BertAttention(\n", + " (self): BertSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): BertSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): BertIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): BertOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (11): BertLayer(\n", + " (attention): BertAttention(\n", + " (self): BertSelfAttention(\n", + " (query): Linear(in_features=768, out_features=768, bias=True)\n", + " (key): Linear(in_features=768, out_features=768, bias=True)\n", + " (value): Linear(in_features=768, out_features=768, bias=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " (output): BertSelfOutput(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " (intermediate): BertIntermediate(\n", + " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", + " )\n", + " (output): BertOutput(\n", + " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", + " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", + " (dropout): Dropout(p=0.1, inplace=False)\n", + " )\n", + " )\n", + " )\n", + " )\n", + " (pooler): BertPooler(\n", + " (dense): Linear(in_features=768, out_features=768, bias=True)\n", + " (activation): Tanh()\n", + " )\n", + " )\n", + " )\n", + " (lm_head_l2r): Linear(in_features=768, out_features=30000, bias=False)\n", + " (lm_head_r2l): Linear(in_features=768, out_features=30000, bias=False)\n", + ")" + ] + }, + "execution_count": 35, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "model.to('cpu')" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "congressional-contract", + "metadata": {}, + "outputs": [ + { + "ename": "NameError", + "evalue": "name 'model' is not defined", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mno_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mb1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;31mNameError\u001b[0m: name 'model' is not defined" + ] + } + ], + "source": [ + "import torch\n", + "with torch.no_grad():\n", + " model(**b1)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.8" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/newlm/lm/elmo/lm_builder.py b/newlm/lm/elmo/lm_builder.py index 590a05f..28bf661 100644 --- a/newlm/lm/elmo/lm_builder.py +++ b/newlm/lm/elmo/lm_builder.py @@ -135,7 +135,7 @@ def preprocess_function(examples): encoded_dataset = dataset.map(preprocess_function, batched=True) return encoded_dataset["train"] - def __get_dataset(self, train_path): + def _get_dataset(self, train_path): dataset = self.__get_dataset_via_ds(train_path)["input_ids"] print(len(dataset)) diff --git a/newlm/lm/elmo/modeling_elmo/elmo_head.py b/newlm/lm/elmo/modeling_elmo/elmo_head.py index 4683fc7..fc3e0cd 100644 --- a/newlm/lm/elmo/modeling_elmo/elmo_head.py +++ b/newlm/lm/elmo/modeling_elmo/elmo_head.py @@ -190,6 +190,8 @@ def forward( last_hidden_states, flip_labels, r2l=True ) + print("l2r_loss", l2r_loss) + print("r2l_loss", r2l_loss) total_loss = l2r_loss + r2l_loss if labels is not None else None return ElmoGPTCausalLMOutput( From fba0c631be31ac6863d0cc36680c82347932f41c Mon Sep 17 00:00:00 2001 From: Made NN Date: Wed, 10 Nov 2021 17:08:19 +0700 Subject: [PATCH 2/9] Elmo sanity check --- Untitled.ipynb | 2348 --------------------------------------- elmo-sanity-check.ipynb | 660 +++++++++++ 2 files changed, 660 insertions(+), 2348 deletions(-) delete mode 100644 Untitled.ipynb create mode 100644 elmo-sanity-check.ipynb diff --git a/Untitled.ipynb b/Untitled.ipynb deleted file mode 100644 index ee6b896..0000000 --- a/Untitled.ipynb +++ /dev/null @@ -1,2348 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "id": "played-perfume", - "metadata": {}, - "outputs": [], - "source": [ - "from transformers import BertModel, BertConfig" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "fatty-broadway", - "metadata": {}, - "outputs": [], - "source": [ - "class BertCausalModel(BertModel):\n", - " def __init__(self, config, add_pooling_layer=True):\n", - " super().__init__(config, add_pooling_layer)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "impressed-thompson", - "metadata": {}, - "outputs": [], - "source": [ - "bconfig = BertConfig()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "brown-canberra", - "metadata": {}, - "outputs": [], - "source": [ - "bconfig" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "fuzzy-portuguese", - "metadata": {}, - "outputs": [], - "source": [ - "bmodel = BertCausalModel(bconfig)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "decent-healthcare", - "metadata": {}, - "outputs": [], - "source": [ - "bmodel.config.pad_token_id" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "treated-sight", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "coordinated-trailer", - "metadata": {}, - "outputs": [], - "source": [ - "import torch\n", - "\n", - "def get_sequence_lengths(pad_token_id, input_ids=None, inputs_embeds=None, **kwargs):\n", - " if input_ids is not None:\n", - " batch_size, sequence_length = input_ids.shape[:2]\n", - " else:\n", - " batch_size, sequence_length = inputs_embeds.shape[:2]\n", - "\n", - " assert (\n", - " pad_token_id is not None or batch_size == 1\n", - " ), \"Cannot handle batch sizes > 1 if no padding token is defined.\"\n", - " if pad_token_id is None:\n", - " sequence_lengths = -1\n", - " else:\n", - " if input_ids is not None:\n", - " sequence_lengths = torch.ne(input_ids, pad_token_id).sum(-1) - 1\n", - " else:\n", - " sequence_lengths = -1\n", - " logger.warning(\n", - " f\"{self.__class__.__name__} will not detect padding tokens in `inputs_embeds`. Results may be \"\n", - " f\"unexpected if using padding tokens in conjunction with `inputs_embeds.`\"\n", - " )\n", - "\n", - " return (batch_size, sequence_lengths)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "desperate-fellow", - "metadata": {}, - "outputs": [], - "source": [ - "bs, seqlen = get_sequence_lengths(0, batch['input_ids'])\n", - "bs, seqlen" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "decreased-logistics", - "metadata": {}, - "outputs": [], - "source": [ - "embed = batch['input_ids']\n", - "embed" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "forty-situation", - "metadata": {}, - "outputs": [], - "source": [ - "embed[range(bs), seqlen]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "adjusted-board", - "metadata": {}, - "outputs": [], - "source": [ - "from transformers import BertModel\n", - "from newlm.lm.bert.modeling_bert.bert_model import BertCausalModel\n", - "\n", - "bm = BertModel.from_pretrained(\"indobenchmark/indobert-base-p1\")\n", - "bcm = BertCausalModel.from_pretrained(\"indobenchmark/indobert-base-p1\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "oriented-truth", - "metadata": {}, - "outputs": [], - "source": [ - "bm" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "sticky-cambodia", - "metadata": {}, - "outputs": [], - "source": [ - "bcm" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "destroyed-particle", - "metadata": {}, - "outputs": [], - "source": [ - "from transformers import BertTokenizerFast\n", - "\n", - "tokenizer = BertTokenizerFast.from_pretrained(\"indobenchmark/indobert-base-p1\")\n", - "\n", - "batch = tokenizer(\n", - " [\"saya pergi ke pasar\", \"saya makan di warung\", \"pada hari senin ku turut ayah ke kota\"],\n", - " padding=True,\n", - " return_tensors=\"pt\"\n", - ")\n", - "batch" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "blond-shepherd", - "metadata": {}, - "outputs": [], - "source": [ - "out = bm(batch)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "extreme-russia", - "metadata": {}, - "outputs": [], - "source": [ - "out2 = bcm(**batch)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "worth-landing", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "killing-weekly", - "metadata": {}, - "outputs": [], - "source": [ - "from transformers import BertTokenizerFast\n", - "\n", - "tokenizer = BertTokenizerFast.from_pretrained(\"indobenchmark/indobert-base-p1\")\n", - "\n", - "batch = tokenizer(\n", - " [\"saya pergi ke pasar\", \"saya makan di warung\", \"pada hari senin ku turut ayah ke kota\"],\n", - " padding=True,\n", - " return_tensors=\"pt\"\n", - ")\n", - "batch" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "spread-debut", - "metadata": {}, - "outputs": [], - "source": [ - "from transformers import BertForSequenceClassification, BertTokenizerFast\n", - "from newlm.lm.bert.modeling_bert.bert_model import BertCausalModel, BertModelCausalForSequenceClassification\n", - "\n", - "model = BertForSequenceClassification.from_pretrained(\n", - " \"indobenchmark/indobert-base-p1\",\n", - " num_labels=3\n", - ")\n", - "model.bert = BertCausalModel(model.config)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "hazardous-plant", - "metadata": {}, - "outputs": [], - "source": [ - "model(**batch)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "incorporate-listing", - "metadata": {}, - "outputs": [], - "source": [ - "from newlm.lm.bert.modeling_bert.bert_model import BertModelCausalForSequenceClassification\n", - "\n", - "model2 = BertModelCausalForSequenceClassification.from_pretrained(\"indobenchmark/indobert-base-p1\", num_labels=3)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "august-horizontal", - "metadata": {}, - "outputs": [], - "source": [ - "model2(**batch)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "handed-correction", - "metadata": {}, - "outputs": [], - "source": [ - "model2(**batch)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "dominican-heath", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "liquid-wellington", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "clean-sheriff", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "markdown", - "id": "vietnamese-trace", - "metadata": {}, - "source": [ - "## Sanity Check" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "grand-richards", - "metadata": {}, - "outputs": [], - "source": [ - "from newlm.lm.elmo.modeling_elmo.elmo_head import ELMOBertLMHeadModel\n", - "from newlm.lm.elmo.lm_builder import ELMOLMBuilder\n", - "from transformers import BertConfig" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "pleased-termination", - "metadata": {}, - "outputs": [], - "source": [ - "from newlm.utils.file_util import read_from_yaml\n", - "config_file = read_from_yaml('examples/configs/run.1-percent-bert-causal.yaml')" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "radical-cooperation", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'vocab_size': 30000,\n", - " 'hidden_size': 768,\n", - " 'num_attention_heads': 12,\n", - " 'num_hidden_layers': 12,\n", - " 'intermediate_size': 3072,\n", - " 'max_position_embeddings': 1024,\n", - " 'is_decoder': True}" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "config_file['lm']['model']['config']" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "compact-confirmation", - "metadata": {}, - "outputs": [], - "source": [ - "elmo_lm_builder = ELMOLMBuilder(\n", - " model_config = config_file['lm']['model']['config'],\n", - " tokenizer=\"bert-base-cased\",\n", - " model_type=\"bert-causal-elmo\"\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "id": "greater-priest", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['Jews',\n", - " 'were',\n", - " 'indeed',\n", - " 'infected',\n", - " 'in',\n", - " 'numbers',\n", - " 'similar',\n", - " 'to',\n", - " 'their',\n", - " 'non',\n", - " '-',\n", - " 'Jewish',\n", - " 'neighbors',\n", - " 'Yet',\n", - " 'they',\n", - " 'were',\n", - " 'still',\n", - " 'made',\n", - " 's',\n", - " '##cape',\n", - " '##go',\n", - " '##ats',\n", - " '.']" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "elmo_lm_builder.tokenizer.tokenize(\"Jews were indeed infected in numbers similar to their non-Jewish neighbors Yet they were still made scapegoats.\")" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "id": "twenty-likelihood", - "metadata": {}, - "outputs": [], - "source": [ - "with open('/mnt/data4/made_workspace/newlm-data/en.1-percent/text.txt', 'r+') as fr:\n", - " lines = fr.readlines()" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "id": "fresh-trainer", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['\\n',\n", - " 'Typha albida is a plant species endemic to Afghanistan.\\n',\n", - " 'It grows in freshwater marshes.\\n',\n", - " '\\n',\n", - " 'Talczyn-Kolonia [ˈtalt͡ʂɨn\\xa0kɔˈlɔɲa] is a village in the administrative district of Gmina Kock, within Lubartów County, Lublin Voivodeship, in eastern Poland.\\n',\n", - " '\\n',\n", - " 'Niveocatharylla romieuxi is a moth in the Crambidae family.\\n',\n", - " 'It was described by Graziano Bassi in 1999. It is found in the Democratic Republic of the Congo.\\n',\n", - " '\\n',\n", - " 'Floyd Burton Jones (November 22, 1910, Cisco, Texas – April 15, 1999, Santa Barbara, California) was an American mathematician, active mainly in topology.\\n',\n", - " \"Jones's father was a pharmacist and local politician in Shackelford County, Texas.\\n\",\n", - " \"As the valedictorian of his high school class, Jones earned a Regents' Scholarship to The University of Texas, intending to study law eventually.\\n\",\n", - " 'Jones soon discovered that he had a poor memory for dates and history, and thus changed his major to chemistry.\\n',\n", - " 'Jones had the extraordinary good fortune to be taught freshman calculus by Robert Lee Moore, a founder of topology in the USA, a legendary mathematics teacher, and the inventor of the Moore method.\\n',\n", - " 'Jones went on to take more mathematics courses than required to be a chemist.\\n',\n", - " 'He displayed sufficient ability in those courses that when he graduated in 1932, Moore invited him to do a Ph.D.\\n',\n", - " 'in mathematics and offered him a part-time job as a math instructor.\\n',\n", - " \"Moore later supervised Jones's Ph.D.\\n\",\n", - " 'dissertation, completed in 1935. Jones then taught at the University of Texas for the next 15 years except during 1942-44, when he was a Research Associate at the Harvard Underwater Sound Laboratory, helping develop scanning sonar for the Navy.\\n',\n", - " 'In 1950, Jones moved to the University of North Carolina, where he eventually headed the Department of Mathematics.\\n',\n", - " 'From 1962 until his 1978 retirement, he was at the University of California at Riverside, where he helped launch the doctoral program in mathematics.\\n',\n", - " 'Over the course of his career, Jones published 67 articles and\\n',\n", - " 'dissertations. In 1987, he endowed a Chair in Topology at the University of California at Riverside.\\n',\n", - " 'Jones taught using a modified version of the Moore method.\\n',\n", - " 'He believed in \"learning by doing\" but unlike Moore, he incorporated textbooks into his courses.\\n',\n", - " 'In 1969, Louis McAuley wrote that \"the magical powers of Jones in the classroom-a master-who breathes the very life of mathematics into his students.\"\\n',\n", - " '\\n',\n", - " 'Paul Cameron (12 June 1904 – 31 December 1978) was a former Australian rules footballer who played with Carlton and South Melbourne in the Victorian Football League (VFL).\\n',\n", - " '\\n',\n", - " \"Abraham F. Sarmiento, Sr. (October 8, 1921 – October 3, 2010) was a Filipino jurist who served as an Associate Justice of the Supreme Court of the Philippines from 1987 to 1991. An active figure in the political opposition against the martial law government of President Ferdinand Marcos, he was appointed to the Court by Marcos' successor, President Corazon Aquino.\\n\",\n", - " 'Sarmiento was born in Santa Cruz, Ilocos Sur.\\n',\n", - " 'He completed his primary and secondary education in Laoag City, Ilocos Norte, graduating as the valedictorian of his high school class.\\n',\n", - " 'Upon the Japanese invasion of the Philippines during World War II, Sarmiento joined the USAFFE and the underground guerilla resistance against the Imperial Japanese Army.\\n',\n", - " 'After the war, Sarmiento completed his law studies at the University of the Philippines College of Law.\\n',\n", - " 'He was a member of the Alpha Phi Beta fraternity.\\n',\n", - " 'Sarmiento earned his Bachelor of Laws degree in 1949. In that year, he authored a biography on the murdered Chief Justice of the Supreme Court of the Philippines, José Abad Santos, entitled Jose Abad Santos: An Apotheosis.\\n',\n", - " 'Upon his admission to the Philippine Bar, Sarmiento entered into private practice.\\n',\n", - " 'In the 1950s, he formed a law partnership with Senators Gerardo Roxas and Justiniano Montano, maintaining his partnership with Roxas until 1967, when he established the Abraham F. Sarmiento Law Office.\\n',\n", - " 'Sarmiento successfully ran for a seat to the 1971 Constitutional Convention as a delegate from Cavite.\\n',\n", - " 'He was elected Vice-President of the Convention, which was tasked with the drafting of a new Constitution.\\n',\n", - " \"President Ferdinand Marcos declared martial law in September 1972. During this time, Sarmiento's eldest son, Ditto, was a student at the University of the Philippines.\\n\",\n", - " 'Ditto became the editor-in-chief of the official university newspaper, the Philippine Collegian, and under his leadership, the paper began publishing editorials critical of Marcos and martial law.\\n',\n", - " \"When Ditto was arrested in 1976, Sarmiento spent months negotiating with government officials to obtain his son's release.\\n\",\n", - " 'Ditto was released after seven months, but died within a year after his health was aggravated by the conditions of his imprisonment.\\n',\n", - " 'After the death of his son, Sarmiento was visibly active in human rights and anti-Marcos groups.\\n',\n", - " 'He co-founded the National Union for Democracy and Freedom, the Philippine Organization for Human Rights, and the National Union for Liberation.\\n',\n", - " 'He was among the founders of the United Nationalists Democratic Organizations (UNIDO), and served as its secretary-general from 1981 to 1983. From 1985 to 1987, Sarmiento served as the Chief Legal Counsel and Member of the Governing Council of the Lakas ng Bayan (LABAN).\\n',\n", - " 'It would be under the auspices of UNIDO and LABAN that Corazon Aquino and Salvador Laurel would form their presidential ticket that challenged Marcos in the 1986 presidential elections.\\n',\n", - " 'Sarmiento was also a member of the Executive Committee and National Council of the Bagong Alyansang Makabayan (BAYAN) from 1985 to 1987. In 1979, Sarmiento co-authored a book, The Road Back to Democracy, with former Philippine President Diosdado Macapagal and three others.\\n',\n", - " 'Later that year, he and Manuel Concordia published a book, Ang Demokrasya sa Pilipinas, which led to their arrest on charges of subversion and inciting to sedition.\\n',\n", - " 'They were later placed under house arrest.\\n',\n", - " 'Upon the assumption to the presidency of Corazon Aquino, following the 1986 People Power Revolution, Sarmiento was appointed to the Board of Directors of San Miguel Corporation, which was then under government sequestration.\\n',\n", - " 'In January 1987, President Aquino appointed Sarmiento as an Associate Justice of the Philippine Supreme Court.\\n',\n", - " 'He would serve on the High Court until he reached the compulsory retirement age of 70 in 1991. During his stint on the Court, Sarmiento held staunch civil libertarian views.\\n',\n", - " 'In People v. Nazario, 165 SCRA 186, the Court through Sarmiento acknowledged the void for vagueness rule as able to invalidate criminal statutes.\\n',\n", - " 'In Pita v. Court of Appeals, G.R.\\n',\n", - " 'No. 80806, 5 October 1989, 163 SCRA 386, he wrote for the Court that any restraint on the publication of purportedly obscene materials must satisfy the clear and present danger test.\\n',\n", - " 'In Salaw v. NLRC, G.R.\\n',\n", - " \"No. 90786, 27 September 1991, 202 SCRA 7, Sarmiento's opinion for the Court held that the dismissal of an employee in the private sector must be attended with procedural due process, a ruling which has since been reversed by the Court.\\n\",\n", - " 'At the same time, in PASE v. Drilon, G.R.\\n',\n", - " \"No. L-81958, 30 June 1988, 178 SCRA 362, Sarmiento's opinion for the Court upheld as a valid police power measure, the Philippine government's right to temporarily ban the deployment abroad of Filipino domestics and household workers. At the same time, Sarmiento dissented from the majority in some high-profile cases.\\n\",\n", - " \"In Marcos v. Manglapus, 178 SCRA 760, Sarmiento dissented from the majority which affirmed President Aquino's ban on the re-entry to the Philippines of Ferdinand Marcos.\\n\",\n", - " 'In Umil v. Ramos, 187 SCRA 311, Sarmiento published a strongly worded dissenting opinion to the majority opinion, which had held that there was no need to procure an arrest warrant to detain persons charged with the crimes of rebellion or subversion.\\n',\n", - " 'Sarmiento invoked the diminution of civil liberties during the Marcos administration, writing: The apprehensions in question chronicle in my mind the increasing pattern of arrests and detention in the country without the sanction of a judicial decree.\\n',\n", - " 'Four years ago at \"EDSA\", and many years before it, although with much fewer of us, we valiantly challenged a dictator and all the evils his regime had stood for: repression of civil liberties and trampling on of human rights.\\n',\n", - " 'We set up a popular government, restored its honored institutions, and crafted a democratic constitution that rests on the guideposts of peace and freedom.\\n',\n", - " \"I feel that with this Court's ruling, we have frittered away, by a stroke of the pen, what we had so painstakingly built in four years of democracy, and almost twenty years of struggle against tyranny.\\n\",\n", - " 'Since 2002, Sarmiento had served as a member of the Board of Regents of the University of the Philippines.\\n',\n", - " 'In 2004, he was among the candidates for appointment as President of the University of the Philippines.\\n',\n", - " 'However, his membership in the Board of Regents of the University of the Philippines was marked with controversy, when he protested the election of UP-Philippine General Hospital (UP-PGH) Director Jose Gonzalez, by questioning the legality of the vote of Student Regent Charisse Banez because of her pending application for residency, thus questioning her representation in the Board of Regents and finally replacing the PGH Director and ouster of the Student Regent.\\n',\n", - " 'But according to the records of his appointment by the President of the Republic of the Philippines, his term as Regent is already expired along with the 2 Malacanang Appointees.\\n',\n", - " 'It is speculated, that his decision to question the status of the Student Regent is politically motivated because the other nominees are close to President Gloria-Macapagal Arroyo and they will continue the private leasing of the Faculty Medical Arts building in the UP-Philippine General Hospital to the Daniel Mercado Medical Center, a private hospital owned by a UP alumnus in Tanauan, Batangas, while Dr. Jose Gonzales wants to revoke the agreement because, it will not benefit the poor patients of the hospital but it will jumpstart commercialization of the UP-PGH.\\n',\n", - " 'Now, his actions hound his reputation as a Martial Law activist.\\n',\n", - " 'Sarmiento was in fact appointed by former President Gloria Macapagal-Arroyo as a Regent until September 29, 2010. However, U.P.\\n',\n", - " 'President Emerlinda R. Roman renominated him as a U.P.\\n',\n", - " 'Regent to President Benigno S. Aquino III and this renomination remained pending even as Sarmiento died in Prague, Czech Republic on October 3, 2010. As of the time of his death, therefore, Sarmiento was still a sitting U.P.\\n',\n", - " 'Regent in a holdover capacity.\\n',\n", - " 'Sarmiento died due to apparent organ failure, while visiting Prague, Czech Republic, was reported in the Philippine media on October 4, 2010.\\n',\n", - " '\\n',\n", - " \"The 1937 World Table Tennis Championships – Women's Singles was the 11th edition of the women's singles championship. Ruth Aarons and Gertrude Pritzi competed in the final which ended in an anticlimax because both players were stopped from finishing the final under the new time limit rules and declared joint champions.\\n\",\n", - " '+ both players disqualified under time limit rules\\n',\n", - " '\\n',\n", - " 'Plaza de Armas of Trujillo is the main square where the Spanish foundation of Trujillo was made, in northern Peru.\\n',\n", - " 'It has been the principal locus of history in this city in the republic era.\\n',\n", - " 'It is located in the central zone of the Historic Centre of Trujillo.\\n',\n", - " 'In the streets that form this main square are located the buildings of Municipality Palace, the Cathedral, among others.\\n',\n", - " 'The Plaza de Armas of Trujillo is formed by the Pizarro, Independencia, Orbegoso and Almagro streets.\\n',\n", - " 'In this square, the proclamation of the independence of Trujillo\\n',\n", - " 'Martin de Estete began the design of the streets of Trujillo by order of Diego de Almagro in the end of the year of 1534, so he designed the structure and dimensions of the Plaza de Armas.\\n',\n", - " 'In 1841, Pedro de Mandalengoitia Sanz de Zarate donated to the square a fountain made of white stone that was built on his farm and mill in Santa Cruz de Carabamba.\\n',\n", - " 'On December 29, 1820, the Plaza de Armas was the scene of the proclamation of Independence of Trujillo by the Marquis of Torre Tagle.\\n',\n", - " 'In this square are located the cathedral of the city and mansions dating back to the Hispanic and republican periods.\\n',\n", - " 'On July 4, 1929, in the middle of this square, the Freedom Monument was opened, which represents the country\\'s independence process and according to \"commentators it represents the most precious of human beings, the love of freedom, the memory and recognition of the distinguished men who gave us independence\".\\n',\n", - " 'The statue was a work in Germany, the materials used are marble and copper, the sculptor was Edmund Moeller and it was armed by Henry Albrecht.\\n',\n", - " 'The Freedom Monument, located in the center of the Plaza de Armas of Trujillo (Peru) and work of sculptor Edmund Möeller, consists of three sections: the first is on a circular platform with pedestals, resting on a granite base, supporting the sculptures represent the art, science, trade and health.\\n',\n", - " 'The second consists of three statues robust.\\n',\n", - " 'A statue of a man who snorts, it is bent, symbolizing oppression or slavery.\\n',\n", - " \"A second statue's arms back, symbolizing the struggle for freedom.\\n\",\n", - " 'The third statue is a man with arms raised and hands making fist, symbolizing liberation.\\n',\n", - " 'Also in this body are located the next plates: the first recalling the proclamation of the independence\\n',\n", - " 'Francisco Pizarro Diego de Almagro Independencia Mariscal Orbegoso In the Plaza de Armas of Trujillo are realized important celebrations y ceremonies such as Corpus Christi in June, parade for Trujillo Marinera Festival in January, independence day of Trujillo in December 29 of every year, etc.\\n',\n", - " '\\n',\n", - " 'In mathematics, specifically transcendental number theory, the six exponentials theorem is a result that, given the right conditions on the exponents, guarantees the transcendence of at least one of a set of exponentials.\\n',\n", - " 'If x₁, x₂,..., xd are d complex numbers that are linearly independent over the rational numbers, and y₁, y₂,...,yₗ are l complex numbers that are also linearly independent over the rational numbers, and if dl\\xa0>\\xa0d\\xa0+\\xa0l, then at least one of the following dl numbers is transcendental: The most interesting case is when d\\xa0=\\xa03 and l\\xa0=\\xa02, in which case there are six exponentials, hence the name of the result.\\n',\n", - " 'They claimed that Carl Ludwig Siegel knew of a proof of this special case, but it is not recorded.\\n',\n", - " 'Using the special case they manage to prove that the ratio of consecutive colossally abundant numbers is always either a prime or a semiprime.\\n',\n", - " 'The theorem was first explicitly stated and proved in its complete form independently by Serge Lang and Kanakanahalli Ramachandra in the 1960s.\\n',\n", - " 'A stronger, related result is the five exponentials theorem, which is as follows.\\n',\n", - " 'Let x₁, x₂ and y₁, y₂ be two pairs of complex numbers, with each pair being linearly independent over the rational numbers, and let γ be a non-zero algebraic number.\\n',\n", - " 'Then at least one of the following five numbers is transcendental: This theorem implies the six exponentials theorem and in turn is implied by the as yet unproven four exponentials conjecture, which says that in fact one of the first four numbers on this list must be transcendental.\\n',\n", - " 'Another related result that implies both the six exponentials theorem and the five exponentials theorem is the sharp six exponentials theorem.\\n',\n", - " 'This theorem is as follows.\\n',\n", - " \"Let x₁, x₂, and x₃ be complex numbers that are linearly independent over the rational numbers, and let y₁ and y₂ be a pair of complex numbers that are linearly independent over the rational numbers, and suppose that βᵢⱼ are six algebraic numbers for 1\\xa0≤\\xa0i\\xa0≤\\xa03 and 1\\xa0≤\\xa0j\\xa0≤\\xa02 such that the following six numbers are algebraic: Then xᵢ\\xa0yⱼ\\xa0=\\xa0βᵢⱼ for 1\\xa0≤\\xa0i\\xa0≤\\xa03 and 1\\xa0≤\\xa0j\\xa0≤\\xa02. The six exponentials theorem then follows by setting βᵢⱼ\\xa0=\\xa00 for every i and j, while the five exponentials theorem follows by setting x₃\\xa0=\\xa0γ/x₁ and using Baker's theorem to ensure that the xᵢ are linearly independent.\\n\",\n", - " 'There is a sharp version of the five exponentials theorem as well, although it as yet unproven so is known as the sharp five exponentials conjecture.\\n',\n", - " 'This conjecture implies both the sharp six exponentials theorem and the five exponentials theorem, and is stated as follows.\\n',\n", - " 'Let x₁, x₂ and y₁, y₂ be two pairs of complex numbers, with each pair being linearly independent over the rational numbers, and let α, β₁₁, β₁₂, β₂₁, β₂₂, and γ be six algebraic numbers with γ\\xa0≠\\xa00 such that the following five numbers are algebraic: Then xᵢ\\xa0yⱼ\\xa0=\\xa0βᵢⱼ for 1\\xa0≤\\xa0i, j\\xa0≤\\xa02 and γx₂\\xa0=\\xa0αx₁.\\n',\n", - " \"A consequence of this conjecture that isn't currently known would be the transcendence of eπ², by setting x₁\\xa0=\\xa0y₁\\xa0=\\xa0β₁₁\\xa0=\\xa01, x₂\\xa0=\\xa0y₂\\xa0=\\xa0iπ, and all the other values in the statement to be zero.\\n\",\n", - " 'A further strengthening of the theorems and conjectures in this area are the strong versions.\\n',\n", - " 'The strong six exponentials theorem is a result proved by Damien Roy that implies the sharp six exponentials theorem.\\n',\n", - " 'This result concerns the vector space over the algebraic numbers generated by 1 and all logarithms of algebraic numbers, denoted here as L∗.\\n',\n", - " 'So L∗ is the set of all complex numbers of the form for some n\\xa0≥\\xa00, where all the βᵢ and αᵢ are algebraic and every branch of the logarithm is considered.\\n',\n", - " 'The strong six exponentials theorem then says that if x₁, x₂, and x₃ are complex numbers that are linearly independent over the algebraic numbers, and if y₁ and y₂ are a pair of complex numbers that are also linearly independent over the algebraic numbers then at least one of the six numbers xᵢ\\xa0yⱼ for 1\\xa0≤\\xa0i\\xa0≤\\xa03 and 1\\xa0≤\\xa0j\\xa0≤\\xa02 is not in L∗.\\n',\n", - " 'This is stronger than the standard six exponentials theorem which says that one of these six numbers is not simply the logarithm of\\n',\n", - " 'There is also a strong five exponentials conjecture formulated by Michel Waldschmidt It would imply both, the strong six exponentials theorem and the sharp five exponentials conjecture.\\n',\n", - " \"This conjecture claims that if x₁, x₂ and y₁, y₂ are two pairs of complex numbers, with each pair being linearly independent over the algebraic numbers, then at least one of the following five numbers is not in L∗: All the above conjectures and theorems are consequences of the unproven extension of Baker's theorem, that logarithms of algebraic numbers that are linearly independent over the rational numbers are automatically algebraically independent too.\\n\",\n", - " 'The diagram on the right shows the logical implications between all these results.\\n',\n", - " 'The exponential function eᶻ uniformizes the exponential map of the multiplicative group Gₘ.\\n',\n", - " 'Therefore, we can reformulate the six exponential theorem more abstractly as follows: Let G = Gₘ × Gₘ and take u\\xa0: C → G(C) to be a non-zero complex-analytic group homomorphism.\\n',\n", - " 'Define L to be the set of complex numbers l for which u(l) is an algebraic point of G. If a minimal generating set of L over Q has more than two elements then the image u(C) is an algebraic subgroup of G(C).\\n',\n", - " '(In order to derive the classical statement, set u(z) =(e ʸ¹ ᶻ; e ʸ² ᶻ) and note that Qx₁ + Qx₂ + Qx₃ is a subset of L).\\n',\n", - " 'In this way, the statement of the six exponentials theorem can be generalized to an arbitrary commutative group variety G over the field of algebraic numbers.\\n',\n", - " 'This generalized six exponential conjecture, however, seems out of scope at the current state of transcendental number theory.\\n',\n", - " 'For the special, but interesting cases G = Gₘ × E and G = E × E′, where E, E′ are elliptic curves over the field of algebraic numbers, results towards the generalized six exponential conjecture were proven by Aleksander Momot.\\n',\n", - " 'These results involve the exponential function eᶻ and a Weierstrass function resp.\\n',\n", - " 'two Weierstrass functions with algebraic invariants , instead of the two exponential functions in the classical statement.\\n',\n", - " 'Let G = Gₘ × E and suppose E is not isogenous to a curve over a real field and that u(C) is not an algebraic subgroup of G(C).\\n',\n", - " 'Then L is generated over Q either by two elements x₁, x₂, or three elements x₁, x₂, x₃ which are not all contained in a real line Rc, where c is a non-zero complex number.\\n',\n", - " 'A similar result is shown for G = E × E′.\\n',\n", - " '\\n',\n", - " \"The 1974 October Revolution Parade was a parade on Red Square dedicated to the 57th anniversary of the October Revolution on November 7th 1974. It was inspected by the Minister of Defense and Marshal of the Soviet Union Andrei Grechko who also made a speech to the Soviet People on Red Square on the grandstand of Lenin's Mausoleum.\\n\",\n", - " 'The Parade Commander is the Commander of the Moscow Garrison Colonel General Vladimir Govorov.\\n',\n", - " \"Also on Lenin's Mausoleum is the General Secretary of the Communist Party of the Soviet Union Leonid Brezhnev and the President Nikolai Podgorny.\\n\",\n", - " 'This Parade is the last to feature heavy nuclear missiles such as Ballistic Missiles.\\n',\n", - " 'Due to the rain that took place that day the parade of workers and athletes after the parade finale was cancelled.Although the demonstrations on Red Square was Cancelled the demonstrations on Palace Square in Leningrad continued following the parade there.\\n',\n", - " '\\n',\n", - " 'East Lake Weir is an unincorporated community in eastern Marion County, Florida, United States, on the east shore of Lake Weir.\\n',\n", - " 'The community is part of the Ocala Metropolitan Statistical Area.\\n',\n", - " 'East Lake Weir is located at 29°01′12″N 81°54′30″W.\\n',\n", - " '\\n',\n", - " 'Point Possession is an unincorporated community on the Kenai Peninsula in Alaska.\\n',\n", - " \"As of April 1, 2010, the village's population was 3. Since 2009 Point Possession has been owned by the federal government and has been made a part of the Kenai National Wildlife Refuge.\\n\",\n", - " '\\n',\n", - " 'Armin van Buuren: In The Mix is a music video game for the Wii console.\\n',\n", - " 'It was announced in May 2009 and was developed by TransGaming Technologies of Canada and published by Foreign Media Games (now Easy Interactive).\\n',\n", - " 'The game was released in the United Kingdom on 12 November 2010.\\n',\n", - " '\\n',\n", - " 'Adenylyl cyclase type 6 is an enzyme that in humans is encoded by the ADCY6 gene.\\n',\n", - " 'This gene encodes adenylyl cyclase 6, which is a membrane-associated enzyme and catalyzes the formation of the secondary messenger cyclic adenosine monophosphate (cAMP).\\n',\n", - " 'The expression of this gene is found in normal thyroid and brain tissues, as well as some tumors;\\n',\n", - " 'and its expression is significantly higher in one hyperfunctioning thyroid tumor than in normal thyroid tissue.\\n',\n", - " 'Alternative splicing generates 2 transcript variants.\\n',\n", - " '\\n',\n", - " 'Millennium Park, located at the historic intersection of Park Row and Broadway is a Greenstreet site designed to replicate a forest amid a busy intersection in Downtown Manhattan. Located to the south of City Hall Park, it is used as a bus turnaround loop, formerly a paved median transformed into a green space at the turn of the 21st century.\\n',\n", - " 'The park was dedicated in November 2000, described by Mayor Rudolph W. Giuliani as “a final gift from the 20th century to New Yorkers of the 21st.” With its completion date in mind, it was named Millennium Park.\\n',\n", - " 'At its southern tip is the David Rockefeller Clock, dedicated by the Downtown Alliance in honor of longtime Chase Manhattan Bank executive David Rockefeller.\\n',\n", - " 'The youngest grandson of John D. Rockefeller, he was instrumental in maintaining the vitality of downtown Manhattan as a business district at a time when many businesses were leaving urban centers for suburban settings.\\n',\n", - " 'Rockefeller’s contributions to the neighborhood includes the preservation of South Street Seaport, and the construction of World Trade Center and Battery Park City.\\n',\n", - " 'Keeping in theme with City Hall Park, the 3,500-pound clock was designed in the late 19th-century style to match the fountain and light fixtures of the Victorian period.\\n',\n", - " '\\n',\n", - " 'A total solar eclipse occurred on August 1, 2008. A solar eclipse occurs when the Moon passes between Earth and the Sun, thereby totally or partly obscuring the image of the Sun for a viewer on Earth.\\n',\n", - " \"A total solar eclipse occurs when the Moon's apparent diameter is larger than the Sun's, blocking all direct sunlight, turning day into darkness.\\n\",\n", - " \"Totality occurs in a narrow path across Earth's surface, with the partial solar eclipse visible over a surrounding region thousands of kilometres wide.\\n\",\n", - " 'It had a magnitude of 1.0394 that was visible from a narrow corridor through northern Canada (Nunavut), Greenland, central Russia, eastern Kazakhstan, western Mongolia and China.\\n',\n", - " 'Occurring north of the arctic circle, it belonged to the so-called midnight sun eclipses.\\n',\n", - " 'The largest city in the path of the eclipse was Novosibirsk in Russia.\\n',\n", - " \"The total eclipse lasted for 2 minutes, and covered 0.4% of the Earth's surface in a 10,200\\xa0km long path.\\n\",\n", - " \"It was the 47th eclipse of the 126th Saros cycle, which began with a partial eclipse on March 10, 1179 and will conclude with a partial eclipse on May 3, 2459. A partial eclipse could be seen from the much broader path of the Moon's penumbra, including northeastern North America and most of Europe and Asia.\\n\",\n", - " 'It was described by observers as \"special for its colours around the horizon. There were wonderful oranges and reds all around, the clouds lit up, some dark in silhouette, some golden, glowing yellowy-orange in the distance. You could see the shadow approaching against the clouds and then rushing away as it left.\"\\n',\n", - " 'Animated path The eclipse began in the far north of Canada in Nunavut at 09:21 UT, the zone of totality being 206\\xa0km wide, and lasting for 1 minute 30 seconds.\\n',\n", - " 'The path of the eclipse then headed north-east, crossing over northern Greenland and reaching the northernmost latitude of 83° 47′ at 09:38 UT before dipping down into Russia.\\n',\n", - " 'The path of totality touched the northeast corner of Kvitøya, an uninhabited Norwegian island in the Svalbard archipelago, at 09:47 UT.\\n',\n", - " 'The eclipse reached the Russian mainland at 10:10 UT, with a path 232\\xa0km wide and a duration of 2 minutes 26 seconds.\\n',\n", - " 'The greatest eclipse occurred shortly after, at 10:21:07 UT at coordinates 65°39′N 72°18′E (close to Nadym), when the path was 237\\xa0km wide, and the duration was 2 minutes 27 seconds. Cities in the path of the total eclipse included Megion, Nizhnevartovsk, Strezhevoy, Novosibirsk and Barnaul.\\n',\n", - " 'Around 10,000 tourists were present in Novosibirsk, the largest city to experience the eclipse.\\n',\n", - " 'The path of the eclipse then moved south-east, crossing into Mongolia and just clipping Kazakhstan at around 10:58 UT.\\n',\n", - " 'The path here was 252\\xa0km wide, but the duration was decreased to 2 minutes 10 seconds.\\n',\n", - " 'The path then ran down the China-Mongolia border, ending in China at 11:18 UT, with an eclipse lasting 1 minute 27 seconds at sunset.\\n',\n", - " 'The total eclipse finished at 11:21 UT.\\n',\n", - " 'The total eclipse passed over Yiwu County, Jiuquan and Xi’an.\\n',\n", - " 'Around 10,000 people were gathered to watch the eclipse in Yiwu.\\n',\n", - " \"A partial eclipse was seen from the much broader path of the Moon's penumbra, including the north east coast of North America and most of Europe and Asia.\\n\",\n", - " 'In London, England, the partial eclipse began at 09:33 BST, with a maximum eclipse of 12% at 10:18 BST, before concluding at 11:05 BST. At Edinburgh the partial eclipse was 23.5% of the sun, whilst it was 36% in Lerwick in the Shetland Isles.\\n',\n", - " 'German charter airline LTU, now trading as Air Berlin, operated a special flight from Düsseldorf to the North Pole to observe the eclipse.\\n',\n", - " 'Flight number LT 1111 spent over 11 hours in the air, returning to base at 6pm after flying a planeload of eclipse chasers, scientists, journalists and TV crews to watch the celestial event.\\n',\n", - " 'The route also included a low-level sightseeing tour of Svalbard before the eclipse and the magnetic pole afterwards.\\n',\n", - " \"Each member in a semester series of solar eclipses repeats approximately every 177 days and 4 hours (a semester) at alternating nodes of the Moon's orbit.\\n\",\n", - " 'It is a part of Saros cycle 126, repeating every 18 years, 11 days, containing 71 events.\\n',\n", - " 'The series started with partial solar eclipse on March 10, 1179. It contains annular eclipses from June 4, 1323 through April 4, 1810 and hybrid eclipses from April 14, 1828 through May 6, 1864. It contains total eclipses from May 17, 1882 through August 23, 2044. The series ends at member 72 as a partial eclipse on May 3, 2459. The longest duration of central eclipse (annular or total) was 5 minutes, 46 seconds of annularity on November 22, 1593. The longest duration of totality was 2 minutes, 36 seconds on July 10, 1972.\\n',\n", - " '\\n',\n", - " 'The Western Canada Soccer League was a soccer league in Canada.\\n',\n", - " 'The league operated for eight seasons over a nine-year period from 1963 to 1971 and eventually operated across nine cities in four Canadian provinces and one American state.\\n',\n", - " 'In the 1960s, the Western Canada Soccer League was one of four major leagues in Canadian soccer alongside the Pacific Coast League, the National Soccer League of Ontario/Quebec, and the Eastern Canada Professional Soccer League.\\n',\n", - " 'In 1963, the league featured 10 clubs across five cities in Alberta and Saskatchewan. In 1967, the league expanded to Manitoba, thus becoming the first Canadian soccer league to operate a league schedule across three provinces.\\n',\n", - " 'The league later added teams in British Columbia and, for one season only, a team in Washington, United States.\\n',\n", - " 'Along with a regular season schedule that usually operated from May to August, the Molson Cup playoffs took place in August or September.\\n',\n", - " \"Teams also participated in other competitions including their provincial playoffs and Canada Soccer's national The Challenge Trophy competition. In 1967, in the weeks after winning the league's playoff Molson Cup, the Calgary Buffalo Kickers became the first and only Western Canada Soccer League team to reach the national final (a 2-1 loss to Toronto Ballymena).\\n\",\n", - " 'The Western Canada Soccer League was co-founded by Alec Castleton and Sam Donaghey, both of whom were involved with the Alberta Soccer Association.\\n',\n", - " \"The league's inaugural match was played on May 4, 1963 in Saskatoon, with opening weekend matches in Calgary and Edmonton postponed because of inclement weather. The price of admission to that opening match at Griffiths Stadium was 50 cents per ticket.\\n\",\n", - " 'Saskatoon Concordia won that first match 7-1 over Saskatoon Hollandia, with Adolf Becker the four-goal hero.\\n',\n", - " \"The most famous player to play for a Western Canada League team, albeit not in league action, was Stanley Matthews who dressed for Calgary ATCO in two exhibition matches in July 1964. Matthews joined the Calgary squad under player-manager Les Medley, who was one of Matthews' former England teammates.\\n\",\n", - " 'The league featured a single table from 1963 to 1969 and then separate Coast and Prairie divisions in 1970 and 1971. There was also a separate playoff (Cup) competition from 1964 to 1970. Regina Concordia SC were the first champions in 1963, but they were upset in the Molson Cup by 10th place Lethbridge Hungária on 24 August 1963. 1963 Regina Concordia SC 1964 Regina Concordia SC 1966 Edmonton Canadians 1967 Edmonton Victoria Canadians 1968 Calgary Buffalo Kickers 1969 Vancouver Spartans 1963 Lethbridge Hungária SC def. Regina Concordia SC 1964 Regina Concordia SC def.\\n',\n", - " 'Lethbridge Hungária SC 1966 Calgary Buffalo Kickers def.\\n',\n", - " 'Regina Concordia SC 1967 Calgary Buffalo Kickers def.\\n',\n", - " 'Saskatoon City SC 1968 (finalists Calgary Buffalo Kickers v Edmonton Victoria Canadians) 1969 Vancouver Spartans def.\\n',\n", - " 'Regina Concordia SC 1970 Victoria Royals def.\\n',\n", - " 'Regina Concordia SC Calgary ATCO (1964) Calgary Kickers (1966-1969) Calgary Croatia (1963-64) Edmonton Canadians / Edmonton Victoria Canadians (1966- 1969) Edmonton City (1970) / Edmonton Eagles (1971) Edmonton Edelweiss (1963-1964) Edmonton Excelsior (1963-1964) Edmonton Hungária (1963-1964) Edmonton Rangers (1963-1964) Lethbridge Hungária SC (1963-1964) Regina Concordia SC (1963-1964, 1966-1971) Regina Internationals (1963) Saskatoon City SC (1966-1969) Saskatoon Concordia (1963-1964) Saskatoon Hollandia (1963) Saskatoon Internationals (1964) Seattle Sea-Tac (1970) Vancouver Cougars (1970) Vancouver Spartans (1969-1971) Victoria Royals (1970-1971) Winnipeg International FC (1967-1968) Winnipeg Blues (1969-1971) Eleven players from the Western Canada Soccer League have since been inducted in the Canada Soccer Hall of Fame as honoured players.\\n',\n", - " 'Bob Bolitho (1970) Tony Chursky (1971) Neil Ellett (1970) Glen Johnson (1969 to 1971) Victor Kodelja (1971) Bob Lenarduzzi (1970 and 1971) Sam Lenarduzzi (1970) Buzz Parsons (1969 and 1970) Ken Pears (1969) Brian Robinson (1970) Bruce Wilson (1970)\\n',\n", - " '\\n',\n", - " 'iRadio is a regional radio station in the Republic of Ireland that broadcasts to the north-east, midlands, north-west and west of the state.\\n',\n", - " 'The station is one of four regional youth oriented stations that were licensed by the Broadcasting Authority of Ireland to challenge the current duopoly in the 15 to 34 age bracket for those outside Dublin by national stations RTÉ 2fm and Today FM.\\n',\n", - " 'It was created by the merger of i102-104FM and i105-107FM in 2011. The station is based in Athlone and broadcasts to 15 counties between the frequencies of 102.1 MHz and 104.4 MHz (96.9 Donegal) in the north-west and 104.7 MHz and 106.7 MHz in the north-east, as well as offering an online stream.\\n',\n", - " '\\n',\n", - " 'Yves Benoist is a French mathematician, known for his work on group dynamics on homogeneous spaces.\\n',\n", - " 'He is currently a Director of Research (Directeur de Recherche) of CNRS at the University of Paris-Sud.\\n',\n", - " 'In 1990 Benoist proved a long open conjecture with Patrick Foulon and François Labourie about Anosov flows on compact, negatively curved manifolds.\\n',\n", - " 'In the 2000s he wrote a series of papers on the divisible convex sets in projective space and periodic tilings by such sets.\\n',\n", - " 'In 2011 he was awarded the Clay Research Award along with his former doctoral student, Jean-François Quint.\\n',\n", - " 'The prize citation highlighted their work on stationary measures and closed orbits for non-abelian group actions on homogeneous spaces, solving a long-standing conjecture of Hillel Furstenberg.\\n',\n", - " 'They showed that in homogeneous spaces of finite volume, orbits of a Zariski dense subgroup of a semisimple group equidistribute towards algebraic measures.\\n',\n", - " 'He gave the 2012 Takagi Lectures in Kyoto at the Research Institute for Mathematical Sciences (RIMS).\\n',\n", - " 'In 2014 he was an Invited speaker at the International Congress of Mathematicians in Seoul.\\n',\n", - " 'With J.-F.\\n',\n", - " 'Quint: Mesures stationaires et fermés invariants of homogeneous espaces.\\n',\n", - " 'Annals of Mathematics, vol 174, 2011, pp 1111–1162. With J.-F.\\n',\n", - " 'Quint: Random walks on finite volume homogeneous spaces.\\n',\n", - " 'Inventiones Mathematicae, vol 187, 2012, pp 37–59.\\n',\n", - " 'Convexes divisibles I: Algebraic groups and arithmetic, 339–374, Tata Inst Fund.. Res., Mumbai in 2004. Convexes divisibles II.\\n',\n", - " 'Duke Math J. 120 (2003), No 1, 97–120.\\n',\n", - " 'Convexes divisibles III: Ann.\\n',\n", - " 'Sci.\\n',\n", - " 'École Norm. Sup.\\n',\n", - " '(4) 38 (2005), no.\\n',\n", - " '5, 793–832.\\n',\n", - " 'Convexes divisibles IV: Invent.\\n',\n", - " 'Math.\\n',\n", - " '164 (2006), no.\\n',\n", - " '2, 249–278.\\n',\n", - " 'Propriétés\\n',\n", - " 'Geom.\\n',\n", - " 'Funct.\\n',\n", - " 'Anal.\\n',\n", - " '7 (1997), no.\\n',\n", - " '1, 1–47.\\n',\n", - " 'Actions propres sur les espaces homogeneous réductifs.\\n',\n", - " 'Ann.\\n',\n", - " 'of Math.\\n',\n", - " '(2) 144 (1996), no. 2, 315–347.\\n',\n", - " \"With P. Foulon, F. Labourie: Flots d'Anosov à distributions stable et instable différentiables.\\n\",\n", - " 'J. Amer.\\n',\n", - " 'Math.\\n',\n", - " 'Soc.\\n',\n", - " '5 (1992), no.\\n',\n", - " '1, 33–74.\\n',\n", - " \"Jean-François Quint: Convexes divisibles, d'après Yves Benoist.\\n\",\n", - " \"Séminaire Bourbaki, June 2008. François Ledrappier: Mesures sur les espaces station aires homogeneous, d'après Yves Benoist et Jean-François Quint Séminaire Bourbaki, 2012.\\n\",\n", - " '\\n',\n", - " 'Doris Baaten (born April 12, 1956 in Maastricht, Limburg) is a Dutch voice actress.\\n',\n", - " 'Baaten performed various female voices on Sesamstraat, the Dutch co-production of Sesame Street.\\n',\n", - " 'She also provided the voice for female roles, such as Countess Dahling Von Dahling and Deena, and was featured in musical numbers during the 1980s. She also performed in a Dutch version of the musical Chicago.\\n',\n", - " '\\n',\n", - " 'Mahonia is a genus of about 70 species of evergreen shrubs and rarely small trees in the family Berberidaceae, native to eastern Asia, the Himalaya, North and Central America.\\n',\n", - " 'They are closely related to the genus Berberis and botanists disagree on whether to recognize a separate Mahonia.\\n',\n", - " 'Some authorities argue Mahonia should be included in Berberis because several species in both genera are able to hybridize, and because when the two genera are looked at as a whole, no consistent morphological separation exists except simple versus compound leaves.\\n',\n", - " 'However, recent DNA-based phylogenetic studies support recognition of Mahonia, though after the removal of several species into the newly-described genera Alloberberis (formerly Mahonia section Horridae) and Moranothamnus (formerly Mahonia claireae).\\n',\n", - " 'Mahonia species typically have large, pinnate leaves 10–50\\xa0cm (3.9–19.7\\xa0in) long with five to 15 leaflets, and flowers in racemes which are 5–20\\xa0cm (2.0–7.9\\xa0in) long.\\n',\n", - " 'The genus name Mahonia honors the Philadelphia horticulturist Bernard McMahon, who introduced the plant from materials collected by the Lewis and Clark expedition.\\n',\n", - " 'The type species of the genus is Mahonia aquifolium, (Oregon-grape) from the Pacific coast of North America.\\n',\n", - " 'Several species are popular garden shrubs, grown for their ornamental, often spiny, evergreen foliage, yellow (or rarely red) flowers in autumn, winter and early spring, and blue-black berries.\\n',\n", - " 'The flowers are borne in terminal clusters or spreading racemes, and may be among the earliest flowers to appear in the growing season.\\n',\n", - " 'The berries are edible, and rich in vitamin C, though with a very sharp flavor.\\n',\n", - " 'Although edible, the plants contain berberine, a compound found in many Berberis and Mahonia species, which can cause vomiting, lowered blood pressure, reduced heart rate, lethargy, and other ill effects when consumed in large quantities.\\n',\n", - " 'The following list includes all currently recognized species of the genus Mahonia as accepted by Tropicos, Missouri Botanical Garden as of February 2016, sorted alphabetically.\\n',\n", - " 'For each, binomial name is followed by author citation. Mahonia aquifolium Mahonia bealei (Fortune) Carrière Mahonia bodinieri Gagnep.\\n',\n", - " 'Mahonia bracteolata Takeda Mahonia breviracema Y.S.\\n',\n", - " 'Wang & P.G.\\n',\n", - " 'Xiao Mahonia cardiophylla T.S.\\n',\n", - " 'Ying & Boufford Mahonia decipiens C.K.\\n',\n", - " 'Schneid.\\n',\n", - " 'Mahonia duclouxiana Gagnep.\\n',\n", - " 'Mahonia eurybracteata Fedde Mahonia fordii C.K.\\n',\n", - " 'Schneid.\\n',\n", - " 'Mahonia fortunei (Lindl.) Fedde Mahonia fremontii (Torr.) Fedde Mahonia gracilipes (Oliv.) Fedde Mahonia hancockiana Takeda Mahonia imbricata T.S.\\n',\n", - " 'Ying & Boufford Mahonia japonica (Thunb.) DC.\\n',\n", - " 'Mahonia leptodonta Gagnep.\\n',\n", - " 'Mahonia longibracteata Takeda Mahonia miccia Buch.-Ham.\\n',\n", - " 'ex D. Don Mahonia microphylla T.S.\\n',\n", - " 'Ying & G.R.\\n',\n", - " 'Long Mahonia monyulensis Ahrendt Mahonia moranensis (Schult. & Schult. f.) I.M. Johnstone Mahonia napaulensis DC.\\n',\n", - " 'Mahonia nitens C.K.\\n',\n", - " 'Schneid.\\n',\n", - " 'Mahonia oiwakensis Hayata Mahonia\\n',\n", - " 'Wu ex S.Y.\\n',\n", - " 'Bao Mahonia polyodonta Fedde Mahonia retinervis P.G. Xiao & Y.S.\\n',\n", - " 'Wang Mahonia setosa Gagnep.\\n',\n", - " 'Mahonia shenii Chun Mahonia sheridaniana C.K.\\n',\n", - " 'Schneid.\\n',\n", - " 'Mahonia subimbricata Chun & F. Chun Mahonia taronensis Hand.-Mazz.\\n',\n", - " 'Mahonia tenuifolia (Lindl.) Loudon ex Fedde Mahonia tinctoria (Terán & Berland.) I.M. Johnst.\\n',\n", - " 'Mahonia volcanica Standl.\\n',\n", - " '& Steyerm.\\n',\n", - " '\\n',\n", - " 'XO Skeletons is an electronic/hardcore/punk group from Waterbury, Connecticut, featuring well-known Boston hardcore legend Wesley Eisold from bands such as American Nightmare and Cold Cave.\\n',\n", - " 'Other members include Chauncey Johnson, F. Sean Martin, and Nicky Money.\\n',\n", - " 'XO Skeletons was notorious for their do-it-yourself mentality.\\n',\n", - " 'The band released their debut album Bored By Heaven for free, stating that \"It\\'s 2007.. Who needs a record label?\"\\n',\n", - " 'Asthmagasm No Muzzles Asthmagasm Suburban Dust XO Skeletons Bored By Heaven Town Crier Spider Earth Between Spider Heaven And Hell Better Babes Stranger Tongues Live From Planet Death Birthdays Under The Ice Of Hell Freezing Over Lascivious Facts (feat. Cage) California Needs You And Me\\n',\n", - " '\\n',\n", - " 'Bishop Solomon Doraiswamy was the second Bishop-in-Trichy-Tanjore Diocese of the Church of South India whose bishopric lasted from 1964 to 1982. Doraiswamy elected as the second Bishop - in - Trichy-Tanjore Diocese of the Church of South India who was principally consecrated in 1964 by Moderator, A. H. Legg and co-consecrated by P. Solomon, the Deputy Moderator.\\n',\n", - " \"During the fourteenth Church of South India Synod held from 10–14 January 1974 at the Women's Christian College, Madras, Doraiswamy was elected as the Deputy Moderator and held the office from 1974 to 1980 for over three terms (1974-1976; 1976-1978 and 1978-1980).\\n\",\n", - " 'Again during the seventeenth Church of South India Synod held from 10–14 January 1980 at Madras Christian College, Tambaram, Doraiswamy became the Moderator and held the office for a term up to 1982. Doraiswamy retired from the bishopric in 1982 on attaining superannuation.\\n',\n", - " 'The Senate of Serampore College (University) awarded an honorary doctorate degree upon Doraiswamy in 1981.\\n',\n", - " '\\n',\n", - " 'The Antilles Episcopal Conference (AEC) is a Roman Catholic episcopal conference.\\n',\n", - " 'Its members are bishops and archbishops from current and former British, Dutch, and French colonies and dependencies in the Caribbean (excluding Haiti), Central America, and northern South America.\\n',\n", - " \"The conference's membership includes five archdioceses, fourteen dioceses, and two missions sui iuris.\\n\",\n", - " 'These particular Churches minister to Catholics in thirteen independent nations, six British Overseas Territories, three departments of France, three countries of the Kingdom of the Netherlands and 3 municipalities of the Netherlands proper.\\n',\n", - " 'The bishop from an American insular area has been granted observer status.\\n',\n", - " 'The episcopal conference is led by a president, who must be a diocesan ordinary and is elected by the membership of the conference for a three-year term.\\n',\n", - " 'The conference also elects a vice president, who has the same qualifications as the president, and a treasurer, who can be a diocesan ordinary, a coadjutor bishop, or an auxiliary bishop.\\n',\n", - " 'Additionally, a permanent board — consisting of the president, vice president, treasurer, the metropolitan archbishops and two other elected members — handles administrative issues between plenary meetings of the conference.\\n',\n", - " 'The president of the conference is currently Gabriel Malzaire, Bishop of Roseau, while the vice president is Charles Jason Gordon, Archbishop of Port of Spain.\\n',\n", - " 'The Holy See appoints an apostolic delegate to the Antilles Episcopal Conference, who also serves as the Apostolic nuncio (papal ambassador) to the independent nations of the conference, except Belize.\\n',\n", - " 'The nunciature is located in Port of Spain, Trinidad and Tobago.\\n',\n", - " 'The current apostolic delegate is Archbishop Fortunatus Nwachukwu, who replaced Archbishop Nicola Girasoli after he was appointed as Apostolic Nuncio to Peru.\\n',\n", - " 'Antilles Episcopal Conference GCatholic\\n',\n", - " '\\n',\n", - " 'Richard \"Dick\" Briefer (January 9, 1915 – December 1980) was an American comic-book artist best known for his various\\n',\n", - " 'Under the pseudonym Dick Hamilton, he also created the superhero team the Target and the Targeteers for Novelty Press.\\n',\n", - " 'Dick Briefer studied at the Art Students League in Manhattan, New York City, and debuted in comic books in 1936 with work in Wow, What A Magazine!, one of the era\\'s proto-comics \"Comic books\": tabloid-sized collections of comic strip reprints in color, which would later include occasional new comic strip-like material.\\n',\n", - " 'Wow was edited by Jerry Iger, and when the comic ceased publication with issue #4 (cover-dated Nov. 1936), Briefer freelanced for the newly formed Eisner & Iger, one of the earliest \"packagers\" that produced complete comics on demand for publishers entering the fledgling medium.\\n',\n", - " \"Briefer's earliest recorded credit is as writer and artist of a five-page story beginning an adaptation of the Victor Hugo novel The Hunchback of Notre Dame, in Jumbo Comics #1-8 & 10 (Sept. 1938 - July 1939 & Nov. 1939), for the Eisner-Iger client Fiction House.\\n\",\n", - " 'Other seminal work includes drawing and possibly writing the science-fiction adventure feature \"Rex Dexter of Mars\", which ran in several issues of Fox Comics\\' Mystery Men Comics;\\n',\n", - " '\"Dynamo\" in Fox\\'s Science Comics;\\n',\n", - " '\"Biff Bannon\" in Harvey Comics\\' Speed Comics;\\n',\n", - " '\"Storm Curtis\" in Prize Comics\\' Prize Comics;\\n',\n", - " 'and \"Crash Parker\" in Fiction House\\'s Planet Comics.\\n',\n", - " 'For Timely Comics, the precursor of Marvel Comics during the 1930s to 1940s period fans and historians call the Golden Age of Comic Books, Briefer created or co-created (writer credit unknown) the single-appearance superhero the Human Top in Red Raven #1 (Aug. 1940).\\n',\n", - " 'Also during this time he also drew the comic strip Pinky Rankin, about a Nazi-fighter, for the American Communist Party newspaper The Daily Worker.\\n',\n", - " 'Briefer, using the pen name Dick Hamilton, created the superhero team the Target and the Targeteers for Novelty Press in 1940. The Target first appeared in Target Comics #10 (Nov. 1940), and the Targeteers the following issue.\\n',\n", - " 'The team starred in Target Comics through issue #95 / vol.\\n',\n", - " '9, #5 (July 1948).\\n',\n", - " 'Target itself ran 10 more issues. In Prize Comics #7 (Dec. 1940), writer-artist Briefer (using the pseudonym \"Frank N. Stein\" in the latter role) introduced the eight-page feature \"New Adventures of Frankenstein\", an updated version of the much-adapted Frankenstein monster created by Mary Shelley in her 1818 novel Frankenstein.\\n',\n", - " \"It's nice.\\n\",\n", - " \"[...] And they're funny as comics.\\n\",\n", - " 'They read well and are beautifully drawn;\\n',\n", - " \"they're full of unforgettable images, like the wizard eating Frankenstein on a hot dog.\\n\",\n", - " \"You'll never forget it, for better or for worse.\\n\",\n", - " \"Briefer's humorous Frankenstein ran through Prize Comics #68 (March 1948), and his humorous Frankenstein ran through issue #17 (Feb. 1949).\\n\",\n", - " 'Three years later, Briefer revived the series with his original, horrific Frankenstein from #18-33 (March 1952 - Nov. 1954).\\n',\n", - " 'Following the cancellation of Frankenstein during an era that put much pressure on horror comics and other violent comic books, leading to the creation of the Comics Code, Briefer left the comic industry for commercial advertising art. At the time of his death, Briefer was living in the Hollywood / Pembroke Pines area of Broward County, Florida.\\n',\n", - " 'Briefer, Dick.\\n',\n", - " 'The Monster of Frankenstein (Idea Men Productions, 2006) ISBN\\xa01-4196-4017-8, ISBN\\xa0978-1-4196-4017-9 Briefer, Dick.\\n',\n", - " \"Dick Briefer's Frankenstein (Library of Horror Comics' Masters, IDW/Yoe Books, 2010) ISBN\\xa01-60010-722-2\\n\",\n", - " '\\n',\n", - " \"Brown Estate Vineyards is the first and only Black-owned estate winery in California's Napa Valley (USA), and is best known as one of that region's most well-regarded zinfandel producers.\\n\",\n", - " 'In addition to zinfandel, Brown Estate produces cabernet sauvignon, chardonnay, and petite sirah.\\n',\n", - " 'Founded in 1995 by siblings Deneen,\\n',\n", - " 'Abandoned for some ten years prior, the land and its two structures - an 1859 stone and redwood barn and an 1885 Queen Anne Victorian home - were derelict. The senior Browns cut roads and brought in plumbing and electricity, ultimately earning an award from the Napa County Historical Society for their restoration of the residence structure.\\n',\n", - " 'In 1985 they planted their first vineyard on the property, approximately nine acres of zinfandel.\\n',\n", - " 'The Chiles Valley microclimate, characterized by extreme temperature\\n',\n", - " 'As well, the Browns sold cabernet sauvignon to the famed Mike Grgich.\\n',\n", - " \"It was through a series of apprenticeships with winemakers who were working with his fruit that David Brown - who by then had been farming his family's vineyards for five years - began learning the art and science of winemaking.\\n\",\n", - " 'In the mid-1990s, as a result of rising demand for their fruit, Deneen and David, then both residing on the vineyard property, made the decision to produce wine under their own label.\\n',\n", - " 'Because they had no winery facility on site, they secured a custom crush contract with Rombauer Vineyards in Saint Helena, and for six years they produced their wines at the Rombauer facility.\\n',\n", - " 'On January 29, 2000, Brown Estate debuted their first two vintages of Napa Valley zinfandel, 1996 and 1997, at the annual Zinfandel Advocates and Producers (ZAP) tasting at Fort Mason in San Francisco.\\n',\n", - " 'ZAP is the largest single varietal wine tasting event in the world, and the Brown offerings were very well received by the trade and public alike, thereby establishing Brown Estate as a winery to watch in the arena of Napa Valley zinfandel.\\n',\n", - " 'Simultaneously, in the January 31, 2000 issue of the Wine Spectator, the 1997 Brown Estate zinfandel received a score of 91 points.\\n',\n", - " \"Then, in June 2000, tragedy struck when a fire broke out at the warehouse facility where Brown's wine library and recently bottled 1998 vintage were stored.\\n\",\n", - " \"With the exception of sixteen bottles - two that went to Robert M. Parker, Jr. for review, two that went to Christie's for auction, and twelve that the Browns had stored at home - all of their 1998 zinfandel was destroyed, as were the remaining cases of their first two vintages.\\n\",\n", - " 'The lost 1998 vintage left the Browns out of the zinfandel market for one whole year, a devastating setback that was mitigated only by the kindness of restaurant and retail accounts who held space on their wine lists and shelves for the next vintage of Brown zin.\\n',\n", - " 'In order to bridge that gap, the Browns accelerated the release of their 1999 Napa Valley zinfandel, which they showed at ZAP in 2001. The previous month, in December 2000, Parker, the most influential wine critic in the world, gave the Brown 1998 zinfandel a score of 90, noting that his review was \"of academic interest only\" since all of the wine had perished in the fire.\\n',\n", - " \"The loss of the 1998 vintage has kept Brown Estate's Napa Valley zinfandel bottlings on a perennially early release schedule.\\n\",\n", - " 'Meanwhile, the Browns and their small production of a few thousand cases slowly were being squeezed out of the Rombauer facility. In order to continue their now quite promising winemaking endeavor, they were compelled to establish a winery facility on their own property.\\n',\n", - " 'The location of the old stone and redwood barn, still in its derelict state - doorless, windowless, and with a dirt floor - all but demanded development of an adjacent subterranean wine cave.\\n',\n", - " \"Winery construction began in Summer of 2002 and by September of that year, the old barn had a new lease on life as Brown Estate's on-site wine production facility.\\n\",\n", - " 'Brown then became a full-fledged estate winery.\\n',\n", - " 'Soon production began increasing, and eventually the barrel room in the lower level of the barn was filled to capacity, transforming the wine cave from notion to necessity.\\n',\n", - " 'Excavation commenced in April 2004, and because the host hillside was solid granite, drilling alone was inadequate.\\n',\n", - " 'For one solid year, muffled explosions rocked the Brown Estate property as one of the most unusual wine caves in the Napa Valley literally was designed by dynamite.\\n',\n", - " \"In Fall 2010, the Brown family celebrated their 30th year in the Napa Valley, and Brown Estate's 15th crush.\\n\",\n", - " 'The family posted the following message on their Website to commemorate these occasions: \"This year we celebrate our thirtieth year in the Napa Valley, and Brown Estate\\'s fifteenth crush. We thank all of you, with all of our hearts, for your continued enjoyment of our wines and your truly stupendous support of our once little endeavor. Each time you open and share a bottle of Brown Estate, you wrap your arms around us. It is an embrace that we cherish, a gesture that gives meaning to all of the passion and commitment we put into what we do. Thank you for making us a part of your lives, and for being a part of ours. Now onward & upward! The sky\\'s the limit!\"\\n',\n", - " \"In November 2010, the Browns released the world's first hashtagged wine, their 2009 Napa Valley Zinfandel. Subsequent vintages of this wine continue to bear the #brownzin hashtag, an homage to Twitter, which has been a source of inspiration and support as Brown Estate has built its reputation as a wine industry leader in the realm of social media.\\n\",\n", - " \"In April 2017, Brown Downtown Napa opened on downtown Napa's re-emerging First Street corridor.\\n\",\n", - " '\\n',\n", - " 'The 2015 Gedling Borough Council election was held on 7 May 2015 to elect all members of Gedling Borough Council in Nottinghamshire, England as part of the English local elections coinciding with the 2015 General Election.\\n',\n", - " 'Each councillor serves a four-year term of office.\\n',\n", - " 'Following Boundary Commission recommendations boundary change took place for this election to provide for 19 wards, many of which are two or three-member, thus electing 41 councillors.\\n',\n", - " 'Labour Party candidates won a majority, exceeding the 21 member threshold needed to reach majority party status, formerly having 64% of councillors.\\n',\n", - " \"The Conservative group was formerly the opposition party on the council, holding 15 seats out of 50 on the larger council between 2011 and 2015. The remainder of councillors comprised Liberal Democrat members, formerly holding 3 seats on the council from 2011 to 2015, returned 1 of the council's 41 councillors.\\n\",\n", - " '\\n',\n", - " 'Superpedestrian Inc., based in Cambridge, MA, develops technological solutions to improve urban mobility worldwide.\\n',\n", - " 'Superpedestrian specializes in micro-mobility technologies which enable the development of small-scale electric vehicles that can sense and adapt to the road and the driver.\\n',\n", - " 'The company works with ride-hailing and electric bike sharing companies to create electric-bike fleets optimized for safety, reliability, and enjoyable riding experiences.\\n',\n", - " 'Their consumer-facing product, the Copenhagen Wheel, is the first human-enhancing transportation technology available to the mass market.\\n',\n", - " \"Superpedestrian's first product, The Copenhagen Wheel, was developed at MIT’s Senseable City Lab in 2009 in partnership with the city of Copenhagen, and unveiled at the 2009 United Nations Climate Change Conference.\\n\",\n", - " 'In December 2012, Assaf Biderman, a co-inventor of the Wheel and Associate Director of the MIT Senseable City Lab, founded Superpedestrian to commercialize the Wheel.\\n',\n", - " 'After several years of engineering, testing, and validation, the Copenhagen Wheel officially launched in April 2017. The world is currently experiencing the fastest wave of urbanization in human history, leading to unprecedented levels of traffic congestion. Superpedestrian’s vehicle technology was developed in direct response to rising urban populations and urban transportation demand.\\n',\n", - " 'The current average car in the U.S. carries only 1.55 people per vehicle, but occupies enough road space to fit five. Many people assume that self-driving cars in the future will be used as shared vehicles, but research shows it is unlikely that autonomous vehicles will be shared for the majority of the time.\\n',\n", - " 'Even with a steep discount in the price of a ride, surveys show only 52% of riders will be willing to share an autonomous vehicle.\\n',\n", - " 'As a result, we are unlikely to meet future transportation needs without turning to micro-mobility - one-to-two person vehicles that enable individualized trips while taking up less road space.\\n',\n", - " 'Superpedestrian’s first foray into micro-mobility was through the bicycle.\\n',\n", - " 'Bicycles have always been the most popular form of single-occupancy vehicle.\\n',\n", - " 'However, since their introduction, the average city has grown 20x in land area, while the bicycle has evolved very little.\\n',\n", - " 'As a result, the bicycle is no longer a viable transportation solution for the many people who commute through these expanded cities.\\n',\n", - " 'Electric bicycles offer the opportunity to transform the humble bicycle into a compelling alternative to the car.\\n',\n", - " 'The Copenhagen Wheel is a rear wheel electric bicycle system that fits most existing bicycle frames - requiring a metal frame and V-brakes.\\n',\n", - " 'It replaces an existing rear wheel, transforming the bicycle into a smart electric hybrid/pedelec.\\n',\n", - " 'The Copenhagen Wheel contains a custom brushless motor, advanced sensors, control systems, and a lithium-ion battery, all contained within the hub of a single rear bicycle wheel.\\n',\n", - " 'Combining actual torque, power, cadence, pedal position, and acceleration sensing with high-speed controllers and actuators, the Wheel generates power that is seamlessly synchronized with your pedaling.\\n',\n", - " 'It’s just like riding a bike, but with a superhuman boost.\\n',\n", - " 'Bluetooth connectivity enables riders to personalize their cycling experience from their smartphone with ride modes such as Turbo, Eco, and Exercise.\\n',\n", - " 'A smartphone also acts as a digital key, activating the Wheel automatically when ready to ride, and communicating with the cloud in real time.\\n',\n", - " 'A self-diagnostic safety system monitors components within the Wheel and proactively responds to events within milliseconds, protecting both rider and Wheel.\\n',\n", - " 'Electronic Braking Assistance allows riders to brake and recharge the battery simultaneously, simply by backpedaling.\\n',\n", - " 'Finally, with over-the-air updates, the Wheel improves with new features over time.\\n',\n", - " 'The Copenhagen Wheel is available throughout the U.S., Canada, and Europe.\\n',\n", - " 'Superpedestrian has manufacturing facilities in both Massachusetts and the Netherlands.\\n',\n", - " 'Superpedestrian offers a complete bike product as well, called Wheel + Bike.\\n',\n", - " 'Customers can choose from a wide range of bicycle frames, from OEM’s like Marin, Public, Montague, Fyxation, Tern, Cinelli, and Fortified to pair with the Copenhagen Wheel.\\n',\n", - " 'Superpedestrian also works with over 260 independent bicycle retailers across the United States and Europe to provide demos, maintenance, and installation services.\\n',\n", - " 'Superpedestrian works with ride-hailing and e-bike-sharing companies worldwide to create fleets of small electric vehicles optimized for safety, reliability, and seamless riding experiences.\\n',\n", - " 'As the industry expands to include e-bikes, scooters, and mopeds, fleet operators contend with the challenges of managing and maintaining large numbers of motorized vehicles.\\n',\n", - " 'Building on Superpedestrian’s robust sensing, monitoring, and control systems developed for the Copenhagen Wheel, Superpedestrian offers solutions to common fleet operation problems.\\n',\n", - " 'Their platform allows operators to remotely monitor fleet health, boost maintenance efficiency, manage user accounts, and improve customer experiences, leading to lower operating costs and increased revenue.\\n',\n", - " 'Superpedestrian is currently working on future versions of the Copenhagen Wheel with updated features and wider compatibility requirements.\\n',\n", - " 'In the meantime, they offer over the air firmware updates to current riders.\\n',\n", - " 'Superpedestrian has raised more than $43 million in venture capital investment from firms including Cambridge-based General Catalyst and Spark Capital.\\n',\n", - " 'Additional investors include Tony Fadell, Charles Kim, Jared Leto, Xavier Niel, Paul Sagan, and Christian Klingler.\\n',\n", - " '\\n',\n", - " \"Alice Doesn't Live Here Anymore is a 1974 American romantic comedy-drama film directed by Martin Scorsese and written by Robert Getchell.\\n\",\n", - " 'It stars Ellen Burstyn as a widow who travels with her preteen son across the Southwestern United States in search of a better life.\\n',\n", - " 'Kris Kristofferson, Billy \"Green\" Bush, Diane Ladd, Valerie Curtin, Lelia Goldoni, Vic Tayback, Jodie Foster, Alfred Lutter and Harvey Keitel are featured in supporting roles.\\n',\n", - " \"Alice Doesn't Live Here Anymore premiered at the 27th Cannes Film Festival where it competed for the Palme d'Or and was released theatrically on December 9, 1974, by Warner Bros. The film was a critical and commercial success, grossing $21 million on a $1.8 million budget.\\n\",\n", - " 'At the 47th Academy Awards, Burstyn won Best Actress, while Ladd and Getchell received nominations for Best Supporting Actress and Best Original Screenplay.\\n',\n", - " \"When Socorro, New Mexico, housewife Alice Hyatt's uncaring husband Donald is killed in an auto accident, she decides to have a garage sale, pack what's left of her meager belongings, and take her precocious son Tommy to her childhood hometown of Monterey, California, where she hopes to pursue the singing career she abandoned when she married.\\n\",\n", - " 'Their financial situation forces them to take temporary lodgings in Phoenix, Arizona, where she finds work as a lounge singer in a seedy bar.\\n',\n", - " 'There she meets Ben, who uses his charm to lure her into a sexual relationship that comes to a sudden end when his wife Rita confronts Alice.\\n',\n", - " \"Ben breaks into Alice's apartment while Alice is there and physically assaults Rita for interfering with his extramarital affair.\\n\",\n", - " 'When Alice tells Ben to calm down, he threatens her also and further smashes up the apartment.\\n',\n", - " 'Fearing for their safety, Alice and Tommy quickly leave town.\\n',\n", - " 'Having spent most of the little money she earned on a new wardrobe, Alice is forced to delay her journey to the West Coast and accept a job as a waitress in Tucson so she can accumulate more cash.\\n',\n", - " \"At the local diner owned by Mel, she eventually bonds with her fellow servers - independent, no-nonsense, outspoken Flo and quiet, timid, incompetent Vera — and meets divorced local rancher David, who soon realizes the way to Alice's heart is through Tommy.\\n\",\n", - " 'Still emotionally wounded from the difficult relationship she had with her uncommunicative husband and the frightening encounter she had with Ben, Alice is hesitant to get involved with another man so quickly.\\n',\n", - " 'However, she finds out that David is a good influence on Tommy, who has befriended wisecracking, shoplifting, Ripple-guzzling Audrey, a slightly older girl forced to fend for herself while her mother makes a living as a prostitute.\\n',\n", - " 'Alice and David warily fall in love, but their relationship is threatened when Alice objects to his discipline of the perpetually bratty Tommy.\\n',\n", - " 'The two reconcile, and David offers to sell his ranch and move to Monterey so Alice can try to fulfill her childhood dream of becoming another Alice Faye.\\n',\n", - " 'In the end, Alice decides to stay in Tucson, coming to the conclusion that she can become a singer anywhere.\\n',\n", - " 'The part of Alice was originally offered to Shirley MacLaine.\\n',\n", - " 'However, MacLaine turned down the role.\\n',\n", - " 'MacLaine admitted in a 2005 interview that she regretted this decision.\\n',\n", - " 'Ellen Burstyn was still in the midst of filming The Exorcist when Warner Bros. executives expressed interest in working with her on another project.\\n',\n", - " 'It hadn\\'t been released yet, so I booked a screening to look at it and I felt that it was exactly what…Alice needed, because [it] was a wonderful script and well written, but for my taste it was a little slick. You know – in a good way, in a kind of Doris Day–Rock Hudson kind of way. I wanted something a bit more gritty.\"\\n',\n", - " 'Burstyn described her collaboration with director Martin Scorsese, making his first Hollywood studio production, as \"one of the best experiences I\\'ve ever had\".\\n',\n", - " 'The director agreed with his star that the film should have a message. \"It\\'s a picture about emotions and feelings and relationships and people in chaos,\" he said.\\n',\n", - " '\"We felt like charting all that and showing the differences and showing people making terrible mistakes ruining their lives and then realizing it and trying to push back when everything is crumbling – without getting into soap opera. We opened ourselves up to a lot of experimentation.\"\\n',\n", - " \"Scorsese's casting director auditioned 300 boys for the role of Tommy before they discovered Alfred Lutter.\\n\",\n", - " '\"I met the kid in my hotel room and he was kind of quiet and shy,\" Scorsese said. But when he paired him with Burstyn and suggested she deviate from the script, he held his own.\\n',\n", - " '\"Usually, when we were improvising with the kids, they would either freeze and look down or go right back to the script. But this kid, you couldn\\'t shut him up.\"\\n',\n", - " 'The film was shot on location predominantly in and around Tucson, but some scenes were shot in Amado, and Phoenix.\\n',\n", - " \"A Mel's Diner still exists in Phoenix.\\n\",\n", - " 'The soundtrack includes \"All the Way from Memphis\" by Mott the Hoople;\\n',\n", - " '\"Roll Away the Stone\" by Leon Russell;\\n',\n", - " '\"Daniel\" by Elton John;\\n',\n", - " '\"Jeepster\" by T-Rex;\\n',\n", - " 'and \"I Will Always Love You\" by Dolly Parton.\\n',\n", - " 'During her lounge act, Alice sings \"Where or When\" by Richard Rodgers and Lorenz Hart;\\n',\n", - " '\"When Your Lover Has Gone\" by Einar Aaron Swan;\\n',\n", - " '\"Gone with the Wind\" by Allie Wrubel and Herb Magidson;\\n',\n", - " 'and \"I\\'ve Got a Crush on You\" by George and Ira Gershwin.\\n',\n", - " 'In a film clip from Coney Island, Betty Grable is heard singing \"Cuddle Up a\\n',\n", - " 'and in a film clip from Hello Frisco, Hello, Alice Faye performs \"You\\'ll Never Know\" by Harry Warren and Mack Gordon.\\n',\n", - " 'Upon its release, the film received critical acclaim and grossed $21 million worldwide.\\n',\n", - " 'Roger Ebert of the Chicago Sun-Times called the film \"one of the most perceptive, funny, occasionally painful portraits of an American woman I\\'ve seen\" and commented, \"The movie has been both attacked and defended on feminist grounds, but I think it belongs somewhere outside ideology, maybe in the area of contemporary myth and romance.\"\\n',\n", - " \"Ebert put the film at #3 of his list of the best films of 1975 (even though the film came out in '74).\\n\",\n", - " 'The film did not go without its detractors, however.\\n',\n", - " 'Variety thought the film was \"a distended bore,\" saying it \"takes a group of well-cast film players and largely wastes them on a smaller-than-life film — one of those \\'little people\\' dramas that makes one despise little people.\"\\n',\n", - " 'TV Guide rated the film three out of four stars, calling it an \"effective but uneven work\" with performances that \"cannot conceal the storyline\\'s shortcomings.\"\\n',\n", - " 'Ellen Burstyn won the Academy Award for Best Actress.\\n',\n", - " 'Diane Ladd was nominated for the Academy Award for Best Supporting Actress but lost to Ingrid Bergman in Murder on the Orient Express, and Robert Getchell was nominated for the Academy Award for Best Original Screenplay but lost to Robert Towne for Chinatown.\\n',\n", - " 'The film won the BAFTA Award for Best Film, and BAFTA Awards went to Burstyn for Best Actress in a Leading Role, to Diane Ladd for Best Actress in a Supporting Role, and to Getchell for Best Screenplay.\\n',\n", - " 'Martin Scorsese was nominated for Best Direction but lost to Stanley Kubrick for Barry Lyndon.\\n',\n", - " \"Getchell was nominated for the Writers Guild of America Award for Best Original Screenplay, Burstyn and Ladd were nominated for Golden Globe Awards for Best Actress in a Motion Picture Drama and Best Supporting Actress in a Motion Picture, respectively, and Scorsese was nominated for the Palme D'Or at the 1975 Cannes Film Festival.\\n\",\n", - " 'The film inspired the sitcom Alice, which was broadcast by CBS from August 1976 through July 1985. The only member of the film cast to reprise his role was Vic Tayback as Mel (though his diner was moved to Phoenix).\\n',\n", - " 'Alfred Lutter portrayed Tommy in the pilot episode but was replaced by Philip McKeon for the series.\\n',\n", - " 'Diane Ladd joined the show later in its run, but in a role different from that she had played in the film.\\n',\n", - " 'Warner Home Video released the film on Region 1 DVD on August 17, 2004. It is in anamorphic widescreen format with audio tracks in English and French and subtitles in English, French, and Spanish.\\n',\n", - " 'Bonus features include commentary by Martin Scorsese, Ellen Burstyn, and Kris Kristofferson and Second Chances, a background look at the making of the film.\\n',\n", - " 'The movie was parodied in The Simpsons by Troy McClure in \"Alice Doesn\\'t Live Anymore\".\\n',\n", - " '\\n',\n", - " 'The Quantic Soul Orchestra is a live band project of musician and DJ, Will Holland.\\n']" - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "lines[1000:1500]" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "advance-daniel", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Using custom data configuration default-e9c2b1c84264b5e7\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Downloading and preparing dataset text/default (download: Unknown size, generated: Unknown size, post-processed: Unknown size, total: Unknown size) to /mnt/data1/hf_dataset_cache/text/default-e9c2b1c84264b5e7/0.0.0/e16f44aa1b321ece1f87b07977cc5d70be93d69b20486d6dacd62e12cf25c9a5...\n" - ] - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=1.0, bar_style='info', layout=Layout(width='20px'), max=1.0), HTML(value=''…" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Dataset text downloaded and prepared to /mnt/data1/hf_dataset_cache/text/default-e9c2b1c84264b5e7/0.0.0/e16f44aa1b321ece1f87b07977cc5d70be93d69b20486d6dacd62e12cf25c9a5. Subsequent calls will reuse this data.\n" - ] - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "c3509c52bbbe44e9aa5a3876b42c3ad8", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "HBox(children=(FloatProgress(value=0.0, max=1.0), HTML(value='')))" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2021-11-10 07:52:58.791 | INFO | newlm.lm.elmo.lm_builder:_get_dataset:142 - Constructing roBERTa style dataset\n", - "100%|██████████| 12/12 [00:00<00:00, 39787.86it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "13\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\n" - ] - } - ], - "source": [ - "# dataset\n", - "train_path = \"/mnt/data4/made_workspace/newlm-data/en.1-percent/text_small.txt\"\n", - "dataset = elmo_lm_builder._get_dataset(train_path)" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "undefined-butler", - "metadata": {}, - "outputs": [], - "source": [ - "# model\n", - "config = BertConfig(**elmo_lm_builder.model_config)\n", - "model = ELMOBertLMHeadModel(config=config)" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "id": "social-refund", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "max_steps is given, it will override any value given in num_train_epochs\n" - ] - } - ], - "source": [ - "# trainer\n", - "from transformers import TrainingArguments, Trainer\n", - "\n", - "output_dir = \"tmp-outputs\"\n", - "\n", - "args = TrainingArguments(\n", - " output_dir=output_dir,\n", - " overwrite_output_dir=True,\n", - " **{\n", - " \"per_device_train_batch_size\": 1,\n", - " \"max_steps\": 50, # scale this based on data size\n", - " \"save_total_limit\": 1,\n", - " \"prediction_loss_only\": True,\n", - " # optimizer, according to the paper\n", - " \"learning_rate\": 0.0001,\n", - " \"warmup_steps\": 10000, # do we scale this as well?\n", - " \"weight_decay\": 0.01,\n", - " },\n", - ")\n", - "trainer = Trainer(\n", - " model=model,\n", - " args=args,\n", - " train_dataset=dataset,\n", - " data_collator=elmo_lm_builder.data_collator,\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "id": "traditional-whole", - "metadata": {}, - "outputs": [], - "source": [ - "data_loader = trainer.get_train_dataloader()" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "id": "considered-cotton", - "metadata": {}, - "outputs": [], - "source": [ - "b1 = next(iter(data_loader))" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "id": "above-entrance", - "metadata": {}, - "outputs": [], - "source": [ - "# elmo_lm_builder.tokenizer.__dict__" - ] - }, - { - "cell_type": "code", - "execution_count": 31, - "id": "foreign-methodology", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "input_ids\n", - "attention_mask\n", - "labels\n" - ] - } - ], - "source": [ - "for k in b1:\n", - " print(k)\n", - " b1[k].to('cuda:0')" - ] - }, - { - "cell_type": "code", - "execution_count": 42, - "id": "sublime-pontiac", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['[CLS]',\n", - " 'National',\n", - " 'Route',\n", - " 'A',\n", - " '##00',\n", - " '##3',\n", - " 'also',\n", - " 'known',\n", - " 'as',\n", - " 'T',\n", - " '##ig',\n", - " '##re',\n", - " 'Access',\n", - " 'is',\n", - " 'an',\n", - " '8',\n", - " '.',\n", - " '8',\n", - " 'km',\n", - " '(',\n", - " '5',\n", - " '.',\n", - " '5',\n", - " 'mi',\n", - " ')',\n", - " '-',\n", - " 'long',\n", - " 'four',\n", - " '-',\n", - " 'lane',\n", - " 'highway',\n", - " '.',\n", - " 'It',\n", - " 'goes',\n", - " 'from',\n", - " 'the',\n", - " 'junction',\n", - " 'with',\n", - " 'National',\n", - " 'Route',\n", - " '9',\n", - " 'and',\n", - " 'Cam',\n", - " '##ino',\n", - " 'de',\n", - " 'C',\n", - " '##int',\n", - " '##ura',\n", - " '(',\n", - " 'Provincial',\n", - " 'Route',\n", - " '4',\n", - " ')',\n", - " 'to',\n", - " 'the',\n", - " 'town',\n", - " 'of',\n", - " 'T',\n", - " '##ig',\n", - " '##re',\n", - " ',',\n", - " 'passing',\n", - " 'the',\n", - " 'towns',\n", - " 'of',\n", - " ':',\n", - " 'Bo',\n", - " '##ulo',\n", - " '##gne',\n", - " 'San',\n", - " 'Is',\n", - " '##id',\n", - " '##ro',\n", - " 'B',\n", - " '##é',\n", - " '##cca',\n", - " '##r',\n", - " 'Victoria',\n", - " 'V',\n", - " '##ir',\n", - " '##rey',\n", - " '##es',\n", - " 'San',\n", - " 'Fernando',\n", - " 'In',\n", - " '1993',\n", - " 'the',\n", - " 'Federal',\n", - " 'Government',\n", - " 'opened',\n", - " 'a',\n", - " 'bid',\n", - " 'for',\n", - " 'the',\n", - " 'Buenos',\n", - " 'Aires',\n", - " 'road',\n", - " 'access',\n", - " 'network',\n", - " '.',\n", - " 'The',\n", - " 'winner',\n", - " 'for',\n", - " 'the',\n", - " 'maintenance',\n", - " 'contract',\n", - " 'for',\n", - " 'the',\n", - " 'North',\n", - " 'Access',\n", - " 'roads',\n", - " 'was',\n", - " 'Auto',\n", - " '##pis',\n", - " '##tas',\n", - " 'del',\n", - " 'Sol',\n", - " '.',\n", - " 'North',\n", - " 'Access',\n", - " 'includes',\n", - " 'Ave',\n", - " '##ni',\n", - " '##da',\n", - " 'General',\n", - " 'Paz',\n", - " ',',\n", - " 'National',\n", - " 'Route',\n", - " 'A',\n", - " '##00',\n", - " '##3',\n", - " ',',\n", - " 'and',\n", - " 'a',\n", - " 'portion',\n", - " 'of',\n", - " 'National',\n", - " 'Route',\n", - " '8',\n", - " ',',\n", - " 'National',\n", - " 'Route',\n", - " '9',\n", - " '.',\n", - " 'Auto',\n", - " '##pis',\n", - " '##tas',\n", - " 'del',\n", - " 'Sol',\n", - " 'signed',\n", - " 'the',\n", - " 'Con',\n", - " '##cession',\n", - " 'Con',\n", - " '##tract',\n", - " 'on',\n", - " '26',\n", - " 'May',\n", - " '1994',\n", - " '.',\n", - " 'To',\n", - " '##lls',\n", - " 'started',\n", - " 'being',\n", - " 'collected',\n", - " 'in',\n", - " 'August',\n", - " '1996',\n", - " ',',\n", - " 'after',\n", - " 'the',\n", - " 'company',\n", - " 'finished',\n", - " 'the',\n", - " 'road',\n", - " 'improvements',\n", - " 'according',\n", - " 'to',\n", - " 'the',\n", - " 'contract',\n", - " '.',\n", - " 'The',\n", - " 'concession',\n", - " 'ends',\n", - " 'on',\n", - " '31',\n", - " 'December',\n", - " '2020',\n", - " '.',\n", - " '[SEP]',\n", - " 'R',\n", - " '##ø',\n", - " '##d',\n", - " '##berg',\n", - " 'is',\n", - " 'the',\n", - " 'administrative',\n", - " 'centre',\n", - " 'of',\n", - " 'Nor',\n", - " '##e',\n", - " 'og',\n", - " 'U',\n", - " '##v',\n", - " '##dal',\n", - " 'municipality',\n", - " 'in',\n", - " 'Bus',\n", - " '##ker',\n", - " '##ud',\n", - " ',',\n", - " 'Norway',\n", - " '.',\n", - " 'R',\n", - " '##ø',\n", - " '##d',\n", - " '##berg',\n", - " 'is',\n", - " 'located',\n", - " 'in',\n", - " 'the',\n", - " 'Norwegian',\n", - " 'traditional',\n", - " 'district',\n", - " 'and',\n", - " 'valley',\n", - " 'of',\n", - " 'N',\n", - " '##ume',\n", - " '##dal',\n", - " '.',\n", - " 'Its',\n", - " 'population',\n", - " '(',\n", - " '2005',\n", - " ')',\n", - " 'is',\n", - " '46',\n", - " '##8',\n", - " ',',\n", - " 'and',\n", - " 'it',\n", - " 'is',\n", - " 'located',\n", - " 'on',\n", - " 'the',\n", - " 'R',\n", - " '##ø',\n", - " '##d',\n", - " '##berg',\n", - " 'dam',\n", - " '(',\n", - " 'Norwegian',\n", - " ':',\n", - " 'R',\n", - " '##ø',\n", - " '##d',\n", - " '##berg',\n", - " '##dam',\n", - " '##men',\n", - " ')',\n", - " 'on',\n", - " 'the',\n", - " 'N',\n", - " '##ume',\n", - " '##dal',\n", - " '##s',\n", - " '##l',\n", - " '##å',\n", - " '##gen',\n", - " 'River',\n", - " '.',\n", - " 'The',\n", - " 'local',\n", - " 'power',\n", - " 'stations',\n", - " ',',\n", - " 'R',\n", - " '##ø',\n", - " '##d',\n", - " '##berg',\n", - " 'Nor',\n", - " '##e',\n", - " '1',\n", - " 'in',\n", - " 'operation',\n", - " '1928',\n", - " 'and',\n", - " 'Nor',\n", - " '##e',\n", - " '2',\n", - " 'in',\n", - " 'operation',\n", - " 'since',\n", - " '1946',\n", - " ',',\n", - " 'utilize',\n", - " 'water',\n", - " '##flow',\n", - " 'from',\n", - " 'the',\n", - " 'R',\n", - " '##ø',\n", - " '##d',\n", - " '##berg',\n", - " 'Dam',\n", - " '.',\n", - " 'The',\n", - " 'plants',\n", - " 'are',\n", - " 'affiliated',\n", - " 'with',\n", - " 'St',\n", - " '##at',\n", - " '##kra',\n", - " '##ft',\n", - " ',',\n", - " 'the',\n", - " 'Norwegian',\n", - " 'state',\n", - " 'owned',\n", - " 'electricity',\n", - " 'company',\n", - " '.',\n", - " 'R',\n", - " '##ø',\n", - " '##d',\n", - " '##berg',\n", - " 'was',\n", - " 'once',\n", - " 'a',\n", - " 'railway',\n", - " 'station',\n", - " ',',\n", - " 'being',\n", - " 'the',\n", - " 'terminal',\n", - " 'station',\n", - " 'of',\n", - " 'the',\n", - " 'now',\n", - " '-',\n", - " 'defunct',\n", - " 'N',\n", - " '##ume',\n", - " '##dal',\n", - " 'railway',\n", - " 'line',\n", - " '(',\n", - " 'N',\n", - " '##ume',\n", - " '##dal',\n", - " '##s',\n", - " '##ban',\n", - " '##en',\n", - " ')',\n", - " 'which',\n", - " 'ran',\n", - " 'up',\n", - " 'the',\n", - " 'N',\n", - " '##ume',\n", - " '##dal',\n", - " 'valley',\n", - " 'between',\n", - " 'Kong',\n", - " '##sberg',\n", - " 'and',\n", - " 'R',\n", - " '##ø',\n", - " '##d',\n", - " '##berg',\n", - " '.',\n", - " 'The',\n", - " 'final',\n", - " 'passenger',\n", - " 'service',\n", - " 'ended',\n", - " 'in',\n", - " '1988',\n", - " '.',\n", - " 'The',\n", - " 'rail',\n", - " 'line',\n", - " 'north',\n", - " 'of',\n", - " 'Roll',\n", - " '##ag',\n", - " 'was',\n", - " 'closed',\n", - " 'in',\n", - " '1989',\n", - " '.',\n", - " '[SEP]']" - ] - }, - "execution_count": 42, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# elmo_lm_builder.tokenizer.convert_ids_to_tokens(b1['input_ids'][0])" - ] - }, - { - "cell_type": "code", - "execution_count": 32, - "id": "broadband-chancellor", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'input_ids': tensor([[ 101, 1305, 3320, 138, 7629, 1495, 1145, 1227, 1112, 157,\n", - " 6512, 1874, 11737, 1110, 1126, 129, 119, 129, 1557, 113,\n", - " 126, 119, 126, 1940, 114, 118, 1263, 1300, 118, 7576,\n", - " 4083, 119, 1135, 2947, 1121, 1103, 6698, 1114, 1305, 3320,\n", - " 130, 1105, 14805, 4559, 1260, 140, 10879, 4084, 113, 9087,\n", - " 3320, 125, 114, 1106, 1103, 1411, 1104, 157, 6512, 1874,\n", - " 117, 3744, 1103, 4281, 1104, 131, 9326, 22806, 8376, 1727,\n", - " 2181, 2386, 2180, 139, 2744, 19495, 1197, 3006, 159, 3161,\n", - " 12210, 1279, 1727, 8834, 1130, 1949, 1103, 3467, 2384, 1533,\n", - " 170, 6875, 1111, 1103, 8883, 8837, 1812, 2469, 2443, 119,\n", - " 1109, 2981, 1111, 1103, 5972, 2329, 1111, 1103, 1456, 11737,\n", - " 4744, 1108, 12983, 19093, 10401, 3687, 17135, 119, 1456, 11737,\n", - " 2075, 15057, 2605, 1810, 1615, 20136, 117, 1305, 3320, 138,\n", - " 7629, 1495, 117, 1105, 170, 3849, 1104, 1305, 3320, 129,\n", - " 117, 1305, 3320, 130, 119, 12983, 19093, 10401, 3687, 17135,\n", - " 1878, 1103, 16752, 16122, 16752, 15017, 1113, 1744, 1318, 1898,\n", - " 119, 1706, 9872, 1408, 1217, 4465, 1107, 1360, 1820, 117,\n", - " 1170, 1103, 1419, 1845, 1103, 1812, 8313, 2452, 1106, 1103,\n", - " 2329, 119, 1109, 18750, 3769, 1113, 1955, 1382, 12795, 119,\n", - " 102, 155, 17945, 1181, 2953, 1110, 1103, 3207, 2642, 1104,\n", - " 16162, 1162, 18631, 158, 1964, 6919, 2667, 1107, 8947, 4188,\n", - " 4867, 117, 4323, 119, 155, 17945, 1181, 2953, 1110, 1388,\n", - " 1107, 1103, 4236, 2361, 1629, 1105, 4524, 1104, 151, 15447,\n", - " 6919, 119, 2098, 1416, 113, 1478, 114, 1110, 3993, 1604,\n", - " 117, 1105, 1122, 1110, 1388, 1113, 1103, 155, 17945, 1181,\n", - " 2953, 6961, 113, 4236, 131, 155, 17945, 1181, 2953, 10775,\n", - " 2354, 114, 1113, 1103, 151, 15447, 6919, 1116, 1233, 16835,\n", - " 4915, 1595, 119, 1109, 1469, 1540, 2930, 117, 155, 17945,\n", - " 1181, 2953, 16162, 1162, 122, 1107, 2805, 3825, 1105, 16162,\n", - " 1162, 123, 1107, 2805, 1290, 3064, 117, 17573, 1447, 12712,\n", - " 1121, 1103, 155, 17945, 1181, 2953, 8732, 119, 1109, 3546,\n", - " 1132, 6559, 1114, 1457, 2980, 27311, 4964, 117, 1103, 4236,\n", - " 1352, 2205, 6495, 1419, 119, 155, 17945, 1181, 2953, 1108,\n", - " 1517, 170, 2529, 1466, 117, 1217, 1103, 6020, 1466, 1104,\n", - " 1103, 1208, 118, 12273, 151, 15447, 6919, 2529, 1413, 113,\n", - " 151, 15447, 6919, 1116, 7167, 1424, 114, 1134, 1868, 1146,\n", - " 1103, 151, 15447, 6919, 4524, 1206, 3462, 19945, 1105, 155,\n", - " 17945, 1181, 2953, 119, 1109, 1509, 4059, 1555, 2207, 1107,\n", - " 2115, 119, 1109, 4356, 1413, 1564, 1104, 9916, 8517, 1108,\n", - " 1804, 1107, 2056, 119, 102]]),\n", - " 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n", - " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n", - " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n", - " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n", - " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n", - " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n", - " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n", - " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n", - " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n", - " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n", - " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n", - " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n", - " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n", - " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n", - " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n", - " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n", - " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]),\n", - " 'labels': tensor([[ 101, 1305, 3320, 138, 7629, 1495, 1145, 1227, 1112, 157,\n", - " 6512, 1874, 11737, 1110, 1126, 129, 119, 129, 1557, 113,\n", - " 126, 119, 126, 1940, 114, 118, 1263, 1300, 118, 7576,\n", - " 4083, 119, 1135, 2947, 1121, 1103, 6698, 1114, 1305, 3320,\n", - " 130, 1105, 14805, 4559, 1260, 140, 10879, 4084, 113, 9087,\n", - " 3320, 125, 114, 1106, 1103, 1411, 1104, 157, 6512, 1874,\n", - " 117, 3744, 1103, 4281, 1104, 131, 9326, 22806, 8376, 1727,\n", - " 2181, 2386, 2180, 139, 2744, 19495, 1197, 3006, 159, 3161,\n", - " 12210, 1279, 1727, 8834, 1130, 1949, 1103, 3467, 2384, 1533,\n", - " 170, 6875, 1111, 1103, 8883, 8837, 1812, 2469, 2443, 119,\n", - " 1109, 2981, 1111, 1103, 5972, 2329, 1111, 1103, 1456, 11737,\n", - " 4744, 1108, 12983, 19093, 10401, 3687, 17135, 119, 1456, 11737,\n", - " 2075, 15057, 2605, 1810, 1615, 20136, 117, 1305, 3320, 138,\n", - " 7629, 1495, 117, 1105, 170, 3849, 1104, 1305, 3320, 129,\n", - " 117, 1305, 3320, 130, 119, 12983, 19093, 10401, 3687, 17135,\n", - " 1878, 1103, 16752, 16122, 16752, 15017, 1113, 1744, 1318, 1898,\n", - " 119, 1706, 9872, 1408, 1217, 4465, 1107, 1360, 1820, 117,\n", - " 1170, 1103, 1419, 1845, 1103, 1812, 8313, 2452, 1106, 1103,\n", - " 2329, 119, 1109, 18750, 3769, 1113, 1955, 1382, 12795, 119,\n", - " 102, 155, 17945, 1181, 2953, 1110, 1103, 3207, 2642, 1104,\n", - " 16162, 1162, 18631, 158, 1964, 6919, 2667, 1107, 8947, 4188,\n", - " 4867, 117, 4323, 119, 155, 17945, 1181, 2953, 1110, 1388,\n", - " 1107, 1103, 4236, 2361, 1629, 1105, 4524, 1104, 151, 15447,\n", - " 6919, 119, 2098, 1416, 113, 1478, 114, 1110, 3993, 1604,\n", - " 117, 1105, 1122, 1110, 1388, 1113, 1103, 155, 17945, 1181,\n", - " 2953, 6961, 113, 4236, 131, 155, 17945, 1181, 2953, 10775,\n", - " 2354, 114, 1113, 1103, 151, 15447, 6919, 1116, 1233, 16835,\n", - " 4915, 1595, 119, 1109, 1469, 1540, 2930, 117, 155, 17945,\n", - " 1181, 2953, 16162, 1162, 122, 1107, 2805, 3825, 1105, 16162,\n", - " 1162, 123, 1107, 2805, 1290, 3064, 117, 17573, 1447, 12712,\n", - " 1121, 1103, 155, 17945, 1181, 2953, 8732, 119, 1109, 3546,\n", - " 1132, 6559, 1114, 1457, 2980, 27311, 4964, 117, 1103, 4236,\n", - " 1352, 2205, 6495, 1419, 119, 155, 17945, 1181, 2953, 1108,\n", - " 1517, 170, 2529, 1466, 117, 1217, 1103, 6020, 1466, 1104,\n", - " 1103, 1208, 118, 12273, 151, 15447, 6919, 2529, 1413, 113,\n", - " 151, 15447, 6919, 1116, 7167, 1424, 114, 1134, 1868, 1146,\n", - " 1103, 151, 15447, 6919, 4524, 1206, 3462, 19945, 1105, 155,\n", - " 17945, 1181, 2953, 119, 1109, 1509, 4059, 1555, 2207, 1107,\n", - " 2115, 119, 1109, 4356, 1413, 1564, 1104, 9916, 8517, 1108,\n", - " 1804, 1107, 2056, 119, 102]])}" - ] - }, - "execution_count": 32, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "b1" - ] - }, - { - "cell_type": "code", - "execution_count": 35, - "id": "knowing-stone", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "ELMOBertLMHeadModel(\n", - " (transformer): ELMOBertModel(\n", - " (l2r_gpt): BertModel(\n", - " (embeddings): BertEmbeddings(\n", - " (word_embeddings): Embedding(30000, 768, padding_idx=0)\n", - " (position_embeddings): Embedding(1024, 768)\n", - " (token_type_embeddings): Embedding(2, 768)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (encoder): BertEncoder(\n", - " (layer): ModuleList(\n", - " (0): BertLayer(\n", - " (attention): BertAttention(\n", - " (self): BertSelfAttention(\n", - " (query): Linear(in_features=768, out_features=768, bias=True)\n", - " (key): Linear(in_features=768, out_features=768, bias=True)\n", - " (value): Linear(in_features=768, out_features=768, bias=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (output): BertSelfOutput(\n", - " (dense): Linear(in_features=768, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (intermediate): BertIntermediate(\n", - " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", - " )\n", - " (output): BertOutput(\n", - " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (1): BertLayer(\n", - " (attention): BertAttention(\n", - " (self): BertSelfAttention(\n", - " (query): Linear(in_features=768, out_features=768, bias=True)\n", - " (key): Linear(in_features=768, out_features=768, bias=True)\n", - " (value): Linear(in_features=768, out_features=768, bias=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (output): BertSelfOutput(\n", - " (dense): Linear(in_features=768, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (intermediate): BertIntermediate(\n", - " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", - " )\n", - " (output): BertOutput(\n", - " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (2): BertLayer(\n", - " (attention): BertAttention(\n", - " (self): BertSelfAttention(\n", - " (query): Linear(in_features=768, out_features=768, bias=True)\n", - " (key): Linear(in_features=768, out_features=768, bias=True)\n", - " (value): Linear(in_features=768, out_features=768, bias=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (output): BertSelfOutput(\n", - " (dense): Linear(in_features=768, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (intermediate): BertIntermediate(\n", - " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", - " )\n", - " (output): BertOutput(\n", - " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (3): BertLayer(\n", - " (attention): BertAttention(\n", - " (self): BertSelfAttention(\n", - " (query): Linear(in_features=768, out_features=768, bias=True)\n", - " (key): Linear(in_features=768, out_features=768, bias=True)\n", - " (value): Linear(in_features=768, out_features=768, bias=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (output): BertSelfOutput(\n", - " (dense): Linear(in_features=768, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (intermediate): BertIntermediate(\n", - " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", - " )\n", - " (output): BertOutput(\n", - " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (4): BertLayer(\n", - " (attention): BertAttention(\n", - " (self): BertSelfAttention(\n", - " (query): Linear(in_features=768, out_features=768, bias=True)\n", - " (key): Linear(in_features=768, out_features=768, bias=True)\n", - " (value): Linear(in_features=768, out_features=768, bias=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (output): BertSelfOutput(\n", - " (dense): Linear(in_features=768, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (intermediate): BertIntermediate(\n", - " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", - " )\n", - " (output): BertOutput(\n", - " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (5): BertLayer(\n", - " (attention): BertAttention(\n", - " (self): BertSelfAttention(\n", - " (query): Linear(in_features=768, out_features=768, bias=True)\n", - " (key): Linear(in_features=768, out_features=768, bias=True)\n", - " (value): Linear(in_features=768, out_features=768, bias=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (output): BertSelfOutput(\n", - " (dense): Linear(in_features=768, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (intermediate): BertIntermediate(\n", - " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", - " )\n", - " (output): BertOutput(\n", - " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (6): BertLayer(\n", - " (attention): BertAttention(\n", - " (self): BertSelfAttention(\n", - " (query): Linear(in_features=768, out_features=768, bias=True)\n", - " (key): Linear(in_features=768, out_features=768, bias=True)\n", - " (value): Linear(in_features=768, out_features=768, bias=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (output): BertSelfOutput(\n", - " (dense): Linear(in_features=768, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (intermediate): BertIntermediate(\n", - " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", - " )\n", - " (output): BertOutput(\n", - " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (7): BertLayer(\n", - " (attention): BertAttention(\n", - " (self): BertSelfAttention(\n", - " (query): Linear(in_features=768, out_features=768, bias=True)\n", - " (key): Linear(in_features=768, out_features=768, bias=True)\n", - " (value): Linear(in_features=768, out_features=768, bias=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (output): BertSelfOutput(\n", - " (dense): Linear(in_features=768, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (intermediate): BertIntermediate(\n", - " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", - " )\n", - " (output): BertOutput(\n", - " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (8): BertLayer(\n", - " (attention): BertAttention(\n", - " (self): BertSelfAttention(\n", - " (query): Linear(in_features=768, out_features=768, bias=True)\n", - " (key): Linear(in_features=768, out_features=768, bias=True)\n", - " (value): Linear(in_features=768, out_features=768, bias=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (output): BertSelfOutput(\n", - " (dense): Linear(in_features=768, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (intermediate): BertIntermediate(\n", - " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", - " )\n", - " (output): BertOutput(\n", - " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (9): BertLayer(\n", - " (attention): BertAttention(\n", - " (self): BertSelfAttention(\n", - " (query): Linear(in_features=768, out_features=768, bias=True)\n", - " (key): Linear(in_features=768, out_features=768, bias=True)\n", - " (value): Linear(in_features=768, out_features=768, bias=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (output): BertSelfOutput(\n", - " (dense): Linear(in_features=768, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (intermediate): BertIntermediate(\n", - " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", - " )\n", - " (output): BertOutput(\n", - " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (10): BertLayer(\n", - " (attention): BertAttention(\n", - " (self): BertSelfAttention(\n", - " (query): Linear(in_features=768, out_features=768, bias=True)\n", - " (key): Linear(in_features=768, out_features=768, bias=True)\n", - " (value): Linear(in_features=768, out_features=768, bias=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (output): BertSelfOutput(\n", - " (dense): Linear(in_features=768, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (intermediate): BertIntermediate(\n", - " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", - " )\n", - " (output): BertOutput(\n", - " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (11): BertLayer(\n", - " (attention): BertAttention(\n", - " (self): BertSelfAttention(\n", - " (query): Linear(in_features=768, out_features=768, bias=True)\n", - " (key): Linear(in_features=768, out_features=768, bias=True)\n", - " (value): Linear(in_features=768, out_features=768, bias=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (output): BertSelfOutput(\n", - " (dense): Linear(in_features=768, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (intermediate): BertIntermediate(\n", - " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", - " )\n", - " (output): BertOutput(\n", - " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " )\n", - " )\n", - " (pooler): BertPooler(\n", - " (dense): Linear(in_features=768, out_features=768, bias=True)\n", - " (activation): Tanh()\n", - " )\n", - " )\n", - " (r2l_gpt): BertModel(\n", - " (embeddings): BertEmbeddings(\n", - " (word_embeddings): Embedding(30000, 768, padding_idx=0)\n", - " (position_embeddings): Embedding(1024, 768)\n", - " (token_type_embeddings): Embedding(2, 768)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (encoder): BertEncoder(\n", - " (layer): ModuleList(\n", - " (0): BertLayer(\n", - " (attention): BertAttention(\n", - " (self): BertSelfAttention(\n", - " (query): Linear(in_features=768, out_features=768, bias=True)\n", - " (key): Linear(in_features=768, out_features=768, bias=True)\n", - " (value): Linear(in_features=768, out_features=768, bias=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (output): BertSelfOutput(\n", - " (dense): Linear(in_features=768, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (intermediate): BertIntermediate(\n", - " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", - " )\n", - " (output): BertOutput(\n", - " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (1): BertLayer(\n", - " (attention): BertAttention(\n", - " (self): BertSelfAttention(\n", - " (query): Linear(in_features=768, out_features=768, bias=True)\n", - " (key): Linear(in_features=768, out_features=768, bias=True)\n", - " (value): Linear(in_features=768, out_features=768, bias=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (output): BertSelfOutput(\n", - " (dense): Linear(in_features=768, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (intermediate): BertIntermediate(\n", - " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", - " )\n", - " (output): BertOutput(\n", - " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (2): BertLayer(\n", - " (attention): BertAttention(\n", - " (self): BertSelfAttention(\n", - " (query): Linear(in_features=768, out_features=768, bias=True)\n", - " (key): Linear(in_features=768, out_features=768, bias=True)\n", - " (value): Linear(in_features=768, out_features=768, bias=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (output): BertSelfOutput(\n", - " (dense): Linear(in_features=768, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (intermediate): BertIntermediate(\n", - " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", - " )\n", - " (output): BertOutput(\n", - " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (3): BertLayer(\n", - " (attention): BertAttention(\n", - " (self): BertSelfAttention(\n", - " (query): Linear(in_features=768, out_features=768, bias=True)\n", - " (key): Linear(in_features=768, out_features=768, bias=True)\n", - " (value): Linear(in_features=768, out_features=768, bias=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (output): BertSelfOutput(\n", - " (dense): Linear(in_features=768, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (intermediate): BertIntermediate(\n", - " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", - " )\n", - " (output): BertOutput(\n", - " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (4): BertLayer(\n", - " (attention): BertAttention(\n", - " (self): BertSelfAttention(\n", - " (query): Linear(in_features=768, out_features=768, bias=True)\n", - " (key): Linear(in_features=768, out_features=768, bias=True)\n", - " (value): Linear(in_features=768, out_features=768, bias=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (output): BertSelfOutput(\n", - " (dense): Linear(in_features=768, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (intermediate): BertIntermediate(\n", - " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", - " )\n", - " (output): BertOutput(\n", - " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (5): BertLayer(\n", - " (attention): BertAttention(\n", - " (self): BertSelfAttention(\n", - " (query): Linear(in_features=768, out_features=768, bias=True)\n", - " (key): Linear(in_features=768, out_features=768, bias=True)\n", - " (value): Linear(in_features=768, out_features=768, bias=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (output): BertSelfOutput(\n", - " (dense): Linear(in_features=768, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (intermediate): BertIntermediate(\n", - " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", - " )\n", - " (output): BertOutput(\n", - " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (6): BertLayer(\n", - " (attention): BertAttention(\n", - " (self): BertSelfAttention(\n", - " (query): Linear(in_features=768, out_features=768, bias=True)\n", - " (key): Linear(in_features=768, out_features=768, bias=True)\n", - " (value): Linear(in_features=768, out_features=768, bias=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (output): BertSelfOutput(\n", - " (dense): Linear(in_features=768, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (intermediate): BertIntermediate(\n", - " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", - " )\n", - " (output): BertOutput(\n", - " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (7): BertLayer(\n", - " (attention): BertAttention(\n", - " (self): BertSelfAttention(\n", - " (query): Linear(in_features=768, out_features=768, bias=True)\n", - " (key): Linear(in_features=768, out_features=768, bias=True)\n", - " (value): Linear(in_features=768, out_features=768, bias=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (output): BertSelfOutput(\n", - " (dense): Linear(in_features=768, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (intermediate): BertIntermediate(\n", - " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", - " )\n", - " (output): BertOutput(\n", - " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (8): BertLayer(\n", - " (attention): BertAttention(\n", - " (self): BertSelfAttention(\n", - " (query): Linear(in_features=768, out_features=768, bias=True)\n", - " (key): Linear(in_features=768, out_features=768, bias=True)\n", - " (value): Linear(in_features=768, out_features=768, bias=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (output): BertSelfOutput(\n", - " (dense): Linear(in_features=768, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (intermediate): BertIntermediate(\n", - " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", - " )\n", - " (output): BertOutput(\n", - " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (9): BertLayer(\n", - " (attention): BertAttention(\n", - " (self): BertSelfAttention(\n", - " (query): Linear(in_features=768, out_features=768, bias=True)\n", - " (key): Linear(in_features=768, out_features=768, bias=True)\n", - " (value): Linear(in_features=768, out_features=768, bias=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (output): BertSelfOutput(\n", - " (dense): Linear(in_features=768, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (intermediate): BertIntermediate(\n", - " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", - " )\n", - " (output): BertOutput(\n", - " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (10): BertLayer(\n", - " (attention): BertAttention(\n", - " (self): BertSelfAttention(\n", - " (query): Linear(in_features=768, out_features=768, bias=True)\n", - " (key): Linear(in_features=768, out_features=768, bias=True)\n", - " (value): Linear(in_features=768, out_features=768, bias=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (output): BertSelfOutput(\n", - " (dense): Linear(in_features=768, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (intermediate): BertIntermediate(\n", - " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", - " )\n", - " (output): BertOutput(\n", - " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (11): BertLayer(\n", - " (attention): BertAttention(\n", - " (self): BertSelfAttention(\n", - " (query): Linear(in_features=768, out_features=768, bias=True)\n", - " (key): Linear(in_features=768, out_features=768, bias=True)\n", - " (value): Linear(in_features=768, out_features=768, bias=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " (output): BertSelfOutput(\n", - " (dense): Linear(in_features=768, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " (intermediate): BertIntermediate(\n", - " (dense): Linear(in_features=768, out_features=3072, bias=True)\n", - " )\n", - " (output): BertOutput(\n", - " (dense): Linear(in_features=3072, out_features=768, bias=True)\n", - " (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n", - " (dropout): Dropout(p=0.1, inplace=False)\n", - " )\n", - " )\n", - " )\n", - " )\n", - " (pooler): BertPooler(\n", - " (dense): Linear(in_features=768, out_features=768, bias=True)\n", - " (activation): Tanh()\n", - " )\n", - " )\n", - " )\n", - " (lm_head_l2r): Linear(in_features=768, out_features=30000, bias=False)\n", - " (lm_head_r2l): Linear(in_features=768, out_features=30000, bias=False)\n", - ")" - ] - }, - "execution_count": 35, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "model.to('cpu')" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "congressional-contract", - "metadata": {}, - "outputs": [ - { - "ename": "NameError", - "evalue": "name 'model' is not defined", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mno_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mb1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[0;31mNameError\u001b[0m: name 'model' is not defined" - ] - } - ], - "source": [ - "import torch\n", - "with torch.no_grad():\n", - " model(**b1)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.8" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/elmo-sanity-check.ipynb b/elmo-sanity-check.ipynb new file mode 100644 index 0000000..a573bfe --- /dev/null +++ b/elmo-sanity-check.ipynb @@ -0,0 +1,660 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "described-startup", + "metadata": {}, + "source": [ + "## Prepare data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "interesting-signal", + "metadata": {}, + "outputs": [], + "source": [ + "with open(\"./examples/data/text_forward.txt\", \"r+\") as fr:\n", + " lines = fr.readlines()" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "hundred-popularity", + "metadata": {}, + "outputs": [], + "source": [ + "lines = [line.strip() for line in lines]" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "brutal-aviation", + "metadata": {}, + "outputs": [], + "source": [ + "lines = [line.split() for line in lines]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "interesting-mandate", + "metadata": {}, + "outputs": [], + "source": [ + "lines = [line[::-1] for line in lines]" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "changed-mixture", + "metadata": {}, + "outputs": [], + "source": [ + "lines = [\" \".join(line) for line in lines]" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "completed-speaker", + "metadata": {}, + "outputs": [], + "source": [ + "lines = lines[::-1]" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "interstate-demonstration", + "metadata": {}, + "outputs": [], + "source": [ + "lines = \"\\n\".join(lines)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "balanced-person", + "metadata": {}, + "outputs": [], + "source": [ + "with open(\"./examples/data/text_backward.txt\", \"w+\") as fw:\n", + " fw.write(lines)" + ] + }, + { + "cell_type": "markdown", + "id": "aerial-drinking", + "metadata": {}, + "source": [ + "## Sanity Check" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "dried-fetish", + "metadata": {}, + "outputs": [], + "source": [ + "import torch\n", + "from newlm.lm.elmo.modeling_elmo.elmo_head import ELMOBertLMHeadModel\n", + "from newlm.lm.elmo.lm_builder import ELMOLMBuilder\n", + "from transformers import BertConfig" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "exotic-craps", + "metadata": {}, + "outputs": [], + "source": [ + "from newlm.utils.file_util import read_from_yaml\n", + "config_file = read_from_yaml('examples/configs/run.1-percent-bert-causal.yaml')\n", + "\n", + "elmo_lm_builder = ELMOLMBuilder(\n", + " model_config = config_file['lm']['model']['config'],\n", + " tokenizer=\"bert-base-cased\",\n", + " model_type=\"bert-causal-elmo\"\n", + ")\n", + "\n", + "# model\n", + "config = BertConfig(**elmo_lm_builder.model_config)\n", + "model = ELMOBertLMHeadModel(config=config)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "junior-efficiency", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Model in eval mode for consistency\n" + ] + } + ], + "source": [ + "model.eval()\n", + "print(\"Model in eval mode for consistency\")" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "white-archives", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2021-11-10 17:03:11.685 | INFO | newlm.lm.elmo.lm_builder:_get_dataset:142 - Constructing roBERTa style dataset\n", + "2021-11-10 17:03:12.892 | INFO | newlm.lm.elmo.lm_builder:_get_dataset:142 - Constructing roBERTa style dataset\n" + ] + } + ], + "source": [ + "%%capture\n", + "\n", + "# dataset-forward\n", + "train_path = \"./examples/data/text_forward.txt\"\n", + "ds_f = elmo_lm_builder._get_dataset(train_path)\n", + "\n", + "# dataset-backward\n", + "train_path = \"./examples/data/text_backward.txt\"\n", + "ds_b = elmo_lm_builder._get_dataset(train_path)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "square-oregon", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "max_steps is given, it will override any value given in num_train_epochs\n", + "max_steps is given, it will override any value given in num_train_epochs\n" + ] + } + ], + "source": [ + "# trainer (for helper)\n", + "from transformers import TrainingArguments, Trainer\n", + "args = TrainingArguments(output_dir=\"tmpout\",**config_file['lm']['hf_trainer']['args'])\n", + "\n", + "trainer = Trainer(model=model, args=args, data_collator=elmo_lm_builder.data_collator,\n", + " train_dataset=ds_f,\n", + ")\n", + "dl_f = trainer.get_train_dataloader() # Data Loader-forward\n", + "\n", + "trainer = Trainer(model=model, args=args,data_collator=elmo_lm_builder.data_collator,\n", + " train_dataset=ds_b,\n", + ")\n", + "dl_b = trainer.get_train_dataloader() # Data Loader-backward" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "aboriginal-brook", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "id": "opening-accident", + "metadata": {}, + "source": [ + "### Compare text_forward and text_backward" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "extra-signature", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(torch.Size([1, 271]), torch.Size([1, 271]))" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "batch_f = next(iter(dl_f))\n", + "batch_b = next(iter(dl_b))\n", + "\n", + "batch_f['input_ids'].shape, batch_b['input_ids'].shape" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "communist-diameter", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
forwardbackward
0[CLS][CLS]
1R.
2##øperiods
3##drepublican
4##bergand
.........
266andR
267republican##ø
268periods##d
269.##berg
270[SEP][SEP]
\n", + "

271 rows × 2 columns

\n", + "
" + ], + "text/plain": [ + " forward backward\n", + "0 [CLS] [CLS]\n", + "1 R .\n", + "2 ##ø periods\n", + "3 ##d republican\n", + "4 ##berg and\n", + ".. ... ...\n", + "266 and R\n", + "267 republican ##ø\n", + "268 periods ##d\n", + "269 . ##berg\n", + "270 [SEP] [SEP]\n", + "\n", + "[271 rows x 2 columns]" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "tokens_f = elmo_lm_builder.tokenizer.convert_ids_to_tokens(batch_f['input_ids'][0])\n", + "tokens_b = elmo_lm_builder.tokenizer.convert_ids_to_tokens(batch_b['input_ids'][0])\n", + "\n", + "import pandas as pd\n", + "pd.DataFrame({\"forward\": tokens_f, \"backward\": tokens_b})" + ] + }, + { + "cell_type": "markdown", + "id": "alone-islam", + "metadata": {}, + "source": [ + "Here we can see that the data is not completely flip when the tokenizer couldn't parse a single word into a single id" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "after-phrase", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "l2r_loss tensor(10.5373, grad_fn=)\n", + "r2l_loss tensor(10.5472, grad_fn=)\n" + ] + } + ], + "source": [ + "res = model(**batch_f)" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "every-cocktail", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "l2r_loss tensor(10.5231, grad_fn=)\n", + "r2l_loss tensor(10.5489, grad_fn=)\n" + ] + } + ], + "source": [ + "res = model(**batch_b)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "julian-dress", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "id": "altered-pharmaceutical", + "metadata": {}, + "source": [ + "### From batch_forward compare Normal vs Rev" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "resident-courage", + "metadata": {}, + "outputs": [], + "source": [ + "import torch\n", + "\n", + "batch_f_input = torch.clone(batch_f['input_ids'])\n", + "batch_f_rev_input = torch.cat(\n", + " (\n", + " batch_f_input[0][0:1],\n", + " torch.flip(batch_f_input[0][1:-1], [0]),\n", + " batch_f_input[0][-1:]\n", + " )\n", + ")\n", + "batch_f_rev_input = batch_f_rev_input.reshape(1,-1)\n", + "\n", + "batch_f_labels = torch.clone(batch_f['labels'])\n", + "batch_f_rev_labels = torch.cat(\n", + " (\n", + " batch_f_labels[0][0:1],\n", + " torch.flip(batch_f_labels[0][1:-1], [0]),\n", + " batch_f_labels[0][-1:]\n", + " )\n", + ")\n", + "batch_f_rev_labels = batch_f_rev_labels.reshape(1,-1)\n", + "\n", + "batch_rev = batch_f.copy()\n", + "batch_rev['input_ids'] = batch_f_rev_input\n", + "batch_rev['labels'] = batch_f_rev_labels" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "creative-sociology", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
forwardreverse
0[CLS][CLS]
1R.
2##øperiods
3##drepublican
4##bergand
.........
266and##berg
267republican##d
268periods##ø
269.R
270[SEP][SEP]
\n", + "

271 rows × 2 columns

\n", + "
" + ], + "text/plain": [ + " forward reverse\n", + "0 [CLS] [CLS]\n", + "1 R .\n", + "2 ##ø periods\n", + "3 ##d republican\n", + "4 ##berg and\n", + ".. ... ...\n", + "266 and ##berg\n", + "267 republican ##d\n", + "268 periods ##ø\n", + "269 . R\n", + "270 [SEP] [SEP]\n", + "\n", + "[271 rows x 2 columns]" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "tokens_f = elmo_lm_builder.tokenizer.convert_ids_to_tokens(batch_f['input_ids'][0])\n", + "tokens_f_rev = elmo_lm_builder.tokenizer.convert_ids_to_tokens(batch_rev['input_ids'][0])\n", + "\n", + "import pandas as pd\n", + "pd.DataFrame({\"forward\": tokens_f, \"reverse\": tokens_f_rev})" + ] + }, + { + "cell_type": "markdown", + "id": "pending-buddy", + "metadata": {}, + "source": [ + "with the exception of [CLS] and [SEP], the data are completely flip" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "owned-dutch", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "l2r_loss tensor(10.5373, grad_fn=)\n", + "r2l_loss tensor(10.5472, grad_fn=)\n" + ] + } + ], + "source": [ + "res = model(**batch_f)" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "lesbian-standing", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "l2r_loss tensor(10.5330, grad_fn=)\n", + "r2l_loss tensor(10.5643, grad_fn=)\n" + ] + } + ], + "source": [ + "res = model(**batch_rev)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.5" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From 36dab6703c4c06efc4deb4b57a6438bc55a4ecaa Mon Sep 17 00:00:00 2001 From: Made NN Date: Thu, 11 Nov 2021 12:23:20 +0000 Subject: [PATCH 3/9] V2 --- elmo-sanity-check-v2.ipynb | 284 +++++++++++++++++++++++++++++++++++++ 1 file changed, 284 insertions(+) create mode 100644 elmo-sanity-check-v2.ipynb diff --git a/elmo-sanity-check-v2.ipynb b/elmo-sanity-check-v2.ipynb new file mode 100644 index 0000000..56af8dc --- /dev/null +++ b/elmo-sanity-check-v2.ipynb @@ -0,0 +1,284 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "unknown-tanzania", + "metadata": {}, + "source": [ + "## Prepare Data + Model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "composed-pennsylvania", + "metadata": {}, + "outputs": [], + "source": [ + "!cat examples/data/text_forward.txt" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "detected-permission", + "metadata": {}, + "outputs": [], + "source": [ + "!ls -al /mnt/data4/made_workspace/newlm-output/elmo-bert-causal-en.1-percent-rerun/model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "lightweight-african", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "silent-paint", + "metadata": {}, + "outputs": [], + "source": [ + "import torch\n", + "from newlm.lm.elmo.modeling_elmo.elmo_head import ELMOBertLMHeadModel\n", + "from newlm.lm.elmo.lm_builder import ELMOLMBuilder\n", + "from transformers import BertConfig" + ] + }, + { + "cell_type": "markdown", + "id": "terminal-denver", + "metadata": {}, + "source": [ + "#### Model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "final-punch", + "metadata": {}, + "outputs": [], + "source": [ + "model = ELMOBertLMHeadModel.from_pretrained(\n", + " \"/mnt/data4/made_workspace/newlm-output/elmo-bert-causal-en.1-percent-rerun/model\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "irish-warner", + "metadata": {}, + "outputs": [], + "source": [ + "model.eval()\n", + "print(\"Model in eval mode for consistency\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "amber-belize", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "id": "wooden-official", + "metadata": {}, + "source": [ + "#### Data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "joined-logan", + "metadata": {}, + "outputs": [], + "source": [ + "%%capture\n", + "\n", + "from newlm.utils.file_util import read_from_yaml\n", + "# lm builder (helper)\n", + "elmo_lm_builder = ELMOLMBuilder(\n", + " model_config = read_from_yaml('examples/configs/run.1-percent-bert-causal.yaml'),\n", + " tokenizer=\"/mnt/data4/made_workspace/newlm-output/elmo-bert-causal-en.1-percent-rerun/model\",\n", + " model_type=\"bert-causal-elmo\"\n", + ")\n", + "\n", + "# dataset-forward\n", + "train_path = \"./examples/data/text_forward.txt\"\n", + "ds_f = elmo_lm_builder._get_dataset(train_path)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "unexpected-garage", + "metadata": {}, + "outputs": [], + "source": [ + "# trainer (helper)\n", + "from transformers import TrainingArguments, Trainer\n", + "args = TrainingArguments(output_dir=\"tmpout\",**config_file['lm']['hf_trainer']['args'])\n", + "\n", + "# dataloader-forward\n", + "trainer = Trainer(model=model, args=args, data_collator=elmo_lm_builder.data_collator, train_dataset=ds_f,)\n", + "dl_f = trainer.get_train_dataloader() # Data Loader-forward" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "weekly-approval", + "metadata": {}, + "outputs": [], + "source": [ + "batch_f = next(iter(dl_f))\n", + "batch_f['input_ids'].shape" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "alert-island", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "id": "adjusted-evans", + "metadata": {}, + "source": [ + "## Sanity Check" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "naughty-staff", + "metadata": {}, + "outputs": [], + "source": [ + "# batch_f" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "awful-stack", + "metadata": {}, + "outputs": [], + "source": [ + "import torch\n", + "\n", + "# reverse input\n", + "batch_f_input = torch.clone(batch_f['input_ids'])\n", + "batch_f_rev_input = torch.cat(\n", + " (\n", + " batch_f_input[0][0:1],\n", + " torch.flip(batch_f_input[0][1:-1], [0]),\n", + " batch_f_input[0][-1:]\n", + " )\n", + ")\n", + "batch_f_rev_input = batch_f_rev_input.reshape(1,-1)\n", + "\n", + "# reverse labels\n", + "batch_f_labels = torch.clone(batch_f['labels'])\n", + "batch_f_rev_labels = torch.cat(\n", + " (\n", + " batch_f_labels[0][0:1],\n", + " torch.flip(batch_f_labels[0][1:-1], [0]),\n", + " batch_f_labels[0][-1:]\n", + " )\n", + ")\n", + "batch_f_rev_labels = batch_f_rev_labels.reshape(1,-1)\n", + "\n", + "batch_rev = batch_f.copy()\n", + "batch_rev['input_ids'] = batch_f_rev_input\n", + "batch_rev['labels'] = batch_f_rev_labels" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "identified-airport", + "metadata": {}, + "outputs": [], + "source": [ + "tokens_f = elmo_lm_builder.tokenizer.convert_ids_to_tokens(batch_f['input_ids'][0])\n", + "tokens_f_rev = elmo_lm_builder.tokenizer.convert_ids_to_tokens(batch_rev['input_ids'][0])\n", + "\n", + "import pandas as pd\n", + "pd.DataFrame({\"forward\": tokens_f, \"reverse\": tokens_f_rev})" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "greenhouse-veteran", + "metadata": {}, + "outputs": [], + "source": [ + "batch_f['input_ids'].shape, batch_rev['input_ids'].shape" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "portuguese-croatia", + "metadata": {}, + "outputs": [], + "source": [ + "res = model(**batch_f) # forward" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "precious-greeting", + "metadata": {}, + "outputs": [], + "source": [ + "res = model(**batch_rev) # reverse" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "random-prophet", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.8" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From f930b73d5c70fed50179aab33b96002f531afaa9 Mon Sep 17 00:00:00 2001 From: Made NN Date: Thu, 11 Nov 2021 19:38:54 +0700 Subject: [PATCH 4/9] Elmo sanity check v2 run --- elmo-sanity-check-v2.ipynb | 276 ++++++++++++++++++++++++++++++------- 1 file changed, 225 insertions(+), 51 deletions(-) diff --git a/elmo-sanity-check-v2.ipynb b/elmo-sanity-check-v2.ipynb index 56af8dc..2de8623 100644 --- a/elmo-sanity-check-v2.ipynb +++ b/elmo-sanity-check-v2.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "markdown", - "id": "unknown-tanzania", + "id": "norwegian-factor", "metadata": {}, "source": [ "## Prepare Data + Model" @@ -10,36 +10,36 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "composed-pennsylvania", + "execution_count": 22, + "id": "institutional-inspector", "metadata": {}, "outputs": [], "source": [ - "!cat examples/data/text_forward.txt" + "# !cat examples/data/text_forward.txt" ] }, { "cell_type": "code", - "execution_count": null, - "id": "detected-permission", + "execution_count": 23, + "id": "alert-immune", "metadata": {}, "outputs": [], "source": [ - "!ls -al /mnt/data4/made_workspace/newlm-output/elmo-bert-causal-en.1-percent-rerun/model" + "# !ls -al ./outputs/en.1-percent.elmo-bert-causal" ] }, { "cell_type": "code", "execution_count": null, - "id": "lightweight-african", + "id": "handed-blair", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", - "execution_count": null, - "id": "silent-paint", + "execution_count": 4, + "id": "designing-rating", "metadata": {}, "outputs": [], "source": [ @@ -51,7 +51,7 @@ }, { "cell_type": "markdown", - "id": "terminal-denver", + "id": "quarterly-algorithm", "metadata": {}, "source": [ "#### Model" @@ -59,22 +59,30 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "final-punch", + "execution_count": 6, + "id": "flexible-upset", "metadata": {}, "outputs": [], "source": [ "model = ELMOBertLMHeadModel.from_pretrained(\n", - " \"/mnt/data4/made_workspace/newlm-output/elmo-bert-causal-en.1-percent-rerun/model\"\n", - ")" + " \"./outputs/en.1-percent.elmo-bert-causal\"\n", + ") # use pre-trained model" ] }, { "cell_type": "code", - "execution_count": null, - "id": "irish-warner", + "execution_count": 13, + "id": "resident-techno", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Model in eval mode for consistency\n" + ] + } + ], "source": [ "model.eval()\n", "print(\"Model in eval mode for consistency\")" @@ -83,14 +91,14 @@ { "cell_type": "code", "execution_count": null, - "id": "amber-belize", + "id": "peaceful-reproduction", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", - "id": "wooden-official", + "id": "heated-article", "metadata": {}, "source": [ "#### Data" @@ -98,18 +106,28 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "joined-logan", + "execution_count": 10, + "id": "appointed-cooking", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2021-11-11 19:28:13.210 | INFO | newlm.lm.elmo.lm_builder:_get_dataset:142 - Constructing roBERTa style dataset\n" + ] + } + ], "source": [ "%%capture\n", "\n", "from newlm.utils.file_util import read_from_yaml\n", + "config_file = read_from_yaml('examples/configs/run.1-percent-elmo-bert-causal.yaml')\n", + "\n", "# lm builder (helper)\n", "elmo_lm_builder = ELMOLMBuilder(\n", - " model_config = read_from_yaml('examples/configs/run.1-percent-bert-causal.yaml'),\n", - " tokenizer=\"/mnt/data4/made_workspace/newlm-output/elmo-bert-causal-en.1-percent-rerun/model\",\n", + " model_config = config_file['lm']['model']['config'],\n", + " tokenizer=\"./outputs/en.1-percent.elmo-bert-causal\", # use pre-trained tokenizer\n", " model_type=\"bert-causal-elmo\"\n", ")\n", "\n", @@ -120,10 +138,18 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "unexpected-garage", + "execution_count": 11, + "id": "desperate-supervisor", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "max_steps is given, it will override any value given in num_train_epochs\n" + ] + } + ], "source": [ "# trainer (helper)\n", "from transformers import TrainingArguments, Trainer\n", @@ -136,10 +162,21 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "weekly-approval", + "execution_count": 12, + "id": "controversial-morgan", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "torch.Size([1, 272])" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "batch_f = next(iter(dl_f))\n", "batch_f['input_ids'].shape" @@ -148,14 +185,14 @@ { "cell_type": "code", "execution_count": null, - "id": "alert-island", + "id": "foreign-industry", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", - "id": "adjusted-evans", + "id": "trained-stadium", "metadata": {}, "source": [ "## Sanity Check" @@ -163,8 +200,8 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "naughty-staff", + "execution_count": 15, + "id": "earned-light", "metadata": {}, "outputs": [], "source": [ @@ -173,8 +210,8 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "awful-stack", + "execution_count": 16, + "id": "insured-cartoon", "metadata": {}, "outputs": [], "source": [ @@ -209,10 +246,118 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "identified-airport", + "execution_count": 17, + "id": "promotional-joint", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
forwardreverse
0[CLS][CLS]
1R.
2##øperiods
3##d##an
4##bergrepublic
.........
267republic##berg
268##an##d
269periods##ø
270.R
271[SEP][SEP]
\n", + "

272 rows × 2 columns

\n", + "
" + ], + "text/plain": [ + " forward reverse\n", + "0 [CLS] [CLS]\n", + "1 R .\n", + "2 ##ø periods\n", + "3 ##d ##an\n", + "4 ##berg republic\n", + ".. ... ...\n", + "267 republic ##berg\n", + "268 ##an ##d\n", + "269 periods ##ø\n", + "270 . R\n", + "271 [SEP] [SEP]\n", + "\n", + "[272 rows x 2 columns]" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "tokens_f = elmo_lm_builder.tokenizer.convert_ids_to_tokens(batch_f['input_ids'][0])\n", "tokens_f_rev = elmo_lm_builder.tokenizer.convert_ids_to_tokens(batch_rev['input_ids'][0])\n", @@ -223,30 +368,59 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "greenhouse-veteran", + "execution_count": 18, + "id": "seeing-liberal", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "(torch.Size([1, 272]), torch.Size([1, 272]))" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "batch_f['input_ids'].shape, batch_rev['input_ids'].shape" ] }, { "cell_type": "code", - "execution_count": null, - "id": "portuguese-croatia", + "execution_count": 19, + "id": "chinese-showcase", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "l2r_loss tensor(4.8444, grad_fn=)\n", + "r2l_loss tensor(8.5702, grad_fn=)\n" + ] + } + ], "source": [ "res = model(**batch_f) # forward" ] }, { "cell_type": "code", - "execution_count": null, - "id": "precious-greeting", + "execution_count": 20, + "id": "beneficial-lloyd", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "l2r_loss tensor(9.3590, grad_fn=)\n", + "r2l_loss tensor(8.4301, grad_fn=)\n" + ] + } + ], "source": [ "res = model(**batch_rev) # reverse" ] @@ -254,7 +428,7 @@ { "cell_type": "code", "execution_count": null, - "id": "random-prophet", + "id": "preliminary-convergence", "metadata": {}, "outputs": [], "source": [] @@ -276,7 +450,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.8" + "version": "3.8.5" } }, "nbformat": 4, From 357a5fcd4c1c911eafc5110a452ed23db915f196 Mon Sep 17 00:00:00 2001 From: Made NN Date: Fri, 12 Nov 2021 08:21:16 +0700 Subject: [PATCH 5/9] Add random case --- elmo-sanity-check-v2.ipynb | 768 +++++++++++++++++++++++++++++++++---- 1 file changed, 701 insertions(+), 67 deletions(-) diff --git a/elmo-sanity-check-v2.ipynb b/elmo-sanity-check-v2.ipynb index 2de8623..1c2aeb4 100644 --- a/elmo-sanity-check-v2.ipynb +++ b/elmo-sanity-check-v2.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "markdown", - "id": "norwegian-factor", + "id": "fifth-father", "metadata": {}, "source": [ "## Prepare Data + Model" @@ -10,8 +10,8 @@ }, { "cell_type": "code", - "execution_count": 22, - "id": "institutional-inspector", + "execution_count": 1, + "id": "hispanic-domain", "metadata": {}, "outputs": [], "source": [ @@ -20,8 +20,8 @@ }, { "cell_type": "code", - "execution_count": 23, - "id": "alert-immune", + "execution_count": 2, + "id": "compliant-society", "metadata": {}, "outputs": [], "source": [ @@ -31,15 +31,15 @@ { "cell_type": "code", "execution_count": null, - "id": "handed-blair", + "id": "careful-present", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", - "execution_count": 4, - "id": "designing-rating", + "execution_count": 3, + "id": "harmful-trust", "metadata": {}, "outputs": [], "source": [ @@ -51,7 +51,7 @@ }, { "cell_type": "markdown", - "id": "quarterly-algorithm", + "id": "active-kentucky", "metadata": {}, "source": [ "#### Model" @@ -59,8 +59,8 @@ }, { "cell_type": "code", - "execution_count": 6, - "id": "flexible-upset", + "execution_count": 4, + "id": "useful-horizon", "metadata": {}, "outputs": [], "source": [ @@ -71,8 +71,8 @@ }, { "cell_type": "code", - "execution_count": 13, - "id": "resident-techno", + "execution_count": 5, + "id": "enclosed-diabetes", "metadata": {}, "outputs": [ { @@ -91,14 +91,14 @@ { "cell_type": "code", "execution_count": null, - "id": "peaceful-reproduction", + "id": "dedicated-washington", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", - "id": "heated-article", + "id": "spectacular-chicken", "metadata": {}, "source": [ "#### Data" @@ -106,15 +106,15 @@ }, { "cell_type": "code", - "execution_count": 10, - "id": "appointed-cooking", + "execution_count": 6, + "id": "working-stanford", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "2021-11-11 19:28:13.210 | INFO | newlm.lm.elmo.lm_builder:_get_dataset:142 - Constructing roBERTa style dataset\n" + "2021-11-12 07:48:28.558 | INFO | newlm.lm.elmo.lm_builder:_get_dataset:142 - Constructing roBERTa style dataset\n" ] } ], @@ -138,8 +138,8 @@ }, { "cell_type": "code", - "execution_count": 11, - "id": "desperate-supervisor", + "execution_count": 7, + "id": "possible-london", "metadata": {}, "outputs": [ { @@ -162,8 +162,8 @@ }, { "cell_type": "code", - "execution_count": 12, - "id": "controversial-morgan", + "execution_count": 8, + "id": "latter-personal", "metadata": {}, "outputs": [ { @@ -172,7 +172,7 @@ "torch.Size([1, 272])" ] }, - "execution_count": 12, + "execution_count": 8, "metadata": {}, "output_type": "execute_result" } @@ -185,14 +185,33 @@ { "cell_type": "code", "execution_count": null, - "id": "foreign-industry", + "id": "ultimate-shift", "metadata": {}, "outputs": [], "source": [] }, + { + "cell_type": "code", + "execution_count": 9, + "id": "derived-links", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Model in eval mode for consistency\n" + ] + } + ], + "source": [ + "model.eval()\n", + "print(\"Model in eval mode for consistency\")" + ] + }, { "cell_type": "markdown", - "id": "trained-stadium", + "id": "informational-tactics", "metadata": {}, "source": [ "## Sanity Check" @@ -200,8 +219,8 @@ }, { "cell_type": "code", - "execution_count": 15, - "id": "earned-light", + "execution_count": 10, + "id": "international-lafayette", "metadata": {}, "outputs": [], "source": [ @@ -210,44 +229,81 @@ }, { "cell_type": "code", - "execution_count": 16, - "id": "insured-cartoon", + "execution_count": 11, + "id": "chubby-problem", "metadata": {}, "outputs": [], "source": [ "import torch\n", "\n", - "# reverse input\n", - "batch_f_input = torch.clone(batch_f['input_ids'])\n", - "batch_f_rev_input = torch.cat(\n", - " (\n", - " batch_f_input[0][0:1],\n", - " torch.flip(batch_f_input[0][1:-1], [0]),\n", - " batch_f_input[0][-1:]\n", + "def reverse_batch(batch_f):\n", + " # reverse input\n", + " batch_f_input = torch.clone(batch_f['input_ids'])\n", + " batch_f_rev_input = torch.cat(\n", + " (\n", + " batch_f_input[0][0:1],\n", + " torch.flip(batch_f_input[0][1:-1], [0]),\n", + " batch_f_input[0][-1:]\n", + " )\n", " )\n", - ")\n", - "batch_f_rev_input = batch_f_rev_input.reshape(1,-1)\n", + " batch_f_rev_input = batch_f_rev_input.reshape(1,-1)\n", "\n", - "# reverse labels\n", - "batch_f_labels = torch.clone(batch_f['labels'])\n", - "batch_f_rev_labels = torch.cat(\n", - " (\n", - " batch_f_labels[0][0:1],\n", - " torch.flip(batch_f_labels[0][1:-1], [0]),\n", - " batch_f_labels[0][-1:]\n", - " )\n", - ")\n", - "batch_f_rev_labels = batch_f_rev_labels.reshape(1,-1)\n", + " # reverse labels\n", + " batch_f_rev_labels = torch.clone(batch_f_rev_input)\n", + " \n", + " # batch_rev\n", + " batch_rev = batch_f.copy()\n", + " batch_rev['input_ids'] = batch_f_rev_input\n", + " batch_rev['labels'] = batch_f_rev_labels\n", + " \n", + " return batch_rev" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "bibliographic-assignment", + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", "\n", - "batch_rev = batch_f.copy()\n", - "batch_rev['input_ids'] = batch_f_rev_input\n", - "batch_rev['labels'] = batch_f_rev_labels" + "def pandas_check(batch_f, batch_rev):\n", + " tokens_f = elmo_lm_builder.tokenizer.convert_ids_to_tokens(batch_f['input_ids'][0])\n", + " tokens_f_rev = elmo_lm_builder.tokenizer.convert_ids_to_tokens(batch_rev['input_ids'][0])\n", + " return pd.DataFrame({\"forward\": tokens_f, \"reverse\": tokens_f_rev})" ] }, { "cell_type": "code", - "execution_count": 17, - "id": "promotional-joint", + "execution_count": null, + "id": "charged-miami", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "id": "ancient-harvey", + "metadata": {}, + "source": [ + "#### Normal vs Reverse" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "actual-moldova", + "metadata": {}, + "outputs": [], + "source": [ + "batch_rev = reverse_batch(batch_f)" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "specialized-monroe", "metadata": {}, "outputs": [ { @@ -353,23 +409,19 @@ "[272 rows x 2 columns]" ] }, - "execution_count": 17, + "execution_count": 21, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "tokens_f = elmo_lm_builder.tokenizer.convert_ids_to_tokens(batch_f['input_ids'][0])\n", - "tokens_f_rev = elmo_lm_builder.tokenizer.convert_ids_to_tokens(batch_rev['input_ids'][0])\n", - "\n", - "import pandas as pd\n", - "pd.DataFrame({\"forward\": tokens_f, \"reverse\": tokens_f_rev})" + "pandas_check(batch_f, batch_rev)" ] }, { "cell_type": "code", - "execution_count": 18, - "id": "seeing-liberal", + "execution_count": 15, + "id": "detailed-fossil", "metadata": {}, "outputs": [ { @@ -378,7 +430,7 @@ "(torch.Size([1, 272]), torch.Size([1, 272]))" ] }, - "execution_count": 18, + "execution_count": 15, "metadata": {}, "output_type": "execute_result" } @@ -389,8 +441,8 @@ }, { "cell_type": "code", - "execution_count": 19, - "id": "chinese-showcase", + "execution_count": 16, + "id": "seventh-standing", "metadata": {}, "outputs": [ { @@ -408,8 +460,8 @@ }, { "cell_type": "code", - "execution_count": 20, - "id": "beneficial-lloyd", + "execution_count": 17, + "id": "ahead-organic", "metadata": {}, "outputs": [ { @@ -428,7 +480,589 @@ { "cell_type": "code", "execution_count": null, - "id": "preliminary-convergence", + "id": "romance-movie", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "id": "common-complement", + "metadata": {}, + "source": [ + "#### Random String" + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "id": "transsexual-accused", + "metadata": {}, + "outputs": [], + "source": [ + "# batch_f#['input_ids'].shape" + ] + }, + { + "cell_type": "code", + "execution_count": 63, + "id": "affected-harassment", + "metadata": {}, + "outputs": [], + "source": [ + "# shuffle data\n", + "batch_f_input = batch_f['input_ids']\n", + "batch_shuffle_input = torch.cat(\n", + " (\n", + " batch_f_input[0][0:1],\n", + " torch.randint(\n", + " low=5, # 0-4 > ['[PAD]', '[UNK]', '[CLS]', '[SEP]', '[MASK]']\n", + " high=29999,\n", + " size=(270,),\n", + " dtype=torch.long\n", + " ),\n", + " batch_f_input[0][-1:]\n", + " )\n", + ")\n", + "batch_shuffle_input = batch_shuffle_input.reshape(1,-1)\n", + "# labels\n", + "batch_shuffle_labels = torch.clone(batch_shuffle_input) \n", + "# batch_shuffle\n", + "batch_shuffle = batch_f.copy()\n", + "batch_shuffle['input_ids'] = batch_shuffle_input\n", + "batch_shuffle['labels'] = batch_shuffle_labels" + ] + }, + { + "cell_type": "code", + "execution_count": 64, + "id": "valuable-costume", + "metadata": {}, + "outputs": [], + "source": [ + "batch_shuffle_rev = reverse_batch(batch_shuffle)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "hawaiian-beverage", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "id": "based-hardwood", + "metadata": {}, + "source": [ + "##### Trial-1" + ] + }, + { + "cell_type": "code", + "execution_count": 49, + "id": "scientific-barrier", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
forwardreverse
0[CLS][CLS]
1statureswirl
2sophistsmok
3dictÁ
4electrified##GE
.........
267##GEelectrified
268Ádict
269smoksophist
270swirlstature
271[SEP][SEP]
\n", + "

272 rows × 2 columns

\n", + "
" + ], + "text/plain": [ + " forward reverse\n", + "0 [CLS] [CLS]\n", + "1 stature swirl\n", + "2 sophist smok\n", + "3 dict Á\n", + "4 electrified ##GE\n", + ".. ... ...\n", + "267 ##GE electrified\n", + "268 Á dict\n", + "269 smok sophist\n", + "270 swirl stature\n", + "271 [SEP] [SEP]\n", + "\n", + "[272 rows x 2 columns]" + ] + }, + "execution_count": 49, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "pandas_check(batch_shuffle, batch_shuffle_rev)" + ] + }, + { + "cell_type": "code", + "execution_count": 50, + "id": "adequate-afghanistan", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "l2r_loss tensor(12.9860, grad_fn=)\n", + "r2l_loss tensor(11.5442, grad_fn=)\n" + ] + } + ], + "source": [ + "res = model(**batch_shuffle) # forward" + ] + }, + { + "cell_type": "code", + "execution_count": 51, + "id": "built-poland", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "l2r_loss tensor(12.8227, grad_fn=)\n", + "r2l_loss tensor(11.2906, grad_fn=)\n" + ] + } + ], + "source": [ + "res = model(**batch_shuffle_rev) # reverse" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "checked-athens", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "id": "polished-session", + "metadata": {}, + "source": [ + "##### Trial-2" + ] + }, + { + "cell_type": "code", + "execution_count": 60, + "id": "private-gravity", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
forwardreverse
0[CLS][CLS]
1blewhelmet
2nationalInternet
3Tabledrawn
4Pomptenant
.........
267tenantPomp
268drawnTable
269Internetnational
270helmetblew
271[SEP][SEP]
\n", + "

272 rows × 2 columns

\n", + "
" + ], + "text/plain": [ + " forward reverse\n", + "0 [CLS] [CLS]\n", + "1 blew helmet\n", + "2 national Internet\n", + "3 Table drawn\n", + "4 Pomp tenant\n", + ".. ... ...\n", + "267 tenant Pomp\n", + "268 drawn Table\n", + "269 Internet national\n", + "270 helmet blew\n", + "271 [SEP] [SEP]\n", + "\n", + "[272 rows x 2 columns]" + ] + }, + "execution_count": 60, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "pandas_check(batch_shuffle, batch_shuffle_rev)" + ] + }, + { + "cell_type": "code", + "execution_count": 61, + "id": "indirect-clothing", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "l2r_loss tensor(12.9374, grad_fn=)\n", + "r2l_loss tensor(11.5165, grad_fn=)\n" + ] + } + ], + "source": [ + "res = model(**batch_shuffle) # forward" + ] + }, + { + "cell_type": "code", + "execution_count": 62, + "id": "honey-workplace", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "l2r_loss tensor(12.9639, grad_fn=)\n", + "r2l_loss tensor(11.6511, grad_fn=)\n" + ] + } + ], + "source": [ + "res = model(**batch_shuffle_rev) # reverse" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "turkish-provincial", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "id": "answering-dynamics", + "metadata": {}, + "source": [ + "##### Trial-3" + ] + }, + { + "cell_type": "code", + "execution_count": 65, + "id": "pointed-stack", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
forwardreverse
0[CLS][CLS]
1hoveredunder
2editingMs
3Whetherguitars
4##hunaura
.........
267aura##hun
268guitarsWhether
269Msediting
270underhovered
271[SEP][SEP]
\n", + "

272 rows × 2 columns

\n", + "
" + ], + "text/plain": [ + " forward reverse\n", + "0 [CLS] [CLS]\n", + "1 hovered under\n", + "2 editing Ms\n", + "3 Whether guitars\n", + "4 ##hun aura\n", + ".. ... ...\n", + "267 aura ##hun\n", + "268 guitars Whether\n", + "269 Ms editing\n", + "270 under hovered\n", + "271 [SEP] [SEP]\n", + "\n", + "[272 rows x 2 columns]" + ] + }, + "execution_count": 65, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "pandas_check(batch_shuffle, batch_shuffle_rev)" + ] + }, + { + "cell_type": "code", + "execution_count": 66, + "id": "lesbian-andrews", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "l2r_loss tensor(12.9565, grad_fn=)\n", + "r2l_loss tensor(11.4332, grad_fn=)\n" + ] + } + ], + "source": [ + "res = model(**batch_shuffle) # forward" + ] + }, + { + "cell_type": "code", + "execution_count": 67, + "id": "certified-canon", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "l2r_loss tensor(12.8039, grad_fn=)\n", + "r2l_loss tensor(11.5467, grad_fn=)\n" + ] + } + ], + "source": [ + "res = model(**batch_shuffle_rev) # reverse" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "powered-cotton", "metadata": {}, "outputs": [], "source": [] From daa6781e75956f0b9d4ed3c40c7e633bc5caf44f Mon Sep 17 00:00:00 2001 From: Made NN Date: Fri, 12 Nov 2021 11:09:23 +0700 Subject: [PATCH 6/9] Fix max_len --- elmo-sanity-check-v2.ipynb | 460 +++++++++++++++++++------------------ elmo-sanity-check.ipynb | 229 +++++++++--------- 2 files changed, 355 insertions(+), 334 deletions(-) diff --git a/elmo-sanity-check-v2.ipynb b/elmo-sanity-check-v2.ipynb index 1c2aeb4..db3f197 100644 --- a/elmo-sanity-check-v2.ipynb +++ b/elmo-sanity-check-v2.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "markdown", - "id": "fifth-father", + "id": "imposed-conducting", "metadata": {}, "source": [ "## Prepare Data + Model" @@ -11,7 +11,7 @@ { "cell_type": "code", "execution_count": 1, - "id": "hispanic-domain", + "id": "tracked-roads", "metadata": {}, "outputs": [], "source": [ @@ -21,7 +21,7 @@ { "cell_type": "code", "execution_count": 2, - "id": "compliant-society", + "id": "adjusted-inquiry", "metadata": {}, "outputs": [], "source": [ @@ -31,7 +31,7 @@ { "cell_type": "code", "execution_count": null, - "id": "careful-present", + "id": "following-tomorrow", "metadata": {}, "outputs": [], "source": [] @@ -39,7 +39,7 @@ { "cell_type": "code", "execution_count": 3, - "id": "harmful-trust", + "id": "grave-interference", "metadata": {}, "outputs": [], "source": [ @@ -51,7 +51,7 @@ }, { "cell_type": "markdown", - "id": "active-kentucky", + "id": "hourly-penetration", "metadata": {}, "source": [ "#### Model" @@ -60,7 +60,7 @@ { "cell_type": "code", "execution_count": 4, - "id": "useful-horizon", + "id": "practical-truth", "metadata": {}, "outputs": [], "source": [ @@ -72,7 +72,7 @@ { "cell_type": "code", "execution_count": 5, - "id": "enclosed-diabetes", + "id": "immediate-bargain", "metadata": {}, "outputs": [ { @@ -91,14 +91,14 @@ { "cell_type": "code", "execution_count": null, - "id": "dedicated-washington", + "id": "naughty-moore", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", - "id": "spectacular-chicken", + "id": "executive-extent", "metadata": {}, "source": [ "#### Data" @@ -107,14 +107,14 @@ { "cell_type": "code", "execution_count": 6, - "id": "working-stanford", + "id": "level-mediterranean", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "2021-11-12 07:48:28.558 | INFO | newlm.lm.elmo.lm_builder:_get_dataset:142 - Constructing roBERTa style dataset\n" + "2021-11-12 10:52:53.587 | INFO | newlm.lm.elmo.lm_builder:_get_dataset:142 - Constructing roBERTa style dataset\n" ] } ], @@ -128,18 +128,19 @@ "elmo_lm_builder = ELMOLMBuilder(\n", " model_config = config_file['lm']['model']['config'],\n", " tokenizer=\"./outputs/en.1-percent.elmo-bert-causal\", # use pre-trained tokenizer\n", - " model_type=\"bert-causal-elmo\"\n", + " model_type=\"bert-causal-elmo\",\n", + " max_len=128\n", ")\n", "\n", "# dataset-forward\n", - "train_path = \"./examples/data/text_forward.txt\"\n", + "train_path = \"./examples/data/text_forward-small.txt\"\n", "ds_f = elmo_lm_builder._get_dataset(train_path)" ] }, { "cell_type": "code", - "execution_count": 7, - "id": "possible-london", + "execution_count": 8, + "id": "governing-industry", "metadata": {}, "outputs": [ { @@ -162,17 +163,17 @@ }, { "cell_type": "code", - "execution_count": 8, - "id": "latter-personal", + "execution_count": 9, + "id": "timely-bobby", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "torch.Size([1, 272])" + "torch.Size([1, 127])" ] }, - "execution_count": 8, + "execution_count": 9, "metadata": {}, "output_type": "execute_result" } @@ -185,15 +186,15 @@ { "cell_type": "code", "execution_count": null, - "id": "ultimate-shift", + "id": "sized-description", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", - "execution_count": 9, - "id": "derived-links", + "execution_count": 10, + "id": "charming-directive", "metadata": {}, "outputs": [ { @@ -211,7 +212,7 @@ }, { "cell_type": "markdown", - "id": "informational-tactics", + "id": "mighty-generator", "metadata": {}, "source": [ "## Sanity Check" @@ -219,8 +220,8 @@ }, { "cell_type": "code", - "execution_count": 10, - "id": "international-lafayette", + "execution_count": 1, + "id": "saving-member", "metadata": {}, "outputs": [], "source": [ @@ -229,8 +230,8 @@ }, { "cell_type": "code", - "execution_count": 11, - "id": "chubby-problem", + "execution_count": 12, + "id": "colored-newman", "metadata": {}, "outputs": [], "source": [ @@ -261,8 +262,8 @@ }, { "cell_type": "code", - "execution_count": 20, - "id": "bibliographic-assignment", + "execution_count": 13, + "id": "affiliated-bangkok", "metadata": {}, "outputs": [], "source": [ @@ -277,14 +278,14 @@ { "cell_type": "code", "execution_count": null, - "id": "charged-miami", + "id": "durable-brass", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", - "id": "ancient-harvey", + "id": "automated-cheese", "metadata": {}, "source": [ "#### Normal vs Reverse" @@ -292,8 +293,8 @@ }, { "cell_type": "code", - "execution_count": 13, - "id": "actual-moldova", + "execution_count": 14, + "id": "infinite-liberty", "metadata": {}, "outputs": [], "source": [ @@ -302,8 +303,8 @@ }, { "cell_type": "code", - "execution_count": 21, - "id": "specialized-monroe", + "execution_count": 15, + "id": "weird-analyst", "metadata": {}, "outputs": [ { @@ -345,17 +346,17 @@ " \n", " 2\n", " ##ø\n", - " periods\n", + " Jews\n", " \n", " \n", " 3\n", " ##d\n", - " ##an\n", + " among\n", " \n", " \n", " 4\n", " ##berg\n", - " republic\n", + " lived\n", " \n", " \n", " ...\n", @@ -363,53 +364,53 @@ " ...\n", " \n", " \n", - " 267\n", - " republic\n", + " 122\n", + " lived\n", " ##berg\n", " \n", " \n", - " 268\n", - " ##an\n", + " 123\n", + " among\n", " ##d\n", " \n", " \n", - " 269\n", - " periods\n", + " 124\n", + " Jews\n", " ##ø\n", " \n", " \n", - " 270\n", + " 125\n", " .\n", " R\n", " \n", " \n", - " 271\n", + " 126\n", " [SEP]\n", " [SEP]\n", " \n", " \n", "\n", - "

272 rows × 2 columns

\n", + "

127 rows × 2 columns

\n", "" ], "text/plain": [ - " forward reverse\n", - "0 [CLS] [CLS]\n", - "1 R .\n", - "2 ##ø periods\n", - "3 ##d ##an\n", - "4 ##berg republic\n", - ".. ... ...\n", - "267 republic ##berg\n", - "268 ##an ##d\n", - "269 periods ##ø\n", - "270 . R\n", - "271 [SEP] [SEP]\n", + " forward reverse\n", + "0 [CLS] [CLS]\n", + "1 R .\n", + "2 ##ø Jews\n", + "3 ##d among\n", + "4 ##berg lived\n", + ".. ... ...\n", + "122 lived ##berg\n", + "123 among ##d\n", + "124 Jews ##ø\n", + "125 . R\n", + "126 [SEP] [SEP]\n", "\n", - "[272 rows x 2 columns]" + "[127 rows x 2 columns]" ] }, - "execution_count": 21, + "execution_count": 15, "metadata": {}, "output_type": "execute_result" } @@ -420,17 +421,17 @@ }, { "cell_type": "code", - "execution_count": 15, - "id": "detailed-fossil", + "execution_count": 16, + "id": "conscious-synthetic", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "(torch.Size([1, 272]), torch.Size([1, 272]))" + "(torch.Size([1, 127]), torch.Size([1, 127]))" ] }, - "execution_count": 15, + "execution_count": 16, "metadata": {}, "output_type": "execute_result" } @@ -441,16 +442,16 @@ }, { "cell_type": "code", - "execution_count": 16, - "id": "seventh-standing", + "execution_count": 17, + "id": "great-saturn", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "l2r_loss tensor(4.8444, grad_fn=)\n", - "r2l_loss tensor(8.5702, grad_fn=)\n" + "l2r_loss tensor(4.3262, grad_fn=)\n", + "r2l_loss tensor(6.8976, grad_fn=)\n" ] } ], @@ -460,16 +461,16 @@ }, { "cell_type": "code", - "execution_count": 17, - "id": "ahead-organic", + "execution_count": 18, + "id": "balanced-convention", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "l2r_loss tensor(9.3590, grad_fn=)\n", - "r2l_loss tensor(8.4301, grad_fn=)\n" + "l2r_loss tensor(9.2779, grad_fn=)\n", + "r2l_loss tensor(6.5343, grad_fn=)\n" ] } ], @@ -480,14 +481,14 @@ { "cell_type": "code", "execution_count": null, - "id": "romance-movie", + "id": "regulation-found", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", - "id": "common-complement", + "id": "revolutionary-procedure", "metadata": {}, "source": [ "#### Random String" @@ -495,18 +496,29 @@ }, { "cell_type": "code", - "execution_count": 44, - "id": "transsexual-accused", + "execution_count": 24, + "id": "crude-dodge", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "torch.Size([1, 127])" + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "# batch_f#['input_ids'].shape" + "batch_f['input_ids'].shape" ] }, { "cell_type": "code", - "execution_count": 63, - "id": "affected-harassment", + "execution_count": 29, + "id": "resident-opinion", "metadata": {}, "outputs": [], "source": [ @@ -518,7 +530,7 @@ " torch.randint(\n", " low=5, # 0-4 > ['[PAD]', '[UNK]', '[CLS]', '[SEP]', '[MASK]']\n", " high=29999,\n", - " size=(270,),\n", + " size=(125,), # modified based on seqlen!\n", " dtype=torch.long\n", " ),\n", " batch_f_input[0][-1:]\n", @@ -535,8 +547,8 @@ }, { "cell_type": "code", - "execution_count": 64, - "id": "valuable-costume", + "execution_count": 30, + "id": "better-hamburg", "metadata": {}, "outputs": [], "source": [ @@ -546,14 +558,14 @@ { "cell_type": "code", "execution_count": null, - "id": "hawaiian-beverage", + "id": "conscious-ancient", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", - "id": "based-hardwood", + "id": "common-breed", "metadata": {}, "source": [ "##### Trial-1" @@ -561,8 +573,8 @@ }, { "cell_type": "code", - "execution_count": 49, - "id": "scientific-barrier", + "execution_count": 21, + "id": "swiss-regression", "metadata": {}, "outputs": [ { @@ -598,23 +610,23 @@ " \n", " \n", " 1\n", - " stature\n", - " swirl\n", + " escaped\n", + " marriage\n", " \n", " \n", " 2\n", - " sophist\n", - " smok\n", + " Develop\n", + " circulated\n", " \n", " \n", " 3\n", - " dict\n", - " Á\n", + " elb\n", + " ъ\n", " \n", " \n", " 4\n", - " electrified\n", - " ##GE\n", + " Madame\n", + " Regional\n", " \n", " \n", " ...\n", @@ -622,53 +634,53 @@ " ...\n", " \n", " \n", - " 267\n", - " ##GE\n", - " electrified\n", + " 122\n", + " Regional\n", + " Madame\n", " \n", " \n", - " 268\n", - " Á\n", - " dict\n", + " 123\n", + " ъ\n", + " elb\n", " \n", " \n", - " 269\n", - " smok\n", - " sophist\n", + " 124\n", + " circulated\n", + " Develop\n", " \n", " \n", - " 270\n", - " swirl\n", - " stature\n", + " 125\n", + " marriage\n", + " escaped\n", " \n", " \n", - " 271\n", + " 126\n", " [SEP]\n", " [SEP]\n", " \n", " \n", "\n", - "

272 rows × 2 columns

\n", + "

127 rows × 2 columns

\n", "" ], "text/plain": [ - " forward reverse\n", - "0 [CLS] [CLS]\n", - "1 stature swirl\n", - "2 sophist smok\n", - "3 dict Á\n", - "4 electrified ##GE\n", - ".. ... ...\n", - "267 ##GE electrified\n", - "268 Á dict\n", - "269 smok sophist\n", - "270 swirl stature\n", - "271 [SEP] [SEP]\n", + " forward reverse\n", + "0 [CLS] [CLS]\n", + "1 escaped marriage\n", + "2 Develop circulated\n", + "3 elb ъ\n", + "4 Madame Regional\n", + ".. ... ...\n", + "122 Regional Madame\n", + "123 ъ elb\n", + "124 circulated Develop\n", + "125 marriage escaped\n", + "126 [SEP] [SEP]\n", "\n", - "[272 rows x 2 columns]" + "[127 rows x 2 columns]" ] }, - "execution_count": 49, + "execution_count": 21, "metadata": {}, "output_type": "execute_result" } @@ -679,16 +691,16 @@ }, { "cell_type": "code", - "execution_count": 50, - "id": "adequate-afghanistan", + "execution_count": 22, + "id": "italian-basement", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "l2r_loss tensor(12.9860, grad_fn=)\n", - "r2l_loss tensor(11.5442, grad_fn=)\n" + "l2r_loss tensor(13.3687, grad_fn=)\n", + "r2l_loss tensor(10.0583, grad_fn=)\n" ] } ], @@ -698,16 +710,16 @@ }, { "cell_type": "code", - "execution_count": 51, - "id": "built-poland", + "execution_count": 23, + "id": "indirect-minnesota", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "l2r_loss tensor(12.8227, grad_fn=)\n", - "r2l_loss tensor(11.2906, grad_fn=)\n" + "l2r_loss tensor(13.1889, grad_fn=)\n", + "r2l_loss tensor(9.9842, grad_fn=)\n" ] } ], @@ -718,14 +730,14 @@ { "cell_type": "code", "execution_count": null, - "id": "checked-athens", + "id": "diverse-pleasure", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", - "id": "polished-session", + "id": "creative-warehouse", "metadata": {}, "source": [ "##### Trial-2" @@ -733,8 +745,8 @@ }, { "cell_type": "code", - "execution_count": 60, - "id": "private-gravity", + "execution_count": 26, + "id": "blessed-company", "metadata": {}, "outputs": [ { @@ -770,23 +782,23 @@ " \n", " \n", " 1\n", - " blew\n", - " helmet\n", + " oscill\n", + " Horatio\n", " \n", " \n", " 2\n", - " national\n", - " Internet\n", + " transluc\n", + " Wig\n", " \n", " \n", " 3\n", - " Table\n", - " drawn\n", + " ##brook\n", + " assuming\n", " \n", " \n", " 4\n", - " Pomp\n", - " tenant\n", + " Perth\n", + " meetings\n", " \n", " \n", " ...\n", @@ -794,53 +806,53 @@ " ...\n", " \n", " \n", - " 267\n", - " tenant\n", - " Pomp\n", + " 122\n", + " meetings\n", + " Perth\n", " \n", " \n", - " 268\n", - " drawn\n", - " Table\n", + " 123\n", + " assuming\n", + " ##brook\n", " \n", " \n", - " 269\n", - " Internet\n", - " national\n", + " 124\n", + " Wig\n", + " transluc\n", " \n", " \n", - " 270\n", - " helmet\n", - " blew\n", + " 125\n", + " Horatio\n", + " oscill\n", " \n", " \n", - " 271\n", + " 126\n", " [SEP]\n", " [SEP]\n", " \n", " \n", "\n", - "

272 rows × 2 columns

\n", + "

127 rows × 2 columns

\n", "" ], "text/plain": [ " forward reverse\n", "0 [CLS] [CLS]\n", - "1 blew helmet\n", - "2 national Internet\n", - "3 Table drawn\n", - "4 Pomp tenant\n", + "1 oscill Horatio\n", + "2 transluc Wig\n", + "3 ##brook assuming\n", + "4 Perth meetings\n", ".. ... ...\n", - "267 tenant Pomp\n", - "268 drawn Table\n", - "269 Internet national\n", - "270 helmet blew\n", - "271 [SEP] [SEP]\n", + "122 meetings Perth\n", + "123 assuming ##brook\n", + "124 Wig transluc\n", + "125 Horatio oscill\n", + "126 [SEP] [SEP]\n", "\n", - "[272 rows x 2 columns]" + "[127 rows x 2 columns]" ] }, - "execution_count": 60, + "execution_count": 26, "metadata": {}, "output_type": "execute_result" } @@ -851,16 +863,16 @@ }, { "cell_type": "code", - "execution_count": 61, - "id": "indirect-clothing", + "execution_count": 27, + "id": "convenient-benefit", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "l2r_loss tensor(12.9374, grad_fn=)\n", - "r2l_loss tensor(11.5165, grad_fn=)\n" + "l2r_loss tensor(13.2914, grad_fn=)\n", + "r2l_loss tensor(10.1427, grad_fn=)\n" ] } ], @@ -870,16 +882,16 @@ }, { "cell_type": "code", - "execution_count": 62, - "id": "honey-workplace", + "execution_count": 28, + "id": "chinese-harvest", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "l2r_loss tensor(12.9639, grad_fn=)\n", - "r2l_loss tensor(11.6511, grad_fn=)\n" + "l2r_loss tensor(13.2515, grad_fn=)\n", + "r2l_loss tensor(10.0191, grad_fn=)\n" ] } ], @@ -890,14 +902,14 @@ { "cell_type": "code", "execution_count": null, - "id": "turkish-provincial", + "id": "peripheral-minnesota", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", - "id": "answering-dynamics", + "id": "opposite-cooperative", "metadata": {}, "source": [ "##### Trial-3" @@ -905,8 +917,8 @@ }, { "cell_type": "code", - "execution_count": 65, - "id": "pointed-stack", + "execution_count": 31, + "id": "competent-verification", "metadata": {}, "outputs": [ { @@ -942,23 +954,23 @@ " \n", " \n", " 1\n", - " hovered\n", - " under\n", + " Stevens\n", + " Lima\n", " \n", " \n", " 2\n", - " editing\n", - " Ms\n", + " Aden\n", + " ##cing\n", " \n", " \n", " 3\n", - " Whether\n", - " guitars\n", + " Devil\n", + " ##wo\n", " \n", " \n", " 4\n", - " ##hun\n", - " aura\n", + " ##ibald\n", + " gate\n", " \n", " \n", " ...\n", @@ -966,53 +978,53 @@ " ...\n", " \n", " \n", - " 267\n", - " aura\n", - " ##hun\n", + " 122\n", + " gate\n", + " ##ibald\n", " \n", " \n", - " 268\n", - " guitars\n", - " Whether\n", + " 123\n", + " ##wo\n", + " Devil\n", " \n", " \n", - " 269\n", - " Ms\n", - " editing\n", + " 124\n", + " ##cing\n", + " Aden\n", " \n", " \n", - " 270\n", - " under\n", - " hovered\n", + " 125\n", + " Lima\n", + " Stevens\n", " \n", " \n", - " 271\n", + " 126\n", " [SEP]\n", " [SEP]\n", " \n", " \n", "\n", - "

272 rows × 2 columns

\n", + "

127 rows × 2 columns

\n", "" ], "text/plain": [ " forward reverse\n", "0 [CLS] [CLS]\n", - "1 hovered under\n", - "2 editing Ms\n", - "3 Whether guitars\n", - "4 ##hun aura\n", + "1 Stevens Lima\n", + "2 Aden ##cing\n", + "3 Devil ##wo\n", + "4 ##ibald gate\n", ".. ... ...\n", - "267 aura ##hun\n", - "268 guitars Whether\n", - "269 Ms editing\n", - "270 under hovered\n", - "271 [SEP] [SEP]\n", + "122 gate ##ibald\n", + "123 ##wo Devil\n", + "124 ##cing Aden\n", + "125 Lima Stevens\n", + "126 [SEP] [SEP]\n", "\n", - "[272 rows x 2 columns]" + "[127 rows x 2 columns]" ] }, - "execution_count": 65, + "execution_count": 31, "metadata": {}, "output_type": "execute_result" } @@ -1023,16 +1035,16 @@ }, { "cell_type": "code", - "execution_count": 66, - "id": "lesbian-andrews", + "execution_count": 32, + "id": "dimensional-trauma", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "l2r_loss tensor(12.9565, grad_fn=)\n", - "r2l_loss tensor(11.4332, grad_fn=)\n" + "l2r_loss tensor(13.3330, grad_fn=)\n", + "r2l_loss tensor(9.8861, grad_fn=)\n" ] } ], @@ -1042,16 +1054,16 @@ }, { "cell_type": "code", - "execution_count": 67, - "id": "certified-canon", + "execution_count": 33, + "id": "typical-president", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "l2r_loss tensor(12.8039, grad_fn=)\n", - "r2l_loss tensor(11.5467, grad_fn=)\n" + "l2r_loss tensor(13.3210, grad_fn=)\n", + "r2l_loss tensor(9.9579, grad_fn=)\n" ] } ], @@ -1062,7 +1074,7 @@ { "cell_type": "code", "execution_count": null, - "id": "powered-cotton", + "id": "arabic-announcement", "metadata": {}, "outputs": [], "source": [] diff --git a/elmo-sanity-check.ipynb b/elmo-sanity-check.ipynb index a573bfe..9dc106d 100644 --- a/elmo-sanity-check.ipynb +++ b/elmo-sanity-check.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "markdown", - "id": "described-startup", + "id": "laden-tsunami", "metadata": {}, "source": [ "## Prepare data" @@ -10,19 +10,19 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "interesting-signal", + "execution_count": 1, + "id": "lovely-flood", "metadata": {}, "outputs": [], "source": [ - "with open(\"./examples/data/text_forward.txt\", \"r+\") as fr:\n", + "with open(\"./examples/data/text_forward-small.txt\", \"r+\") as fr:\n", " lines = fr.readlines()" ] }, { "cell_type": "code", "execution_count": 2, - "id": "hundred-popularity", + "id": "false-bernard", "metadata": {}, "outputs": [], "source": [ @@ -32,7 +32,7 @@ { "cell_type": "code", "execution_count": 3, - "id": "brutal-aviation", + "id": "funded-listing", "metadata": {}, "outputs": [], "source": [ @@ -42,7 +42,7 @@ { "cell_type": "code", "execution_count": 4, - "id": "interesting-mandate", + "id": "medical-visiting", "metadata": {}, "outputs": [], "source": [ @@ -52,7 +52,7 @@ { "cell_type": "code", "execution_count": 5, - "id": "changed-mixture", + "id": "eastern-publicity", "metadata": {}, "outputs": [], "source": [ @@ -62,7 +62,7 @@ { "cell_type": "code", "execution_count": 6, - "id": "completed-speaker", + "id": "alien-nightmare", "metadata": {}, "outputs": [], "source": [ @@ -72,7 +72,7 @@ { "cell_type": "code", "execution_count": 7, - "id": "interstate-demonstration", + "id": "acute-black", "metadata": {}, "outputs": [], "source": [ @@ -82,17 +82,17 @@ { "cell_type": "code", "execution_count": 8, - "id": "balanced-person", + "id": "modified-links", "metadata": {}, "outputs": [], "source": [ - "with open(\"./examples/data/text_backward.txt\", \"w+\") as fw:\n", + "with open(\"./examples/data/text_backward-small.txt\", \"w+\") as fw:\n", " fw.write(lines)" ] }, { "cell_type": "markdown", - "id": "aerial-drinking", + "id": "absolute-world", "metadata": {}, "source": [ "## Sanity Check" @@ -100,8 +100,8 @@ }, { "cell_type": "code", - "execution_count": 1, - "id": "dried-fetish", + "execution_count": 9, + "id": "hollow-newsletter", "metadata": {}, "outputs": [], "source": [ @@ -111,10 +111,18 @@ "from transformers import BertConfig" ] }, + { + "cell_type": "markdown", + "id": "chronic-crack", + "metadata": {}, + "source": [ + "Model from scratch" + ] + }, { "cell_type": "code", - "execution_count": 2, - "id": "exotic-craps", + "execution_count": 10, + "id": "identified-ivory", "metadata": {}, "outputs": [], "source": [ @@ -122,9 +130,10 @@ "config_file = read_from_yaml('examples/configs/run.1-percent-bert-causal.yaml')\n", "\n", "elmo_lm_builder = ELMOLMBuilder(\n", - " model_config = config_file['lm']['model']['config'],\n", - " tokenizer=\"bert-base-cased\",\n", - " model_type=\"bert-causal-elmo\"\n", + " model_config = config_file['lm']['model']['config'], # no pretrained model\n", + " tokenizer=\"./outputs/en.1-percent.elmo-bert-causal\", # use pre-trained tokenizer\n", + " model_type=\"bert-causal-elmo\",\n", + " max_len=128\n", ")\n", "\n", "# model\n", @@ -134,8 +143,8 @@ }, { "cell_type": "code", - "execution_count": 3, - "id": "junior-efficiency", + "execution_count": 11, + "id": "boring-workplace", "metadata": {}, "outputs": [ { @@ -153,16 +162,16 @@ }, { "cell_type": "code", - "execution_count": 4, - "id": "white-archives", + "execution_count": 12, + "id": "deluxe-expert", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "2021-11-10 17:03:11.685 | INFO | newlm.lm.elmo.lm_builder:_get_dataset:142 - Constructing roBERTa style dataset\n", - "2021-11-10 17:03:12.892 | INFO | newlm.lm.elmo.lm_builder:_get_dataset:142 - Constructing roBERTa style dataset\n" + "2021-11-12 11:07:27.881 | INFO | newlm.lm.elmo.lm_builder:_get_dataset:142 - Constructing roBERTa style dataset\n", + "2021-11-12 11:07:29.324 | INFO | newlm.lm.elmo.lm_builder:_get_dataset:142 - Constructing roBERTa style dataset\n" ] } ], @@ -170,18 +179,18 @@ "%%capture\n", "\n", "# dataset-forward\n", - "train_path = \"./examples/data/text_forward.txt\"\n", + "train_path = \"./examples/data/text_forward-small.txt\"\n", "ds_f = elmo_lm_builder._get_dataset(train_path)\n", "\n", "# dataset-backward\n", - "train_path = \"./examples/data/text_backward.txt\"\n", + "train_path = \"./examples/data/text_backward-small.txt\"\n", "ds_b = elmo_lm_builder._get_dataset(train_path)" ] }, { "cell_type": "code", - "execution_count": 5, - "id": "square-oregon", + "execution_count": 13, + "id": "immune-quick", "metadata": {}, "outputs": [ { @@ -212,14 +221,14 @@ { "cell_type": "code", "execution_count": null, - "id": "aboriginal-brook", + "id": "enabling-business", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", - "id": "opening-accident", + "id": "preceding-evening", "metadata": {}, "source": [ "### Compare text_forward and text_backward" @@ -227,17 +236,17 @@ }, { "cell_type": "code", - "execution_count": 6, - "id": "extra-signature", + "execution_count": 14, + "id": "promising-panama", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "(torch.Size([1, 271]), torch.Size([1, 271]))" + "(torch.Size([1, 127]), torch.Size([1, 127]))" ] }, - "execution_count": 6, + "execution_count": 14, "metadata": {}, "output_type": "execute_result" } @@ -252,7 +261,7 @@ { "cell_type": "code", "execution_count": 15, - "id": "communist-diameter", + "id": "italic-treaty", "metadata": {}, "outputs": [ { @@ -294,17 +303,17 @@ " \n", " 2\n", " ##ø\n", - " periods\n", + " Jews\n", " \n", " \n", " 3\n", " ##d\n", - " republican\n", + " among\n", " \n", " \n", " 4\n", " ##berg\n", - " and\n", + " lived\n", " \n", " \n", " ...\n", @@ -312,50 +321,50 @@ " ...\n", " \n", " \n", - " 266\n", - " and\n", + " 122\n", + " lived\n", " R\n", " \n", " \n", - " 267\n", - " republican\n", + " 123\n", + " among\n", " ##ø\n", " \n", " \n", - " 268\n", - " periods\n", + " 124\n", + " Jews\n", " ##d\n", " \n", " \n", - " 269\n", + " 125\n", " .\n", " ##berg\n", " \n", " \n", - " 270\n", + " 126\n", " [SEP]\n", " [SEP]\n", " \n", " \n", "\n", - "

271 rows × 2 columns

\n", + "

127 rows × 2 columns

\n", "" ], "text/plain": [ - " forward backward\n", - "0 [CLS] [CLS]\n", - "1 R .\n", - "2 ##ø periods\n", - "3 ##d republican\n", - "4 ##berg and\n", - ".. ... ...\n", - "266 and R\n", - "267 republican ##ø\n", - "268 periods ##d\n", - "269 . ##berg\n", - "270 [SEP] [SEP]\n", + " forward backward\n", + "0 [CLS] [CLS]\n", + "1 R .\n", + "2 ##ø Jews\n", + "3 ##d among\n", + "4 ##berg lived\n", + ".. ... ...\n", + "122 lived R\n", + "123 among ##ø\n", + "124 Jews ##d\n", + "125 . ##berg\n", + "126 [SEP] [SEP]\n", "\n", - "[271 rows x 2 columns]" + "[127 rows x 2 columns]" ] }, "execution_count": 15, @@ -373,7 +382,7 @@ }, { "cell_type": "markdown", - "id": "alone-islam", + "id": "attached-nursing", "metadata": {}, "source": [ "Here we can see that the data is not completely flip when the tokenizer couldn't parse a single word into a single id" @@ -382,15 +391,15 @@ { "cell_type": "code", "execution_count": 16, - "id": "after-phrase", + "id": "graphic-paintball", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "l2r_loss tensor(10.5373, grad_fn=)\n", - "r2l_loss tensor(10.5472, grad_fn=)\n" + "l2r_loss tensor(10.5425, grad_fn=)\n", + "r2l_loss tensor(10.4237, grad_fn=)\n" ] } ], @@ -401,15 +410,15 @@ { "cell_type": "code", "execution_count": 17, - "id": "every-cocktail", + "id": "frequent-scheduling", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "l2r_loss tensor(10.5231, grad_fn=)\n", - "r2l_loss tensor(10.5489, grad_fn=)\n" + "l2r_loss tensor(10.5305, grad_fn=)\n", + "r2l_loss tensor(10.4198, grad_fn=)\n" ] } ], @@ -420,14 +429,14 @@ { "cell_type": "code", "execution_count": null, - "id": "julian-dress", + "id": "amber-auckland", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", - "id": "altered-pharmaceutical", + "id": "unique-jungle", "metadata": {}, "source": [ "### From batch_forward compare Normal vs Rev" @@ -435,8 +444,8 @@ }, { "cell_type": "code", - "execution_count": 10, - "id": "resident-courage", + "execution_count": 18, + "id": "right-coalition", "metadata": {}, "outputs": [], "source": [ @@ -469,8 +478,8 @@ }, { "cell_type": "code", - "execution_count": 14, - "id": "creative-sociology", + "execution_count": 19, + "id": "terminal-withdrawal", "metadata": {}, "outputs": [ { @@ -512,17 +521,17 @@ " \n", " 2\n", " ##ø\n", - " periods\n", + " Jews\n", " \n", " \n", " 3\n", " ##d\n", - " republican\n", + " among\n", " \n", " \n", " 4\n", " ##berg\n", - " and\n", + " lived\n", " \n", " \n", " ...\n", @@ -530,53 +539,53 @@ " ...\n", " \n", " \n", - " 266\n", - " and\n", + " 122\n", + " lived\n", " ##berg\n", " \n", " \n", - " 267\n", - " republican\n", + " 123\n", + " among\n", " ##d\n", " \n", " \n", - " 268\n", - " periods\n", + " 124\n", + " Jews\n", " ##ø\n", " \n", " \n", - " 269\n", + " 125\n", " .\n", " R\n", " \n", " \n", - " 270\n", + " 126\n", " [SEP]\n", " [SEP]\n", " \n", " \n", "\n", - "

271 rows × 2 columns

\n", + "

127 rows × 2 columns

\n", "" ], "text/plain": [ - " forward reverse\n", - "0 [CLS] [CLS]\n", - "1 R .\n", - "2 ##ø periods\n", - "3 ##d republican\n", - "4 ##berg and\n", - ".. ... ...\n", - "266 and ##berg\n", - "267 republican ##d\n", - "268 periods ##ø\n", - "269 . R\n", - "270 [SEP] [SEP]\n", + " forward reverse\n", + "0 [CLS] [CLS]\n", + "1 R .\n", + "2 ##ø Jews\n", + "3 ##d among\n", + "4 ##berg lived\n", + ".. ... ...\n", + "122 lived ##berg\n", + "123 among ##d\n", + "124 Jews ##ø\n", + "125 . R\n", + "126 [SEP] [SEP]\n", "\n", - "[271 rows x 2 columns]" + "[127 rows x 2 columns]" ] }, - "execution_count": 14, + "execution_count": 19, "metadata": {}, "output_type": "execute_result" } @@ -591,7 +600,7 @@ }, { "cell_type": "markdown", - "id": "pending-buddy", + "id": "continued-recommendation", "metadata": {}, "source": [ "with the exception of [CLS] and [SEP], the data are completely flip" @@ -599,16 +608,16 @@ }, { "cell_type": "code", - "execution_count": 19, - "id": "owned-dutch", + "execution_count": 20, + "id": "narrow-pipeline", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "l2r_loss tensor(10.5373, grad_fn=)\n", - "r2l_loss tensor(10.5472, grad_fn=)\n" + "l2r_loss tensor(10.5425, grad_fn=)\n", + "r2l_loss tensor(10.4237, grad_fn=)\n" ] } ], @@ -618,16 +627,16 @@ }, { "cell_type": "code", - "execution_count": 18, - "id": "lesbian-standing", + "execution_count": 21, + "id": "desirable-peripheral", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "l2r_loss tensor(10.5330, grad_fn=)\n", - "r2l_loss tensor(10.5643, grad_fn=)\n" + "l2r_loss tensor(10.5323, grad_fn=)\n", + "r2l_loss tensor(10.4094, grad_fn=)\n" ] } ], From d20d64d17cef2f337cff5ff27b492d048e1738d7 Mon Sep 17 00:00:00 2001 From: Made NN Date: Sat, 13 Nov 2021 00:32:17 +0700 Subject: [PATCH 7/9] Fix model --- elmo-sanity-check-v2.ipynb | 334 ++++++++++++++++++------------------- 1 file changed, 167 insertions(+), 167 deletions(-) diff --git a/elmo-sanity-check-v2.ipynb b/elmo-sanity-check-v2.ipynb index db3f197..9004f6f 100644 --- a/elmo-sanity-check-v2.ipynb +++ b/elmo-sanity-check-v2.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "markdown", - "id": "imposed-conducting", + "id": "working-fisher", "metadata": {}, "source": [ "## Prepare Data + Model" @@ -11,7 +11,7 @@ { "cell_type": "code", "execution_count": 1, - "id": "tracked-roads", + "id": "quiet-macro", "metadata": {}, "outputs": [], "source": [ @@ -21,17 +21,17 @@ { "cell_type": "code", "execution_count": 2, - "id": "adjusted-inquiry", + "id": "resistant-denver", "metadata": {}, "outputs": [], "source": [ - "# !ls -al ./outputs/en.1-percent.elmo-bert-causal" + "# !ls -al ./outputs/en.1-percent.elmo-bert-causal-fixr2l" ] }, { "cell_type": "code", "execution_count": null, - "id": "following-tomorrow", + "id": "egyptian-association", "metadata": {}, "outputs": [], "source": [] @@ -39,7 +39,7 @@ { "cell_type": "code", "execution_count": 3, - "id": "grave-interference", + "id": "robust-strategy", "metadata": {}, "outputs": [], "source": [ @@ -51,7 +51,7 @@ }, { "cell_type": "markdown", - "id": "hourly-penetration", + "id": "beneficial-franchise", "metadata": {}, "source": [ "#### Model" @@ -60,19 +60,19 @@ { "cell_type": "code", "execution_count": 4, - "id": "practical-truth", + "id": "supported-strategy", "metadata": {}, "outputs": [], "source": [ "model = ELMOBertLMHeadModel.from_pretrained(\n", - " \"./outputs/en.1-percent.elmo-bert-causal\"\n", + " \"./outputs/en.1-percent.elmo-bert-causal-fixr2l\"\n", ") # use pre-trained model" ] }, { "cell_type": "code", "execution_count": 5, - "id": "immediate-bargain", + "id": "emotional-moisture", "metadata": {}, "outputs": [ { @@ -91,14 +91,14 @@ { "cell_type": "code", "execution_count": null, - "id": "naughty-moore", + "id": "following-execution", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", - "id": "executive-extent", + "id": "sized-enemy", "metadata": {}, "source": [ "#### Data" @@ -107,14 +107,14 @@ { "cell_type": "code", "execution_count": 6, - "id": "level-mediterranean", + "id": "previous-crack", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "2021-11-12 10:52:53.587 | INFO | newlm.lm.elmo.lm_builder:_get_dataset:142 - Constructing roBERTa style dataset\n" + "2021-11-13 00:29:52.826 | INFO | newlm.lm.elmo.lm_builder:_get_dataset:142 - Constructing roBERTa style dataset\n" ] } ], @@ -127,7 +127,7 @@ "# lm builder (helper)\n", "elmo_lm_builder = ELMOLMBuilder(\n", " model_config = config_file['lm']['model']['config'],\n", - " tokenizer=\"./outputs/en.1-percent.elmo-bert-causal\", # use pre-trained tokenizer\n", + " tokenizer=\"./outputs/en.1-percent.elmo-bert-causal-fixr2l\", # use pre-trained tokenizer\n", " model_type=\"bert-causal-elmo\",\n", " max_len=128\n", ")\n", @@ -139,8 +139,8 @@ }, { "cell_type": "code", - "execution_count": 8, - "id": "governing-industry", + "execution_count": 7, + "id": "mighty-major", "metadata": {}, "outputs": [ { @@ -163,8 +163,8 @@ }, { "cell_type": "code", - "execution_count": 9, - "id": "timely-bobby", + "execution_count": 8, + "id": "attempted-tomato", "metadata": {}, "outputs": [ { @@ -173,7 +173,7 @@ "torch.Size([1, 127])" ] }, - "execution_count": 9, + "execution_count": 8, "metadata": {}, "output_type": "execute_result" } @@ -186,15 +186,15 @@ { "cell_type": "code", "execution_count": null, - "id": "sized-description", + "id": "fallen-statement", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", - "execution_count": 10, - "id": "charming-directive", + "execution_count": 9, + "id": "paperback-luther", "metadata": {}, "outputs": [ { @@ -212,7 +212,7 @@ }, { "cell_type": "markdown", - "id": "mighty-generator", + "id": "fluid-product", "metadata": {}, "source": [ "## Sanity Check" @@ -220,8 +220,8 @@ }, { "cell_type": "code", - "execution_count": 1, - "id": "saving-member", + "execution_count": 10, + "id": "dimensional-sight", "metadata": {}, "outputs": [], "source": [ @@ -230,8 +230,8 @@ }, { "cell_type": "code", - "execution_count": 12, - "id": "colored-newman", + "execution_count": 11, + "id": "environmental-experience", "metadata": {}, "outputs": [], "source": [ @@ -262,8 +262,8 @@ }, { "cell_type": "code", - "execution_count": 13, - "id": "affiliated-bangkok", + "execution_count": 12, + "id": "informed-termination", "metadata": {}, "outputs": [], "source": [ @@ -278,14 +278,14 @@ { "cell_type": "code", "execution_count": null, - "id": "durable-brass", + "id": "complex-music", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", - "id": "automated-cheese", + "id": "tight-bennett", "metadata": {}, "source": [ "#### Normal vs Reverse" @@ -293,8 +293,8 @@ }, { "cell_type": "code", - "execution_count": 14, - "id": "infinite-liberty", + "execution_count": 13, + "id": "respective-beauty", "metadata": {}, "outputs": [], "source": [ @@ -303,8 +303,8 @@ }, { "cell_type": "code", - "execution_count": 15, - "id": "weird-analyst", + "execution_count": 14, + "id": "direct-warren", "metadata": {}, "outputs": [ { @@ -410,7 +410,7 @@ "[127 rows x 2 columns]" ] }, - "execution_count": 15, + "execution_count": 14, "metadata": {}, "output_type": "execute_result" } @@ -421,8 +421,8 @@ }, { "cell_type": "code", - "execution_count": 16, - "id": "conscious-synthetic", + "execution_count": 15, + "id": "fourth-salmon", "metadata": {}, "outputs": [ { @@ -431,7 +431,7 @@ "(torch.Size([1, 127]), torch.Size([1, 127]))" ] }, - "execution_count": 16, + "execution_count": 15, "metadata": {}, "output_type": "execute_result" } @@ -442,16 +442,16 @@ }, { "cell_type": "code", - "execution_count": 17, - "id": "great-saturn", + "execution_count": 16, + "id": "violent-shell", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "l2r_loss tensor(4.3262, grad_fn=)\n", - "r2l_loss tensor(6.8976, grad_fn=)\n" + "l2r_loss tensor(4.4801, grad_fn=)\n", + "r2l_loss tensor(4.3589, grad_fn=)\n" ] } ], @@ -461,16 +461,16 @@ }, { "cell_type": "code", - "execution_count": 18, - "id": "balanced-convention", + "execution_count": 17, + "id": "related-mambo", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "l2r_loss tensor(9.2779, grad_fn=)\n", - "r2l_loss tensor(6.5343, grad_fn=)\n" + "l2r_loss tensor(9.1742, grad_fn=)\n", + "r2l_loss tensor(9.0804, grad_fn=)\n" ] } ], @@ -481,14 +481,14 @@ { "cell_type": "code", "execution_count": null, - "id": "regulation-found", + "id": "retired-bahamas", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", - "id": "revolutionary-procedure", + "id": "broad-agriculture", "metadata": {}, "source": [ "#### Random String" @@ -496,8 +496,8 @@ }, { "cell_type": "code", - "execution_count": 24, - "id": "crude-dodge", + "execution_count": 18, + "id": "remarkable-sailing", "metadata": {}, "outputs": [ { @@ -506,7 +506,7 @@ "torch.Size([1, 127])" ] }, - "execution_count": 24, + "execution_count": 18, "metadata": {}, "output_type": "execute_result" } @@ -518,7 +518,7 @@ { "cell_type": "code", "execution_count": 29, - "id": "resident-opinion", + "id": "fifth-booth", "metadata": {}, "outputs": [], "source": [ @@ -548,7 +548,7 @@ { "cell_type": "code", "execution_count": 30, - "id": "better-hamburg", + "id": "white-juice", "metadata": {}, "outputs": [], "source": [ @@ -558,14 +558,14 @@ { "cell_type": "code", "execution_count": null, - "id": "conscious-ancient", + "id": "addressed-collective", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", - "id": "common-breed", + "id": "abroad-johnson", "metadata": {}, "source": [ "##### Trial-1" @@ -574,7 +574,7 @@ { "cell_type": "code", "execution_count": 21, - "id": "swiss-regression", + "id": "inclusive-station", "metadata": {}, "outputs": [ { @@ -610,23 +610,23 @@ " \n", " \n", " 1\n", - " escaped\n", - " marriage\n", + " Lex\n", + " ##leg\n", " \n", " \n", " 2\n", - " Develop\n", - " circulated\n", + " neither\n", + " squinting\n", " \n", " \n", " 3\n", - " elb\n", - " ъ\n", + " fever\n", + " ы\n", " \n", " \n", " 4\n", - " Madame\n", - " Regional\n", + " Leeds\n", + " Russell\n", " \n", " \n", " ...\n", @@ -635,23 +635,23 @@ " \n", " \n", " 122\n", - " Regional\n", - " Madame\n", + " Russell\n", + " Leeds\n", " \n", " \n", " 123\n", - " ъ\n", - " elb\n", + " ы\n", + " fever\n", " \n", " \n", " 124\n", - " circulated\n", - " Develop\n", + " squinting\n", + " neither\n", " \n", " \n", " 125\n", - " marriage\n", - " escaped\n", + " ##leg\n", + " Lex\n", " \n", " \n", " 126\n", @@ -664,18 +664,18 @@ "" ], "text/plain": [ - " forward reverse\n", - "0 [CLS] [CLS]\n", - "1 escaped marriage\n", - "2 Develop circulated\n", - "3 elb ъ\n", - "4 Madame Regional\n", - ".. ... ...\n", - "122 Regional Madame\n", - "123 ъ elb\n", - "124 circulated Develop\n", - "125 marriage escaped\n", - "126 [SEP] [SEP]\n", + " forward reverse\n", + "0 [CLS] [CLS]\n", + "1 Lex ##leg\n", + "2 neither squinting\n", + "3 fever ы\n", + "4 Leeds Russell\n", + ".. ... ...\n", + "122 Russell Leeds\n", + "123 ы fever\n", + "124 squinting neither\n", + "125 ##leg Lex\n", + "126 [SEP] [SEP]\n", "\n", "[127 rows x 2 columns]" ] @@ -692,15 +692,15 @@ { "cell_type": "code", "execution_count": 22, - "id": "italian-basement", + "id": "hungarian-soundtrack", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "l2r_loss tensor(13.3687, grad_fn=)\n", - "r2l_loss tensor(10.0583, grad_fn=)\n" + "l2r_loss tensor(13.5902, grad_fn=)\n", + "r2l_loss tensor(13.4444, grad_fn=)\n" ] } ], @@ -711,15 +711,15 @@ { "cell_type": "code", "execution_count": 23, - "id": "indirect-minnesota", + "id": "employed-yukon", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "l2r_loss tensor(13.1889, grad_fn=)\n", - "r2l_loss tensor(9.9842, grad_fn=)\n" + "l2r_loss tensor(13.7210, grad_fn=)\n", + "r2l_loss tensor(13.3258, grad_fn=)\n" ] } ], @@ -730,14 +730,14 @@ { "cell_type": "code", "execution_count": null, - "id": "diverse-pleasure", + "id": "voluntary-necessity", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", - "id": "creative-warehouse", + "id": "judicial-republic", "metadata": {}, "source": [ "##### Trial-2" @@ -746,7 +746,7 @@ { "cell_type": "code", "execution_count": 26, - "id": "blessed-company", + "id": "reported-sweden", "metadata": {}, "outputs": [ { @@ -782,23 +782,23 @@ " \n", " \n", " 1\n", - " oscill\n", - " Horatio\n", + " Byrne\n", + " Clean\n", " \n", " \n", " 2\n", - " transluc\n", - " Wig\n", + " vanilla\n", + " Wear\n", " \n", " \n", " 3\n", - " ##brook\n", - " assuming\n", + " departed\n", + " flickered\n", " \n", " \n", " 4\n", - " Perth\n", - " meetings\n", + " ##ische\n", + " ##ghter\n", " \n", " \n", " ...\n", @@ -807,23 +807,23 @@ " \n", " \n", " 122\n", - " meetings\n", - " Perth\n", + " ##ghter\n", + " ##ische\n", " \n", " \n", " 123\n", - " assuming\n", - " ##brook\n", + " flickered\n", + " departed\n", " \n", " \n", " 124\n", - " Wig\n", - " transluc\n", + " Wear\n", + " vanilla\n", " \n", " \n", " 125\n", - " Horatio\n", - " oscill\n", + " Clean\n", + " Byrne\n", " \n", " \n", " 126\n", @@ -836,18 +836,18 @@ "" ], "text/plain": [ - " forward reverse\n", - "0 [CLS] [CLS]\n", - "1 oscill Horatio\n", - "2 transluc Wig\n", - "3 ##brook assuming\n", - "4 Perth meetings\n", - ".. ... ...\n", - "122 meetings Perth\n", - "123 assuming ##brook\n", - "124 Wig transluc\n", - "125 Horatio oscill\n", - "126 [SEP] [SEP]\n", + " forward reverse\n", + "0 [CLS] [CLS]\n", + "1 Byrne Clean\n", + "2 vanilla Wear\n", + "3 departed flickered\n", + "4 ##ische ##ghter\n", + ".. ... ...\n", + "122 ##ghter ##ische\n", + "123 flickered departed\n", + "124 Wear vanilla\n", + "125 Clean Byrne\n", + "126 [SEP] [SEP]\n", "\n", "[127 rows x 2 columns]" ] @@ -864,15 +864,15 @@ { "cell_type": "code", "execution_count": 27, - "id": "convenient-benefit", + "id": "equipped-ordinary", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "l2r_loss tensor(13.2914, grad_fn=)\n", - "r2l_loss tensor(10.1427, grad_fn=)\n" + "l2r_loss tensor(13.4740, grad_fn=)\n", + "r2l_loss tensor(13.1427, grad_fn=)\n" ] } ], @@ -883,15 +883,15 @@ { "cell_type": "code", "execution_count": 28, - "id": "chinese-harvest", + "id": "celtic-spider", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "l2r_loss tensor(13.2515, grad_fn=)\n", - "r2l_loss tensor(10.0191, grad_fn=)\n" + "l2r_loss tensor(13.3491, grad_fn=)\n", + "r2l_loss tensor(13.3746, grad_fn=)\n" ] } ], @@ -902,14 +902,14 @@ { "cell_type": "code", "execution_count": null, - "id": "peripheral-minnesota", + "id": "rough-kelly", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", - "id": "opposite-cooperative", + "id": "invisible-white", "metadata": {}, "source": [ "##### Trial-3" @@ -918,7 +918,7 @@ { "cell_type": "code", "execution_count": 31, - "id": "competent-verification", + "id": "chubby-progressive", "metadata": {}, "outputs": [ { @@ -954,23 +954,23 @@ " \n", " \n", " 1\n", - " Stevens\n", - " Lima\n", + " Severance\n", + " Hild\n", " \n", " \n", " 2\n", - " Aden\n", - " ##cing\n", + " endorse\n", + " nor\n", " \n", " \n", " 3\n", - " Devil\n", - " ##wo\n", + " reviewed\n", + " ##frey\n", " \n", " \n", " 4\n", - " ##ibald\n", - " gate\n", + " Alone\n", + " Fred\n", " \n", " \n", " ...\n", @@ -979,23 +979,23 @@ " \n", " \n", " 122\n", - " gate\n", - " ##ibald\n", + " Fred\n", + " Alone\n", " \n", " \n", " 123\n", - " ##wo\n", - " Devil\n", + " ##frey\n", + " reviewed\n", " \n", " \n", " 124\n", - " ##cing\n", - " Aden\n", + " nor\n", + " endorse\n", " \n", " \n", " 125\n", - " Lima\n", - " Stevens\n", + " Hild\n", + " Severance\n", " \n", " \n", " 126\n", @@ -1008,18 +1008,18 @@ "" ], "text/plain": [ - " forward reverse\n", - "0 [CLS] [CLS]\n", - "1 Stevens Lima\n", - "2 Aden ##cing\n", - "3 Devil ##wo\n", - "4 ##ibald gate\n", - ".. ... ...\n", - "122 gate ##ibald\n", - "123 ##wo Devil\n", - "124 ##cing Aden\n", - "125 Lima Stevens\n", - "126 [SEP] [SEP]\n", + " forward reverse\n", + "0 [CLS] [CLS]\n", + "1 Severance Hild\n", + "2 endorse nor\n", + "3 reviewed ##frey\n", + "4 Alone Fred\n", + ".. ... ...\n", + "122 Fred Alone\n", + "123 ##frey reviewed\n", + "124 nor endorse\n", + "125 Hild Severance\n", + "126 [SEP] [SEP]\n", "\n", "[127 rows x 2 columns]" ] @@ -1036,15 +1036,15 @@ { "cell_type": "code", "execution_count": 32, - "id": "dimensional-trauma", + "id": "partial-devil", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "l2r_loss tensor(13.3330, grad_fn=)\n", - "r2l_loss tensor(9.8861, grad_fn=)\n" + "l2r_loss tensor(13.9457, grad_fn=)\n", + "r2l_loss tensor(13.5547, grad_fn=)\n" ] } ], @@ -1055,15 +1055,15 @@ { "cell_type": "code", "execution_count": 33, - "id": "typical-president", + "id": "structured-farmer", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "l2r_loss tensor(13.3210, grad_fn=)\n", - "r2l_loss tensor(9.9579, grad_fn=)\n" + "l2r_loss tensor(13.5305, grad_fn=)\n", + "r2l_loss tensor(13.4759, grad_fn=)\n" ] } ], @@ -1074,7 +1074,7 @@ { "cell_type": "code", "execution_count": null, - "id": "arabic-announcement", + "id": "funky-space", "metadata": {}, "outputs": [], "source": [] From ea5e7286e05097f7768d2a57980c9dd6230694d8 Mon Sep 17 00:00:00 2001 From: Made NN Date: Mon, 22 Nov 2021 19:43:46 +0700 Subject: [PATCH 8/9] sanity 40k --- elmo-sanity-check-v2-elmo_40k.ipynb | 535 ++++++++++ elmo-sanity-check-v2-elmo_small_40k.ipynb | 1104 +++++++++++++++++++++ 2 files changed, 1639 insertions(+) create mode 100644 elmo-sanity-check-v2-elmo_40k.ipynb create mode 100644 elmo-sanity-check-v2-elmo_small_40k.ipynb diff --git a/elmo-sanity-check-v2-elmo_40k.ipynb b/elmo-sanity-check-v2-elmo_40k.ipynb new file mode 100644 index 0000000..1d4b38d --- /dev/null +++ b/elmo-sanity-check-v2-elmo_40k.ipynb @@ -0,0 +1,535 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "passive-benchmark", + "metadata": {}, + "source": [ + "## Prepare Data + Model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "entire-diagram", + "metadata": {}, + "outputs": [], + "source": [ + "# !cat examples/data/text_forward.txt" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "diverse-brother", + "metadata": {}, + "outputs": [], + "source": [ + "# !ls -al ./outputs/en.1-percent.elmo-bert-causal-fixr2l" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "tight-loading", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "instant-proof", + "metadata": {}, + "outputs": [], + "source": [ + "import torch\n", + "from newlm.lm.elmo.modeling_elmo.elmo_head import ELMOBertLMHeadModel\n", + "from newlm.lm.elmo.lm_builder import ELMOLMBuilder\n", + "from transformers import BertConfig" + ] + }, + { + "cell_type": "markdown", + "id": "amino-trainer", + "metadata": {}, + "source": [ + "#### Model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "informative-sword", + "metadata": {}, + "outputs": [], + "source": [ + "model = ELMOBertLMHeadModel.from_pretrained(\n", + " \"./outputs/en.100-percent.elmo-bert-causal.40k\"\n", + ") # use pre-trained model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "olive-regular", + "metadata": {}, + "outputs": [], + "source": [ + "model.eval()\n", + "print(\"Model in eval mode for consistency\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "designed-telling", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "id": "specific-belief", + "metadata": {}, + "source": [ + "#### Data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "exotic-hepatitis", + "metadata": {}, + "outputs": [], + "source": [ + "%%capture\n", + "\n", + "from newlm.utils.file_util import read_from_yaml\n", + "config_file = read_from_yaml('examples/configs_gcloud/run-100-percent.elmo-bert-causal.yaml')\n", + "\n", + "# lm builder (helper)\n", + "elmo_lm_builder = ELMOLMBuilder(\n", + " model_config = config_file['lm']['model']['config'],\n", + " tokenizer=\"./outputs/en.100-percent.elmo-bert-causal.40k\", # use pre-trained tokenizer\n", + " model_type=\"bert-causal-elmo\",\n", + " max_len=128\n", + ")\n", + "\n", + "# dataset-forward\n", + "train_path = \"./examples/data/text_forward-small.txt\"\n", + "ds_f = elmo_lm_builder._get_dataset(train_path)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "considered-discharge", + "metadata": {}, + "outputs": [], + "source": [ + "# trainer (helper)\n", + "from transformers import TrainingArguments, Trainer\n", + "args = TrainingArguments(output_dir=\"tmpout\",**config_file['lm']['hf_trainer']['args'])\n", + "\n", + "# dataloader-forward\n", + "trainer = Trainer(model=model, args=args, data_collator=elmo_lm_builder.data_collator, train_dataset=ds_f,)\n", + "dl_f = trainer.get_train_dataloader() # Data Loader-forward" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "studied-catholic", + "metadata": {}, + "outputs": [], + "source": [ + "batch_f = next(iter(dl_f))\n", + "batch_f['input_ids'].shape" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "pending-hands", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "frequent-conspiracy", + "metadata": {}, + "outputs": [], + "source": [ + "model.eval()\n", + "print(\"Model in eval mode for consistency\")" + ] + }, + { + "cell_type": "markdown", + "id": "challenging-tracker", + "metadata": {}, + "source": [ + "## Sanity Check" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "damaged-decimal", + "metadata": {}, + "outputs": [], + "source": [ + "# batch_f" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "distributed-ireland", + "metadata": {}, + "outputs": [], + "source": [ + "import torch\n", + "\n", + "def reverse_batch(batch_f):\n", + " # reverse input\n", + " batch_f_input = torch.clone(batch_f['input_ids'])\n", + " batch_f_rev_input = torch.cat(\n", + " (\n", + " batch_f_input[0][0:1],\n", + " torch.flip(batch_f_input[0][1:-1], [0]),\n", + " batch_f_input[0][-1:]\n", + " )\n", + " )\n", + " batch_f_rev_input = batch_f_rev_input.reshape(1,-1)\n", + "\n", + " # reverse labels\n", + " batch_f_rev_labels = torch.clone(batch_f_rev_input)\n", + " \n", + " # batch_rev\n", + " batch_rev = batch_f.copy()\n", + " batch_rev['input_ids'] = batch_f_rev_input\n", + " batch_rev['labels'] = batch_f_rev_labels\n", + " \n", + " return batch_rev" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "alpine-moore", + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "\n", + "def pandas_check(batch_f, batch_rev):\n", + " tokens_f = elmo_lm_builder.tokenizer.convert_ids_to_tokens(batch_f['input_ids'][0])\n", + " tokens_f_rev = elmo_lm_builder.tokenizer.convert_ids_to_tokens(batch_rev['input_ids'][0])\n", + " return pd.DataFrame({\"forward\": tokens_f, \"reverse\": tokens_f_rev})" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "forty-essence", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "id": "speaking-defeat", + "metadata": {}, + "source": [ + "#### Normal vs Reverse" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "major-gates", + "metadata": {}, + "outputs": [], + "source": [ + "batch_rev = reverse_batch(batch_f)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "linear-abuse", + "metadata": {}, + "outputs": [], + "source": [ + "pandas_check(batch_f, batch_rev)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "weird-australian", + "metadata": {}, + "outputs": [], + "source": [ + "batch_f['input_ids'].shape, batch_rev['input_ids'].shape" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "noted-equivalent", + "metadata": {}, + "outputs": [], + "source": [ + "res = model(**batch_f) # forward" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "yellow-progressive", + "metadata": {}, + "outputs": [], + "source": [ + "res = model(**batch_rev) # reverse" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "golden-major", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "id": "every-value", + "metadata": {}, + "source": [ + "#### Random String" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fiscal-society", + "metadata": {}, + "outputs": [], + "source": [ + "batch_f['input_ids'].shape" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "senior-breast", + "metadata": {}, + "outputs": [], + "source": [ + "# shuffle data\n", + "batch_f_input = batch_f['input_ids']\n", + "batch_shuffle_input = torch.cat(\n", + " (\n", + " batch_f_input[0][0:1],\n", + " torch.randint(\n", + " low=5, # 0-4 > ['[PAD]', '[UNK]', '[CLS]', '[SEP]', '[MASK]']\n", + " high=29999,\n", + " size=(121,), # modified based on seqlen!\n", + " dtype=torch.long\n", + " ),\n", + " batch_f_input[0][-1:]\n", + " )\n", + ")\n", + "batch_shuffle_input = batch_shuffle_input.reshape(1,-1)\n", + "# labels\n", + "batch_shuffle_labels = torch.clone(batch_shuffle_input) \n", + "# batch_shuffle\n", + "batch_shuffle = batch_f.copy()\n", + "batch_shuffle['input_ids'] = batch_shuffle_input\n", + "batch_shuffle['labels'] = batch_shuffle_labels" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "downtown-spouse", + "metadata": {}, + "outputs": [], + "source": [ + "batch_shuffle_rev = reverse_batch(batch_shuffle)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "otherwise-department", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "id": "surface-saying", + "metadata": {}, + "source": [ + "##### Trial-1" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "sexual-wilderness", + "metadata": {}, + "outputs": [], + "source": [ + "pandas_check(batch_shuffle, batch_shuffle_rev)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "gothic-understanding", + "metadata": {}, + "outputs": [], + "source": [ + "res = model(**batch_shuffle) # forward" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "boolean-holiday", + "metadata": {}, + "outputs": [], + "source": [ + "res = model(**batch_shuffle_rev) # reverse" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "talented-spanish", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "id": "australian-logistics", + "metadata": {}, + "source": [ + "##### Trial-2" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "italic-responsibility", + "metadata": {}, + "outputs": [], + "source": [ + "pandas_check(batch_shuffle, batch_shuffle_rev)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "three-simpson", + "metadata": {}, + "outputs": [], + "source": [ + "res = model(**batch_shuffle) # forward" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "classified-pennsylvania", + "metadata": {}, + "outputs": [], + "source": [ + "res = model(**batch_shuffle_rev) # reverse" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "greenhouse-reader", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "id": "announced-economics", + "metadata": {}, + "source": [ + "##### Trial-3" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "tutorial-circuit", + "metadata": {}, + "outputs": [], + "source": [ + "pandas_check(batch_shuffle, batch_shuffle_rev)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "round-watts", + "metadata": {}, + "outputs": [], + "source": [ + "res = model(**batch_shuffle) # forward" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "juvenile-walter", + "metadata": {}, + "outputs": [], + "source": [ + "res = model(**batch_shuffle_rev) # reverse" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dying-congress", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.5" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/elmo-sanity-check-v2-elmo_small_40k.ipynb b/elmo-sanity-check-v2-elmo_small_40k.ipynb new file mode 100644 index 0000000..679bcb5 --- /dev/null +++ b/elmo-sanity-check-v2-elmo_small_40k.ipynb @@ -0,0 +1,1104 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "swedish-breast", + "metadata": {}, + "source": [ + "## Prepare Data + Model" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "former-delta", + "metadata": {}, + "outputs": [], + "source": [ + "# !cat examples/data/text_forward.txt" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "noted-jacksonville", + "metadata": {}, + "outputs": [], + "source": [ + "# !ls -al ./outputs/en.1-percent.elmo-bert-causal-fixr2l" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "requested-nudist", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "atomic-orbit", + "metadata": {}, + "outputs": [], + "source": [ + "import torch\n", + "from newlm.lm.elmo.modeling_elmo.elmo_head import ELMOBertLMHeadModel\n", + "from newlm.lm.elmo.lm_builder import ELMOLMBuilder\n", + "from transformers import BertConfig" + ] + }, + { + "cell_type": "markdown", + "id": "empirical-forestry", + "metadata": {}, + "source": [ + "#### Model" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "outside-consultancy", + "metadata": {}, + "outputs": [], + "source": [ + "model = ELMOBertLMHeadModel.from_pretrained(\n", + " \"./outputs/en.100-percent.elmo-small-bert-causal.40k\"\n", + ") # use pre-trained model" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "fewer-harmony", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Model in eval mode for consistency\n" + ] + } + ], + "source": [ + "model.eval()\n", + "print(\"Model in eval mode for consistency\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "impressive-convert", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "id": "radical-secret", + "metadata": {}, + "source": [ + "#### Data" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "sexual-wisconsin", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2021-11-22 19:37:37.967 | INFO | newlm.lm.elmo.lm_builder:_get_dataset:142 - Constructing roBERTa style dataset\n" + ] + } + ], + "source": [ + "%%capture\n", + "\n", + "from newlm.utils.file_util import read_from_yaml\n", + "config_file = read_from_yaml('examples/configs_gcloud/run-100-percent.elmo-small-bert-causal.yaml')\n", + "\n", + "# lm builder (helper)\n", + "elmo_lm_builder = ELMOLMBuilder(\n", + " model_config = config_file['lm']['model']['config'],\n", + " tokenizer=\"./outputs/en.100-percent.elmo-small-bert-causal.40k\", # use pre-trained tokenizer\n", + " model_type=\"bert-causal-elmo\",\n", + " max_len=128\n", + ")\n", + "\n", + "# dataset-forward\n", + "train_path = \"./examples/data/text_forward-small.txt\"\n", + "ds_f = elmo_lm_builder._get_dataset(train_path)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "sophisticated-texas", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "max_steps is given, it will override any value given in num_train_epochs\n" + ] + } + ], + "source": [ + "# trainer (helper)\n", + "from transformers import TrainingArguments, Trainer\n", + "args = TrainingArguments(output_dir=\"tmpout\",**config_file['lm']['hf_trainer']['args'])\n", + "\n", + "# dataloader-forward\n", + "trainer = Trainer(model=model, args=args, data_collator=elmo_lm_builder.data_collator, train_dataset=ds_f,)\n", + "dl_f = trainer.get_train_dataloader() # Data Loader-forward" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "attractive-insight", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "torch.Size([1, 123])" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "batch_f = next(iter(dl_f))\n", + "batch_f['input_ids'].shape" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "incorporate-integration", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "original-voluntary", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Model in eval mode for consistency\n" + ] + } + ], + "source": [ + "model.eval()\n", + "print(\"Model in eval mode for consistency\")" + ] + }, + { + "cell_type": "markdown", + "id": "empirical-accessory", + "metadata": {}, + "source": [ + "## Sanity Check" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "impaired-soviet", + "metadata": {}, + "outputs": [], + "source": [ + "# batch_f" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "activated-platinum", + "metadata": {}, + "outputs": [], + "source": [ + "import torch\n", + "\n", + "def reverse_batch(batch_f):\n", + " # reverse input\n", + " batch_f_input = torch.clone(batch_f['input_ids'])\n", + " batch_f_rev_input = torch.cat(\n", + " (\n", + " batch_f_input[0][0:1],\n", + " torch.flip(batch_f_input[0][1:-1], [0]),\n", + " batch_f_input[0][-1:]\n", + " )\n", + " )\n", + " batch_f_rev_input = batch_f_rev_input.reshape(1,-1)\n", + "\n", + " # reverse labels\n", + " batch_f_rev_labels = torch.clone(batch_f_rev_input)\n", + " \n", + " # batch_rev\n", + " batch_rev = batch_f.copy()\n", + " batch_rev['input_ids'] = batch_f_rev_input\n", + " batch_rev['labels'] = batch_f_rev_labels\n", + " \n", + " return batch_rev" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "adopted-tamil", + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "\n", + "def pandas_check(batch_f, batch_rev):\n", + " tokens_f = elmo_lm_builder.tokenizer.convert_ids_to_tokens(batch_f['input_ids'][0])\n", + " tokens_f_rev = elmo_lm_builder.tokenizer.convert_ids_to_tokens(batch_rev['input_ids'][0])\n", + " return pd.DataFrame({\"forward\": tokens_f, \"reverse\": tokens_f_rev})" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "opponent-grant", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "id": "electoral-effort", + "metadata": {}, + "source": [ + "#### Normal vs Reverse" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "elegant-occasions", + "metadata": {}, + "outputs": [], + "source": [ + "batch_rev = reverse_batch(batch_f)" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "demanding-wealth", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
forwardreverse
0[CLS][CLS]
1[UNK].
2isJews
3locatedamong
4inlived
.........
118livedin
119amonglocated
120Jewsis
121.[UNK]
122[SEP][SEP]
\n", + "

123 rows × 2 columns

\n", + "
" + ], + "text/plain": [ + " forward reverse\n", + "0 [CLS] [CLS]\n", + "1 [UNK] .\n", + "2 is Jews\n", + "3 located among\n", + "4 in lived\n", + ".. ... ...\n", + "118 lived in\n", + "119 among located\n", + "120 Jews is\n", + "121 . [UNK]\n", + "122 [SEP] [SEP]\n", + "\n", + "[123 rows x 2 columns]" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "pandas_check(batch_f, batch_rev)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "stuffed-lightweight", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(torch.Size([1, 123]), torch.Size([1, 123]))" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "batch_f['input_ids'].shape, batch_rev['input_ids'].shape" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "norman-pricing", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "l2r_loss tensor(4.0284, grad_fn=)\n", + "r2l_loss tensor(3.9578, grad_fn=)\n" + ] + } + ], + "source": [ + "res = model(**batch_f) # forward" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "divine-joseph", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "l2r_loss tensor(9.0815, grad_fn=)\n", + "r2l_loss tensor(9.1060, grad_fn=)\n" + ] + } + ], + "source": [ + "res = model(**batch_rev) # reverse" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "hairy-logan", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "id": "expanded-portal", + "metadata": {}, + "source": [ + "#### Random String" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "understanding-knock", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "torch.Size([1, 123])" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "batch_f['input_ids'].shape" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "raising-deviation", + "metadata": {}, + "outputs": [], + "source": [ + "# shuffle data\n", + "batch_f_input = batch_f['input_ids']\n", + "batch_shuffle_input = torch.cat(\n", + " (\n", + " batch_f_input[0][0:1],\n", + " torch.randint(\n", + " low=5, # 0-4 > ['[PAD]', '[UNK]', '[CLS]', '[SEP]', '[MASK]']\n", + " high=29999,\n", + " size=(121,), # modified based on seqlen!\n", + " dtype=torch.long\n", + " ),\n", + " batch_f_input[0][-1:]\n", + " )\n", + ")\n", + "batch_shuffle_input = batch_shuffle_input.reshape(1,-1)\n", + "# labels\n", + "batch_shuffle_labels = torch.clone(batch_shuffle_input) \n", + "# batch_shuffle\n", + "batch_shuffle = batch_f.copy()\n", + "batch_shuffle['input_ids'] = batch_shuffle_input\n", + "batch_shuffle['labels'] = batch_shuffle_labels" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "headed-aside", + "metadata": {}, + "outputs": [], + "source": [ + "batch_shuffle_rev = reverse_batch(batch_shuffle)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "distinct-steel", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "id": "dirty-doctor", + "metadata": {}, + "source": [ + "##### Trial-1" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "virtual-church", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
forwardreverse
0[CLS][CLS]
1##wichbattery
2deskNer
3Collectionought
4fisherMug
.........
118Mugfisher
119oughtCollection
120Nerdesk
121battery##wich
122[SEP][SEP]
\n", + "

123 rows × 2 columns

\n", + "
" + ], + "text/plain": [ + " forward reverse\n", + "0 [CLS] [CLS]\n", + "1 ##wich battery\n", + "2 desk Ner\n", + "3 Collection ought\n", + "4 fisher Mug\n", + ".. ... ...\n", + "118 Mug fisher\n", + "119 ought Collection\n", + "120 Ner desk\n", + "121 battery ##wich\n", + "122 [SEP] [SEP]\n", + "\n", + "[123 rows x 2 columns]" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "pandas_check(batch_shuffle, batch_shuffle_rev)" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "artificial-salem", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "l2r_loss tensor(14.4735, grad_fn=)\n", + "r2l_loss tensor(13.7806, grad_fn=)\n" + ] + } + ], + "source": [ + "res = model(**batch_shuffle) # forward" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "blond-pitch", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "l2r_loss tensor(14.0861, grad_fn=)\n", + "r2l_loss tensor(13.5071, grad_fn=)\n" + ] + } + ], + "source": [ + "res = model(**batch_shuffle_rev) # reverse" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "clean-favorite", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "id": "another-penetration", + "metadata": {}, + "source": [ + "##### Trial-2" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "brilliant-antarctica", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
forwardreverse
0[CLS][CLS]
1##wichbattery
2deskNer
3Collectionought
4fisherMug
.........
118Mugfisher
119oughtCollection
120Nerdesk
121battery##wich
122[SEP][SEP]
\n", + "

123 rows × 2 columns

\n", + "
" + ], + "text/plain": [ + " forward reverse\n", + "0 [CLS] [CLS]\n", + "1 ##wich battery\n", + "2 desk Ner\n", + "3 Collection ought\n", + "4 fisher Mug\n", + ".. ... ...\n", + "118 Mug fisher\n", + "119 ought Collection\n", + "120 Ner desk\n", + "121 battery ##wich\n", + "122 [SEP] [SEP]\n", + "\n", + "[123 rows x 2 columns]" + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "pandas_check(batch_shuffle, batch_shuffle_rev)" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "animal-break", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "l2r_loss tensor(14.4735, grad_fn=)\n", + "r2l_loss tensor(13.7806, grad_fn=)\n" + ] + } + ], + "source": [ + "res = model(**batch_shuffle) # forward" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "unlikely-table", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "l2r_loss tensor(14.0861, grad_fn=)\n", + "r2l_loss tensor(13.5071, grad_fn=)\n" + ] + } + ], + "source": [ + "res = model(**batch_shuffle_rev) # reverse" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "boolean-supplier", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "id": "mexican-canadian", + "metadata": {}, + "source": [ + "##### Trial-3" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "assured-bahrain", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
forwardreverse
0[CLS][CLS]
1##wichbattery
2deskNer
3Collectionought
4fisherMug
.........
118Mugfisher
119oughtCollection
120Nerdesk
121battery##wich
122[SEP][SEP]
\n", + "

123 rows × 2 columns

\n", + "
" + ], + "text/plain": [ + " forward reverse\n", + "0 [CLS] [CLS]\n", + "1 ##wich battery\n", + "2 desk Ner\n", + "3 Collection ought\n", + "4 fisher Mug\n", + ".. ... ...\n", + "118 Mug fisher\n", + "119 ought Collection\n", + "120 Ner desk\n", + "121 battery ##wich\n", + "122 [SEP] [SEP]\n", + "\n", + "[123 rows x 2 columns]" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "pandas_check(batch_shuffle, batch_shuffle_rev)" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "promotional-panel", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "l2r_loss tensor(14.4735, grad_fn=)\n", + "r2l_loss tensor(13.7806, grad_fn=)\n" + ] + } + ], + "source": [ + "res = model(**batch_shuffle) # forward" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "moving-karma", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "l2r_loss tensor(14.0861, grad_fn=)\n", + "r2l_loss tensor(13.5071, grad_fn=)\n" + ] + } + ], + "source": [ + "res = model(**batch_shuffle_rev) # reverse" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "clean-gibson", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.5" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From 86c31545a7c9cca2e0d457c64e0bdb4c9513afb1 Mon Sep 17 00:00:00 2001 From: Made NN Date: Mon, 22 Nov 2021 19:46:58 +0700 Subject: [PATCH 9/9] run --- elmo-sanity-check-v2-elmo_40k.ipynb | 759 ++++++++++++++++++++++++---- 1 file changed, 664 insertions(+), 95 deletions(-) diff --git a/elmo-sanity-check-v2-elmo_40k.ipynb b/elmo-sanity-check-v2-elmo_40k.ipynb index 1d4b38d..d2c4073 100644 --- a/elmo-sanity-check-v2-elmo_40k.ipynb +++ b/elmo-sanity-check-v2-elmo_40k.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "markdown", - "id": "passive-benchmark", + "id": "refined-creek", "metadata": {}, "source": [ "## Prepare Data + Model" @@ -10,8 +10,8 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "entire-diagram", + "execution_count": 1, + "id": "generous-timothy", "metadata": {}, "outputs": [], "source": [ @@ -20,8 +20,8 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "diverse-brother", + "execution_count": 2, + "id": "valid-lingerie", "metadata": {}, "outputs": [], "source": [ @@ -31,15 +31,15 @@ { "cell_type": "code", "execution_count": null, - "id": "tight-loading", + "id": "optional-motivation", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", - "execution_count": null, - "id": "instant-proof", + "execution_count": 3, + "id": "tired-commodity", "metadata": {}, "outputs": [], "source": [ @@ -51,7 +51,7 @@ }, { "cell_type": "markdown", - "id": "amino-trainer", + "id": "outdoor-recognition", "metadata": {}, "source": [ "#### Model" @@ -59,8 +59,8 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "informative-sword", + "execution_count": 4, + "id": "fifty-projector", "metadata": {}, "outputs": [], "source": [ @@ -71,10 +71,18 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "olive-regular", + "execution_count": 5, + "id": "former-neighbor", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Model in eval mode for consistency\n" + ] + } + ], "source": [ "model.eval()\n", "print(\"Model in eval mode for consistency\")" @@ -83,14 +91,14 @@ { "cell_type": "code", "execution_count": null, - "id": "designed-telling", + "id": "ultimate-posting", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", - "id": "specific-belief", + "id": "living-consumption", "metadata": {}, "source": [ "#### Data" @@ -98,10 +106,18 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "exotic-hepatitis", + "execution_count": 6, + "id": "italic-resort", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2021-11-22 19:46:05.850 | INFO | newlm.lm.elmo.lm_builder:_get_dataset:142 - Constructing roBERTa style dataset\n" + ] + } + ], "source": [ "%%capture\n", "\n", @@ -123,10 +139,18 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "considered-discharge", + "execution_count": 7, + "id": "forward-litigation", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "max_steps is given, it will override any value given in num_train_epochs\n" + ] + } + ], "source": [ "# trainer (helper)\n", "from transformers import TrainingArguments, Trainer\n", @@ -139,10 +163,21 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "studied-catholic", + "execution_count": 8, + "id": "subject-exposure", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "torch.Size([1, 123])" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "batch_f = next(iter(dl_f))\n", "batch_f['input_ids'].shape" @@ -151,17 +186,25 @@ { "cell_type": "code", "execution_count": null, - "id": "pending-hands", + "id": "supposed-relief", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", - "execution_count": null, - "id": "frequent-conspiracy", + "execution_count": 9, + "id": "international-debate", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Model in eval mode for consistency\n" + ] + } + ], "source": [ "model.eval()\n", "print(\"Model in eval mode for consistency\")" @@ -169,7 +212,7 @@ }, { "cell_type": "markdown", - "id": "challenging-tracker", + "id": "substantial-warning", "metadata": {}, "source": [ "## Sanity Check" @@ -177,8 +220,8 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "damaged-decimal", + "execution_count": 10, + "id": "weekly-reverse", "metadata": {}, "outputs": [], "source": [ @@ -187,8 +230,8 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "distributed-ireland", + "execution_count": 11, + "id": "optimum-italy", "metadata": {}, "outputs": [], "source": [ @@ -219,8 +262,8 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "alpine-moore", + "execution_count": 12, + "id": "eastern-transfer", "metadata": {}, "outputs": [], "source": [ @@ -235,14 +278,14 @@ { "cell_type": "code", "execution_count": null, - "id": "forty-essence", + "id": "flush-syracuse", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", - "id": "speaking-defeat", + "id": "human-reynolds", "metadata": {}, "source": [ "#### Normal vs Reverse" @@ -250,8 +293,8 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "major-gates", + "execution_count": 13, + "id": "sexual-meditation", "metadata": {}, "outputs": [], "source": [ @@ -260,40 +303,177 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "linear-abuse", + "execution_count": 14, + "id": "rotary-ontario", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
forwardreverse
0[CLS][CLS]
1[UNK].
2isJews
3locatedamong
4inlived
.........
118livedin
119amonglocated
120Jewsis
121.[UNK]
122[SEP][SEP]
\n", + "

123 rows × 2 columns

\n", + "
" + ], + "text/plain": [ + " forward reverse\n", + "0 [CLS] [CLS]\n", + "1 [UNK] .\n", + "2 is Jews\n", + "3 located among\n", + "4 in lived\n", + ".. ... ...\n", + "118 lived in\n", + "119 among located\n", + "120 Jews is\n", + "121 . [UNK]\n", + "122 [SEP] [SEP]\n", + "\n", + "[123 rows x 2 columns]" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "pandas_check(batch_f, batch_rev)" ] }, { "cell_type": "code", - "execution_count": null, - "id": "weird-australian", + "execution_count": 15, + "id": "beginning-committee", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "(torch.Size([1, 123]), torch.Size([1, 123]))" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "batch_f['input_ids'].shape, batch_rev['input_ids'].shape" ] }, { "cell_type": "code", - "execution_count": null, - "id": "noted-equivalent", + "execution_count": 16, + "id": "cultural-block", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "l2r_loss tensor(3.9125, grad_fn=)\n", + "r2l_loss tensor(3.8753, grad_fn=)\n" + ] + } + ], "source": [ "res = model(**batch_f) # forward" ] }, { "cell_type": "code", - "execution_count": null, - "id": "yellow-progressive", + "execution_count": 17, + "id": "sublime-rider", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "l2r_loss tensor(8.8075, grad_fn=)\n", + "r2l_loss tensor(8.8464, grad_fn=)\n" + ] + } + ], "source": [ "res = model(**batch_rev) # reverse" ] @@ -301,14 +481,14 @@ { "cell_type": "code", "execution_count": null, - "id": "golden-major", + "id": "european-examination", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", - "id": "every-value", + "id": "synthetic-volleyball", "metadata": {}, "source": [ "#### Random String" @@ -316,18 +496,29 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "fiscal-society", + "execution_count": 18, + "id": "ecological-advocate", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "torch.Size([1, 123])" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "batch_f['input_ids'].shape" ] }, { "cell_type": "code", - "execution_count": null, - "id": "senior-breast", + "execution_count": 19, + "id": "cosmetic-shame", "metadata": {}, "outputs": [], "source": [ @@ -356,8 +547,8 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "downtown-spouse", + "execution_count": 20, + "id": "respective-receiver", "metadata": {}, "outputs": [], "source": [ @@ -367,14 +558,14 @@ { "cell_type": "code", "execution_count": null, - "id": "otherwise-department", + "id": "religious-camping", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", - "id": "surface-saying", + "id": "advisory-subject", "metadata": {}, "source": [ "##### Trial-1" @@ -382,30 +573,156 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "sexual-wilderness", + "execution_count": 21, + "id": "rotary-reduction", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
forwardreverse
0[CLS][CLS]
1##wichbattery
2deskNer
3Collectionought
4fisherMug
.........
118Mugfisher
119oughtCollection
120Nerdesk
121battery##wich
122[SEP][SEP]
\n", + "

123 rows × 2 columns

\n", + "
" + ], + "text/plain": [ + " forward reverse\n", + "0 [CLS] [CLS]\n", + "1 ##wich battery\n", + "2 desk Ner\n", + "3 Collection ought\n", + "4 fisher Mug\n", + ".. ... ...\n", + "118 Mug fisher\n", + "119 ought Collection\n", + "120 Ner desk\n", + "121 battery ##wich\n", + "122 [SEP] [SEP]\n", + "\n", + "[123 rows x 2 columns]" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "pandas_check(batch_shuffle, batch_shuffle_rev)" ] }, { "cell_type": "code", - "execution_count": null, - "id": "gothic-understanding", + "execution_count": 22, + "id": "presidential-impossible", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "l2r_loss tensor(13.4756, grad_fn=)\n", + "r2l_loss tensor(13.2225, grad_fn=)\n" + ] + } + ], "source": [ "res = model(**batch_shuffle) # forward" ] }, { "cell_type": "code", - "execution_count": null, - "id": "boolean-holiday", + "execution_count": 23, + "id": "figured-carroll", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "l2r_loss tensor(13.2165, grad_fn=)\n", + "r2l_loss tensor(13.3072, grad_fn=)\n" + ] + } + ], "source": [ "res = model(**batch_shuffle_rev) # reverse" ] @@ -413,14 +730,14 @@ { "cell_type": "code", "execution_count": null, - "id": "talented-spanish", + "id": "fabulous-dutch", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", - "id": "australian-logistics", + "id": "dominant-falls", "metadata": {}, "source": [ "##### Trial-2" @@ -428,30 +745,156 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "italic-responsibility", + "execution_count": 24, + "id": "professional-visitor", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
forwardreverse
0[CLS][CLS]
1##wichbattery
2deskNer
3Collectionought
4fisherMug
.........
118Mugfisher
119oughtCollection
120Nerdesk
121battery##wich
122[SEP][SEP]
\n", + "

123 rows × 2 columns

\n", + "
" + ], + "text/plain": [ + " forward reverse\n", + "0 [CLS] [CLS]\n", + "1 ##wich battery\n", + "2 desk Ner\n", + "3 Collection ought\n", + "4 fisher Mug\n", + ".. ... ...\n", + "118 Mug fisher\n", + "119 ought Collection\n", + "120 Ner desk\n", + "121 battery ##wich\n", + "122 [SEP] [SEP]\n", + "\n", + "[123 rows x 2 columns]" + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "pandas_check(batch_shuffle, batch_shuffle_rev)" ] }, { "cell_type": "code", - "execution_count": null, - "id": "three-simpson", + "execution_count": 25, + "id": "reflected-thursday", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "l2r_loss tensor(13.4756, grad_fn=)\n", + "r2l_loss tensor(13.2225, grad_fn=)\n" + ] + } + ], "source": [ "res = model(**batch_shuffle) # forward" ] }, { "cell_type": "code", - "execution_count": null, - "id": "classified-pennsylvania", + "execution_count": 26, + "id": "stupid-violin", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "l2r_loss tensor(13.2165, grad_fn=)\n", + "r2l_loss tensor(13.3072, grad_fn=)\n" + ] + } + ], "source": [ "res = model(**batch_shuffle_rev) # reverse" ] @@ -459,14 +902,14 @@ { "cell_type": "code", "execution_count": null, - "id": "greenhouse-reader", + "id": "quantitative-memory", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", - "id": "announced-economics", + "id": "marine-settle", "metadata": {}, "source": [ "##### Trial-3" @@ -474,30 +917,156 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "tutorial-circuit", + "execution_count": 27, + "id": "informal-wright", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
forwardreverse
0[CLS][CLS]
1##wichbattery
2deskNer
3Collectionought
4fisherMug
.........
118Mugfisher
119oughtCollection
120Nerdesk
121battery##wich
122[SEP][SEP]
\n", + "

123 rows × 2 columns

\n", + "
" + ], + "text/plain": [ + " forward reverse\n", + "0 [CLS] [CLS]\n", + "1 ##wich battery\n", + "2 desk Ner\n", + "3 Collection ought\n", + "4 fisher Mug\n", + ".. ... ...\n", + "118 Mug fisher\n", + "119 ought Collection\n", + "120 Ner desk\n", + "121 battery ##wich\n", + "122 [SEP] [SEP]\n", + "\n", + "[123 rows x 2 columns]" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "pandas_check(batch_shuffle, batch_shuffle_rev)" ] }, { "cell_type": "code", - "execution_count": null, - "id": "round-watts", + "execution_count": 28, + "id": "devoted-brush", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "l2r_loss tensor(13.4756, grad_fn=)\n", + "r2l_loss tensor(13.2225, grad_fn=)\n" + ] + } + ], "source": [ "res = model(**batch_shuffle) # forward" ] }, { "cell_type": "code", - "execution_count": null, - "id": "juvenile-walter", + "execution_count": 29, + "id": "supported-entrance", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "l2r_loss tensor(13.2165, grad_fn=)\n", + "r2l_loss tensor(13.3072, grad_fn=)\n" + ] + } + ], "source": [ "res = model(**batch_shuffle_rev) # reverse" ] @@ -505,7 +1074,7 @@ { "cell_type": "code", "execution_count": null, - "id": "dying-congress", + "id": "inclusive-poker", "metadata": {}, "outputs": [], "source": []