-
Notifications
You must be signed in to change notification settings - Fork 0
/
pos.py
153 lines (153 loc) · 5.01 KB
/
pos.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
#
#
# from typing import Iterator, List, Dict
#
# import torch
# import torch.optim as optim
# import numpy as np
#
# from allennlp.data import Instance
# from allennlp.data.fields import TextField, SequenceLabelField
#
# from allennlp.data.dataset_readers import DatasetReader
#
# from allennlp.common.file_utils import cached_path
#
# from allennlp.data.token_indexers import TokenIndexer, SingleIdTokenIndexer
# from allennlp.data.tokenizers import Token
#
# from allennlp.data.vocabulary import Vocabulary
#
# from allennlp.models import Model
#
# from allennlp.modules.text_field_embedders import TextFieldEmbedder, BasicTextFieldEmbedder
# from allennlp.modules.token_embedders import Embedding
# from allennlp.modules.seq2seq_encoders import Seq2SeqEncoder, PytorchSeq2SeqWrapper
# from allennlp.nn.util import get_text_field_mask, sequence_cross_entropy_with_logits
#
# from allennlp.training.metrics import CategoricalAccuracy
#
# from allennlp.data.iterators import BucketIterator
#
# from allennlp.training.trainer import Trainer
#
# from allennlp.predictors import SentenceTaggerPredictor
#
# torch.manual_seed(1)
#
# class PosDatasetReader(DatasetReader):
# """
# DatasetReader for PoS tagging data, one sentence per line, like
#
# The###DET dog###NN ate###V the###DET apple###NN
# """
#
# def __init__(self, token_indexers: Dict[str, TokenIndexer] = None) -> None:
# super().__init__(lazy=False)
# self.token_indexers = token_indexers or {"tokens": SingleIdTokenIndexer()}
#
# def text_to_instance(self, tokens: List[Token], tags: List[str] = None) -> Instance:
# sentence_field = TextField(tokens, self.token_indexers)
# fields = {"sentence": sentence_field}
#
# if tags:
# label_field = SequenceLabelField(labels=tags, sequence_field=sentence_field)
# fields["labels"] = label_field
#
# return Instance(fields)
#
# def _read(self, file_path: str) -> Iterator[Instance]:
# with open(file_path) as f:
# for line in f:
# pairs = line.strip().split()
# sentence, tags = zip(*(pair.split("###") for pair in pairs))
# yield self.text_to_instance([Token(word) for word in sentence], tags)
#
# class LstmTagger(Model):
#
# def __init__(self,
#
# word_embeddings: TextFieldEmbedder,
#
# encoder: Seq2SeqEncoder,
#
# vocab: Vocabulary) -> None:
#
# super().__init__(vocab)
# self.word_embeddings = word_embeddings
# self.encoder = encoder
#
# self.hidden2tag = torch.nn.Linear(in_features=encoder.get_output_dim(),
# out_features=vocab.get_vocab_size('labels'))
#
# self.accuracy = CategoricalAccuracy()
#
# def forward(self,
# sentence: Dict[str, torch.Tensor],
# labels: torch.Tensor = None) -> torch.Tensor:
#
# mask = get_text_field_mask(sentence)
#
# embeddings = self.word_embeddings(sentence)
#
# encoder_out = self.encoder(embeddings, mask)
#
# tag_logits = self.hidden2tag(encoder_out)
# output = {"tag_logits": tag_logits}
#
# if labels is not None:
# self.accuracy(tag_logits, labels, mask)
# output["loss"] = sequence_cross_entropy_with_logits(tag_logits, labels, mask)
#
# return output
#
# def get_metrics(self, reset: bool = False) -> Dict[str, float]:
# return {"accuracy": self.accuracy.get_metric(reset)}
#
# reader = PosDatasetReader()
#
# train_dataset = reader.read(cached_path(
# 'https://raw.githubusercontent.com/allenai/allennlp'
# '/master/tutorials/tagger/training.txt'))
# validation_dataset = reader.read(cached_path(
# 'https://raw.githubusercontent.com/allenai/allennlp'
# '/master/tutorials/tagger/validation.txt'))
#
# vocab = Vocabulary.from_instances(train_dataset + validation_dataset)
#
# EMBEDDING_DIM = 6
# HIDDEN_DIM = 6
#
# token_embedding = Embedding(num_embeddings=vocab.get_vocab_size('tokens'),
# embedding_dim=EMBEDDING_DIM)
# word_embeddings = BasicTextFieldEmbedder({"tokens": token_embedding})
#
# lstm = PytorchSeq2SeqWrapper(torch.nn.LSTM(EMBEDDING_DIM, HIDDEN_DIM, batch_first=True))
#
# model = LstmTagger(word_embeddings, lstm, vocab)
#
# optimizer = optim.SGD(model.parameters(), lr=0.1)
#
# iterator = BucketIterator(batch_size=2, sorting_keys=[("sentence", "num_tokens")])
#
# iterator.index_with(vocab)
#
# trainer = Trainer(model=model,
# optimizer=optimizer,
# iterator=iterator,
# train_dataset=train_dataset,
# validation_dataset=validation_dataset,
# patience=10,
# num_epochs=1000)
#
# trainer.train()
#
# predictor = SentenceTaggerPredictor(model, dataset_reader=reader)
#
# tag_logits = predictor.predict("The dog ate the apple")['tag_logits']
#
# tag_ids = np.argmax(tag_logits, axis=-1)
#
# print([model.vocab.get_token_from_index(i, 'labels') for i in tag_ids])
#
#