From b14c01f028358f1d86cbef7793ed29ab96a9393c Mon Sep 17 00:00:00 2001 From: aakbik Date: Thu, 13 Sep 2018 10:20:34 +0200 Subject: [PATCH 001/113] GH-34: update torch version --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index fafd808e20..fdda2ad34c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -torch==0.4.0 +torch==0.4.1 awscli==1.14.32 gensim==3.4.0 typing==3.6.4 diff --git a/setup.py b/setup.py index 1d25df6ac3..1a337088b6 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ packages=find_packages(exclude='test'), # same as name license='MIT', install_requires=[ - 'torch==0.4.0', + 'torch==0.4.1', 'awscli==1.14.32', 'gensim==3.4.0', 'typing==3.6.4', From a48ae758fe5f97dc41805b39d5fde03731c0bdd9 Mon Sep 17 00:00:00 2001 From: aakbik Date: Thu, 13 Sep 2018 20:07:38 +0200 Subject: [PATCH 002/113] GH-75: added span class --- flair/data.py | 201 +++++++++++++++++++++++++++------------- flair/training_utils.py | 3 +- 2 files changed, 136 insertions(+), 68 deletions(-) diff --git a/flair/data.py b/flair/data.py index 01a352649b..f655984d0f 100644 --- a/flair/data.py +++ b/flair/data.py @@ -190,6 +190,25 @@ def embedding(self): return self.get_embedding() +class Span: + def __init__(self, tokens: List[Token], tag: str = None): + self.tokens = tokens + self.tag = tag + + @property + def text(self) -> str: + return(' '.join([t.text for t in self.tokens])) + + def __str__(self) -> str: + ids = ','.join([str(t.idx) for t in self.tokens]) + return '{}-span [{}]: "{}"'.format(self.tag, ids, self.text) \ + if self.tag is not None else 'span [{}]: "{}"'.format(ids, self.text) + + def __repr__(self) -> str: + ids = ','.join([str(t.idx) for t in self.tokens]) + return '<{}-span ({}): "{}">'.format(self.tag, ids, self.text) \ + if self.tag is not None else ''.format(ids, self.text) + class Sentence: def __init__(self, text: str = None, use_tokenizer: bool = False, labels: Union[List[Label], List[str]] = None): @@ -242,43 +261,66 @@ def __init__(self, text: str = None, use_tokenizer: bool = False, labels: Union[ token = Token(word) self.add_token(token) - def _infer_space_after(self): - """ - Heuristics in case you wish to infer whitespace_after values for tokenized text. This is useful for some old NLP - tasks (such as CoNLL-03 and CoNLL-2000) that provide only tokenized data with no info of original whitespacing. - :return: - """ - last_token = None - quote_count: int = 0 - # infer whitespace after field - + def get_token(self, token_id: int) -> Token: for token in self.tokens: - if token.text == '"': - quote_count += 1 - if quote_count % 2 != 0: - token.whitespace_after = False - elif last_token is not None: - last_token.whitespace_after = False + if token.idx == token_id: + return token - if last_token is not None: + def add_token(self, token: Token): + self.tokens.append(token) - if token.text in ['.', ':', ',', ';', ')', 'n\'t', '!', '?']: - last_token.whitespace_after = False + # set token idx if not set + token.sentence = self + if token.idx is None: + token.idx = len(self.tokens) - if token.text.startswith('\''): - last_token.whitespace_after = False + def get_spans(self, tag_type: str) -> List[Span]: - if token.text in ['(']: - token.whitespace_after = False + spans: List[Span] = [] - last_token = token - return self + current_span = [] + tags = [] + for token in self: - def __getitem__(self, idx: int) -> Token: - return self.tokens[idx] + tag = token.get_tag(tag_type) - def __iter__(self): - return iter(self.tokens) + # non-set tags are OUT tags + if len(tag) < 2: tag = 'O-' + + # anything that is not a BIOES tag is a SINGLE tag + if tag[0:2] not in ['B-', 'I-', 'O-', 'E-', 'S-']: + tag = 'S-' + tag + + # anything that is not OUT is IN + in_span = False + if tag[0:2] not in ['O-']: + in_span = True + + # single and begin tags start a new span + starts_new_span = False + if tag[0:2] in ['B-', 'S-']: + starts_new_span = True + + if (starts_new_span or not in_span) and len(current_span) > 0: + tag = Counter(tags).most_common(1)[0][0] + spans.append(Span(current_span, tag)) + current_span = [] + tags = [] + + if in_span: + current_span.append(token) + tags.append(tag[2:]) + + if tag[0:2] in ['S-']: + spans.append(Span(current_span, tag[2:])) + current_span = [] + tags = [] + + if len(current_span) > 0: + tag = Counter(tags).most_common(1)[0][0] + spans.append(Span(current_span, tag)) + + return spans def add_label(self, label: Union[Label, str]): if type(label) is Label: @@ -294,33 +336,13 @@ def add_labels(self, labels: Union[List[Label], List[str]]): def get_label_names(self) -> List[str]: return [label.name for label in self.labels] - def get_token(self, token_id: int) -> Token: - for token in self.tokens: - if token.idx == token_id: - return token - - def add_token(self, token: Token): - self.tokens.append(token) - - # set token idx if not set - token.sentence = self - if token.idx is None: - token.idx = len(self.tokens) + @property + def embedding(self): + return self.get_embedding() def set_embedding(self, name: str, vector): self._embeddings[name] = vector.cpu() - def clear_embeddings(self, also_clear_word_embeddings: bool = True): - self._embeddings: Dict = {} - - if also_clear_word_embeddings: - for token in self: - token.clear_embeddings() - - def cpu_embeddings(self): - for name, vector in self._embeddings.items(): - self._embeddings[name] = vector.cpu() - def get_embedding(self) -> torch.autograd.Variable: embeddings = [] for embed in sorted(self._embeddings.keys()): @@ -332,11 +354,18 @@ def get_embedding(self) -> torch.autograd.Variable: return torch.FloatTensor() - @property - def embedding(self): - return self.get_embedding() + def clear_embeddings(self, also_clear_word_embeddings: bool = True): + self._embeddings: Dict = {} - def to_tagged_string(self) -> str: + if also_clear_word_embeddings: + for token in self: + token.clear_embeddings() + + def cpu_embeddings(self): + for name, vector in self._embeddings.items(): + self._embeddings[name] = vector.cpu() + + def to_tagged_string(self, main_tag = None) -> str: list = [] for token in self.tokens: list.append(token.text) @@ -344,6 +373,8 @@ def to_tagged_string(self) -> str: tags = [] for tag_type in token.tags.keys(): + if main_tag is not None and main_tag != tag_type: continue + if token.get_tag(tag_type) == '' or token.get_tag(tag_type) == 'O': continue tags.append(token.get_tag(tag_type)) all_tags = '<' + '/'.join(tags) + '>' @@ -351,6 +382,16 @@ def to_tagged_string(self) -> str: list.append(all_tags) return ' '.join(list) + def to_tokenized_string(self) -> str: + return ' '.join([t.text for t in self.tokens]) + + def to_plain_string(self): + plain = '' + for token in self.tokens: + plain += token.text + if token.whitespace_after: plain += ' ' + return plain.rstrip() + def convert_tag_scheme(self, tag_type: str = 'ner', target_scheme: str = 'iob'): tags: List[str] = [] @@ -368,6 +409,44 @@ def convert_tag_scheme(self, tag_type: str = 'ner', target_scheme: str = 'iob'): for index, tag in enumerate(tags): self.tokens[index].add_tag(tag_type, tag) + def _infer_space_after(self): + """ + Heuristics in case you wish to infer whitespace_after values for tokenized text. This is useful for some old NLP + tasks (such as CoNLL-03 and CoNLL-2000) that provide only tokenized data with no info of original whitespacing. + :return: + """ + last_token = None + quote_count: int = 0 + # infer whitespace after field + + for token in self.tokens: + if token.text == '"': + quote_count += 1 + if quote_count % 2 != 0: + token.whitespace_after = False + elif last_token is not None: + last_token.whitespace_after = False + + if last_token is not None: + + if token.text in ['.', ':', ',', ';', ')', 'n\'t', '!', '?']: + last_token.whitespace_after = False + + if token.text.startswith('\''): + last_token.whitespace_after = False + + if token.text in ['(']: + token.whitespace_after = False + + last_token = token + return self + + def __getitem__(self, idx: int) -> Token: + return self.tokens[idx] + + def __iter__(self): + return iter(self.tokens) + def __repr__(self): return 'Sentence: "' + ' '.join([t.text for t in self.tokens]) + '" - %d Tokens' % len(self) @@ -387,16 +466,6 @@ def __str__(self) -> str: def __len__(self) -> int: return len(self.tokens) - def to_tokenized_string(self) -> str: - return ' '.join([t.text for t in self.tokens]) - - def to_plain_string(self): - plain = '' - for token in self.tokens: - plain += token.text - if token.whitespace_after: plain += ' ' - return plain.rstrip() - class TaggedCorpus: def __init__(self, train: List[Sentence], dev: List[Sentence], test: List[Sentence]): diff --git a/flair/training_utils.py b/flair/training_utils.py index bbba465293..f4023a05c0 100644 --- a/flair/training_utils.py +++ b/flair/training_utils.py @@ -53,8 +53,7 @@ def __str__(self): self.name, self.precision(), self.recall(), self.accuracy(), self.f_score()) def print(self): - print('{0:<20}\tprecision: {1:.4f} - recall: {2:.4f} - accuracy: {3:.4f} - f1-score: {4:.4f}'.format( - self.name, self.precision(), self.recall(), self.accuracy(), self.f_score())) + print(self) def clear_embeddings(sentences: List[Sentence]): From e191ac4f3f92ba7796ea2dcff960f850ad451637 Mon Sep 17 00:00:00 2001 From: aakbik Date: Fri, 14 Sep 2018 13:51:35 +0200 Subject: [PATCH 003/113] GH-75: added native span-F1 evaluation method --- flair/data.py | 38 +++++--- flair/data_fetcher.py | 4 +- flair/trainers/sequence_tagger_trainer.py | 101 ++++++++++++---------- flair/training_utils.py | 22 ++++- tests/test_utils.py | 14 ++- 5 files changed, 116 insertions(+), 63 deletions(-) diff --git a/flair/data.py b/flair/data.py index f655984d0f..d7b9432df2 100644 --- a/flair/data.py +++ b/flair/data.py @@ -191,13 +191,17 @@ def embedding(self): class Span: + """ + This class represents one textual span consisting of Tokens. A span may have a tag. + """ + def __init__(self, tokens: List[Token], tag: str = None): self.tokens = tokens self.tag = tag @property def text(self) -> str: - return(' '.join([t.text for t in self.tokens])) + return ' '.join([t.text for t in self.tokens]) def __str__(self) -> str: ids = ','.join([str(t.idx) for t in self.tokens]) @@ -209,7 +213,12 @@ def __repr__(self) -> str: return '<{}-span ({}): "{}">'.format(self.tag, ids, self.text) \ if self.tag is not None else ''.format(ids, self.text) + class Sentence: + """ + A Sentence is a list of Tokens and is used to represent a sentence or text fragment. + """ + def __init__(self, text: str = None, use_tokenizer: bool = False, labels: Union[List[Label], List[str]] = None): super(Sentence, self).__init__() @@ -279,7 +288,9 @@ def get_spans(self, tag_type: str) -> List[Span]: spans: List[Span] = [] current_span = [] - tags = [] + import flair.training_utils + tags: flair.training_utils.ItemWeigher = flair.training_utils.ItemWeigher() + previous_tag = '' for token in self: tag = token.get_tag(tag_type) @@ -301,24 +312,24 @@ def get_spans(self, tag_type: str) -> List[Span]: if tag[0:2] in ['B-', 'S-']: starts_new_span = True + # if previous_tag[0:2] in ['S-', 'E-']: + # starts_new_span = True + if (starts_new_span or not in_span) and len(current_span) > 0: - tag = Counter(tags).most_common(1)[0][0] - spans.append(Span(current_span, tag)) + spans.append(Span(current_span, tags.best())) current_span = [] - tags = [] + tags = flair.training_utils.ItemWeigher() if in_span: current_span.append(token) - tags.append(tag[2:]) + weight = 1.1 if starts_new_span else 1.0 + tags.add(tag[2:], weight) - if tag[0:2] in ['S-']: - spans.append(Span(current_span, tag[2:])) - current_span = [] - tags = [] + # remember previous tag + previous_tag = tag if len(current_span) > 0: - tag = Counter(tags).most_common(1)[0][0] - spans.append(Span(current_span, tag)) + spans.append(Span(current_span, tags.best())) return spans @@ -365,7 +376,7 @@ def cpu_embeddings(self): for name, vector in self._embeddings.items(): self._embeddings[name] = vector.cpu() - def to_tagged_string(self, main_tag = None) -> str: + def to_tagged_string(self, main_tag=None) -> str: list = [] for token in self.tokens: list.append(token.text) @@ -577,7 +588,6 @@ def print_statistics(self): Print statistics about the class distribution (only labels of sentences are taken into account) and sentence sizes. """ - self._print_statistics_for(self.train, "TRAIN") self._print_statistics_for(self.test, "TEST") self._print_statistics_for(self.dev, "DEV") diff --git a/flair/data_fetcher.py b/flair/data_fetcher.py index e3b73c94b7..34c51f600f 100644 --- a/flair/data_fetcher.py +++ b/flair/data_fetcher.py @@ -64,7 +64,8 @@ def fetch_data(task: NLPTask) -> TaggedCorpus: train_file='eng.train', test_file='eng.testb', dev_file='eng.testa', - tag_to_biloes='ner') + tag_to_biloes='ner', + ) # the CoNLL 03 task for German has an additional lemma column if task == NLPTask.CONLL_03_GERMAN: @@ -210,6 +211,7 @@ def fetch_column_corpus( for sentence in sentences_train + sentences_test + sentences_dev: sentence: Sentence = sentence sentence.convert_tag_scheme(tag_type=tag_to_biloes, target_scheme='iobes') + # sentence.convert_tag_scheme(tag_type=tag_to_biloes, target_scheme='iob') return TaggedCorpus(sentences_train, sentences_dev, sentences_test) diff --git a/flair/trainers/sequence_tagger_trainer.py b/flair/trainers/sequence_tagger_trainer.py index 26f2ee4483..1b84f34f70 100644 --- a/flair/trainers/sequence_tagger_trainer.py +++ b/flair/trainers/sequence_tagger_trainer.py @@ -33,7 +33,7 @@ def train(self, train_with_dev: bool = False): evaluation_method = 'F1' - if self.model.tag_type in ['ner', 'np', 'srl']: evaluation_method = 'span-F1' + # if self.model.tag_type in ['ner', 'np', 'srl']: evaluation_method = 'span-F1' if self.model.tag_type in ['pos', 'upos']: evaluation_method = 'accuracy' print(evaluation_method) @@ -151,9 +151,6 @@ def train(self, def evaluate(self, evaluation: List[Sentence], out_path=None, evaluation_method: str = 'F1', embeddings_in_memory: bool = True): - tp: int = 0 - fp: int = 0 - batch_no: int = 0 mini_batch_size = 32 batches = [evaluation[x:x + mini_batch_size] for x in @@ -175,13 +172,39 @@ def evaluate(self, evaluation: List[Sentence], out_path=None, evaluation_method: # Step 3. Run our forward pass. score, tag_seq = self.model.predict_scores(sentence) + # add predicted tags + for (token, pred_id) in zip(sentence.tokens, tag_seq): + token: Token = token + # get the predicted tag + predicted_tag = self.model.tag_dictionary.get_item_for_index(pred_id) + token.add_tag('predicted', predicted_tag) + + gold_tags = [] + predicted_tags = [] + + # get spans + for tag in sentence.get_spans(self.model.tag_type): + gold_tags.append(tag.__str__()) + + for tag in sentence.get_spans('predicted'): + predicted_tags.append(tag.__str__()) + + for prediction in predicted_tags: + if prediction in gold_tags: + metric.tp() + else: + metric.fp() + + for gold in gold_tags: + if gold not in predicted_tags: + metric.fn() + # Step 5. Compute predictions predicted_id = tag_seq for (token, pred_id) in zip(sentence.tokens, predicted_id): token: Token = token # get the predicted tag predicted_tag = self.model.tag_dictionary.get_item_for_index(pred_id) - token.add_tag('predicted', predicted_tag) # get the gold tag gold_tag = token.get_tag(self.model.tag_type) @@ -189,24 +212,6 @@ def evaluate(self, evaluation: List[Sentence], out_path=None, evaluation_method: # append both to file for evaluation eval_line = token.text + ' ' + gold_tag + ' ' + predicted_tag + "\n" - # positives - if predicted_tag != '': - # true positives - if predicted_tag == gold_tag: - metric.tp() - # false positive - if predicted_tag != gold_tag: - metric.fp() - - # negatives - if predicted_tag == '': - # true negative - if predicted_tag == gold_tag: - metric.tn() - # false negative - if predicted_tag != gold_tag: - metric.fn() - lines.append(eval_line) lines.append('\n') @@ -214,33 +219,39 @@ def evaluate(self, evaluation: List[Sentence], out_path=None, evaluation_method: if not embeddings_in_memory: self.clear_embeddings_in_batch(batch) + span_f1_score = metric.f_score() * 100 + span_f1_score = "{0:.2f}".format(span_f1_score) + if out_path is not None: test_tsv = os.path.join(out_path, "test.tsv") with open(test_tsv, "w", encoding='utf-8') as outfile: outfile.write(''.join(lines)) - if evaluation_method == 'span-F1': - - # get the eval script - eval_script = cached_path('https://s3.eu-central-1.amazonaws.com/alan-nlp/resources/scripts/conll03_eval_script.pl', cache_dir='scripts') - os.chmod(eval_script, 0o777) - - eval_data = ''.join(lines) - - p = run(eval_script, stdout=PIPE, input=eval_data, encoding='utf-8') - main_result = p.stdout - print(main_result) - - main_result = main_result.split('\n')[1] - - # parse the result file - main_result = re.sub(';', ' ', main_result) - main_result = re.sub('precision', 'p', main_result) - main_result = re.sub('recall', 'r', main_result) - main_result = re.sub('accuracy', 'acc', main_result) - - f_score = float(re.findall(r'\d+\.\d+$', main_result)[0]) - return f_score, metric._fp, main_result + # if evaluation_method == 'span-F1': + # + # # get the eval script + # eval_script = cached_path('https://s3.eu-central-1.amazonaws.com/alan-nlp/resources/scripts/conll03_eval_script.pl', cache_dir='scripts') + # eval_script = 'resources/tasks/eval_script_mod' + # + # os.chmod(eval_script, 0o777) + # + # eval_data = ''.join(lines) + # + # p = run(eval_script, stdout=PIPE, input=eval_data, encoding='utf-8') + # main_result = p.stdout + # print(span_f1_score) + # print(main_result) + # + # main_result = main_result.split('\n')[1] + # + # # parse the result file + # main_result = re.sub(';', ' ', main_result) + # main_result = re.sub('precision', 'p', main_result) + # main_result = re.sub('recall', 'r', main_result) + # main_result = re.sub('accuracy', 'acc', main_result) + # + # f_score = float(re.findall(r'\d+\.\d+$', main_result)[0]) + # return f_score, metric._fp, main_result if evaluation_method == 'accuracy': score = metric.accuracy() diff --git a/flair/training_utils.py b/flair/training_utils.py index f4023a05c0..6928aed202 100644 --- a/flair/training_utils.py +++ b/flair/training_utils.py @@ -1,4 +1,4 @@ -from typing import List +from typing import List, Dict import os import numpy as np @@ -56,6 +56,24 @@ def print(self): print(self) +class ItemWeigher(object): + """ + Simple helper class for counting (optionally weighed) strings. + """ + def __init__(self): + self.dict: Dict[str, float] = {} + + def add(self, item:str, weight:float = 1.0): + new_val: float = weight if item not in self.dict else weight + self.dict[item] + self.dict[item] = new_val + + def best(self) -> str: + sorted_by_value = sorted(self.dict.items(), key=lambda kv: kv[1]) + sorted_by_value.reverse() + # print(sorted_by_value) + return sorted_by_value[0][0] + + def clear_embeddings(sentences: List[Sentence]): """ Clears the embeddings from all given sentences. @@ -152,4 +170,4 @@ def calculate_class_metrics(y_true: List[List[int]], y_pred: List[List[int]], la metrics.append(metric) - return metrics \ No newline at end of file + return metrics diff --git a/tests/test_utils.py b/tests/test_utils.py index 8788aa821f..77b8f5cfef 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,7 +1,7 @@ import pytest from flair.data import Dictionary -from flair.training_utils import calculate_micro_avg_metric, calculate_class_metrics +from flair.training_utils import calculate_micro_avg_metric, calculate_class_metrics, ItemWeigher @pytest.fixture @@ -52,3 +52,15 @@ def test_calculate_class_metrics(): assert(1 == metrics_dict['class-3']._tn) assert(1 == metrics_dict['class-3']._fn) + +def test_item_weigher(): + weigher = ItemWeigher() + weigher.add('one element') + weigher.add('one element') + weigher.add('another element') + + assert('one element' == weigher.best()) + + weigher.add('another element', weight=1.1) + assert('another element' == weigher.best()) + From 38a033372b27a4bbff487a9341f6cef67ccd880c Mon Sep 17 00:00:00 2001 From: aakbik Date: Fri, 14 Sep 2018 17:33:02 +0200 Subject: [PATCH 004/113] GH-97: test for span getters --- flair/data.py | 4 +-- tests/test_data.py | 77 ++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 77 insertions(+), 4 deletions(-) diff --git a/flair/data.py b/flair/data.py index d7b9432df2..e804de1cc8 100644 --- a/flair/data.py +++ b/flair/data.py @@ -312,8 +312,8 @@ def get_spans(self, tag_type: str) -> List[Span]: if tag[0:2] in ['B-', 'S-']: starts_new_span = True - # if previous_tag[0:2] in ['S-', 'E-']: - # starts_new_span = True + if previous_tag[0:2] in ['S-'] and previous_tag[2:] != tag[2:] and in_span: + starts_new_span = True if (starts_new_span or not in_span) and len(current_span) > 0: spans.append(Span(current_span, tags.best())) diff --git a/tests/test_data.py b/tests/test_data.py index a63bf0b68b..bb648801d7 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -1,8 +1,9 @@ import os import pytest +from typing import List -from flair.data import Sentence, Label, Token, Dictionary, TaggedCorpus +from flair.data import Sentence, Label, Token, Dictionary, TaggedCorpus, Span from flair.data_fetcher import NLPTaskDataFetcher, NLPTask @@ -330,7 +331,6 @@ def test_tagged_corpus_statistics_multi_label(): assert (4 == tokens_in_sentences[2]) - def test_tagged_corpus_downsample(): sentence = Sentence('I love Berlin.', labels=[Label('class_1')], use_tokenizer=True) @@ -342,3 +342,76 @@ def test_tagged_corpus_downsample(): corpus.downsample(percentage=0.3, only_downsample_train=True) assert (3 == len(corpus.train)) + + +def test_spans(): + sentence = Sentence('Zalando Research is located in Berlin .') + + # bioes tags + sentence[0].add_tag('ner', 'B-ORG') + sentence[1].add_tag('ner', 'E-ORG') + sentence[5].add_tag('ner', 'S-LOC') + + spans: List[Span] = sentence.get_spans('ner') + + assert (2 == len(spans)) + assert ('Zalando Research' == spans[0].text) + assert ('ORG' == spans[0].tag) + assert ('Berlin' == spans[1].text) + assert ('LOC' == spans[1].tag) + + # bio tags + sentence[0].add_tag('ner', 'B-ORG') + sentence[1].add_tag('ner', 'I-ORG') + sentence[5].add_tag('ner', 'B-LOC') + + spans: List[Span] = sentence.get_spans('ner') + + assert ('Zalando Research' == spans[0].text) + assert ('ORG' == spans[0].tag) + assert ('Berlin' == spans[1].text) + assert ('LOC' == spans[1].tag) + + # broken tags + sentence[0].add_tag('ner', 'I-ORG') + sentence[1].add_tag('ner', 'E-ORG') + sentence[5].add_tag('ner', 'I-LOC') + + spans: List[Span] = sentence.get_spans('ner') + + assert ('Zalando Research' == spans[0].text) + assert ('ORG' == spans[0].tag) + assert ('Berlin' == spans[1].text) + assert ('LOC' == spans[1].tag) + + # all tags + sentence[0].add_tag('ner', 'I-ORG') + sentence[1].add_tag('ner', 'E-ORG') + sentence[2].add_tag('ner', 'aux') + sentence[3].add_tag('ner', 'verb') + sentence[4].add_tag('ner', 'preposition') + sentence[5].add_tag('ner', 'I-LOC') + + spans: List[Span] = sentence.get_spans('ner') + assert (5 == len(spans)) + assert ('Zalando Research' == spans[0].text) + assert ('ORG' == spans[0].tag) + assert ('Berlin' == spans[4].text) + assert ('LOC' == spans[4].tag) + + # all weird tags + sentence[0].add_tag('ner', 'I-ORG') + sentence[1].add_tag('ner', 'S-LOC') + sentence[2].add_tag('ner', 'aux') + sentence[3].add_tag('ner', 'B-relation') + sentence[4].add_tag('ner', 'E-preposition') + sentence[5].add_tag('ner', 'S-LOC') + + spans: List[Span] = sentence.get_spans('ner') + assert (5 == len(spans)) + assert ('Zalando' == spans[0].text) + assert ('ORG' == spans[0].tag) + assert ('Research' == spans[1].text) + assert ('LOC' == spans[1].tag) + assert ('located in' == spans[3].text) + assert ('relation' == spans[3].tag) \ No newline at end of file From b567e8a6c9528ab160ac055cb7f032ee5802cd2d Mon Sep 17 00:00:00 2001 From: aakbik Date: Fri, 14 Sep 2018 17:44:42 +0200 Subject: [PATCH 005/113] GH-97: comment out unused code --- flair/data_fetcher.py | 1 - flair/trainers/sequence_tagger_trainer.py | 1 - 2 files changed, 2 deletions(-) diff --git a/flair/data_fetcher.py b/flair/data_fetcher.py index 34c51f600f..fb91563179 100644 --- a/flair/data_fetcher.py +++ b/flair/data_fetcher.py @@ -211,7 +211,6 @@ def fetch_column_corpus( for sentence in sentences_train + sentences_test + sentences_dev: sentence: Sentence = sentence sentence.convert_tag_scheme(tag_type=tag_to_biloes, target_scheme='iobes') - # sentence.convert_tag_scheme(tag_type=tag_to_biloes, target_scheme='iob') return TaggedCorpus(sentences_train, sentences_dev, sentences_test) diff --git a/flair/trainers/sequence_tagger_trainer.py b/flair/trainers/sequence_tagger_trainer.py index 1b84f34f70..1a360ed641 100644 --- a/flair/trainers/sequence_tagger_trainer.py +++ b/flair/trainers/sequence_tagger_trainer.py @@ -33,7 +33,6 @@ def train(self, train_with_dev: bool = False): evaluation_method = 'F1' - # if self.model.tag_type in ['ner', 'np', 'srl']: evaluation_method = 'span-F1' if self.model.tag_type in ['pos', 'upos']: evaluation_method = 'accuracy' print(evaluation_method) From aa3a1cd035154a7537fd63c988fe18a3158ff9cf Mon Sep 17 00:00:00 2001 From: aakbik Date: Fri, 14 Sep 2018 17:46:56 +0200 Subject: [PATCH 006/113] GH-97: comment out unused code --- flair/trainers/sequence_tagger_trainer.py | 29 ----------------------- 1 file changed, 29 deletions(-) diff --git a/flair/trainers/sequence_tagger_trainer.py b/flair/trainers/sequence_tagger_trainer.py index 1a360ed641..72849a6fe8 100644 --- a/flair/trainers/sequence_tagger_trainer.py +++ b/flair/trainers/sequence_tagger_trainer.py @@ -218,40 +218,11 @@ def evaluate(self, evaluation: List[Sentence], out_path=None, evaluation_method: if not embeddings_in_memory: self.clear_embeddings_in_batch(batch) - span_f1_score = metric.f_score() * 100 - span_f1_score = "{0:.2f}".format(span_f1_score) - if out_path is not None: test_tsv = os.path.join(out_path, "test.tsv") with open(test_tsv, "w", encoding='utf-8') as outfile: outfile.write(''.join(lines)) - # if evaluation_method == 'span-F1': - # - # # get the eval script - # eval_script = cached_path('https://s3.eu-central-1.amazonaws.com/alan-nlp/resources/scripts/conll03_eval_script.pl', cache_dir='scripts') - # eval_script = 'resources/tasks/eval_script_mod' - # - # os.chmod(eval_script, 0o777) - # - # eval_data = ''.join(lines) - # - # p = run(eval_script, stdout=PIPE, input=eval_data, encoding='utf-8') - # main_result = p.stdout - # print(span_f1_score) - # print(main_result) - # - # main_result = main_result.split('\n')[1] - # - # # parse the result file - # main_result = re.sub(';', ' ', main_result) - # main_result = re.sub('precision', 'p', main_result) - # main_result = re.sub('recall', 'r', main_result) - # main_result = re.sub('accuracy', 'acc', main_result) - # - # f_score = float(re.findall(r'\d+\.\d+$', main_result)[0]) - # return f_score, metric._fp, main_result - if evaluation_method == 'accuracy': score = metric.accuracy() return score, metric._fp, str(score) From b5372984365eaa3dd2343239e09ac5d09bf0b538 Mon Sep 17 00:00:00 2001 From: aakbik Date: Mon, 17 Sep 2018 11:33:25 +0200 Subject: [PATCH 007/113] GH-97: replace ItemWeigher with defaultdict --- flair/data.py | 13 ++++----- flair/trainers/sequence_tagger_trainer.py | 33 +++++++---------------- flair/training_utils.py | 18 ------------- tests/test_utils.py | 17 ++---------- 4 files changed, 18 insertions(+), 63 deletions(-) diff --git a/flair/data.py b/flair/data.py index e804de1cc8..96ac2a79c4 100644 --- a/flair/data.py +++ b/flair/data.py @@ -288,8 +288,9 @@ def get_spans(self, tag_type: str) -> List[Span]: spans: List[Span] = [] current_span = [] - import flair.training_utils - tags: flair.training_utils.ItemWeigher = flair.training_utils.ItemWeigher() + + tags = defaultdict(lambda: 0.0) + previous_tag = '' for token in self: @@ -316,20 +317,20 @@ def get_spans(self, tag_type: str) -> List[Span]: starts_new_span = True if (starts_new_span or not in_span) and len(current_span) > 0: - spans.append(Span(current_span, tags.best())) + spans.append(Span(current_span, sorted(tags.items(), key=lambda k_v: k_v[1], reverse=True)[0][0])) current_span = [] - tags = flair.training_utils.ItemWeigher() + tags = defaultdict(lambda: 0.0) if in_span: current_span.append(token) weight = 1.1 if starts_new_span else 1.0 - tags.add(tag[2:], weight) + tags[tag[2:]] += weight # remember previous tag previous_tag = tag if len(current_span) > 0: - spans.append(Span(current_span, tags.best())) + spans.append(Span(current_span, sorted(tags.items(), key=lambda k_v: k_v[1], reverse=True)[0][0])) return spans diff --git a/flair/trainers/sequence_tagger_trainer.py b/flair/trainers/sequence_tagger_trainer.py index 72849a6fe8..7fa766c3a7 100644 --- a/flair/trainers/sequence_tagger_trainer.py +++ b/flair/trainers/sequence_tagger_trainer.py @@ -178,16 +178,18 @@ def evaluate(self, evaluation: List[Sentence], out_path=None, evaluation_method: predicted_tag = self.model.tag_dictionary.get_item_for_index(pred_id) token.add_tag('predicted', predicted_tag) - gold_tags = [] - predicted_tags = [] + # append both to file for evaluation + eval_line = token.text + ' ' + token.get_tag(self.model.tag_type) + ' ' + predicted_tag + "\n" + lines.append(eval_line) + lines.append('\n') - # get spans - for tag in sentence.get_spans(self.model.tag_type): - gold_tags.append(tag.__str__()) + # make list of gold tags + gold_tags = [str(tag) for tag in sentence.get_spans(self.model.tag_type)] - for tag in sentence.get_spans('predicted'): - predicted_tags.append(tag.__str__()) + # make list of predicted tags + predicted_tags = [str(tag) for tag in sentence.get_spans('predicted')] + # check for true positives, false positives and false negatives for prediction in predicted_tags: if prediction in gold_tags: metric.tp() @@ -198,23 +200,6 @@ def evaluate(self, evaluation: List[Sentence], out_path=None, evaluation_method: if gold not in predicted_tags: metric.fn() - # Step 5. Compute predictions - predicted_id = tag_seq - for (token, pred_id) in zip(sentence.tokens, predicted_id): - token: Token = token - # get the predicted tag - predicted_tag = self.model.tag_dictionary.get_item_for_index(pred_id) - - # get the gold tag - gold_tag = token.get_tag(self.model.tag_type) - - # append both to file for evaluation - eval_line = token.text + ' ' + gold_tag + ' ' + predicted_tag + "\n" - - lines.append(eval_line) - - lines.append('\n') - if not embeddings_in_memory: self.clear_embeddings_in_batch(batch) diff --git a/flair/training_utils.py b/flair/training_utils.py index 6928aed202..fa8226b692 100644 --- a/flair/training_utils.py +++ b/flair/training_utils.py @@ -56,24 +56,6 @@ def print(self): print(self) -class ItemWeigher(object): - """ - Simple helper class for counting (optionally weighed) strings. - """ - def __init__(self): - self.dict: Dict[str, float] = {} - - def add(self, item:str, weight:float = 1.0): - new_val: float = weight if item not in self.dict else weight + self.dict[item] - self.dict[item] = new_val - - def best(self) -> str: - sorted_by_value = sorted(self.dict.items(), key=lambda kv: kv[1]) - sorted_by_value.reverse() - # print(sorted_by_value) - return sorted_by_value[0][0] - - def clear_embeddings(sentences: List[Sentence]): """ Clears the embeddings from all given sentences. diff --git a/tests/test_utils.py b/tests/test_utils.py index 77b8f5cfef..46c53c4bdb 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,7 +1,7 @@ import pytest from flair.data import Dictionary -from flair.training_utils import calculate_micro_avg_metric, calculate_class_metrics, ItemWeigher +from flair.training_utils import calculate_micro_avg_metric, calculate_class_metrics @pytest.fixture @@ -50,17 +50,4 @@ def test_calculate_class_metrics(): assert(1 == metrics_dict['class-3']._tp) assert(0 == metrics_dict['class-3']._fp) assert(1 == metrics_dict['class-3']._tn) - assert(1 == metrics_dict['class-3']._fn) - - -def test_item_weigher(): - weigher = ItemWeigher() - weigher.add('one element') - weigher.add('one element') - weigher.add('another element') - - assert('one element' == weigher.best()) - - weigher.add('another element', weight=1.1) - assert('another element' == weigher.best()) - + assert(1 == metrics_dict['class-3']._fn) \ No newline at end of file From 27d421214996d804035503c4f8d7902008797aad Mon Sep 17 00:00:00 2001 From: aakbik Date: Mon, 17 Sep 2018 11:35:11 +0200 Subject: [PATCH 008/113] GH-75: inferspaceafter() is not public method --- flair/data.py | 2 +- flair/data_fetcher.py | 4 ++-- tests/test_data.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/flair/data.py b/flair/data.py index 96ac2a79c4..0fff309f54 100644 --- a/flair/data.py +++ b/flair/data.py @@ -421,7 +421,7 @@ def convert_tag_scheme(self, tag_type: str = 'ner', target_scheme: str = 'iob'): for index, tag in enumerate(tags): self.tokens[index].add_tag(tag_type, tag) - def _infer_space_after(self): + def infer_space_after(self): """ Heuristics in case you wish to infer whitespace_after values for tokenized text. This is useful for some old NLP tasks (such as CoNLL-03 and CoNLL-2000) that provide only tokenized data with no info of original whitespacing. diff --git a/flair/data_fetcher.py b/flair/data_fetcher.py index fb91563179..d9d735b586 100644 --- a/flair/data_fetcher.py +++ b/flair/data_fetcher.py @@ -246,7 +246,7 @@ def read_column_data(path_to_column_file: str, if line == '': if len(sentence) > 0: - sentence._infer_space_after() + sentence.infer_space_after() sentences.append(sentence) sentence: Sentence = Sentence() @@ -260,7 +260,7 @@ def read_column_data(path_to_column_file: str, sentence.add_token(token) if len(sentence.tokens) > 0: - sentence._infer_space_after() + sentence.infer_space_after() sentences.append(sentence) return sentences diff --git a/tests/test_data.py b/tests/test_data.py index bb648801d7..310898d751 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -72,13 +72,13 @@ def test_sentence_infer_tokenization(): sentence.add_token(Token('"')) sentence.add_token(Token('abc')) sentence.add_token(Token('"')) - sentence._infer_space_after() + sentence.infer_space_after() assert ('xyz " abc "' == sentence.to_tokenized_string()) assert ('xyz "abc"' == sentence.to_plain_string()) sentence: Sentence = Sentence('xyz " abc "') - sentence._infer_space_after() + sentence.infer_space_after() assert ('xyz " abc "' == sentence.to_tokenized_string()) assert ('xyz "abc"' == sentence.to_plain_string()) From cb3ae5d794600359eeb670b988731b854b21a211 Mon Sep 17 00:00:00 2001 From: aakbik Date: Mon, 17 Sep 2018 17:40:20 +0200 Subject: [PATCH 009/113] GH-48: added word dropout | moved dropouts into new flair.nn module --- flair/models/sequence_tagger_model.py | 53 ++++++++++++--------------- flair/nn.py | 37 +++++++++++++++++++ 2 files changed, 60 insertions(+), 30 deletions(-) create mode 100644 flair/nn.py diff --git a/flair/models/sequence_tagger_model.py b/flair/models/sequence_tagger_model.py index ef461cb917..2b0517efc4 100644 --- a/flair/models/sequence_tagger_model.py +++ b/flair/models/sequence_tagger_model.py @@ -1,7 +1,8 @@ import warnings import torch.autograd as autograd -import torch.nn as nn +import torch.nn +import flair.nn import torch import numpy as np @@ -58,7 +59,7 @@ def pad_tensors(tensor_list, type_=torch.FloatTensor): return template, lens_ -class SequenceTagger(nn.Module): +class SequenceTagger(torch.nn.Module): def __init__(self, hidden_size: int, @@ -67,7 +68,8 @@ def __init__(self, tag_type: str, use_crf: bool = True, use_rnn: bool = True, - rnn_layers: int = 1 + rnn_layers: int = 1, + use_word_dropout: bool = False, ): super(SequenceTagger, self).__init__() @@ -90,40 +92,42 @@ def __init__(self, self.nlayers: int = rnn_layers self.hidden_word = None - # self.dropout = nn.Dropout(0.5) - self.dropout: nn.Module = LockedDropout(0.5) + # dropouts + self.dropout: torch.nn.Module = flair.nn.LockedDropout(0.5) + + self.use_word_dropout: bool = use_word_dropout + if self.use_word_dropout: + self.word_dropout = flair.nn.WordDropout(0.05) rnn_input_dim: int = self.embeddings.embedding_length self.relearn_embeddings: bool = True if self.relearn_embeddings: - self.embedding2nn = nn.Linear(rnn_input_dim, rnn_input_dim) + self.embedding2nn = torch.nn.Linear(rnn_input_dim, rnn_input_dim) # bidirectional LSTM on top of embedding layer self.rnn_type = 'LSTM' if self.rnn_type in ['LSTM', 'GRU']: if self.nlayers == 1: - self.rnn = getattr(nn, self.rnn_type)(rnn_input_dim, hidden_size, + self.rnn = getattr(torch.nn, self.rnn_type)(rnn_input_dim, hidden_size, num_layers=self.nlayers, bidirectional=True) else: - self.rnn = getattr(nn, self.rnn_type)(rnn_input_dim, hidden_size, + self.rnn = getattr(torch.nn, self.rnn_type)(rnn_input_dim, hidden_size, num_layers=self.nlayers, dropout=0.5, bidirectional=True) - self.nonlinearity = nn.Tanh() - # final linear map to tag space if self.use_rnn: - self.linear = nn.Linear(hidden_size * 2, len(tag_dictionary)) + self.linear = torch.nn.Linear(hidden_size * 2, len(tag_dictionary)) else: - self.linear = nn.Linear(self.embeddings.embedding_length, len(tag_dictionary)) + self.linear = torch.nn.Linear(self.embeddings.embedding_length, len(tag_dictionary)) if self.use_crf: - self.transitions = nn.Parameter( + self.transitions = torch.nn.Parameter( torch.randn(self.tagset_size, self.tagset_size)) self.transitions.data[self.tag_dictionary.get_idx_for_item(START_TAG), :] = -10000 self.transitions.data[:, self.tag_dictionary.get_idx_for_item(STOP_TAG)] = -10000 @@ -220,6 +224,10 @@ def forward(self, sentences: List[Sentence]) -> Tuple[List, List]: # -------------------------------------------------------------------- sentence_tensor = self.dropout(sentence_tensor) + # use word dropout if set + if self.use_word_dropout: + sentence_tensor = self.word_dropout(sentence_tensor) + if self.relearn_embeddings: sentence_tensor = self.embedding2nn(sentence_tensor) @@ -362,7 +370,7 @@ def neg_log_likelihood(self, sentences: List[Sentence], tag_type: str): tag_tensor = autograd.Variable(torch.cuda.LongTensor(sentence_tags)) else: tag_tensor = autograd.Variable(torch.LongTensor(sentence_tags)) - score += nn.functional.cross_entropy(sentence_feats, tag_tensor) + score += torch.nn.functional.cross_entropy(sentence_feats, tag_tensor) return score @@ -572,19 +580,4 @@ def load(model: str): if model_file is not None: tagger: SequenceTagger = SequenceTagger.load_from_file(model_file) - return tagger - - -class LockedDropout(nn.Module): - def __init__(self, dropout_rate=0.5): - super(LockedDropout, self).__init__() - self.dropout_rate = dropout_rate - - def forward(self, x): - if not self.training or not self.dropout_rate: - return x - - m = x.data.new(1, x.size(1), x.size(2)).bernoulli_(1 - self.dropout_rate) - mask = torch.autograd.Variable(m, requires_grad=False) / (1 - self.dropout_rate) - mask = mask.expand_as(x) - return mask * x + return tagger \ No newline at end of file diff --git a/flair/nn.py b/flair/nn.py new file mode 100644 index 0000000000..d7abbdac6e --- /dev/null +++ b/flair/nn.py @@ -0,0 +1,37 @@ +import torch.nn + + +class LockedDropout(torch.nn.Module): + """ + Implementation of locked (or variational) dropout. Randomly drops out entire parameters in embedding space. + """ + def __init__(self, dropout_rate=0.5): + super(LockedDropout, self).__init__() + self.dropout_rate = dropout_rate + + def forward(self, x): + if not self.training or not self.dropout_rate: + return x + + m = x.data.new(1, x.size(1), x.size(2)).bernoulli_(1 - self.dropout_rate) + mask = torch.autograd.Variable(m, requires_grad=False) / (1 - self.dropout_rate) + mask = mask.expand_as(x) + return mask * x + + +class WordDropout(torch.nn.Module): + """ + Implementation of word dropout. Randomly drops out entire words (or characters) in embedding space. + """ + def __init__(self, dropout_rate=0.05): + super(WordDropout, self).__init__() + self.dropout_rate = dropout_rate + + def forward(self, x): + if not self.training or not self.dropout_rate: + return x + + m = x.data.new(x.size(0), 1, 1).bernoulli_(1 - self.dropout_rate) + mask = torch.autograd.Variable(m, requires_grad=False) + mask = mask.expand_as(x) + return mask * x \ No newline at end of file From 2bba00f1ae3ca98438573ebdd1eecd9f88eced7a Mon Sep 17 00:00:00 2001 From: tabergma Date: Thu, 13 Sep 2018 13:49:32 +0200 Subject: [PATCH 010/113] GH-108: Define loss function at the beginning. --- flair/models/text_classification_model.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/flair/models/text_classification_model.py b/flair/models/text_classification_model.py index 5c83ec5095..5693c25bfe 100644 --- a/flair/models/text_classification_model.py +++ b/flair/models/text_classification_model.py @@ -34,6 +34,11 @@ def __init__(self, self._init_weights() + if multi_label: + self.loss_function = nn.BCELoss() + else: + self.loss_function = nn.CrossEntropyLoss() + # auto-spawn on GPU if available if torch.cuda.is_available(): self.cuda() @@ -151,7 +156,7 @@ def _get_multi_label(self, label_scores) -> List[Label]: for idx, conf in enumerate(results): if conf > 0.5: label = self.label_dictionary.get_item_for_index(idx) - labels.append(Label(label, conf.item())) + labels.append(Label(label, conf.item(), idx)) return labels @@ -159,16 +164,14 @@ def _get_single_label(self, label_scores) -> List[Label]: conf, idx = torch.max(label_scores, 0) label = self.label_dictionary.get_item_for_index(idx.item()) - return [Label(label, conf.item())] + return [Label(label, conf.item(), idx)] def _calculate_multi_label_loss(self, label_scores, sentences: List[Sentence]) -> float: - loss_function = nn.BCELoss() sigmoid = nn.Sigmoid() - return loss_function(sigmoid(label_scores), self._labels_to_one_hot(sentences)) + return self.loss_function(sigmoid(label_scores), self._labels_to_one_hot(sentences)) def _calculate_single_label_loss(self, label_scores, sentences: List[Sentence]) -> float: - loss_function = nn.CrossEntropyLoss() - return loss_function(label_scores, self._labels_to_indices(sentences)) + return self.loss_function(label_scores, self._labels_to_indices(sentences)) def _labels_to_one_hot(self, sentences: List[Sentence]): label_list = [sentence.get_label_names() for sentence in sentences] From ee8a88008dedecc28c8d518e8d4e2865014d2294 Mon Sep 17 00:00:00 2001 From: tabergma Date: Thu, 13 Sep 2018 13:50:42 +0200 Subject: [PATCH 011/113] GH-108: Refactor labels to one hot. --- flair/training_utils.py | 13 +------------ tests/test_utils.py | 17 +++++++++++++++-- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/flair/training_utils.py b/flair/training_utils.py index fa8226b692..c3aaf58a09 100644 --- a/flair/training_utils.py +++ b/flair/training_utils.py @@ -86,18 +86,7 @@ def convert_labels_to_one_hot(label_list: List[List[str]], label_dict: Dictionar :param label_dict: label dictionary :return: converted label list """ - converted_label_list = [] - - for labels in label_list: - arr = np.empty(len(label_dict)) - arr.fill(0) - - for label in labels: - arr[label_dict.get_idx_for_item(label)] = 1 - - converted_label_list.append(arr.tolist()) - - return converted_label_list + return [[1 if l in labels else 0 for l in label_dict.get_items()] for labels in label_list] def calculate_micro_avg_metric(y_true: List[List[int]], y_pred: List[List[int]], labels: Dictionary) -> Metric: diff --git a/tests/test_utils.py b/tests/test_utils.py index 46c53c4bdb..7dad1ed69f 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,7 +1,7 @@ import pytest from flair.data import Dictionary -from flair.training_utils import calculate_micro_avg_metric, calculate_class_metrics +from flair.training_utils import calculate_micro_avg_metric, calculate_class_metrics, convert_labels_to_one_hot @pytest.fixture @@ -50,4 +50,17 @@ def test_calculate_class_metrics(): assert(1 == metrics_dict['class-3']._tp) assert(0 == metrics_dict['class-3']._fp) assert(1 == metrics_dict['class-3']._tn) - assert(1 == metrics_dict['class-3']._fn) \ No newline at end of file + assert(1 == metrics_dict['class-3']._fn) + + +def test_convert_labels_to_one_hot(): + label_dict = Dictionary(add_unk=False) + label_dict.add_item('class-1') + label_dict.add_item('class-2') + label_dict.add_item('class-3') + + one_hot = convert_labels_to_one_hot([['class-2']], label_dict) + + assert(one_hot[0][0] == 0) + assert(one_hot[0][1] == 1) + assert(one_hot[0][2] == 0) From 515a49cf184687408a268c91e43919262359b87f Mon Sep 17 00:00:00 2001 From: tabergma Date: Thu, 13 Sep 2018 15:14:19 +0200 Subject: [PATCH 012/113] GH-108: Refactor trainer. Add tracemalloc. --- flair/data.py | 2 +- flair/models/text_classification_model.py | 4 +- flair/trainers/text_classification_trainer.py | 107 +++++++++++------- 3 files changed, 71 insertions(+), 42 deletions(-) diff --git a/flair/data.py b/flair/data.py index 0fff309f54..4ca3953402 100644 --- a/flair/data.py +++ b/flair/data.py @@ -122,7 +122,7 @@ def confidence(self, confidence): if 0.0 <= confidence <= 1.0: self._confidence = confidence else: - self._confidence = 0.0 + self._confidence = 1.0 def __str__(self): return "{} ({})".format(self._name, self._confidence) diff --git a/flair/models/text_classification_model.py b/flair/models/text_classification_model.py index 5693c25bfe..8018e7e2a5 100644 --- a/flair/models/text_classification_model.py +++ b/flair/models/text_classification_model.py @@ -156,7 +156,7 @@ def _get_multi_label(self, label_scores) -> List[Label]: for idx, conf in enumerate(results): if conf > 0.5: label = self.label_dictionary.get_item_for_index(idx) - labels.append(Label(label, conf.item(), idx)) + labels.append(Label(label, conf.item())) return labels @@ -164,7 +164,7 @@ def _get_single_label(self, label_scores) -> List[Label]: conf, idx = torch.max(label_scores, 0) label = self.label_dictionary.get_item_for_index(idx.item()) - return [Label(label, conf.item(), idx)] + return [Label(label, conf.item())] def _calculate_multi_label_loss(self, label_scores, sentences: List[Sentence]) -> float: sigmoid = nn.Sigmoid() diff --git a/flair/trainers/text_classification_trainer.py b/flair/trainers/text_classification_trainer.py index 67b308248d..a2d490416e 100644 --- a/flair/trainers/text_classification_trainer.py +++ b/flair/trainers/text_classification_trainer.py @@ -10,6 +10,9 @@ from flair.models.text_classification_model import TextClassifier from flair.training_utils import convert_labels_to_one_hot, calculate_micro_avg_metric, init_output_file, clear_embeddings, \ calculate_class_metrics +import tracemalloc + +MICRO_AVG_METRIC = 'MICRO_AVG' class TextClassifierTrainer: @@ -22,6 +25,7 @@ def __init__(self, model: TextClassifier, corpus: TaggedCorpus, label_dict: Dict self.corpus: TaggedCorpus = corpus self.label_dict: Dictionary = label_dict self.test_mode: bool = test_mode + self.snapshot = None def train(self, base_path: str, @@ -44,6 +48,8 @@ def train(self, :param train_with_dev: boolean value indicating, if the dev data set should be used for training or not """ + tracemalloc.start() + loss_txt = init_output_file(base_path, 'loss.txt') with open(loss_txt, 'a') as f: f.write('EPOCH\tITERATION\tDEV_LOSS\tTRAIN_LOSS\tDEV_F_SCORE\tTRAIN_F_SCORE\tDEV_ACC\tTRAIN_ACC\n') @@ -69,17 +75,19 @@ def train(self, for epoch in range(max_epochs): print('-' * 100) + if not self.test_mode: random.shuffle(train_data) - batches = [train_data[x:x + mini_batch_size] for x in range(0, len(train_data), mini_batch_size)] + self.model.train() + + batches = [self.corpus.train[x:x + mini_batch_size] for x in + range(0, len(self.corpus.train), mini_batch_size)] current_loss: float = 0 seen_sentences = 0 modulo = max(1, int(len(batches) / 10)) - self.model.train() - for batch_no, batch in enumerate(batches): scores = self.model.forward(batch) loss = self.model.calculate_loss(scores, batch) @@ -98,52 +106,43 @@ def train(self, if batch_no % modulo == 0: print("epoch {0} - iter {1}/{2} - loss {3:.8f}".format(epoch + 1, batch_no, len(batches), current_loss / seen_sentences)) - - iteration = epoch * len(batches) + batch_no - self._extract_weigths(iteration, weights_index, weights_txt) + self.trace_print() + # iteration = epoch * len(batches) + batch_no + # self._extract_weights(iteration, weights_index, weights_txt) current_loss /= len(train_data) - # IMPORTANT: Switch to eval mode self.model.eval() print('-' * 100) - train_metrics, train_loss = self.evaluate(self.corpus.train, mini_batch_size=mini_batch_size, - embeddings_in_memory=embeddings_in_memory) - train_f_score = train_metrics['MICRO_AVG'].f_score() - train_acc = train_metrics['MICRO_AVG'].accuracy() - print("{0:<7} epoch {1} - loss {2:.8f} - f-score {3:.4f} - acc {4:.4f}".format( - 'TRAIN:', epoch, train_loss, train_f_score, train_acc)) + + train_acc, train_f_score, train_loss = self._calculate_evaluation_results_for( + 'TRAIN', self.corpus.train, embeddings_in_memory, epoch, mini_batch_size) dev_f_score = dev_acc = dev_loss = 0 if not train_with_dev: - dev_metrics, dev_loss = self.evaluate(self.corpus.dev, mini_batch_size=mini_batch_size, - embeddings_in_memory=embeddings_in_memory) - dev_f_score = dev_metrics['MICRO_AVG'].f_score() - dev_acc = dev_metrics['MICRO_AVG'].accuracy() - print("{0:<7} epoch {1} - loss {2:.8f} - f-score {3:.4f} - acc {4:.4f}".format( - 'DEV:', epoch, dev_loss, dev_f_score, dev_acc)) + dev_acc, dev_f_score, dev_loss = self._calculate_evaluation_results_for( + 'DEV', self.corpus.dev, embeddings_in_memory, epoch, mini_batch_size) with open(loss_txt, 'a') as f: f.write('{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\n'.format( epoch, epoch * len(batches), dev_loss, train_loss, dev_f_score, train_f_score, dev_acc, train_acc)) - # IMPORTANT: Switch back to train mode self.model.train() - # anneal against train loss if training with dev, otherwise anneal against dev score - scheduler.step(current_loss) if train_with_dev else scheduler.step(dev_f_score) + # # anneal against train loss if training with dev, otherwise anneal against dev score + # scheduler.step(current_loss) if train_with_dev else scheduler.step(dev_f_score) - is_best_model_so_far: bool = False - current_score = dev_f_score if not train_with_dev else train_f_score - - if current_score > best_score: - best_score = current_score - is_best_model_so_far = True - - if is_best_model_so_far: - if save_model: - self.model.save(base_path + "/model.pt") + # is_best_model_so_far: bool = False + # current_score = dev_f_score if not train_with_dev else train_f_score + # + # if current_score > best_score: + # best_score = current_score + # is_best_model_so_far = True + # + # if is_best_model_so_far: + # if save_model: + # self.model.save(base_path + "/model.pt") self.model.save(base_path + "/final-model.pt") @@ -151,14 +150,16 @@ def train(self, self.model = TextClassifier.load_from_file(base_path + "/model.pt") print('-' * 100) - print('testing...') + print('Testing using best model ...') + self.model.eval() test_metrics, test_loss = self.evaluate( self.corpus.test, mini_batch_size=mini_batch_size, eval_class_metrics=True, embeddings_in_memory=embeddings_in_memory) for metric in test_metrics.values(): metric.print() + self.model.train() print('-' * 100) @@ -171,6 +172,33 @@ def train(self, model_save_file.close() print('done') + def _calculate_evaluation_results_for(self, dataset_name, dataset, embeddings_in_memory, epoch, mini_batch_size): + metrics, loss = self.evaluate(dataset, mini_batch_size=mini_batch_size, + embeddings_in_memory=embeddings_in_memory) + + f_score = metrics[MICRO_AVG_METRIC].f_score() + acc = metrics[MICRO_AVG_METRIC].accuracy() + + print("{0:<7} epoch {1} - loss {2:.8f} - f-score {3:.4f} - acc {4:.4f}".format( + dataset_name, epoch + 1, loss, f_score, acc)) + + return acc, f_score, loss + + def trace_print(self): + snapshot2 = tracemalloc.take_snapshot() + snapshot2 = snapshot2.filter_traces(( + tracemalloc.Filter(False, ""), + tracemalloc.Filter(False, ""), + tracemalloc.Filter(False, tracemalloc.__file__) + )) + + if self.snapshot is not None: + print("================================== Begin Trace:") + top_stats = snapshot2.compare_to(self.snapshot, 'lineno', cumulative=True) + for stat in top_stats[:10]: + print(stat) + self.snapshot = snapshot2 + def evaluate(self, sentences: List[Sentence], eval_class_metrics: bool = False, mini_batch_size: int = 32, embeddings_in_memory: bool = True) -> (dict, float): """ @@ -185,8 +213,9 @@ def evaluate(self, sentences: List[Sentence], eval_class_metrics: bool = False, range(0, len(sentences), mini_batch_size)] y_pred = [] - y_true = [] + y_true = convert_labels_to_one_hot([sentence.get_label_names() for batch in batches for sentence in batch], self.label_dict) + j = 0 for batch in batches: scores = self.model.forward(batch) labels = self.model.obtain_labels(scores) @@ -194,14 +223,14 @@ def evaluate(self, sentences: List[Sentence], eval_class_metrics: bool = False, eval_loss += loss - y_true.extend([sentence.get_label_names() for sentence in batch]) - y_pred.extend([[label.name for label in sent_labels] for sent_labels in labels]) + y_pred.extend(convert_labels_to_one_hot([[label.name for label in sent_labels] for sent_labels in labels], self.label_dict)) if not embeddings_in_memory: clear_embeddings(batch) - y_pred = convert_labels_to_one_hot(y_pred, self.label_dict) - y_true = convert_labels_to_one_hot(y_true, self.label_dict) + j += 1 + if j % 10 == 0: + self.trace_print() metrics = [calculate_micro_avg_metric(y_true, y_pred, self.label_dict)] if eval_class_metrics: @@ -213,7 +242,7 @@ def evaluate(self, sentences: List[Sentence], eval_class_metrics: bool = False, return metrics_dict, eval_loss - def _extract_weigths(self, iteration, weights_index, weights_txt): + def _extract_weights(self, iteration, weights_index, weights_txt): for key in self.model.state_dict().keys(): vec = self.model.state_dict()[key] From 2bbc5ee62bc0c37a0847ee823009535f0f518dbf Mon Sep 17 00:00:00 2001 From: tabergma Date: Mon, 17 Sep 2018 09:31:20 +0200 Subject: [PATCH 013/113] GH-108: Remove comments. --- flair/trainers/text_classification_trainer.py | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/flair/trainers/text_classification_trainer.py b/flair/trainers/text_classification_trainer.py index a2d490416e..118d26d5bc 100644 --- a/flair/trainers/text_classification_trainer.py +++ b/flair/trainers/text_classification_trainer.py @@ -106,9 +106,9 @@ def train(self, if batch_no % modulo == 0: print("epoch {0} - iter {1}/{2} - loss {3:.8f}".format(epoch + 1, batch_no, len(batches), current_loss / seen_sentences)) - self.trace_print() - # iteration = epoch * len(batches) + batch_no - # self._extract_weights(iteration, weights_index, weights_txt) + # self.trace_print() + iteration = epoch * len(batches) + batch_no + self._extract_weights(iteration, weights_index, weights_txt) current_loss /= len(train_data) @@ -130,19 +130,19 @@ def train(self, self.model.train() - # # anneal against train loss if training with dev, otherwise anneal against dev score - # scheduler.step(current_loss) if train_with_dev else scheduler.step(dev_f_score) + # anneal against train loss if training with dev, otherwise anneal against dev score + scheduler.step(current_loss) if train_with_dev else scheduler.step(dev_f_score) - # is_best_model_so_far: bool = False - # current_score = dev_f_score if not train_with_dev else train_f_score - # - # if current_score > best_score: - # best_score = current_score - # is_best_model_so_far = True - # - # if is_best_model_so_far: - # if save_model: - # self.model.save(base_path + "/model.pt") + is_best_model_so_far: bool = False + current_score = dev_f_score if not train_with_dev else train_f_score + + if current_score > best_score: + best_score = current_score + is_best_model_so_far = True + + if is_best_model_so_far: + if save_model: + self.model.save(base_path + "/model.pt") self.model.save(base_path + "/final-model.pt") @@ -213,9 +213,9 @@ def evaluate(self, sentences: List[Sentence], eval_class_metrics: bool = False, range(0, len(sentences), mini_batch_size)] y_pred = [] - y_true = convert_labels_to_one_hot([sentence.get_label_names() for batch in batches for sentence in batch], self.label_dict) + y_true = convert_labels_to_one_hot([sentence.get_label_names() for sentence in sentences], self.label_dict) - j = 0 + # j = 0 for batch in batches: scores = self.model.forward(batch) labels = self.model.obtain_labels(scores) @@ -228,9 +228,9 @@ def evaluate(self, sentences: List[Sentence], eval_class_metrics: bool = False, if not embeddings_in_memory: clear_embeddings(batch) - j += 1 - if j % 10 == 0: - self.trace_print() + # j += 1 + # if j % 10 == 0: + # self.trace_print() metrics = [calculate_micro_avg_metric(y_true, y_pred, self.label_dict)] if eval_class_metrics: From 6cddf8f458fe5b848d725074d2995700a2423e78 Mon Sep 17 00:00:00 2001 From: tabergma Date: Thu, 20 Sep 2018 09:22:20 +0200 Subject: [PATCH 014/113] GH-108: Print trace after every epoch. --- flair/trainers/text_classification_trainer.py | 40 +++++++++---------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/flair/trainers/text_classification_trainer.py b/flair/trainers/text_classification_trainer.py index 118d26d5bc..224a692d6c 100644 --- a/flair/trainers/text_classification_trainer.py +++ b/flair/trainers/text_classification_trainer.py @@ -106,10 +106,11 @@ def train(self, if batch_no % modulo == 0: print("epoch {0} - iter {1}/{2} - loss {3:.8f}".format(epoch + 1, batch_no, len(batches), current_loss / seen_sentences)) - # self.trace_print() iteration = epoch * len(batches) + batch_no self._extract_weights(iteration, weights_index, weights_txt) + self.trace_print() + current_loss /= len(train_data) self.model.eval() @@ -184,21 +185,6 @@ def _calculate_evaluation_results_for(self, dataset_name, dataset, embeddings_in return acc, f_score, loss - def trace_print(self): - snapshot2 = tracemalloc.take_snapshot() - snapshot2 = snapshot2.filter_traces(( - tracemalloc.Filter(False, ""), - tracemalloc.Filter(False, ""), - tracemalloc.Filter(False, tracemalloc.__file__) - )) - - if self.snapshot is not None: - print("================================== Begin Trace:") - top_stats = snapshot2.compare_to(self.snapshot, 'lineno', cumulative=True) - for stat in top_stats[:10]: - print(stat) - self.snapshot = snapshot2 - def evaluate(self, sentences: List[Sentence], eval_class_metrics: bool = False, mini_batch_size: int = 32, embeddings_in_memory: bool = True) -> (dict, float): """ @@ -215,7 +201,6 @@ def evaluate(self, sentences: List[Sentence], eval_class_metrics: bool = False, y_pred = [] y_true = convert_labels_to_one_hot([sentence.get_label_names() for sentence in sentences], self.label_dict) - # j = 0 for batch in batches: scores = self.model.forward(batch) labels = self.model.obtain_labels(scores) @@ -228,9 +213,7 @@ def evaluate(self, sentences: List[Sentence], eval_class_metrics: bool = False, if not embeddings_in_memory: clear_embeddings(batch) - # j += 1 - # if j % 10 == 0: - # self.trace_print() + self.trace_print() metrics = [calculate_micro_avg_metric(y_true, y_pred, self.label_dict)] if eval_class_metrics: @@ -278,4 +261,19 @@ def _init_weights_index(self, key, weights_index, weights_to_watch): indices[i] = cur_indices i += 1 - weights_index[key] = indices \ No newline at end of file + weights_index[key] = indices + + def trace_print(self): + snapshot2 = tracemalloc.take_snapshot() + snapshot2 = snapshot2.filter_traces(( + tracemalloc.Filter(False, ""), + tracemalloc.Filter(False, ""), + tracemalloc.Filter(False, tracemalloc.__file__) + )) + + if self.snapshot is not None: + print("================================== Begin Trace:") + top_stats = snapshot2.compare_to(self.snapshot, 'lineno', cumulative=True) + for stat in top_stats[:10]: + print(stat) + self.snapshot = snapshot2 \ No newline at end of file From 1787d6165e616b3cf2fb457b0a966b9ff6b57a77 Mon Sep 17 00:00:00 2001 From: tabergma Date: Thu, 20 Sep 2018 09:43:16 +0200 Subject: [PATCH 015/113] GH-108: Remove tracemalloc. --- flair/trainers/text_classification_trainer.py | 24 +------------------ 1 file changed, 1 insertion(+), 23 deletions(-) diff --git a/flair/trainers/text_classification_trainer.py b/flair/trainers/text_classification_trainer.py index 224a692d6c..cb693f83ea 100644 --- a/flair/trainers/text_classification_trainer.py +++ b/flair/trainers/text_classification_trainer.py @@ -10,7 +10,6 @@ from flair.models.text_classification_model import TextClassifier from flair.training_utils import convert_labels_to_one_hot, calculate_micro_avg_metric, init_output_file, clear_embeddings, \ calculate_class_metrics -import tracemalloc MICRO_AVG_METRIC = 'MICRO_AVG' @@ -48,8 +47,6 @@ def train(self, :param train_with_dev: boolean value indicating, if the dev data set should be used for training or not """ - tracemalloc.start() - loss_txt = init_output_file(base_path, 'loss.txt') with open(loss_txt, 'a') as f: f.write('EPOCH\tITERATION\tDEV_LOSS\tTRAIN_LOSS\tDEV_F_SCORE\tTRAIN_F_SCORE\tDEV_ACC\tTRAIN_ACC\n') @@ -109,8 +106,6 @@ def train(self, iteration = epoch * len(batches) + batch_no self._extract_weights(iteration, weights_index, weights_txt) - self.trace_print() - current_loss /= len(train_data) self.model.eval() @@ -213,8 +208,6 @@ def evaluate(self, sentences: List[Sentence], eval_class_metrics: bool = False, if not embeddings_in_memory: clear_embeddings(batch) - self.trace_print() - metrics = [calculate_micro_avg_metric(y_true, y_pred, self.label_dict)] if eval_class_metrics: metrics.extend(calculate_class_metrics(y_true, y_pred, self.label_dict)) @@ -261,19 +254,4 @@ def _init_weights_index(self, key, weights_index, weights_to_watch): indices[i] = cur_indices i += 1 - weights_index[key] = indices - - def trace_print(self): - snapshot2 = tracemalloc.take_snapshot() - snapshot2 = snapshot2.filter_traces(( - tracemalloc.Filter(False, ""), - tracemalloc.Filter(False, ""), - tracemalloc.Filter(False, tracemalloc.__file__) - )) - - if self.snapshot is not None: - print("================================== Begin Trace:") - top_stats = snapshot2.compare_to(self.snapshot, 'lineno', cumulative=True) - for stat in top_stats[:10]: - print(stat) - self.snapshot = snapshot2 \ No newline at end of file + weights_index[key] = indices \ No newline at end of file From 9173df81de7ba8199b7a5beccee4cf15e1f9575b Mon Sep 17 00:00:00 2001 From: tabergma Date: Thu, 20 Sep 2018 09:47:16 +0200 Subject: [PATCH 016/113] GH-108: Fix test. --- flair/trainers/text_classification_trainer.py | 1 - tests/test_data.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/flair/trainers/text_classification_trainer.py b/flair/trainers/text_classification_trainer.py index cb693f83ea..31e7d216b0 100644 --- a/flair/trainers/text_classification_trainer.py +++ b/flair/trainers/text_classification_trainer.py @@ -24,7 +24,6 @@ def __init__(self, model: TextClassifier, corpus: TaggedCorpus, label_dict: Dict self.corpus: TaggedCorpus = corpus self.label_dict: Dictionary = label_dict self.test_mode: bool = test_mode - self.snapshot = None def train(self, base_path: str, diff --git a/tests/test_data.py b/tests/test_data.py index 310898d751..aa28cb2308 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -230,7 +230,7 @@ def test_tagged_corpus_make_vocab_dictionary(): def test_label_set_confidence(): label = Label('class_1', 3.2) - assert (0.0 == label.confidence) + assert (1.0 == label.confidence) assert ('class_1' == label.name) label.confidence = 0.2 From 00d38d58557c47819d0cf525e25da59451aeaf25 Mon Sep 17 00:00:00 2001 From: tabergma Date: Thu, 20 Sep 2018 13:08:30 +0200 Subject: [PATCH 017/113] GH-108: Remove tracemalloc. --- flair/embeddings.py | 1 - flair/trainers/text_classification_trainer.py | 9 +++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/flair/embeddings.py b/flair/embeddings.py index 885e93290d..d3b3ed9c23 100644 --- a/flair/embeddings.py +++ b/flair/embeddings.py @@ -381,7 +381,6 @@ def embedding_length(self) -> int: return self.__embedding_length def _add_embeddings_internal(self, sentences: List[Sentence]) -> List[Sentence]: - # get text sentences text_sentences = [sentence.to_tokenized_string() for sentence in sentences] diff --git a/flair/trainers/text_classification_trainer.py b/flair/trainers/text_classification_trainer.py index 31e7d216b0..9e053edd99 100644 --- a/flair/trainers/text_classification_trainer.py +++ b/flair/trainers/text_classification_trainer.py @@ -24,6 +24,7 @@ def __init__(self, model: TextClassifier, corpus: TaggedCorpus, label_dict: Dict self.corpus: TaggedCorpus = corpus self.label_dict: Dictionary = label_dict self.test_mode: bool = test_mode + self.snapshot = None def train(self, base_path: str, @@ -179,7 +180,7 @@ def _calculate_evaluation_results_for(self, dataset_name, dataset, embeddings_in return acc, f_score, loss - def evaluate(self, sentences: List[Sentence], eval_class_metrics: bool = False, mini_batch_size: int = 32, + def evaluate(self, sentences: List[Sentence], eval_class_metrics: bool = False, mini_batch_size: int = 16, embeddings_in_memory: bool = True) -> (dict, float): """ Evaluates the model with the given list of sentences. @@ -200,13 +201,13 @@ def evaluate(self, sentences: List[Sentence], eval_class_metrics: bool = False, labels = self.model.obtain_labels(scores) loss = self.model.calculate_loss(scores, batch) + if not embeddings_in_memory: + clear_embeddings(batch) + eval_loss += loss y_pred.extend(convert_labels_to_one_hot([[label.name for label in sent_labels] for sent_labels in labels], self.label_dict)) - if not embeddings_in_memory: - clear_embeddings(batch) - metrics = [calculate_micro_avg_metric(y_true, y_pred, self.label_dict)] if eval_class_metrics: metrics.extend(calculate_class_metrics(y_true, y_pred, self.label_dict)) From 7e2c669d8f620e484cf2661e06f91f440d3a80a9 Mon Sep 17 00:00:00 2001 From: tabergma Date: Thu, 20 Sep 2018 14:05:58 +0200 Subject: [PATCH 018/113] GH-108: Add params for training. --- flair/embeddings.py | 3 +-- flair/trainers/text_classification_trainer.py | 23 +++++++++++++------ 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/flair/embeddings.py b/flair/embeddings.py index d3b3ed9c23..32bb443b09 100644 --- a/flair/embeddings.py +++ b/flair/embeddings.py @@ -307,8 +307,6 @@ class CharLMEmbeddings(TokenEmbeddings): """Contextual string embeddings of words, as proposed in Akbik et al., 2018.""" def __init__(self, model, detach: bool = True): - super().__init__() - """ Contextual string embeddings of words, as proposed in Akbik et al., 2018. @@ -321,6 +319,7 @@ def __init__(self, model, detach: bool = True): if set to false, the gradient will propagate into the language model. this dramatically slows down training and often leads to worse results, so not recommended. """ + super().__init__() # news-english-forward if model.lower() == 'news-forward': diff --git a/flair/trainers/text_classification_trainer.py b/flair/trainers/text_classification_trainer.py index 9e053edd99..7eef04fc27 100644 --- a/flair/trainers/text_classification_trainer.py +++ b/flair/trainers/text_classification_trainer.py @@ -24,27 +24,31 @@ def __init__(self, model: TextClassifier, corpus: TaggedCorpus, label_dict: Dict self.corpus: TaggedCorpus = corpus self.label_dict: Dictionary = label_dict self.test_mode: bool = test_mode - self.snapshot = None def train(self, base_path: str, learning_rate: float = 0.1, mini_batch_size: int = 32, + eval_mini_batch_size: int = 8, max_epochs: int = 100, anneal_factor: float = 0.5, patience: int = 2, save_model: bool = True, embeddings_in_memory: bool = True, - train_with_dev: bool = False): + train_with_dev: bool = False, + eval_on_train: bool = False): """ Trains the model using the training data of the corpus. :param base_path: the directory to which any results should be written to :param learning_rate: the learning rate :param mini_batch_size: the mini batch size + :param eval_mini_batch_size: the mini batch size for evaluation :param max_epochs: the maximum number of epochs to train :param save_model: boolean value indicating, whether the model should be saved or not :param embeddings_in_memory: boolean value indicating, if embeddings should be kept in memory or not :param train_with_dev: boolean value indicating, if the dev data set should be used for training or not + :param eval_on_train: boolean value indicating, if evaluation metrics should be calculated on training data set + or not """ loss_txt = init_output_file(base_path, 'loss.txt') @@ -101,8 +105,11 @@ def train(self, clear_embeddings(batch) if batch_no % modulo == 0: - print("epoch {0} - iter {1}/{2} - loss {3:.8f}".format(epoch + 1, batch_no, len(batches), - current_loss / seen_sentences)) + for group in optimizer.param_groups: + learning_rate = group['lr'] + + print("epoch {0} - iter {1}/{2} - loss {3:.8f} - lr {4:.4f} - bad epochs {5}".format( + epoch + 1, batch_no, len(batches), current_loss / seen_sentences, learning_rate, scheduler.num_bad_epochs)) iteration = epoch * len(batches) + batch_no self._extract_weights(iteration, weights_index, weights_txt) @@ -112,13 +119,15 @@ def train(self, print('-' * 100) - train_acc, train_f_score, train_loss = self._calculate_evaluation_results_for( - 'TRAIN', self.corpus.train, embeddings_in_memory, epoch, mini_batch_size) + train_f_score = train_acc = train_loss = 0 + if eval_on_train: + train_acc, train_f_score, train_loss = self._calculate_evaluation_results_for( + 'TRAIN', self.corpus.train, embeddings_in_memory, epoch, eval_mini_batch_size) dev_f_score = dev_acc = dev_loss = 0 if not train_with_dev: dev_acc, dev_f_score, dev_loss = self._calculate_evaluation_results_for( - 'DEV', self.corpus.dev, embeddings_in_memory, epoch, mini_batch_size) + 'DEV', self.corpus.dev, embeddings_in_memory, epoch, eval_mini_batch_size) with open(loss_txt, 'a') as f: f.write('{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\n'.format( From 7132735c667c8ba6c0c524a635f9084815df722e Mon Sep 17 00:00:00 2001 From: tabergma Date: Thu, 20 Sep 2018 16:09:08 +0200 Subject: [PATCH 019/113] GH-48: Read only docs with certain amount of tokens. --- flair/data_fetcher.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/flair/data_fetcher.py b/flair/data_fetcher.py index d9d735b586..35ddaf12eb 100644 --- a/flair/data_fetcher.py +++ b/flair/data_fetcher.py @@ -313,7 +313,7 @@ def read_conll_ud(path_to_conll_file: str) -> List[Sentence]: return sentences @staticmethod - def read_text_classification_file(path_to_file): + def read_text_classification_file(path_to_file, max_tokens_per_doc=-1): """ Reads a data file for text classification. The file should contain one document/text per line. The line should have the following format: @@ -321,6 +321,8 @@ def read_text_classification_file(path_to_file): If you have a multi class task, you can have as many labels as you want at the beginning of the line, e.g., __label__ __label__ :param path_to_file: the path to the data file + :param max_tokens_per_doc: Take only documents that contain number of tokens less or equal to this value. If + set to -1 all documents are taken. :return: list of sentences """ label_prefix = '__label__' @@ -346,7 +348,9 @@ def read_text_classification_file(path_to_file): text = line[l_len:].strip() if text and labels: - sentences.append(Sentence(text, labels=labels, use_tokenizer=True)) + sentence = Sentence(text, labels=labels, use_tokenizer=True) + if max_tokens_per_doc == -1 or len(sentence.tokens) <= max_tokens_per_doc: + sentences.append(sentence) return sentences From 64da98e21bc6e31e6971a976fef7590ac2d421c8 Mon Sep 17 00:00:00 2001 From: tabergma Date: Thu, 20 Sep 2018 16:16:06 +0200 Subject: [PATCH 020/113] GH-48: Add word dropout to document lstm embeddings --- flair/embeddings.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/flair/embeddings.py b/flair/embeddings.py index 32bb443b09..8750faf6cf 100644 --- a/flair/embeddings.py +++ b/flair/embeddings.py @@ -1,5 +1,4 @@ import os -import pickle import re from abc import abstractmethod from typing import List, Union @@ -8,7 +7,8 @@ import numpy as np import torch -from .data import Dictionary, Token, Sentence, TaggedCorpus +from .nn import LockedDropout, WordDropout +from .data import Dictionary, Token, Sentence from .file_utils import cached_path @@ -491,7 +491,7 @@ class DocumentLSTMEmbeddings(DocumentEmbeddings): def __init__(self, token_embeddings: List[TokenEmbeddings], hidden_states=128, num_layers=1, reproject_words: bool = True, reproject_words_dimension: int = None, bidirectional: bool = False, - use_first_representation: bool = False): + use_first_representation: bool = False, use_word_dropout: bool = True): """The constructor takes a list of embeddings to be combined. :param token_embeddings: a list of token embeddings :param hidden_states: the number of hidden states in the lstm @@ -503,6 +503,7 @@ def __init__(self, token_embeddings: List[TokenEmbeddings], hidden_states=128, n :param bidirectional: boolean value, indicating whether to use a bidirectional lstm or not :param use_first_representation: boolean value, indicating whether to concatenate the first and last representation of the lstm to be used as final document embedding. + :param use_word_dropout: boolean value, indicating whether to use word dropout or not. """ super().__init__() @@ -534,7 +535,13 @@ def __init__(self, token_embeddings: List[TokenEmbeddings], hidden_states=128, n self.embeddings_dimension) self.rnn = torch.nn.GRU(self.embeddings_dimension, hidden_states, num_layers=num_layers, bidirectional=self.bidirectional) - self.dropout = torch.nn.Dropout(0.5) + + # dropouts + self.dropout: torch.nn.Module = LockedDropout(0.5) + + self.use_word_dropout: bool = use_word_dropout + if self.use_word_dropout: + self.word_dropout = WordDropout(0.05) torch.nn.init.xavier_uniform_(self.word_reprojection_map.weight) @@ -598,6 +605,10 @@ def embed(self, sentences: Union[List[Sentence], Sentence]): # -------------------------------------------------------------------- # FF PART # -------------------------------------------------------------------- + # use word dropout if set + if self.use_word_dropout: + sentence_tensor = self.word_dropout(sentence_tensor) + if self.reproject_words: sentence_tensor = self.word_reprojection_map(sentence_tensor) From afbc805bd81bde2053406bfb095a0d2446d0db88 Mon Sep 17 00:00:00 2001 From: tabergma Date: Thu, 20 Sep 2018 16:38:57 +0200 Subject: [PATCH 021/113] GH-16: Move weight extraction to training utils. --- flair/trainers/text_classification_trainer.py | 49 ++--------------- flair/training_utils.py | 54 +++++++++++++++++-- 2 files changed, 54 insertions(+), 49 deletions(-) diff --git a/flair/trainers/text_classification_trainer.py b/flair/trainers/text_classification_trainer.py index 7eef04fc27..c3eabeedd5 100644 --- a/flair/trainers/text_classification_trainer.py +++ b/flair/trainers/text_classification_trainer.py @@ -1,6 +1,4 @@ import random -from collections import defaultdict -from functools import reduce from typing import List import torch @@ -8,8 +6,8 @@ from flair.data import Sentence, TaggedCorpus, Dictionary from flair.models.text_classification_model import TextClassifier -from flair.training_utils import convert_labels_to_one_hot, calculate_micro_avg_metric, init_output_file, clear_embeddings, \ - calculate_class_metrics +from flair.training_utils import convert_labels_to_one_hot, calculate_micro_avg_metric, init_output_file, \ + clear_embeddings, calculate_class_metrics, WeightExtractor MICRO_AVG_METRIC = 'MICRO_AVG' @@ -54,9 +52,8 @@ def train(self, loss_txt = init_output_file(base_path, 'loss.txt') with open(loss_txt, 'a') as f: f.write('EPOCH\tITERATION\tDEV_LOSS\tTRAIN_LOSS\tDEV_F_SCORE\tTRAIN_F_SCORE\tDEV_ACC\tTRAIN_ACC\n') - weights_txt = init_output_file(base_path, 'weights.txt') - weights_index = defaultdict(lambda: defaultdict(lambda: list())) + weight_extractor = WeightExtractor(base_path) optimizer = torch.optim.SGD(self.model.parameters(), lr=learning_rate) @@ -111,7 +108,7 @@ def train(self, print("epoch {0} - iter {1}/{2} - loss {3:.8f} - lr {4:.4f} - bad epochs {5}".format( epoch + 1, batch_no, len(batches), current_loss / seen_sentences, learning_rate, scheduler.num_bad_epochs)) iteration = epoch * len(batches) + batch_no - self._extract_weights(iteration, weights_index, weights_txt) + weight_extractor.extract_weights(self.model.state_dict(), iteration) current_loss /= len(train_data) @@ -226,41 +223,3 @@ def evaluate(self, sentences: List[Sentence], eval_class_metrics: bool = False, metrics_dict = {metric.name: metric for metric in metrics} return metrics_dict, eval_loss - - def _extract_weights(self, iteration, weights_index, weights_txt): - for key in self.model.state_dict().keys(): - - vec = self.model.state_dict()[key] - weights_to_watch = min(10, reduce(lambda x, y: x*y, list(vec.size()))) - - if key not in weights_index: - self._init_weights_index(key, weights_index, weights_to_watch) - - for i in range(weights_to_watch): - vec = self.model.state_dict()[key] - for index in weights_index[key][i]: - vec = vec[index] - - value = vec.item() - - with open(weights_txt, 'a') as f: - f.write('{}\t{}\t{}\t{}\n'.format(iteration, key, i, float(value))) - - def _init_weights_index(self, key, weights_index, weights_to_watch): - indices = {} - - i = 0 - while len(indices) < weights_to_watch: - vec = self.model.state_dict()[key] - cur_indices = [] - - for x in range(len(vec.size())): - index = random.randint(0, len(vec) - 1) - vec = vec[index] - cur_indices.append(index) - - if cur_indices not in list(indices.values()): - indices[i] = cur_indices - i += 1 - - weights_index[key] = indices \ No newline at end of file diff --git a/flair/training_utils.py b/flair/training_utils.py index c3aaf58a09..555e167ed4 100644 --- a/flair/training_utils.py +++ b/flair/training_utils.py @@ -1,9 +1,9 @@ -from typing import List, Dict - +import random import os -import numpy as np - +from collections import defaultdict +from typing import List from flair.data import Dictionary, Sentence +from functools import reduce class Metric(object): @@ -56,6 +56,52 @@ def print(self): print(self) +class WeightExtractor(object): + + def __init__(self, directory: str, number_of_weights: int = 10): + self.weights_file = init_output_file(directory, 'weights.txt') + self.weights_dict = defaultdict(lambda: defaultdict(lambda: list())) + self.number_of_weights = number_of_weights + + def extract_weights(self, state_dict, iteration): + for key in state_dict.keys(): + + vec = state_dict[key] + weights_to_watch = min(self.number_of_weights, reduce(lambda x, y: x*y, list(vec.size()))) + + if key not in self.weights_dict: + self._init_weights_index(key, state_dict, weights_to_watch) + + for i in range(weights_to_watch): + vec = state_dict[key] + for index in self.weights_dict[key][i]: + vec = vec[index] + + value = vec.item() + + with open(self.weights_file, 'a') as f: + f.write('{}\t{}\t{}\t{}\n'.format(iteration, key, i, float(value))) + + def _init_weights_index(self, key, state_dict, weights_to_watch): + indices = {} + + i = 0 + while len(indices) < weights_to_watch: + vec = state_dict[key] + cur_indices = [] + + for x in range(len(vec.size())): + index = random.randint(0, len(vec) - 1) + vec = vec[index] + cur_indices.append(index) + + if cur_indices not in list(indices.values()): + indices[i] = cur_indices + i += 1 + + self.weights_dict[key] = indices + + def clear_embeddings(sentences: List[Sentence]): """ Clears the embeddings from all given sentences. From 352637e6af64f7ea3ff587beeb904675a4462c76 Mon Sep 17 00:00:00 2001 From: tabergma Date: Thu, 20 Sep 2018 17:35:29 +0200 Subject: [PATCH 022/113] GH-16: Standardized log output in text classifier and sequence labeler. --- flair/trainers/sequence_tagger_trainer.py | 75 +++++++++---------- flair/trainers/text_classification_trainer.py | 44 ++++++----- flair/training_utils.py | 4 + 3 files changed, 64 insertions(+), 59 deletions(-) diff --git a/flair/trainers/sequence_tagger_trainer.py b/flair/trainers/sequence_tagger_trainer.py index 7fa766c3a7..e36f2493eb 100644 --- a/flair/trainers/sequence_tagger_trainer.py +++ b/flair/trainers/sequence_tagger_trainer.py @@ -12,7 +12,7 @@ from flair.file_utils import cached_path from flair.models.sequence_tagger_model import SequenceTagger from flair.data import Sentence, Token, TaggedCorpus -from flair.training_utils import Metric +from flair.training_utils import Metric, init_output_file, WeightExtractor class SequenceTaggerTrainer: @@ -36,10 +36,11 @@ def train(self, if self.model.tag_type in ['pos', 'upos']: evaluation_method = 'accuracy' print(evaluation_method) - os.makedirs(base_path, exist_ok=True) + loss_txt = init_output_file(base_path, 'loss.txt') + with open(loss_txt, 'a') as f: + f.write('EPOCH\tTIMESTAMP\tTRAIN_LOSS\tTRAIN_METRICS\tDEV_LOSS\tDEV_METRICS\tTEST_LOSS\tTEST_METRICS\n') - loss_txt = os.path.join(base_path, "loss.txt") - open(loss_txt, "w", encoding='utf-8').close() + weight_extractor = WeightExtractor(base_path) optimizer = torch.optim.SGD(self.model.parameters(), lr=learning_rate) @@ -56,12 +57,8 @@ def train(self, # At any point you can hit Ctrl + C to break out of training early. try: - for epoch in range(0, max_epochs): - - current_loss: int = 0 - - for group in optimizer.param_groups: - learning_rate = group['lr'] + for epoch in range(max_epochs): + print('-' * 100) if not self.test_mode: random.shuffle(train_data) @@ -69,14 +66,15 @@ def train(self, self.model.train() - batch_no: int = 0 + current_loss: float = 0 + seen_sentences = 0 + modulo = max(1, int(len(batches) / 10)) - for batch in batches: - batch: List[Sentence] = batch - batch_no += 1 + for group in optimizer.param_groups: + learning_rate = group['lr'] - if batch_no % 100 == 0: - print("%d of %d (%f)" % (batch_no, len(batches), float(batch_no / len(batches)))) + for batch_no, batch in enumerate(batches): + batch: List[Sentence] = batch optimizer.zero_grad() @@ -84,35 +82,34 @@ def train(self, loss = self.model.neg_log_likelihood(batch, self.model.tag_type) current_loss += loss.item() + seen_sentences += len(batch) loss.backward() - torch.nn.utils.clip_grad_norm_(self.model.parameters(), 5.0) - optimizer.step() - sys.stdout.write('.') - sys.stdout.flush() - if not embeddings_in_memory: self.clear_embeddings_in_batch(batch) + if batch_no % modulo == 0: + print("epoch {0} - iter {1}/{2} - loss {3:.8f}".format( + epoch + 1, batch_no, len(batches), current_loss / seen_sentences)) + iteration = epoch * len(batches) + batch_no + weight_extractor.extract_weights(self.model.state_dict(), iteration) + current_loss /= len(train_data) # switch to eval mode self.model.eval() + print('-' * 100) + if not train_with_dev: - print('.. evaluating... dev... ') - dev_score, dev_fp, dev_result = self.evaluate(self.corpus.dev, base_path, + dev_score, dev_metric = self.evaluate(self.corpus.dev, base_path, evaluation_method=evaluation_method, embeddings_in_memory=embeddings_in_memory) - else: - dev_fp = 0 - dev_result = '_' - print('test... ') - test_score, test_fp, test_result = self.evaluate(self.corpus.test, base_path, + test_score, test_metric = self.evaluate(self.corpus.test, base_path, evaluation_method=evaluation_method, embeddings_in_memory=embeddings_in_memory) @@ -122,16 +119,16 @@ def train(self, # anneal against train loss if training with dev, otherwise anneal against dev score scheduler.step(current_loss) if train_with_dev else scheduler.step(dev_score) - summary = '%d' % epoch + '\t({:%H:%M:%S})'.format(datetime.datetime.now()) \ - + '\t%f\t%d\t%f\tDEV %d\t' % ( - current_loss, scheduler.num_bad_epochs, learning_rate, dev_fp) + dev_result - summary = summary.replace('\n', '') - summary += '\tTEST \t%d\t' % test_fp + test_result + if not train_with_dev: + print("{0:<7} epoch {1} - lr {2:.4f} - bad epochs {3} - f-score {4:.4f} - acc {5:.4f}".format( + 'DEV', epoch + 1, learning_rate, scheduler.num_bad_epochs, dev_metric.f_score(), dev_metric.accuracy())) + print("{0:<7} epoch {1} - lr {2:.4f} - bad epochs {3} - f-score {4:.4f} - acc {5:.4f}".format( + 'TEST', epoch + 1, learning_rate, scheduler.num_bad_epochs, test_metric.f_score(), test_metric.accuracy())) - print(summary) - with open(loss_txt, "a") as loss_file: - loss_file.write('%s\n' % summary) - loss_file.close() + with open(loss_txt, 'a') as f: + dev_metric_str = dev_metric.to_csv() if dev_metric is not None else '_' + f.write('{}\t{:%H:%M:%S}\t{}\t{}\t{}\t{}\t{}\t{}\n'.format( + epoch, datetime.datetime.now(), '_', '_', '_', dev_metric_str, '_', test_metric.to_csv())) # save if model is current best and we use dev data for model selection if save_model and not train_with_dev and dev_score == scheduler.best: @@ -210,11 +207,11 @@ def evaluate(self, evaluation: List[Sentence], out_path=None, evaluation_method: if evaluation_method == 'accuracy': score = metric.accuracy() - return score, metric._fp, str(score) + return score, metric if evaluation_method == 'F1': score = metric.f_score() - return score, metric._fp, str(metric) + return score, metric def clear_embeddings_in_batch(self, batch: List[Sentence]): for sentence in batch: diff --git a/flair/trainers/text_classification_trainer.py b/flair/trainers/text_classification_trainer.py index c3eabeedd5..c741e60f92 100644 --- a/flair/trainers/text_classification_trainer.py +++ b/flair/trainers/text_classification_trainer.py @@ -1,3 +1,4 @@ +import datetime import random from typing import List @@ -51,7 +52,7 @@ def train(self, loss_txt = init_output_file(base_path, 'loss.txt') with open(loss_txt, 'a') as f: - f.write('EPOCH\tITERATION\tDEV_LOSS\tTRAIN_LOSS\tDEV_F_SCORE\tTRAIN_F_SCORE\tDEV_ACC\tTRAIN_ACC\n') + f.write('EPOCH\tTIMESTAMP\tTRAIN_LOSS\tTRAIN_METRICS\tDEV_LOSS\tDEV_METRICS\tTEST_LOSS\tTEST_METRICS\n') weight_extractor = WeightExtractor(base_path) @@ -86,6 +87,9 @@ def train(self, seen_sentences = 0 modulo = max(1, int(len(batches) / 10)) + for group in optimizer.param_groups: + learning_rate = group['lr'] + for batch_no, batch in enumerate(batches): scores = self.model.forward(batch) loss = self.model.calculate_loss(scores, batch) @@ -102,11 +106,8 @@ def train(self, clear_embeddings(batch) if batch_no % modulo == 0: - for group in optimizer.param_groups: - learning_rate = group['lr'] - - print("epoch {0} - iter {1}/{2} - loss {3:.8f} - lr {4:.4f} - bad epochs {5}".format( - epoch + 1, batch_no, len(batches), current_loss / seen_sentences, learning_rate, scheduler.num_bad_epochs)) + print("epoch {0} - iter {1}/{2} - loss {3:.8f}".format( + epoch + 1, batch_no, len(batches), current_loss / seen_sentences)) iteration = epoch * len(batches) + batch_no weight_extractor.extract_weights(self.model.state_dict(), iteration) @@ -116,27 +117,30 @@ def train(self, print('-' * 100) - train_f_score = train_acc = train_loss = 0 + dev_metric = train_metric = None + dev_loss = train_loss = '_' + if eval_on_train: - train_acc, train_f_score, train_loss = self._calculate_evaluation_results_for( - 'TRAIN', self.corpus.train, embeddings_in_memory, epoch, eval_mini_batch_size) + train_metric, train_loss = self._calculate_evaluation_results_for( + 'TRAIN', self.corpus.train, embeddings_in_memory, epoch, eval_mini_batch_size, learning_rate, scheduler.num_bad_epochs) - dev_f_score = dev_acc = dev_loss = 0 if not train_with_dev: - dev_acc, dev_f_score, dev_loss = self._calculate_evaluation_results_for( - 'DEV', self.corpus.dev, embeddings_in_memory, epoch, eval_mini_batch_size) + dev_metric, dev_loss = self._calculate_evaluation_results_for( + 'DEV', self.corpus.dev, embeddings_in_memory, epoch, eval_mini_batch_size, learning_rate, scheduler.num_bad_epochs) with open(loss_txt, 'a') as f: - f.write('{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\n'.format( - epoch, epoch * len(batches), dev_loss, train_loss, dev_f_score, train_f_score, dev_acc, train_acc)) + train_metric_str = train_metric.to_csv() if train_metric is not None else '_' + dev_metric_str = dev_metric.to_csv() if dev_metric is not None else '_' + f.write('{}\t{:%H:%M:%S}\t{}\t{}\t{}\t{}\t{}\t{}\n'.format( + epoch, datetime.datetime.now(), train_loss, train_metric_str, dev_loss, dev_metric_str, '_', '_')) self.model.train() # anneal against train loss if training with dev, otherwise anneal against dev score - scheduler.step(current_loss) if train_with_dev else scheduler.step(dev_f_score) + scheduler.step(current_loss) if train_with_dev else scheduler.step(dev_metric.f_score()) is_best_model_so_far: bool = False - current_score = dev_f_score if not train_with_dev else train_f_score + current_score = dev_metric.f_score() if not train_with_dev else train_metric.f_score() if current_score > best_score: best_score = current_score @@ -174,17 +178,17 @@ def train(self, model_save_file.close() print('done') - def _calculate_evaluation_results_for(self, dataset_name, dataset, embeddings_in_memory, epoch, mini_batch_size): + def _calculate_evaluation_results_for(self, dataset_name, dataset, embeddings_in_memory, epoch, mini_batch_size, learning_rate, num_bad_epochs): metrics, loss = self.evaluate(dataset, mini_batch_size=mini_batch_size, embeddings_in_memory=embeddings_in_memory) f_score = metrics[MICRO_AVG_METRIC].f_score() acc = metrics[MICRO_AVG_METRIC].accuracy() - print("{0:<7} epoch {1} - loss {2:.8f} - f-score {3:.4f} - acc {4:.4f}".format( - dataset_name, epoch + 1, loss, f_score, acc)) + print("{0:<7} epoch {1} - lr {2:.4f} - bad epochs {3} - loss {4:.8f} - f-score {5:.4f} - acc {6:.4f}".format( + dataset_name, epoch + 1, learning_rate, num_bad_epochs, loss, f_score, acc)) - return acc, f_score, loss + return metrics[MICRO_AVG_METRIC], loss def evaluate(self, sentences: List[Sentence], eval_class_metrics: bool = False, mini_batch_size: int = 16, embeddings_in_memory: bool = True) -> (dict, float): diff --git a/flair/training_utils.py b/flair/training_utils.py index 555e167ed4..837175e04d 100644 --- a/flair/training_utils.py +++ b/flair/training_utils.py @@ -48,6 +48,10 @@ def accuracy(self): return (self._tp + self._tn) / (self._tp + self._tn + self._fp + self._fn) return 0.0 + def to_csv(self): + return '{},{},{},{},{},{},{},{}'.format( + self._tp, self._tn, self._fp, self._fn, self.precision(), self.recall(), self.f_score(), self.accuracy()) + def __str__(self): return '{0:<20}\tprecision: {1:.4f} - recall: {2:.4f} - accuracy: {3:.4f} - f1-score: {4:.4f}'.format( self.name, self.precision(), self.recall(), self.accuracy(), self.f_score()) From 3c25909296b8c4c1b0a3de3e52569eb2e5c0d133 Mon Sep 17 00:00:00 2001 From: tabergma Date: Thu, 20 Sep 2018 17:53:02 +0200 Subject: [PATCH 023/113] GH-16: Adapt epoch summary. --- flair/trainers/sequence_tagger_trainer.py | 9 +++++---- flair/trainers/text_classification_trainer.py | 11 ++++++----- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/flair/trainers/sequence_tagger_trainer.py b/flair/trainers/sequence_tagger_trainer.py index e36f2493eb..b31532f4d2 100644 --- a/flair/trainers/sequence_tagger_trainer.py +++ b/flair/trainers/sequence_tagger_trainer.py @@ -119,11 +119,12 @@ def train(self, # anneal against train loss if training with dev, otherwise anneal against dev score scheduler.step(current_loss) if train_with_dev else scheduler.step(dev_score) + print("EPOCH {0}: lr {1:.4f} - bad epochs {2}".format(epoch + 1, learning_rate, scheduler.num_bad_epochs)) if not train_with_dev: - print("{0:<7} epoch {1} - lr {2:.4f} - bad epochs {3} - f-score {4:.4f} - acc {5:.4f}".format( - 'DEV', epoch + 1, learning_rate, scheduler.num_bad_epochs, dev_metric.f_score(), dev_metric.accuracy())) - print("{0:<7} epoch {1} - lr {2:.4f} - bad epochs {3} - f-score {4:.4f} - acc {5:.4f}".format( - 'TEST', epoch + 1, learning_rate, scheduler.num_bad_epochs, test_metric.f_score(), test_metric.accuracy())) + print("{0:<4}: f-score {1:.4f} - acc {2:.4f} - tp {3} - fp {4} - fn {5} - tn {6}".format( + 'DEV', dev_metric.f_score(), dev_metric.accuracy(), dev_metric._tp, dev_metric._fp, dev_metric._fn, dev_metric._tn)) + print("{0:<4}: f-score {1:.4f} - acc {2:.4f} - tp {3} - fp {4} - fn {5} - tn {6}".format( + 'TEST', test_metric.f_score(), test_metric.accuracy(), test_metric._tp, test_metric._fp, test_metric._fn, test_metric._tn)) with open(loss_txt, 'a') as f: dev_metric_str = dev_metric.to_csv() if dev_metric is not None else '_' diff --git a/flair/trainers/text_classification_trainer.py b/flair/trainers/text_classification_trainer.py index c741e60f92..1148d41f6b 100644 --- a/flair/trainers/text_classification_trainer.py +++ b/flair/trainers/text_classification_trainer.py @@ -116,17 +116,18 @@ def train(self, self.model.eval() print('-' * 100) + print("EPOCH {0}: lr {1:.4f} - bad epochs {2}".format(epoch + 1, learning_rate, scheduler.num_bad_epochs)) dev_metric = train_metric = None dev_loss = train_loss = '_' if eval_on_train: train_metric, train_loss = self._calculate_evaluation_results_for( - 'TRAIN', self.corpus.train, embeddings_in_memory, epoch, eval_mini_batch_size, learning_rate, scheduler.num_bad_epochs) + 'TRAIN', self.corpus.train, embeddings_in_memory, eval_mini_batch_size) if not train_with_dev: dev_metric, dev_loss = self._calculate_evaluation_results_for( - 'DEV', self.corpus.dev, embeddings_in_memory, epoch, eval_mini_batch_size, learning_rate, scheduler.num_bad_epochs) + 'DEV', self.corpus.dev, embeddings_in_memory, eval_mini_batch_size) with open(loss_txt, 'a') as f: train_metric_str = train_metric.to_csv() if train_metric is not None else '_' @@ -178,15 +179,15 @@ def train(self, model_save_file.close() print('done') - def _calculate_evaluation_results_for(self, dataset_name, dataset, embeddings_in_memory, epoch, mini_batch_size, learning_rate, num_bad_epochs): + def _calculate_evaluation_results_for(self, dataset_name, dataset, embeddings_in_memory, mini_batch_size): metrics, loss = self.evaluate(dataset, mini_batch_size=mini_batch_size, embeddings_in_memory=embeddings_in_memory) f_score = metrics[MICRO_AVG_METRIC].f_score() acc = metrics[MICRO_AVG_METRIC].accuracy() - print("{0:<7} epoch {1} - lr {2:.4f} - bad epochs {3} - loss {4:.8f} - f-score {5:.4f} - acc {6:.4f}".format( - dataset_name, epoch + 1, learning_rate, num_bad_epochs, loss, f_score, acc)) + print("{0:<5}: loss {1:.8f} - f-score {2:.4f} - acc {3:.4f}".format( + dataset_name, loss, f_score, acc)) return metrics[MICRO_AVG_METRIC], loss From 145acda6874d71c1ec7af213f4a90bc2c9be7d5e Mon Sep 17 00:00:00 2001 From: tabergma Date: Thu, 20 Sep 2018 18:03:19 +0200 Subject: [PATCH 024/113] GH-16: Add logger. --- flair/__init__.py | 9 ++++++ flair/trainers/sequence_tagger_trainer.py | 29 +++++++++---------- flair/trainers/text_classification_trainer.py | 26 +++++++++-------- 3 files changed, 37 insertions(+), 27 deletions(-) diff --git a/flair/__init__.py b/flair/__init__.py index fe00a5bb01..37250c116b 100644 --- a/flair/__init__.py +++ b/flair/__init__.py @@ -1,2 +1,11 @@ from . import data from . import models + +import sys +import logging + +logger = logging.getLogger(__name__) + +FORMAT = '%(asctime)-15s %(message)s' + +logging.basicConfig(level=logging.INFO, format=FORMAT, stream=sys.stdout) \ No newline at end of file diff --git a/flair/trainers/sequence_tagger_trainer.py b/flair/trainers/sequence_tagger_trainer.py index b31532f4d2..5a86ca16d2 100644 --- a/flair/trainers/sequence_tagger_trainer.py +++ b/flair/trainers/sequence_tagger_trainer.py @@ -1,19 +1,18 @@ -from subprocess import run, PIPE from typing import List import datetime import os import random -import re -import sys +import logging import torch from torch.optim.lr_scheduler import ReduceLROnPlateau -from flair.file_utils import cached_path from flair.models.sequence_tagger_model import SequenceTagger from flair.data import Sentence, Token, TaggedCorpus from flair.training_utils import Metric, init_output_file, WeightExtractor +log = logging.getLogger(__name__) + class SequenceTaggerTrainer: def __init__(self, model: SequenceTagger, corpus: TaggedCorpus, test_mode: bool = False) -> None: @@ -34,7 +33,7 @@ def train(self, evaluation_method = 'F1' if self.model.tag_type in ['pos', 'upos']: evaluation_method = 'accuracy' - print(evaluation_method) + log.info(evaluation_method) loss_txt = init_output_file(base_path, 'loss.txt') with open(loss_txt, 'a') as f: @@ -58,7 +57,7 @@ def train(self, try: for epoch in range(max_epochs): - print('-' * 100) + log.info('-' * 100) if not self.test_mode: random.shuffle(train_data) @@ -92,7 +91,7 @@ def train(self, self.clear_embeddings_in_batch(batch) if batch_no % modulo == 0: - print("epoch {0} - iter {1}/{2} - loss {3:.8f}".format( + log.info("epoch {0} - iter {1}/{2} - loss {3:.8f}".format( epoch + 1, batch_no, len(batches), current_loss / seen_sentences)) iteration = epoch * len(batches) + batch_no weight_extractor.extract_weights(self.model.state_dict(), iteration) @@ -102,7 +101,7 @@ def train(self, # switch to eval mode self.model.eval() - print('-' * 100) + log.info('-' * 100) if not train_with_dev: dev_score, dev_metric = self.evaluate(self.corpus.dev, base_path, @@ -119,11 +118,11 @@ def train(self, # anneal against train loss if training with dev, otherwise anneal against dev score scheduler.step(current_loss) if train_with_dev else scheduler.step(dev_score) - print("EPOCH {0}: lr {1:.4f} - bad epochs {2}".format(epoch + 1, learning_rate, scheduler.num_bad_epochs)) + log.info("EPOCH {0}: lr {1:.4f} - bad epochs {2}".format(epoch + 1, learning_rate, scheduler.num_bad_epochs)) if not train_with_dev: - print("{0:<4}: f-score {1:.4f} - acc {2:.4f} - tp {3} - fp {4} - fn {5} - tn {6}".format( + log.info("{0:<4}: f-score {1:.4f} - acc {2:.4f} - tp {3} - fp {4} - fn {5} - tn {6}".format( 'DEV', dev_metric.f_score(), dev_metric.accuracy(), dev_metric._tp, dev_metric._fp, dev_metric._fn, dev_metric._tn)) - print("{0:<4}: f-score {1:.4f} - acc {2:.4f} - tp {3} - fp {4} - fn {5} - tn {6}".format( + log.info("{0:<4}: f-score {1:.4f} - acc {2:.4f} - tp {3} - fp {4} - fn {5} - tn {6}".format( 'TEST', test_metric.f_score(), test_metric.accuracy(), test_metric._tp, test_metric._fp, test_metric._fn, test_metric._tn)) with open(loss_txt, 'a') as f: @@ -139,11 +138,11 @@ def train(self, if save_model and train_with_dev: self.model.save(base_path + "/final-model.pt") except KeyboardInterrupt: - print('-' * 89) - print('Exiting from training early') - print('saving model') + log.info('-' * 89) + log.info('Exiting from training early') + log.info('saving model') self.model.save(base_path + "/final-model.pt") - print('done') + log.info('done') def evaluate(self, evaluation: List[Sentence], out_path=None, evaluation_method: str = 'F1', embeddings_in_memory: bool = True): diff --git a/flair/trainers/text_classification_trainer.py b/flair/trainers/text_classification_trainer.py index 1148d41f6b..9533985766 100644 --- a/flair/trainers/text_classification_trainer.py +++ b/flair/trainers/text_classification_trainer.py @@ -1,5 +1,6 @@ import datetime import random +import logging from typing import List import torch @@ -12,6 +13,7 @@ MICRO_AVG_METRIC = 'MICRO_AVG' +log = logging.getLogger(__name__) class TextClassifierTrainer: """ @@ -73,7 +75,7 @@ def train(self, best_score = 0 for epoch in range(max_epochs): - print('-' * 100) + log.info('-' * 100) if not self.test_mode: random.shuffle(train_data) @@ -106,7 +108,7 @@ def train(self, clear_embeddings(batch) if batch_no % modulo == 0: - print("epoch {0} - iter {1}/{2} - loss {3:.8f}".format( + log.info("epoch {0} - iter {1}/{2} - loss {3:.8f}".format( epoch + 1, batch_no, len(batches), current_loss / seen_sentences)) iteration = epoch * len(batches) + batch_no weight_extractor.extract_weights(self.model.state_dict(), iteration) @@ -115,8 +117,8 @@ def train(self, self.model.eval() - print('-' * 100) - print("EPOCH {0}: lr {1:.4f} - bad epochs {2}".format(epoch + 1, learning_rate, scheduler.num_bad_epochs)) + log.info('-' * 100) + log.info("EPOCH {0}: lr {1:.4f} - bad epochs {2}".format(epoch + 1, learning_rate, scheduler.num_bad_epochs)) dev_metric = train_metric = None dev_loss = train_loss = '_' @@ -156,8 +158,8 @@ def train(self, if save_model: self.model = TextClassifier.load_from_file(base_path + "/model.pt") - print('-' * 100) - print('Testing using best model ...') + log.info('-' * 100) + log.info('Testing using best model ...') self.model.eval() test_metrics, test_loss = self.evaluate( @@ -168,16 +170,16 @@ def train(self, metric.print() self.model.train() - print('-' * 100) + log.info('-' * 100) except KeyboardInterrupt: - print('-' * 89) - print('Exiting from training early') - print('saving model') + log.info('-' * 89) + log.info('Exiting from training early') + log.info('saving model') with open(base_path + "/final-model.pt", 'wb') as model_save_file: torch.save(self.model, model_save_file, pickle_protocol=4) model_save_file.close() - print('done') + log.info('done') def _calculate_evaluation_results_for(self, dataset_name, dataset, embeddings_in_memory, mini_batch_size): metrics, loss = self.evaluate(dataset, mini_batch_size=mini_batch_size, @@ -186,7 +188,7 @@ def _calculate_evaluation_results_for(self, dataset_name, dataset, embeddings_in f_score = metrics[MICRO_AVG_METRIC].f_score() acc = metrics[MICRO_AVG_METRIC].accuracy() - print("{0:<5}: loss {1:.8f} - f-score {2:.4f} - acc {3:.4f}".format( + log.info("{0:<5}: loss {1:.8f} - f-score {2:.4f} - acc {3:.4f}".format( dataset_name, loss, f_score, acc)) return metrics[MICRO_AVG_METRIC], loss From ffc0b8612c3ef0e9bd3e8799201c54786f159244 Mon Sep 17 00:00:00 2001 From: tabergma Date: Tue, 25 Sep 2018 10:04:55 +0200 Subject: [PATCH 025/113] GH-16: Metric to tsv --- flair/trainers/sequence_tagger_trainer.py | 10 +++++--- flair/trainers/text_classification_trainer.py | 13 +++++----- flair/training_utils.py | 25 +++++++++++++++---- 3 files changed, 33 insertions(+), 15 deletions(-) diff --git a/flair/trainers/sequence_tagger_trainer.py b/flair/trainers/sequence_tagger_trainer.py index 5a86ca16d2..e20bec6e79 100644 --- a/flair/trainers/sequence_tagger_trainer.py +++ b/flair/trainers/sequence_tagger_trainer.py @@ -35,9 +35,10 @@ def train(self, if self.model.tag_type in ['pos', 'upos']: evaluation_method = 'accuracy' log.info(evaluation_method) - loss_txt = init_output_file(base_path, 'loss.txt') + loss_txt = init_output_file(base_path, 'loss.tsv') with open(loss_txt, 'a') as f: - f.write('EPOCH\tTIMESTAMP\tTRAIN_LOSS\tTRAIN_METRICS\tDEV_LOSS\tDEV_METRICS\tTEST_LOSS\tTEST_METRICS\n') + f.write('EPOCH\tTIMESTAMP\tTRAIN_LOSS\t{}\tDEV_LOSS\t{}\tTEST_LOSS\t{}\n'.format( + Metric.tsv_header('TRAIN'), Metric.tsv_header('DEV'), Metric.tsv_header('TEST'))) weight_extractor = WeightExtractor(base_path) @@ -103,6 +104,7 @@ def train(self, log.info('-' * 100) + dev_score = dev_metric = None if not train_with_dev: dev_score, dev_metric = self.evaluate(self.corpus.dev, base_path, evaluation_method=evaluation_method, @@ -126,9 +128,9 @@ def train(self, 'TEST', test_metric.f_score(), test_metric.accuracy(), test_metric._tp, test_metric._fp, test_metric._fn, test_metric._tn)) with open(loss_txt, 'a') as f: - dev_metric_str = dev_metric.to_csv() if dev_metric is not None else '_' + dev_metric_str = dev_metric.to_tsv() if dev_metric is not None else Metric.to_empty_tsv() f.write('{}\t{:%H:%M:%S}\t{}\t{}\t{}\t{}\t{}\t{}\n'.format( - epoch, datetime.datetime.now(), '_', '_', '_', dev_metric_str, '_', test_metric.to_csv())) + epoch, datetime.datetime.now(), '_', Metric.to_empty_tsv(), '_', dev_metric_str, '_', test_metric.to_tsv())) # save if model is current best and we use dev data for model selection if save_model and not train_with_dev and dev_score == scheduler.best: diff --git a/flair/trainers/text_classification_trainer.py b/flair/trainers/text_classification_trainer.py index 9533985766..f220196bcf 100644 --- a/flair/trainers/text_classification_trainer.py +++ b/flair/trainers/text_classification_trainer.py @@ -9,7 +9,7 @@ from flair.data import Sentence, TaggedCorpus, Dictionary from flair.models.text_classification_model import TextClassifier from flair.training_utils import convert_labels_to_one_hot, calculate_micro_avg_metric, init_output_file, \ - clear_embeddings, calculate_class_metrics, WeightExtractor + clear_embeddings, calculate_class_metrics, WeightExtractor, Metric MICRO_AVG_METRIC = 'MICRO_AVG' @@ -52,9 +52,10 @@ def train(self, or not """ - loss_txt = init_output_file(base_path, 'loss.txt') + loss_txt = init_output_file(base_path, 'loss.tsv') with open(loss_txt, 'a') as f: - f.write('EPOCH\tTIMESTAMP\tTRAIN_LOSS\tTRAIN_METRICS\tDEV_LOSS\tDEV_METRICS\tTEST_LOSS\tTEST_METRICS\n') + f.write('EPOCH\tTIMESTAMP\tTRAIN_LOSS\t{}\tDEV_LOSS\t{}\tTEST_LOSS\t{}\n'.format( + Metric.tsv_header('TRAIN'), Metric.tsv_header('DEV'), Metric.tsv_header('TEST'))) weight_extractor = WeightExtractor(base_path) @@ -132,10 +133,10 @@ def train(self, 'DEV', self.corpus.dev, embeddings_in_memory, eval_mini_batch_size) with open(loss_txt, 'a') as f: - train_metric_str = train_metric.to_csv() if train_metric is not None else '_' - dev_metric_str = dev_metric.to_csv() if dev_metric is not None else '_' + train_metric_str = train_metric.to_tsv() if train_metric is not None else Metric.to_empty_tsv() + dev_metric_str = dev_metric.to_tsv() if dev_metric is not None else Metric.to_empty_tsv() f.write('{}\t{:%H:%M:%S}\t{}\t{}\t{}\t{}\t{}\t{}\n'.format( - epoch, datetime.datetime.now(), train_loss, train_metric_str, dev_loss, dev_metric_str, '_', '_')) + epoch, datetime.datetime.now(), train_loss, train_metric_str, dev_loss, dev_metric_str, '_', Metric.to_empty_tsv())) self.model.train() diff --git a/flair/training_utils.py b/flair/training_utils.py index 837175e04d..fbe70303fc 100644 --- a/flair/training_utils.py +++ b/flair/training_utils.py @@ -1,4 +1,5 @@ import random +import logging import os from collections import defaultdict from typing import List @@ -6,6 +7,9 @@ from functools import reduce +log = logging.getLogger(__name__) + + class Metric(object): def __init__(self, name): @@ -48,17 +52,28 @@ def accuracy(self): return (self._tp + self._tn) / (self._tp + self._tn + self._fp + self._fn) return 0.0 - def to_csv(self): - return '{},{},{},{},{},{},{},{}'.format( + def to_tsv(self): + return '{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}'.format( self._tp, self._tn, self._fp, self._fn, self.precision(), self.recall(), self.f_score(), self.accuracy()) + def print(self): + log.info(self) + + @staticmethod + def tsv_header(prefix=None): + if prefix: + return '{0}_TP\t{0}_TN\t{0}_FP\t{0}_FN\t{0}_PRECISION\t{0}_RECALL\t{0}_F-SCORE\t{0}_ACCURACY'.format(prefix) + + return 'TP\tTN\tFP\tFN\tPRECISION\tRECALL\tF-SCORE\tACCURACY' + + @staticmethod + def to_empty_tsv(): + return '_\t_\t_\t_\t_\t_\t_\t_' + def __str__(self): return '{0:<20}\tprecision: {1:.4f} - recall: {2:.4f} - accuracy: {3:.4f} - f1-score: {4:.4f}'.format( self.name, self.precision(), self.recall(), self.accuracy(), self.f_score()) - def print(self): - print(self) - class WeightExtractor(object): From 167c496ce2c614d6aeeedf5bc58e619b35c574e8 Mon Sep 17 00:00:00 2001 From: tabergma Date: Thu, 27 Sep 2018 09:09:25 +0200 Subject: [PATCH 026/113] GH-16: Replace print with log.info --- flair/__init__.py | 3 ++- flair/data.py | 27 +++++++++++++++++------ flair/data_fetcher.py | 7 ++++-- flair/trainers/language_model_trainer.py | 27 +++++++++++++---------- flair/trainers/sequence_tagger_trainer.py | 2 +- 5 files changed, 43 insertions(+), 23 deletions(-) diff --git a/flair/__init__.py b/flair/__init__.py index 37250c116b..c4a7b48757 100644 --- a/flair/__init__.py +++ b/flair/__init__.py @@ -8,4 +8,5 @@ FORMAT = '%(asctime)-15s %(message)s' -logging.basicConfig(level=logging.INFO, format=FORMAT, stream=sys.stdout) \ No newline at end of file +logging.basicConfig(level=logging.INFO, format=FORMAT, stream=sys.stdout) +logging.getLogger('flair').setLevel(logging.INFO) \ No newline at end of file diff --git a/flair/data.py b/flair/data.py index 4ca3953402..1bc71889d8 100644 --- a/flair/data.py +++ b/flair/data.py @@ -1,6 +1,7 @@ from typing import List, Dict, Union import torch +import logging from collections import Counter from collections import defaultdict @@ -10,6 +11,9 @@ from segtok.tokenizer import word_tokenizer +log = logging.getLogger(__name__) + + class Dictionary: """ This class holds a dictionary that maps strings to IDs, used to generate one-hot encodings of strings. @@ -601,14 +605,23 @@ def _print_statistics_for(sentences, name): classes_to_count = TaggedCorpus._get_classes_to_count(sentences) tokens_per_sentence = TaggedCorpus._get_tokens_per_sentence(sentences) - print(name) - print("total size: " + str(len(sentences))) + size_dict = {} for l, c in classes_to_count.items(): - print("size of class {}: {}".format(l, c)) - print("total # of tokens: " + str(sum(tokens_per_sentence))) - print("min # of tokens: " + str(min(tokens_per_sentence))) - print("max # of tokens: " + str(max(tokens_per_sentence))) - print("avg # of tokens: " + str(sum(tokens_per_sentence) / len(sentences))) + size_dict = { l: c } + size_dict['total'] = len(sentences) + + stats = { + 'dataset': name, + 'number_of_documents': size_dict, + 'number_of_tokens': { + 'total': sum(tokens_per_sentence), + 'min': min(tokens_per_sentence), + 'max': max(tokens_per_sentence), + 'avg': sum(tokens_per_sentence) / len(sentences) + } + } + + log.info(stats) @staticmethod def _get_tokens_per_sentence(sentences): diff --git a/flair/data_fetcher.py b/flair/data_fetcher.py index 35ddaf12eb..73e17f1225 100644 --- a/flair/data_fetcher.py +++ b/flair/data_fetcher.py @@ -1,9 +1,12 @@ from typing import List, Dict import re import os +import logging from enum import Enum -from flair.data import Sentence, TaggedCorpus, Token, Label +from flair.data import Sentence, TaggedCorpus, Token + +log = logging.getLogger(__name__) class NLPTask(Enum): @@ -43,7 +46,7 @@ def fetch_data(task: NLPTask) -> TaggedCorpus: """ data_folder = os.path.join('resources', 'tasks', str(task.value).lower()) - print("reading data from {}".format(data_folder)) + log.info("Reading data from {}".format(data_folder)) # the CoNLL 2000 task on chunking has three columns: text, pos and np (chunk) if task == NLPTask.CONLL_2000: diff --git a/flair/trainers/language_model_trainer.py b/flair/trainers/language_model_trainer.py index 8fa3129306..d3e2023597 100644 --- a/flair/trainers/language_model_trainer.py +++ b/flair/trainers/language_model_trainer.py @@ -1,7 +1,7 @@ import time, datetime import os import random - +import logging import math import torch from torch.autograd import Variable @@ -10,6 +10,10 @@ from flair.data import Dictionary from flair.models import LanguageModel + +log = logging.getLogger(__name__) + + class TextCorpus(object): def __init__(self, path, dictionary: Dictionary, forward: bool = True, character_level: bool = True): @@ -70,7 +74,6 @@ def charsplit(self, path: str, expand_vocab=False, forward=True, split_on_char=T else: chars = line.split() - # print(chars) tokens += len(chars) # Add chars to the dictionary @@ -185,7 +188,7 @@ def train(self, for split in range(1, max_epochs + 1): - print('Split %d' % split + '\t - ({:%H:%M:%S})'.format(datetime.datetime.now())) + log.info('Split %d' % split + '\t - ({:%H:%M:%S})'.format(datetime.datetime.now())) for group in optimizer.param_groups: learning_rate = group['lr'] @@ -193,7 +196,7 @@ def train(self, train_slice = self.corpus.get_next_train_slice() train_data = self._batchify(train_slice, mini_batch_size) - print('\t({:%H:%M:%S})'.format(datetime.datetime.now())) + log.info('\t({:%H:%M:%S})'.format(datetime.datetime.now())) # go into train mode self.model.train() @@ -237,7 +240,7 @@ def train(self, if batch % self.log_interval == 0 and batch > 0: cur_loss = total_loss.item() / self.log_interval elapsed = time.time() - start_time - print('| split {:3d} /{:3d} | {:5d}/{:5d} batches | ms/batch {:5.2f} | ' + log.info('| split {:3d} /{:3d} | {:5d}/{:5d} batches | ms/batch {:5.2f} | ' 'loss {:5.2f} | ppl {:8.2f}'.format( split, number_of_splits, batch, len(train_data) // sequence_length, elapsed * 1000 / self.log_interval, cur_loss, @@ -245,7 +248,7 @@ def train(self, total_loss = 0 start_time = time.time() - print('training done! \t({:%H:%M:%S})'.format(datetime.datetime.now())) + log.info('training done! \t({:%H:%M:%S})'.format(datetime.datetime.now())) ############################################################################### # TEST @@ -254,7 +257,7 @@ def train(self, val_loss = self.evaluate(val_data, mini_batch_size, sequence_length) scheduler.step(val_loss) - print('best loss so far {:5.2f}'.format(best_val_loss)) + log.info('best loss so far {:5.2f}'.format(best_val_loss)) # Save the model if the validation loss is the best we've seen so far. if val_loss < best_val_loss: @@ -264,7 +267,7 @@ def train(self, ############################################################################### # print info ############################################################################### - print('-' * 89) + log.info('-' * 89) local_split_number = split % number_of_splits if local_split_number == 0: local_split_number = number_of_splits @@ -281,12 +284,12 @@ def train(self, with open(loss_txt, "a") as myfile: myfile.write('%s\n' % summary) - print(summary) - print('-' * 89) + log.info(summary) + log.info('-' * 89) except KeyboardInterrupt: - print('-' * 89) - print('Exiting from training early') + log.info('-' * 89) + log.info('Exiting from training early') def evaluate(self, data_source, eval_batch_size, sequence_length): # Turn on evaluation mode which disables dropout. diff --git a/flair/trainers/sequence_tagger_trainer.py b/flair/trainers/sequence_tagger_trainer.py index e20bec6e79..39dca80a0e 100644 --- a/flair/trainers/sequence_tagger_trainer.py +++ b/flair/trainers/sequence_tagger_trainer.py @@ -33,7 +33,7 @@ def train(self, evaluation_method = 'F1' if self.model.tag_type in ['pos', 'upos']: evaluation_method = 'accuracy' - log.info(evaluation_method) + log.info('Evaluation method: {}'.format(evaluation_method)) loss_txt = init_output_file(base_path, 'loss.tsv') with open(loss_txt, 'a') as f: From c57557237d837cd4619e12224a6cd41f51f9c82d Mon Sep 17 00:00:00 2001 From: tabergma Date: Thu, 27 Sep 2018 09:10:58 +0200 Subject: [PATCH 027/113] GH-16: Update log level. --- flair/__init__.py | 2 +- flair/trainers/sequence_tagger_trainer.py | 8 ++++---- flair/trainers/text_classification_trainer.py | 8 ++++---- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/flair/__init__.py b/flair/__init__.py index c4a7b48757..2b74ad0143 100644 --- a/flair/__init__.py +++ b/flair/__init__.py @@ -8,5 +8,5 @@ FORMAT = '%(asctime)-15s %(message)s' -logging.basicConfig(level=logging.INFO, format=FORMAT, stream=sys.stdout) +logging.basicConfig(level=logging.WARNING, format=FORMAT, stream=sys.stdout) logging.getLogger('flair').setLevel(logging.INFO) \ No newline at end of file diff --git a/flair/trainers/sequence_tagger_trainer.py b/flair/trainers/sequence_tagger_trainer.py index 39dca80a0e..bdff9c4c94 100644 --- a/flair/trainers/sequence_tagger_trainer.py +++ b/flair/trainers/sequence_tagger_trainer.py @@ -140,11 +140,11 @@ def train(self, if save_model and train_with_dev: self.model.save(base_path + "/final-model.pt") except KeyboardInterrupt: - log.info('-' * 89) - log.info('Exiting from training early') - log.info('saving model') + log.info('-' * 100) + log.info('Exiting from training early.') + log.info('Saving model ...') self.model.save(base_path + "/final-model.pt") - log.info('done') + log.info('Done.') def evaluate(self, evaluation: List[Sentence], out_path=None, evaluation_method: str = 'F1', embeddings_in_memory: bool = True): diff --git a/flair/trainers/text_classification_trainer.py b/flair/trainers/text_classification_trainer.py index f220196bcf..66e222d018 100644 --- a/flair/trainers/text_classification_trainer.py +++ b/flair/trainers/text_classification_trainer.py @@ -174,13 +174,13 @@ def train(self, log.info('-' * 100) except KeyboardInterrupt: - log.info('-' * 89) - log.info('Exiting from training early') - log.info('saving model') + log.info('-' * 100) + log.info('Exiting from training early.') + log.info('Saving model ...') with open(base_path + "/final-model.pt", 'wb') as model_save_file: torch.save(self.model, model_save_file, pickle_protocol=4) model_save_file.close() - log.info('done') + log.info('Done.') def _calculate_evaluation_results_for(self, dataset_name, dataset, embeddings_in_memory, mini_batch_size): metrics, loss = self.evaluate(dataset, mini_batch_size=mini_batch_size, From b0a02b1556aad22be91ec224ce354648bbf5a67e Mon Sep 17 00:00:00 2001 From: aakbik Date: Tue, 18 Sep 2018 17:12:50 +0200 Subject: [PATCH 028/113] GH-99: split save_model into checkpoint and save_final_model for better saving options --- flair/models/sequence_tagger_model.py | 33 ++++-------- flair/trainers/sequence_tagger_trainer.py | 63 +++++++++++++---------- 2 files changed, 47 insertions(+), 49 deletions(-) diff --git a/flair/models/sequence_tagger_model.py b/flair/models/sequence_tagger_model.py index 2b0517efc4..e734682b34 100644 --- a/flair/models/sequence_tagger_model.py +++ b/flair/models/sequence_tagger_model.py @@ -426,29 +426,16 @@ def _forward_alg(self, feats, lens_): return alpha - def predict_scores(self, sentence: Sentence): - feats, tags = self.forward([sentence]) - feats = feats[0] - tags = tags[0] - if self.use_crf: - score, tag_seq = self.viterbi_decode(feats) - else: - score, tag_seq = torch.max(feats, 1) - tag_seq = list(tag_seq.cpu().data) - - return score, tag_seq - - def predict_old(self, sentence: Sentence) -> Sentence: - - score, tag_seq = self.predict_scores(sentence) - predicted_id = tag_seq - for (token, pred_id) in zip(sentence.tokens, predicted_id): - token: Token = token - # get the predicted tag - predicted_tag = self.tag_dictionary.get_item_for_index(pred_id) - token.add_tag(self.tag_type, predicted_tag) - - return sentence + # def predict_scores(self, sentence: Sentence): + # feats, tags = self.forward([sentence]) + # feats = feats[0] + # if self.use_crf: + # score, tag_seq = self.viterbi_decode(feats) + # else: + # score, tag_seq = torch.max(feats, 1) + # tag_seq = list(tag_seq.cpu().data) + # + # return score, tag_seq def predict(self, sentences: Union[List[Sentence], Sentence], mini_batch_size=32) -> List[Sentence]: diff --git a/flair/trainers/sequence_tagger_trainer.py b/flair/trainers/sequence_tagger_trainer.py index 7fa766c3a7..43240af205 100644 --- a/flair/trainers/sequence_tagger_trainer.py +++ b/flair/trainers/sequence_tagger_trainer.py @@ -1,15 +1,13 @@ -from subprocess import run, PIPE +import copy from typing import List import datetime import os import random -import re import sys import torch from torch.optim.lr_scheduler import ReduceLROnPlateau -from flair.file_utils import cached_path from flair.models.sequence_tagger_model import SequenceTagger from flair.data import Sentence, Token, TaggedCorpus from flair.training_utils import Metric @@ -28,9 +26,11 @@ def train(self, max_epochs: int = 100, anneal_factor: float = 0.5, patience: int = 2, - save_model: bool = True, + train_with_dev: bool = False, embeddings_in_memory: bool = True, - train_with_dev: bool = False): + checkpoint: bool = False, + save_final_model: bool = True, + ): evaluation_method = 'F1' if self.model.tag_type in ['pos', 'upos']: evaluation_method = 'accuracy' @@ -53,6 +53,9 @@ def train(self, if train_with_dev: train_data.extend(self.corpus.dev) + # variable to store best model during training + best_model = None + # At any point you can hit Ctrl + C to break out of training early. try: @@ -116,15 +119,12 @@ def train(self, evaluation_method=evaluation_method, embeddings_in_memory=embeddings_in_memory) - # switch back to train mode - self.model.train() - # anneal against train loss if training with dev, otherwise anneal against dev score scheduler.step(current_loss) if train_with_dev else scheduler.step(dev_score) summary = '%d' % epoch + '\t({:%H:%M:%S})'.format(datetime.datetime.now()) \ + '\t%f\t%d\t%f\tDEV %d\t' % ( - current_loss, scheduler.num_bad_epochs, learning_rate, dev_fp) + dev_result + current_loss, scheduler.num_bad_epochs, learning_rate, dev_fp) + dev_result summary = summary.replace('\n', '') summary += '\tTEST \t%d\t' % test_fp + test_result @@ -133,12 +133,19 @@ def train(self, loss_file.write('%s\n' % summary) loss_file.close() - # save if model is current best and we use dev data for model selection - if save_model and not train_with_dev and dev_score == scheduler.best: - self.model.save(base_path + "/best-model.pt") + if checkpoint: + self.model.save(base_path + "/checkpoint.pt") + + # if we use dev data, remember best model based on dev evaluation score + if not train_with_dev and dev_score == scheduler.best: + best_model = copy.deepcopy(self.model) # if we do not use dev data for model selection, save final model - if save_model and train_with_dev: self.model.save(base_path + "/final-model.pt") + if save_final_model: + if train_with_dev: + self.model.save(base_path + "/final-model.pt") + else: + best_model.save(base_path + "/best-model.pt") except KeyboardInterrupt: print('-' * 89) @@ -148,12 +155,12 @@ def train(self, print('done') def evaluate(self, evaluation: List[Sentence], out_path=None, evaluation_method: str = 'F1', + eval_batch_size: int = 32, embeddings_in_memory: bool = True): batch_no: int = 0 - mini_batch_size = 32 - batches = [evaluation[x:x + mini_batch_size] for x in - range(0, len(evaluation), mini_batch_size)] + batches = [evaluation[x:x + eval_batch_size] for x in + range(0, len(evaluation), eval_batch_size)] metric = Metric('') @@ -162,24 +169,28 @@ def evaluate(self, evaluation: List[Sentence], out_path=None, evaluation_method: for batch in batches: batch_no += 1 - self.model.embeddings.embed(batch) + score, tag_seq = self.model._predict_scores_batch(batch) + predicted_id = tag_seq + all_tokens = [] + for sentence in batch: + all_tokens.extend(sentence.tokens) + + for (token, pred_id) in zip(all_tokens, predicted_id): + token: Token = token + # get the predicted tag + predicted_tag = self.model.tag_dictionary.get_item_for_index(pred_id) + token.add_tag('predicted', predicted_tag) for sentence in batch: sentence: Sentence = sentence - # Step 3. Run our forward pass. - score, tag_seq = self.model.predict_scores(sentence) - # add predicted tags for (token, pred_id) in zip(sentence.tokens, tag_seq): - token: Token = token - # get the predicted tag - predicted_tag = self.model.tag_dictionary.get_item_for_index(pred_id) - token.add_tag('predicted', predicted_tag) - # append both to file for evaluation - eval_line = token.text + ' ' + token.get_tag(self.model.tag_type) + ' ' + predicted_tag + "\n" + eval_line = '{} {} {}\n'.format(token.text, + token.get_tag(self.model.tag_type), + token.get_tag('predicted')) lines.append(eval_line) lines.append('\n') From fa3fe2d246d41126171c9b5644a8bab0b070f751 Mon Sep 17 00:00:00 2001 From: aakbik Date: Mon, 24 Sep 2018 17:50:07 +0200 Subject: [PATCH 029/113] GH-38: add confidence for tagger predictions | GH-37: small refactorings for speed improvements --- flair/data.py | 94 +++++------ flair/embeddings.py | 53 ++++++- flair/models/sequence_tagger_model.py | 182 +++++++++++----------- flair/trainers/sequence_tagger_trainer.py | 77 +++++---- flair/training_utils.py | 4 +- tests/test_data.py | 16 +- 6 files changed, 245 insertions(+), 181 deletions(-) diff --git a/flair/data.py b/flair/data.py index 4ca3953402..7525035fcc 100644 --- a/flair/data.py +++ b/flair/data.py @@ -101,6 +101,7 @@ class Label: def __init__(self, name: str, confidence: float = 1.0): self.name = name self.confidence = confidence + super().__init__() @property def name(self): @@ -108,7 +109,7 @@ def name(self): @name.setter def name(self, name): - if not name: + if not name and name != '': raise ValueError('Incorrect label name provided. Label name needs to be set.') else: self._name = name @@ -124,8 +125,8 @@ def confidence(self, confidence): else: self._confidence = 1.0 - def __str__(self): - return "{} ({})".format(self._name, self._confidence) + # def __str__(self): + # return "{} ({})".format(self._name, self._confidence) def __repr__(self): return "{} ({})".format(self._name, self._confidence) @@ -150,23 +151,24 @@ def __init__(self, self.sentence: Sentence = None self._embeddings: Dict = {} - self.tags: Dict[str, str] = {} + self.tags: Dict[str, Label] = {} - def add_tag(self, tag_type: str, tag_value: str): - self.tags[tag_type] = tag_value + def add_tag(self, tag_type: str, tag_value: str, confidence=1.0): + tag = Label(tag_value, confidence) + self.tags[tag_type] = tag - def get_tag(self, tag_type: str) -> str: + def get_tag(self, tag_type: str) -> Label: if tag_type in self.tags: return self.tags[tag_type] - return '' + return Label('') def get_head(self): return self.sentence.get_token(self.head_id) def __str__(self) -> str: - return 'Token: %d %s' % (self.idx, self.text) + return 'Token: %d %s' % (self.idx, self.text) if self.idx is not None else 'Token: %s' % (self.text) def __repr__(self) -> str: - return 'Token: %d %s' % (self.idx, self.text) + return 'Token: %d %s' % (self.idx, self.text) if self.idx is not None else 'Token: %s' % (self.text) def set_embedding(self, name: str, vector: torch.autograd.Variable): self._embeddings[name] = vector.cpu() @@ -174,11 +176,9 @@ def set_embedding(self, name: str, vector: torch.autograd.Variable): def clear_embeddings(self): self._embeddings: Dict = {} - def get_embedding(self) -> torch.autograd.Variable: + def get_embedding(self) -> torch.FloatTensor: - embeddings = [] - for embed in sorted(self._embeddings.keys()): - embeddings.append(self._embeddings[embed]) + embeddings = [self._embeddings[embed] for embed in sorted(self._embeddings.keys())] if embeddings: return torch.cat(embeddings, dim=0) @@ -291,29 +291,30 @@ def get_spans(self, tag_type: str) -> List[Span]: tags = defaultdict(lambda: 0.0) - previous_tag = '' + previous_tag_value: str = 'O' for token in self: - tag = token.get_tag(tag_type) + tag: Label = token.get_tag(tag_type) + tag_value = tag.name # non-set tags are OUT tags - if len(tag) < 2: tag = 'O-' + if len(tag_value) < 2: tag_value = 'O-' # anything that is not a BIOES tag is a SINGLE tag - if tag[0:2] not in ['B-', 'I-', 'O-', 'E-', 'S-']: - tag = 'S-' + tag + if tag_value[0:2] not in ['B-', 'I-', 'O-', 'E-', 'S-']: + tag_value = 'S-' + tag_value # anything that is not OUT is IN in_span = False - if tag[0:2] not in ['O-']: + if tag_value[0:2] not in ['O-']: in_span = True # single and begin tags start a new span starts_new_span = False - if tag[0:2] in ['B-', 'S-']: + if tag_value[0:2] in ['B-', 'S-']: starts_new_span = True - if previous_tag[0:2] in ['S-'] and previous_tag[2:] != tag[2:] and in_span: + if previous_tag_value[0:2] in ['S-'] and previous_tag_value[2:] != tag_value[2:] and in_span: starts_new_span = True if (starts_new_span or not in_span) and len(current_span) > 0: @@ -324,13 +325,13 @@ def get_spans(self, tag_type: str) -> List[Span]: if in_span: current_span.append(token) weight = 1.1 if starts_new_span else 1.0 - tags[tag[2:]] += weight + tags[tag_value[2:]] += weight # remember previous tag - previous_tag = tag + previous_tag_value = tag_value if len(current_span) > 0: - spans.append(Span(current_span, sorted(tags.items(), key=lambda k_v: k_v[1], reverse=True)[0][0])) + spans.append(Span(current_span, sorted(tags.items(), key=lambda k_v: k_v[1], reverse=True)[0][0])) return spans @@ -382,13 +383,13 @@ def to_tagged_string(self, main_tag=None) -> str: for token in self.tokens: list.append(token.text) - tags = [] + tags: List[str] = [] for tag_type in token.tags.keys(): if main_tag is not None and main_tag != tag_type: continue - if token.get_tag(tag_type) == '' or token.get_tag(tag_type) == 'O': continue - tags.append(token.get_tag(tag_type)) + if token.get_tag(tag_type).name == '' or token.get_tag(tag_type).name == 'O': continue + tags.append(token.get_tag(tag_type).name) all_tags = '<' + '/'.join(tags) + '>' if all_tags != '<>': list.append(all_tags) @@ -406,7 +407,7 @@ def to_plain_string(self): def convert_tag_scheme(self, tag_type: str = 'ner', target_scheme: str = 'iob'): - tags: List[str] = [] + tags: List[Label] = [] for token in self.tokens: token: Token = token tags.append(token.get_tag(tag_type)) @@ -514,7 +515,7 @@ def make_tag_dictionary(self, tag_type: str) -> Dictionary: for sentence in self.get_all_sentences(): for token in sentence.tokens: token: Token = token - tag_dictionary.add_item(token.get_tag(tag_type)) + tag_dictionary.add_item(token.get_tag(tag_type).name) tag_dictionary.add_item('') tag_dictionary.add_item('') return tag_dictionary @@ -632,19 +633,20 @@ def iob2(tags): Tags in IOB1 format are converted to IOB2. """ for i, tag in enumerate(tags): - if tag == 'O': + # print(tag) + if tag.name == 'O': continue - split = tag.split('-') + split = tag.name.split('-') if len(split) != 2 or split[0] not in ['I', 'B']: return False if split[0] == 'B': continue - elif i == 0 or tags[i - 1] == 'O': # conversion IOB1 to IOB2 - tags[i] = 'B' + tag[1:] - elif tags[i - 1][1:] == tag[1:]: + elif i == 0 or tags[i - 1].name == 'O': # conversion IOB1 to IOB2 + tags[i].name = 'B' + tag.name[1:] + elif tags[i - 1].name[1:] == tag.name[1:]: continue else: # conversion IOB1 to IOB2 - tags[i] = 'B' + tag[1:] + tags[i].name = 'B' + tag.name[1:] return True @@ -654,20 +656,20 @@ def iob_iobes(tags): """ new_tags = [] for i, tag in enumerate(tags): - if tag == 'O': - new_tags.append(tag) - elif tag.split('-')[0] == 'B': + if tag.name == 'O': + new_tags.append(tag.name) + elif tag.name.split('-')[0] == 'B': if i + 1 != len(tags) and \ - tags[i + 1].split('-')[0] == 'I': - new_tags.append(tag) + tags[i + 1].name.split('-')[0] == 'I': + new_tags.append(tag.name) else: - new_tags.append(tag.replace('B-', 'S-')) - elif tag.split('-')[0] == 'I': + new_tags.append(tag.name.replace('B-', 'S-')) + elif tag.name.split('-')[0] == 'I': if i + 1 < len(tags) and \ - tags[i + 1].split('-')[0] == 'I': - new_tags.append(tag) + tags[i + 1].name.split('-')[0] == 'I': + new_tags.append(tag.name) else: - new_tags.append(tag.replace('I-', 'E-')) + new_tags.append(tag.name.replace('I-', 'E-')) else: raise Exception('Invalid IOB format!') return new_tags diff --git a/flair/embeddings.py b/flair/embeddings.py index 8750faf6cf..7bff1cc7ac 100644 --- a/flair/embeddings.py +++ b/flair/embeddings.py @@ -1,7 +1,7 @@ import os import re from abc import abstractmethod -from typing import List, Union +from typing import List, Union, Dict import gensim import numpy as np @@ -100,7 +100,7 @@ def __init__(self, embeddings: List[TokenEmbeddings], detach: bool = True): self.name = 'Stack' self.static_embeddings = True - self.__embedding_type: int = embeddings[0].embedding_type + self.__embedding_type: str = embeddings[0].embedding_type self.__embedding_length: int = 0 for embedding in embeddings: @@ -221,6 +221,55 @@ def _add_embeddings_internal(self, sentences: List[Sentence]) -> List[Sentence]: return sentences +class MemoryEmbeddings(TokenEmbeddings): + + def __init__(self, tag_type: str, tag_dictionary: Dictionary): + + self.name = "memory" + self.static_embeddings = False + self.tag_type: str = tag_type + self.tag_dictionary: Dictionary = tag_dictionary + self.__embedding_length: int = len(tag_dictionary) + + self.memory: Dict[str:List] = {} + + super().__init__() + + @property + def embedding_length(self) -> int: + return self.__embedding_length + + def train(self, mode=True): + super().train(mode=mode) + if mode: + self.memory: Dict[str:List] = {} + + def update_embedding(self, text: str, tag: str): + self.memory[text][self.tag_dictionary.get_idx_for_item(tag)] += 1 + + def _add_embeddings_internal(self, sentences: List[Sentence]) -> List[Sentence]: + + for i, sentence in enumerate(sentences): + + for token, token_idx in zip(sentence.tokens, range(len(sentence.tokens))): + token: Token = token + + if token.text not in self.memory: + self.memory[token.text] = [0] * self.__embedding_length + + word_embedding = torch.FloatTensor(self.memory[token.text]) + import torch.nn.functional as F + word_embedding = F.normalize(word_embedding, p=2, dim=0) + + token.set_embedding(self.name, word_embedding) + + # add label if in training mode + if self.training: + self.update_embedding(token.text, token.get_tag(self.tag_type).name) + + return sentences + + class CharacterEmbeddings(TokenEmbeddings): """Character embeddings of words, as proposed in Lample et al., 2016.""" diff --git a/flair/models/sequence_tagger_model.py b/flair/models/sequence_tagger_model.py index e734682b34..0e1d70dab3 100644 --- a/flair/models/sequence_tagger_model.py +++ b/flair/models/sequence_tagger_model.py @@ -4,7 +4,6 @@ import torch.nn import flair.nn import torch -import numpy as np import flair.embeddings from flair.data import Dictionary, Sentence, Token @@ -14,7 +13,6 @@ from flair.training_utils import clear_embeddings - START_TAG: str = '' STOP_TAG: str = '' @@ -112,13 +110,13 @@ def __init__(self, if self.nlayers == 1: self.rnn = getattr(torch.nn, self.rnn_type)(rnn_input_dim, hidden_size, - num_layers=self.nlayers, - bidirectional=True) + num_layers=self.nlayers, + bidirectional=True) else: self.rnn = getattr(torch.nn, self.rnn_type)(rnn_input_dim, hidden_size, - num_layers=self.nlayers, - dropout=0.5, - bidirectional=True) + num_layers=self.nlayers, + dropout=0.5, + bidirectional=True) # final linear map to tag space if self.use_rnn: @@ -170,7 +168,7 @@ def load_from_file(cls, model_file): model = model.cuda() return model - def forward(self, sentences: List[Sentence]) -> Tuple[List, List]: + def forward(self, sentences: List[Sentence]): self.zero_grad() @@ -180,42 +178,31 @@ def forward(self, sentences: List[Sentence]) -> Tuple[List, List]: self.embeddings.embed(sentences) - all_sentence_tensors = [] - lengths: List[int] = [] + lengths: List[int] = [len(sentence.tokens) for sentence in sentences] tag_list: List = [] - padding = torch.FloatTensor(np.zeros(self.embeddings.embedding_length, dtype='float')).unsqueeze(0) - - for sentence in sentences: - - # get the tags in this sentence - tag_idx: List[int] = [] - - lengths.append(len(sentence.tokens)) - - word_embeddings = [] + # initialize zero-padded word embeddings tensor + sentence_tensor = torch.zeros([len(sentences), + longest_token_sequence_in_batch, + self.embeddings.embedding_length], + dtype=torch.float) - for token in sentence: - # get the tag - tag_idx.append(self.tag_dictionary.get_idx_for_item(token.get_tag(self.tag_type))) - # get the word embeddings - word_embeddings.append(token.get_embedding().unsqueeze(0)) + for s_id, sentence in enumerate(sentences): - # pad shorter sentences out - for add in range(longest_token_sequence_in_batch - len(sentence.tokens)): - word_embeddings.append(padding) - - word_embeddings_tensor = torch.cat(word_embeddings, 0) + # fill values with word embeddings + sentence_tensor[s_id][:len(sentence)] = torch.cat([token.get_embedding().unsqueeze(0) + for token in sentence], 0) + # get the tags in this sentence + tag_idx: List[int] = [self.tag_dictionary.get_idx_for_item(token.get_tag(self.tag_type).name) + for token in sentence] + # add tags as tensor if torch.cuda.is_available(): tag_list.append(torch.cuda.LongTensor(tag_idx)) else: tag_list.append(torch.LongTensor(tag_idx)) - all_sentence_tensors.append(word_embeddings_tensor.unsqueeze(1)) - - # padded tensor for entire batch - sentence_tensor = torch.cat(all_sentence_tensors, 1) + sentence_tensor = sentence_tensor.transpose_(0, 1) if torch.cuda.is_available(): sentence_tensor = sentence_tensor.cuda() @@ -242,27 +229,18 @@ def forward(self, sentences: List[Sentence]) -> Tuple[List, List]: features = self.linear(sentence_tensor) - predictions_list = [] - for sentence_no, length in enumerate(lengths): - sentence_predictions = [] - for token_no in range(length): - sentence_predictions.append( - features[token_no, sentence_no, :].unsqueeze(0) - ) - predictions_list.append(torch.cat(sentence_predictions, 0)) - - return predictions_list, tag_list + return features.transpose_(0, 1), lengths, tag_list def _score_sentence(self, feats, tags, lens_): if torch.cuda.is_available(): start = torch.cuda.LongTensor([ - self.tag_dictionary.get_idx_for_item(START_TAG) + self.tag_dictionary.get_idx_for_item(START_TAG) ]) start = start[None, :].repeat(tags.shape[0], 1) stop = torch.cuda.LongTensor([ - self.tag_dictionary.get_idx_for_item(STOP_TAG) + self.tag_dictionary.get_idx_for_item(STOP_TAG) ]) stop = stop[None, :].repeat(tags.shape[0], 1) @@ -272,14 +250,14 @@ def _score_sentence(self, feats, tags, lens_): torch.cat([tags, stop], 1) else: start = torch.LongTensor([ - self.tag_dictionary.get_idx_for_item(START_TAG) + self.tag_dictionary.get_idx_for_item(START_TAG) ]) start = start[None, :].repeat(tags.shape[0], 1) stop = torch.LongTensor([ - self.tag_dictionary.get_idx_for_item(STOP_TAG) + self.tag_dictionary.get_idx_for_item(STOP_TAG) ]) - + stop = stop[None, :].repeat(tags.shape[0], 1) pad_start_tags = torch.cat([start, tags], 1) @@ -308,52 +286,66 @@ def _score_sentence(self, feats, tags, lens_): def viterbi_decode(self, feats): backpointers = [] + backscores = [] + backscores_2 = [] + init_vvars = torch.Tensor(1, self.tagset_size).fill_(-10000.) init_vvars[0][self.tag_dictionary.get_idx_for_item(START_TAG)] = 0 forward_var = autograd.Variable(init_vvars) if torch.cuda.is_available(): forward_var = forward_var.cuda() + import torch.nn.functional as F for feat in feats: next_tag_var = forward_var.view(1, -1).expand(self.tagset_size, self.tagset_size) + self.transitions _, bptrs_t = torch.max(next_tag_var, dim=1) bptrs_t = bptrs_t.squeeze().data.cpu().numpy() + backscores_2.append(next_tag_var) next_tag_var = next_tag_var.data.cpu().numpy() viterbivars_t = next_tag_var[range(len(bptrs_t)), bptrs_t] viterbivars_t = autograd.Variable(torch.FloatTensor(viterbivars_t)) if torch.cuda.is_available(): viterbivars_t = viterbivars_t.cuda() forward_var = viterbivars_t + feat + backscores.append(forward_var) backpointers.append(bptrs_t) terminal_var = forward_var + self.transitions[self.tag_dictionary.get_idx_for_item(STOP_TAG)] terminal_var.data[self.tag_dictionary.get_idx_for_item(STOP_TAG)] = -10000. terminal_var.data[self.tag_dictionary.get_idx_for_item(START_TAG)] = -10000. best_tag_id = argmax(terminal_var.unsqueeze(0)) - path_score = terminal_var[best_tag_id] + best_path = [best_tag_id] + for bptrs_t in reversed(backpointers): best_tag_id = bptrs_t[best_tag_id] best_path.append(best_tag_id) + + best_scores = [] + for backscore in backscores: + softmax = F.softmax(backscore, dim=0) + _, idx = torch.max(backscore, 0) + prediction = idx.item() + best_scores.append(softmax[prediction].item()) + start = best_path.pop() assert start == self.tag_dictionary.get_idx_for_item(START_TAG) best_path.reverse() - return path_score, best_path - - def neg_log_likelihood(self, sentences: List[Sentence], tag_type: str): - feats, tags = self.forward(sentences) + return best_scores, best_path - if torch.cuda.is_available(): - feats, lens_ = pad_tensors(feats, torch.cuda.FloatTensor) - tags, _ = pad_tensors(tags, torch.cuda.LongTensor) - else: - feats, lens_ = pad_tensors(feats) - tags, _ = pad_tensors(tags, torch.LongTensor) + def neg_log_likelihood(self, sentences: List[Sentence]): + features, lengths, tags = self.forward(sentences) if self.use_crf: - forward_score = self._forward_alg(feats, lens_) - gold_score = self._score_sentence(feats, tags, lens_) + # pad tags if using batch-CRF decoder + if torch.cuda.is_available(): + tags, _ = pad_tensors(tags, torch.cuda.LongTensor) + else: + tags, _ = pad_tensors(tags, torch.LongTensor) + + forward_score = self._forward_alg(features, lengths) + gold_score = self._score_sentence(features, tags, lengths) score = forward_score - gold_score @@ -362,9 +354,8 @@ def neg_log_likelihood(self, sentences: List[Sentence], tag_type: str): else: score = 0 - for i in range(len(feats)): - sentence_feats = feats[i] - sentence_tags = tags[i] + for sentence_feats, sentence_tags, sentence_length in zip(features, tags, lengths): + sentence_feats = sentence_feats[:sentence_length] if torch.cuda.is_available(): tag_tensor = autograd.Variable(torch.cuda.LongTensor(sentence_tags)) @@ -391,13 +382,12 @@ def _forward_alg(self, feats, lens_): forward_var = forward_var.cuda() transitions = self.transitions.view( - 1, + 1, self.transitions.shape[0], self.transitions.shape[1], ).repeat(feats.shape[0], 1, 1) for i in range(feats.shape[1]): - emit_score = feats[:, i, :] tag_var = \ @@ -408,7 +398,7 @@ def _forward_alg(self, feats, lens_): max_tag_var, _ = torch.max(tag_var, dim=2) tag_var = tag_var - \ - max_tag_var[:, :, None].repeat(1, 1, transitions.shape[2]) + max_tag_var[:, :, None].repeat(1, 1, transitions.shape[2]) agg_ = torch.log(torch.sum(torch.exp(tag_var), dim=2)) @@ -420,23 +410,13 @@ def _forward_alg(self, feats, lens_): forward_var = forward_var[range(forward_var.shape[0]), lens_, :] terminal_var = forward_var + \ - self.transitions[self.tag_dictionary.get_idx_for_item(STOP_TAG)][None, :].repeat(forward_var.shape[0], 1) + self.transitions[self.tag_dictionary.get_idx_for_item(STOP_TAG)][None, :].repeat( + forward_var.shape[0], 1) alpha = log_sum_exp_batch(terminal_var) return alpha - # def predict_scores(self, sentence: Sentence): - # feats, tags = self.forward([sentence]) - # feats = feats[0] - # if self.use_crf: - # score, tag_seq = self.viterbi_decode(feats) - # else: - # score, tag_seq = torch.max(feats, 1) - # tag_seq = list(tag_seq.cpu().data) - # - # return score, tag_seq - def predict(self, sentences: Union[List[Sentence], Sentence], mini_batch_size=32) -> List[Sentence]: if type(sentences) is Sentence: @@ -449,38 +429,50 @@ def predict(self, sentences: Union[List[Sentence], Sentence], mini_batch_size=32 batches = [sentences[x:x + mini_batch_size] for x in range(0, len(sentences), mini_batch_size)] for batch in batches: - score, tag_seq = self._predict_scores_batch(batch) - predicted_id = tag_seq + scores, predicted_ids = self._predict_scores_batch(batch) all_tokens = [] for sentence in batch: all_tokens.extend(sentence.tokens) - for (token, pred_id) in zip(all_tokens, predicted_id): + for (token, score, predicted_id) in zip(all_tokens, scores, predicted_ids): token: Token = token # get the predicted tag - predicted_tag = self.tag_dictionary.get_item_for_index(pred_id) - token.add_tag(self.tag_type, predicted_tag) + predicted_tag = self.tag_dictionary.get_item_for_index(predicted_id) + token.add_tag(self.tag_type, predicted_tag, score) return sentences def _predict_scores_batch(self, sentences: List[Sentence]): - all_feats, tags = self.forward(sentences) + feature, lengths, tags = self.forward(sentences) - overall_score = 0 + all_confidences = [] all_tags_seqs = [] - for feats in all_feats: - # viterbi to get tag_seq + for feats, length in zip(feature, lengths): + if self.use_crf: - score, tag_seq = self.viterbi_decode(feats) + confidences, tag_seq = self.viterbi_decode(feats[:length]) else: - score, tag_seq = torch.max(feats, 1) - tag_seq = list(tag_seq.cpu().data) + import torch.nn.functional as F + + tag_seq = [] + confidences = [] + for backscore in feats[:length]: + softmax = F.softmax(backscore, dim=0) + _, idx = torch.max(backscore, 0) + prediction = idx.item() + tag_seq.append(prediction) + confidences.append(softmax[prediction].item()) + + # softmax = F.softmax(feats[:length], dim=0) + # confidences, tag_seq = torch.max(F.normalize(feats[:length], p=2, dim=1), 1) + + # tag_seq = list(tag_seq.cpu().data) - # overall_score += score all_tags_seqs.extend(tag_seq) + all_confidences.extend(confidences) - return overall_score, all_tags_seqs + return all_confidences, all_tags_seqs @staticmethod def load(model: str): @@ -567,4 +559,4 @@ def load(model: str): if model_file is not None: tagger: SequenceTagger = SequenceTagger.load_from_file(model_file) - return tagger \ No newline at end of file + return tagger diff --git a/flair/trainers/sequence_tagger_trainer.py b/flair/trainers/sequence_tagger_trainer.py index 43240af205..6ed976abff 100644 --- a/flair/trainers/sequence_tagger_trainer.py +++ b/flair/trainers/sequence_tagger_trainer.py @@ -8,8 +8,9 @@ import torch from torch.optim.lr_scheduler import ReduceLROnPlateau +from flair.embeddings import MemoryEmbeddings from flair.models.sequence_tagger_model import SequenceTagger -from flair.data import Sentence, Token, TaggedCorpus +from flair.data import Sentence, Token, TaggedCorpus, Label from flair.training_utils import Metric @@ -25,16 +26,22 @@ def train(self, mini_batch_size: int = 32, max_epochs: int = 100, anneal_factor: float = 0.5, - patience: int = 2, + patience: int = 3, train_with_dev: bool = False, embeddings_in_memory: bool = True, checkpoint: bool = False, save_final_model: bool = True, + memory: MemoryEmbeddings = None, ): evaluation_method = 'F1' if self.model.tag_type in ['pos', 'upos']: evaluation_method = 'accuracy' - print(evaluation_method) + print('evaluation method: {}'.format(evaluation_method)) + + # if memory is not None, set as field and eval batch size to 1 + self.memory = memory + eval_batch_size = 1 if self.memory else mini_batch_size + print('evaluation eval_batch_size: {}'.format(eval_batch_size)) os.makedirs(base_path, exist_ok=True) @@ -46,16 +53,12 @@ def train(self, anneal_mode = 'min' if train_with_dev else 'max' scheduler: ReduceLROnPlateau = ReduceLROnPlateau(optimizer, factor=anneal_factor, patience=patience, mode=anneal_mode) - train_data = self.corpus.train # if training also uses dev data, include in training set if train_with_dev: train_data.extend(self.corpus.dev) - # variable to store best model during training - best_model = None - # At any point you can hit Ctrl + C to break out of training early. try: @@ -66,6 +69,10 @@ def train(self, for group in optimizer.param_groups: learning_rate = group['lr'] + if learning_rate < 0.001: + print('learning rate too small - quitting training!') + break + if not self.test_mode: random.shuffle(train_data) batches = [train_data[x:x + mini_batch_size] for x in range(0, len(train_data), mini_batch_size)] @@ -84,7 +91,7 @@ def train(self, optimizer.zero_grad() # Step 4. Compute the loss, gradients, and update the parameters by calling optimizer.step() - loss = self.model.neg_log_likelihood(batch, self.model.tag_type) + loss = self.model.neg_log_likelihood(batch) current_loss += loss.item() @@ -105,47 +112,43 @@ def train(self, # switch to eval mode self.model.eval() + # if set, make a model checkpoint before evaluation + if checkpoint: + self.model.save(base_path + "/checkpoint.pt") + if not train_with_dev: print('.. evaluating... dev... ') dev_score, dev_fp, dev_result = self.evaluate(self.corpus.dev, base_path, evaluation_method=evaluation_method, - embeddings_in_memory=embeddings_in_memory) + embeddings_in_memory=embeddings_in_memory, + eval_batch_size=eval_batch_size, + ) else: - dev_fp = 0 dev_result = '_' print('test... ') test_score, test_fp, test_result = self.evaluate(self.corpus.test, base_path, evaluation_method=evaluation_method, - embeddings_in_memory=embeddings_in_memory) + embeddings_in_memory=embeddings_in_memory, + eval_batch_size=eval_batch_size, + ) # anneal against train loss if training with dev, otherwise anneal against dev score scheduler.step(current_loss) if train_with_dev else scheduler.step(dev_score) - summary = '%d' % epoch + '\t({:%H:%M:%S})'.format(datetime.datetime.now()) \ - + '\t%f\t%d\t%f\tDEV %d\t' % ( - current_loss, scheduler.num_bad_epochs, learning_rate, dev_fp) + dev_result - summary = summary.replace('\n', '') - summary += '\tTEST \t%d\t' % test_fp + test_result + summary = '{} ({:%H:%M:%S})\t{}\t{}\t{} DEV {} TEST {}'.format(epoch, datetime.datetime.now(), + current_loss, scheduler.num_bad_epochs, + learning_rate, dev_result, test_result) print(summary) with open(loss_txt, "a") as loss_file: loss_file.write('%s\n' % summary) loss_file.close() - if checkpoint: - self.model.save(base_path + "/checkpoint.pt") - - # if we use dev data, remember best model based on dev evaluation score - if not train_with_dev and dev_score == scheduler.best: - best_model = copy.deepcopy(self.model) - # if we do not use dev data for model selection, save final model if save_final_model: if train_with_dev: self.model.save(base_path + "/final-model.pt") - else: - best_model.save(base_path + "/best-model.pt") except KeyboardInterrupt: print('-' * 89) @@ -169,28 +172,36 @@ def evaluate(self, evaluation: List[Sentence], out_path=None, evaluation_method: for batch in batches: batch_no += 1 - score, tag_seq = self.model._predict_scores_batch(batch) - predicted_id = tag_seq + scores, tag_seq = self.model._predict_scores_batch(batch) + predicted_ids = tag_seq all_tokens = [] for sentence in batch: all_tokens.extend(sentence.tokens) - for (token, pred_id) in zip(all_tokens, predicted_id): + for (token, score, predicted_id) in zip(all_tokens, scores, predicted_ids): token: Token = token # get the predicted tag - predicted_tag = self.model.tag_dictionary.get_item_for_index(pred_id) - token.add_tag('predicted', predicted_tag) + predicted_tag = self.model.tag_dictionary.get_item_for_index(predicted_id) + token.add_tag('predicted', predicted_tag, score) for sentence in batch: sentence: Sentence = sentence # add predicted tags - for (token, pred_id) in zip(sentence.tokens, tag_seq): + for token in sentence.tokens: + + predicted_tag: Label = token.get_tag('predicted') + # append both to file for evaluation eval_line = '{} {} {}\n'.format(token.text, - token.get_tag(self.model.tag_type), - token.get_tag('predicted')) + token.get_tag(self.model.tag_type).name, + predicted_tag.name) + + # self-supervised learning from high-confidence predicted labels + if self.memory is not None and predicted_tag.confidence > 0.95: + self.memory.update_embedding(token.text, predicted_tag.name) + lines.append(eval_line) lines.append('\n') diff --git a/flair/training_utils.py b/flair/training_utils.py index c3aaf58a09..635b46b77d 100644 --- a/flair/training_utils.py +++ b/flair/training_utils.py @@ -49,8 +49,8 @@ def accuracy(self): return 0.0 def __str__(self): - return '{0:<20}\tprecision: {1:.4f} - recall: {2:.4f} - accuracy: {3:.4f} - f1-score: {4:.4f}'.format( - self.name, self.precision(), self.recall(), self.accuracy(), self.f_score()) + return '{0:<10}\ttp: {1} - fp: {2} - fn: {3} - precision: {4:.4f} - recall: {5:.4f} - accuracy: {6:.4f} - f1-score: {7:.4f}'.format( + self.name, self._tp, self._fp, self._fn, self.precision(), self.recall(), self.accuracy(), self.f_score()) def print(self): print(self) diff --git a/tests/test_data.py b/tests/test_data.py index aa28cb2308..7456387a8c 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -237,8 +237,8 @@ def test_label_set_confidence(): assert (0.2 == label.confidence) - with pytest.raises(ValueError): - label.name = '' + # with pytest.raises(ValueError): + # label.name = '' def test_tagged_corpus_make_label_dictionary(): @@ -414,4 +414,14 @@ def test_spans(): assert ('Research' == spans[1].text) assert ('LOC' == spans[1].tag) assert ('located in' == spans[3].text) - assert ('relation' == spans[3].tag) \ No newline at end of file + assert ('relation' == spans[3].tag) + + sentence = Sentence( + 'A woman was charged on Friday with terrorist offences after three Irish Republican Army mortar bombs were found in a Belfast house , police said . ') + sentence[11].add_tag('ner', 'S-MISC') + sentence[12].add_tag('ner', 'B-MISC') + sentence[13].add_tag('ner', 'E-MISC') + spans: List[Span] = sentence.get_spans('ner') + assert (2 == len(spans)) + assert ('Irish' == spans[0].text) + assert ('Republican Army' == spans[1].text) \ No newline at end of file From 3d93884d8b82d2b2a9dd13f17580a99f472f6b2a Mon Sep 17 00:00:00 2001 From: aakbik Date: Wed, 26 Sep 2018 16:51:36 +0200 Subject: [PATCH 030/113] GH-121: load GPU language models on CPU if required --- flair/models/language_model.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/flair/models/language_model.py b/flair/models/language_model.py index 83d64688f4..1516145818 100644 --- a/flair/models/language_model.py +++ b/flair/models/language_model.py @@ -115,7 +115,12 @@ def initialize(self, matrix): @classmethod def load_language_model(cls, model_file): - state = torch.load(model_file) + + if not torch.cuda.is_available(): + state = torch.load(model_file, map_location='cpu') + else: + state = torch.load(model_file) + model = LanguageModel(state['dictionary'], state['is_forward_lm'], state['hidden_size'], From 2f5415fd9db9be1835559badaf67f86e328b4e11 Mon Sep 17 00:00:00 2001 From: aakbik Date: Wed, 26 Sep 2018 16:52:04 +0200 Subject: [PATCH 031/113] GH-38: add confidence for span predictions --- flair/data.py | 23 ++++++++++++--- flair/models/sequence_tagger_model.py | 5 ---- flair/trainers/sequence_tagger_trainer.py | 36 +++++++---------------- tests/test_data.py | 26 +++++++++++++++- 4 files changed, 54 insertions(+), 36 deletions(-) diff --git a/flair/data.py b/flair/data.py index 7525035fcc..f6263c39be 100644 --- a/flair/data.py +++ b/flair/data.py @@ -195,9 +195,10 @@ class Span: This class represents one textual span consisting of Tokens. A span may have a tag. """ - def __init__(self, tokens: List[Token], tag: str = None): + def __init__(self, tokens: List[Token], tag: str = None, score=1.): self.tokens = tokens self.tag = tag + self.score = score @property def text(self) -> str: @@ -283,7 +284,7 @@ def add_token(self, token: Token): if token.idx is None: token.idx = len(self.tokens) - def get_spans(self, tag_type: str) -> List[Span]: + def get_spans(self, tag_type: str, min_score=-1) -> List[Span]: spans: List[Span] = [] @@ -318,7 +319,14 @@ def get_spans(self, tag_type: str) -> List[Span]: starts_new_span = True if (starts_new_span or not in_span) and len(current_span) > 0: - spans.append(Span(current_span, sorted(tags.items(), key=lambda k_v: k_v[1], reverse=True)[0][0])) + scores = [t.get_tag(tag_type).confidence for t in current_span] + span_score = sum(scores) / len(scores) + if span_score > min_score: + spans.append(Span( + current_span, + tag=sorted(tags.items(), key=lambda k_v: k_v[1], reverse=True)[0][0], + score=span_score) + ) current_span = [] tags = defaultdict(lambda: 0.0) @@ -331,7 +339,14 @@ def get_spans(self, tag_type: str) -> List[Span]: previous_tag_value = tag_value if len(current_span) > 0: - spans.append(Span(current_span, sorted(tags.items(), key=lambda k_v: k_v[1], reverse=True)[0][0])) + scores = [t.get_tag(tag_type).confidence for t in current_span] + span_score = sum(scores) / len(scores) + if span_score > min_score: + spans.append(Span( + current_span, + tag=sorted(tags.items(), key=lambda k_v: k_v[1], reverse=True)[0][0], + score=span_score) + ) return spans diff --git a/flair/models/sequence_tagger_model.py b/flair/models/sequence_tagger_model.py index 0e1d70dab3..428d623634 100644 --- a/flair/models/sequence_tagger_model.py +++ b/flair/models/sequence_tagger_model.py @@ -464,11 +464,6 @@ def _predict_scores_batch(self, sentences: List[Sentence]): tag_seq.append(prediction) confidences.append(softmax[prediction].item()) - # softmax = F.softmax(feats[:length], dim=0) - # confidences, tag_seq = torch.max(F.normalize(feats[:length], p=2, dim=1), 1) - - # tag_seq = list(tag_seq.cpu().data) - all_tags_seqs.extend(tag_seq) all_confidences.extend(confidences) diff --git a/flair/trainers/sequence_tagger_trainer.py b/flair/trainers/sequence_tagger_trainer.py index 6ed976abff..e9011747e2 100644 --- a/flair/trainers/sequence_tagger_trainer.py +++ b/flair/trainers/sequence_tagger_trainer.py @@ -1,4 +1,3 @@ -import copy from typing import List import datetime @@ -8,7 +7,6 @@ import torch from torch.optim.lr_scheduler import ReduceLROnPlateau -from flair.embeddings import MemoryEmbeddings from flair.models.sequence_tagger_model import SequenceTagger from flair.data import Sentence, Token, TaggedCorpus, Label from flair.training_utils import Metric @@ -31,18 +29,11 @@ def train(self, embeddings_in_memory: bool = True, checkpoint: bool = False, save_final_model: bool = True, - memory: MemoryEmbeddings = None, ): - evaluation_method = 'F1' - if self.model.tag_type in ['pos', 'upos']: evaluation_method = 'accuracy' + evaluation_method = 'F1' if self.model.tag_type not in ['pos', 'upos'] else 'accuracy' print('evaluation method: {}'.format(evaluation_method)) - # if memory is not None, set as field and eval batch size to 1 - self.memory = memory - eval_batch_size = 1 if self.memory else mini_batch_size - print('evaluation eval_batch_size: {}'.format(eval_batch_size)) - os.makedirs(base_path, exist_ok=True) loss_txt = os.path.join(base_path, "loss.txt") @@ -75,15 +66,15 @@ def train(self, if not self.test_mode: random.shuffle(train_data) + # make batches batches = [train_data[x:x + mini_batch_size] for x in range(0, len(train_data), mini_batch_size)] + # switch to train mode self.model.train() - batch_no: int = 0 - - for batch in batches: + for batch_no, batch in enumerate(batches): + # each batch is a list of sentences batch: List[Sentence] = batch - batch_no += 1 if batch_no % 100 == 0: print("%d of %d (%f)" % (batch_no, len(batches), float(batch_no / len(batches)))) @@ -121,7 +112,7 @@ def train(self, dev_score, dev_fp, dev_result = self.evaluate(self.corpus.dev, base_path, evaluation_method=evaluation_method, embeddings_in_memory=embeddings_in_memory, - eval_batch_size=eval_batch_size, + eval_batch_size=mini_batch_size, ) else: dev_result = '_' @@ -130,15 +121,15 @@ def train(self, test_score, test_fp, test_result = self.evaluate(self.corpus.test, base_path, evaluation_method=evaluation_method, embeddings_in_memory=embeddings_in_memory, - eval_batch_size=eval_batch_size, + eval_batch_size=mini_batch_size, ) # anneal against train loss if training with dev, otherwise anneal against dev score scheduler.step(current_loss) if train_with_dev else scheduler.step(dev_score) summary = '{} ({:%H:%M:%S})\t{}\t{}\t{} DEV {} TEST {}'.format(epoch, datetime.datetime.now(), - current_loss, scheduler.num_bad_epochs, - learning_rate, dev_result, test_result) + current_loss, scheduler.num_bad_epochs, + learning_rate, dev_result, test_result) print(summary) with open(loss_txt, "a") as loss_file: @@ -182,15 +173,12 @@ def evaluate(self, evaluation: List[Sentence], out_path=None, evaluation_method: token: Token = token # get the predicted tag predicted_tag = self.model.tag_dictionary.get_item_for_index(predicted_id) - token.add_tag('predicted', predicted_tag, score) + token.add_tag('predicted', predicted_tag.name, score) for sentence in batch: - sentence: Sentence = sentence - # add predicted tags for token in sentence.tokens: - predicted_tag: Label = token.get_tag('predicted') # append both to file for evaluation @@ -198,10 +186,6 @@ def evaluate(self, evaluation: List[Sentence], out_path=None, evaluation_method: token.get_tag(self.model.tag_type).name, predicted_tag.name) - # self-supervised learning from high-confidence predicted labels - if self.memory is not None and predicted_tag.confidence > 0.95: - self.memory.update_embedding(token.text, predicted_tag.name) - lines.append(eval_line) lines.append('\n') diff --git a/tests/test_data.py b/tests/test_data.py index 7456387a8c..d5de96fe3c 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -424,4 +424,28 @@ def test_spans(): spans: List[Span] = sentence.get_spans('ner') assert (2 == len(spans)) assert ('Irish' == spans[0].text) - assert ('Republican Army' == spans[1].text) \ No newline at end of file + assert ('Republican Army' == spans[1].text) + + sentence = Sentence('Zalando Research is located in Berlin .') + + # tags with confidence + sentence[0].add_tag('ner', 'B-ORG', 1.0) + sentence[1].add_tag('ner', 'E-ORG', 0.9) + sentence[5].add_tag('ner', 'S-LOC', 0.5) + + spans: List[Span] = sentence.get_spans('ner', min_score=0.) + + assert (2 == len(spans)) + assert ('Zalando Research' == spans[0].text) + assert ('ORG' == spans[0].tag) + assert (0.95 == spans[0].score) + + assert ('Berlin' == spans[1].text) + assert ('LOC' == spans[1].tag) + assert (0.5 == spans[1].score) + + spans: List[Span] = sentence.get_spans('ner', min_score=0.6) + assert (1 == len(spans)) + + spans: List[Span] = sentence.get_spans('ner', min_score=0.99) + assert (0 == len(spans)) \ No newline at end of file From 77048b3470584b7c50b62f686aa11623b4261451 Mon Sep 17 00:00:00 2001 From: aakbik Date: Thu, 27 Sep 2018 13:21:43 +0200 Subject: [PATCH 032/113] GH-94: new default patience value --- flair/models/sequence_tagger_model.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/flair/models/sequence_tagger_model.py b/flair/models/sequence_tagger_model.py index 428d623634..b244918af2 100644 --- a/flair/models/sequence_tagger_model.py +++ b/flair/models/sequence_tagger_model.py @@ -287,7 +287,6 @@ def _score_sentence(self, feats, tags, lens_): def viterbi_decode(self, feats): backpointers = [] backscores = [] - backscores_2 = [] init_vvars = torch.Tensor(1, self.tagset_size).fill_(-10000.) init_vvars[0][self.tag_dictionary.get_idx_for_item(START_TAG)] = 0 @@ -300,7 +299,6 @@ def viterbi_decode(self, feats): next_tag_var = forward_var.view(1, -1).expand(self.tagset_size, self.tagset_size) + self.transitions _, bptrs_t = torch.max(next_tag_var, dim=1) bptrs_t = bptrs_t.squeeze().data.cpu().numpy() - backscores_2.append(next_tag_var) next_tag_var = next_tag_var.data.cpu().numpy() viterbivars_t = next_tag_var[range(len(bptrs_t)), bptrs_t] viterbivars_t = autograd.Variable(torch.FloatTensor(viterbivars_t)) From a4a7c4aa2928611f3c68e0f1211f1f48a420209f Mon Sep 17 00:00:00 2001 From: aakbik Date: Thu, 27 Sep 2018 13:31:59 +0200 Subject: [PATCH 033/113] GH-38: Labels now have a value and a score --- flair/data.py | 94 +++++++++---------- flair/embeddings.py | 2 +- flair/models/sequence_tagger_model.py | 2 +- flair/models/text_classification_model.py | 2 +- flair/trainers/sequence_tagger_trainer.py | 6 +- flair/trainers/text_classification_trainer.py | 2 +- tests/test_data.py | 8 +- tests/test_text_classifier.py | 4 +- tests/test_text_classifier_trainer.py | 12 +-- 9 files changed, 66 insertions(+), 66 deletions(-) diff --git a/flair/data.py b/flair/data.py index f6263c39be..79694b910d 100644 --- a/flair/data.py +++ b/flair/data.py @@ -94,42 +94,42 @@ def load(cls, name: str): class Label: """ - This class represents a label of a sentence. Each label has a name and optional a confidence value. The confidence - value needs to be between 0.0 and 1.0. Default value for the confidence is 1.0. + This class represents a label of a sentence. Each label has a value and optionally a confidence score. The + score needs to be between 0.0 and 1.0. Default value for the score is 1.0. """ - def __init__(self, name: str, confidence: float = 1.0): - self.name = name - self.confidence = confidence + def __init__(self, value: str, score: float = 1.0): + self.value = value + self.score = score super().__init__() @property - def name(self): - return self._name + def value(self): + return self._value - @name.setter - def name(self, name): - if not name and name != '': - raise ValueError('Incorrect label name provided. Label name needs to be set.') + @value.setter + def value(self, value): + if not value and value != '': + raise ValueError('Incorrect label value provided. Label value needs to be set.') else: - self._name = name + self._value = value @property - def confidence(self): - return self._confidence + def score(self): + return self._score - @confidence.setter - def confidence(self, confidence): - if 0.0 <= confidence <= 1.0: - self._confidence = confidence + @score.setter + def score(self, score): + if 0.0 <= score <= 1.0: + self._score = score else: - self._confidence = 1.0 + self._score = 1.0 - # def __str__(self): - # return "{} ({})".format(self._name, self._confidence) + def __str__(self): + return "{} ({})".format(self._value, self._score) def __repr__(self): - return "{} ({})".format(self._name, self._confidence) + return "{} ({})".format(self._value, self._score) class Token: @@ -296,7 +296,7 @@ def get_spans(self, tag_type: str, min_score=-1) -> List[Span]: for token in self: tag: Label = token.get_tag(tag_type) - tag_value = tag.name + tag_value = tag.value # non-set tags are OUT tags if len(tag_value) < 2: tag_value = 'O-' @@ -319,7 +319,7 @@ def get_spans(self, tag_type: str, min_score=-1) -> List[Span]: starts_new_span = True if (starts_new_span or not in_span) and len(current_span) > 0: - scores = [t.get_tag(tag_type).confidence for t in current_span] + scores = [t.get_tag(tag_type).score for t in current_span] span_score = sum(scores) / len(scores) if span_score > min_score: spans.append(Span( @@ -339,7 +339,7 @@ def get_spans(self, tag_type: str, min_score=-1) -> List[Span]: previous_tag_value = tag_value if len(current_span) > 0: - scores = [t.get_tag(tag_type).confidence for t in current_span] + scores = [t.get_tag(tag_type).score for t in current_span] span_score = sum(scores) / len(scores) if span_score > min_score: spans.append(Span( @@ -362,7 +362,7 @@ def add_labels(self, labels: Union[List[Label], List[str]]): self.add_label(label) def get_label_names(self) -> List[str]: - return [label.name for label in self.labels] + return [label.value for label in self.labels] @property def embedding(self): @@ -403,8 +403,8 @@ def to_tagged_string(self, main_tag=None) -> str: if main_tag is not None and main_tag != tag_type: continue - if token.get_tag(tag_type).name == '' or token.get_tag(tag_type).name == 'O': continue - tags.append(token.get_tag(tag_type).name) + if token.get_tag(tag_type).value == '' or token.get_tag(tag_type).value == 'O': continue + tags.append(token.get_tag(tag_type).value) all_tags = '<' + '/'.join(tags) + '>' if all_tags != '<>': list.append(all_tags) @@ -530,7 +530,7 @@ def make_tag_dictionary(self, tag_type: str) -> Dictionary: for sentence in self.get_all_sentences(): for token in sentence.tokens: token: Token = token - tag_dictionary.add_item(token.get_tag(tag_type).name) + tag_dictionary.add_item(token.get_tag(tag_type).value) tag_dictionary.add_item('') tag_dictionary.add_item('') return tag_dictionary @@ -580,7 +580,7 @@ def _get_most_common_tokens(self, max_tokens, min_freq) -> List[str]: return tokens def _get_all_label_names(self) -> List[str]: - return [label.name for sent in self.train for label in sent.labels] + return [label.value for sent in self.train for label in sent.labels] def _get_all_tokens(self) -> List[str]: tokens = list(map((lambda s: s.tokens), self.train)) @@ -635,7 +635,7 @@ def _get_classes_to_count(sentences): classes_to_count = defaultdict(lambda: 0) for sent in sentences: for label in sent.labels: - classes_to_count[label.name] += 1 + classes_to_count[label.value] += 1 return classes_to_count def __str__(self) -> str: @@ -649,19 +649,19 @@ def iob2(tags): """ for i, tag in enumerate(tags): # print(tag) - if tag.name == 'O': + if tag.value == 'O': continue - split = tag.name.split('-') + split = tag.value.split('-') if len(split) != 2 or split[0] not in ['I', 'B']: return False if split[0] == 'B': continue - elif i == 0 or tags[i - 1].name == 'O': # conversion IOB1 to IOB2 - tags[i].name = 'B' + tag.name[1:] - elif tags[i - 1].name[1:] == tag.name[1:]: + elif i == 0 or tags[i - 1].value == 'O': # conversion IOB1 to IOB2 + tags[i].value = 'B' + tag.value[1:] + elif tags[i - 1].value[1:] == tag.value[1:]: continue else: # conversion IOB1 to IOB2 - tags[i].name = 'B' + tag.name[1:] + tags[i].value = 'B' + tag.value[1:] return True @@ -671,20 +671,20 @@ def iob_iobes(tags): """ new_tags = [] for i, tag in enumerate(tags): - if tag.name == 'O': - new_tags.append(tag.name) - elif tag.name.split('-')[0] == 'B': + if tag.value == 'O': + new_tags.append(tag.value) + elif tag.value.split('-')[0] == 'B': if i + 1 != len(tags) and \ - tags[i + 1].name.split('-')[0] == 'I': - new_tags.append(tag.name) + tags[i + 1].value.split('-')[0] == 'I': + new_tags.append(tag.value) else: - new_tags.append(tag.name.replace('B-', 'S-')) - elif tag.name.split('-')[0] == 'I': + new_tags.append(tag.value.replace('B-', 'S-')) + elif tag.value.split('-')[0] == 'I': if i + 1 < len(tags) and \ - tags[i + 1].name.split('-')[0] == 'I': - new_tags.append(tag.name) + tags[i + 1].value.split('-')[0] == 'I': + new_tags.append(tag.value) else: - new_tags.append(tag.name.replace('I-', 'E-')) + new_tags.append(tag.value.replace('I-', 'E-')) else: raise Exception('Invalid IOB format!') return new_tags diff --git a/flair/embeddings.py b/flair/embeddings.py index 7bff1cc7ac..690388444b 100644 --- a/flair/embeddings.py +++ b/flair/embeddings.py @@ -265,7 +265,7 @@ def _add_embeddings_internal(self, sentences: List[Sentence]) -> List[Sentence]: # add label if in training mode if self.training: - self.update_embedding(token.text, token.get_tag(self.tag_type).name) + self.update_embedding(token.text, token.get_tag(self.tag_type).value) return sentences diff --git a/flair/models/sequence_tagger_model.py b/flair/models/sequence_tagger_model.py index b244918af2..e1d56a2d5a 100644 --- a/flair/models/sequence_tagger_model.py +++ b/flair/models/sequence_tagger_model.py @@ -194,7 +194,7 @@ def forward(self, sentences: List[Sentence]): for token in sentence], 0) # get the tags in this sentence - tag_idx: List[int] = [self.tag_dictionary.get_idx_for_item(token.get_tag(self.tag_type).name) + tag_idx: List[int] = [self.tag_dictionary.get_idx_for_item(token.get_tag(self.tag_type).value) for token in sentence] # add tags as tensor if torch.cuda.is_available(): diff --git a/flair/models/text_classification_model.py b/flair/models/text_classification_model.py index 8018e7e2a5..51323542f9 100644 --- a/flair/models/text_classification_model.py +++ b/flair/models/text_classification_model.py @@ -184,7 +184,7 @@ def _labels_to_one_hot(self, sentences: List[Sentence]): def _labels_to_indices(self, sentences: List[Sentence]): indices = [ - torch.LongTensor([self.label_dictionary.get_idx_for_item(label.name) for label in sentence.labels]) + torch.LongTensor([self.label_dictionary.get_idx_for_item(label.value) for label in sentence.labels]) for sentence in sentences ] diff --git a/flair/trainers/sequence_tagger_trainer.py b/flair/trainers/sequence_tagger_trainer.py index e9011747e2..b940e0b43a 100644 --- a/flair/trainers/sequence_tagger_trainer.py +++ b/flair/trainers/sequence_tagger_trainer.py @@ -173,7 +173,7 @@ def evaluate(self, evaluation: List[Sentence], out_path=None, evaluation_method: token: Token = token # get the predicted tag predicted_tag = self.model.tag_dictionary.get_item_for_index(predicted_id) - token.add_tag('predicted', predicted_tag.name, score) + token.add_tag('predicted', predicted_tag.value, score) for sentence in batch: @@ -183,8 +183,8 @@ def evaluate(self, evaluation: List[Sentence], out_path=None, evaluation_method: # append both to file for evaluation eval_line = '{} {} {}\n'.format(token.text, - token.get_tag(self.model.tag_type).name, - predicted_tag.name) + token.get_tag(self.model.tag_type).value, + predicted_tag.value) lines.append(eval_line) lines.append('\n') diff --git a/flair/trainers/text_classification_trainer.py b/flair/trainers/text_classification_trainer.py index 7eef04fc27..19b8424343 100644 --- a/flair/trainers/text_classification_trainer.py +++ b/flair/trainers/text_classification_trainer.py @@ -215,7 +215,7 @@ def evaluate(self, sentences: List[Sentence], eval_class_metrics: bool = False, eval_loss += loss - y_pred.extend(convert_labels_to_one_hot([[label.name for label in sent_labels] for sent_labels in labels], self.label_dict)) + y_pred.extend(convert_labels_to_one_hot([[label.value for label in sent_labels] for sent_labels in labels], self.label_dict)) metrics = [calculate_micro_avg_metric(y_true, y_pred, self.label_dict)] if eval_class_metrics: diff --git a/tests/test_data.py b/tests/test_data.py index d5de96fe3c..e4f0a30660 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -230,12 +230,12 @@ def test_tagged_corpus_make_vocab_dictionary(): def test_label_set_confidence(): label = Label('class_1', 3.2) - assert (1.0 == label.confidence) - assert ('class_1' == label.name) + assert (1.0 == label.score) + assert ('class_1' == label.value) - label.confidence = 0.2 + label.score = 0.2 - assert (0.2 == label.confidence) + assert (0.2 == label.score) # with pytest.raises(ValueError): # label.name = '' diff --git a/tests/test_text_classifier.py b/tests/test_text_classifier.py index 033997992a..789df66682 100644 --- a/tests/test_text_classifier.py +++ b/tests/test_text_classifier.py @@ -26,7 +26,7 @@ def test_labels_to_indices(): result = model._labels_to_indices(corpus.train) for i in range(len(corpus.train)): - expected = label_dict.get_idx_for_item(corpus.train[i].labels[0].name) + expected = label_dict.get_idx_for_item(corpus.train[i].labels[0].value) actual = result[i].item() assert(expected == actual) @@ -38,7 +38,7 @@ def test_labels_to_one_hot(): result = model._labels_to_one_hot(corpus.train) for i in range(len(corpus.train)): - expected = label_dict.get_idx_for_item(corpus.train[i].labels[0].name) + expected = label_dict.get_idx_for_item(corpus.train[i].labels[0].value) actual = result[i] for idx in range(len(label_dict)): diff --git a/tests/test_text_classifier_trainer.py b/tests/test_text_classifier_trainer.py index e9b1b4cbe5..244f2cbd23 100644 --- a/tests/test_text_classifier_trainer.py +++ b/tests/test_text_classifier_trainer.py @@ -24,9 +24,9 @@ def test_text_classifier_single_label(): for s in model.predict(sentence): for l in s.labels: - assert(l.name is not None) - assert(0.0 <= l.confidence <= 1.0) - assert(type(l.confidence) is float) + assert(l.value is not None) + assert(0.0 <= l.score <= 1.0) + assert(type(l.score) is float) # clean up results directory shutil.rmtree('./results') @@ -48,9 +48,9 @@ def test_text_classifier_mulit_label(): for s in model.predict(sentence): for l in s.labels: - assert(l.name is not None) - assert(0.0 <= l.confidence <= 1.0) - assert(type(l.confidence) is float) + assert(l.value is not None) + assert(0.0 <= l.score <= 1.0) + assert(type(l.score) is float) # clean up results directory shutil.rmtree('./results') \ No newline at end of file From 6c47a8617cb9483c2a764b18d88950109d2e3ae9 Mon Sep 17 00:00:00 2001 From: aakbik Date: Thu, 27 Sep 2018 13:42:14 +0200 Subject: [PATCH 034/113] GH-38: fixed label value bug --- flair/trainers/sequence_tagger_trainer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flair/trainers/sequence_tagger_trainer.py b/flair/trainers/sequence_tagger_trainer.py index b940e0b43a..e553ff4099 100644 --- a/flair/trainers/sequence_tagger_trainer.py +++ b/flair/trainers/sequence_tagger_trainer.py @@ -172,8 +172,8 @@ def evaluate(self, evaluation: List[Sentence], out_path=None, evaluation_method: for (token, score, predicted_id) in zip(all_tokens, scores, predicted_ids): token: Token = token # get the predicted tag - predicted_tag = self.model.tag_dictionary.get_item_for_index(predicted_id) - token.add_tag('predicted', predicted_tag.value, score) + predicted_value = self.model.tag_dictionary.get_item_for_index(predicted_id) + token.add_tag('predicted', predicted_value, score) for sentence in batch: From d2c3c251abb28e0cdc0f653bcb4c2116a011f653 Mon Sep 17 00:00:00 2001 From: tabergma Date: Thu, 27 Sep 2018 17:21:32 +0200 Subject: [PATCH 035/113] GH-48: fix warning: flatten parameters in language model --- flair/models/language_model.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/flair/models/language_model.py b/flair/models/language_model.py index 1516145818..e5851f3e46 100644 --- a/flair/models/language_model.py +++ b/flair/models/language_model.py @@ -66,6 +66,8 @@ def forward(self, input, hidden, ordered_sequence_lengths=None): encoded = self.encoder(input) emb = self.drop(encoded) + self.rnn.flatten_parameters() + output, hidden = self.rnn(emb, hidden) if self.proj is not None: From 280ffd4e64ab376334f4706603f3badb8b2e5018 Mon Sep 17 00:00:00 2001 From: tabergma Date: Thu, 27 Sep 2018 17:24:28 +0200 Subject: [PATCH 036/113] GH-48: Use locked dropout only if specified. --- flair/embeddings.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/flair/embeddings.py b/flair/embeddings.py index 690388444b..0fc16c0356 100644 --- a/flair/embeddings.py +++ b/flair/embeddings.py @@ -540,7 +540,7 @@ class DocumentLSTMEmbeddings(DocumentEmbeddings): def __init__(self, token_embeddings: List[TokenEmbeddings], hidden_states=128, num_layers=1, reproject_words: bool = True, reproject_words_dimension: int = None, bidirectional: bool = False, - use_first_representation: bool = False, use_word_dropout: bool = True): + use_first_representation: bool = False, use_word_dropout: bool = False, use_locked_dropout: bool = False): """The constructor takes a list of embeddings to be combined. :param token_embeddings: a list of token embeddings :param hidden_states: the number of hidden states in the lstm @@ -553,6 +553,7 @@ def __init__(self, token_embeddings: List[TokenEmbeddings], hidden_states=128, n :param use_first_representation: boolean value, indicating whether to concatenate the first and last representation of the lstm to be used as final document embedding. :param use_word_dropout: boolean value, indicating whether to use word dropout or not. + :param use_locked_dropout: boolean value, indicating whether to use locked dropout or not. """ super().__init__() @@ -586,7 +587,10 @@ def __init__(self, token_embeddings: List[TokenEmbeddings], hidden_states=128, n bidirectional=self.bidirectional) # dropouts - self.dropout: torch.nn.Module = LockedDropout(0.5) + if use_locked_dropout: + self.dropout: torch.nn.Module = LockedDropout(0.5) + else: + self.dropout = torch.nn.Dropout(0.5) self.use_word_dropout: bool = use_word_dropout if self.use_word_dropout: From 801a4bde7f9951a7107705f9e06bb5de928d769f Mon Sep 17 00:00:00 2001 From: tabergma Date: Fri, 28 Sep 2018 08:48:58 +0200 Subject: [PATCH 037/113] GH-48: Rewrite eval method. --- flair/trainers/text_classification_trainer.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/flair/trainers/text_classification_trainer.py b/flair/trainers/text_classification_trainer.py index 55e7f2ea01..5d74010ca1 100644 --- a/flair/trainers/text_classification_trainer.py +++ b/flair/trainers/text_classification_trainer.py @@ -215,12 +215,14 @@ def evaluate(self, sentences: List[Sentence], eval_class_metrics: bool = False, labels = self.model.obtain_labels(scores) loss = self.model.calculate_loss(scores, batch) + eval_loss += loss + + y_pred.extend([[label.value for label in sent_labels] for sent_labels in labels]) + if not embeddings_in_memory: clear_embeddings(batch) - eval_loss += loss - - y_pred.extend(convert_labels_to_one_hot([[label.value for label in sent_labels] for sent_labels in labels], self.label_dict)) + y_pred = convert_labels_to_one_hot(y_pred, self.label_dict) metrics = [calculate_micro_avg_metric(y_true, y_pred, self.label_dict)] if eval_class_metrics: From a2975c2809f114ffb053ce76ef87f2f0cc861d23 Mon Sep 17 00:00:00 2001 From: tabergma Date: Fri, 28 Sep 2018 09:13:46 +0200 Subject: [PATCH 038/113] GH-48: Fix corpus statistic --- flair/data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flair/data.py b/flair/data.py index b5be47e615..704b5c17d1 100644 --- a/flair/data.py +++ b/flair/data.py @@ -623,7 +623,7 @@ def _print_statistics_for(sentences, name): size_dict = {} for l, c in classes_to_count.items(): - size_dict = { l: c } + size_dict[l] = c size_dict['total'] = len(sentences) stats = { From 03258f33a3d7f060a4038f5a0ac6f7a4499e870d Mon Sep 17 00:00:00 2001 From: tabergma Date: Fri, 28 Sep 2018 09:19:23 +0200 Subject: [PATCH 039/113] GH-48: Set train loss to current loss per default --- flair/trainers/text_classification_trainer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/flair/trainers/text_classification_trainer.py b/flair/trainers/text_classification_trainer.py index 5d74010ca1..e51c68979f 100644 --- a/flair/trainers/text_classification_trainer.py +++ b/flair/trainers/text_classification_trainer.py @@ -122,7 +122,8 @@ def train(self, log.info("EPOCH {0}: lr {1:.4f} - bad epochs {2}".format(epoch + 1, learning_rate, scheduler.num_bad_epochs)) dev_metric = train_metric = None - dev_loss = train_loss = '_' + dev_loss = '_' + train_loss = current_loss if eval_on_train: train_metric, train_loss = self._calculate_evaluation_results_for( From 6b225f18d65929136df20257d6100192100c54bd Mon Sep 17 00:00:00 2001 From: tabergma Date: Fri, 28 Sep 2018 10:02:27 +0200 Subject: [PATCH 040/113] GH-48: Update eval method. --- flair/models/language_model.py | 2 -- flair/trainers/text_classification_trainer.py | 4 +++- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/flair/models/language_model.py b/flair/models/language_model.py index e5851f3e46..1516145818 100644 --- a/flair/models/language_model.py +++ b/flair/models/language_model.py @@ -66,8 +66,6 @@ def forward(self, input, hidden, ordered_sequence_lengths=None): encoded = self.encoder(input) emb = self.drop(encoded) - self.rnn.flatten_parameters() - output, hidden = self.rnn(emb, hidden) if self.proj is not None: diff --git a/flair/trainers/text_classification_trainer.py b/flair/trainers/text_classification_trainer.py index e51c68979f..3e739dae68 100644 --- a/flair/trainers/text_classification_trainer.py +++ b/flair/trainers/text_classification_trainer.py @@ -209,7 +209,7 @@ def evaluate(self, sentences: List[Sentence], eval_class_metrics: bool = False, range(0, len(sentences), mini_batch_size)] y_pred = [] - y_true = convert_labels_to_one_hot([sentence.get_label_names() for sentence in sentences], self.label_dict) + y_true = [] for batch in batches: scores = self.model.forward(batch) @@ -218,11 +218,13 @@ def evaluate(self, sentences: List[Sentence], eval_class_metrics: bool = False, eval_loss += loss + y_true.extend([sentence.get_label_names() for sentence in batch]) y_pred.extend([[label.value for label in sent_labels] for sent_labels in labels]) if not embeddings_in_memory: clear_embeddings(batch) + y_true = convert_labels_to_one_hot(y_true, self.label_dict) y_pred = convert_labels_to_one_hot(y_pred, self.label_dict) metrics = [calculate_micro_avg_metric(y_true, y_pred, self.label_dict)] From 4a389acc60b608c82e486c9f012d35ff581a5e68 Mon Sep 17 00:00:00 2001 From: Duncan Blythe Date: Mon, 13 Aug 2018 11:03:23 +0200 Subject: [PATCH 041/113] GH-61: used sklearn --- flair/visual/__init__.py | 1 + flair/visual/tsne.py | 31 + resources/data/snippet.txt | 2000 ++++++++++++++++++++++++++++++++++++ tests/test_visual.py | 34 + 4 files changed, 2066 insertions(+) create mode 100644 flair/visual/__init__.py create mode 100644 flair/visual/tsne.py create mode 100644 resources/data/snippet.txt create mode 100644 tests/test_visual.py diff --git a/flair/visual/__init__.py b/flair/visual/__init__.py new file mode 100644 index 0000000000..bb50a64823 --- /dev/null +++ b/flair/visual/__init__.py @@ -0,0 +1 @@ +from .tsne import tSNE \ No newline at end of file diff --git a/flair/visual/tsne.py b/flair/visual/tsne.py new file mode 100644 index 0000000000..5b8e5fa539 --- /dev/null +++ b/flair/visual/tsne.py @@ -0,0 +1,31 @@ +from sklearn.manifold import TSNE +import tqdm +import numpy + + +class tSNE: + def __init__(self, embeddings): + + self.embeddings = embeddings + + self.transform = \ + TSNE(n_components=2, verbose=1, perplexity=40, n_iter=300) + + def _prepare(self, sentences): + + X = [] + + print('computing embeddings') + for sentence in tqdm.tqdm(sentences): + self.embeddings.embed(sentence) + + for token in sentence: + X.append(token.embedding.detach().numpy()[None, :]) + + X = numpy.concatenate(X, 0) + + return X + + def fit(self, sentences): + + return self.transform.fit_transform(self._prepare(sentences)) diff --git a/resources/data/snippet.txt b/resources/data/snippet.txt new file mode 100644 index 0000000000..96b66ed166 --- /dev/null +++ b/resources/data/snippet.txt @@ -0,0 +1,2000 @@ +The U.S. Centers for Disease Control and Prevention initially advised school systems to close if outbreaks occurred , then reversed itself , saying the apparent mildness of the virus meant most schools and day care centers should stay open , even if they had confirmed cases of swine flu . +When Ms. Winfrey invited Suzanne Somers to share her controversial views about bio-identical hormone treatment on her syndicated show in 2009 , it won Ms. Winfrey a rare dollop of unflattering press , including a Newsweek cover story titled " Crazy Talk : Oprah , Wacky Cures & You . " +Elk calling -- a skill that hunters perfected long ago to lure game with the promise of a little romance -- is now its own sport . +Don 't ! +Fish , ranked 98th in the world , fired 22 aces en route to a 6-3 , 6-7 ( 5 / 7 ) , 7-6 ( 7 / 4 ) win over seventh-seeded Argentinian David Nalbandian . +Why does everything have to become such a big issue ? +AMMAN ( Reuters ) - King Abdullah of Jordan will meet U.S. President Barack Obama in Washington on April 21 to lobby on behalf of Arab states for a stronger U.S. role in Middle East peacemaking , palace officials said on Sunday . +To help keep traffic flowing the Congestion Charge will remain in operation through-out the strike and TfL will be suspending road works on major London roads wherever possible . +If no candidate wins an absolute majority , there will be a runoff between the top two contenders , most likely in mid-October . +Authorities previously served search warrants at Murray 's Las Vegas home and his businesses in Las Vegas and Houston . +Brent North Sea crude for November delivery rose 84 cents to 68.88 dollars a barrel . +That seems to have been their model up til now . +Gordon will join Luol Deng on the GB team ; their respective NBA teams , the Detroit Pistons and the Chicago Bulls , play tonight . +Nikam maintains the attacks were masterminded by the Muslim militant group Lashkar-e-Taiba . +Last year , Williams was unseeded , ranked 81st and coming off one of her worst losses on tour -- in a Tier 4 event at Hobart -- yet she beat six seeded players en route to the title at Melbourne Park . +It said that two officers involved in the case had been disciplined . +" There is more intelligence now being gathered , " the official said , adding that such efforts would continue for some time . +The majority will be of the standard 6X6 configuration for carrying personnel . +" Consequently , necessary actions may not be taken to reduce the risks to children of sexual exploitation and drug or alcohol misuse , " the report said . • Almost two-thirds of inspected schools were good or outstanding , but the number of underperforming secondaries remained " stubborn and persistent . " +What a World Cup . +But , there have also been many cases of individuals and small groups of people protesting , as in the case of Rongye Adak , a nomad who called for the return of the Dalai Lama and for the freedom of Tibet during the Lithang Horse Racing Festival , in eastern Tibet . +James Duncan , head of transportation at Bournemouth Borough Council , said : " Our legal team is reviewing the entitlement of taxis to drop and pick up passengers at bus stops , only for as long as is absolutely necessary to fulfil that function and for no other reason . +To Mo concerning the food log you kept -- Dr. Buchholz recommends the same thing . +The CBO estimates that only 23 percent of that would be spent in 2009 and 2010 . +Even so , Democrats slammed Bush as out of touch . +An information campaign will be launched later to raise awareness of employment rights and how to enforce them . +At the gallery the concept is less vague , as Ms. Piper cites specific instances of racial violence , political assassinations and the devastation of Hurricane Katrina . +There have been some exceptions -- such as Medicare in 1965 . +The government guidance will be reviewed early next year after a period of public comment . +It wasn 't the most seaworthy of prizes . +LOUIS - A former Anheuser-Busch executive who was the company 's top-ranking woman has sued the brewer claiming it encourages a " frat party " atmosphere and pays women less in salary and bonuses than men . +What does it say about Pelosi that she lost 95 dem votes when she has all the power . +Skea , a 3-month-old Taipei store that has nothing to do with Ikea , sells custom-made , small-scale buildings as well as cars , boats , baseball mitts or just about anything else that can be made from paper , cardboard and light wood . +A dozen athletes with connections to Lake Placid competed in Vancouver . +They owe $ 10,500 on their car . +" We need them to move on and accept change , because their problems are a distraction from our goal of building a more integrated Asean , " said Ong Keng Yong , the association 's secretary general . +The internet is sort-of-40 this year . +His approach was inquisitive , a meeting of artful hesitation with fluid technique . +Katrin 's father , Dr Edmund Radmacher , inherited a chemistry company that he transformed into a flourishing concern . +On that basis Mr Perry is not a man fit for higher office . +Grand ladies in exquisitely tailored houndstooth check may have had husbands who bought them their rarefied Dior clothing , but it is only too clear who was wearing the trousers . +The two men reportedly lived alone in the shelter , a two-bedroom flat that is run by a private social care company , financed partly by the city authorities . +A lot rode on the opening lead . +Why you support the FAKE war on terror I understand , you are just following orders . +The motive for the attack on the pope remains unclear but it has not been linked to Islamic issues . +A post-mortem examination is due to be carried out in the next few days , a spokesman said . +HBOS is the lead bank in a syndicate that includes Lloyds TSB , Royal Bank of Scotland , HSBC and National Australia Bank and granted New Star the loan last year . +You do , however , still have to choose between those technologies on most larger screens . +There are currently some 5 million homeowners that are 90 days or more past due on their mortgages , according to Fannie Mae chief economist Doug Duncan . +There was an extended paralysis of racing in the major horse racing states of New South Wales and Queensland , including the loss of popular traditional racing carnivals . +They wanted things to be monumental , so they could be seen from a distance , " explains Laurence Mouillefarine , associate curator of the show , picking out some big , showy brooches to illustrate the point . +More than 45,000 women are diagnosed with breast cancer each year in the UK , and more than 12,000 die from the disease . +That agency 's grant panels do not use culturally specific criteria when awarding money . +In the five months since he started the company , he has proven his hunch : People are willing to pay for New York City tap water . +They also maintain that because Google 's system for identifying and displaying ads is more lucrative than Yahoo 's approach , the deal will generate additional revenue for Yahoo that will it make it a more formidable competitor to both Google and Microsoft . +Forecasts called for plunging temperatures and afternoon rain storms in the early Southern Hemisphere autumn . +Kloppers said BHP would continue talking to its customers about the merger . +Sailing is a passion . +I would argue that the media caters just to that , if not simple minded , to those who are not disciplined enough to see out the truth , THEIR OWN truth . +Android is being developed , " assures Mobiclip 's vice-president of marketing Denis Pagnac . +LONDON ( Reuters ) - Want to know your chance of having a baby through in-vitro fertilization ? +NEW YORK ( Reuters ) - Toll Brothers ( TOL.N : Quote , Profile , Research ) said on Thursday that it expected to report a 36 percent drop in quarterly home-building revenue , while net orders for new homes fell more steeply than in prior quarters , reflecting last month 's deepening decline in the U.S. housing market . +The six , who call themselves " Ztohoven , " claim that the aim of the project - called Media Reality - was to show how reality can be manipulated by the media . +Stupak had pledged to oppose the health care package unless given greater assurance that it would not allow federal funding of elective abortions . +Sheik Salman al-Feraiji , al-Sadr 's chief representative in Sadr City , issued a statement with demands to quell the discontent , including the release of Sadrist detainees , an end to military operations against them and al-Maliki 's resignation . +Cowles declined to comment to ABC News for this story . +In fairness , the Scots finished the half more adeptly than they 'd started and McFadden was unlucky not to win a penalty when he was obstructed in the box . +Results from dozens of clinical studies will be released at the gathering opening Friday and running through June 2 . +There , 1 per cent of the population owns 49 per cent of the land ; here , 0.3 per cent owns 69 per cent . +" One wall is enough , " he said . +" Next question , " said Fisher with a smile . +He seems genuinely affected and I almost suggest we meet again , informally as friends , but retrieve the last scraps of my dignity . +Prosecutors said Plotkin ran the schemes , enlisting David Pajcin , a former Goldman Sachs Group Inc. analyst who has pleaded guilty to charges in the case and is cooperating with the government . +In anticipation of prolonged outages , Rockford city officials coordinated assistance to provide access to warming centers for residents . +From TPP-- that 's how i see it. it 's a resource . +In the nearby village of Tepalcatepec , concrete homes built with dollars sent home by migrants stand next to tin-roof shacks . +They do not listen , or even know how to listen , they are operating on " eternal truths " that are not open to debate . +Most vacationers visit America 's national parks during the summer months--when the roads are crowded , restaurants are packed and prices are at their peak . +Good , there you go ! +" In World War II , we knew who the enemy was and we fought with them . +To some this may seem puzzling , given that dishonour is spread fairly evenly between both of the main parties . +3 ( UPI ) -- An Arizona elementary school principal placed on leave for writing a letter jokingly calling children " stupid " has apologized to parents , officials said . +So embrace the idea of big . +Reclusive North Korean leader Kim Jong Il isn 't always easy to explain . +Items were stolen during the incident . +Santiago tried to stop the fight after the 11th round , but Cotto went back out to take even more punishment before a final flurry along the ropes prompted referee Kenny Bayless to end it . +Knecht and Benner presented the fossil last month in Krakow , Poland , during the Second International Congress on Ichnology . +John McMahon , manager of the Torquay Tigers Football Club , said the pictures of Klan members were posted on the Internet by a volunteer , a retired member of the club , without the knowledge of officials , The Age reported Thursday . +The sub-Billy Liar / Elliot one , where the gruff father is disgusted by his son 's airy-fairy wholemeal ambitions , is particularly good . +Meanwhile , many Algerians say they have not felt the effects of the country 's new wealth . +A secret review of Fletcher 's death concluded two years ago that there was enough evidence to charge two Libyans , The Daily Telegraph reported earlier this month . +Earlier , state television said the blast happened in central Iran , without giving details . +Coach K and North Carolina 's Roy Williams are busy arguing over injury reports , so we know they didn 't hear Capel on Tobacco Road . +Lance , Inc. has scheduled a conference call and presentation with investors at 9 : 00 am eastern time on Friday , July 24 , 2009 to discuss financial results . To participate in the conference call , the dial-in number is ( 800 ) 789-3681 for U.S. callers or ( 702 ) 696-4943 for international callers . A continuous telephone replay of the call will be available beginning at 12 : 00 pm on July 24th and running through July 31st at midnight . The replay telephone number is ( 800 ) 642-1687 for U.S. callers or ( 706 ) 645-9291 for international callers . The replay access code is 17972828 . Investors may also access a web-based replay of the conference call at Lance 's website http : / / ir.lance.com. +We all have an interest in the success of these institutions . +Translated , that means debt for each household grew by $ 582 in the last six months , compared with $ 418 in the previous six . +It turns out that woman have sex for all of these reasons and more , and that their choices are not arbitrary ; there may be evolutionary explanations at work . +For months , the panel had been investigating the Democratic senator 's appointment and seating . +Because most blacks today enjoy improved education and quality of life , " dealing with the African-American community is the same as dealing with any other community , " said Dick Harpootlian , a white Obama supporter and former chairman of the South Carolina Democratic Party . +Ditto her bum-grazing cut-offs . +Though the cards haven 't prompted much opposition in the Bay Area , the matter of government-issued identification cards for illegal immigrants is controversial nationwide . +" October delivers some of the worst driving conditions of the year as the weather changes and the clocks go back but drivers continue as if they were in the summer months . +At the beginning of my day with the G20 protesters , I had every hope that my presence as an independent observer on behalf of Parliament would be redundant . +SocGen issues its first public statement outlining ' an exceptional fraud ' at the bank . +Rose has pledged to split his controversial double role as chief executive and chairman by 2011 , although he will come under pressure from shareholders at next week 's annual meeting to bring this forward by a year . +It is very tempting , but I don 't want to do so knowing that I won 't see him for weeks . +A major earthquake -- of magnitude 7 or higher -- is capable of causing widespread and heavy damage . +During his party conference Mr Clegg called for " savage cuts . " +" This is a very distressing time for the family , and our thoughts are with them at this time , " said Det Sgt Emma Hampson . +Two independent forecasters who did foresee the return of Arctic weather , and are predicting that there will be more , are Britain 's Piers Corbyn ( who relies on solar activity for much of his prediction ) and the US meteorologist Joe Bastardi of AccuWeather , who largely bases his forecasts on movements of air and ocean currents . +Saint Anthony Medical Center in Crown Point will implement the program March 9 ; Saint Anthony Memorial Hospital in Michigan City , March 23 . +Because we have the best team in the world , " said the 69-year-old Aragones , who will resign as Spain 's most successful coach despite repeated pleas from his players Monday to remain . +The pundits and analysts can talk themselves silly all day Sunday . +Cobham is developing only its fourth generation of refuelling equipment since 1934 . +According to the Ministry of Justice the current cost of the alternative - a by election should an MP stand down - is £ 70,000 - £ 80,000 , depending on the size of the constituency . +PINZOLO , Italy ( AP ) -- David Trezeguet is quitting international football after the confirmation of the reappointment of Raymond Domenech as national coach . +Later in the afternoon , the first couple returned to the base from their Kailua rental home to attend a midday Christmas Day meal at its Anderson Hall chow hall , the newspaper said . +There is intense irritation with Mr King in Downing Street , where he was regarded as slow to react to the banking crisis and naive in giving briefings to the Tories that were later leaked . +But first , this part is about creating a sense of shared experiences . +Nowitzki , who is averaging 23.6 points and 8.7 rebounds per game , will sit out Thursday 's home game against the Houston Rockets . +The warning was later lifted after a 4-inch tsunami rolled ashore . +The defense argued Mineo exaggerated the encounter and made up the assault to get a bigger payday from the city . +From 2016 all new homes will have to be zero-carbon and rental properties may have to have Energy Performance Certificates . +Housing Executive head of research Joe Frey said indications that the market was stabilising were welcome , but he warned it was " important to be cautious at this stage . " +That was the unappealing choice facing UBS , a Swiss bank which has been badly hurt by the carnage in America 's mortgage market . +Juergen Weckherlin , a German businessman in Hong Kong , said people were losing money because they cannot make trips that require face-to-face contact . +29 ( UPI ) -- A spokesman for Hamas in Syria said Tuesday leaders were still discussing an Israeli offer to release hundreds of prisoners for captured soldier Gilad Shalit . +Thirty years after a paedophilia case forced him to flee America , the film director Roman Polanski has filed a request in Los Angeles to have charges against him dismissed . +Tanzanite is believed to be limited to east Africa 's Rift Valley region and the pits where the accident happened are located in the heart of Maasai land , a short distance from Mount Kilimanjaro . +Ray Barrell , a research fellow at the NIESR , said that the Chancellor was wrongly assuming that revenues from the housing and financial services sectors would quickly return to levels seen in the boom years - while the spending cuts and tax rises announced would not be enough to restore the government budget to balance by Mr Darling 's target date of 2017-18 . +This breezy notion brooked no caveat that the climate might have changed greatly by 2050 , and we will possibly not be offering even the lukewarm welcome to migrants in the UK that we do today . +Pinch hitter Colin Walsh hit an RBI single and pinch hitter Ben Clowe followed with a three-run homer to left . +Most of the 350 crowd had a whale of a time , even the ones whose nappies were being changed on the specially provided benches . +" We learned from the Ronald Reagan years how generational support for a candidate can ripple through the demographics for years to come , " said one leading GOP strategist close to the McCain campaign . +Selection standards are extremely high -- all recruits must be unmarried Roman Catholic males between the ages of 19 and 30 who are able to endure grueling 24-hour shifts . +France was forced to import UK electricity earlier this year when low river water caused 14 of its nuclear power stations to grind to a halt . +It might be argued that if the market price is only an average , it should be open to anyone with better than average information to beat the market . +And to date there have been no reports of any British citizens caught up in the violence . +The important thing is the presence of light and the atmospheric conditions which affect light . +( CNN ) -- French Open champion Svetlana Kuznetsova crashed to defeat in her opening Wimbledon warm-up encounter at the AEGON International in Eastbourne on Tuesday . +Elsewhere , Dallas and Denver were beaten for the first time , while Buffalo and Tennessee remained spotless . +Libi , once an al-Qaeda spokesman , is believed to have died during an air attack in Pakistan in late January . +The Lakes Line Community Rail Partnership works alongside First TransPennine Express on the service between Oxenholme and Windermere . +14 beachfront wedding in New Jersey after visiting him in jail soon after his arrest . +Mr. Cameron calls his wife , Samantha , his " secret weapon , " and she is turning out to be just that , with an increasingly prominent role in the campaign . +More and more this is shaping up as the heavyweight bout everyone ( outside of Tuscaloosa , State College , Norman , Gainesville , etc . ) is dying to see . +Body odours also change perceptibly according to mood . +The teams will be accompanied by USAID disaster experts who will assist with assessments of the situation , the agency said . +Regulators have been encouraging banks to change the way they award bonuses to help avoid excessive risk-taking after the financial crisis . +A UCLA sports information official said Morgan would rejoin the team in Tempe for Saturday 's Arizona State game . +What he brings to " She 's Out of My League , " in addition to the geek and the gawk , is a dash of the debonair , which might seem impossible and yet he does . +Memorable moment In 2008 , Gwyneth Paltrow started an A-list stampede for his designs when she returned to the red carpet , post baby break , wearing a sexy , lace-panelled shift . +U.S. Border Patrol Chief David V. Aguilar released the data to the Senate Judiciary subcommittee on immigration , refugees and border security , noting that the number of Border Patrol agents has more than doubled from 9,000 in 2001 to a projected 20,000 by September . +Just three weeks ago , the Jaguars mauled the Steelers at their own game on a snowy day at Heinz Field , delivering to the Steelers their first loss at home this season . +Both methods are complex and rely on intricate genetic manipulation because the mammoth DNA is not suitable for cloning . +Overall , Brown led Poizner 53 to 22 % . +While other villages usually harpoon their quarry out at sea , the particularly bloody killing methods in Taiji have made the town a focal point of animal rights activists worldwide . +According to the American Cancer Society , 1,500 Americans die each day of some form of the disease . +Eriksson helped Sweden win a bronze medal at this year 's world championships and is a candidate for the 2010 Vancouver Olympic team . +In the past several years , many investors who put their money into emerging markets enjoyed annual returns of more than 30 percent , attracting capital from Japanese housewives and American pensioners . +He was convicted of a string of sex offences , including multiple rapes , against two young girls . +RYAN : And there are in every state . +Originally aired on the internet during the writers ' strike , " Dr Horrible " became a sensation , and went on to win seven Streamy awards ( given for web television ) including the audience choice for best web series . +On the eve of last week 's hearing , Kercher 's sister Stephanie read out a statement in a steady voice at a Perugia hotel to say the family was " pleased that we have reached a new phase in the process , hoping that justice will soon be done for Meredith . " +Neither player has yet been convincing in the first team . +At its best , even in the midst of all this , there is laughter too , often centred on my joker son . +( AP ) - Colt McCoy completed his first 11 passes and produced touchdowns on No. 3 Texas ' first three drives , helping to put Missouri away early in a 41-7 victory on Saturday night . +The gender wars are important but take a back seat to survival . +Amphibians lay jelly-coated eggs that are unsuited to development on land , so they must return to water in order to spawn . +NEW YORK ( AP ) - An undercover cop chased a Times Square scam artist through sidewalks crowded with holiday shoppers and tourists Thursday , exchanged gunfire with the suspect and killed him near a landmark hotel , police said . +Although industrial policy is often equated with protectionism , Ford and other speakers said the US needs to be tougher with trading partners to maintain prosperity . +Hold down the power button for ~ 10 seconds . +JAKARTA , Indonesia ( AP ) - Economic growth in Southeast Asia 's largest economy is expected to hold steady at just over 6 percent in 2009 , Indonesian 's president said Friday in his State of the Union address . +The Spaniard was dumped out of Wimbledon on Tuesday in a 6-0 , 6-0 , 6-4 drubbing at the hands of Leonardo Mayer , the Argentina player . +Endeavour 's internet home page at http : / / www.endeavourcorp.com. +He made his professional operatic debut at Glyndebourne in Richard Strauss 's Capriccio in 1964 . +So far these deals -- worth a total of nearly $ 10bn ( £ 6bn ) according to Swiss Re -- have been tailored to the requirements of the insurer or pension fund looking to offload the risk . +Buddy Piszel , Freddie Mac 's chief financial officer , said the regulator 's move would allow it to buy mortgage securities at attractive prices . +That was a key reason the unemployment rate fell so far in December . +" I want those who would use terror against British citizens to know beyond doubt that we and our allies will pursue them relentlessly , and that they will meet the justice they deserve , " Brown said in a statement . +Justice demands no less . +But another 158,000 homeowners who signed up have dropped out--either because they didn 't make payments or failed to return the necessary documents . +John Meyer of Fairfax said disruption at the Grasberg mine could reduce global copper concentrate supply by around 5 per cent and that combined problems at these mines / smelters could cut more than 10 per cent , around 36,000 tonnes of copper production a week . +Within a few weeks , Mr. O 'Brien had assembled a group of engineers , architects and managers . +Extra police and street pastors are being brought in to patrol the streets of Devon and Cornwall later . +Phelps R. Hope , vice president of meetings and expositions for Kellen Meetings , said some of his clients had opted to meet in the United States instead of taking their events abroad . +The president pushed Congress to pass legislation to more tightly regulate Fannie Mae and Freddie Mac , the government-sponsored companies that finance home loans . +A sharp rise in gasoline prices will help drive U.S. consumer prices 0.3 percent higher in October but core inflation , which excludes food and energy , should be more moderate , according to a Reuters poll of economists . +25 -- The leaders of the world 's 20 leading economies have agreed to restrict bankers ' pay but not to impose hard limits , according to the latest draft of a joint communique to be issued late Friday afternoon . +Janet Hoffman , global managing director of retail for Accenture , dubbed it a " shift to thrift " that spanned all income levels . +Each requires the written support of 12 to 15 other MPs , at least three of whom must come from a different party from their own . +Megan is more Baby Jane Holzer than Jane Eyre ; she 's a lovely , refined Yale graduate who was raised in nearby Fort Lauderdale but seems quite at ease in high society . +The number includes almost 90,000 Eastern Europeans in the last year alone , although separate figures show a downward trend in recent months . +Overall , only about one in 10 voters said race was an important factor in deciding whom to pick -- and a majority of them voted for Obama . +Charlie Sheen shares custody of two young daughters with ex-wife Denise Richards and also has an older daughter with a former girlfriend . +Tomorrow , when it 's all over , we will settle in for a long winter . +Gore , who won a Nobel Peace Prize for his work on climate change , joined the foreign ministers of Norway and Denmark in presenting two new reports on melting Arctic ice . +Violence has been a major concern as tensions rise in Lebanon , where Shiites , Sunnis and Christians each make up about a third of the country 's 4 million people . +SEOUL , South Korea ( AP ) - North Korea 's military threatened to use " merciless physical force " in response to annual military drills by the United States and South Korea that begin Monday , further raising tension on the Korean peninsula . +Hundreds of MPs , including the three main party leaders , have already agreed to repay sums totalling thousands of pounds . +The Moon is only the latest . +The Fatah supporters pelted Hamas troops with stones , surging forward even as they were met by heavy bursts of gunfire . +Ties have improved dramatically since Beijing-friendly Ma Ying-jeou became Taiwan 's president last year . +Commodities prices were mixed , while the dollar fell slightly against other major currencies . +BARBOURSVILLE , W.Va. +" It would have been one boat or the other . " +Liverpool avoided a potential Europa League embarrassment in Romania as they came from behind against Unirea Urziceni to book their place in the last 16 . +The military denied having fired at anybody in that area , however , and local Palestinians said the boy had been missing since Monday , raising questions about whether he could have been the victim of internal violence . +Presented by Numark and Beatport , Remix Hotel Miami will also feature master classes in Ableton Live , Apple Logic and Propellerhead Reason among others . +" They are not foolproof . " +Running UBS 's and Credit Suisse 's wealth , investment banking and domestic banking businesses as autonomous units with independent capital and risk-taking frameworks -- rather than breaking up the banks outright -- is a better solution . +Colin Grahamslaw , from the Royal Club , said the decision would help secure the sport 's future . +Australia have a big problem here , a big 16 stone problem who is doing pretty much as he pleases . +Energy and railroads were among the worst performers , as was Staples . +OSLO ( Reuters ) - Norway could become " Europe 's battery " by developing huge sea-based wind parks costing up to $ 44 billion by 2025 , Norway 's Oil and Energy Minister said on Monday . +Still , iPhone sales pale compared with those of established mobile phone makers , such as Nokia Oy , which sells almost 10 million phones each week , or Samsung Electronics Co Ltd and LG Electronics Inc , which each ship more than 100 million phones a year . +Robinson paid his way to New York , then at 6 a.m. +The protests were against delays caused by a baggage handlers ' strike and a walk-out by ticket counter workers . +When he arrived in New York in July 1824 , Lafayette was cheered by 50,000 people on a parade up Broadway to City Hall . +Dealers are limited to five offers per issue , and the minimum offer size is $ 1 million , with a minimum increment of $ 1 million . +The first game that Brosnan lent his voice to . +It doesn 't make any sense , " he said Wednesday outside his firm in nearby Quakertown . +" The Spy Next Door " operates on familial bonding by way of humiliating pranks , which in better circumstances might go by the name of slapstick . +" I chose Quadrangle to manage my investments because of the quality of the team and the firm 's track record of delivering value , and those are the reasons I intend to maintain this partnership going forward , " Mr. Bloomberg said in a statement . +And there is a good book , Living with Bipolar Disorder , by Dr Neel Burton ( Sheldon Press ) . +An investigation was underway to determine the cause of the crash . +5 ( UPI ) -- The Chicago Cubs have fired hitting coach Von Joshua four months after he replaced Gerald Perry , team officials said . +Give it back to the Indians . +Kooley High : " Everywhere Is Summer " The North Carolina-based hip-hop nostalgists turn a remixed version of Common 's " Everywhere " into a soulful , summery reminder of the mid- ' 90s , when jazz-influenced hip-hop outfits like A Tribe Called Quest roamed the Earth . +Its two Paris events this month are likely to produce lower revenues than last year . +PARIS ( Reuters ) - The body of the pilot of an Air France plane that crashed into the Atlantic on June 1 has been identified among dozens that have been recovered from the ocean , the airline said Thursday . +A ladder usually costs $ 1,000 to $ 2,000 , and it takes 8 to 10 weeks to make . +He gave us medical care . +For the period , Lowe 's earned $ 938 million , or 64 cents per share , in the three months ended Aug . +They never planned for the aftermath . +McLean , Va.-based Freddie Mac neither admitted nor denied wrongdoing under the accord with the Securities and Exchange Commission announced Thursday , but it agreed to refrain from future violations of securities laws . +But today President Obama told allies and enemies that world skepticism and distrust of America has been a convenient cop-out . +I don 't think the basic flight itself cost more than 20 per cent of the total -- the rest was fees , taxes and surcharges . +The results of his study appear to contradict hospital guidelines from the Centers for Disease Control and Prevention that say better hand hygiene -- through frequent washing or use of hand gels -- has been shown to cut the spread of hospital infections . +" Nobody put the brakes on . +Western observers have criticized Sunday 's election as not fully democratic but Putin said the vote was held in strict accordance with the constitution . +Han Sang-choon , deputy head of Mirae Asset Investment Education Institute , said investors ignored warnings over the past few months that trouble lay ahead for stocks and held on to funds thinking a big pay day was just around the corner . +Be it the evil or the commies or imaginary terrorist , no enemy = no GOP. because she would be harder to beat than Obama . +Scholarship America is currently piloting the use of Zinch.com with one of its community-based scholarship foundations ( called Dollars for Scholars chapters ) . +That was because an hour of weight lifting and 90 minutes of film work awaited him . +Pester power is keeping the movie industry afloat . +Elizabeth King , associate director at the Securities and Exchange Commission said she was " very supportive " of increased transparency . +She said she 's worried about the highway 's impact on her 16-year-old son , who had severe asthma when he was younger , and on several neighbors with asthma , including one who sometimes relies on an oxygen tank . +All that insistent , throbbing bass rhythm -- what can he possibly see in it ? +Developed countries are preparing to relent on their demand that developing countries agree to long-term cuts in greenhouse gas emissions in a concession that could form the basis of a global deal on climate change . +As part of a six-nation disarmament deal , North Korea is disabling its Yongbyon nuclear reactor and related nuclear facilities , and is obligated to provide a " complete and correct " declaration of all their nuclear programs , weapons , and materiel by Dec . +Many people have stayed at home rather than making the usual Christmas journeys . +The admissions process is daunting . +Millions are going to catch STIs as they journey through life . +Meteorologists are forecasting wind gusts of up to 126 kilometres an hour . +The old conception of individualism needed to be replaced by a new system in which the citizen " married his interests to the state , " in Wilson 's words . +I despise them with a passion . +The fact the Christmas post has been secured is a relief to Royal Mail , which doubled annual profits to £ 321m last year but has been haemorrhaging customers . +A version appeared on p35 of the Comment & debate section of the Guardian on Friday 27 November 2009 . +That combined with the weather could keep Schaub 's numbers closer to earth this week . +He 'd called her saying he needed to talk . +Kapalua , owned by Maui Land & Pineapple Co . , sponsored the event 's inaugural event last year , which was won by its touring pro , Morgan Pressel . +That is the same tumor-nourishing protein blocked by Avastin , the blockbuster treatment for colon and lung cancer sold by Genentech and Roche Holding . +Despite the ordeal , Mannie says he 's just happy to be alive . +Adada said he hoped 80 percent of the force would deploy by year end . +But independent observers believe the bulk of the organisation has remained united under the clandestine leadership of Hafiz Saeed . +Sales gained 37 % to $ 1.29 billion , in line with analyst estimates . +I 'm not an Obama supporter , and I certainly don 't like the spendulous bill and the bail-outs . +Although the company built its reputation catering to tech geeks , its product lineup now goes well beyond computers to include such items as digital cameras , cookware and electric shavers . +Kari Keaton is the sort of customer most businesses used to hate . +Johnny Carson , who reigned over late-night TV for 30 years , was chosen as the No. 1 TV icon . +California State University , Stanislaus president said a portion of the contract between a school foundation and Sarah Palin for an upcoming speech was taken from a recycling bin inside the office of Susana Gajic-Bruyea , vice president for university advancement . +First of all , you make a point about Medicare that 's very important . +Roethlisberger was due a $ 2.95 million bonus this month that , if paid , would have put the Steelers over the salary cap . +And then I decide . +He won a second five-year term in July with promises of economic growth , political reform and good governance . +He remembers , at 5 , seeing " The Blob " on TV and thinking , hey -- that 's the theater where I watch Disney movies . +Anti-NATO demonstrators have vowed to disrupt the summit and riot police clashed with hundreds of protesters Thursday in Strasbourg in France , repeatedly firing tear gas and rubber bullets and arresting around 200 youths . +" Where it 's hard we can 't recruit and we can 't retain " Afghan forces . +And Carey feels Friday 's game will give his side a chance to bounce back . +Should Ellis win that particular contest , what started as a difficult season will end as a golden one . +Peter Stalkus , a fellow maritime captain who has worked with him for 23 years , said Phillips is known as " the Larry Bird of Chief Mates , " according to the Boston Globe . +P : Right , that 's what it 's all about , the middle class and government needing to work for them . +Gerber Products Co. and the Food and Drug Administration have found no evidence of tampering with Gerber products . +Who knew that following politics could prepare you so well for this stuff ! +Glendale Mayor Frank Quintero also honored Lakeside for Lakeside 's generous sponsorship of Glendale 's 21st Annual Great American Clean-Up . +These White House sessions are designed to put the presidency above politics , temporarily at least . +He exaggerated , but did so because of the great fear in France that the game is up for their currency . +She said other bodies had already been buried due to advanced decomposition . +Every day can be exhausting and you 're spending all day long with children . +The assets it is selling to CVC account for less than 3 per cent of the brewer 's total underlying earnings and are a collection of comparatively small brewing businesses . +But Mr Brown will tell John Bercow , the Commons Speaker , in a special evidence session as part of the drive to modernise of Parliament , that homosexual MPs and peers should have the same privilege . +Gordon Brown last week announced plans to allow local authorities to give priority to local people , in what was seen as an attempt to head off BNP claims . +But then rain swept in again and turned the pistes mushy , forcing the postponement of the blue riband event , the men 's downhill , until tomorrow. the women 's super-combined , which had been due to start today , has been put back to Friday . +The settlement will be drawn from a government-financed insurance fund . A lawyer representing more than nine thousand workers said he was pleased that men and women he described as heroic would finally receive just compensation . +Tensions in the region are already high . +" We really donʼt know where it came from , so I guess you could consider any possibility , " said Clayton Police Chief Thomas Byrne . +The pile-up occurred shortly before 11 p.m. +Acute mountain sickness affects people at high altitudes . +He said the findings " bridge a critical knowledge gap for stem cell biologists , enabling them to better understand the enormously complex process by which DNA is repackaged during differentiation " -- when embryonic stem cells adopt specialized functions . +Again , City knew about the massive collapse coming , ignored it , raped the taxpayers for billions , lined their pockets , paid outrageous bonuses , and laughed like all hell . +His trenchant analysis was supported , naturally , by statistics and citations , but there were a couple of places where the book moved into warmly personal focus . +At New Year 's Eve at the Sandy Lane hotel , when Green chose the 12-course " degustation " menu , Cowell went to the opposite extreme and had chicken pot pie with mash . +" I 'm even more excited because I think we can take a major step defensively . +Previous research has shown that these pride and shame gestures are widely recognized around the world in many different cultures . +Urban dystopias . +The kids had been on the roof for a few hours when PMQs began . +Selected by the Packers in the fifth round of the 1954 draft , McGee spent two years in the Air Force as a pilot following his rookie year before returning in 1957 to play 11 more seasons . +The MoD was also concerned that MoD police could not ensure the safety of protesters from attacks by local residents , which had happened in the past , he said . +Partly it 's that we 're both from Pittsburgh , and both began our careers as paperboys for the Pittsburgh Press . +Higher up , lush rainforest half-hides retiring blue monkeys that swing through the lianas , until eventually we emerge on to a plateau of sweeping grassland . +The Sayre Fire destroyed 510 structures , including those at the Oakridge Mobile Home Park , the Times reported . +SEOUL ( Reuters ) - North Korea is making final preparations for a rocket launch the United States said could come as early as Saturday , pushing ahead with a plan widely seen as a disguised long-range missile test . +In comparison , his successor , Maria , a teaching assistant from Poland , seemed positively staid . +While he didn 't have exact figures , he said roughly 60 passengers were killed and more than 20 people injured were taken to local hospitals , he said . +They have asked Highland councillors to approve an allocation of £ 650,000 towards the work . +On June 17th , Mr Obama announced that he will extend some ( although not full ) benefits to the same-sex partners of federal employees . +OTTAWA , April 15 ( UPI ) -- Male nurses in Canadian hospitals and nursing homes in 2005 were much more likely to be physically assaulted by a patient than female nurses , officials said . +Robert and Tonya Harris matched all five numbers plus the Mega Ball in Friday night 's drawing , the only such ticket in the multistate game . +Paste-on hymens . +It is ENTIRELY Obama 's fault that illegal workers still routinely penetrated thousands of miles into our territory . +Four days after the bombing attempt , Obama emerged from his vacation seclusion to cite what he called " human and systemic failures " in U.S. intelligence leading up to the incident . +White House press secretary Robert Gibbs said Obama would urge lawmakers to forgo part of their August recess to continue working on health care legislation . +Liukin is the reigning Olympic champion , joining Mary Lou Retton and Carly Patterson as the only American women to win the all-around title . +The comparisons don 't end there . +To help make this year 's Labor Day holiday period a safe one , AAA Michigan will once again partner with the Michigan State Police ( MSP ) in supporting Operation C.A.R.E. +Gotham Bowl officials had high hopes for the 1962 game . +Meanwhile , the volatility in financial and housing markets is increasing the riskiness of the assets that they already hold - so that is also forcing them to cut back on lending to maintain even plausibly healthy balance sheets . +A wreath with 36 roses marks Jerry Zovko 's birthday . +The case was appealed this year to Iran 's Supreme Court . +Human rights groups condemned the event as provocative , while pro-Tibet activists argued that the leg should have been cancelled in the wake of the unrest there in March . +The inclusion of Snoop Dogg in the line-up is likely to cause controversy . +The americans are a different breed altogether and invite people with talent and zeal to come and work , the Europeans seem to dictate terms and conditions to the talented folks ! +" I don 't feel pressure , " he said . +A third House committee , Energy and Commerce , also was considering the measure today , but the road was expected to be rougher there . +" I bent down slightly under the counter , and he started panicking and pistol-whipped me a couple of times . +" The people of New Orleans needed me , " she said simply . +Now the self-styled political junkie and former Mississippi state legislator has written a book that 's more political intrigue than legal thriller . +The sex education lesson for 15-year-olds is not part of India 's national syllabus , and the exclusive private school teaching the subject is taking a risk by passing on the facts of life to its pupils . +Sri Lankan President Mahinda Rajapaksa , left , has his strongest support from the Sinhalese south . +Stir and continue adding the stock until the rice is cooked , about 20 minutes . · Remove from the heat and stir in the remaining herbs , butter and parmesan . · Preheat the oven to 150C . · Wash the pears . +Notes : @ Cleveland scored six runs in the fifth for a second straight night . ... +It was initially thought 30 pupils at the Islamic Birchfield Independent School for Girls had TB but tests have shown the figure is lower . +He said that Todd Palin submitted " objections " to the subpoena that needed to be answered before he would take the stand . +In that case , you are entitled to what you contributed , but the money your employer put up will be taken away regardless of whether you do a direct rollover or you keep it in the company plan , says Doug Flynn of Flynn Zito Capital Management . +He is handed a four-match touchline ban . +Read previous Green Lantern columns here . +Government forces struck a suspected al-Qaida training camp in the region in December , killing more than 30 in the raid . +When this fish actually makes it to plate , will sushi-lovers in Hong Kong and Tokyo know they are sitting down to a meal worthy of the kind of elite endangered dining club featured in the 1990 film The Freshman ? +" I believe this tax is not helpful at this time , " said Stefano Pessina , the executive chairman of Alliance Boots . +The innuendo-filled radio adverts for Mattesons sausages asked listeners where they would like to " stick it . " +We want permanent tax relief ! +In the opening 20 minutes at Eastlands , City appeared crude by comparison . +Beth Ehlers plays an Army lieutenant on " All My Children . " +In 1990 at the peak of his tenure , Unilever sold £ 23 billion of cosmetics , foods , detergents and oils . +Cordelle said he had received a letter terminating his services last month , after more than 12 years at SocGen . +As home prices come back down to earth , many of these borrowers will find themselves with negative equity -- owing more than their houses are worth . +Armstrong said tattoo removal is no fun , with the repeated laser treatments being described as being smacked over and over with a rubber band , or like hot grease against the skin . +This may help the pragmatic conservatives . +Thirteen other people were wounded . +One student , Marta , had a clear advantage , she was Portuguese . +Day two Which isn 't to say that aesthetic limitations can 't be singled out . +11 attacks has paid $ 6 billion to 2,880 families of those who died in the attacks and more than $ 1 billion to 2,680 injured victims . +( What I 'm trying to tell you , between the lines , Jay , is that Sarah 's kind of corrupt , and no stranger to cashing in on her situation long before John McCain had ever heard of her .... assuming he ever has . +The pay-TV group BSkyB has been ordered to reduce its stake in rival ITV by a UK court . +Last year , Pioneer announced a capital and operations tie-up with another electronics maker , Sharp Corp. The alliance allows Pioneer to procure Sharp 's liquid crystal displays . +Instead , they changed course . +Five Western states -- Oregon , California , Washington , Idaho and Montana -- pocketed more than 80 percent of the total timber payments from 2000 to 2007 . +For those aged 5 to 9 , rates of death fell 27 percent to 14.5 deaths per 100,000 in 2005 . +A penthouse in a Birmingham skyscraper is the first flat in the city to sell for £ 1 million , say developers . +Iraq is assuming responsibility for almost all the funding of large-scale reconstruction projects , and our share of security costs is dropping as well . +The supplies were being distributed Wednesday . +Do you know why one of James Bulger 's killers has been returned to custody ? +The tallest woman picks the shortest man , and the shortest woman picks the tallest man . +The swine flu pandemic sweeping the world might not have happened without a laboratory accident in the 1970s , a new study claims . +China 's exports and imports declined in May , but this was offset by better-than-expected growth in urban fixed-asset investment - indicating the world 's third-largest economy is well on the road to recovery . +This season Coach Ben Howland has waved that white flag and gone zone . +" After Simon , I can handle Jason , " Sinitta says . +He met Burlington in Italy , and their passion for classical forms can be seen at Chiswick in a grand processional avenue lined with stone urns and sphinxes . +In New York , critics referenced a pair of big Johns , Cheever and Updike , in their rave reviews , while here the play seemed a more jagged and spelt-out version of Pinter 's dreamy Old Times , with the triangle rearranged against the distaff side . +But the overall business climate is starting to change , as investors eye Africa 's growth potential , and that could lead to more options for expatriates wanting to return home , Patel from HSBC said . +During that time , David Cameron and William Hague have repeatedly said that the undertakings were being met . +" Unpredictable " seems to be the word used most often by experts to describe the outbreak of swine flu , writes Clive Cookson . +Mercado has dodged more eliminations than any finalist in " Idol " history , so she 's no stranger to danger . +And now you know how silly , plastic-surgery addicted Jacko came to his end . +" Golf is all about performing in the majors but I 'm not going to this major and thinking it has to happen , " Harrington said of his bid for a rare third major win in a row . +If you walk down Karl Johans Gate , the main drag of central Oslo , a tree-lined promenade bordered by restaurants , cafes and upscale stores , you 'll eventually find yourself face-to-face with the Royal Palace , the mammoth , cream-colored home of the Norwegian royal family . +The title search did not find the lien . +As for Ucas points , these are used mainly as a general guide to applicants . +The 12-time All-Star was appearing in his first game since admitting he took performance-enhancing drugs while playing for the Texas Rangers from 2001 to 2003 . +The company added that it was slashing staff in its North American development division from 70 people to 10 . +Fox , 8 p.m. +Schools are legally required to arrange full-time , suitable education for pupils excluded for six days or more . +The MNF said the three were thought to be involved in the construction and distribution of IEDs in the Baghdad region . +Stay in school . +He said there was no case to answer following defence submissions at the end of two months of prosecution evidence . +The island is home to the Chincoteague National Wildlife Refuge , the Assateague Island National Seashore and Assateague State Park . +He would love to get his hands on Sea The Stars . +Casa Ferreirinha 's Fernando Nicolau de Almeida created Barca Velha in 1952 , using the port grapes to make the region 's first high-quality table wine and in five decades it has only been made in 15 vintages , the most recent being the 2000 . +Foxx , who is best-known for his performances in " Ray " and " Dreamgirls , " will play Prentice Earl Sanders , one of two black detectives determined to crack a series of racially motivated serial killings in 1973-74 San Francisco , the trade paper reported . +At that point , descending hills or time spent idling will result in a fall in the temperature gauge reading . +Police raided the home of Robert Savage , at Ballycarry Street in Belfast in July 2007 , following a tip-off . +Still , print and radio news organizations need not pack up their computers . +The group , without any central direction , constructs temporary facilities every year . +That , say local brokers , could encourage people to go down the private health care route because of fears that standards in the public sector could decline . +The company has presented income tax benefit ( expense ) after adjusting for the tax effect of unusual items including charges related to restructuring , impairment and other income or charges ( " Adjusted Income Tax Benefit ( Expense ) " ) . The company presents Adjusted Income Tax Benefit ( Expense ) to provide investors further information regarding the tax effects of adjustments used in determining the non-GAAP financial measure Adjusted ( Loss ) Income from Continuing Operations ( as defined below ) . The difference between Adjusted Income Tax Benefit ( Expense ) and income tax benefit ( expense ) ( the most comparable GAAP measure ) is the tax effect of adjusting items . The limitation of this measure is that it excludes the financial impact of items that would otherwise either increase or decrease income tax benefit ( expense ) . +For example , obtaining an ex-directory telephone number cost £ 75 , while a car number-plate check to match it to an address cost £ 150 . +Terra is implementing an Interactive Call Center solution in many of the locations , and other upgrades like the Aruba Wireless Network which is FIPS140-2 compliant for their federal secured wireless network . +The restaurant serves it over crisp crostini topped with guanciale to make a perfect appetizer . +Social conservatives have criticised Labour 's record on the family , claiming that the benefits system creates a financial incentive for parents to live apart . +It is boosting sales of alcohol by accepting credit cards instead of cash . +Could I risk it ? +Her husband was a cabinet installer at Quality Woodworks but was not suspected of any crime , Varnau said . +CANYON DE CHELLY NATIONAL MONUMENT , Arizona ( AP ) -- This majestic canyon is lined with towering red sandstones . +On Tuesday , he testified at a Congressional oversight hearing on the use of performance-enhancing drugs in baseball . +Mandating write-downs in home equity loans would be a particularly bad idea , he said , because these loans were simply used to consume rather than pay for housing . +Toyota 's plug-in travels 14.5 miles as an electric vehicle on a single charge , and gets 35.42 miles a liter in mileage , the equivalent of about 135 miles a gallon . +According to the petition , it 's critical for ice floes used for pupping to remain stable until pups are independent . +On Thursday , Cairn said that it had struck a $ 310 million ( £ 190 million ) deal with Petronas , of Malaysia , to farm out a 10 per cent block off the coast of Greenland . +He was the third of the four children ( two boys and two girls ) of Siegfried Wagner and his British wife Winifred Williams , whom Siegfried ( Richard 's only son ) had married in 1915 when he was 46 and she was 18 . +It takes you from 7000 B.C. to A.D. 500 on a beautifully displayed and well-described sweep through Greek history , from prehistoric and Mycenaean artifacts through the evolution of classical Greek statuary . +The campaign : Obama was the only candidate visiting Nebraska . +Moyes 's contract expires at the end of next season , but while he has been in talks with Bill Kenwright , the Everton chairman , since the turn of the year about a lucrative new deal , an agreement has not been reached . +His reply was swift , when moments later he broke free inside the Falkirk penalty area and smashed home a high , right-footed drive from a tight angle on the right-hand side of the six yard box to give Hearts some hope of a recovery . +Biden " should clearly explain the reasons that led the Obama administration to its decision not to build a radar in the Czech Republic , " Former Prime Minister Mirek Topolanek said in a statement prior to the vice president 's arrival .. +Presented in the committee of the lower house of parliament by the Catalan party Convergence and Union on March 10 , the bill is designed , according to spokesperson Josep Sánchez y Llibre , " to protect citizens against those acts that attack their dignity or invade their privacy . " +Do not drag him because he will only resist and make the matter worse . +Caution should be observed when initiating DIOVAN in patients with heart failure or recent myocardial infarction , and in patients undergoing surgery or dialysis . +Polling day now looks certain to be on 6 May . +The region still places a heavy burden on the Kremlin 's budget today . +Northern Rock has been lending again , in a modest way , but its figures this week showed a £ 1.2 billion loss of savings , which must concern potential buyers . +And for many it will be a way to save on child care . +For all the proposals stuffed into its 1,336 pages , the financial reform bill that is headed to the full Senate will change very little for banks , Wall Street and financial firms . +The first-term Alaska governor has been accused of firing a state commissioner to settle a family dispute . +PERIA JERRY , DT , Mississippi , may be as good . +Jean-Sebastien Giguere made 37 saves before he left the game after two periods and was replaced by Jonas Hiller . +When the paramedics arrived , they were amazed that Rafaëlle had managed to continue CPR for almost half an hour ; normally the effort makes your arms numb after a few minutes . +Police are continuing to search 10 properties across the north-west of England in connection with an alleged planned terror attack . +Coach Louis van Gaal has demanded more from his Bayern Munich side despite their recent improvement . +A Department of Environmental Protection biologist warned state officials before the attack that Travis could seriously hurt someone if he felt threatened , noting that he was large and strong . +When the war ended , they danced in the streets . +Can the new generations of workers in Russia , the United States and around the world replicate the mind-set of the people who opened the space frontier ? +1256 : Manchester United striker Wayne Rooney drops to the bench and Carlos Tevez starts , while centre-back Rio Ferdinand is out and Jonny Evans starts in his place . +Observers are warning that such growth cannot continue , with Nationwide suggesting that Northern Ireland will underperform all other regions in 2008 . +The analyst firm also recognized Autonomy as the fastest growing of the leading vendors in the report with 17.6 percent growth from 2007 to 2008 . +BNY Mellon Asset Servicing offers clients worldwide a broad spectrum of specialized asset servicing capabilities , including custody and fund services , securities lending , performance and analytics , and execution services . BNY Mellon Asset Servicing offers its products and services through The Bank of New York Mellon and other subsidiaries of The Bank of New York Mellon Corporation . +And I found it immensely frustrating that one of the few great races of the season , behind Monaco and the sodden chaos of Silverstone , is in the hands of lawyers and could take weeks to unravel . +Cable companies that once rejected the firm 's ads are now having second thoughts , said Noel Biderman , president of Ashley Madison Agency . +President Tamara Mellon follows in the footsteps of Matthew Williamson , Roberto Cavalli and Karl Lagerfeld in the deal with the retail giant . +His father , Felipe , emigrated from Mexico 's Jalisco state in 1951 and within two years mastered English , then was disabled in an industrial accident when John was a baby . +If confirmed , it would be the second missile strike in two days in a tribal region . +Others add so much sugar that their calories content is ultimately not far off what were they were meant to replace . +A number of reports suggested that Britton was quitting the show because she was unhappy at being paid less and having a lower profile than Schofield . +But that view appears far from universal . +That said , " The Four Seasons " has benefited , like other ubiquitous Baroque and Classical works , from the early music world 's constant redefinition of period sound and style . +Madoff is serving a 150-year prison sentence . +That 's the message sent by an international team of scientists who say they 've discovered a protected lunar " lava tube " -- a deep , giant hole -- that might be well suited for a moon colony or a lunar base . +" With the closure of the Laura Ashley factory , we have a valuable opportunity to celebrate the legacy of Laura Ashley , at the same time supporting the economic development of the Carno region . +" Iran 's action , first the expulsion of two diplomats and now the arrest of a number of our locally engaged staff , is unacceptable , unjustified and without foundation , " Prime Minister Gordon Brown told reporters in London . +The jagged and oft-snow-topped blocks of rock that dominate the horizon in every direction make this place feel a bit like Shangri-La : a secret valley , cut off from the world and stranded in happy seclusion . +The suspicion is that Ferguson was not really sure how he would use Cole -- and it has taken Ancelotti some time to assess that , too . +Later that evening , freshly showered and pink-cheeked from the day 's exertions , I enjoy a final blow-out at the Hotel Eden in Les Praz , just on the outskirts of Chamonix . +A barrel of oil vaulted to a record above $ 147 , raising more concerns about inflation and the overall economy . +I am six years divorced now and look forward to new challenges , like meeting a new partner . +What was her favorite thing about Paris ? +But Sir Richard stressed he was not endorsing the policy of any one party . +But he and Bristol split up three months after the birth and Mr Johnston has since been a thorn in Mrs Palin 's side , using a series of interviews with US magazines and television shows to criticise her . +Several Starbucks representatives did not immediately respond to calls and e-mails seeking reaction to that request . +Those who believe Marwan was a double agent loyal to Egypt claim he provided wrong information about the exact hour of the actual 1973 attack . +Another example of this can be seen with my own district nurses . +Such statements involve inherent risks and uncertainties . +Charlotte 's Fresnel lens was moved to another lighthouse and then disappeared . +Not because they are die-hard political pundits or waiting for the truth to emerge from the mouth of the leader of the free world , but because they are playing the six-year-old State of the Union drinking game . +" While we can 't say that this was George Washington 's pipe , we can wonder about it , " Levy said . +He also had his Porsche taken during the raid , among other things , with the car later found burnt-out in West Derby . +The more vivid the view from the bridge , the greater her discomfort . +Is this difficult ? +Mr. Pan also maps the government 's efforts to paper over the horrors of its recent history -- from the Anti-Rightist Campaign begun in the 1950s to the Cultural Revolution ( which began in 1966 and ended a decade later ) to the crushing of the Tiananmen Square protests of 1989 -- while chronicling the brave , sometimes thwarted , efforts of citizens to pay tribute to the victims of those years and to pry a truthful version of the past from the gears of the government 's mammoth propaganda machine . +They will also receive 50 per cent of any sell-on for the German under-21 international who arrived at Upton Park in January , from Brescia , but has failed to make an impact . +It claimed it needed the arms to resist Israel . +Twitter users are becoming increasingly wary about the secret services monitoring Twitter . +The stream of people walking out at last summer 's Proms showed that it was less good on stage . +Students stick fingers in their ears . +This chick is going to make it illegal to be white .. +Not many out-of-state students will be around on Jan . +The European Union bans more than 90 airlines , including six registered in Swaziland , in a move to boost air safety . +Castillo , who was stopped on his way back to California , said he knows federal law requires him to be able to converse in English with an officer but he thought his language skills were good enough to avoid a ticket . +He attempted to describe the confusing politics of Cambodia in the late 1960s and early 70s , as the Vietnam War raged on Cambodia 's eastern border and the Khmer Rouge tried to recruit peasants and intellectuals angry with Sihanouk 's autocratic regime . +There were a couple of notable casting changes this year , with US bass-baritone Alan Titus taking over as the new Hans Sachs and Viennese baritone Adrian Eroed as Beckmesser . +Erdogan contradicted Babacan , telling the state-run Anatolya news agency that economic measures were not yet being implemented . +Daimon Hoyt now faces three charges of soliciting murder , the Las Vegas Review-Journal reported . +A third suspect , Ivory Coast immigrant Rudy Guede , has been convicted and sentenced to 30 years in prison in relation to the student 's death . +" That 's where our primary weight of effort is . +The ship will probably take its cargo to Asia . +One mother said : " It is the nanny state gone too far - generations of my family have grown up on Marmite . +" Israeli security forces will continue to operate in order to ensure freedom of religion and a normal way of life for Palestinians uninvolved in terrorism , " the statement said . +During the regular session , the stock gained $ 2.28 , or 6.2 % , to close at $ 39.15 in a broad market rally . +The film , as far as we can tell , is lacking any real narrative structure -- it 's just a series of shots of Zoller aiming and felling yet another faceless soldier . +It is a difficult operation , " once rescue worker said . +It meant I was approaching the border before Julie had even left the tarmac at Edinburgh airport . +Clin Res Cardiol . +It is the largest renovation in the museum 's century-long history and a transformation of its largest exhibition space , making it as much about the museum 's future as about the ocean 's . +" It is a homing receptor for lymphocytes to get to the gut . +The Cougars last won at the Coliseum in 2000 , 33-27 . . . . +( NYSE : LII ) today reported financial results for the first quarter of 2009 . +And she still sat in her living room , a brooding presence , waiting for something that was never going to happen . +This is one example of the power of partnerships in action . +" I think it is important to pivot as quickly as possible for the superdelegates or others to make a decision as quickly as possible , " Obama said , to give the nominee time to choose a running mate and plan for the party 's convention in August . +10 , 1957 , in Inglewood and graduated from Western High School in Anaheim and Cal State Fullerton . +" We will give them plenty of respect and the Slovakia game was good preparation as they are similar to the Ukraine . +Civil aviation officials in New Delhi accused Continental of gross violation of Indian security rules that prohibit pre-embarkation body checks on certain dignitaries like a former president . +In other markets , US bond yields were steady , while oil prices fell toward $ 51 a barrel , after the International Energy Agency forecast lower global crude demand . +He threw himself into fighting against the deregulation of gas and oil prices , called for tighter regulation of the insurance industry and fought for national health insurance . +" I have ignored my health for years , but recent developments have forced me to re-evaluate my priorities of faith and family . +A third party company took 288 new products and calculated the average price from eBay sales , including postal charges , the auction site said . +14 , the W.H.O. said that cholera had been clinically confirmed in more than 1,055 cases so far in Kurdistan and was suspected in more than 24,500 cases of diarrhea and vomiting . +Watch the full NBC replay of Troy Dumais ' springboard victory at the U.S. Olympic Diving Trials on June 21 . +Wilson allegedly told detectives she then went inside and fell asleep . +The Peak District is a classic British misnomer ; although it holds more than its fair share of cliffs , gorges and high moorland , proper pointy peaks with 360 ° panoramas are in short supply . +But even for Lipsett , who found the training helpful , there 's still an element of resignation when it comes to drunken clubgoers . +Sir Sean Connery has settled a lawsuit with a golf club for an undisclosed sum . +Rarely by a first lieutenant , and virtually never by a captain . +In a recent study , 15-year-olds in the U.S. ranked 21st out of 30 countries in science scores . +To the " STARS " --Come on now , loosen up after all these Great Years of Music and let us ALL enjoy - not just those with BIG BUCKS ! +Its chairman , Sir Ian Kennedy , has pledged to have a new expenses system in place in the next Parliament . +The prince , who wore desert fatigues , also visited the capital , Kabul , " the only royal in living memory " to have done so , according to Clarence House . +She said : " We picked up a T-shirt ... but next to it was this T-shirt with a dagger and blood running down the side of it . +Whether rocking spray-on pencil skirts or fluoro jumpsuits , Scutt has built a reputation for doing things her own way , and her designs have become the epitome of vixen chic . +On Friday , Hugo Chavez , the President of Venezuela and one of Mr Zelaya 's key supporters , said the talks in Costa Rica were dead and that it was " horrible " to see the " usurper " Mr Micheletti being treated with deference by Mr Arias . +Inaugural dresses worn by Helen Taft , Eleanor Roosevelt , Rosalynn Carter and Laura Bush , among others , will also be exhibited . +Each firm raised suspicion because its default rate was at least twice the average of peers in their area . +Hospital and police officials said the two wounded children were 9 and 12 . +Mr Sarkozy came up with the idea of a mini-treaty to revive the key parts of the Constitution while Mrs Merkel drew it up under the German EU presidency last year . +So I felt Uefa was wrong and welcomed the U-turn . +While board members don 't get salaries , some who are ministers get paid for speaking at church events through offerings and honorariums , Swicegood confirmed . +He and Sadeer Saleem were going to join the future bombers Mohammed Siddique Khan and Shehzad Tanweer , who had already left . +From the Carolinas to New England , forecasters called for a second consecutive weekend of choppy seas , dangerous rip currents and heavy rains as Danny was expected to pass to the east of the mainland Saturday . +Three months after the incident , a trip home to Belfast is proving a morale booster but he was not able to be there for the birth of his son Carter . +The remaining non-managerial workers would share no more than $ 1.62 million . +It had been replaced and the machine was being switched back on when the latest problem occurred . +" You have to use careful selection . +Lonnie Lynam , a self-employed carpenter in Pipe Creek , Tex . , specialized in spiral staircases . +Reports suggest Adriano may be considering his footballing future after his fortunes have nosedived . +But the governor offered a different account when spotted in the Atlanta , Georgia , airport after nearly a week out of pocket . +Ahmadinejad said this was rude . +Why is the death of people that have chosen that lifestyle a bad thing . +Days after reports of Mehsud 's death surfaced , Pakistani Interior Minister Rehman Malik told reporters that clashes had erupted in a meeting , or shura , to decide a successor and that one or both of the leading contenders - Hakimullah Mehsud and Waliur Rehman - had been killed or wounded . +Libyan leaders told Asharq al-Awsat that Gadhafi had no issues with Lebanon . +Wilkinson 's name on an England team sheet may still send ticket prices into orbit , but even in his pomp , he was not the all-seeing genius some people made him out to be . +The order states he cannot act in a manner which causes alarm , harassment or distress to any person in England and Wales . +There 's nothing new about SUP , though . +( AP ) Washington Mutual 's holding company is suing federal regulators for billions of dollars , saying the fire sale of the bank 's assets to J.P. Morgan Chase violated its rights . +Remember , when the gov 't gives me money back , that means they didn 't spend it on something frivilous . +Anxiety is also being felt among the businesses that cater to Wall Street and its high-income workers . +The mayor 's office had no comment Thursday . +A wisdom , however , that has not yet made it to the administrator 's office at Columbia . +The Knicks came out shooting well from long range , converting five 3-pointers in the first quarter , when they took a 30-26 lead . +Detective Chief Superintendent Tony Porter , head of the North West Counter-Terrorism Unit , said the raids were part of a continuing operation and that police had acted on intelligence received . +Broader stock indexes also fell . +Yettaw , who was taken to Suu Kyi 's home Thursday by officials to re-enact his visit , told the court Wednesday he had been sent by God to warn Suu Kyi of his premonition that she would be assassinated by terrorists , said Nyan Win , one of Suu Kyi 's lawyers . +There are Eastside Petsitters , Eastside Surfers , Eastside Real Estate and Eastside Studios , and for years , locals came together for an Eastside Art Crawl . +Elsewhere , two schools have collapsed , each trapping nearly 1,000 students and staff . +Guardian home exchange allows you to swap homes and like a local all over the world. wrap. cardiff bay , wales . £ 28-35K plus an enhancement to cover benefits which are normally available to permanent staff. metropolitan housing partnership-1. spirita changes the lives of literally thousands o ... . £ 15000 - £ 19999 per annum + £ 17,361 PA pro rata per hours worked. veterinary laboratories agency. south east england . £ 42,000 - £ 52,865 per annum .. gardens , antique shops , art galleries , heritage farms , working studios , wineries , farmers markets , and an arts center with year round fine art classes , are but ... . wa. park , indoor recreation facility , new performing arts center and restored movie house , and many of our ... charm of their historic heritage. live , work and play ... . oh. cultural activities at your fingertips , theatre , arts , museums , etc. sports enthusiast paradise , for ... is rich with culture and heritage , breathtaking ... . nm . +South Africa 's government has been criticized for its slow reaction to the violence and for not addressing the poverty that is widely blamed for the bloodshed . +Forterraʼs civilian platform , the On-Line Interactive Virtual Environment , is one of many products being used or developed for training first responders to cope with chemical , biological or radiological incidents . +About two dozen students also flew into seats in front of them or onto the floor . +Spector alleges that Shapiro 's work on the case was inadequate and may have led prosecutors to file formal charges against Spector . +Head south for about 150 yards and then turn left ( east ) to follow the contour along Westfield , through a coppice and emerge through a kissing gate on to the Westfield road . +Several bicycle shops rent bikes all year round from $ 15 ( 75 kroner ) a day -- http : / / cykelboersen.dk / en / . +Russia began negotiations for WTO membership in 1993 , but the talks have hit numerous roadblocks over the years and today Russia remains the world 's largest economy still outside the WTO . +Then we started getting closer . +Alice Waters , founder of Berkeley 's Chez Panisse cafe , is known worldwide for her unbending fidelity to locally grown food and organic agriculture . +Martin Brodeur glided past Sean Avery like a man preoccupied with what he was going to do with himself for the next two months until he could open up hockey 's proverbial postseason cottage on the lake . +The database , which has been seen by The Times , raises important data protection concerns . +" We find a loss of weight , loss of waist-to-hip ratio , loss of Body Mass Index -- in significant numbers even though they 're not enormous -- and a drop in blood pressure , " she says . +The wife of a Virginia school administrator was not pleased when a 17-year-old student called her home last week . +So what do this year 's casting decisions mean , if anything ? +The yield , at 13 per cent with a commitment fee of 5 per cent , a 2 per cent exit fee and a 1 per cent fee for undrawn commitments , is a heavy burden for a company which then lends its funds to its small and medium-size business customers at a far lower rate . +But the reform treaty will not come into force unless all of them do so - and it still has to run the gauntlet of a second Irish referendum later this year . +He said Sunday that another important focus of the G8 meeting would be to ensure that emerging nations like China and India sign up to measures to cap the production of greenhouse gases . +Earning A 's in her major subjects , history and international relations , she had fulfilled all the requirements for her bachelor of arts degree , except for physical education . +I had this argument with him awhile back and when I asked him back then about whether he cares if the troops are put in danger during a withdrawal he ran and hid and I expect he will do the same this time . +He does not encourage initiative ; he likes people who do what they are told . +The Menier Chocolate Factory production of the Tony Award-winning show is to open April 18 at a Shubert theater in Manhattan to be announced . +Out here , the economy rests on four fuzzy legs . +Marine reserves have been created off the north-central and south-central portions of the state . +Shareholders balked at an announcement by the National Bureau of Economic Research that the US economy entered recession in December last year . +In the mid Eighties , when she was in her teens , she cooked for her bohemian supper club of " about 20 to 40 people , every Thursday , who all paid a fiver . " +By intervening aggressively , Hank Paulson and his kindred spirits at the Fed haven 't quite ensured a continuation of the status quo -- some reforms will come , and banks and their regulators will tread more gingerly for at least a few years -- but they do seem to have headed off a re-enactment of the New Deal . +The nine foreign nationals were held in Southampton , Hampshire , on Wednesday by officers from the Serious Organised Crime Agency ( Soca ) . +Like corals , they secrete calcium carbonate . +While Americans have gotten much heavier since then , they 've been able to lower their cholesterol with powerful drugs that carry few if any side effects . +The French retailing company Auchan hired Intertek to test toys it stocks ; it discovered the first Mattel lead-paint problem on June 6 . +In " Octopus , " he sings : " So high you go , so low you creep . . . The squeaking door will always squeak . " +They have to rebuild . +I also like everyday vanilla ice cream with the sides of the sandwich rolled in flaked coconut ( do this just after you fill the cookies so the ice cream is still soft enough for the flakes to adhere ) . +The data showed a ratio of 2.9 birth defects per 1,000 live births in Kettleman City during those years . +Still , the Hornets , with Hilton Armstrong starting at center for Chandler ( ankle ) , went toe-to-toe with the Nuggets until Denver 's third-quarter run started the celebration . +ROUEN , France , Feb . +a definitive destination for advertisements from football 's biggest night. vote online for best ads during the live broadcast. capability to share and embed their commercial picks . +If someone allows water to shoot up the nose--say , by doing a somersault in chest-deep water--the amoeba can latch onto the olfactory nerve . +Data show that from Friday through Tuesday night , almost $ 48 billion was moved from prime money funds . +Long before the Kasbah du Toubkal and Richard Branson 's Tamadot opened in the nearby Imlil valley , this was the main crossroads in this part of the High Atlas . +Yusuf spoke to Reuters while seated upright in his hospital bed , where he was watching television and speaking with aides . +The cylinder thermostat should not need to be set higher than 60 ° C / 140 ° F. And dripping taps can waste enough water in a week to fill a bath . +" We remain cautious about trading prospects for the year but our cost cutting measures are having a positive impact and we are currently in line with the board 's expectations , " a statement said . +Meanwhile , Essex 's Ravi Bopara was unable to train at Green Park because of a stomach complaint and left-arm seamer Ryan Sidebottom has discomfort in his lower back . +The IRG boats aggressively approached the U.S. ships during daylight , reportedly getting as close as 200 yards . +The primary goal of this program is to discover and characterize planetary systems and Earth-like planets around other stars . +A 16-year-old boy has been charged with causing criminal damage after almost 30 vehicles were attacked in a tyre slashing spree on Tyneside . +They 're both present from the start . +The Good Schools Guide recommends Belmont Primary School . +Montoya shot Rios and his girlfriend as they slept , then walked out of the rebel camp with Rios ' memory sticks , passport and hand to present to the army . +And , as with any extended foreign travel , it does help to speak at least a few words of the language . +But in a statement issued to the BBC , the MP said he utterly refuted the allegation . +If only dieting were this easy . +It was published on guardian.co.uk at 00.05 GMT on Sunday 28 February 2010. one of the uk 's top architectural practices is loo ... . £ 20,000 - £ 25,000 depending on experience . +The measure now goes to the Senate , where passage is expected . +" I think if someone wants to evacuate , regardless of their economic situation , they 're going to evacuate , " he said . +In early 2003 , he met a collector named Tom Hoyt , who said he was a computer executive anxious to get help in building his collection . +That 's it .. how do you lose ? +The concentration of inorganic arsenic -- considered the more toxic form -- in the rice grains was nearly three times greater under flooded conditions . +But , he adds , the rise in unethical acts today may be more a function of economic uncertainty . +Easy , the set-up of the network is done simply by pushing a button . +In Caia Park , Wrexham , a community-based organisation has been founded to plan for 80 new allotments . +Ross Perot Jr , Perot Systems ' chairman , is expected to be appointed to the Dell board . +" Chelsea is a great team because it has 25 great players , and for me it is impossible to choose the first team . +The Cubs ' offense clicked in the third off starter Jesse Litsch . +Michael Longley won in 2000 and Nobel winner Seamus Heaney added it to his list of awards in 2006 . +A sports club located near the gymnastics venue for the 2012 Games could be forced to close as it is unable to keep up with the rent . +Isner 's win helped him improve on last year 's quarterfinal exit in Auckland , when he reached the main draw through qualifying , and to avenge his narrow , three-set loss to Robredo while representing the United States at last week 's Hopman Cup . +A severe shaved head made the former Neighbours star barely recognisable , but he looked super-sleek in a custom-tailored Burberry tux , dress shirt and swish shoes . +Prowse says Duncan 's charm and generosity now make him regret getting the senior Tory into trouble but that politicians ' attitudes towards their part in the expenses scandal needed to be exposed . +Mr Herbert was very persistent though and he eventually gave him permission to search in the field . +Headley pleaded guilty to conspiracy to bomb public places in India ; conspiracy to murder and maim persons in India ; six counts of aiding and abetting the murder of U.S. citizens in India ; conspiracy to provide material support to terrorism in India ; conspiracy to murder and maim persons in Denmark ; conspiracy to provide material support to terrorism in Denmark ; and conspiracy to provide material support to Lashkar-e-Toiba . +I will come talk to you , ' " Mr. Mohamed said . +Donald has not played since the U.S. Open and will also miss the British Open at Royal Birkdale this week . +How that is best done is what has divided the EU bloc of 27 nations . +He was referring to the frenetic bargaining and deal-making that is apparently taking place in the Zimbabwean capital , Harare , as politicians and the military top brass adjust to the post election political scene . +Thirty-three people lost their lives in the crush . +Griner blocked four shots in the first half ; her long-armed presence seemed to unnerve the Lady Vols . +It 's not known if the suspects were looking for the second safe , Beasley said . +As Elizabeth Palmer reports , at least five demonstrators were killed in this series of clashes with police . +For someone who has been conspicuously stable while closing out matches , rarely letting slip opportunities to win when they present themselves , Nadal 's words superficially contained an element of surprise . +Both have come on the second playoff hole . +The higher fees will make the sale and purchase of a property - the conveyancing process - slightly more expensive . +" But I said very clearly--and I 'd be glad to get a record of what I said--I said the government has to enact reform to prevent the kind of crisis we have , and there was a role for government , and I supported a bipartisan solution , " McCain said . +Ms Blears , the Communities Sectretary , announced she was leaving Cabinet this morning , fuelling the sense of crisis surrounding the Prime Minister after news of Ms Smith 's resignation emerged yesterday . +On Monday , Netanyahu repeated he had no intention of implementing a complete settlement freeze , saying any halt would be temporary , would not extend to east Jerusalem and would exclude some 2,500 units already under construction . +The media center was no more than 50 yards away . +Some Canadian companies have invested in Mexico : Bombardier has factories making aircraft parts and trains , while Scotiabank is Mexico 's seventh-biggest bank . +Many of the 120 students from the housing project have not been to school since March 7 because they fear retaliation after a reputed gang member from ABLA shot and killed another student who lived on a rival gang 's turf . +27 ( UPI ) -- Gil Morgan managed an even-par 72 in the wind Saturday to maintain his two-stroke lead through two rounds of the seniors Turtle Bay Championship . +She said the diagnosis was serious but she was confident she could be in remission quickly and hoped to get back to " my old self very soon . " +A bomb attack killed three security personnel outside an army barracks in Pakistani-administered Kashmir on Wednesday , a police official said . +Levi-Strauss ' death at age 100 was announced in Paris on Tuesday . +There are an estimated four million carriers of the virus in Europe , and between three and four million people in the United States are chronically infected , according to the WHO . +Jenkins said even if terrorists did build a nuclear bomb , it is likely to be a low-level device , perhaps one tenth of a kiloton -- about 10 times the size of the largest truck bomb -- rather than the " standard " assumption of 10 kilotons -- about the size of the bomb the United States dropped on Hiroshima . +The idea for a monumental symbol of American expansion into the West originated in the 1930s but was shelved during World War II . +Taylor Paschal-Placker , 13 , and her best friend Skyla Jade Whitaker , 11 , died after being shot multiple times the afternoon of June 4 , 2008 , less than a half mile from Taylor 's rural Weleetka , Okla . , home . +Do you have a favorite Jenna line ? +Hillary Clinton was somewhere else in China , so couldn 't provide relief from the monotony and static manliness of the delegates . +Theo Walcott and Rooney were to the right and left respectively of Emile Heskey , with Steven Gerrard and Frank Lampard to either side of the holding man , Gareth Barry , in midfield . +Barney Frank , a Democrat and chairman of the House Financial Services Committee , yesterday managed to secure a concession that the US Government will have the authority to seize stakes in banks seeking to benefit from the bailout . +The 6,000-plus candidates have been highly visible and accessible to media in the campaign ( which ends 48 hours before the ballot ) , creating some scepticism . +THE PLACE Everything is just right about Krogbar ( 112 Krog St. ; 404 / 524-1618 ; dinner for two $ 40 ) , a pocket-size wine bar from the owners of Rathbun 's next door : the amber lighting and the chic log-cabin feel ; the careful sourcing behind the anchovies , salumi and cheeses ; and the infectious enthusiasm of sommelier Jon Allen ( he 'll pour you a taste of any of his 50 wines by the glass ) . +Ages 5 and up. batteries ( not included ) . +South Waziristan is part of the lawless tribal region along Pakistan 's border with Afghanistan , and top Taliban and al-Qaida leaders are believed to be hiding there , helping plot attacks on American troops across the border . +The FDA has struggled with the issue of acetaminophen 's safety since at least 1977 , when an agency committee suggested that labels for pain relievers contain a warning that they can damage a patient 's liver . +The consortiumʼs goal to map 50 cancer genomes is 25,000 times larger than the output of the Human Genome Project , Hudson told the Toronto Star . +Note : Adapted from " Doña Tomás : Discovering Authentic Mexican Cooking " by Thomas Schnetz and Dona Savitsky with Mike Wille . +Michael Tamvakis , professor of commodity economics and trade at Cass Business School in London , said it was hard to predict an upturn . +The offer values 100 percent of Opel equity at 515 million euros and assigns the company an enterprise value of 4.45 billion euros as of the end of May . +" Only a small proportion of prison budgets is spent on activity intended to reduce reoffending by prisoners on short sentences , despite the fact that 60 % of such prisoners are reconvicted within a year of release , at an estimated economic and social cost of £ 7bn to £ 10bn a year , " says the NAO report . +A similar confrontation now looms . +Moore was a man of great charm and capability who tended to hide his talent behind a natural diffidence . +Nobody is being enriched through testifying . +BT Vision combines Freeview channels with a library of 3,500 hours of on-demand programming as well as several channel options . +You really don 't , do you ? +Madison 's Impala , a butterscotch two-door with peanut butter-colored interior and 24-inch wheel rims , is expected to make its training camp debut next week . +Pope Benedict will be given the first set of the work , published by the Vatican Secret Archives in collaboration with Italy 's Scrinium cultural foundation , which acted as curator and will have exclusive world distribution rights . +Thirty of the past 35 Gold Cup winners to have run again the same spring failed on their next start . +But the hip-hop heavyweight successfully appealed the ban and received entry clearance from an asylum and immigration tribunal in January . +The senior manager told the inquiry it had partially collapsed less than 24 hours before Ms Hume fell in . +The paid trolls here can cut and paste all day long , but the vast majority of U.S. citizens eagerly await the opportunity to repudiate Obama 's policies . +Scientists have known for a while that having a close relationship with an obese person , whether a friend or a spouse , makes you more likely to become obese . +The enduring erosion of the coastline , caused by bad water management all the way from the upper Mississippi River and the coastal canals , erodes the marshes and leaves coastal Louisiana exposed to the full brunt of future hurricanes . +Tracking her down seemed impossible . +He likened abortion to the Holocaust and the Sept . +Hernandez escaped trouble in the third and fourth innings , then led off the fifth by hitting Byrd . +Hosted by Ryan Seacrest , American Idol remains the most popular show in the United States , with around 26 million viewers tuning in to watch each episode . +Versus the dollar , sterling climbed 0.2 per cent to $ 1.6307 , having jumped as high as $ 1.64 , a three week high . +Government official Rao Iftikhar said four gunmen were killed , including three by army snipers . +First reported by Military.com , the pirates , who have halted all talks with the ship 's owners , are talking to a woman named Michele Lynn Ballarin , instead . +19 , the group had raised almost $ 4.4 million , more than seven times as much as was raised by groups opposing the ballot measure . +The state department will also be giving evidence on Tuesday to present its side of the story . +Early in the month , this planet will be negative fourth magnitude , or very bright . +" China is poised to have more impact on the world over the next 20 years than any other country , " he said . +The damaged link - part of a network of eight similar pieces - is about 2 inches thick and was cracked halfway through . +The Irish News says the scale and ferocity of the rioting has " brought shame " on all those involved . +Correction : " Mugabe and the White African " has not , as we originally wrote , been shown on the BBC . +It will indicate how companies are doing , but also give clues about consumers , simply because when Americans are working , they 're in a better position to feed economic growth . +Provisional dates for a Four Nations tournament involving Scotland , Northern Ireland , Wales and Republic of Ireland have been announced . +Wisely , Mr. Marsh , who based his film on a book Mr. Petit published in 2002 , never alludes to Sept . +Microsoft executives announced that " All You Need Is Love " by The Beatles will be released as a download for Xbox 360 consoles , with proceeds going to humanitarian group Doctors Without Borders . +In the hospital wards , many former lepers are receiving ongoing medical treatment . +He took advantage of cooling track temperatures under clouding skies at Sepang to set the day 's best lap of 1 minute , 35.055 seconds . +" ( With ) this financial meltdown and the constant barrage of negative economic news ... +This article was published on guardian.co.uk at 11.33 GMT on Friday 19 March 2010 . +Yet almost at once the brigade was pinned down in some of the hardest fighting the Army has experienced since the Korean War , and things have gone downhill since . +But Prince couldn 't talk and he couldn 't eat , and he knew something still was wrong . +Open and serve . +The damage assessment training could lead to new construction standards in Haiti that take quakes into consideration . +There is life left in the cleaned-up meaning of hot mess , which has come to mean " disheveled " or " incompetent , " as in " I was a hot mess this morning before I hit the shower . " +No matter what kind of blade does the work , the sculptures have one thing in common . +Meetings are held twice a week and conducted in Creole . +The Senate Judiciary Committee held a hearing on West last week but has not scheduled a vote on his nomination . +Sales during the quarter rose 10 percent to $ 1.31 billion , the company said . +Now I here there are some Added Delegates [ 76 ] that come in later . +At 83 , she recently suffered through a painful bout of shingles , a reactivation of the dormant virus that causes chickenpox . +The trial by the Special Court for Sierra Leone , in The Hague , Netherlands , began last June but adjourned after one day when Taylor fired his lawyer . +At the same time , the unemployment rate increased to a four year high of 4.8 per cent , and the number of jobs on offer fell to a seven-year low with only half as many jobs as applicants . +She knew where practically everything on the menu had come from , or was going ( when I eyed a slice of the " world-famous " carrot cake , she told me that she mailed it to Queen Noor of Jordan whenever she came to Washington , D.C. ) . +more than 5,000 hours of construction . +Samoa 's highlight came with a first-half try from Gavin Williams and the appearance in a record fifth World Cup for Brian Lima , on as a replacement . +He claims that the White House politicised the honour under the Bush regime . +Mike Clasper , former head of BAA , was appointed non-executive senior independent director to replace the outgoing Sir James Crosby , the ex-HBOS chief executive , earlier this week . +Sweet treats from the CD include : " Mostly Ghostly " - a wailing pipe organ introduction ; " Vampire Empire " - a succulent waltz with harpsichord , organ , and kick drums thumping like heartbeats ; and two versions of the traditional " Souling Song " - " All Hallows Version , " which explains the medieval Christian history of All Hallows with cathedral bells and organ , and " Samhain Version , " which explains the pagan origins with drums , organ , and guitars . As Kristen 's singing and compositions range from richly intense to delightfully quirky , this CD will appeal to all ages and types of Trick-or-Treaters . +Turnout at the ballot was 87 per cent , with 57 per cent voting in favour and 43 per cent against . +Clinton matched the mood of the moment after Oklahoma , and saw his approval ratings rise , after his own mid-term election drubbing by Republicans . +The cure is that we stop consuming and start saving , and that 's a recession . +Poor Indian tax payers , do not get any security with this weapons . +Kate Ferranti , a union spokeswoman , said that in those settlements the union has gone far toward its goals of creating national standards for janitors , transforming many jobs from part time to full time and getting many janitors family health coverage . +This poor performance is often blamed on the fact that millions of Americans lack health insurance . +The 1st Infantry Brigade Combat Team from the 10th Mountain Division based in Fort Drum , New York , had been scheduled to relieve another combat brigade in Iraq in January but will not deploy , the Defense Department said in a statement . +If we keep on spending without saving , many of us will be headed straight for personal financial ruin . +Hayden said claims by the European Parliament that at least 1,245 CIA flights transited European airspace or airports are misleading because they implied that most of those flights were rendition flights . +Sheikh Muszaphar reported on his Weblog that the grass around the capsule began burning quite heavily after touchdown . +Videojug is slowly setting about its aim of producing millions of videos that cater for every conceivable problem that people turn to the internet to solve ; unsurprisingly , one of their biggest categories is food and drink , but love and dating also score highly , as do beauty and style . +Currently , Hong Kong 's leader , the chief executive , is picked by an 800-seat committee under the influence of the Communist leadership in Beijing . +Of course , community values aren 't solely the province of people with children . +It may also explain why , in some cases , standard chemotherapy treatment for brain tumours works at first but then loses its ability to beat back tumour growth . +He was described by Martin O 'Neill as " one of the best players in the world , absolutely at the top of his form " on Wednesday . +First in BCS history . +" We intend the same thing to happen with the Twenty20 cricketers coming into the squad . " +Environmental group Greenpeace and animal rights activist group Sea Shepherd said they would track the hunt . +According to a regular participant in last week 's demonstrations , the protesters yesterday hoped to form large groups in side streets before bursting onto the main highway , thwarting attempts to disperse them . +But politics come down to such shallow triggers at times . +" He is thrilled not to have to go shopping , " she said . +Football star Cristiano Ronaldo 's 177mph Porsche 911 convertible is up for sale on Auto Trader . +It seems so unfair . +Although the children try to escape a society built on lying , when they arrive at the animal oasis , they find no single truth : only a seemingly safe environment , with hints of malevolence . +Do consumers suffer ? +" It was a constructive telephone conversation , " Uribe said , adding he needed to be prudent in answering whether Obama indicated he would be open to a vote on the pact this year . +A possible deal between the countries on civilian nuclear power was not mentioned . +Bits of plastic were the most frequently found items , contributing to a 148 % rise in the density of plastic waste since the survey began in 1994 . +Associated Press writer Leila Saralayeva in Bishkek contributed to this report . +Cruise portrays Claus Schenk von Stauffenberg , an aristocratic officer who mounted a failed plot to kill Adolf Hitler in 1944 as Germany was losing the war , and was executed at the Bendlerblock along with his fellow conspirators . +She was told her cancer was terminal in February 2009 and died on 22 March 2009 , aged 27 . +Andrew and I want to meet people who were there - to hear what it felt like , and what the creation of the NHS did for our country . +Consequently most of the extras we used in the film were from the actual school for the Performing Arts . +However Garry Jones , global head of derivatives , said NYSE Liffe would be prepared to sell up to 49 per cent of the equity . +" We 're burning through our reserves right now , so it 's nice to have extra help , " said Sable , who is talking to another local corporate firm this week about taking in subsidized attorneys . +White House press secretary Dana Perino said on Tuesday the JEC was " known for being partisan and political " and that Republican members had not been consulted . +Automakers have been offering substantial discounts on some models and shutting down the plants that make them to keep inventories from growing larger . +While they and their millions of supporters may be powerless to confront the system 's instruments of enforcement , their declarations raise issues that go to the heart of the Islamic Republic , its identity and values , and the legitimacy of those now running it . +Former D.C. first lady Cora Masters Barry 's nonprofit group will get to stay in the recreation center that she was instrumental in building a decade ago , a decision made after negotiations with the city . +Abhisit said there would be no " iron fist " approach to ending the violence and he urged people to prevent rebels from creating rifts between Muslims and the region 's minority Buddhists . +Daewoo Logistics complained it had already invested " not a small amount . " +Look at how people have turned their creativity loose on the iPhone . +The Free Software Foundation 's Coughlan said the difficulties of resolving the issue within the ISO framework illustrated a growing awareness that the implications of such decisions went far beyond the software industry . +West Indies closed their second innings on 96 for one , needing to bat through Wednesday 's final day to secure a draw or reach an improbable victory target of 437 at Providence Stadium . +" The last time I had a conversation with John I had to hold the phone three feet from my head , " says Randy Pullen , the Republican state chairman , whom McCain has repeatedly tried to replace . +Tan said the vessel 's 10 tanks carrying 1,700 tonnes of crude oil remained intact , but fuel had spilled from the engine system , which emergency workers were trying to contain with a floating 500-metre-long barrier . +Automakers were generally higher , particularly truck makers . +Candidates seemed secure on some aspects of the solar system and space but over 20 % of candidates thought the Sun orbited the Earth . +It is the city 's largest gay parade outside Manhattan and it has " a real neighborhood feeling , with people coming out with their folding chairs , their coolers , like you 'd see on Memorial Day , " said City Council Speaker Christine C. Quinn , who is a lesbian and has been one of the parade 's most loyal participants . +Other Hollywood figures such as director Steven Spielberg , executive Jeffrey Katzenberg , and actor Kevin Bacon are also among Madoff 's victims . +Iran today blamed Pakistan as well as the US and Britain for a suicide bombing that killed six of its commanders and 37 others in one of the country 's most unstable provinces . +You can 't even remember the last time you made someone cry . +The broadcasters account for 70 per cent of television viewing . +There is added intrigue over the more prominent of the two films that were withdrawn from the Palm Springs festival : " City of Life and Death " ( also known as " Nanjing ! +AOL president and chief operating officer Ron Grant will leave with Falco after a transitional period of a few weeks , The Washington Post ( NYSE : WPO ) reported Thursday . +Although the absolute risk of dying is still low -- less than 1 % -- such patients also endure longer hospitalizations and other complications and add substantially to the nation 's healthcare costs , researchers from the Duke Clinical Research Institute in Durham , N.C. , reported in the Journal of the American Medical Assn . +( TMS ) . The scholarships are valued at $ 20,000 ( for national winners like Sperry ) or $ 10,000 each , over four years , for study at a four-year college or university starting in the fall of 2009 . +Massive ! +" It feels like we are playing against any other team , " he said . +Britain has complained that the revised law will lead to the withdrawal of certain fungicides and cause sharp reductions in wheat yields . +No-one is talking about hard-core ideologues with links to al-Qaeda . +" I got an idea , none of us can decide who is most in love with her so we all have to make out with her and see , " Robertson joked . +But their gratitude does not extend to support for his Workers ' Party . +Those with the knowledge and hands-on experience with their horses must be ready with the right answers in this important ongoing debate . +Both the video game industry and the music industry are clambering to harness all the revenue making momentum these music games can muster . +Along with several fellow experts Dr Grant spent 36 hours sifting through the documents , comparing them to other pieces Barot was known to have written . +( NYSE : ME ) announced today that Scott Josey , Chairman , President and Chief Executive Officer , will present at the Barclays Capital 2009 CEO Energy / Power Conference in New York City on Thursday , September 10 , 2009 at 3 : 45 p.m. +And with the transfer deadline nearing , the manager is also hoping to strengthen his attacking options with some late additions to the squad . +Roosevelt Principal Kathie Danielson said science teachers " can 't be sure " the pigs came from Roosevelt . +A startup called All inPlay offers online games , including poker , designed to allow play between blind and sighted users . +After exchanging pleasantries with the Taiwanese visitor , Mr Hu said that improving relations between the sides needs efforts from both parties . +Abbas remains out of the country : Amman Monday , Yemen today , Rome later this week . +Does this mean he is losing interest in me ? +" We 've seen it before and we 're seeing it again -- ugly phone calls , misleading mail , misleading TV ads , careless , outrageous comments , " Obama said . +As key partners , the three organizations will create campaign awareness , provide participant encouragement and facilitate communication among agencies . +He said the study , which he was not involved in , pointed to a genetic switch that turns off the body 's defenses against cigarette smoke and leads to lung damage . +Jay-Z and Lil Wayne drew the second-most nominations with seven apiece , while T.I. received six nods . +Insurers and retailers were to blame for the weak performance . +A turquoise Whiting & Davis mesh and Lucite clutch was $ 79 ; the same bag on Buy.com is $ 200 . +" A lot of people just feel uneasy about troops kicking in their doors and driving like hell through the streets , " he says . +He attacked and killed Mr Blair with a carpet knife when his neighbour repeated the allegations . +Trickles of lava also rolled down the 8,000-foot mountain , which towers over the Albay Gulf in the central Philippines , and five new ash explosions shook Mayon 's steep slopes . +If I step outside myself , no . +It sounded like a large animal , or at least a rat . +( AP ) - Republican presidential candidate Mitt Romney on Tuesday said rival Rudy Giuliani " had nothing but praise " when then-first lady Hillary Rodham Clinton proposed a universal health care plan . +A government spokesman said the investigation into the Sept . +First , developing countries such as China and India must play a critical role in any global solution to climate change . +But Rangers took the lead in the 22nd minute when Kris Boyd flicked the ball into the path of Nacho Novo , who rifled it home . +A scrapbook containing what Heritage says are previously unpublished photos of the dead Che are also being auctioned . +Nancy edged Valenciennes 1-0 to climb into 14th place with 15 points , while Valenciennes slipped to last place on nine points . +EDT , valuing it at around 12.2 billion pounds . +Siena is not a team to be dismissed . +In 3 1 / 2 years , however , her quiet existence might change when all the donations sent to her when she was a baby mature into a payment of $ 1 million or more . +In May , Lockton Denver was awarded the Denver Business Journal 's Partners in Philanthropy Award , which recognizes businesses who make significant contributions to the local community . +" A little bit of patience by everyone is appreciated when you 're talking about making investments like this , " Anderson said after signing the contract with Duke at a ceremony at the state Capitol . +Their minimum wage incomes barred her from very basic student aid , like tens of thousands of others . +NEW DELHI ( Reuters ) - India 's monsoon rainfall deficit has widened further , increasing the risk of crop damage , but its impact on the country 's economy was offset by high growth in June industrial output due to buoyant consumer demand . +Mr Tomback said the bill for repairs at Apethorpe , which have involved more than 150,000 man hours , was in fact money well spent given the " invaluable " heritage of the hall . +So far , the search has not yielded any strong links , Orrey said . +Ten years ago , there were 10,000 arcades in the nation , and now the number is close to 3,000 , according to the American Amusement Machine Association . +There are about 25,000 of these , valued at £ 13bn . +NEW YORK ( AP ) -- Gillian Anderson and her boyfriend , Mark Griffiths , are expecting their second child this fall , her manager , Connie Freiberg , said Wednesday . +The government said it would prevent farm trucks carrying protesters from traveling to Bangkok and warned it would use the ISA to censor or close community radio and television channels that incited violence . +Mr Duncan Smith lost the vote the next day . +NEW YORK ( AP ) - Wall Street showed relief early Monday over the government 's plan to bail out Citigroup Inc.--a move it hopes will help address some of the uncertainty hounding the financial sector . +Don 't Read This appeared on a self-publishing website . +Some legal experts say the fact that Murray was trying to get Jackson off propofol , also known as Diprivan , can be interpreted to imply he was aware the drug was harmful to his patient . +They create programmes about exotic wildlife and locations that can be shown anywhere , foreignness being part of their appeal . +The creditors , holding 42 million dollars out of 6.9 billion dollars of Chrysler debt , argue fundamental creditor protections were ignored in a sale forced through by Washington . +Then I remembered what my mother did when she washed my hair as a child ( you are going to love this ) -- that a cup of vinegar mixed in a gallon of water cuts the soap fast . +The results were nearly identical to the year-by-year increases in humidity . +And consultants are already lining up interpreters and forging alliances with community advocates , tenant-association presidents , ministers and others who might help promote their candidates . +Adding fuel to the fire was a decision by the Texas Department of Insurance to adopt a rule allowing title insurance companies to claim a " blanket exception " on their responsibility to determine whether a landowner owns the mineral rights for a piece of property . +The winner will be announced at an awards dinner on 3 September , hosted by Lauren Laverne from The Culture Show . +Furthermore , participation of Afghan civilians is also lacking . +Elana Glatt said they had reluctantly paid for the flowers in advance , with a cashier 's check for $ 27,435.14 . +Will there be an after ? +In her speech Wednesday night , Palin belittled Sen. Barack Obama 's experience as a community organizer in Chicago , Illinois , and remarked that the Democratic presidential nominee and senator had written two memoirs but no major legislation . +Local media reports said the last stumbling block was an Israeli demand that some of the prisoners slated for release be deported to the Gaza Strip or elsewhere after they are freed . +Mr. Obama reached agreement with Mr. Nazarbayev to fly supplies to Afghanistan over Kazakhstan , with no real public criticism of the country . +to use , hardware independent , ZFS based software. information products , database marketing and processing services . -- Turn-up and configuration services are also now available. at http : / / www.nexenta.com or call : ( 877 ) 862-7770 . +A 22-year-old man has been charged with causing her death by dangerous driving and is due to appear in court again in August . +Electricity was knocked out across a wide area of the southern state stretching from Miami up to Daytona Beach , causing traffic jams as signals malfunctioned , and forcing some shops to close before power was restored to most customers by Friday evening . +The woman said the man , who appeared to be homeless , quickly ran from the porch . +As the death toll among British troops rose to 231 , Mr Ainsworth acknowledged that support for the campaign had been " dented " by recent losses . +Bush met on Friday in Peru with Chinese President Hu Jintao for talks focused in part on the global economic turmoil . +INDIANAPOLIS ( AP ) -- The drugmaker Eli Lilly and Co. said Tuesday that Sidney Taurel will retire as chief executive on March 31 after nearly a decade in the post and will step down as chairman of the board at the end of 2008 . +HORNCASTLE , England , June 2 ( UPI ) -- A British man who admitted biting off another man 's ear while dressed in a chicken suit was sentenced to 15 months in jail . +Already cautioned , he unnecessarily caught Lucas 's heels . +According to state regulations , anyone receiving unemployment benefits who works one day and earns less than $ 405 will have his check for the week reduced by 25 % . +Our Sports Bog colleague dug up his 2007 video dispatch from the America 's Polo Cup -- yes , Tareq Salahi 's organization -- and recalls crazy good times hanging out with an exuberant Michaele ( who kept shouting at the crowd , " Hi , beautiful people ! " ) . +The interest rate on one-month sterling in the interbank market rose to 6.715 per cent this week , its highest rate in nine years . +It too was published in real life by Hyperion . +Unveiling the charges against Mr Kilpatrick and Ms Beatty yesterday , Ms Worthy issued a stern lecture on the importance of telling the truth . +Johnson said the number of occupied trailers on the Gulf Coast , which peaked at more than 143,000 after the hurricanes , has dropped to about 34,000 as FEMA rushes to move people into safer housing . +HARARE , Zimbabwe ( AP ) - Zimbabwe 's Election Commission released a handful of results Monday from presidential and legislative elections , announcing an equal number of wins for both parties after a delay that raised tensions amid fears of rigging . +They believe their voices will help avoid similar bloodshed during the next election in 2012 , though early signs suggest that ethnic mistrust is again being stoked by politicians . +2000 - Crackdown on official corruption intensifies , with the execution for bribe taking of a former deputy chairman of the National People 's Congress . +The head of MI5 broke cover last night to defend the service 's foreign intelligence links with countries accused of torturing detainees , saying British lives had been saved as a direct result . +The shares will be offered at £ 14 to UK shareholders and $ 28.29 to Australian shareholders . +A Microsoft spokesman said the company had always maintained it reserves the right to exercise all options but declined to comment specifically on the DealBook report . +But Salim Abdullah , a Sunni lawmaker from the Iraqi Islamic Party , said his group is confident and unafraid of competition by the awakening councils . +Shinichi Ichikawa , equity strategist at Credit Suisse , believes investors should be wary of DPJ campaign pledges that could hurt companies , such as the opposition 's plan for a higher minimum wage . +There is no going back from that . +When robbing an illegal gaming room , take a quick look around to make sure the police aren 't in ... +First up , that if you 're going to attempt them , you have to go the whole hog . +THOMASVILLE , Ala . , July 31 / PRNewswire-FirstCall / -- United Security Bancshares , Inc . +The oil giant 's results were also boosted by higher production volumes , which rose 5 per cent on a year ago to more than 3.6 million barrels a day after the return to service of Thunderhorse , a platform in the Gulf of Mexico . +On top of potentially losing our membership of the Famous Five , there is another threat to Australia 's dignity : the very real prospect that we will lose our other major Olympic title . +Detectives have rejected that claim , and an autopsy found that Lauterbach died of blunt force trauma to the head . +Vessels able to navigate the debris-filled canals are scarce and efforts to import trucks and other vehicles have been hampered by governmental red tape . +The Scottish Football Association believes Uefa 's decision to expand the European Championships to 24 teams will prohibit small nations from acting as hosts . +But the jury -- what jury ? -- is still out . +The U.S. military said an al-Qaida-linked militant connected to his death and a plot to kill other tribal leaders Fallah Khalifa Hiyas Fayyas al-Jumayli , an Iraqi also known as Abu Khamis was seized Saturday during a raid west of Balad , and the search continued for other suspects . +They will use an Opposition Day debate to put the matter to a vote in parliament , and will be hoping to encourage rebel Labour MPs to support them . +Millions of Chinese-made toys have been recalled in recent months . +Jackson 's birthplace of Gary , Indiana , also is planning a memorial service for the deceased singer on July 10 , according to a spokeswoman for the Gary mayor 's office . +Say , if the study points out known offenders like Jose Conseco or David Segui or Jason Giambi -- who has elliptically implied his past steroid use -- there won 't be a whole lot of new information there . +Cramers put the death toll at 21 and said most of the victims were people on the ground . +The group has routinely trained gunmen -- called " fedayeen , " or fighters who volunteer to sacrifice themselves in battle -- to carry out operations in Kashmir and elsewhere in India . +This week in Europeans , " Hans von der Brelie , euronews reporter . +He did have his hands full late with Chase for the Sprint Cup drivers Kasey Kahne and Jeff Gordon , but neither could beat him on a series of late restarts , and Johnson ultimately made it look easy . +Next Article in Opinion ( 6 of 28 ) » A version of this article appeared in print on January 13 , 2010 , on page A30 of the New York edition . +The pressure for a rate cut comes as UK retailers face a crucial test of the strength of consumer demand in the run-up to Christmas . +The Orlando Convention & Visitors Bureau has a Bundles of Free Smiles promotion for hotel stays as well as shopping , dining , attractions , and golf . +Workers from nearby businesses scrambled to rescue the injured before firefighters reached the scene . +Do people really " hate " Clinton , Obama Or even Bush ? +The first time , the leadoff man Shannon Stewart nicked a foul and seemed so shaken that his helmet fell off . +But over five years of observing a group of bonobos the researchers recorded about 10 instances when a group of the apes set out on hunting trips in search of monkeys . +In honor of Madonna 's big day , we asked iReporters to share their stories about turning 50 and what the milestone means to them . +" You don 't have the gigantic gulf of difference you had earlier this decade , " said Philip A. Baggaley , a senior credit analyst with Standard & Poor 's Rating Services . +Japan and North Korea opened talks here on Saturday in a bid to resolve long-standing bilateral disputes and normalise relations , officials said . +The recovery work will begin next Tuesday in the presence of government representatives from France , Britain and Australia after a religious blessing of the site . +The Information Commissioner 's Officer confirmed that it met Google last year to discuss how Street View would be implemented . +A Merseyside Police spokesman confirmed on Tuesday the officer has been sacked . +Of course , in the late-Eighties dance music was the underground , the preserve of a small band of clubbers and DJs in London , Manchester and Liverpool . +" We haven 't always got it right . +Doing so means the items would forever become military property . +Instead , it has taken exposure at the higher level of the Heineken Cup for all these chickens to come home to roost . +Republican Pat McCrory , the mayor of Charlotte , says education was the top issue in the past . +It must ultimately be approved by the County Council . +But now the Senate is moving closer to the House position . +Analysts also are closely watching how government energy policy under the fledgling Obama administration could affect the coal industry . +Galliani said Milan wanted Beckham , one of world 's most famous and marketable athletes , for footballing reasons , not commercial ones . +Uribe will meet with Bush at the White House on Saturday , giving them another chance to urge congressional action . +But Scricca said those differences can be a source of business opportunities . +Often bargain price if you book late afternoon for same day . +We speak to Marc Gander of the Consumer Action Group ( CAG ) and BBC personal finance reporter Ian Pollock . +The first few years as a senator were difficult for Mr. Helms . +" He does not appear to understand how desperate the problem in Zimbabwe is , and the solutions he proposes are too small , " Tsvangirai said in a statement issued on the second day of the talks in Johannesburg . +The leftist party has pledged to block any attempt to privatize the industry . +The International Tennis Federation said it had appealed that decision to the Court of Arbitration for Sport , but for now Gasquet has his tour status . +The CDC is aware of 28 deaths of pregnant women and about 50 of children . +Some trader got there first and the share price has already blasted off . +Abbott is seen by many voters as a climate change skeptic after becoming opposition leader last December on the back of opposing the government 's climate change policy of a carbon emissions trading scheme . +Chances are , folks in Cleveland won 't be talking about the NBA draft Thursday . +Israel plans to free Kantar and four other prisoners Wednesday . +Of the 59,000 defendants facing crown courts in 2007 , around 20,000 " could on the face of it been dealt with by magistrates , " he said . +Spitzer hasn 't been charged in the case , but he apologized and resigned March 12 , shortly after the case became public . +Apparently surprised that some airports long have allowed guns in unsecured areas , Rep. Bennie Thompson , D-Miss . , said the new Georgia legislation represents a significant hole in national security and a threat to travelers . +I learnt that walking is companionable and draws its participants close together , that it oils the tongue and unblocks the spirit , that it challenges and probes and sets puzzles , that of all the outdoor activities it is the one most open to everyone . +With Wednesday 's blizzard compounding the effects of last week 's storm in Washington , the federal government announced another snow day . +NEW YORK , March 31 -- U.S. stocks opened higher this morning , rallying after a steep fall yesterday despite glum news from the housing sector . +BURBANK , Calif . , June 9 / PRNewswire / -- Fire up your rockets and blast off with ZulaWorld.com ( TM ) , an incredible new online destination where kids ages six and older , along with their peers , parents and teachers , can spend hours exploring the Galaxies in fun and challenging ways . From racing around Saturn 's rings while playing the Bumper Spaceships Game , to riding a skateboard at Greater Crater Planetary Park , ZulaWorld.com ( TM ) offers out-of-this-world entertainment that encourages discovery and excitement. www.zulaworld.com. +Some Reverend Wright on steroids might burst into public view . +Tim Ruskell has been pushed out as president and general manager of the Seahawks , leaving the door open for Mike Holmgren 's possible return to Seattle . +It would be difficult and dangerous for the Sunni to relaunch the sectarian civil war in which they were routed a couple of years ago . +He told the programme 's makers : " My most irritating habit is wanting things done properly all the time . +The crew members were arrested after UK Border Agency officers made the discovery . +The lawsuit alleges that three Blackwater guards on a rooftop killed three guards for the Iraqi Media Network and that 20 other Blackwater employees refused to cooperate with Iraqi officials investigating the Feb . +Apple 's Tablet : the future of computing ? 9 Can a Muslim say happy Christmas to his friends ? 3 The Big Question : Has the divide between Britain 's social classes really narrowed ? +A make-up artist helped make disguises for a number of men accused of carrying out the £ 53m Securitas robbery in Kent , the Old Bailey has heard . +In 1990 elections , her National League for Democracy won a landslide victory , but the military annulled the results and has ruled by fiat ever since . +The bank denied reports that it would post huge losses linked to US sub-prime mortgages and filed a statement with the Hong Kong Stock Exchange that said it was " not aware of any reasons for such movement . " +The preliminary hearing is scheduled to resume today . +He said : " With the bin strikes in Leeds and Liverpool , continuing action by our members in the transport sector , the postal workers ' dispute and the attack on firefighters ' jobs and conditions , it is no wonder that there are growing calls for an alternative political option that backs the workers against the bosses and the political elite . +They want to come back here and that is because the business opportunity is here . +I only have to admit that my neck pinches a bit , " he said on his website . +After spending two " hard , long , lonely " years in prison , the two said they were looking forward to spending time with their families and putting this chapter of their lives behind them . +Later , she returned to Indiana to get Lorraine and Johnny Jr . , whom the family called Jay . +" I know this , Man United is the game people really look forward to , a really important one for the supporters , and they can be decisive over the course of the season , " says Torres . +And in the real-life drama of the Israeli Air Force 's raid on a target in Syria last month , there are two particular " dogs " that have not barked . +To talk more about the issue , I called Cindy Pearson , a long-time women 's health activist and executive director of the National Women 's Health Network . +In a hopeful sign for political progress , lawmakers from Iraq 's largest Sunni Arab bloc on Monday ended their boycott of parliament over the " house arrest " of their leader , Adnan al-Dulaimi , who also attended the session . +The visiting restrictions are said to be a precautionary measure to help control the spread of the illness . +" We are in the compassion business and we are seeing families reunited , " said James Winans , development director of the famed Bowery Mission in Manhattan . +" After all this time the thought of actually finishing frightened me to death , " he said . +Tehran has said it has not been notified of any such plans . +I 'd rather have him as CoS than in the House leadership . +Streleski had been a graduate student for 19 years . +In the video , Mr. Azizuddin , wearing a newly grown gray beard , appealed to the Pakistani government to meet the demands of his captors as soon as possible . +( 296 Hanover St. , www.vittoriacaffe.com ) . +On Monday , the state-run al-Sabah daily said Iraq 's Foreign Affairs Ministry had sent a protest note to Tehran . +But the two sides have not been able to agree on the value of the company , nor who would take on the debt . +It 's more likely that the # s are , for example , 50 % approve / 40 % disapprove / 10 % neutral or unsure . +" We will field a team , " BOA chief Simon Clegg told BBC One 's Inside Sport . +A blog called Knife Tricks ( knifetricks.blogspot.com ) noted an increasing propensity toward articles about models , but we are probably still a long way from stippled versions of the Page 3 girls of the British tabloids . +Alternatively , stimulating a different area in the brain may be able to reduce tremors but not affect decision making . +It costs less to create a radio commercial than it does to make a TV commercial and local radio stations can hit more targeted demographics . +Penalties _ Smid , Edm ( tripping ) , 7 : 33 Mottau , NJ ( high-sticking ) , 10 : 18 Tarnstrom , Edm ( high-sticking ) , 14 : 07 . +But then I suppose their attitude was born out of the stupidity of Americans forcing a beautiful country to its knees while ignoring the heinous crimes they inflicted on the American red indians . +Nearly 6,000 copper alloy coins were found buried in two pots in a field at Sully , Vale of Glamorgan by a local metal detector enthusiast in April . +It derailed anyway , the first such derailment in the bullet train 's 40-year history . +A spokesman for the council said it had a duty to take companies to court if it believed they were in breach of regulations , and pointed out that Mr Whalley himself had insisted on the case going to crown court rather than being dealt with by magistrates . • A council is spending £ 400,000 on a training course that includes controversial " neuro-linguistic programming " techniques as practised by television hypnotist Paul McKenna . +For more information on how to vote on the proposed resolution shareholders should contact Georgeson Shareholder Communications Canada Inc . , the Company 's proxy solicitation agent , toll free at 1-866-676-2882 . +The search giant tests Beijing 's appetite for democracy. as long as the new and old customers to buy the corresponding product on this site , both a gift , so stay tuned ! ! 1. sport shoes : Jordan , Nike , adidas , Puma , Gucci , LV , UGG , etc. including women shoes and kids shoes . +About the difference between # 3 and the classic Monty Hall problem . +Compared with a placebo group , women taking flibanserin for 24 weeks reported a statistically significant increase in the number of " satisfying sexual events " they experienced , recorded increased sexual desire in their diaries and had less distress about sex . +However , that doesn 't mean you wouldn 't want to be aware of all your choices -- and have the chance to make the most educated decision about your health care . +Umar Farouk Abdulmutalla , the Nigerian man charged with Friday 's bombing attempt , allegedly told authorities in Detroit that al-Qaida members in Yemen engineered the explosive device found sewn into his underwear , ABC News reported Sunday . +" You find it in coatings boosting the efficiency of auto engines , in protecting electronic devices and making cholesterol-reducing drugs more effective . +The Dow Jones industrials added 181.84 points , or 1.42 percent , to close at 12,980.88 . +The link between family structure and later success in life is established by empirical study . +It appears that no neighborhood is safer than the other . +To participate and ask questions during the conference call , dial the local country telephone number and the confirmation code 4312643 . The toll-free numbers are 888-708-5692 in the United States , 0 800 051 7166 in the United Kingdom , and 800 191 83 in Norway . Other international callers should dial 913-312-0962 ( tolls apply ) . +If our current system goes the way of the dinosaurs its bacause it is a 19th century system trying to meet 21st century problems . +The Post reported that Napper 's documentation as well as the observations of three other employees and a grieving son have sparked an investigation by the Virginia Board of Funeral Directors and Embalmers . +" The risk reductions associated with some whole grain foods and fiber provide general support for the hypothesis that whole grains are better than more refined and sweetened grains for pancreatic cancer prevention , " Chan said . +The trip starts with visits to the Pyramids of Giza and the mysterious Sphinx . +" The jetliner 's computer systems ultimately failed , and the plane broke apart likely in midair as it crashed into the Atlantic " - CBS News , no one knows what happened , so why do you print stuff like this ? +At the targeted sites on Thursday , worshippers said they were relieved that the plot was halted , but they didn 't intend to change their daily routines . +There is no going back , and the manufacturing jobs that have been lost are gone forever . +But her voice was clear . +As he prepared to take office , Kennedy was concerned with not only what he would say , but how he would look , said Stacey Bredhoff , the museum 's curator . +Mr Ban has just returned from a holiday in his home country . +But I think the two of us would have a great time in a car yelling at the idiots cutting us off . +The 5.7 percent decrease was more than economists had expected . +9 Ready to Wear : Is it fair to lambast such an innovative designer in this way ? +Poland gets over 90 % of its electricity from coal . +WASHINGTON - President Barack Obama and Brazilian President Luiz Inacio Lula da Silva discussed the economy , energy and the environment Saturday during their first White House meeting . +Golf No amount of noisy parakeets , even louder jets or sunbathing nudists nearby could distract Louis Oosthuizen from claiming his first European Tour title . +He left his collection to his friend , Paul Kelly who has put it up for auction . +Q. Doesn 't this just hit the very wealthy who can afford it ? +Also , Catholic school girl outfits . +He wrestled his cousins wearing funny masks worn by the locals . +Chrysler is in the process of seeking approval from a U.S. Bankruptcy Court judge to sell itself to a new entity jointly owned by Fiat , the United Auto Workers and the U.S. and Canadian governments . +The first three days were the most difficult , John Westhoff said , especially for his father , who once briefly left the hospital after an operation to grab a sandwich . +Guillermo del Toro has been confirmed as directing The Hobbit and its sequel . +The Planning Board serves as the council 's principal adviser on land-use and community planning , and oversees the county 's parks . +Investigators interviewed a Salvadoran immigrant who has denied any involvement in Levy 's disappearance and killing . +Produced by Universal Pictures , the remake will be set in New York . +The graph in Black 's article looks like the ice is melting in the Arctic at about the same rate as last year , but your graph show a clear lag this year . +It has long been assumed that President Vladimir Putin , whose term of office expires next March , would prefer to remain in power . +" Each hand is a new hand and I 've been really focused on staying in the moment even though my phone is ringing off the hook , the media have been all over me . +That means each dress must be cut separately . +It was set up in 1980 in the Netherlands and in 1995 opened for business in the UK with an office in Bristol , where Mercury Provident , an independent bank that merged with Triodos , was based . +The debate he has called for is needed , but the more urgent matter he unwittingly raises is whether our police chiefs should be elected , and hence whether Sir Ian should be invited to join the debate at all . +" The third-generation iPod shuffle is significantly smaller than a AA battery , holds up to 1,000 songs and is easier to use with all of the controls conveniently located on the earphone cord , " Apple said in a statement . +Nicola Sanders , runner-up to British team-mate Christine Ohuruogu in the 2007 World Championships , continues her comeback from injury . +I assure my fellow readers that Dr. Kass 's sentiments do no represent those of all Christians , just as the views of some who have commented here do not represent the views of all scientists or other concerned individuals . +" People ran out into the streets because of how prolonged the quake was . +1344 : As half time approaches , Mozambique come close to scoring the opening goal . +Unfortunately , the answer to that question is probably nothing . +The project should help protect a small group of homes at the south end of the property that are regularly flooded . +Diplomats in Washington believe that the Iranian leadership is hoping -- possibly in vain -- that the US election will signal a decisive shift in relations with the West . +The tenor of the argument suggested , however , that rather than simply decide the case in favor of the state , a majority of five justices would go further and rule that the challenge to the statute , the strictest voter-identification law in the country , was improperly brought in the first place . +They want to boost the personal income tax rate temporarily by up to 75 percent , pushing the current rate of 3 percent as high as 5.25 percent . +Nigeria , Africa 's most populous nation with 140 million people , is the continent 's largest supplier of peacekeepers and the country has spearheaded or hosted mediation efforts for many of Africa 's hot spots , including Liberia , Sierra Leone and Sudan 's Darfur , where rebels have battled government and allied militia forces for years . +The defense argues Mattis was improperly influenced by a military lawyer , Col. John Ewers , who investigated the killings of 24 Iraqis in Haditha in November 2005 and later became a top aide when the general was deciding who should be charged . +Peter Williams , chief executive of the Quality Assurance Agency ( QAA ) , says the current degree classification system is " arbitrary and unreliable . " +So here is Zuckerman 's exit sign : a necessary book , in the scheme of things , but not a very gratifying one , unless perhaps for its creator . +They say he also didn 't mention his past as a Hitler Youth movement member . +Or Mr. Mailer , who reeled in Joan Didion , Don DeLillo and Gay Talese ? +Harbhajan will miss next month 's triangular one-day series in Bangladesh after being handed a five-game ban . +But a regulation now called the " 90-10 " rule , which requires colleges to collect at least 10 percent of their revenue from nongovernment sources , drove hundreds of shady operations under and lowered the proportion . +Therefore , the minister said , natural disasters would continue to occur . +Raven 's Pass , a three-year-old chestnut ridden by Frankie Dettori , out muscled his rivals in the Santa Anita stretch to win by one-and-three-quarter lengths over Irish raider Henrythenavigator . +With a penchant for Americana , alternative folk , post-rock and dream pop , they have Beach House , the Low Anthem , the Dirty Three and Explosions in the Sky on their roster . +A nine-year-old boy and his 10-year-old sister remain critically ill in hospital after a crash in the Cotswolds , which claimed seven lives . +But the proposed package of middle-class tax cuts and spending on projects like roads and bridges may be a tough sell , even in a Democratic-led Congress . +25 , leading to speculation the court will rule on the results ' validity before then , the report said . +Details of the Montgomery plan are pending , county spokeswoman Esther Bowring said , but if the program mirrors others in use , it would work like this : When drivers arrive at a designated parking space , they would call the phone number on the meter and punch in the meter number , the amount it costs to park and a credit card number . +Family and friends have paid tribute to a heart expert and his GP wife from Barnsley who were killed in a motorbike accident during a holiday in Florida . +The reader is cautioned not to place undue reliance on these forward-looking statements , which are not guarantees of future performance . +They are not voting for the spouse and I doubt -- however many price tags are attached to their high-street clothes -- whether a loving partner will swing a single punter in the polling booth , but my every instinct in 2005 was to put private life on hold and give it my all . +Even the business gloomsters will eventually have to adjust . +It is quite helpful , too , when you need emergency medical care but , of course , the patient has probably paid for that in taxes . +The tweaked bill will return to the House . +But this latest leaflet campaign seems to have touched a particularly raw nerve . +The crash site of Continental Connnection Flight 3407 in Clarence Center , N.Y. Commuter plane crashes into home , 50 people killed . +Champion trainer Paul Nicholls has yet to taste success in the National and his main hopes are Big Fella Thanks , the mount of stable jockey and two-time National victor Ruby Walsh , and Tricky Trickster . +Besides Becker , the Obama administration appointments appear to be mostly technocratic , not ideological . +Pakistan captain Younis Khan said his team were lucky to escape the attack on Sri Lanka 's team bus Tuesday . +Log on to www.hoochymail.com , a free online service that allows couples to e-mail each other customised fantasies . +Brown , who made 32 of 40 field goals in the regular season and playoffs , would not mind getting the tag again , but he wants a lengthy contract . +The campaign attracted nearly 34,700 video entries from almost 200 countries , surpassing all expectations in promoting tourism in the Australian state . +For depression , the drugs are used to augment antidepressant therapy , but Simon said there are several other options , including switching to another antidepressant . +Yesterday , its Bertolli olive oil business was added to that list . +It is marketed as a plant food . +For Storrie , the proof will only come on the debt-repayment dates , when he asks al-Fahim for the money . +She has to watch herself and also watch Bill Clinton to make sure he is in line . +Visitors also learn about visual effects and the company that pioneered digital moviemaking . +That has prompted the Polish government to lead the battle against moves to reform European Union climate laws forcing its coal-burning utilities to pay for permits to continue emitting such large amounts of greenhouse gases . +" England would be a game of massive interest to the fans and while it is not in the bag yet , we are closing in on an agreement . +Mehsud 's aide Kafayat Ullah told The Associated Press a day earlier that Mehsud was killed with one of his two wives Wednesday in his stronghold in the South Waziristan tribal region . +The company 's quarterly profit plunged 64 % to $ 940m . +Dennis Quaid stars as grumpy English professor Lawrence Wetherhold , who has been generally disagreeable to just about everyone since the death of his wife several years earlier . +Martina Navratilova has revealed that she is being treated for breast cancer . +That is why we are working towards a bilateral trade target of £ 60 billion by 2010 . +He also tried to remove J. Edgar Hoover 's name from the FBI building after learning that Hoover had the FBI spy on a former Senate colleague . +Right now , airlines schedule more flights than the runways , terminals and air traffic control system can accommodate . +" That 's how I built my business and that 's the approach I 've brought to a city government that was insular , provincial and married to the conventional , " Bloomberg said , offering a hint of how he might wish to transform Washington . +In the absence of a time-sensitive problem , she said , the commissioners did not have much incentive to push a change through . +No , because it would mean admitting that some teachers need a kick up the backside and some need the sack -- a taboo in the unreal world of the NUT . +The baker , Lobels , said it had exhausted its 4,000-ton reserve of wheat flour and had but two days ' supply left . +" The very idea of competing nation-states that scramble for markets , power and resources will become passé , " he writes , introducing a reasoned plea for one-worldism . +The researchers based the stud book on a review of the handwritten zoo records in Rabat kept from 1969 to 1998 , plus a detailed review of breeding records across zoos worldwide kept from 1974 onwards . +National Express said engineers would be working overnight on Friday . +It could have been 20 minutes or an hour . +China has 24 reactors under construction with total generating capacity of 25.4 gigawatts , he said . +It was NOT meant as a put down to him , but saying that what it represents ( this collection was 50s based ) is a woman who was of HER time , not this one . +While that philosophy still exists to some extent , more commanders in today 's military are trained to be able to spot mental problems , said Brown , who speaks on the issue for the Iraq and Afghanistan Veterans of America . +So civilian suffering and loss of life continues , and the chances of any kind of political settlement recede . +Click here to read more from The New York Times . +Now if I try to sell it I will lose a lot of money , " Li said . +America is now likely to play a major role in Europe 's 2016 ExoMars mission . +Ultimate All-Star Interactive Luncheon -- Step into the kitchen and play sous-chef to an all-star lineup of Wynn Las Vegas and Encore Las Vegas chefs at this incredible interactive luncheon at Wynn | Encore with Paul Bartolotta , Frederic Robert , Alex Stratta and David Walzog . +Longley said she shudders to think what could have happened if there were complications during the birth . +Search-and-rescue operations were to continue at least through the weekend . +Hey , it 's all professional . +Beijing and Shanghai-based gold industry analysts said the country had almost doubled its bullion holdings . +" The policy of the Department of Defense is to treat prisoners humanely , and those who have violated that policy have been investigated and disciplined , " he said . +It will be fun to watch how this develops . +The president also was to visit the Berkeley Plantation in Charles City County , Va . , where it is claimed the first Thanksgiving took place Dec . +Excluding the aforementioned adjustments , on a non-GAAP basis for the three months ended June 30 , 2009 , PAREXEL 's consolidated service revenue would have been $ 268.4 million , operating income would have been $ 35.5 million , taxes would have been $ 10.9 million , net income would have been approximately $ 15.6 million , and earnings per diluted share would have been approximately $ 0.27 . +On the plus side , Sheridan coaxes two genuinely astounding performances out of the child actors Bailee Madison and Mare Winningham , playing Sam 's daughters . +" A motorcyclist riding a green-coloured Kawasaki motorbike and a pillion passenger were involved in a collision with a number of other vehicles , some of which may not be aware and have left the scene , " the spokesman said . +His spokesman Father Michael Canny said he felt " it did not go far enough to condemn violence and people who propose it to be a legitimate means . " +At the time Hunter Biden was receiving consulting payments from MBNA , he also was a Washington lobbyist at a firm he had co-founded . +In a blog post on Thursday , Icahn had downplayed the annual meeting 's importance and said he would not attend . +Like a trainer who promises a washboard stomach in 60 days , Mr. Mount throws out a short Latin inscription -- the epitaph on the tomb of the great Renaissance humanist Leonardo Bruni -- and swears that anyone who makes it through his little books will be able to translate it at the end . +Matt Cullen got the lone goal for Ottawa . +Los Angeles County Fire Captain Mark Savage says Santa Ana winds spurred the blaze to jump a fireline early Monday and burn toward the freeway and populated areas , most of which have already been evacuated . +It was last updated at 10.28 BST on Wednesday 15 July 2009. morgan hunt. a vacancy for an experienced geography lecturer ha ... . £ 24500 - £ 35000 per annum. bedford borough council. bedford . £ 30,546 - £ 33,328 pa + Essential Car User Allowance. badenoch & clark. badenoch & clark is looking to recruit an exec ... . £ 600 - £ 900 per day . +Many of them are working in Kaohsiung County , a mountainous farming region in southern Taiwan . +He has avoided any public scolding of his hosts . +Get Me Out of Here ! experience reminded her of Peter Andre . +Make a couple of calls ! +In a second round which saw one major upset - French favourite Gregory Bauge being pushed into the repechages by diminutive but wily Malaysian Azizulhasni Awang - Britain 's Matthew Crampton also qualified directly for the quarters . +NEW YORK - Madonna 's love letters and erotic answering machine messages to an ex-boyfriend are up for sale in New York City . +On Tuesday , former workers will gather outside 10 Downing Street and the Senedd ahead of a review by actuary Andrew Young to see if financial help for people who lost their pensions could be increased . +" I 'm glad I could do that for my team-mates especially for how hard they 've worked in the last few days , " added the Team Columbia rider . +The breaking of the embargo , a rare agreement in Britain 's usually free-for-all media environment , infuriated the military . +The Town Council has been under fire before . +Last month , a suicide bomber blew up a vehicle outside the main gate of a military base used by North Atlantic Treaty Organization troops near the Kabul airport , killing several Afghans . +Johan Franzen then had a chance to continue the shootout , but his shot was turned aside by Mason , who had 26 saves in regulation and overtime . +But the results are likely to solidify in investors ' minds ( and probably those of regulators and policymakers as well ) the notion that GS is truly different from other markets-oriented banks . +But one leading political analyst doubted Bloomberg will run because he in all likelihood cannot win , noting there is no precedent for a successful independent campaign and suggesting Bloomberg might simply siphon enough votes from the Democrats to hand the election to the Republican nominee . +J Ren Nutr 2007 ; 17 ( 6 ) : 372-380 . +The IPL governing council -- basically Modi in another name -- has increased its number of overseas players to ten per team . +But the Palestinians demanded Israel first commit itself to ceasing all settlement activity , as called for under the long-stalled " road map " peace plan . +Van Commenee said : " It would be silly as a foreigner to come in and dictate and I won 't shut my eyes to the great history of British athletics . +The sanctions , imposed for alleged human rights violations , electoral fraud and suppression of political opposition , include travel bans and the freezing of assets for Mugabe and his allies . +Fifteen out of 30 DJIA stocks were down in midmorning trading , but overall index was up 44.89 points , 0.42 percent , to 10,654.54 . +" I 'm here because this is the only festival in which you can declare your Russianness . +" Birds provide an accurate and easy-to-read environmental barometer , allowing us to see clearly the pressures our current way of life are putting on the world 's biodiversity , " said Mike Rands , chief executive of the alliance of conservation groups . +STAR spokeswoman Jannie Poon said Fox STAR Studios will operate from several locations , with its first unit based in India 's Mumbai , home of the Bollywood industry . +The stimulus will provide some relief in the second and third quarters , but will not follow through to 2009 , it added . +At 317 ¼ p , the shares sit 5 per cent above NAV and offer a 6.4 per cent yield . +Commodities were mixed as the US oil price remained below $ 50 a barrel amid lingering fears about the US car industry , but copper climbed 3.7 per cent to regain $ 4,000 a tonne and gold rose modestly . +" We 've covered most of the aspects of human life , " he said . +I wonder why ? ? ? ? ? ? ? +On Monday , Fame 's share price hit its permitted daily ceiling on Indian bourses , having already tripled this year . +If they get up a head of steam , opponents have trouble bringing them back to within striking distance . +She recently had two shows of her visual art in L.A. galleries , much of it assembled collage-style with disparate images and ideas colliding , as Picasso also did in his Cubist period . +But the Center for Strategic Research , led by former nuclear negotiator Hasan Rowhani , said Ahmadinejad has attempted to downplay the role of his predecessors in developing Iran 's nuclear program , which was started in the 1980s under former Prime Minister Mir Hossein Mousavi , the president 's leading election challenger . +The photo finish in Minnesota 's Senate race came after months of intense campaigning and millions of dollars in ad spending . +So many newcomers for Colorado and the wear and tear became more evident as the season began to grind the kiddie corps down . +Like many people , you 're possibly afraid to share your views on animal experiments , because you don 't want anyone digging up your grandmother 's grave , or setting fire to your house , or stuff like that . +We were even more excited when we went to the site where Dr. Guillotine--yes the very inventor of the grisly machine--had his shop . +Colin Gant , 41 , of Lowestoft , who drives more than 500 miles a week in his job , was forced to take detours to avoid the detection devices . +The good roads take you toward and away from it , making you play a game of topographical peekaboo . +The results are unlikely to be known until next week and will determine the next stage of the investigation . +France and the UK want similar one-off taxes to be adopted across the European Union . +In 2008 , Oxycodone had U.S. sales of approximately U.S. $ 2 billion . +At Chelsea , similar tactics were not needed . +The recommendations by the all-party Parliamentary Football Group would , if implemented , prevent takeovers similar to those of the Glazer family at Manchester United and Tom Hicks and George Gillett at Liverpool . +The unwritten rule is to leave the lavatory as clean as you found it , which means at least wiping out the sink with a paper towel . +" I don 't think I barely resemble myself to myself or to my family , " said Kay Warren . +There are only 1,460 pairs of eight-eyes and 1,461 pairs of three-eyes available globally , per color . +One of the biggest businesses in sport , announced Roger Goodell , league commissioner , was bucking the economic recession . +In a scrappy opening to the game , Chris Baird made the first attempt on goal after eight minutes , striking a low shot from outside the area which Al Habsi got down to his left to save comfortably . +And they understand that freedom really works . +Shannon was last seen on 19 February on her way home from Westmoor Junior School in Dewsbury , West Yorkshire . +Hazelton 's home took 10 weeks to build . +Some of the largest mortgage servicers are scrambling to make the most of this shift . +But in recent months , the group has made a point of branching out beyond music copyrights . +Analysis revealed that both his parents originally came from the Windward Islands in the Caribbean . +Every close line call went against her , including a serve that looked wide but was called good and gave Safina match point . +I think that the only way to rectify the AIG mess is to change its present management and replace them with trustworthy competent people because no one is indispensible . +Sereno also unearthed the Nigersaurus , a plant-eating dinosaur with a huge jaw studded with 500 teeth that lived in the same geologic period , the Cretaceous . +European runner Man of Iron got the upset-filled Championships off to a minor surprise by winning the $ 500,000 Marathon by a nose over 9-year-old Cloudy 's Knight . +Dr Bekinschtein , who began the work as a PhD student in Argentina , said they also found that patients who started to anticipate the puff of air were more likely to show signs of recovery later on , in terms of increased ability to communicate . +But another national teachers union and a group that has supported the law 's goals of holding schools accountable for student progress criticized the proposal . +Three high-profile late withdrawals have , however , taken the shine off some events . +The " right direction " number is up 8 points since February and a remarkable 31 points since October , the month before Obama 's election . +The point is not that the polls were " wrong , " but that people weren 't honest with the pollsters . +" A lot of this has to do with communication , " says Mahesh Rangarajan , a political analyst and professor of history at Delhi University . +You should consider the letter of instruction as a master list , detailing where you keep your financial documents . +His ability to anticipate the location of Karlovic 's serves improved as the game went on . +Moreover , Looking For Eric boasts the most rousing finale of any Loach film . +I put my mother in the car . +Friday and took off his boots , probably so he wouldn 't make any noise while sneaking around , police said . +One of the names they gave out was Paul Farrelly , MP for Newcastle-under-Lyme , who said yesterday that he had neither signed , nor even seen , any letter calling on Mr Brown to resign . +Messages left at a telephone listing for a Karen Robbins in Clinton and with the county public defender 's office were not immediately returned . +The job is a Treasury appointment , although Gordon Brown is taking an interest . +This time , Webb 's throw beat the runner but it could not be held by Tracy , allowing the runners to move up . +A Beijing police spokesman confirmed to AFP an American tourist had been killed and the other two had been injured , but gave no further details . +The premise is that the hospital cannot succeed without a successful city . +Everybody KNOWS ! ! ! +The new subfamily , one of 21 ant subfamilies , is the first new one to be named by scientists since 1967 . +Minnesota legislative leaders have indicated that any stadium-financing assistance would be addressed in 2009 , said William Lester , the executive director of the Metropolitan Sports Facilities Commission , which owns and operates the Metrodome , the longtime home of the Vikings and the Minnesota Twins . +However , nine starters on defense are expected to be back next season , including sophomore cornerback Asher Allen , who had two interceptions , and freshman linebacker Rennie Curran , who had a pair of sacks . +Yinka Saunders , 24 , admitted stealing the cash to pay off £ 40,000 credit-card debts run up by her partner . +From a classified church to a seventies concrete community centre , Norway is developing new ways of renovating buildings saving both money and the planet . +KBR moved to have the claim heard in arbitration as Jonesʼ contract provided . +Nice. if he had a legitimate film career then he wouldn 't have to do those lowly tv ads just for the money . +There were times , White admitted , when he got nervous . +Moscow retains full control of Abkhazia and South Ossetia and says it has the right to establish an buffer zone beyond taking in stretches of the highway linking Tbilisi to Poti . +It 's probably the most powerful development since the Industrial Revolution , but I was a little slow to see it coming . +Neil Hendy , the Bhs design director , left the business to join Marks & Spencer in April last year , and Beverley Churchill , the marketing director , left in October 2006 after only a year in the job . +Obama has a personal interest in both sites . +A visit to Martin O 'Neill 's side , despite their Boxing Day humbling at the Emirates , is hardly the ideal fixture for Benítez , though , given his side 's fragile confidence . +Sophomore Tiana Myers scored 16 points and sophomore Sara Mostafa posted her second straight double-double to lead the Colonials . +By 11 : 30 p.m. , Sen. John McCain , the Republican front-runner , has won seven states ( his home state Arizona , Connecticut , Delaware , Illinois , New Jersey , New York and Oklahoma ) , former Gov. Mike Huckabee has won four states ( his home state of Arkansas , Alabama , Georgia and West Virginia ) and former Gov. Mitt Romney has also won four states ( his home state Massachusetts , Minnesota , North Dakota and Utah ) . +My mother does not even have the strength to come to court . +" I 'm not a performer , " he said . +8 ( UPI ) -- The U.S. game show , " Deal or No Deal , " will go into national syndication next fall , NBC Universal Domestic Television Distribution said Monday . +( 28 ) " Castle , " ABC , 10.96 million viewers . +Asked whether the incident-ridden game might have been his last in charge , Domenech said his only immediate plans were to marry his partner , television journalist Estelle Denis . +I would recommend that you choose your surgeon carefully and do your research on that Doctor . +His accuser , Brian McNamee , brought two photos showing syringes and vials and even a crumpled beer can in a bid to bolster his side of the story . +Pledging to uphold The Lessig Principles as specified in the ccMixter.org RFP , start-up ATM is taking stewardship of the Top-5 Free Music Site . +Bearded & Tschorn : The Times ' coverage ... +NEW YORK ( Reuters Health ) - People with cancers of the head or neck seem to have better survival odds if they have private health insurance , research hints . +Wasps made a bright start and Mark van Gisbergen scored on five minutes . +The baby will be his eighth . +United Nations envoy Ad Melkert said the elections were well-organized and orderly , the Daily Telegraph reported . +Now Dennis , who stood down as team principal in favour of Martin Whitmarsh earlier this year , fears the title is already out of Hamilton 's reach . +It 's the American dream . +Google met Mr Thür in the run-up to launching Street View in Switzerland in September , and was initially given the green light . +And trust me , when this finally evens out years from now , the average American will have amnesia and make the same bad decisions . +He won everything in his career and the only thing lacking was the world record . +" You have to look to where we 're going to be , " Gore urged the Bali delegates . +Browsing through Ibis 's small list is like wandering into a literary cafe filled with a vital spirit of intellectual engagement . +GM has begun a search for a permanent CEO although Whitacre has told directors he is prepared to see the automaker through the next stage of its restructuring and possibly through June when it is due to pay off $ 6.7 billion in U.S. government loans . +PEMBROKE PINES , Fla . , Aug . +organization of its kind in the country. improves the health of Minnesotans by reducing the harm caused by tobacco. with 3 percent of the state 's 1998 tobacco settlement . +The research from the International Institute for Environment and Development ( IIED ) , a non-profit London-based think tank , challenges the common perception in the developed world that waves of refugees will try to move there permanently to escape the impact of global warming . +And going . +Revisions to the back data saw the Q4 year-on-year GDP growth revised to -3.3 per cent from -3.2 per cent . +Bond yields were mixed , with the yield on the two-year Treasury note rising 1 basis point to 0.955 per cent and on the 10-year Treasury note losing 1bp to 2.874 per cent . +He estimated that he could process 5 billion coins annually , separating out 1.2 billion copper pennies . +Rude people who just care about themselves and everyone else be damned . +Descent International had been struggling for months but continued to take deposits as it attempted to finalise a rescue plan . +There have already been a number of protests by haulage companies in Wales and the UK over fuel costs . +" This is the worst tragedy since the start of the Millennium , " said Guido Bertolaso , the head of Italy 's Civil Protection Department . +Associated Press writers Gary D. Robertson , Estes Thompson and Martha Waggoner in Raleigh ; Page Ivey , Susanne M. Schafer and Jim Davenport in Columbia , S.C. ; Bruce Smith in Charleston , S.C. ; and Karen Testa in Boston contributed to this report . +Following domestic and international pressure , the station was reopened . +Football has been a way out for dozens of young men from this sugarcane farming country along Lake Okeechobee , a place so poor that many people can 't afford cars and merely hop rides on pickup trucks around town . +After Walthamstow closes , only Wimbledon in south London and Crayford and Romford , on the city 's fringes , will remain in the capital . +The Leicester forward suffered the injury during England 's defeat by Wales in the opening match of this year 's Six Nations tournament and a consultation with a Swedish specialist has confirmed his worst fears . +China and India have a perfect right to retore original names of their cities and not retain the distorted names the English and other westerners gave who were unable to pronounce Asian names . +" The fact of doing a guy who , when he was 40 , had an affair with a 12-year-old . . . that was the challenge that as an actor you kind of gravitate toward . +The important thing is to get it right , not to get stuck on some specific day or some specific time , " Conrad said . +She opened with an 11-stroke victory in Singapore , defended her Safeway International title with a seven-stroke romp , won the season 's first major by five shots and then led from start to finish last week in Mexico and won by 11 . +The technology would also be available to iPod Touch users , although they would have to buy a microphone and headphones to make calls , PC World reported . +" I don 't usually bet , only the odd pound here and there . +Dominic Grieve , shadow justice secretary , said : " The public will rightly be shocked to learn that not only are the Government failing to deport foreign prisoners but taxpayers are also paying the price for the Government 's incompetence . " +" As interest rates come down , banks will want to lend more to consumers , so you will see a pick-up in loan growth in emerging markets , " he adds . +Much of the damage came when massive storage tanks , stainless steel vats more than 15 feet high , toppled . +A number of the dealerships have already closed . +Oakland is trying to return to normal after a horrendous and atypical bout of black-white racial tension . +The terrier had to be rescued from the jaws of the bull mastiff by both dog owners and was uninjured . +" I don 't think I want to see our party go back to that kind of campaigning , " Romney said in his most pointed rebuttal yet to front-runner McCain 's claim that the former Massachusetts governor favors a timetable for withdrawing troops from Iraq . +Hundreds of research studies have shown that mantras , prayer and meditation techniques can successfully lower blood pressure , help decrease chronic pain , anxiety , digestive problems and ease emotional problems . +Evan Longoria and Carlos Pena had solo home runs for the Rays , who took a 2-1 lead in the best-of-seven series with the four-homer attack . +BUSH : And that 's where we are today . +A colder feeling day with temperatures of eight Celsius . +For the Welsh National Opera ( 1967 ) he sang Don Pasquale and for Glyndebourne Touring Opera ( 1968 ) Dr Dulcamara in L 'Elisir D 'Amore . +In this case hardship was ameliorated by a hope adapted from the biblical dream of Zion , that someday blacks might return to a land from which they were exiled : Ethiopia . +She said she heard male TSA agents snickering as she took out the ring . +Chicago Police commander John Lewin runs the information services department . +Now Cox-Foster says she and colleagues are trying to reproduce CCD 's effects on bee colonies by seeding healthy hives with the agents -- the biomedical equivalent of getting a killer to confess . +Schools also receive 20 cents a lunch to spend on government-issued food . +A 12-year-old girl had returned to Paignton Community and Sports College for two days following a trip to Mexico , unaware she had the virus . +To date , they have preferred breezy slogans . +Obama was asked , twice , whether the Bush administration engaged in torture , which would be illegal . +Department of Homeland Security Secretary Janet Napolitano will announce the nomination Monday with Harding by her side , according to one administration official . +She told officers that Rodriguez had been taken by ambulance to a hospital for a mental evaluation a few days earlier but left without being discharged . +But the item always eventually turned up . +One hedge fund reckons they will provide 70 % of profits in the long run . +He found a conspicuous absence of black comedy shows , so he started plotting how to fill the sketch-comedy void left by Dave Chappelle 's abrupt exit from series TV in early 2006 . +Profile America -- Saturday , November 14th . Each year , November marks National Diabetes Month . The goal is to make the public aware of the serious nature of the disease , and how to detect and control it . When our bodies are unable to maintain a normal blood sugar level , many complications may follow , including kidney failure . The disease is also the leading cause of new cases of blindness . Diabetes in the U.S. is on the rise , and some public health experts even refer to it as an epidemic , fueled by our overweight and inactive lifestyle . Across the country , more than 75,000 people a year die from complications of the disease . You can find these and more facts about America from the U.S. Census Bureau online at http : / / www.census.gov. +Those toxic assets , according to TIME 's calculations , come to around $ 37 billion on U.S. bank books . +Sowell is charged with attempted murder , and two counts each of Rape , Kidnapping and Felonious Assault . +Abortion is legal up to 22 to 24 weeks in Spain , Switzerland and the Netherlands . +His next project , Smith House , Vancouver ( 1966 ) , for the artist Gordon Smith , was a stripped-back cubist structure of glass and timber built into a hillside and strongly-evoking Frank Lloyd Wright 's masterpiece , Falling Water in Pennslyvannia ( 1934 ) . +Ben Bernanke -- for it was he , of course -- has found himself in an even more privileged position to learn such lessons . +Many of them , supporters of the change pointed out , were in traditionally conservative states such as Alabama , Georgia , North Carolina and Tennessee . +BRIC nations represent about 40 percent of the world 's population and 20 percent of global economic output but , as a group , they are just beginning to flex their muscle , after the first summit of the four in Russia last year . +In addition , the company 's former outside lawyer from Mayer Brown , Joseph P. Collins , received a seven-year sentence after a jury convicted him of helping to hide the debts from investors . +But investigators say Abbott had no intentions of staying in jail for long . +He was referring to his greatest asset as a fast bowler : his speed . +Large volumes are stored in cylinders of underground storage facilities . +But she had a double bogey on her 13th hole of the day after she hit her second shot over the green into heavy rough on the 518-yard , par-five No. 4 . +Hey moron .... if we went into Iraq for the oil ..... tell me why all the oil being pumped is still going to the same places it was before we invaded ? +The truck pulled over and the driver and a passenger got out . +When Long entered the room to take orders from the group , she did a visible double-take upon seeing Noonan . +The singer has been in the headlines all year after reports of drug use , rehabilitation treatment and cancelled performances - as well as awards success and a best-selling album . +But many feminists and civil rights leaders cite a long history of separate and unequal education for girls , and argue that segregation will perpetuate damaging stereotypes . +He 's super supportive : He 's got a career and is secure in his work life , so he 's totally supportive of your ambitions . +In the final weeks of the campaign , the Arizona senator struck a more negative tone and , along with vice presidential running mate Gov. Sarah Palin of Alaska , began attacking Obama 's relationships and judgment . +The Kings fell behind 3-0 in the first period against a longtime antagonist . +Tony Hillerman was one of the great novelists of the American South West , using the medium of the modern detective novel to explore traditional Native American beliefs , including notions of witches , ghosts and " skinwalkers " or shape-shifters , and the continued survival of such beliefs and practices within the borders of a modern international superpower . +One good effect of high gasoline prices ? +The yield on the 10-year Treasury bond dropped to 2.919 percent from 3.041 percent Friday and that on the 30-year bond eased to 3.649 percent from 3.722 percent . +It withdrew medical and engineering military units from Afghanistan last year . +But Russo insisted that " a new meeting must take place . " +And then another one . +Osher and Rami were hit in the street . +Detectives said they want to hear from anyone who was in the area at the time . +Sharif , then in his second stint as prime minister , refused to let an airliner carrying his dismissed military chief , Gen. Pervez Musharraf , and 200 other passengers land in Pakistan . +Leave us guys out ! +After Barnes gave the Suns a 113-109 advantage by making two free throws , Houston missed two shots and committed a traveling violation to wipe out its hopes . +Among Chinese developers listed in Hong Kong , China Overseas Land dropped 5.6 percent , Shimao Property fell 7.3 percent and Guangzhou R & F Properties tumbled 11.4 percent . +The uncharitable view would be that , guided by an acute understanding of the nature of commerce , servers are told by restaurant managers to hustle clients through a meal and as many bottles of wine as possible . +It was not possible to confirm casualty figures because of the remote locations of the fighting and the dangers of traveling there . +Industrial demand for silver , used in gizmos such as mobile telephones , is pegged to economic recovery in manufacturing economies such as Japan , the second largest consumer of Mexican silver after the US . +The existing sea route eastwards round the Cape of Good Hope , to India , China and the Spice Islands with their rich resources of pepper , cinnamon and nutmeg , was long and dangerous . +It is just like the tortured media to make anything less than a Dem sweep in next year 's elections a failure on the part of the president and a repudiation of his policies . +Iran is accused of pursuing a weapons program under the guise of peaceful nuclear power and Syria has recently come under fire for allegedly building a covert nuclear facility . +The museum has lost an estimated 15,000 pieces . +While disorganized Republicans and major health-care companies wait for President Obama and Democratic leaders to reveal the details of their plan before criticizing it , Scott is using $ 5 million of his own money and up to $ 15 million more from supporters to try to build resistance to any government-run program . +The forum also wants suicide prevention in drug users to be made a key priority , pointing out that approximately 23 % of drug-related deaths were either intentional or of undetermined intent . +( CBS ) If you lost a bag at the aiport in Phoenix recently - it may have been stolen . +The Met Office spokesman added : " For winds like that you need a showery day . +King has made peace with the possibility that his rivals might go home with his money . +Today they can quietly communicate with each other by text messaging through their cell phones . +The discovery near Alness last Tuesday by a member of the public shocked animal cruelty prevention officers . +Severe weather conditions are adding to the difficulties and police say there is no parking for drivers who are being turned away from the tunnel . +But there had been little publicity of the strike before Tuesday , and morning traffic was typically heavy in the capital as stores and banks opened as usual . +The new rules aim to make it more difficult for potential terrorists to enter the United States from places such as France , Germany , Switzerland , Britain , Belgium , Portugal , Spain , Singapore , New Zealand , Japan and Australia , the government said . +The Board of Regents was set to approve Hauck 's contract in an afternoon meeting , less than one week after he coached the Grizzlies in the Football Championship Subdivision title game . +The Beijing museums to offer free admission include public museums and memorial halls run by municipal or district cultural and heritage protection departments , as well as patriotic educational bases , Xinhua said . +The review could take up to a year and likely would include an examination of the CIA 's detention program and secret prisons , the aide said . +He added that the talks between the two sides would tackle the Sudanese-Ethiopian relations as well as the relations between the Government of Southern Sudan and the Ethiopian government besides the progress of implementation of the Comprehensive Peace Agreement ( CPA ) and the positive developments in this regard in light of the meeting of the Presidency Tuesday night in Khartoum .. +Furthermore , Mr. Dimitroff added , " we are also following closely the current political dialogue regarding the election reforms and the discussions taking place on the formation of the Supreme Commission for Elections and Referendum ( SCER ) . " +" The main drivers were a stronger than expected performance in agrochemicals , as well as cost cutting measures in the chemical segments , " Commerzbank Equity Research analyst Stephan Kippe said in research report . +Nearly 59 percent or 77 individuals reported they never experienced it . +At many of its 18 stops , the relay has been beset by protests against China 's human rights record and a recent crackdown in Tibet following anti-government riots . +" Often when you move anywhere , there is the ' allergic honeymoon , ' which is a respite for a few years , until people become allergic to the new things they 're breathing " and develop runny noses and other actual symptoms . +It is unlikely to crumble like Netscape , the most recent start-up to vie to become a new platform , since hypervisors are much harder to replace than browsers . +Three years later , she not only takes care of her own children , but she also teaches preschool . +The index takes a random sample of species from a taxonomic group to calculate the trends in extinction risk within that group . +SAN FRANCISCO - Yahoo Inc. is setting up a new chain of command amid the turmoil triggered by the embattled Internet icon 's snub of Microsoft Corp. ' s $ 47.5 billion takeover bid . +437 . +" I 'm a man who strives to achieve his expectations , " Kuby said of his modest goal . +It is not clear whether the government could delay access to a lawyer for military detainees within the United States . +If the discount persists IndoChina intends to hold another tender offer next summer - again for 20 per cent . +It was a Sunday , with a ball of sun blazing above and little birds peeping on branches to signal the onrushing spring . +Brook says he acted in self-defense to protect his child . +It can also reduce the perverse incentive for homeowners to go into default to get a better mortgage deal . +But an expert who tracks evangelical Christianity , Larry Eskridge , said few are addressing the subject as directly as Daystar . +MPC Steamship was founded in 2001 and now manages 17 vessels . +The move would be a huge step toward ratifying the deal , which sets out conditions for U.S. military conduct in Iraq as well as a timeline for troops ' withdrawal from the country by the end of 2011 . +Besides having a three-year head start , Gardasil also defends against two more HPV types that cause 90 percent of genital warts , which Cervarix does not target . +There is another key Catholic notion here . +Jason Fogelson , who writes about motorcycling for youmustbetrippin.com , recommends a ramble from New York to New Hope , Pa . , on the Delaware River , where bikers have long congregated on weekends . +A lot of names date and really identify someone as being from a certain generation . +The project would replace a vacant Teledyne building , three low-slung brick office buildings and a large asphalt parking lot . +A top diplomat said Thursday he is heading to Honduras to demand the return of the president toppled at gunpoint -- a mission he said is likely to meet rejection , bringing diplomatic and economic punishment for the impoverished Central American nation . +Investigators probing the loss of Air France flight 447 hope they may yet explain one of the most baffling air crashes of recent times . +But last year the legislature rejected a bill requiring background checks for private sales at gun shows and repealed a law that Mr. Kaine had supported to prohibit anyone from carrying concealed weapons into a club or restaurant where alcohol is served . +Spokeswoman Kendra Barkoff said the Interior Department will follow Congress 's directive and put the new firearms law in effect in late February . +( AP ) - A woman who befriended a Florida lottery winner who later went missing was charged Tuesday with trying to conceal his death , five days after his body was found buried in her backyard . +Everyone would win , including the hard-pressed NHS . +The more than 40,000 Pennsylvania farms located within the watershed discharge 46 percent of the nitrogen and 58 percent of the phosphorus into these waterways and , consequently , farmers today face more stringent water quality requirements . +For tickets and accommodations , this is shaping up to be a particularly challenging Olympics . +In 2001 , he looked set to become one of the decade 's biggest stars . +A visit to the city 's jam-packed Yu Yuan Garden gave her an authentic taste of Shanghainese " xiaolongbao " juicy steam buns and flower tea and the crowding common in the city of more than 20 million people . +The shoes are a reminder to appreciate people 's differences . � � The National Christmas Tree was lit last night on the Ellipse behind the White House . � � More seasonal cheer : There will be a free family concert and singalong Saturday from noon to 2 p.m. at the U.S. Navy Memorial , 701 Pennsylvania Ave . +The property is in Alpine Meadows , a residential neighborhood with an even mix of full-time residents and vacationers , according to the listing agent . +If this is true then all of the government 's targets for reducing our CO2 emissions will make no discernible difference to the planet . +Like the objects she feels compelled to create , Helen is herself posited here as an exotic work of art , one prone to be cherished and misinterpreted . +In another burst of violence , police said at least 30 people died in clashes between Sunni and Shiite Muslims in Parachinar , a town near the Afghan border . +But local officials have not confirmed a link to the scandal . +At a time when so many competent and committed workers are losing their jobs , or are being asked to take a pay cut due to a financial crisis brought about largely by the reckless behaviour of crass bankers like the head of Barclays , the exodus of these people is more than welcome news . +Spain could play on the counter and that was important . +Wexler , a Democrat , has represented parts of Palm Beach and Broward counties since 1997 . +This simply can 't be true . +" I 'd have a little smile on my face , " Matta said . +They hope the scientific meeting , which ends on Friday , will highlight the range of measures needed , such as agreements that covered the entire catchment areas of the wetlands . +Marquette appears to be hitting its stride after struggling past Seton Hall in the opening round . +Another part of AMerican comedy ( or tragedy ) . +With goals by Brad Stuart and Tomas Holmstrom in the first period and another strong defensive effort on Monday night , the Detroit Red Wings beat the visiting Pittsburgh Penguins , 3-0 , to take a 2-0 lead in the Stanley Cup finals . +He had to create his own because the concept of rehabilitating fighting dogs is so unusual . +" I 'm trying to become a better person each and every day , " Woods said . +PITTSBURGH ( Reuters ) - The Pittsburgh Penguins appointed Dan Bylsma as head coach on Tuesday after his successful two-month stint as interim coach . +The proposals were " a jumble of national wish lists " and there should be real " European projects " instead , he added . +Excluding performance-related payments , costs declined £ 1.1m. +Usually , the awards , which were founded in 1996 , recognize individual Web sites , online ads or film and video . +Sri Lanka has many deep-seated problems in need of attention after decades of war against minority Tamil separatists , including resettlement of the estimated 100,000 Tamils still in internment camps , political and legal reform , economic development , and ethnic and political reconciliation . +COPENHAGEN , Denmark - Denmark 's National Gallery says it will exhibit some 100 works by Bob Dylan , including some of his newest acrylic paintings . +And he feels that he has been ' hung out to dry ' over the situation , having suffered a reaction to his knee problem after scoring the winner for Manchester City on Saturday in their friendly with Celtic at Eastlands . +SNP Westminster leader , Angus Robertson , called on the MoD to explain how it was that two subs carrying weapons of mass destruction could possibly have collided " in the middle of the world 's second-largest ocean . " +When an entry-level middle-class home in major markets on either coast lists at $ 500,000 and pays out a 6 percent sales commission , it is easy to see how an agent could get excited . +" Given the circumstances of the last 10 years and our attempts to give assistance in Zimbabwe , which have been thwarted and resisted , it is not possible for us to attend this summit and sit down with President Mugabe , " Brown said in London Tuesday . +Antonucci and others " repeatedly singled out Antonucci 's $ 6.5 million capital investment as evidence that the bank was viable and deserving of TARP funds , " it said . +( CBS ) Tom Engelhardt , co-founder of the American Empire Project , runs the Nation Institute 's TomDispatch.com. +At an event Friday in Barranquilla , Uribe said he would respect the ruling and urged that his successor keep the nation secure . +Property managers are under orders to open up their houses and make them feel unique and lived-in , while visitors will be treated as house guests instead of paying customers . +They are usually low earners , with wages up to £ 25,000 a year , and with either bad or no credit histories . +" Twenty-five percent of the combined white-collar and supplemental workforce " positions will be eliminated , Chrysler spokesman David Elshoff told AFP . +Some 40 tonnes of transformer oil leaked into the river after the explosion , killing fish and raising fears of chemical pollution . +Hunter 's best season was with Philadelphia in 2006-07 when he averaged 6.5 points and 4.8 rebounds starting 41 games . +Rowling , whose mother had multiple sclerosis , is also a patron of MS Scotland . +The case dates back to the late 1970s , when the Justice Department accused Demjanjuk of being a Nazi guard known as " Ivan the Terrible . " +The other two were diagnosed after 52 weeks and 104 weeks of treatment . +Other than the distortion to fit Stewie 's head , the helmet is pretty much the same . +A quick way to cut costs would be to reduce personnel , one analyst suggesting that the total size of the Army , Royal Navy and Royal Air Force will have to fall by more than 30,000 to 142,000 in six years . +Nuremberg nearly soured the start of Munich 's Oktoberfest after Maxim Choupo-Moting evened the score following Ivica Olic 's goal for Bayern . +The BBC 's Mame Less Camara in Dakar says some of those arrested are Mr Senghor 's security guards . +I received a generous gift certificate to Gramercy Tavern , one of my favorite restaurants . +The misdemeanor charge she pleaded to carried a maximum of six months in prison . +The public has almost forgotten the case is still going on . +Corruption at every level of Afghan society has undermined the population 's confidence in the government -- confidence Washington says is sorely needed before the Taliban can be defeated . +Former colleagues say that he is a supporter of a Tobin Tax -- a levy on financial transactions -- which Britain opposed for years until a policy about-turn this summer by Gordon Brown . +But approval ratings have rarely been a good predictor of presidential success . +Sven-Göran Eriksson has reacted angrily to accusations from England players that a " circus " engulfed the team 's appearance in the last World Cup finals in Germany . +A projector is duct-taped to a pedestal , enhancing the transient aura of a museum or art school lecture hall . +Coach K made his first Final Four with Duke in 1986 and hadn 't had this long of a gap between trips . +Bath were hosting Edinburgh in Pool 4 where Ulster are setting the pace after their 23-13 win in Belfast on Saturday over Stade Francais . +The ( Toronto ) Star reported Sunday that about 400 of Canadaʼs 1,600 active tuberculosis cases are in Toronto , a figure experts expect to rise given the cityʼs growing immigrant community . +Scores of students had been among the 500 watching when Jakarta 's mayor unveiled the statue in the park in December . +NEW YORK , March 4 / PRNewswire / -- Pica9 , Inc . , ( www.pica9.com ) , the leading provider of marketing automation solutions for major brands , today announced that it has become an authorized reseller of Google AdWords ( TM ) . +WASHINGTON ( AP ) - Construction of new homes posted the biggest increase in more than two years in April , a rare bit of good news in what has been the worst downturn in housing in more than two decades . +Prior to the FlyDubai and Etihad announcements , Boeing had taken orders for 475 new planes this year , primarily for the short-to-medium-range 737 . +Staple goods , all in higher demand . +Marion scored just four points in 40 minutes of his fifth game since joining Miami . +( AP ) - The NCAA is investigating Arizona 's basketball program for alleged recruiting violations , according to a letter sent to university president Robert Shelton . +The charity responded to a variety of SOS calls , from rescuing children cut off by the tide , to sinking fishing vessels and stricken whales . +These factors can include the genetic makeup of the patient , genetic changes that occur with diseases such as cancer , as well as diet , age , environmental factors . +However , the Commons Transport Committee said it was " not convinced " the project was adequate for the number of trains which could end up using the station and urged the government to look into the issue . +It was the first decline since March 2007 and the biggest drop in more than three years , since inventories fell by 0.3 percent in July 2005 . +Sykora-ML has been delivering solutions to the AS / 400 marketplace for over 20 years . +" Dozens of houses have been destroyed in the past three days by adult elephants entering human settlements to look for their wandering calves , " said the local magistrate , L.S. Changsan . +Hasan Rasouli , one of Mr Khatami 's campaign managers , said the former president announce his official decision to pull out of the race by the Iranian New Year , on March 21 . +And be thankful none of the action will involve any mention of Jonathan Ross or falling house prices . +The Arghandab district where the bombing occurred has long been known as a haven for Taliban forces and their sympathizers . +Ahmadinejad also blamed the West for the unrest following his disputed reelection as a third batch of his opponents was hauled into a courtroom to confess to plotting against the state . +I 've stayed faithful even in Vienna , home of the traditional Kaffeehaus , much to the disgust of an Austrian friend who waited outside in shame . +GM retains a 35 per cent stake and workers will hold 10 per cent . +Let everyone know that this is divine justice . " +WRAP has conducted similar tests of thinner glass and plastic bottles , with equally promising results . +Garrido , eldest son of former Ryder Cup player Antonio , finished strongly on the opening day and played superbly in round two to finish on 15-under-par 129 . +Ryan never had any hesitation about going back to his favorite receiver . +This was true of Rogers 's firm , BGR Holding , which made its name with blue-chip connections in elite Republican circles . +Attorney General Mike Cox " has concerns about the actions that 's he 's read about on the blog and what we 've heard about as far as the trespassing issue with the University of Michigan , " spokesman John Sellek said . +The news isn 't always black and white . +In a 2003 sermon , Mr Wright said blacks should condemn the US . +Its fatwas are nonbinding . +Speaking to the American Israel Public Affairs Committee , Barack Obama endorsed a two-state , Israel-Palestine settlement , and took an implicit poke at President Bush . +Finance Minister Ali Rodriguez said the government is intervening to back deposits in Stanford Bank SA after U.S. fraud charges against other Stanford banking companies provoked a wave of withdrawals by customers in the South American country . +His lifetime record with the Yomiuri Giants ¿ Japan 's equivalent to the esteemed Yankees ¿ was 112-62 with a 3.01 ERA . +The admission has sparked a torrent of protests from Israeli and foreign journalists who claim such a move will jeopardize the objectivity and safety of journalists . +Other participating entertainers included Mariah Carey , Bono , Miley Cyrus and Snoop Dogg . +To quote the authors of your article , " The country seems , if not adrift , becalmed at the moment , even unsure of itself and its identity . " +Lord Turner will also launch a crackdown on the use of offshore , off balance sheet funding vehicles by banks . +A health crisis also loomed a week after the 7.0-magnitude tremor , as aid workers struggled to tend to the homeless and injured amid deteriorating security in Port-au-Prince . +Rooney had to watch from the sidelines as Emile Heskey returned from his own international exile to bring out the best in Owen as England put their qualification bid back on track with vital home wins against Israel and Russia . +( Dan ) Ellis has been named president and chief executive officer and appointed to the board of directors . Concurrently , the company announced that Peter T. Cyrus has resigned from his positions as interim president and chief executive officer , and a member of the company 's board of directors , effective June 9 , 2009 . +We have an immediate gratification culture which has spread to financial services with online sales of car finance , loans , insurance , savings and even mortgages . +" When you run into guys like that , you need to have good pitching , and we didn 't have that , " Tigers manager Jim Leyland said . +The Medicines and Healthcare products Regulatory Agency ( MHRA ) has upheld a complaint over the online advertising of two remedies , Duchy Herbals Echina-Relief Tincture and Duchy Herbals Hyperi-Lift Tincture , which are sold for £ 10 for 50ml in selected Boots and Waitrose stores . +Uddin refused to answer our repeated inquiries about the address . +The free eBook and audiobook downloads are part of an ongoing effort to educate general readers on pulp fiction stories written in the 1920s , 1930s and 1940s . +Three others were on the run , said police , who also detained 13 civilians . +Iranian state television reported that eight people had died in the street violence Sunday , but independent confirmation of the casualty toll was virtually impossible because of curbs on media coverage . +Friday , Ankiel politely declined to specify what they asked or what he told them . +Actually , stormer , you are wrong again . +British cinema was born in a hall on Regent Street in central London in 1896 - when the Lumiere brothers put on the first public show of moving pictures in the UK . +When Jerry Hall told her to open her eyes to the truth about Bailey 's newest model , Catherine , Helvin took a good , hard look . +" In both instances when the small packages were opened , there was a dissemination of smoke and a smell , that 's the best description we have right now , " he said . +All of Pettigrew 's competitive results since January 1997 will be disqualified . +On the 767s and 757s , the BusinessFirst seats recline 156 degrees and measure 21 and 20-inches wide , respectively . +" This really gives information on steps that people can take that 's going to improve their health , " said Fradkin , who had no role in the latest research . +Twenty-seven of the task force 's recommendations , including the express flight paths , could be implemented by next summer , with help from the Federal Aviation Administration , the Port Authority said . +In today 's tough economy , every supply chain participant wants to streamline operations and cut costs while continuing to increase efficiencies . +But it is not necessarily even about being an original songwriter . +Hopefully the audience will maintain . +Italy 's fashion and textile industry employs 800,000 workers in 30,000 companies , the Industry Ministry noted in a statement . +BASKING RIDGE , N.J. , and ATLANTA , July 14 / PRNewswire / -- Verizon Wireless , the nation 's leading wireless provider , and AirSage , a world leader in using wireless signaling data for value added applications such as traffic information , predictive analytics and location services , have entered into an agreement where non-identifying data from Verizon Wireless ' network will be used to help AirSage provide real-time traffic information . +His last pitch was to John Buck , who lined it over the right field fence for a three-run homer to make it 8-2 . +KEITH BOTSFORD has just flown from Costa Rica , where he lives in an angular , steel-covered house overlooking the Caribbean , to Boston , where his wife -- a molecular biologist some 52 years his junior -- lives and works . +The A4 2.0T , ( $ 37,525 , MSRP as tested ) is powered by a 211-hp , 2.0-liter , turbocharged four-cylinder engine that provides 22 mpg overall , on premium fuel , which is good considering the car 's quick acceleration and all wheel drive . +Much of the remainder would be used to maintain those systems . +He made a face ? +We have two other boys , Mitchell , 6 , and Donovan , 10 , and we would have needed a variety of child care arrangements when the baby was born . +Saunders , a Midwest native who , in addition to teaching , runs water conservation workshop for brewers and cheese makers , says she has a " palate for grains " and has loved the taste of beer for as long as she can remember . +But prospects are uncertain in the Senate , where Republicans and coal-state Democrats oppose it . +Liverpool and England player Gerrard , 28 , remains on bail after appearing in court charged with assault and affray last month . +Other airlines with the A380 in their fleet are Emirates , Qantas Airways and Singapore Airlines . +Among men with cancer that had spread beyond the prostate ( metastatic cancer ) , the death rate in the first month was more than triple that in the general population . +Some of the players are duking it out over who owns the patent to commercially clone animals in the first place . +Yet when it comes to scheduling working hours , the U.S. does have more flexibility than Europe . +He asked more questions of himself , and his ability to turn things around , than his interrogators . +" It 's a very , very exciting day for football fans because it 's a ground-breaking move , " he told the BBC . +American University professor Jane Hall told FOXNews.com that the race issue is an unfortunate diversion for Obama , who drew supporters hoping this would not be a campaign about race . +But Gov. Jon Corzine and most of the state legislators are big Rutgers football fans , and that does not make for much of a check on spending . +Abreu added that " it took a long time " to get his normal swing back . +On the other hand , the Chargers have some pleasant memories of Indy , winning there 26-17 two years ago , when the Colts were 13-0 . +Inflation would work the same magic on government debt . +President Hu and Prime Minister Wen are basking in good will from their management of the Olympics , so far widely perceived as a triumph for China . +But he said that troops on each side had not fully pulled back , and that the status of the oil-rich Abiye border region , claimed by each party , had not been resolved . +" I have requested both the ministers to vacate the hotel rooms and go to their respective bhavans ( guest houses ) , " said Pranab Mukherjee , the finance minister . +Joseph under went numerous operations following his accident when he suffered burns over 76 % of his body . +Back home , he plays gigs with his band . +The procedure will take place next week at the Cleveland Clinic . +" Corals face threats ranging from ocean acidification , pollution , and warming to overfishing and starfish outbreaks , " says Dr. Ian Poiner , Chief Executive Officer of the Australian Institute of Marine Science ( AIMS ) , which led the research . +" This is a rare instance where something good can arise from tragedy . +MCCAIN : As a cold political calculation , I could not be more pleased . +On the other hand , if you eat more calories than you burn up , the extra calories are stored in your body as fat -- and you gain weight . +She is a medical school graduate who defends those who preach intolerance of homosexuals . +The probe can also remove a tissue sample which can be removed and examined under a microscope for cancer cells . +Former double world champion Fernando Alonso insisted he has put last season 's last gasp title disappointment behind him as Ferrari prepare for the new Formula One season . +" After several minutes shooting our security guards repulsed the attackers and killed three of them , " the U.N. official told Reuters . +I am not a Hillary hater , but I must admit that I find some facets of her personality grating . +The Security Council , in its first joint formal action since last month 's pro-democracy rallies were crushed , said it " strongly deplores " the repression and called for the release of political prisoners , amid growing concern over the fate of nearly 1,000 demonstrators still being held . +Mr Morse drove the hunt 's 4x4 to the disused airfield and stood in front of it to prevent it taking off . +It has been in business for more than 213 years . +Unlike the formal and intense competition at the Sambadrome , anything goes at the blocos , and they are becoming more and more popular because fans can join the fun . +WASHINGTON ( Reuters ) - The trade deficit narrowed more than expected in March on a record plunge in the value of imports , even as average oil prices surged to a new record , a Commerce Department report on Friday showed . +Indeed Armstrong had hinted to that effect . +Make your request after the wedding . · Anniversary greeting : An anniversary card will be sent to couples who are celebrating their 50th ( or greater ) wedding anniversary . +The comments by the two MPs threaten once again to embarrass Tory leader David Cameron after he was forced last week to reprimand senior frontbencher Alan Duncan for suggesting MPs were living on rations in the wake of the expenses scandal . +IDS said there had also been a number of freezes in the finance sector , media , airlines and road transport . +The events called attention to Beijing 's human rights record in the Himalayan region at a time when Chinese leaders had hoped for a peaceful run-up to the Olympics . +He called her into his office before the start of a season and mused in admiration at how wonderful she was . +The ads cited his criticism of the Haditha incident as well as his comment about " racist " voting tendencies of many western Pennsylvania residents . +To a certain extent , Brown has been undone by events beyond his control that have nearly paralyzed the government in recent months . +" The recovery will be slow and uneven , " he added . +COPENHAGEN ( AP ) - President Barack Obama squeezed two impromptu meetings into his tight schedule and made an animated plea for compromise Friday , making plain his frustration over the difficulty of pushing world leaders to settle on a plan to combat global warming . +" This whole idea of change and disease processes throughout life is something that a medical student should dwell on and hear a lot about and think a lot about , " he said . +Instead , Cliburn did the honors a year later . +This article is somewhat disingenuous about the extent of support Mitt Romney had from so called Conservative leaders . +Associated Press Writer Matt Moore in Berlin contributed to this report . +She prepared me for the interview not by showing me her work but making me a momentary participant . +SANAA ( Reuters ) - A Saudi suspected of belonging to al Qaeda has handed himself in to authorities in Yemen where al Qaeda militancy is on the rise , a government website said on Thursday . +The police investigation of the attempted jewelry store robbery and hostage incident is continuing . +" Yes , I cried , " he says . +" I 'm not convinced we are winning in Afghanistan . +US stocks looked set to show some resilience on Wednesday morning , with buyers being tempted back into the market following two days of selling . +She followed with a series of blockbuster interviews--Mike Tyson and Evander Holyfield , exclusives with singer Whitney Houston and ESPN 's Erin Andrews , and just this week , former Alaska governor , GOP vice presidential candidate Sarah Palin . +Reviewers must complete profiles , listing their e-mail addresses . +Abbey said it had " fully investigated " Mr Bates 's claims but it could not say whether any disciplinary action had been taken . +Privately , local leaders admit it might be a concern if it made a specific commitment to hampering oil and gas exploration , but the general view was that , with the exception of Chavez on the Queen , they have heard it all before . +In von Maltzahn 's approach , one set of nanoparticles lodge in a tumor 's blood vessels and cause local bleeding . The bleeding prompts clotting factors to be produced , which in turn , attracts a second set of nanoparticles that have been programmed to be attracted to the clotting factors and that deliver a cancer drug . The use of the clotting factors dramatically increases the number of targets for the drug-carrying particles . +" Brits going to Australia , New Zealand and Canada are looking for a better lifestyle not to make their fortune , " says Harwood . +Made in Britain , each coat is finished to exceptional quality , with all the original details. east thames group limited. in this varied and challenging role you will provi ... . £ 22,514 - £ 24,246 per Annum. english heritage. bedfordshire . £ 29,500 - £ 33,000 per annum. imperial college. south kensington , london . £ 26,580 - £ 30,360 p.a .. their beautiful waterfront location , the breadth of arts and cultural offerings rivaling those of cities ... reflect their cherished heritage and a famous lineup ... . wi. performing arts. the duties of the heritage chair in ... arts , the john f. kennedy center for the performing arts , and mason 's center for the arts. the ... . va. their beautiful waterfront location , the breadth of arts and cultural offerings rivaling those of cities ... reflect their cherished heritage and a famous lineup ... . wi . +" The funny thing is , one of the groups that was seeing ' Legally Blonde , ' when the band director told his kids they were now going to see ' Spelling Bee , ' the kids were more excited , " he said . +I think there was some disappointment with some of the recent , earlier Saw films that came out before Saw VI . +And for all of the commotion over the Mitchell report that linked about 90 major league players to steroids , a new set of baseball initials , H.G.H. , alias human growth hormone , is about to upstage all the others . +" Peter Galbraith , the highest-ranking American in Afghanistan working for the United Nations Mission there -- recalled today by U.N.Secretary General Ban Ki-moon over disagreements with the head of the mission there , special envoy Kai Eide -- says the Afghan elections recently carried out are the worst the U.N. has overseen . +Yentob took over in 2002 when Massow resigned after criticising the preponderance of conceptual art in the art world . +The bustling commercial and residential area in north London , with its open markets and a scenic canal , is one of the British capital 's top shopping and tourist destinations . +However , the company 's latest big-screen release , " Precious , " a drama about the brutal home life of a teenage girl from Harlem , could be a positive contributor . +It doesn 't get more authentique than that . +CAEN , France - Ugly words on the playground were his first hurtful clue . +22 ) : Your zeal is at a peak but it 's important to hold back . +In Toronto , the snowstorm caused some 600 traffic accidents on Sunday morning alone , the provincial police reprted . +Within hours , he said , a battle erupted ; Ahmed was shot in the leg . +Not because I think the Big Four banks are evil , but because I 'm pretty sure we 'll get a better deal and much better service from one of the small guys . +Damage from the quake may be severe but it is limited to one region . +With any luck , the worms will soon be eating as much as three pounds of food scraps a week , a way to turn kitchen wastes into a rich , dark , earth-smelling soil conditioner you can use in your garden , in flowerpots or even spread over your lawn . +In October , the UN announced that it had been allowed to collaborate with regional authorities to supply relief food , medicine , and veterinary services as well as setting up offices in a key town there . +And the design scheme allowed them to market the 43-story Broadway tower , one of many postwar apartment towers on Broadway , as if it had a coveted Central Park Address , by linking the tower through a rear drive and garden , and its mailing address , to Central Park West . +" Are there lots of errors in the data ? +Despite the comments , Hamas has shown little tolerance for dissent in Gaza . +Stir in turkey , using a wooden spoon to break up the meat . +I guess if he called me he was thinking about a comedy . +After a series of corrections failed to straighten out the glitch , Britt-Crane yielded to the inevitable . +Can you really establish trust by falling backwards and hoping Rick from accounting will , in fact , catch you ? +Gates also made a point Tuesday of saying that the United States is " ready to help resolve disputes over boundaries and hydrocarbons , " a reference to widening tensions between Arabs and Kurds . +Afghanistan , having been largely ignored through the early weeks of the campaign , bounded on to the agenda three weeks ago after more than 70 Afghans were killed in a US air strike called down , in a perverse twist of fate , by a German officer . +She emerged as a heroine from the conflict , during which she led a team of 38 female nurses ministering to wounded soldiers in extremely difficult conditions . +That led them to Mendoza and Tobar-Campos , police said . +The Irish News reports how the former Taoiseach , Albert Reynolds , is behind plans to bring a huge bio-energy project to Londonderry . +Heinz said it now expects earnings of $ 2.54 to $ 2.62 per share for the full fiscal year . +In 2007 , authorities said two Tennessee men set fire to the home of a suspected child pornographer , killing his wife . +Last week there was a modest revival of demand for UK bank shares . +Regulators can fine Microsoft up to 10 percent of yearly global turnover without seeking evidence if the company doesn 't stick to this commitment . +Satyendra Narayan Yadav , from Nararipuri village in Madhubani district , says he is another harassed Act applicant . +Now imagine that a strange phenomenon occurs , in which it is noticed ( but , oddly , not widely reported ) that Americans on the Canadian border seem to stream across into Canada to buy their Big Macs . +Do you worry , though , that being green might just be another trend for musicians and will be forgotten in a few years ? +" That is not a sign that the system worked . +That is , the solution is merely a matter of fully understanding the problem , and not one of needing genuine creativity. re : 58 . +Gesine Schwan , a 65-year-old political scientist backed by the Social Democrats ( SPD ) , said this week that she saw " very , very good chances ( of being elected ) . +He was never arrested or charged , and when he sued the government for ruining his career , a federal judge found " not a scintilla of evidence " linking Hatfill to the mailings . +The global fame and the " odd bit of negative press " in the past week have been too overwhelming for her , judge Piers Morgan told CNN 's " Larry King Live " on Friday . +Let 's go ! " barked the instructor in camouflage fatigues as we shuffled into a cavernous corrugated-tin warehouse in an open field at the Farm . +Eight people have been arrested in last week 's shootings of Byrd and Melanie Billings in their sprawling home near Pensacola , Florida , which they shared with their many children . +Activity | The following two activities are designed so that students find and peruse Times articles on a range of topics related to childhood obesity , nutrition and exercise . +EST , underperforming the German blue-chip DAX .GDAXI , which fell 6.6 percent . +TCU ( 1-1 ) will play the College of Charleston during the evening . +Their captors made off with a sum of cash . +Some in the shrinking community of global warming skeptics and those downplaying the issue , were dubious , however . +The researchers discovered that long before the fly leaps it calculates the location of the threat and comes up with an escape plan . +Continuing that custody while developing a permanency plan , or sending the children ' home ' , will be the subject of subsequent timely hearings . +Nor is Mr. Mukasey ( pronounced mew-KAY-see ) a Washington insider with experience in managing a federal bureaucracy . +There is also Cannibal ! - The Musical , based on the cult film by Trey Parker , the creator of the South Park cartoon series . +The main opposition candidate , Almazbek Atambayev , has dismissed the election as fraudulent and is demanding a rerun . +Thus ensue lectures on genetically modified foods , demonstrations of toxic fumes leaking from mercury-laced fillings and paeans to the benefit of frequent coffee enemas . +True progress would moving away from the courts " deciding " what rights someone has and has not . +For the poll , more than 16,000 people who had just voted were selected at random to fill out questionnaires . +But Fenty 's budget would increase funds per student by $ 175 in both traditional and charter schools , would add $ 2.7 million for summer camps and facility improvements to recreation centers and would add $ 2.4 million for staff and equipment for the new Wilson and upcoming Deanwood pools . +The National Assembly will select a vice president and the entire 32 strong executive Council of State . +I was driven to Loch Fyne in a car stinking of creosote . +The latter , in particular , is simply brilliant ; he makes us chuckle with an eyebrow raised or slight turn of his head . +The two sides are at a stand-off and are apparently determined to take the issue to court . +Nissan 's car sales increased 11 percent , but truck sales were flat . +The new book The Coke Machine : The Dirty Truth Behind The World 's Favorite Soft Drink ( $ 26 / € 19 ) by Michael Blanding , released on September 16 , explains the original Coca-Cola recipe did have small amounts of cocaine present . +New York recently mandated 90 days notice for cuts of 250 or more at one site . +A self-taught painter , Boeke keeps a steady hand crucial for pinstriping and custom painting by abstaining from alcohol and making sure his painting is interruption-free . +Others just want to give their guests or adolescent children privacy . +The audience cheered ; he shouted louder . +Irene from Virginia - we can 't even elect our PM . +" We have now seen all the investment and infrastructure coming together to bring a turnaround in the number of people using bus services . +It is no good coming up with a similar-ish product to todays iPhone ( it 's actually just far more complicated and overwhelming to use it seems ) , but they have to move ahead of it , because Apple doesn 't stand still . +The rolling action is designed to disrupt services while ensuring workers lose only one day 's pay . +With the exceptions of Argentina , Saudi Arabia and South Africa , all of the member countries fall within the list of the top 20 biggest state G.D.P. ' s in the world . +Wicks suggests setting a provisional retirement age for planning purposes -- perhaps 66 , when he would be eligible for a state pension . +One of the many infamous quotes from Seminerio was when he said in the State Assembly if Native Americans didn 't like their life in the US then they could walk back across the Bering Strait to Russia . +The Angels took a lead in the third when rookie shortstop Jed Lowrie went to field a grounder with two outs and could not come up with the ball cleanly , allowing Garret Anderson to move to second and Vladimir Guerrero to reach first safely . +I have no doubt that he cares dearly for these soldiers , but I think he 's way off the mark there . +A Spanish website containing abuse aimed at Lewis Hamilton was condemned yesterday by the FIA , motor sport 's governing body . +Belmont Park opened in 1905 . +That could show up again today when chain-store sales are reported by the International Council of Shopping Centers . +The survey saw more than 400 people interviewed over the phone . +Douglas Carswell , a leading critic of Mr Martin , said a new Speaker was needed with the " moral authority " to pass reforms . +Their members are scattered throughout Russia , Asia and the Western Hemisphere , shunning much of the modern way of life . +CAMBRIDGE , Mass . , March 5 / PRNewswire / -- Cultural Care Au Pair , the leading provider of intercultural childcare in the United States has announced the launch of a brand new program created to address the concerns of American families during this difficult economic time . The Cultural Care " Peace of Mind Guarantee " ( www.culturalcare.com / peaceofmind ) assures that if a host parent loses their job , Cultural Care Au Pair will guarantee a pro-rated refund of all unused program weeks . +Teachers and communities often reach their own arrangements with the Taleban . +Heretofore , my chief anxiety about traveling to France had always been that , at some point during my trip , I would be called upon to pronounce the name of the town Ypres . +The ultimate insult : he was only in charge of the accessories . +They 'd sold their house in Fairfield at a nice profit , so they had money in the bank for the first time . +In a statement , Detective Inspector Ronnie Walker , who led the police inquiry , said : " Jim Ingram hoarded money obsessively and the thought of parting with a large divorce settlement triggered the tragic events of 13 March . +Lynne Spears : Do I promote teen sex ? +Another extraordinary case arrived in court yesterday after O. J. Simpson , the former American football player and murder suspect , burst into a hotel room in Las Vegas last month in an attempt to take sporting memorabilia that , he said , belonged to him . +I don 't see any other front where this is available , " he said . +The intersection will continue to " fail , " highway officials said , but the new lanes should reduce delays there by an average of 15 to 30 percent . +Concerns about China 's plans to limit economic growth and proposed regulatory bank changes from Washington also have pummeled the market . +At one point , as many as 150 students were at the building protesting the tuition increase , Morain said . +1995 - 20th anniversary of the Indonesian invasion marked by protest by 112 East Timorese and sympathisers who enter Russian and Dutch embassies in Jakarta . +The FSA said that as the wine was below 9 % alcohol it could not be legally called a wine and must be labelled a " wine-based drink . " +To watch the video of the lock gates arriving in the Cumberland Basin and to listen to the interview with Matt Ewing , the site manager , click on the link below . +A spike in defaults in the coming months could force the insurers to pay billions of dollars in claims . +The ICSC said that the late Easter had a negative impact on the sales figures , and that a clearer picture of US consumer spending would emerge when looking at March and April 's figures together . +Opponents of his plan said backing away from the larger project would send a negative message amid efforts to borrow against future lottery revenue . +Officials of Disney and Dell declined to comment on specific allegations , but both companies say they carefully monitor factories in China and take action when they find problems or unfair labor practices . +" We have no experience of marketing or customer service . +Mr Brown has struggled to convince the US of the value of a transaction tax . +" We hit a point in growth that we needed people who have done this before , " he said . +Also , more than 1,800 parents were informed and more than 18,000 teenagers directed to " positive " activities . +0943 Peter Robinson is telling jokes . +The glass holding the candles can break during use , presenting a fire hazard , the commission said . +Michael Gilbert , who had been living in Blackburn , Lancashire , was identified through his fingerprints , police said . +" We have been open for 20 years now and have not had an E. coli incident so it 's alarming and distressing that we 've had it now , " she said . +It was enough to frighten his bravest rivals as , dreaming of dancing in London on an Olympic night in 2012 , The World 's Fastest Man sped away into the shimmering distance . +Powerset , a 2-1 / 2-year-old start-up , has licensed natural language processing technology and related machine processing methods developed over three decades at the Xerox PARC research center in Silicon Valley . +He can 't buy his way out of problems as Labour did . +" We were very fortunate that the head of the fire ... was stopped at the backyards of those homes , " said Los Angeles County deputy fire chief John Tripp . +Brain dead today aChangeOfIdeas ? +In adults , researchers have shown an association between metabolic syndrome and a group of diseases called nonalcoholic fatty liver disease ( NAFLD ) , which at its most severe , may progress to irreversible liver damage . +But it doesn 't mean we won 't have a health care bill , " she said . +The publicity material refers to the magazine as " she " and letters from correspondents are signed " Miss " and " Mrs. " +It has chosen an unlikely candidate : Chris Ostrowski , 28 , a manager for John Lewis and the party 's seventh-choice candidate in the Eastern region in the European elections . +Sunderland , which lost its first home league game of the season last month , fielded a weakened team and was beaten , 2-1 , by Notts County , which is struggling in League One , the third tier of the English game . +Also through to the last 16 in early action was German veteran Tommy Haas who defeated France 's Jeremy Chardy 7-5 , 6-3 , 4-6 , 6-4 . +Police Commissioner Raymond Kelly , speaking from Times Square on a Webcast , said his department had many officers in the crowd , both uniformed and plainclothes . +NEW YORK ( Reuters Health ) - China now has more people with diabetes than any other country , a new report shows , making it clear that the nation 's soaring economic growth is taking a toll on public health . +Shiites account for a majority of Iraq 's population , and in the last national elections , in 2005 , Shiite parties ran together under a broad umbrella endorsed by their top religious authority , Ayatollah Ali Sistani , leaving little doubt as to who would emerge the winner . +This is a notion with which Govan himself takes umbrage -- even though part of the reason he took the job in Los Angeles was to take charge of an ambitious three-phase building project ; the first phase alone will cost $ 156 million . +He made the building a co-op again in 1952 . +If you prefer to plan your own activities at each port , Alaska 's official tourism site is a good resource . +" We feel proud but it doesn 't mean we are over-confident , " he said . +More than half of the world 's attempts to land on Mars have ended in failures . +With Libya once more back in the international fold , it is now as easy to sell your music over the net from Benghazi as it is from Brighton , Budapest or Bangalore - and some previously rare tunes are easier to obtain than they have ever been before . +" We are very serious to implement this project since the food crisis is turning serious day by day , " Manjhi , who has eaten rats , told Reuters . +While few cyber crooks are attacking Mac users through Safari and Firefox at the moment , that may change soon if a large number of Windows users migrate to Windows 7 , the successor to Windows Vista , due to be released sometime later this year . +LOS ANGELES , California ( CNN ) -- A 24-year-old gang member was arrested Thursday in connection with a shooting at a Los Angeles bus stop in which eight people were wounded , city officials said . +Oil prices averaged over $ 120 a barrel in the second quarter -- almost double the level in the same period of 2007 -- before rising to a record high above $ 147 / barrel on July 11 . +More important , she said : " There is a huge difference between singing and having a vision for yourself . +Classes explaining scientific method and basic concepts will be included in the induction programme for all Tory MPs after the next election , and sitting members and peers will also be offered the opportunity to attend , The Times has learnt . +Hilton , which is controlled by Blackstone , also owns brands such as Hampton Inn , Hilton Garden Inn and Doubletree . +Many Israeli leaders over the years have requested Pollard 's release , Netanyahu observed in his letter to Obama . +WASHINGTON , June 25 ( UPI ) -- Anderson Hernandez finished off a four-run first with a two-run single Thursday and the Washington Nationals went on to defeat Boston 9-3 . +More than that and runaway global warming becomes much more likely , ice sheets melt and sea levels could rise by several metres . +But Mosley is mostly known for his crime books -- a fact that he blames on marketing . +MONACO ( Reuters ) - Booming global investor interest in an emerging industry to supply clean energy alternatives to fossil fuels , such as wind and solar , has temporarily peaked in the wake of a widespread credit squeeze , a U.N. official said . +But the Red Wings would not surrender their crown without a fight , rookie Ericsson blasting a shot from the point past Fleury with just over six minutes to set up a dramatic finish . +Sony 's shares rose $ 1.15 , or 3.1 percent , to $ 38.17 in midday trading . +It is mothers who pay out a fortune in shops like Lush for daughters whose idea of going green is to buy as many new " green " things as possible , then carry them virtuously home in a paper bag . +Mr. Buffett did not attend the news conference , but said in a statement that he was impressed with Mr. Wang 's record as a manager . +In Diwaniyah , a city south of Baghdad , gunmen killed a policeman in a drive-by shooting , said another police official . +Maybe people find that a little eerie . +The company said the purchase " will be earnings enhancing in 2008 . " +Producer Adonis Shropshire confirmed to E ! that Rihanna , the 21-year-old Grammy-winning singer was indeed back in the studio . +The Tiffen Photo fx 2.0 allows users to enhance images from photos taken or stored on their iPhone or iPod Touch . +Unmarried women went the opposite way , voting for Senator John Kerry by a margin of 62 percent to 37 percent . +I 've often said that it helped when I got married and had an Italian last name . +Arsenal ( 4-1-4-1 ) : L Fabianski -- E Eboué , A Song , M Silvestre , A Traoré -- C Eastmond ( sub : S Watt , 68 ) -- J Wilshere , A Ramsey , T Rosicky , F Mérida -- C Vela . +The win takes United two points clear of Arsenal on 69 points after 31 games . +Net charge-offs rose 6 percent to $ 168 million , and nonperforming assets nearly tripled to $ 1.66 billion . +Even in this traumatic week for him , he has had six endorsements , compared with four for Hillary Clinton . +At the same time , she doesn 't plan on taking her bike for a spin anytime soon . +" They masturbated me , they made me masturbate them , they sodomized me , " he said . +Tamilnet.com said the rebels did not suffer casualties . +PHILADELPHIA - In the battle against black unemployment , places like the Opportunities Industrialization Center are ground zero . +Last month , the two companies sliced their dividends and sold billions of dollars of special stock to raise capital and shore up their finances . +" He was bleeding pretty badly so I asked someone else to sit on the shooter 's ankles and I went into Safeway and asked them for some paper towels and I made a compress . +But after the failed attempt to buy the London International Financial Futures Exchange ( Liffe ) in her first year , and the bids she rebuffed from Deutsche Börse , Euronext , the Macquarie banking group and Nasdaq - twice - she hasn 't always received favourable notices . +U.S. Defense Secretary Robert Gates arrived in Seoul Wednesday for the annual Security Consultative Meeting , which is focused on steps to deter military threats from North Korea . +Hotel Kabuki , 1625 Post Street , San Francisco , 800-533-4567 ; www.jdvhotels.com / kabuki . +He tasked George Mitchell , his envoy to the region , to continue meeting with Israeli and Palestinian officials . +Financial markets are concerned that many Spanish borrowers will not be able to refinance their debts this year at a time when investors are nervous about taking on any European risk . +But provincial authorities identified him as Attaullah Wahab , who served as both the deputy chief and security head of the national police in the province . +Brown 's office gave O 'Keefe and Giles immunity from a state privacy law that prevents unauthorized taping in exchange for a complete set of their videos , the report said . +According to the program , what are the differences between a meteoroid , a meteor and a meteorite ? +Coffee chain retailer Peet 's Coffee & Tea Inc. said Monday it will buy wholesale roaster and distributor Diedrich Coffee Inc. for $ 213 million in a move to enter the rapidly growing single-cup coffee market . +Marcus Gad Binghi Thompson , 30 , of Bowfield Road , Firth Park , Sheffield , appeared at the city 's magistrates ' court charged with murder . +Although the vaccine is destined to be used on girls , it has mainly been tested on adults in the United States , where the vaccination programme started in June last year . +A fight is brewing over the practice of feeding chicken feces and other poultry farm waste to cattle . +Catches taken in a one-day match against South Africa in 2008 . +In his 2007 ad , Brock said the mayor backdated her signature on a document and that he had asked his lawyers to report her for possible criminal investigation . +The result is a kind of Alfie-to-Alfie ( or is that alpha-to-alpha ? ) combat , as these two accomplished actors , playing equally crafty characters , engage in the kind of mirroring that is one of the visual leitmotifs of the film itself . +There was no immediate Palestinian comment . +Yeah , probably , a little , if you judge things so parlously as to measure your life in tenths of a second . +But she had long been afflicted with a genetic heart defect , and the stage was reached when she could neither eat nor speak . +She agrees that newcomers might be taken aback by the full-on rock assault of the group 's live show . +I ignored it and kept on eating . +I find it interesting that the amount of " noise " on this message thread e.g. climate change deniers , increased multifold immediately after my posting # 118 , which debunked much of what " M. " of # 66 said about textbook publishing . +An SFA review panel examined footage of the midfielder 's sending-off for violent conduct in Sunday 's 2-1 Clydesdale Bank Premier League defeat at New Douglas Park and determined it warranted only a yellow card for " adopting a threatening and / or aggressive attitude . " +" You think I 'm bad with all the stuff that goes on in the N.F.L. ? " he said . +Al-Hayat alleged Facebook had come under pressure from Israeli and U.S. lobbyists demanding Palestinian resistance movements such as Hamas be removed . +Scholz , who said Goudreau left the band more than 25 years ago after a three-year stint , objects to the implication that the band and one of its members has endorsed Huckabee 's candidacy . +They presented fans with a slightly different singer than the outrageous legend who was even then awaiting trial for alleged onstage exposure -- podgier and less lizardly than in his previous sex-god image . +On Wednesday , four Republican lawmakers demanded an audit of the $ 787 billion stimulus program following reports of exaggerated or inaccurate accounts of the number of jobs created . +Manning , when he went out , had completed 8 of 10 passes for 173 yards and 2 touchdowns . +To him , she was " My Dear Dusia " and he signed his letters " Br " -- short for brother . +The debate this week will be whether or not he would have survived Chappaquidick -- or the William Kennedy Smith trial -- in the digital age . +But ministers in England are still refusing to drop the £ 6.85 charge . +Shelton was given an interim ban until sentencing on 15 January . +And , arguably , there 's a third Razorback in the race , given the candidacy of Sen. Hillary Clinton , D-N.Y. The state favored President Bush over then-Vice President Al Gore in 2000 and went even more solidly for the sitting president over Sen. John Kerry , D-Mass . , in 2004 . +The company also is less subject to other forms of federal involvement . +Lehtonen was caught leaning to his right , leaving half the net exposed as Vasicek crossed in front of the net for the shot . +They included a AA battery , several tissues , a toenail clipper fashioned into a sharp piece of metal , and a device made of razor blade . +Apps that locate specific shops are ten-a-penny , but what 's the use of getting directions to the nearest supermarket when it has run out of the ingredients you want ? +The Prime Minister also condemned the protest and expressed disappointment that a " tiny minority " of the crowd tried to disrupt the event . +19 when Curran , the Georgia linebacker , recovered one against Arkansas in the Bulldogs ' 52-41 win . +Kellogg products are manufactured in 19 countries and marketed in more than 180 countries . For more information , visit www.kelloggcompany.com. Kellogg Company 's Corporate Responsibility report including its approach , progress and future direction in the marketplace , workplace , environment and community can be found at www.kelloggcompany.com / CR . +Little Obama is 6 months old , born May 3 , weighing 6 pounds 5 ounces . +Announcing the Budget details , the minister said : " Next year , my department intends to spend some £ 245m and disburse over € 300m to farmers and rural dwellers , a very significant investment . +Then perhaps Norman you would like to pay off my share of the national debt ? +Paterson 's microphone sounded like it was turned to " murky punk gig from the late 70s " setting , but this only enhanced the brooding menace of Ramalama and the renegade thrill of the entire performance . +It hardly seems like the power dynamic is skewed toward the passenger right now . +The two-year hiatus ensued , with Lane Fox having to fight every step of the way to recover . +The worst tornado hit the night of Feb . +The rate of home price declines has accelerated on a quarterly basis too . +TechCrunch even posted a video of the Zynga chief executive , Mark Pincus , admitting at a presentation this spring that , " I knew that I wanted to control my destiny , so I knew I needed revenues , right f * * * * * * now . +America wake up ! ! +Some studies say they are too conservative and will not provide the returns future retirees need to finance their golden years . +See how much oil you are actually adding to your food use a spray or measure out oil with a teaspoon instead of pouring it straight from the bottle . +Stack spilled a long range dig from Glen in the final minute but recovered in time , with some assistance from Hogg , to form enough of a barrier to foil the onrushing Driver . +Ali Jerba scored twice as Dons beat Hartlepool United 3-1 . +" This trial separation was agreed to with the goal of ultimately strengthening our marriage , " she said . +The | impact will be most evident in his foreign policy . +The company also said it would no longer obey censorship rules . +The United States is a rich nation . +Kartika Sari Dewi Shukarno says that if a caning is meant to teach a lesson , then it should be in public . +22 ( UPI ) -- A new , minimally invasive surgical instrument to remove varicose veins has been developed by a University of California-Los Angeles researcher . +Geller has even been criticized by other Islamophobes for her extremism and for supporting far-right groups in Europe . +One is that people who live in states with few blacks seem more open to the idea of a president who is not white . +Hatoyama repeated that the party would not raise the politically sensitive consumption tax from its current five percent for next four years . +After a week which saw the UK 's heaviest snowfall for 18 years , closing schools and crippling public transport , the rain has heightened the risk of flooding . +Aren 't travel companies -- particularly online travel agencies -- just victims of this scheme , like us ? +For many shoppers , it paid to wait given the plentiful offerings and good deals . +The first clue was the announcement last year by the American Society of Plastic Surgeons that the number of pectoral implants had tripled . +MiCTA is an association of non-profit organizations , colleges , universities , K-12 schools systems , federal state and local governments . +Ducks : Oct . +His staying power and his consistency , and his approach is unique . +The new bus would have four more seats than a Routemaster and standing room for 30 people , twice as many as its predecessor . +A Balearic Islands Superior Tribunal statement Tuesday says the autopsy shows Gately , 33 , died as a result of a pulmonary edema , or fluid in the lungs . +And it 's taking less time to pick up a truck , which he sees as a sign that there 's less work to keep them on the road--and out of his reposessors ' reach . +Tony Woodley , the leader of Unite , Britain 's biggest trade union told his members to boycott The Sun newspaper after the tabloid abandoned its support for the Labour Party . +Ms. Esfandiari was released from prison on Aug . +He was due to have retired next year . +McClement deflected in a shot from the point by Alex Pietrangelo to tie it with 4 : 25 left . +For Democrats , a break in that fight could allow them to focus on issues that voters say demand attention . +But the 23-year-old has stopped talking to her father , even though they live in the same house . +Typhoon Ketsana hit the Philippines and parts of Vietnam , leaving at least 300 people dead and hundreds of thousands more homeless . +" They physically jumped off , " Police Commissioner Raymond Kelly said . +Next month , Vodafone will increase its minimum call charge for pay-as-you-go customers by 30 per cent , from 15p to 20p , adding to pressure on consumers hit by rising gas and food bills as well as soaring inflation . +One argument is that industry intakes are gender specific . +Rather than being provoked by the attacks , 1st Scots Guards took pains to establish good relations with the local population and press . +While it seems like common sense to pump money into an economy that is pulling the bedcovers over its head , the problem with most social interventions is that they target not robots and machines but human beings -- who regularly respond to interventions in contrarian , paradoxical and unpredictable ways . +In a world that blends flamboyant costumes with decidedly bland personalities , figure skater Christopher Bowman stood out for his irrepressible individuality and unapologetic , undisciplined approach to the sport . +In Europe 's largest economy , Germany , the benchmark DAX index is off slightly more than 20 percent this year , and the CAC-40 in France is down almost 22 percent . +This appraisal of life in a more acidic ocean was if anything conservative , Dr Hall-Spencer said , because it mimicked future ecosystems only partially . +After yesterday 's thrillers with both finalists overcoming opponent fightbacks , this year 's final promises to be a cracker . +The fast-moving developments left liberals in a quandary . +The victims had moved to Palm Beach County from the Brownsville , Texas , area just a few months before they were killed . +In the previous week , 30-year , fixed-rate mortgages averaged 5.29 percent . +His daughter , Tiffany Ward , continues her father 's legacy as executive producer of the feature films " The Adventures of Rocky & Bullwinkle , " Dudley Do-Right " and " George of the Jungle , " and Cartoon Network 's ( sister channel to CNN ) new " George of the Jungle " animated TV series . +Goalies _ Anaheim , Hiller 6-5-1 ( 30 shots-29 saves ) . +She says she tried to talk Boyle into being " more out there " and to stray from India 's Bollywood traditions . +For instance , it 's not just about manufacturing those wind turbines in the cities . +" I showed Gattuso the midfield of the top five teams in Europe and he would not have been a starter in any of those . +" Starting over is not an option , though that 's the battle cry of the congressional wing " of the GOP , Emanuel said . +Now that has failed , they are trying to split the Shia . " +BAD RAP extensively evaluated all 49 dogs from the Vick property and found that 48 of them were safe with people . They selected 10 of the most outgoing and social dogs to bring back to the Bay Area for care , training and re-homing . +Could this be a Harry Potter-like pancultural behemoth ? +The probes were launched in the aftermath of the June 1 crash in the Atlantic Ocean -- when Air France Flight 447 was flying to Paris , France , from Rio de Janeiro , Brazil . +Nicholas is a writer in our Washington bureau . +A mother last night welcomed the decision to extend the prison sentences of the two men responsible for a fatal attack on her son . +During the conference , a policeman asked for their documents and they were subsequently detained , police said . +He was , for many , the ultimate performer . +His resignation after three years on the board of the Cupertino-based company comes after the US government announced in May that it was beginning an antitrust investigation into the interlocking directorships between Apple and Google . +Had it been Obama delivering a message , I 'm sure that King Jong IL would be blasting some more rockets off after a ten minute session with the Obamessiah . +A long bamboo fence divides two fields but every so often Sangram has left open a booby-trapped entrance . +The tonnes of coal and anthracite carved out of the mountains fed the trains and factory furnaces : Hillary Clinton recalled , as a child , marvelling at the Lackawanna River running through the city , black with coal dust , and the burning piles on the horizon . +However recent datapoints are giving us cause to expect some reversal . +We were grown adults . +Down three , minus 800 . +Q : I 've booked a flight during the strike period . +Leah 's Secret was fourth and the favorite , Visit , was never in the mix , finishing sixth . +Billy Tauzin , president of the industry 's trade association , highlighted the gloomy prediction in a June 1 letter to President Obama shortly before striking the deal to cut drug costs by $ 80 billion . +Before that , the final keynote speech comes from Secretary of State Hillary Rodham Clinton . +" I was even more wrong , both personally and professionally , to ask for your help in encouraging others to vote for the film and to comment on another movie . +GM barely won the global sales race with Toyota last year , but Toyota overtook it as the world 's top automaker as measured by global vehicle production in 2007 . +They are angry not only over his long-standing opposition to much modern architecture , but his efforts to block a major steel-and-glass tower project on the site of an old army barracks in the posh Chelsea neighborhood of London . +The Illinois senator has said the war in Afghanistan , where Taliban and al-Qaida-linked militants are resurgent , deserves more troops and more attention as opposed to the conflict in Iraq . +The remarks , published Friday in The Jerusalem Post daily , came just days before President Bush arrives in the region to build on the momentum created at a Mideast peace conference in Annapolis , Md . +Political sketchwriter Ann Treneman of The Times , had been a regular tormentor , her memorable descriptions of Sir Ming ranging from " a wrinkled old turtle " to " looking as if he had just escaped from a care home . " +My wife and I watched the moon rise over the Atlantic Yards site in Brooklyn on Tuesday evening . +Members of the Rashtriya Swayamsevak Sangh are often called chaddi wallahs because their uniform includes baggy khaki shorts . +Initially , it was thought the burial site dated from the Iron Age ( from BC 800 ) to early Roman times ( from AD 43 ) after examining pottery in the pit , later identified as a Roman quarry . +A nunchaku is made up of two wooden sticks joined by a chain or rope . +" The differences in terms of life satisfaction and attitudes towards the future underline the significant inequalities in living conditions and in the experience of daily life for Europeans , " said Eurofound director Jorma Karppinen . +Law enforcement officials say that such sting operations are an extremely effective means of lowering crime rates and stopping the criminally minded before they commit worse offenses . +It was suggested that his brawny physique and square-jawed poses masked a wooden style . +" My son was able to rescue two women ... but unfortunately the second blind rocket fell and claimed his life , " said Sadiq Abbas . +It was centered offshore , about 41 miles southwest of Eureka and 208 miles northwest of Sacramento . +This article was published on guardian.co.uk at 00.47 GMT on Saturday 19 December 2009 . +( AP ) -- The Oakland Raiders informed coach Tom Cable on Tuesday that they will not bring him back as coach next season even though he led them to their best record in eight years this season . +Police have said they do not know the exact reason why Walters attacked him . +That is just one position , " Boycott added . +Nancy Keenan , president of NARAL Pro-Choice America , said the federal ban is " unfair and insulting " and should be repealed . +Lt Col David Wakefield , spokesman for Task Force Helmand , said he would " not be forgotten . " +Overall rent has soared by 22 per cent in just three years , with the average undergraduate paying £ 99 a week in 2009 / 10 . +Zac Efron is more than busy promoting " High School Musical 3 : Senior Year , " but the actor tells Access Hollywood that although he plans on attending USC in the future , he has no plans of heading to Broadway . +MAPLE LEAFS 4 , LIGHTNING 3 Phil Kessel scored at 3 : 33 of overtime to give host Toronto the win . +Industrial production in the eurozone has risen for the fifth consecutive month , sparking hopes that GDP figures to be published on Friday could be higher than originally expected . +If young musicians are to have a chance of enjoying a fruitful career , then we need to establish the principle of artists ' rights throughout the Internet -- and we need to do it now . +First whiff of scandal for Obama ? +The lion 's share , however , will be found by raiding other parts of the overstretched defence budget , with planned cuts that would realise around £ 1.5 billion over three years . +When the House committee met late on July 7 to consider the Financial Services spending bill ( HR 3170 ) , LaTourette offered an amendment to require automakers that have taken government funding , namely General Motors Corp. and Chrysler LLC , to reinstate agreements with auto dealers dropped as part of their restructuring . +Whether it 's how to build something , understand something or create something , we hope you 'll find a book here that interests you . +A new ring of housing , the four- and five-story , Socialist-style apartment compounds , began to envelop the city . +Small said he thinks residents are comfortable with the idea because he 's only gotten a handful of e-mail complaints . +Next season 's play list for the Costa Mesa company , which made its name on the national scene as a launching pad for new work , features four world premieres out of nine main stage selections -- plus two new shows in the theater 's three-play Theatre for Young Audiences series . +During Watergate he scored an exclusive interview with a security guard for the Nixon re-election campaign who had been involved in the break-in at the headquarters of the Democratic National Committee . +The conjuring act has proven harder to pull off this campaign . +The typically bustling Lake Arrowhead resembled a ghost town , with abandoned shops and homes . +Legal experts say ratings agencies may prove harder to sue but that will not stop people trying or stop politicians calling for blood . +Fininvest , of which Mr Berlusconi owns 63.3 % , with the rest owned by his five children , reported net profits on June 18th of € 242m for 2008 and cash holdings of € 729m at year-end . +For example , with acute lymphoblastic leukemia , " we haven 't really added a new drug in 30 to 40 years , " Kamen said . +The announcement came three days after the orca pulled Brancheau into the water by her ponytail in front of park visitors . +So where do you go after interviewing the President ? +Fulham 's profligacy tested the crowd 's patience but they made the breakthrough after 56 minutes , when Dickson Etuhu met Paul Konchesky 's out-swinging corner and directed a header inside the far post . +He said the decline of the dollar , which has shed some 8 percent against the euro this year and hit a record low against a basket of major currencies , has been orderly and added that the exchange rate is set by markets . +Muller is the only man from Luxembourg to play in a Grand Slam tournament in the 40-year Open era--and now the first man or woman from the country to reach the fourth round . +The video clip has since been removed from such major state-linked sites as sina.com but is still a hot viewing item on smaller private video-sharing suppliers and has become the most talked about event in Chinese cyberspace this week . +He also defended the Treasury 's actions after it realised that new Whitehall accounting rules introduced in 2002 to cut costs had actually allowed the MoD to increase spending . +America is truly dead when we agree to pay a freak dog torturer $ 1.6 million to play a kid 's game . +In December a businessman living in Buckinghamshire returned home with his family to find " 3 " masked burglars in their home . +Mr Farage nimbly steered clear of the " spontaneous " demonstration taking place outside the hotel where he was speaking . +Analysts , on average , had expected the same-store sales to rise 1.8 percent , according to Thomson Financial . +The former president doesn 't join his wife on the campaign trail that often . +She died after suffering a brain haemorrhage on 10 September 2007 . +She says she and her husband Sean are focusing on what they have , not on what they might lose . +Isla Palenque , a new island resort community in Panama 's Gulf of Chiriqui , will offer superior service to residents and be a haven for those who decide to retire abroad . +The Pakistan army is currently fighting fierce battles against militants in Bajaur , at the north east extreme of the tribal belt , and Swat , an alpine valley in a more settled region close to the tribal lands . +A death abroad is subject to an inquest if the body is returned to Britain . +The loan comes as President Viktor Yushchenko and Prime Minister Yulia Tymoshenko are locked in a fierce battle over the president 's decision to call fresh elections . +One bore a phone number that rang at a custom ironworks company . +But even if a Conservative government proves to be more ferociously partisan compared with poor old timid New Labour I doubt if it will have cause to eject the Speaker . +The Denali pipeline consortium aims to deliver Alaskan gas to Canada and possibly to markets in the Lower 48 . +Coca cultivation surged by almost 30 percent last year in Colombia , which still provides 90 percent of the cocaine found in the United States . +The six-hitter followed his first career complete game , when he allowed three hits in a 6-3 victory over Atlanta last Wednesday . +Dow Jones industrial average futures rose 15 to 8,464 . +The exchange between Romney and Giuliani set the tone for the night in what is likely to turn into a very bitter battle for the Republican nomination . +They are knowledge intensive , rich in intellectual property and require high-level systems-integration skills . +Detectives and prosecutors stating there was " insufficient evidence " to prosecute . +Aaron Brooks had 19 points for the Rockets , who defeated Portland in six games this past spring in the first round of the playoffs . +Wolfson then read the first two paragraphs of the poll , which explained that in the 20 states where Clinton claimed popular vote victories , she led McCain by 50 percent to 43 percent for the general election . +Iteris is headquartered in Santa Ana , California . +I think Julia changed after moving for a few months and having an affair . +" The outcry from those in need of loans is substantial , " said Lee Sachs , special advisor to Treasury Secretary Timothy F. Geithner . +Seven men and one woman , aged between 18 and 53 , were due to appear at Cheltenham Magistrates Court . +Andre McGee matched his career high with 18 points and Derrick Caracter had 11 points and eight rebounds as Louisville ( 2-0 ) had little trouble with the Tigers ( 0-4 ) . +In April , the bank received a $ 7 billion private-equity investment , which helped build capital reserves . +You walk in off the street -- well , off that whopping new terminal -- straight on to a street , paved , lined with shops , bars , cafés and a pizzeria . +Michigan tourism leaders were also keeping close watch on behalf of Detroit , where casinos bring in $ 1.3 billion a year with the help with thousands of Ohio gamblers . +A central point of the president 's plan is to create a government-sponsored health insurance program that would be an option for all Americans , similar to how Medicare is now an option for Americans over age 65 . +Illinois offenders face similar restrictions , including a ban on dressing in a costume . +Assuming it is a game , of course , rather than , say , an official police procedure or a pitched and genuine battle between rival villages . +This strategy is widely considered to have been a mistake as it led to infighting between liberal and moderate Democrats , creating disillusionment among the public that led to support for healthcare reform declining the longer the debate continued . +Chu , who raised nearly $ 1 million , also had the support of the California Democratic Party and Los Angeles County Federation of Labor , which spent about $ 150,000 campaigning on her behalf . +The coffee table-ization of comics , said Mr. Liang , is partly because of another factor : the consistent growth of trade paperback editions of graphic novels , like Mr. Miller 's " The Dark Knight Returns " or Alan Moore 's " Watchmen , " since the late 1990s . +Instead , trailing by four near the end , they had to go for the try and did so from a close-range lineout won by Jonny Fa 'amatuainu . +To take the temperature of the current crop of albariños in the marketplace , the wine panel recently sampled 25 bottles , 17 from the 2006 vintage and 8 from the 2007 . +" It remains to be seen whether this [ grenade attack ] was caused by hooligans ... or other causes , " said Moldovan Interior Minister Viktor Katan . +In 2002 and 2003 the group ran programmes analysing urine from Afghans . +Joe Biden , in a lot of his speeches , is delivering them in a preacher sort of fashion that tunes really well . +" Thanks Billy , " she says , giving him the thumbs up . +Mr Kadyrov denied any involvement and promised to investigate the killing personally . +But Mr Hague said he had only found out in " the last few months " that the peer , who has donated millions to the Tories , had later renegotiated the deal with government officials . +Indeed , as soon as the motorcade arrived at his downtown hotel , agents began preparing for an imminent departure to an undisclosed dinner location . +The pictures , which capture well-known local scenes , will be on display at the Hadley Gallery . +The filmmakers ' decision to include a four-minute montage tracing the back story of Carl Fredricksen and his wife was a bold stroke that enriched the entire movie . +Overall , family income of blacks in their 30s was $ 35,000 , 58 percent that of comparable whites , a gap that did not surprise researchers . +On the security side , by keeping on George Bush 's first-rate defence secretary , Robert Gates , and ( probably ) by choosing a former general , Jim Jones , as his national security adviser , Mr Obama is showing that he will not let himself be tagged with the " Defeaticrat " label . +My vacations inevitably last an additional week as I return to work , fully jet-lagged . +The firing came just five months before her scheduled retirement . +Mr. Reilly joined Korn / Ferry in June 2001 as Chairman and Chief Executive Officer . +That means it will most likely be obligated to finance a considerably smaller amount as well as tie up less bank capital , though it still could amount to billions of dollars . +" It 's a backdoor tax increase , " said Sen. George Runner ( R-Lancaster ) . +" I 'm a Christian . +The topic of several biographies , Thomas , who was confirmed to the court by a 52-48 margin , has also been described as disinterested because he does not ask questions during oral presentations before the court . +" I 'm used to it blowing in Hawaii , but here it is 30 degrees colder . +The company pitch was that , for a $ 50,000 investment , people could buy homes and not worry about the price because Metro Dream Homes would make their mortgage payments . +The recommendation targets men with a normal reading on the prostate-specific antigen or PSA , test , which is considered the best indicator of the presence of a tumor , because clinical trials of the drug covered only such men . +Students , staff and faculty aim to paralyze the University of California on what is the first day of class on most campuses . +It is not a dish for the chaotic cook or anyone short of time . +An old , yellowed hotel register bears the signatures of Presidents Benjamin Harrison and Grover Cleveland . +The wife of embattled S.C. governor Mark Sanford speaks out for the first time , ... +It was his franchise record 16th RBI of the postseason . +I would have understood completely had he told ( England manager ) Fabio Capello he did not want to play against Egypt . +" The access expansions are a significant step forward , but this legislation will exacerbate the health care costs crisis facing many working families and small businesses , " Karen Ignagni , president of the America 's Health Insurance Plans group , said in a statement . +Yet the drones are unpopular with many Pakistanis , who see them as a violation of their country 's sovereignty -- one reason the United States refuses to officially acknowledge the attacks . +The Penguins managed one shot through the first 15 minutes , but finally turned it around late in the period . +LVG went into administration last Thursday . +The event took place in a hotel ballroom in the heart of Berlin . +1 , 1917 , in Leechburg , Pa . , and graduated from Monmouth College in 1938 . +A few thousand people in Spain 's northern Basque region have police or private bodyguards due to threats by ETA , which is blamed for more than 800 deaths in its long fight for Basque independence . +When the winter rains start , there is little vegetation left in the burn areas to prevent water , rocks , grit and branches from coursing down steep canyons and ravines toward thousands of homes . +However , Mr Donaldson said he should not have claimed for these items . +The Phillies got their first win while sporting a new alternate home uniform , a throwback model from the 1940s minus the customary red pinstripes the team has worn since 1950 . +They don 't care where we are . +I encountered that scam as well with United Healthcare . +Anywhere from 40,000 to 50,000 users are active at any time . +The Shea Stadium boos , which had been reserved for Chipper Jones and Luis Castillo until that point , rang out in full force . +Then Goma became a launching pad for two civil wars , one of which escalated into a regional conflict known as Africa 's First World War . +No one was hurt there , The Associated Press reported . +Further , the new bank holding company model adopted by Goldman Sachs and Morgan Stanley after rival Lehman Brothers was allowed to sink into bankruptcy , comes with regulations that did not apply to these titans when they were investment banks . +Advancing issues outnumbered decliners by about 3 to 1 on the New York Stock Exchange , where volume came to 940.1 million shares . +Homeowners can apply for a grant of up to £ 2,500 to install the technology . +The Europeans can get information and tell the world our demands . +Dallas activist Elizabeth Villafranca praised the News for its courage in giving the immigration debate such importance . +So Neil Gaiman , author of " Coraline " and other dark tales , can get weirded out from time to time . +By the time the team arrived , it was clear that word of the trip had spread like wildfire in the city . +" I honestly believe I can get a lot better . +6 , 2001 , to trigger the plaintiffs ' duty to investigate the alleged fraud . +Thornton later lost the ball to Pistons center Jason Maxiell , leading to an alley-oop dunk by Prince , forcing Saunders to call a timeout . +Most Kenyans cannot get hold of a copy . +The government has already announced plans to add a new course called " economic wellbeing " to PSHE from September 2008 , but it will remain voluntary . +One idea he has floated is a reduction , to 25 percent from 68 percent , in the state 's annual share of the revenue from the 1,500 slot machines that he plans to move from the Monticello Raceway . +The case comes in a state that was at the heart of the U.S. civil rights movement in the 1960s . +He then devoured a parade of oysters , steak , macaroni and cheese , mashed potatoes and several desserts . +After conquering the world 's tallest peak on May 29 , 1953 , Hillary returned to Nepal several times and founded the Himalayan Trust , which has built 27 schools , two hospitals and 12 clinics around Mount Everest . +Others will not be so kind . +" If you just traveled to Mexico and you 're home and feeling well you should get on with your daily life and not worry about it , " Evans said . +Dolsi ( 0-1 ) intentionally walked Torii Hunter before Seay came on . +A delegation including officials from the Chinese Development Bank recently spent a week in Guinea and discussing a deal which could see billions of dollars of Chinese investment in return for mining rights . +Alfred Mann , 87 , has struggled to eat and speak since he was injured in the land mine blast while serving with Royal Army Medical Corps in Italy in 1944 . +The doctor observed that a person suffering from Mr Doran 's symptoms would , in a civilian setting , normally be admitted to a hospital . +Parry has considered leaving Anfield once already this season when he was sounded out about the chief executive 's position at England 's 2018 World Cup bid . +You can look down into the space -- and at its bold graffiti-painted walls . +The woman , said as her " humiliation and terror " mounted she did her best to " switch off , " realising resistance was not only futile but could end in her murder . +A strike ballot is due to be held at Airwave from 24 February until 1 March . +Acting on police advice , the organisers decided to cancel the exhibition of artwork by 11 young ethnic Albanians from the Kosovo capital Pristina , Ljubica Beljanski Ristic told the news agency . +Gossage was anything but soft , and though he was good at his new job he didn 't relish it initially . +Capt. Thomas Ohlhaber of the U.S. Public Health Service -- who serves as deputy director of the Food and Drug Administration 's division of mammography quality and radiation programs -- describes X-rays as one of medicine 's most remarkable achievements . +Lombardi : As far as being able to throw out runners , I 'm not overly concerned about that issue . +" I think our number is either right on or negative equity may be even a little worse , " he said . +The correct figure of around 160,000 who have benefited from " time to pay " has been used by the chancellor , Alistair Darling , and Stephen Timms , the Treasury minister . +But if you show the least sign of nerves , the place gets nasty . +Roach immediately came on to pepper him with short balls and after he got one hook shot away , he fended one straight to short leg to be out for two to leave Australia on 134 for 8 . +But he said 2009 had been a bad year , with poachers killing 12 black rhinos and six white ones . +With public anger over the financial crisis and Wall Street bailouts channeled against the Fed , there is pressure on those backing Mr Bernanke to distance themselves from the central bank in a subsequent vote . +States classified ever larger numbers of young offenders as adults . +So where does this leave Obama 's promise to replace the fist of his White House predecessor and extend an open hand to Iran ? +D 'Arcy had been due to fly to Britain this week to compete in next month 's short-course world championships in Manchester but the Brisbane university student withdrew from the short-course team to consult his lawyers . +Shaken , she then rang her 32 year-old husband , Rob , who is also a vet . +It wasn 't your first effort at a project like this . +It occurs to me that perhaps you can be in fashion and normal after all . +Second-quarter premiums rose 25 percent to $ 3.49 billion , primarily due to a 17 percent increase in average membership and higher per member premium revenues . +Such systems do not seem to lead to the normalisation of unnatural death , and thus a steadily rising number of suicides ; rather , after an initial surge , the number seeking help to die drops and then stays steady . +" We 've got an agreement on an offer that will be made to the government of Iran , " David Miliband , the British foreign secretary , said for the six governments . +It looks for salvation in a revival of what Mr. Gioni calls the " mythopoetic function of art " and in the fantasies of the prophet in the wilderness . +Judging from the noises emanating from some corners of Washington these days , the federal debt has assumed pride of place as the source of national anxiety . +Construction has begun on approximately 60 new homes in a Jewish settlement in Israeli-occupied East Jerusalem , the Israeli campaign group Peace Now says . +" We were nomads , but that is being extinguished by this policy . +Finger pointing and namecalling isn 't becoming of a president . +ONStor , the ONStor logo , Pantera and Bobcat are trademarks of ONStor , Inc. in the U.S. and other countries. be superseded by subsequent documents and is subject to change without notice . +Wickmayer is seeking her second tournament victory in a row after winning last week 's title in Linz , Austria . +The North also appeared to be preparing for more missile tests , including one believed to be capable of reaching the U.S. A U.S. measure imposed on Banco Delta Asia , a bank in the Chinese territory of Macau , in 2005 effectively led to the North being severed from the international financial system , as other institutions voluntarily severed their dealings with the bank and the North . +A spokeswoman said : " Refurbishment of the coping stones and railings was well overdue . +But the last eight wickets fell for 50 runs and Scotland must now beat UAE on Friday , while hoping Afghanistan and Netherlands lose their final games . +That only inspired Ferrer , winner of the Valencia title last month , with the second seed re-breaking and serving out the set to bring on a deciding third . +THC and CBD appear to help people with MS control their bladders . +For the first six weeks , Jan and Sam shuttled back and forth from their home in Surbiton to the neonatal unit at Kingston Hospital . +The front two , facing the street , are carpeted , and one has a walk-in closet . +The rocket that narrowly missed the oncoming Sri Lankan bus had slammed into a parade of shops - reducing one shopfront to cinders . +About 6 hours after Capt. Thurman stopped by the side of the road to help out , his truck was hit by an IED . +The world 's key money markets were in a state of near-seizure for much of yesterday as the shockwaves from Lehman 's closure sparked a panicky scramble by investors to dump shares and risky assets and secure short-term cash to see them through the worsening financial storm. seven years . +England manager Fabio Capello has said Manchester United are not the " war machine " that once ruled the Premier League . +Regarding the private military contractor Blackwater , Bush said he was sorry " that innocent lives were lost , " but declined to comment further until an investigation was complete . +Senator Obama rebutted , point-by-point , each of the sleazy Rove-like attacks on his patriotism and vision . +They 've won three times by 20 points or more . +He increased his volume out of public view , in an intrasquad game and two simulations , and used one of those simulated games to practice how he would pitch to Boston hitters . diff --git a/tests/test_visual.py b/tests/test_visual.py new file mode 100644 index 0000000000..17ed06fce0 --- /dev/null +++ b/tests/test_visual.py @@ -0,0 +1,34 @@ +from flair.visual import tSNE +from flair.data import Sentence +from flair.embeddings import CharLMEmbeddings, StackedEmbeddings +import unittest +import numpy + + +class TesttSNE(unittest.TestCase): + def test(self): + with open('resources/data/snippet.txt') as f: + sentences = [x for x in f.read().split('\n') if x] + + sentences = [Sentence(x) for x in sentences[:100]] + + charlm_embedding_forward = CharLMEmbeddings('news-forward') + charlm_embedding_backward = CharLMEmbeddings('news-backward') + + embeddings = StackedEmbeddings( + [charlm_embedding_backward, charlm_embedding_forward] + ) + + trans_ = tSNE(embeddings) + + embeddings = trans_.fit(sentences) + + numpy.save(embeddings, 'resources/data/embeddings.npy') + + +if __name__ == '__main__': + unittest.main() + + + + From 3654ab777598ffd92273a561bf6985179def68ae Mon Sep 17 00:00:00 2001 From: Duncan Blythe Date: Mon, 13 Aug 2018 15:03:34 +0200 Subject: [PATCH 042/113] GH-61: added visualization and word/char visual tSNE makes sense on the word level. Bidirectional character embeddings don't make much sense. --- .gitignore | 3 + flair/visual/__init__.py | 2 +- flair/visual/manifold.py | 133 +++++++++++++++++++++++++++++++++++++++ flair/visual/tsne.py | 31 --------- tests/test_visual.py | 92 +++++++++++++++++++++++++-- 5 files changed, 223 insertions(+), 38 deletions(-) create mode 100644 flair/visual/manifold.py delete mode 100644 flair/visual/tsne.py diff --git a/.gitignore b/.gitignore index bcdbbb3b2d..4d3de671f2 100644 --- a/.gitignore +++ b/.gitignore @@ -104,3 +104,6 @@ venv.bak/ # mypy .mypy_cache/ + +# data directory +resources/data diff --git a/flair/visual/__init__.py b/flair/visual/__init__.py index bb50a64823..8ea3a4d75a 100644 --- a/flair/visual/__init__.py +++ b/flair/visual/__init__.py @@ -1 +1 @@ -from .tsne import tSNE \ No newline at end of file +from .manifold import tSNE, uMap, show, prepare_word_embeddings, prepare_char_embeddings, word_contexts, char_contexts \ No newline at end of file diff --git a/flair/visual/manifold.py b/flair/visual/manifold.py new file mode 100644 index 0000000000..44b54f36dc --- /dev/null +++ b/flair/visual/manifold.py @@ -0,0 +1,133 @@ +from sklearn.manifold import TSNE +from umap import UMAP +import tqdm +import numpy +import blessings + + +t = blessings.Terminal() + + +def prepare_word_embeddings(embeddings, sentences): + X = [] + + print('computing embeddings') + for sentence in tqdm.tqdm(sentences): + embeddings.embed(sentence) + + for i, token in enumerate(sentence): + X.append(token.embedding.detach().numpy()[None, :]) + + X = numpy.concatenate(X, 0) + + return X + + +def word_contexts(sentences): + contexts = [] + + for sentence in sentences: + + strs = [x.text for x in sentence.tokens] + + for i, token in enumerate(strs): + prop = ' {token} '.format( + token=token) + + prop = ' '.join(strs[max(i - 4, 0):i]) + prop + prop = prop + ' '.join(strs[i + 1:min(len(strs), i + 5)]) + + contexts.append('

' + prop + '

') + + return contexts + + +def prepare_char_embeddings(embeddings, sentences): + + X = [] + + print('computing embeddings') + for sentence in tqdm.tqdm(sentences): + + sentence = ' '.join([x.text for x in sentence]) + + hidden = embeddings.lm.get_representation(sentence) + X.append(hidden.squeeze().detach().numpy()) + + X = numpy.concatenate(X, 0) + + return X + + +def char_contexts(sentences): + + contexts = [] + + for sentence in sentences: + sentence = ' '.join([token.text for token in sentence]) + + for i, char in enumerate(sentence): + + context = '{}'.format(char) + context = ''.join(sentence[max(i - 30, 0):i]) + context + context = context + ''.join(sentence[i + 1:min(len(sentence), i + 30)]) + + contexts.append(context) + + return contexts + + +class _Transform: + def __init__(self): + pass + + def fit(self, X): + return self.transform.fit_transform(X) + + +class tSNE(_Transform): + def __init__(self): + + super().__init__() + + self.transform = \ + TSNE(n_components=2, verbose=1, perplexity=40, n_iter=300) + + +class uMap(_Transform): + def __init__(self): + + super().__init__() + + self.transform = UMAP( + n_neighbors = 5, + min_dist = 0.3, + metric = 'correlation', + ) + + +def show(X, contexts): + import matplotlib.pyplot + import mpld3 + + fig, ax = matplotlib.pyplot.subplots() + + ax.grid(True, alpha=0.3) + + points = ax.plot(X[:, 0], X[:, 1], 'o', color='b', + mec='k', ms=5, mew=1, alpha=.6) + + ax.set_xlabel('x') + ax.set_ylabel('y') + ax.set_title('Hover mouse to reveal context', size=20) + + tooltip = mpld3.plugins.PointHTMLTooltip( + points[0], + contexts, + voffset=10, + hoffset=10 + ) + + mpld3.plugins.connect(fig, tooltip) + + mpld3.show() diff --git a/flair/visual/tsne.py b/flair/visual/tsne.py deleted file mode 100644 index 5b8e5fa539..0000000000 --- a/flair/visual/tsne.py +++ /dev/null @@ -1,31 +0,0 @@ -from sklearn.manifold import TSNE -import tqdm -import numpy - - -class tSNE: - def __init__(self, embeddings): - - self.embeddings = embeddings - - self.transform = \ - TSNE(n_components=2, verbose=1, perplexity=40, n_iter=300) - - def _prepare(self, sentences): - - X = [] - - print('computing embeddings') - for sentence in tqdm.tqdm(sentences): - self.embeddings.embed(sentence) - - for token in sentence: - X.append(token.embedding.detach().numpy()[None, :]) - - X = numpy.concatenate(X, 0) - - return X - - def fit(self, sentences): - - return self.transform.fit_transform(self._prepare(sentences)) diff --git a/tests/test_visual.py b/tests/test_visual.py index 17ed06fce0..aa6027f805 100644 --- a/tests/test_visual.py +++ b/tests/test_visual.py @@ -1,12 +1,12 @@ -from flair.visual import tSNE +from flair.visual import * from flair.data import Sentence from flair.embeddings import CharLMEmbeddings, StackedEmbeddings import unittest import numpy -class TesttSNE(unittest.TestCase): - def test(self): +class Test(unittest.TestCase): + def test_prepare(self): with open('resources/data/snippet.txt') as f: sentences = [x for x in f.read().split('\n') if x] @@ -19,11 +19,91 @@ def test(self): [charlm_embedding_backward, charlm_embedding_forward] ) - trans_ = tSNE(embeddings) + X = prepare_word_embeddings(embeddings, sentences) + contexts = word_contexts(sentences) + + numpy.save('resources/data/embeddings', X) + + with open('resources/data/contexts.txt', 'w') as f: + f.write('\n'.join(contexts)) + + def test_tSNE(self): + + X = numpy.load('resources/data/embeddings.npy') + trans_ = tSNE() + reduced = trans_.fit(X) + + numpy.save('resources/data/tsne', reduced) + + def test__prepare_char(self): + + with open('resources/data/snippet.txt') as f: + sentences = [x for x in f.read().split('\n') if x] + + sentences = [Sentence(x) for x in sentences[:100]] + + embeddings = CharLMEmbeddings('news-forward') + + X_forward = prepare_char_embeddings(embeddings, sentences) + + embeddings = CharLMEmbeddings('news-backward') + + X_backward = prepare_char_embeddings(embeddings, sentences) + + X = numpy.concatenate([X_forward, X_backward], axis=1) + + numpy.save('resources/data/char_embeddings', X) + + def test_tSNE_char(self): + + X = numpy.load('resources/data/char_embeddings.npy') + trans_ = tSNE() + reduced = trans_.fit(X) + + numpy.save('resources/data/char_tsne', reduced) + + def test_char_contexts(self): + + with open('resources/data/snippet.txt') as f: + sentences = [x for x in f.read().split('\n') if x] + + sentences = [Sentence(x) for x in sentences[:100]] + + contexts = char_contexts(sentences) + + with open('resources/data/char_contexts.txt', 'w') as f: + f.write('\n'.join(contexts)) + + +class TestuMap(unittest.TestCase): + def test(self): + + X = numpy.load('resources/data/embeddings.npy') + + reduced = uMap().fit(X) + + numpy.save('resources/data/umap', reduced) + + +class Test_show(unittest.TestCase): + def test_word(self): + + reduced = numpy.load('resources/data/umap.npy') + + with open('resources/data/contexts.txt') as f: + contexts = f.read().split('\n') + + show(reduced, contexts) + + def test_char(self): + + reduced = numpy.load('resources/data/char_tsne.npy') + + with open('resources/data/char_contexts.txt') as f: + contexts = f.read().split('\n') - embeddings = trans_.fit(sentences) + show(reduced, contexts) - numpy.save(embeddings, 'resources/data/embeddings.npy') if __name__ == '__main__': From 76662247da8e5c0063a6bbfe65b35e5a85189e86 Mon Sep 17 00:00:00 2001 From: Duncan Blythe Date: Mon, 13 Aug 2018 15:41:10 +0200 Subject: [PATCH 043/113] GH-61: added requirements and char level visual --- flair/visual/manifold.py | 5 --- requirements.txt | 5 ++- tests/test_visual.py | 83 +++++++++++++++++++++++++++++++++++++--- 3 files changed, 81 insertions(+), 12 deletions(-) diff --git a/flair/visual/manifold.py b/flair/visual/manifold.py index 44b54f36dc..74f0b50181 100644 --- a/flair/visual/manifold.py +++ b/flair/visual/manifold.py @@ -1,11 +1,6 @@ from sklearn.manifold import TSNE -from umap import UMAP import tqdm import numpy -import blessings - - -t = blessings.Terminal() def prepare_word_embeddings(embeddings, sentences): diff --git a/requirements.txt b/requirements.txt index fdda2ad34c..350fc679cb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,4 +4,7 @@ gensim==3.4.0 typing==3.6.4 pytest==3.6.4 tqdm -segtok \ No newline at end of file +segtok +matplotlib +mpld3 +sklearn \ No newline at end of file diff --git a/tests/test_visual.py b/tests/test_visual.py index aa6027f805..1e87a13908 100644 --- a/tests/test_visual.py +++ b/tests/test_visual.py @@ -62,6 +62,27 @@ def test_tSNE_char(self): numpy.save('resources/data/char_tsne', reduced) + def test_prepare_char_uni(self): + + with open('resources/data/snippet.txt') as f: + sentences = [x for x in f.read().split('\n') if x] + + sentences = [Sentence(x) for x in sentences[:100]] + + embeddings = CharLMEmbeddings('news-forward') + + X = prepare_char_embeddings(embeddings, sentences) + + numpy.save('resources/data/uni_embeddings', X) + + def test_tSNE_char_uni(self): + + X = numpy.load('resources/data/uni_embeddings.npy') + trans_ = tSNE() + reduced = trans_.fit(X) + + numpy.save('resources/data/uni_tsne', reduced) + def test_char_contexts(self): with open('resources/data/snippet.txt') as f: @@ -74,21 +95,48 @@ def test_char_contexts(self): with open('resources/data/char_contexts.txt', 'w') as f: f.write('\n'.join(contexts)) + def test_benchmark(self): -class TestuMap(unittest.TestCase): - def test(self): + import time + + with open('resources/data/snippet.txt') as f: + sentences = [x for x in f.read().split('\n') if x] + + sentences = [Sentence(x) for x in sentences[:10]] - X = numpy.load('resources/data/embeddings.npy') - reduced = uMap().fit(X) + charlm_embedding_forward = CharLMEmbeddings('news-forward') + charlm_embedding_backward = CharLMEmbeddings('news-backward') + + embeddings = StackedEmbeddings( + [charlm_embedding_backward, charlm_embedding_forward] + ) - numpy.save('resources/data/umap', reduced) + tic = time.time() + + prepare_word_embeddings(embeddings, sentences) + + current_elaped = time.time() - tic + + print('current implementation: {} sec/ sentence'.format(current_elaped / 10)) + + embeddings_f = CharLMEmbeddings('news-forward') + embeddings_b = CharLMEmbeddings('news-backward') + + tic = time.time() + + prepare_char_embeddings(embeddings_f, sentences) + prepare_char_embeddings(embeddings_b, sentences) + + current_elaped = time.time() - tic + + print('pytorch implementation: {} sec/ sentence'.format(current_elaped / 10)) class Test_show(unittest.TestCase): def test_word(self): - reduced = numpy.load('resources/data/umap.npy') + reduced = numpy.load('resources/data/tsne.npy') with open('resources/data/contexts.txt') as f: contexts = f.read().split('\n') @@ -104,6 +152,29 @@ def test_char(self): show(reduced, contexts) + def test_uni_sentence(self): + + reduced = numpy.load('resources/data/uni_tsne.npy') + + with open('resources/data/snippet.txt') as f: + sentences = [x for x in f.read().split('\n') if x] + + l = len(sentences[0]) + + with open('resources/data/char_contexts.txt') as f: + contexts = f.read().split('\n') + + show(reduced[:l], contexts[:l]) + + def test_uni(self): + + reduced = numpy.load('resources/data/uni_tsne.npy') + + with open('resources/data/char_contexts.txt') as f: + contexts = f.read().split('\n') + + show(reduced, contexts) + if __name__ == '__main__': From fcd0c3f2532392eee1005add20829785cad06b7a Mon Sep 17 00:00:00 2001 From: Duncan Blythe Date: Mon, 13 Aug 2018 16:41:07 +0200 Subject: [PATCH 044/113] GH-61: added highlighter over paragraph --- flair/visual/__init__.py | 3 +- flair/visual/activations.py | 66 +++++++++++++++++++++++++++++++++++++ flair/visual/manifold.py | 12 ------- tests/test_visual.py | 15 +++++++++ 4 files changed, 83 insertions(+), 13 deletions(-) create mode 100644 flair/visual/activations.py diff --git a/flair/visual/__init__.py b/flair/visual/__init__.py index 8ea3a4d75a..4b35bff9f1 100644 --- a/flair/visual/__init__.py +++ b/flair/visual/__init__.py @@ -1 +1,2 @@ -from .manifold import tSNE, uMap, show, prepare_word_embeddings, prepare_char_embeddings, word_contexts, char_contexts \ No newline at end of file +from .manifold import tSNE, show, prepare_word_embeddings, prepare_char_embeddings, word_contexts, char_contexts +from .activations import Highlighter \ No newline at end of file diff --git a/flair/visual/activations.py b/flair/visual/activations.py new file mode 100644 index 0000000000..f9cabd0726 --- /dev/null +++ b/flair/visual/activations.py @@ -0,0 +1,66 @@ +import numpy + + +class Highlighter: + def __init__(self): + self.color_map = [ + "#ff0000", + "#ff4000", + "#ff8000", + "#ffbf00", + "#ffff00", + "#bfff00", + "#80ff00", + "#40ff00", + "#00ff00", + "#00ff40", + "#00ff80", + "#00ffbf", + "#00ffff", + "#00bfff", + "#0080ff", + "#0040ff", + "#0000ff", + "#4000ff", + "#8000ff", + "#bf00ff", + "#ff00ff", + "#ff00bf", + "#ff0080", + "#ff0040", + "#ff0000", + ] + + def highlight(self, activation, text, file_='resources/data/highlight.html'): + + activation = activation.detach().numpy() + + step_size = (max(activation) - min(activation)) / len(self.color_map) + + lookup = numpy.array(list( + numpy.arange(min(activation), max(activation), step_size) + )) + + colors = [] + + for i, act in enumerate(activation): + + try: + colors.append( + self.color_map[numpy.where(act > lookup)[0][-1]] + ) + except IndexError: + colors.append(len(self.color_map) - 1) + + str_ = '' + + for i, (char, color) in enumerate(zip(list(text), colors)): + str_ += self._render(char, color) + + with open(file_, 'w') as f: + f.write(str_) + + def _render(self, char, color): + return '{}'.format(color, char) + + diff --git a/flair/visual/manifold.py b/flair/visual/manifold.py index 74f0b50181..373b9a9d13 100644 --- a/flair/visual/manifold.py +++ b/flair/visual/manifold.py @@ -89,18 +89,6 @@ def __init__(self): TSNE(n_components=2, verbose=1, perplexity=40, n_iter=300) -class uMap(_Transform): - def __init__(self): - - super().__init__() - - self.transform = UMAP( - n_neighbors = 5, - min_dist = 0.3, - metric = 'correlation', - ) - - def show(X, contexts): import matplotlib.pyplot import mpld3 diff --git a/tests/test_visual.py b/tests/test_visual.py index 1e87a13908..12ba4a6b8a 100644 --- a/tests/test_visual.py +++ b/tests/test_visual.py @@ -176,6 +176,21 @@ def test_uni(self): show(reduced, contexts) +class TestHighlighter(unittest.TestCase): + def test(self): + + i = numpy.random.choice(2048) + + with open('resources/data/snippet.txt') as f: + sentences = [x for x in f.read().split('\n') if x] + + embeddings = CharLMEmbeddings('news-forward') + + features = embeddings.lm.get_representation(sentences[0]).squeeze() + + Highlighter().highlight(features[:, i], sentences[0]) + + if __name__ == '__main__': unittest.main() From cc65d798e590ec08d9f8d679c77169280fe58b76 Mon Sep 17 00:00:00 2001 From: Duncan Blythe Date: Mon, 13 Aug 2018 16:44:33 +0200 Subject: [PATCH 045/113] GH-61: added new line characters to html rendering --- flair/visual/activations.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/flair/visual/activations.py b/flair/visual/activations.py index f9cabd0726..2cf9b91638 100644 --- a/flair/visual/activations.py +++ b/flair/visual/activations.py @@ -57,6 +57,9 @@ def highlight(self, activation, text, file_='resources/data/highlight.html'): for i, (char, color) in enumerate(zip(list(text), colors)): str_ += self._render(char, color) + if i % 100 == 0 and i > 0: + str_ += '
' + with open(file_, 'w') as f: f.write(str_) From cd8d2a3b6daa39fcb6b94ffff53ee763e3d626a5 Mon Sep 17 00:00:00 2001 From: Duncan Blythe Date: Tue, 14 Aug 2018 14:42:46 +0200 Subject: [PATCH 046/113] GH-61-visualization fixed bug in calling language model --- flair/visual/activations.py | 18 +++++++++++++++--- flair/visual/manifold.py | 2 +- tests/test_visual.py | 2 +- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/flair/visual/activations.py b/flair/visual/activations.py index 2cf9b91638..c7837f3958 100644 --- a/flair/visual/activations.py +++ b/flair/visual/activations.py @@ -31,7 +31,7 @@ def __init__(self): "#ff0000", ] - def highlight(self, activation, text, file_='resources/data/highlight.html'): + def highlight(self, activation, text): activation = activation.detach().numpy() @@ -52,7 +52,7 @@ def highlight(self, activation, text, file_='resources/data/highlight.html'): except IndexError: colors.append(len(self.color_map) - 1) - str_ = '' + str_ = '

' for i, (char, color) in enumerate(zip(list(text), colors)): str_ += self._render(char, color) @@ -60,8 +60,20 @@ def highlight(self, activation, text, file_='resources/data/highlight.html'): if i % 100 == 0 and i > 0: str_ += '
' + return str_ + + def highlight_selection(self, activations, text, file_='resources/data/highlight.html', n=10): + + ix = numpy.random.choice(activations.shape[1], size=n) + + rendered = '' + + for i in ix: + + rendered += self.highlight(activations[:, i], text) + with open(file_, 'w') as f: - f.write(str_) + f.write(rendered) def _render(self, char, color): return '{}'.format(color, char) diff --git a/flair/visual/manifold.py b/flair/visual/manifold.py index 373b9a9d13..9ea486c52d 100644 --- a/flair/visual/manifold.py +++ b/flair/visual/manifold.py @@ -46,7 +46,7 @@ def prepare_char_embeddings(embeddings, sentences): sentence = ' '.join([x.text for x in sentence]) - hidden = embeddings.lm.get_representation(sentence) + hidden = embeddings.lm.get_representation([sentence]) X.append(hidden.squeeze().detach().numpy()) X = numpy.concatenate(X, 0) diff --git a/tests/test_visual.py b/tests/test_visual.py index 12ba4a6b8a..4e51d4d2e4 100644 --- a/tests/test_visual.py +++ b/tests/test_visual.py @@ -188,7 +188,7 @@ def test(self): features = embeddings.lm.get_representation(sentences[0]).squeeze() - Highlighter().highlight(features[:, i], sentences[0]) + Highlighter().highlight_selection(features, sentences[0], n=1000) From b1929e1d51413d411a581d5f8194f65c0bebe7a2 Mon Sep 17 00:00:00 2001 From: tabergma Date: Thu, 4 Oct 2018 10:07:14 +0200 Subject: [PATCH 047/113] GH-61: Read line one by one. --- flair/data_fetcher.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/flair/data_fetcher.py b/flair/data_fetcher.py index 73e17f1225..097edfc992 100644 --- a/flair/data_fetcher.py +++ b/flair/data_fetcher.py @@ -332,9 +332,7 @@ def read_text_classification_file(path_to_file, max_tokens_per_doc=-1): sentences = [] with open(path_to_file) as f: - lines = f.readlines() - - for line in lines: + for line in f: words = line.split() labels = [] From 1427c739e40491091c52de5ea479e1836c7cef8c Mon Sep 17 00:00:00 2001 From: tabergma Date: Thu, 4 Oct 2018 10:07:38 +0200 Subject: [PATCH 048/113] GH-61: Update imports in __init__ --- flair/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/flair/__init__.py b/flair/__init__.py index 2b74ad0143..98200efa19 100644 --- a/flair/__init__.py +++ b/flair/__init__.py @@ -1,5 +1,6 @@ from . import data from . import models +from . import visual import sys import logging From f9d791a1a461757c768a4dd1e4d92cd9ea8e6dab Mon Sep 17 00:00:00 2001 From: tabergma Date: Thu, 4 Oct 2018 10:08:30 +0200 Subject: [PATCH 049/113] GH-61: Add plotter for training curves. --- flair/visual/training_curves.py | 192 ++++++++++++++++++++++++++++++++ train.py | 7 +- 2 files changed, 197 insertions(+), 2 deletions(-) create mode 100644 flair/visual/training_curves.py diff --git a/flair/visual/training_curves.py b/flair/visual/training_curves.py new file mode 100644 index 0000000000..734f92b3f2 --- /dev/null +++ b/flair/visual/training_curves.py @@ -0,0 +1,192 @@ +from collections import defaultdict + +import numpy as np +import csv +import os + +import matplotlib +import math +import sys + +matplotlib.use('Agg') + +import matplotlib.pyplot as plt + + +# header for 'loss.tsv' +TRAIN_LOSS = 2 +TRAIN_F_SCORE = 9 +TRAIN_ACCURACY = 10 +DEV_LOSS = 11 +DEV_F_SCORE = 18 +DEV_ACCURACY = 19 +TEST_LOSS = 20 +TEST_F_SCORE = 27 +TEST_ACCURACY = 28 + +# header for 'weights.txt' +WEIGHT_NAME = 1 +WEIGHT_NUMBER = 2 +WEIGHT_VALUE = 3 + + +class Plotter(object): + """ + Plots training curves (loss, f-score, and accuracy) and training weights over time. + Input files are the output files loss.tsv and weights.txt from training either a sequence label tagger or text + classification model. + """ + + @staticmethod + def _extract_evaluation_data(file_name) -> dict: + training_curves = { + 'train': { + 'loss': [], + 'f_score': [], + 'acc': [] + }, + 'test': { + 'loss': [], + 'f_score': [], + 'acc': [] + }, + 'dev': { + 'loss': [], + 'f_score': [], + 'acc': [] + } + } + + with open(file_name, 'r') as tsvin: + tsvin = csv.reader(tsvin, delimiter='\t') + next(tsvin, None) + + for row in tsvin: + if row[TRAIN_LOSS] != '_': training_curves['train']['loss'].append(row[TRAIN_LOSS]) + if row[TRAIN_F_SCORE] != '_': training_curves['train']['f_score'].append(row[TRAIN_F_SCORE]) + if row[TRAIN_ACCURACY] != '_': training_curves['train']['acc'].append(row[TRAIN_ACCURACY]) + if row[DEV_LOSS] != '_': training_curves['dev']['loss'].append(row[DEV_LOSS]) + if row[DEV_F_SCORE] != '_': training_curves['dev']['f_score'].append(row[DEV_F_SCORE]) + if row[DEV_ACCURACY] != '_': training_curves['dev']['acc'].append(row[DEV_ACCURACY]) + if row[TEST_LOSS] != '_': training_curves['test']['loss'].append(row[TEST_LOSS]) + if row[TEST_F_SCORE] != '_': training_curves['test']['f_score'].append(row[TEST_F_SCORE]) + if row[TEST_ACCURACY] != '_': training_curves['test']['acc'].append(row[TEST_ACCURACY]) + + return training_curves + + @staticmethod + def _extract_weight_data(file_name) -> dict: + weights = defaultdict(lambda: defaultdict(lambda: list())) + + with open(file_name, 'r') as tsvin: + tsvin = csv.reader(tsvin, delimiter='\t') + + for row in tsvin: + name = row[WEIGHT_NAME] + param = row[WEIGHT_NUMBER] + value = float(row[WEIGHT_VALUE]) + + weights[name][param].append(value) + + return weights + + def plot_weights(self, file_name): + weights = self._extract_weight_data(file_name) + + total = len(weights) + columns = 2 + rows = max(2, int(math.ceil(total / columns))) + + figsize = (5, 5) + if rows != columns: + figsize = (5, rows + 5) + + fig = plt.figure() + f, axarr = plt.subplots(rows, columns, figsize=figsize) + + c = 0 + r = 0 + for name, values in weights.items(): + # plot i + axarr[r, c].set_title(name, fontsize=6) + for _, v in values.items(): + axarr[r, c].plot(np.arange(0, len(v)), v, linewidth=0.35) + axarr[r, c].set_yticks([]) + axarr[r, c].set_xticks([]) + c += 1 + if c == columns: + c = 0 + r += 1 + + while r != rows and c != columns: + axarr[r, c].set_yticks([]) + axarr[r, c].set_xticks([]) + c += 1 + if c == columns: + c = 0 + r += 1 + + # save plots + f.subplots_adjust(hspace=0.5) + plt.tight_layout(pad=1.0) + path = os.path.join(os.path.dirname(file_name), 'weights.png') + plt.savefig(path, dpi=300) + + plt.close(fig) + + def plot_training_curves(self, file_name): + fig = plt.figure(figsize=(15, 10)) + + training_curves = self._extract_evaluation_data(file_name) + + # plot 1 + plt.subplot(3, 1, 1) + if training_curves['train']['loss']: + x = np.arange(0, len(training_curves['train']['loss'])) + plt.plot(x, training_curves['train']['loss'], label='training loss') + if training_curves['dev']['loss']: + x = np.arange(0, len(training_curves['dev']['loss'])) + plt.plot(x, training_curves['dev']['loss'], label='validation loss') + if training_curves['test']['loss']: + x = np.arange(0, len(training_curves['test']['loss'])) + plt.plot(x, training_curves['test']['loss'], label='test loss') + plt.legend(bbox_to_anchor=(1.04, 0), loc="lower left", borderaxespad=0) + plt.ylabel('loss') + plt.xlabel('epochs') + + # plot 2 + plt.subplot(3, 1, 2) + if training_curves['train']['acc']: + x = np.arange(0, len(training_curves['train']['acc'])) + plt.plot(x, training_curves['train']['acc'], label='training accuracy') + if training_curves['dev']['acc']: + x = np.arange(0, len(training_curves['dev']['acc'])) + plt.plot(x, training_curves['dev']['acc'], label='validation accuracy') + if training_curves['test']['acc']: + x = np.arange(0, len(training_curves['test']['acc'])) + plt.plot(x, training_curves['test']['acc'], label='test accuracy') + plt.legend(bbox_to_anchor=(1.04, 0), loc="lower left", borderaxespad=0) + plt.ylabel('accuracy') + plt.xlabel('epochs') + + # plot 3 + plt.subplot(3, 1, 3) + if training_curves['train']['f_score']: + x = np.arange(0, len(training_curves['train']['f_score'])) + plt.plot(x, training_curves['train']['f_score'], label='training f1-score') + if training_curves['dev']['f_score']: + x = np.arange(0, len(training_curves['dev']['f_score'])) + plt.plot(x, training_curves['dev']['f_score'], label='validation f1-score') + if training_curves['test']['f_score']: + x = np.arange(0, len(training_curves['test']['f_score'])) + plt.plot(x, training_curves['test']['f_score'], label='test f1-score') + plt.legend(bbox_to_anchor=(1.04, 0), loc="lower left", borderaxespad=0) + plt.ylabel('f1-score') + plt.xlabel('epochs') + + # save plots + plt.tight_layout(pad=1.0) + path = os.path.join(os.path.dirname(file_name), 'training.png') + plt.savefig(path, dpi=300) + + plt.close(fig) diff --git a/train.py b/train.py index 1fe2cff712..3ac737b08c 100644 --- a/train.py +++ b/train.py @@ -1,10 +1,9 @@ from typing import List -import torch - from flair.data_fetcher import NLPTaskDataFetcher, NLPTask from flair.data import TaggedCorpus from flair.embeddings import TokenEmbeddings, WordEmbeddings, StackedEmbeddings, CharLMEmbeddings, CharacterEmbeddings +from flair.visual.training_curves import Plotter # 1. get the corpus corpus: TaggedCorpus = NLPTaskDataFetcher.fetch_data(NLPTask.CONLL_03).downsample(0.1) @@ -49,3 +48,7 @@ trainer: SequenceTaggerTrainer = SequenceTaggerTrainer(tagger, corpus, test_mode=True) trainer.train('resources/taggers/example-ner', learning_rate=0.1, mini_batch_size=32, max_epochs=20) + +plotter = Plotter() +plotter.plot_training_curves('resources/taggers/example-ner/loss.tsv') +plotter.plot_weights('resources/taggers/example-ner/weights.txt') \ No newline at end of file From 981efc1aaea9f79b091aec262efd7cec371113ac Mon Sep 17 00:00:00 2001 From: tabergma Date: Thu, 4 Oct 2018 11:24:42 +0200 Subject: [PATCH 050/113] GH-61: Update documentation --- resources/data/snippet.txt | 2000 ------------------- resources/docs/TUTORIAL_TRAINING_A_MODEL.md | 20 +- 2 files changed, 19 insertions(+), 2001 deletions(-) delete mode 100644 resources/data/snippet.txt diff --git a/resources/data/snippet.txt b/resources/data/snippet.txt deleted file mode 100644 index 96b66ed166..0000000000 --- a/resources/data/snippet.txt +++ /dev/null @@ -1,2000 +0,0 @@ -The U.S. Centers for Disease Control and Prevention initially advised school systems to close if outbreaks occurred , then reversed itself , saying the apparent mildness of the virus meant most schools and day care centers should stay open , even if they had confirmed cases of swine flu . -When Ms. Winfrey invited Suzanne Somers to share her controversial views about bio-identical hormone treatment on her syndicated show in 2009 , it won Ms. Winfrey a rare dollop of unflattering press , including a Newsweek cover story titled " Crazy Talk : Oprah , Wacky Cures & You . " -Elk calling -- a skill that hunters perfected long ago to lure game with the promise of a little romance -- is now its own sport . -Don 't ! -Fish , ranked 98th in the world , fired 22 aces en route to a 6-3 , 6-7 ( 5 / 7 ) , 7-6 ( 7 / 4 ) win over seventh-seeded Argentinian David Nalbandian . -Why does everything have to become such a big issue ? -AMMAN ( Reuters ) - King Abdullah of Jordan will meet U.S. President Barack Obama in Washington on April 21 to lobby on behalf of Arab states for a stronger U.S. role in Middle East peacemaking , palace officials said on Sunday . -To help keep traffic flowing the Congestion Charge will remain in operation through-out the strike and TfL will be suspending road works on major London roads wherever possible . -If no candidate wins an absolute majority , there will be a runoff between the top two contenders , most likely in mid-October . -Authorities previously served search warrants at Murray 's Las Vegas home and his businesses in Las Vegas and Houston . -Brent North Sea crude for November delivery rose 84 cents to 68.88 dollars a barrel . -That seems to have been their model up til now . -Gordon will join Luol Deng on the GB team ; their respective NBA teams , the Detroit Pistons and the Chicago Bulls , play tonight . -Nikam maintains the attacks were masterminded by the Muslim militant group Lashkar-e-Taiba . -Last year , Williams was unseeded , ranked 81st and coming off one of her worst losses on tour -- in a Tier 4 event at Hobart -- yet she beat six seeded players en route to the title at Melbourne Park . -It said that two officers involved in the case had been disciplined . -" There is more intelligence now being gathered , " the official said , adding that such efforts would continue for some time . -The majority will be of the standard 6X6 configuration for carrying personnel . -" Consequently , necessary actions may not be taken to reduce the risks to children of sexual exploitation and drug or alcohol misuse , " the report said . • Almost two-thirds of inspected schools were good or outstanding , but the number of underperforming secondaries remained " stubborn and persistent . " -What a World Cup . -But , there have also been many cases of individuals and small groups of people protesting , as in the case of Rongye Adak , a nomad who called for the return of the Dalai Lama and for the freedom of Tibet during the Lithang Horse Racing Festival , in eastern Tibet . -James Duncan , head of transportation at Bournemouth Borough Council , said : " Our legal team is reviewing the entitlement of taxis to drop and pick up passengers at bus stops , only for as long as is absolutely necessary to fulfil that function and for no other reason . -To Mo concerning the food log you kept -- Dr. Buchholz recommends the same thing . -The CBO estimates that only 23 percent of that would be spent in 2009 and 2010 . -Even so , Democrats slammed Bush as out of touch . -An information campaign will be launched later to raise awareness of employment rights and how to enforce them . -At the gallery the concept is less vague , as Ms. Piper cites specific instances of racial violence , political assassinations and the devastation of Hurricane Katrina . -There have been some exceptions -- such as Medicare in 1965 . -The government guidance will be reviewed early next year after a period of public comment . -It wasn 't the most seaworthy of prizes . -LOUIS - A former Anheuser-Busch executive who was the company 's top-ranking woman has sued the brewer claiming it encourages a " frat party " atmosphere and pays women less in salary and bonuses than men . -What does it say about Pelosi that she lost 95 dem votes when she has all the power . -Skea , a 3-month-old Taipei store that has nothing to do with Ikea , sells custom-made , small-scale buildings as well as cars , boats , baseball mitts or just about anything else that can be made from paper , cardboard and light wood . -A dozen athletes with connections to Lake Placid competed in Vancouver . -They owe $ 10,500 on their car . -" We need them to move on and accept change , because their problems are a distraction from our goal of building a more integrated Asean , " said Ong Keng Yong , the association 's secretary general . -The internet is sort-of-40 this year . -His approach was inquisitive , a meeting of artful hesitation with fluid technique . -Katrin 's father , Dr Edmund Radmacher , inherited a chemistry company that he transformed into a flourishing concern . -On that basis Mr Perry is not a man fit for higher office . -Grand ladies in exquisitely tailored houndstooth check may have had husbands who bought them their rarefied Dior clothing , but it is only too clear who was wearing the trousers . -The two men reportedly lived alone in the shelter , a two-bedroom flat that is run by a private social care company , financed partly by the city authorities . -A lot rode on the opening lead . -Why you support the FAKE war on terror I understand , you are just following orders . -The motive for the attack on the pope remains unclear but it has not been linked to Islamic issues . -A post-mortem examination is due to be carried out in the next few days , a spokesman said . -HBOS is the lead bank in a syndicate that includes Lloyds TSB , Royal Bank of Scotland , HSBC and National Australia Bank and granted New Star the loan last year . -You do , however , still have to choose between those technologies on most larger screens . -There are currently some 5 million homeowners that are 90 days or more past due on their mortgages , according to Fannie Mae chief economist Doug Duncan . -There was an extended paralysis of racing in the major horse racing states of New South Wales and Queensland , including the loss of popular traditional racing carnivals . -They wanted things to be monumental , so they could be seen from a distance , " explains Laurence Mouillefarine , associate curator of the show , picking out some big , showy brooches to illustrate the point . -More than 45,000 women are diagnosed with breast cancer each year in the UK , and more than 12,000 die from the disease . -That agency 's grant panels do not use culturally specific criteria when awarding money . -In the five months since he started the company , he has proven his hunch : People are willing to pay for New York City tap water . -They also maintain that because Google 's system for identifying and displaying ads is more lucrative than Yahoo 's approach , the deal will generate additional revenue for Yahoo that will it make it a more formidable competitor to both Google and Microsoft . -Forecasts called for plunging temperatures and afternoon rain storms in the early Southern Hemisphere autumn . -Kloppers said BHP would continue talking to its customers about the merger . -Sailing is a passion . -I would argue that the media caters just to that , if not simple minded , to those who are not disciplined enough to see out the truth , THEIR OWN truth . -Android is being developed , " assures Mobiclip 's vice-president of marketing Denis Pagnac . -LONDON ( Reuters ) - Want to know your chance of having a baby through in-vitro fertilization ? -NEW YORK ( Reuters ) - Toll Brothers ( TOL.N : Quote , Profile , Research ) said on Thursday that it expected to report a 36 percent drop in quarterly home-building revenue , while net orders for new homes fell more steeply than in prior quarters , reflecting last month 's deepening decline in the U.S. housing market . -The six , who call themselves " Ztohoven , " claim that the aim of the project - called Media Reality - was to show how reality can be manipulated by the media . -Stupak had pledged to oppose the health care package unless given greater assurance that it would not allow federal funding of elective abortions . -Sheik Salman al-Feraiji , al-Sadr 's chief representative in Sadr City , issued a statement with demands to quell the discontent , including the release of Sadrist detainees , an end to military operations against them and al-Maliki 's resignation . -Cowles declined to comment to ABC News for this story . -In fairness , the Scots finished the half more adeptly than they 'd started and McFadden was unlucky not to win a penalty when he was obstructed in the box . -Results from dozens of clinical studies will be released at the gathering opening Friday and running through June 2 . -There , 1 per cent of the population owns 49 per cent of the land ; here , 0.3 per cent owns 69 per cent . -" One wall is enough , " he said . -" Next question , " said Fisher with a smile . -He seems genuinely affected and I almost suggest we meet again , informally as friends , but retrieve the last scraps of my dignity . -Prosecutors said Plotkin ran the schemes , enlisting David Pajcin , a former Goldman Sachs Group Inc. analyst who has pleaded guilty to charges in the case and is cooperating with the government . -In anticipation of prolonged outages , Rockford city officials coordinated assistance to provide access to warming centers for residents . -From TPP-- that 's how i see it. it 's a resource . -In the nearby village of Tepalcatepec , concrete homes built with dollars sent home by migrants stand next to tin-roof shacks . -They do not listen , or even know how to listen , they are operating on " eternal truths " that are not open to debate . -Most vacationers visit America 's national parks during the summer months--when the roads are crowded , restaurants are packed and prices are at their peak . -Good , there you go ! -" In World War II , we knew who the enemy was and we fought with them . -To some this may seem puzzling , given that dishonour is spread fairly evenly between both of the main parties . -3 ( UPI ) -- An Arizona elementary school principal placed on leave for writing a letter jokingly calling children " stupid " has apologized to parents , officials said . -So embrace the idea of big . -Reclusive North Korean leader Kim Jong Il isn 't always easy to explain . -Items were stolen during the incident . -Santiago tried to stop the fight after the 11th round , but Cotto went back out to take even more punishment before a final flurry along the ropes prompted referee Kenny Bayless to end it . -Knecht and Benner presented the fossil last month in Krakow , Poland , during the Second International Congress on Ichnology . -John McMahon , manager of the Torquay Tigers Football Club , said the pictures of Klan members were posted on the Internet by a volunteer , a retired member of the club , without the knowledge of officials , The Age reported Thursday . -The sub-Billy Liar / Elliot one , where the gruff father is disgusted by his son 's airy-fairy wholemeal ambitions , is particularly good . -Meanwhile , many Algerians say they have not felt the effects of the country 's new wealth . -A secret review of Fletcher 's death concluded two years ago that there was enough evidence to charge two Libyans , The Daily Telegraph reported earlier this month . -Earlier , state television said the blast happened in central Iran , without giving details . -Coach K and North Carolina 's Roy Williams are busy arguing over injury reports , so we know they didn 't hear Capel on Tobacco Road . -Lance , Inc. has scheduled a conference call and presentation with investors at 9 : 00 am eastern time on Friday , July 24 , 2009 to discuss financial results . To participate in the conference call , the dial-in number is ( 800 ) 789-3681 for U.S. callers or ( 702 ) 696-4943 for international callers . A continuous telephone replay of the call will be available beginning at 12 : 00 pm on July 24th and running through July 31st at midnight . The replay telephone number is ( 800 ) 642-1687 for U.S. callers or ( 706 ) 645-9291 for international callers . The replay access code is 17972828 . Investors may also access a web-based replay of the conference call at Lance 's website http : / / ir.lance.com. -We all have an interest in the success of these institutions . -Translated , that means debt for each household grew by $ 582 in the last six months , compared with $ 418 in the previous six . -It turns out that woman have sex for all of these reasons and more , and that their choices are not arbitrary ; there may be evolutionary explanations at work . -For months , the panel had been investigating the Democratic senator 's appointment and seating . -Because most blacks today enjoy improved education and quality of life , " dealing with the African-American community is the same as dealing with any other community , " said Dick Harpootlian , a white Obama supporter and former chairman of the South Carolina Democratic Party . -Ditto her bum-grazing cut-offs . -Though the cards haven 't prompted much opposition in the Bay Area , the matter of government-issued identification cards for illegal immigrants is controversial nationwide . -" October delivers some of the worst driving conditions of the year as the weather changes and the clocks go back but drivers continue as if they were in the summer months . -At the beginning of my day with the G20 protesters , I had every hope that my presence as an independent observer on behalf of Parliament would be redundant . -SocGen issues its first public statement outlining ' an exceptional fraud ' at the bank . -Rose has pledged to split his controversial double role as chief executive and chairman by 2011 , although he will come under pressure from shareholders at next week 's annual meeting to bring this forward by a year . -It is very tempting , but I don 't want to do so knowing that I won 't see him for weeks . -A major earthquake -- of magnitude 7 or higher -- is capable of causing widespread and heavy damage . -During his party conference Mr Clegg called for " savage cuts . " -" This is a very distressing time for the family , and our thoughts are with them at this time , " said Det Sgt Emma Hampson . -Two independent forecasters who did foresee the return of Arctic weather , and are predicting that there will be more , are Britain 's Piers Corbyn ( who relies on solar activity for much of his prediction ) and the US meteorologist Joe Bastardi of AccuWeather , who largely bases his forecasts on movements of air and ocean currents . -Saint Anthony Medical Center in Crown Point will implement the program March 9 ; Saint Anthony Memorial Hospital in Michigan City , March 23 . -Because we have the best team in the world , " said the 69-year-old Aragones , who will resign as Spain 's most successful coach despite repeated pleas from his players Monday to remain . -The pundits and analysts can talk themselves silly all day Sunday . -Cobham is developing only its fourth generation of refuelling equipment since 1934 . -According to the Ministry of Justice the current cost of the alternative - a by election should an MP stand down - is £ 70,000 - £ 80,000 , depending on the size of the constituency . -PINZOLO , Italy ( AP ) -- David Trezeguet is quitting international football after the confirmation of the reappointment of Raymond Domenech as national coach . -Later in the afternoon , the first couple returned to the base from their Kailua rental home to attend a midday Christmas Day meal at its Anderson Hall chow hall , the newspaper said . -There is intense irritation with Mr King in Downing Street , where he was regarded as slow to react to the banking crisis and naive in giving briefings to the Tories that were later leaked . -But first , this part is about creating a sense of shared experiences . -Nowitzki , who is averaging 23.6 points and 8.7 rebounds per game , will sit out Thursday 's home game against the Houston Rockets . -The warning was later lifted after a 4-inch tsunami rolled ashore . -The defense argued Mineo exaggerated the encounter and made up the assault to get a bigger payday from the city . -From 2016 all new homes will have to be zero-carbon and rental properties may have to have Energy Performance Certificates . -Housing Executive head of research Joe Frey said indications that the market was stabilising were welcome , but he warned it was " important to be cautious at this stage . " -That was the unappealing choice facing UBS , a Swiss bank which has been badly hurt by the carnage in America 's mortgage market . -Juergen Weckherlin , a German businessman in Hong Kong , said people were losing money because they cannot make trips that require face-to-face contact . -29 ( UPI ) -- A spokesman for Hamas in Syria said Tuesday leaders were still discussing an Israeli offer to release hundreds of prisoners for captured soldier Gilad Shalit . -Thirty years after a paedophilia case forced him to flee America , the film director Roman Polanski has filed a request in Los Angeles to have charges against him dismissed . -Tanzanite is believed to be limited to east Africa 's Rift Valley region and the pits where the accident happened are located in the heart of Maasai land , a short distance from Mount Kilimanjaro . -Ray Barrell , a research fellow at the NIESR , said that the Chancellor was wrongly assuming that revenues from the housing and financial services sectors would quickly return to levels seen in the boom years - while the spending cuts and tax rises announced would not be enough to restore the government budget to balance by Mr Darling 's target date of 2017-18 . -This breezy notion brooked no caveat that the climate might have changed greatly by 2050 , and we will possibly not be offering even the lukewarm welcome to migrants in the UK that we do today . -Pinch hitter Colin Walsh hit an RBI single and pinch hitter Ben Clowe followed with a three-run homer to left . -Most of the 350 crowd had a whale of a time , even the ones whose nappies were being changed on the specially provided benches . -" We learned from the Ronald Reagan years how generational support for a candidate can ripple through the demographics for years to come , " said one leading GOP strategist close to the McCain campaign . -Selection standards are extremely high -- all recruits must be unmarried Roman Catholic males between the ages of 19 and 30 who are able to endure grueling 24-hour shifts . -France was forced to import UK electricity earlier this year when low river water caused 14 of its nuclear power stations to grind to a halt . -It might be argued that if the market price is only an average , it should be open to anyone with better than average information to beat the market . -And to date there have been no reports of any British citizens caught up in the violence . -The important thing is the presence of light and the atmospheric conditions which affect light . -( CNN ) -- French Open champion Svetlana Kuznetsova crashed to defeat in her opening Wimbledon warm-up encounter at the AEGON International in Eastbourne on Tuesday . -Elsewhere , Dallas and Denver were beaten for the first time , while Buffalo and Tennessee remained spotless . -Libi , once an al-Qaeda spokesman , is believed to have died during an air attack in Pakistan in late January . -The Lakes Line Community Rail Partnership works alongside First TransPennine Express on the service between Oxenholme and Windermere . -14 beachfront wedding in New Jersey after visiting him in jail soon after his arrest . -Mr. Cameron calls his wife , Samantha , his " secret weapon , " and she is turning out to be just that , with an increasingly prominent role in the campaign . -More and more this is shaping up as the heavyweight bout everyone ( outside of Tuscaloosa , State College , Norman , Gainesville , etc . ) is dying to see . -Body odours also change perceptibly according to mood . -The teams will be accompanied by USAID disaster experts who will assist with assessments of the situation , the agency said . -Regulators have been encouraging banks to change the way they award bonuses to help avoid excessive risk-taking after the financial crisis . -A UCLA sports information official said Morgan would rejoin the team in Tempe for Saturday 's Arizona State game . -What he brings to " She 's Out of My League , " in addition to the geek and the gawk , is a dash of the debonair , which might seem impossible and yet he does . -Memorable moment In 2008 , Gwyneth Paltrow started an A-list stampede for his designs when she returned to the red carpet , post baby break , wearing a sexy , lace-panelled shift . -U.S. Border Patrol Chief David V. Aguilar released the data to the Senate Judiciary subcommittee on immigration , refugees and border security , noting that the number of Border Patrol agents has more than doubled from 9,000 in 2001 to a projected 20,000 by September . -Just three weeks ago , the Jaguars mauled the Steelers at their own game on a snowy day at Heinz Field , delivering to the Steelers their first loss at home this season . -Both methods are complex and rely on intricate genetic manipulation because the mammoth DNA is not suitable for cloning . -Overall , Brown led Poizner 53 to 22 % . -While other villages usually harpoon their quarry out at sea , the particularly bloody killing methods in Taiji have made the town a focal point of animal rights activists worldwide . -According to the American Cancer Society , 1,500 Americans die each day of some form of the disease . -Eriksson helped Sweden win a bronze medal at this year 's world championships and is a candidate for the 2010 Vancouver Olympic team . -In the past several years , many investors who put their money into emerging markets enjoyed annual returns of more than 30 percent , attracting capital from Japanese housewives and American pensioners . -He was convicted of a string of sex offences , including multiple rapes , against two young girls . -RYAN : And there are in every state . -Originally aired on the internet during the writers ' strike , " Dr Horrible " became a sensation , and went on to win seven Streamy awards ( given for web television ) including the audience choice for best web series . -On the eve of last week 's hearing , Kercher 's sister Stephanie read out a statement in a steady voice at a Perugia hotel to say the family was " pleased that we have reached a new phase in the process , hoping that justice will soon be done for Meredith . " -Neither player has yet been convincing in the first team . -At its best , even in the midst of all this , there is laughter too , often centred on my joker son . -( AP ) - Colt McCoy completed his first 11 passes and produced touchdowns on No. 3 Texas ' first three drives , helping to put Missouri away early in a 41-7 victory on Saturday night . -The gender wars are important but take a back seat to survival . -Amphibians lay jelly-coated eggs that are unsuited to development on land , so they must return to water in order to spawn . -NEW YORK ( AP ) - An undercover cop chased a Times Square scam artist through sidewalks crowded with holiday shoppers and tourists Thursday , exchanged gunfire with the suspect and killed him near a landmark hotel , police said . -Although industrial policy is often equated with protectionism , Ford and other speakers said the US needs to be tougher with trading partners to maintain prosperity . -Hold down the power button for ~ 10 seconds . -JAKARTA , Indonesia ( AP ) - Economic growth in Southeast Asia 's largest economy is expected to hold steady at just over 6 percent in 2009 , Indonesian 's president said Friday in his State of the Union address . -The Spaniard was dumped out of Wimbledon on Tuesday in a 6-0 , 6-0 , 6-4 drubbing at the hands of Leonardo Mayer , the Argentina player . -Endeavour 's internet home page at http : / / www.endeavourcorp.com. -He made his professional operatic debut at Glyndebourne in Richard Strauss 's Capriccio in 1964 . -So far these deals -- worth a total of nearly $ 10bn ( £ 6bn ) according to Swiss Re -- have been tailored to the requirements of the insurer or pension fund looking to offload the risk . -Buddy Piszel , Freddie Mac 's chief financial officer , said the regulator 's move would allow it to buy mortgage securities at attractive prices . -That was a key reason the unemployment rate fell so far in December . -" I want those who would use terror against British citizens to know beyond doubt that we and our allies will pursue them relentlessly , and that they will meet the justice they deserve , " Brown said in a statement . -Justice demands no less . -But another 158,000 homeowners who signed up have dropped out--either because they didn 't make payments or failed to return the necessary documents . -John Meyer of Fairfax said disruption at the Grasberg mine could reduce global copper concentrate supply by around 5 per cent and that combined problems at these mines / smelters could cut more than 10 per cent , around 36,000 tonnes of copper production a week . -Within a few weeks , Mr. O 'Brien had assembled a group of engineers , architects and managers . -Extra police and street pastors are being brought in to patrol the streets of Devon and Cornwall later . -Phelps R. Hope , vice president of meetings and expositions for Kellen Meetings , said some of his clients had opted to meet in the United States instead of taking their events abroad . -The president pushed Congress to pass legislation to more tightly regulate Fannie Mae and Freddie Mac , the government-sponsored companies that finance home loans . -A sharp rise in gasoline prices will help drive U.S. consumer prices 0.3 percent higher in October but core inflation , which excludes food and energy , should be more moderate , according to a Reuters poll of economists . -25 -- The leaders of the world 's 20 leading economies have agreed to restrict bankers ' pay but not to impose hard limits , according to the latest draft of a joint communique to be issued late Friday afternoon . -Janet Hoffman , global managing director of retail for Accenture , dubbed it a " shift to thrift " that spanned all income levels . -Each requires the written support of 12 to 15 other MPs , at least three of whom must come from a different party from their own . -Megan is more Baby Jane Holzer than Jane Eyre ; she 's a lovely , refined Yale graduate who was raised in nearby Fort Lauderdale but seems quite at ease in high society . -The number includes almost 90,000 Eastern Europeans in the last year alone , although separate figures show a downward trend in recent months . -Overall , only about one in 10 voters said race was an important factor in deciding whom to pick -- and a majority of them voted for Obama . -Charlie Sheen shares custody of two young daughters with ex-wife Denise Richards and also has an older daughter with a former girlfriend . -Tomorrow , when it 's all over , we will settle in for a long winter . -Gore , who won a Nobel Peace Prize for his work on climate change , joined the foreign ministers of Norway and Denmark in presenting two new reports on melting Arctic ice . -Violence has been a major concern as tensions rise in Lebanon , where Shiites , Sunnis and Christians each make up about a third of the country 's 4 million people . -SEOUL , South Korea ( AP ) - North Korea 's military threatened to use " merciless physical force " in response to annual military drills by the United States and South Korea that begin Monday , further raising tension on the Korean peninsula . -Hundreds of MPs , including the three main party leaders , have already agreed to repay sums totalling thousands of pounds . -The Moon is only the latest . -The Fatah supporters pelted Hamas troops with stones , surging forward even as they were met by heavy bursts of gunfire . -Ties have improved dramatically since Beijing-friendly Ma Ying-jeou became Taiwan 's president last year . -Commodities prices were mixed , while the dollar fell slightly against other major currencies . -BARBOURSVILLE , W.Va. -" It would have been one boat or the other . " -Liverpool avoided a potential Europa League embarrassment in Romania as they came from behind against Unirea Urziceni to book their place in the last 16 . -The military denied having fired at anybody in that area , however , and local Palestinians said the boy had been missing since Monday , raising questions about whether he could have been the victim of internal violence . -Presented by Numark and Beatport , Remix Hotel Miami will also feature master classes in Ableton Live , Apple Logic and Propellerhead Reason among others . -" They are not foolproof . " -Running UBS 's and Credit Suisse 's wealth , investment banking and domestic banking businesses as autonomous units with independent capital and risk-taking frameworks -- rather than breaking up the banks outright -- is a better solution . -Colin Grahamslaw , from the Royal Club , said the decision would help secure the sport 's future . -Australia have a big problem here , a big 16 stone problem who is doing pretty much as he pleases . -Energy and railroads were among the worst performers , as was Staples . -OSLO ( Reuters ) - Norway could become " Europe 's battery " by developing huge sea-based wind parks costing up to $ 44 billion by 2025 , Norway 's Oil and Energy Minister said on Monday . -Still , iPhone sales pale compared with those of established mobile phone makers , such as Nokia Oy , which sells almost 10 million phones each week , or Samsung Electronics Co Ltd and LG Electronics Inc , which each ship more than 100 million phones a year . -Robinson paid his way to New York , then at 6 a.m. -The protests were against delays caused by a baggage handlers ' strike and a walk-out by ticket counter workers . -When he arrived in New York in July 1824 , Lafayette was cheered by 50,000 people on a parade up Broadway to City Hall . -Dealers are limited to five offers per issue , and the minimum offer size is $ 1 million , with a minimum increment of $ 1 million . -The first game that Brosnan lent his voice to . -It doesn 't make any sense , " he said Wednesday outside his firm in nearby Quakertown . -" The Spy Next Door " operates on familial bonding by way of humiliating pranks , which in better circumstances might go by the name of slapstick . -" I chose Quadrangle to manage my investments because of the quality of the team and the firm 's track record of delivering value , and those are the reasons I intend to maintain this partnership going forward , " Mr. Bloomberg said in a statement . -And there is a good book , Living with Bipolar Disorder , by Dr Neel Burton ( Sheldon Press ) . -An investigation was underway to determine the cause of the crash . -5 ( UPI ) -- The Chicago Cubs have fired hitting coach Von Joshua four months after he replaced Gerald Perry , team officials said . -Give it back to the Indians . -Kooley High : " Everywhere Is Summer " The North Carolina-based hip-hop nostalgists turn a remixed version of Common 's " Everywhere " into a soulful , summery reminder of the mid- ' 90s , when jazz-influenced hip-hop outfits like A Tribe Called Quest roamed the Earth . -Its two Paris events this month are likely to produce lower revenues than last year . -PARIS ( Reuters ) - The body of the pilot of an Air France plane that crashed into the Atlantic on June 1 has been identified among dozens that have been recovered from the ocean , the airline said Thursday . -A ladder usually costs $ 1,000 to $ 2,000 , and it takes 8 to 10 weeks to make . -He gave us medical care . -For the period , Lowe 's earned $ 938 million , or 64 cents per share , in the three months ended Aug . -They never planned for the aftermath . -McLean , Va.-based Freddie Mac neither admitted nor denied wrongdoing under the accord with the Securities and Exchange Commission announced Thursday , but it agreed to refrain from future violations of securities laws . -But today President Obama told allies and enemies that world skepticism and distrust of America has been a convenient cop-out . -I don 't think the basic flight itself cost more than 20 per cent of the total -- the rest was fees , taxes and surcharges . -The results of his study appear to contradict hospital guidelines from the Centers for Disease Control and Prevention that say better hand hygiene -- through frequent washing or use of hand gels -- has been shown to cut the spread of hospital infections . -" Nobody put the brakes on . -Western observers have criticized Sunday 's election as not fully democratic but Putin said the vote was held in strict accordance with the constitution . -Han Sang-choon , deputy head of Mirae Asset Investment Education Institute , said investors ignored warnings over the past few months that trouble lay ahead for stocks and held on to funds thinking a big pay day was just around the corner . -Be it the evil or the commies or imaginary terrorist , no enemy = no GOP. because she would be harder to beat than Obama . -Scholarship America is currently piloting the use of Zinch.com with one of its community-based scholarship foundations ( called Dollars for Scholars chapters ) . -That was because an hour of weight lifting and 90 minutes of film work awaited him . -Pester power is keeping the movie industry afloat . -Elizabeth King , associate director at the Securities and Exchange Commission said she was " very supportive " of increased transparency . -She said she 's worried about the highway 's impact on her 16-year-old son , who had severe asthma when he was younger , and on several neighbors with asthma , including one who sometimes relies on an oxygen tank . -All that insistent , throbbing bass rhythm -- what can he possibly see in it ? -Developed countries are preparing to relent on their demand that developing countries agree to long-term cuts in greenhouse gas emissions in a concession that could form the basis of a global deal on climate change . -As part of a six-nation disarmament deal , North Korea is disabling its Yongbyon nuclear reactor and related nuclear facilities , and is obligated to provide a " complete and correct " declaration of all their nuclear programs , weapons , and materiel by Dec . -Many people have stayed at home rather than making the usual Christmas journeys . -The admissions process is daunting . -Millions are going to catch STIs as they journey through life . -Meteorologists are forecasting wind gusts of up to 126 kilometres an hour . -The old conception of individualism needed to be replaced by a new system in which the citizen " married his interests to the state , " in Wilson 's words . -I despise them with a passion . -The fact the Christmas post has been secured is a relief to Royal Mail , which doubled annual profits to £ 321m last year but has been haemorrhaging customers . -A version appeared on p35 of the Comment & debate section of the Guardian on Friday 27 November 2009 . -That combined with the weather could keep Schaub 's numbers closer to earth this week . -He 'd called her saying he needed to talk . -Kapalua , owned by Maui Land & Pineapple Co . , sponsored the event 's inaugural event last year , which was won by its touring pro , Morgan Pressel . -That is the same tumor-nourishing protein blocked by Avastin , the blockbuster treatment for colon and lung cancer sold by Genentech and Roche Holding . -Despite the ordeal , Mannie says he 's just happy to be alive . -Adada said he hoped 80 percent of the force would deploy by year end . -But independent observers believe the bulk of the organisation has remained united under the clandestine leadership of Hafiz Saeed . -Sales gained 37 % to $ 1.29 billion , in line with analyst estimates . -I 'm not an Obama supporter , and I certainly don 't like the spendulous bill and the bail-outs . -Although the company built its reputation catering to tech geeks , its product lineup now goes well beyond computers to include such items as digital cameras , cookware and electric shavers . -Kari Keaton is the sort of customer most businesses used to hate . -Johnny Carson , who reigned over late-night TV for 30 years , was chosen as the No. 1 TV icon . -California State University , Stanislaus president said a portion of the contract between a school foundation and Sarah Palin for an upcoming speech was taken from a recycling bin inside the office of Susana Gajic-Bruyea , vice president for university advancement . -First of all , you make a point about Medicare that 's very important . -Roethlisberger was due a $ 2.95 million bonus this month that , if paid , would have put the Steelers over the salary cap . -And then I decide . -He won a second five-year term in July with promises of economic growth , political reform and good governance . -He remembers , at 5 , seeing " The Blob " on TV and thinking , hey -- that 's the theater where I watch Disney movies . -Anti-NATO demonstrators have vowed to disrupt the summit and riot police clashed with hundreds of protesters Thursday in Strasbourg in France , repeatedly firing tear gas and rubber bullets and arresting around 200 youths . -" Where it 's hard we can 't recruit and we can 't retain " Afghan forces . -And Carey feels Friday 's game will give his side a chance to bounce back . -Should Ellis win that particular contest , what started as a difficult season will end as a golden one . -Peter Stalkus , a fellow maritime captain who has worked with him for 23 years , said Phillips is known as " the Larry Bird of Chief Mates , " according to the Boston Globe . -P : Right , that 's what it 's all about , the middle class and government needing to work for them . -Gerber Products Co. and the Food and Drug Administration have found no evidence of tampering with Gerber products . -Who knew that following politics could prepare you so well for this stuff ! -Glendale Mayor Frank Quintero also honored Lakeside for Lakeside 's generous sponsorship of Glendale 's 21st Annual Great American Clean-Up . -These White House sessions are designed to put the presidency above politics , temporarily at least . -He exaggerated , but did so because of the great fear in France that the game is up for their currency . -She said other bodies had already been buried due to advanced decomposition . -Every day can be exhausting and you 're spending all day long with children . -The assets it is selling to CVC account for less than 3 per cent of the brewer 's total underlying earnings and are a collection of comparatively small brewing businesses . -But Mr Brown will tell John Bercow , the Commons Speaker , in a special evidence session as part of the drive to modernise of Parliament , that homosexual MPs and peers should have the same privilege . -Gordon Brown last week announced plans to allow local authorities to give priority to local people , in what was seen as an attempt to head off BNP claims . -But then rain swept in again and turned the pistes mushy , forcing the postponement of the blue riband event , the men 's downhill , until tomorrow. the women 's super-combined , which had been due to start today , has been put back to Friday . -The settlement will be drawn from a government-financed insurance fund . A lawyer representing more than nine thousand workers said he was pleased that men and women he described as heroic would finally receive just compensation . -Tensions in the region are already high . -" We really donʼt know where it came from , so I guess you could consider any possibility , " said Clayton Police Chief Thomas Byrne . -The pile-up occurred shortly before 11 p.m. -Acute mountain sickness affects people at high altitudes . -He said the findings " bridge a critical knowledge gap for stem cell biologists , enabling them to better understand the enormously complex process by which DNA is repackaged during differentiation " -- when embryonic stem cells adopt specialized functions . -Again , City knew about the massive collapse coming , ignored it , raped the taxpayers for billions , lined their pockets , paid outrageous bonuses , and laughed like all hell . -His trenchant analysis was supported , naturally , by statistics and citations , but there were a couple of places where the book moved into warmly personal focus . -At New Year 's Eve at the Sandy Lane hotel , when Green chose the 12-course " degustation " menu , Cowell went to the opposite extreme and had chicken pot pie with mash . -" I 'm even more excited because I think we can take a major step defensively . -Previous research has shown that these pride and shame gestures are widely recognized around the world in many different cultures . -Urban dystopias . -The kids had been on the roof for a few hours when PMQs began . -Selected by the Packers in the fifth round of the 1954 draft , McGee spent two years in the Air Force as a pilot following his rookie year before returning in 1957 to play 11 more seasons . -The MoD was also concerned that MoD police could not ensure the safety of protesters from attacks by local residents , which had happened in the past , he said . -Partly it 's that we 're both from Pittsburgh , and both began our careers as paperboys for the Pittsburgh Press . -Higher up , lush rainforest half-hides retiring blue monkeys that swing through the lianas , until eventually we emerge on to a plateau of sweeping grassland . -The Sayre Fire destroyed 510 structures , including those at the Oakridge Mobile Home Park , the Times reported . -SEOUL ( Reuters ) - North Korea is making final preparations for a rocket launch the United States said could come as early as Saturday , pushing ahead with a plan widely seen as a disguised long-range missile test . -In comparison , his successor , Maria , a teaching assistant from Poland , seemed positively staid . -While he didn 't have exact figures , he said roughly 60 passengers were killed and more than 20 people injured were taken to local hospitals , he said . -They have asked Highland councillors to approve an allocation of £ 650,000 towards the work . -On June 17th , Mr Obama announced that he will extend some ( although not full ) benefits to the same-sex partners of federal employees . -OTTAWA , April 15 ( UPI ) -- Male nurses in Canadian hospitals and nursing homes in 2005 were much more likely to be physically assaulted by a patient than female nurses , officials said . -Robert and Tonya Harris matched all five numbers plus the Mega Ball in Friday night 's drawing , the only such ticket in the multistate game . -Paste-on hymens . -It is ENTIRELY Obama 's fault that illegal workers still routinely penetrated thousands of miles into our territory . -Four days after the bombing attempt , Obama emerged from his vacation seclusion to cite what he called " human and systemic failures " in U.S. intelligence leading up to the incident . -White House press secretary Robert Gibbs said Obama would urge lawmakers to forgo part of their August recess to continue working on health care legislation . -Liukin is the reigning Olympic champion , joining Mary Lou Retton and Carly Patterson as the only American women to win the all-around title . -The comparisons don 't end there . -To help make this year 's Labor Day holiday period a safe one , AAA Michigan will once again partner with the Michigan State Police ( MSP ) in supporting Operation C.A.R.E. -Gotham Bowl officials had high hopes for the 1962 game . -Meanwhile , the volatility in financial and housing markets is increasing the riskiness of the assets that they already hold - so that is also forcing them to cut back on lending to maintain even plausibly healthy balance sheets . -A wreath with 36 roses marks Jerry Zovko 's birthday . -The case was appealed this year to Iran 's Supreme Court . -Human rights groups condemned the event as provocative , while pro-Tibet activists argued that the leg should have been cancelled in the wake of the unrest there in March . -The inclusion of Snoop Dogg in the line-up is likely to cause controversy . -The americans are a different breed altogether and invite people with talent and zeal to come and work , the Europeans seem to dictate terms and conditions to the talented folks ! -" I don 't feel pressure , " he said . -A third House committee , Energy and Commerce , also was considering the measure today , but the road was expected to be rougher there . -" I bent down slightly under the counter , and he started panicking and pistol-whipped me a couple of times . -" The people of New Orleans needed me , " she said simply . -Now the self-styled political junkie and former Mississippi state legislator has written a book that 's more political intrigue than legal thriller . -The sex education lesson for 15-year-olds is not part of India 's national syllabus , and the exclusive private school teaching the subject is taking a risk by passing on the facts of life to its pupils . -Sri Lankan President Mahinda Rajapaksa , left , has his strongest support from the Sinhalese south . -Stir and continue adding the stock until the rice is cooked , about 20 minutes . · Remove from the heat and stir in the remaining herbs , butter and parmesan . · Preheat the oven to 150C . · Wash the pears . -Notes : @ Cleveland scored six runs in the fifth for a second straight night . ... -It was initially thought 30 pupils at the Islamic Birchfield Independent School for Girls had TB but tests have shown the figure is lower . -He said that Todd Palin submitted " objections " to the subpoena that needed to be answered before he would take the stand . -In that case , you are entitled to what you contributed , but the money your employer put up will be taken away regardless of whether you do a direct rollover or you keep it in the company plan , says Doug Flynn of Flynn Zito Capital Management . -He is handed a four-match touchline ban . -Read previous Green Lantern columns here . -Government forces struck a suspected al-Qaida training camp in the region in December , killing more than 30 in the raid . -When this fish actually makes it to plate , will sushi-lovers in Hong Kong and Tokyo know they are sitting down to a meal worthy of the kind of elite endangered dining club featured in the 1990 film The Freshman ? -" I believe this tax is not helpful at this time , " said Stefano Pessina , the executive chairman of Alliance Boots . -The innuendo-filled radio adverts for Mattesons sausages asked listeners where they would like to " stick it . " -We want permanent tax relief ! -In the opening 20 minutes at Eastlands , City appeared crude by comparison . -Beth Ehlers plays an Army lieutenant on " All My Children . " -In 1990 at the peak of his tenure , Unilever sold £ 23 billion of cosmetics , foods , detergents and oils . -Cordelle said he had received a letter terminating his services last month , after more than 12 years at SocGen . -As home prices come back down to earth , many of these borrowers will find themselves with negative equity -- owing more than their houses are worth . -Armstrong said tattoo removal is no fun , with the repeated laser treatments being described as being smacked over and over with a rubber band , or like hot grease against the skin . -This may help the pragmatic conservatives . -Thirteen other people were wounded . -One student , Marta , had a clear advantage , she was Portuguese . -Day two Which isn 't to say that aesthetic limitations can 't be singled out . -11 attacks has paid $ 6 billion to 2,880 families of those who died in the attacks and more than $ 1 billion to 2,680 injured victims . -( What I 'm trying to tell you , between the lines , Jay , is that Sarah 's kind of corrupt , and no stranger to cashing in on her situation long before John McCain had ever heard of her .... assuming he ever has . -The pay-TV group BSkyB has been ordered to reduce its stake in rival ITV by a UK court . -Last year , Pioneer announced a capital and operations tie-up with another electronics maker , Sharp Corp. The alliance allows Pioneer to procure Sharp 's liquid crystal displays . -Instead , they changed course . -Five Western states -- Oregon , California , Washington , Idaho and Montana -- pocketed more than 80 percent of the total timber payments from 2000 to 2007 . -For those aged 5 to 9 , rates of death fell 27 percent to 14.5 deaths per 100,000 in 2005 . -A penthouse in a Birmingham skyscraper is the first flat in the city to sell for £ 1 million , say developers . -Iraq is assuming responsibility for almost all the funding of large-scale reconstruction projects , and our share of security costs is dropping as well . -The supplies were being distributed Wednesday . -Do you know why one of James Bulger 's killers has been returned to custody ? -The tallest woman picks the shortest man , and the shortest woman picks the tallest man . -The swine flu pandemic sweeping the world might not have happened without a laboratory accident in the 1970s , a new study claims . -China 's exports and imports declined in May , but this was offset by better-than-expected growth in urban fixed-asset investment - indicating the world 's third-largest economy is well on the road to recovery . -This season Coach Ben Howland has waved that white flag and gone zone . -" After Simon , I can handle Jason , " Sinitta says . -He met Burlington in Italy , and their passion for classical forms can be seen at Chiswick in a grand processional avenue lined with stone urns and sphinxes . -In New York , critics referenced a pair of big Johns , Cheever and Updike , in their rave reviews , while here the play seemed a more jagged and spelt-out version of Pinter 's dreamy Old Times , with the triangle rearranged against the distaff side . -But the overall business climate is starting to change , as investors eye Africa 's growth potential , and that could lead to more options for expatriates wanting to return home , Patel from HSBC said . -During that time , David Cameron and William Hague have repeatedly said that the undertakings were being met . -" Unpredictable " seems to be the word used most often by experts to describe the outbreak of swine flu , writes Clive Cookson . -Mercado has dodged more eliminations than any finalist in " Idol " history , so she 's no stranger to danger . -And now you know how silly , plastic-surgery addicted Jacko came to his end . -" Golf is all about performing in the majors but I 'm not going to this major and thinking it has to happen , " Harrington said of his bid for a rare third major win in a row . -If you walk down Karl Johans Gate , the main drag of central Oslo , a tree-lined promenade bordered by restaurants , cafes and upscale stores , you 'll eventually find yourself face-to-face with the Royal Palace , the mammoth , cream-colored home of the Norwegian royal family . -The title search did not find the lien . -As for Ucas points , these are used mainly as a general guide to applicants . -The 12-time All-Star was appearing in his first game since admitting he took performance-enhancing drugs while playing for the Texas Rangers from 2001 to 2003 . -The company added that it was slashing staff in its North American development division from 70 people to 10 . -Fox , 8 p.m. -Schools are legally required to arrange full-time , suitable education for pupils excluded for six days or more . -The MNF said the three were thought to be involved in the construction and distribution of IEDs in the Baghdad region . -Stay in school . -He said there was no case to answer following defence submissions at the end of two months of prosecution evidence . -The island is home to the Chincoteague National Wildlife Refuge , the Assateague Island National Seashore and Assateague State Park . -He would love to get his hands on Sea The Stars . -Casa Ferreirinha 's Fernando Nicolau de Almeida created Barca Velha in 1952 , using the port grapes to make the region 's first high-quality table wine and in five decades it has only been made in 15 vintages , the most recent being the 2000 . -Foxx , who is best-known for his performances in " Ray " and " Dreamgirls , " will play Prentice Earl Sanders , one of two black detectives determined to crack a series of racially motivated serial killings in 1973-74 San Francisco , the trade paper reported . -At that point , descending hills or time spent idling will result in a fall in the temperature gauge reading . -Police raided the home of Robert Savage , at Ballycarry Street in Belfast in July 2007 , following a tip-off . -Still , print and radio news organizations need not pack up their computers . -The group , without any central direction , constructs temporary facilities every year . -That , say local brokers , could encourage people to go down the private health care route because of fears that standards in the public sector could decline . -The company has presented income tax benefit ( expense ) after adjusting for the tax effect of unusual items including charges related to restructuring , impairment and other income or charges ( " Adjusted Income Tax Benefit ( Expense ) " ) . The company presents Adjusted Income Tax Benefit ( Expense ) to provide investors further information regarding the tax effects of adjustments used in determining the non-GAAP financial measure Adjusted ( Loss ) Income from Continuing Operations ( as defined below ) . The difference between Adjusted Income Tax Benefit ( Expense ) and income tax benefit ( expense ) ( the most comparable GAAP measure ) is the tax effect of adjusting items . The limitation of this measure is that it excludes the financial impact of items that would otherwise either increase or decrease income tax benefit ( expense ) . -For example , obtaining an ex-directory telephone number cost £ 75 , while a car number-plate check to match it to an address cost £ 150 . -Terra is implementing an Interactive Call Center solution in many of the locations , and other upgrades like the Aruba Wireless Network which is FIPS140-2 compliant for their federal secured wireless network . -The restaurant serves it over crisp crostini topped with guanciale to make a perfect appetizer . -Social conservatives have criticised Labour 's record on the family , claiming that the benefits system creates a financial incentive for parents to live apart . -It is boosting sales of alcohol by accepting credit cards instead of cash . -Could I risk it ? -Her husband was a cabinet installer at Quality Woodworks but was not suspected of any crime , Varnau said . -CANYON DE CHELLY NATIONAL MONUMENT , Arizona ( AP ) -- This majestic canyon is lined with towering red sandstones . -On Tuesday , he testified at a Congressional oversight hearing on the use of performance-enhancing drugs in baseball . -Mandating write-downs in home equity loans would be a particularly bad idea , he said , because these loans were simply used to consume rather than pay for housing . -Toyota 's plug-in travels 14.5 miles as an electric vehicle on a single charge , and gets 35.42 miles a liter in mileage , the equivalent of about 135 miles a gallon . -According to the petition , it 's critical for ice floes used for pupping to remain stable until pups are independent . -On Thursday , Cairn said that it had struck a $ 310 million ( £ 190 million ) deal with Petronas , of Malaysia , to farm out a 10 per cent block off the coast of Greenland . -He was the third of the four children ( two boys and two girls ) of Siegfried Wagner and his British wife Winifred Williams , whom Siegfried ( Richard 's only son ) had married in 1915 when he was 46 and she was 18 . -It takes you from 7000 B.C. to A.D. 500 on a beautifully displayed and well-described sweep through Greek history , from prehistoric and Mycenaean artifacts through the evolution of classical Greek statuary . -The campaign : Obama was the only candidate visiting Nebraska . -Moyes 's contract expires at the end of next season , but while he has been in talks with Bill Kenwright , the Everton chairman , since the turn of the year about a lucrative new deal , an agreement has not been reached . -His reply was swift , when moments later he broke free inside the Falkirk penalty area and smashed home a high , right-footed drive from a tight angle on the right-hand side of the six yard box to give Hearts some hope of a recovery . -Biden " should clearly explain the reasons that led the Obama administration to its decision not to build a radar in the Czech Republic , " Former Prime Minister Mirek Topolanek said in a statement prior to the vice president 's arrival .. -Presented in the committee of the lower house of parliament by the Catalan party Convergence and Union on March 10 , the bill is designed , according to spokesperson Josep Sánchez y Llibre , " to protect citizens against those acts that attack their dignity or invade their privacy . " -Do not drag him because he will only resist and make the matter worse . -Caution should be observed when initiating DIOVAN in patients with heart failure or recent myocardial infarction , and in patients undergoing surgery or dialysis . -Polling day now looks certain to be on 6 May . -The region still places a heavy burden on the Kremlin 's budget today . -Northern Rock has been lending again , in a modest way , but its figures this week showed a £ 1.2 billion loss of savings , which must concern potential buyers . -And for many it will be a way to save on child care . -For all the proposals stuffed into its 1,336 pages , the financial reform bill that is headed to the full Senate will change very little for banks , Wall Street and financial firms . -The first-term Alaska governor has been accused of firing a state commissioner to settle a family dispute . -PERIA JERRY , DT , Mississippi , may be as good . -Jean-Sebastien Giguere made 37 saves before he left the game after two periods and was replaced by Jonas Hiller . -When the paramedics arrived , they were amazed that Rafaëlle had managed to continue CPR for almost half an hour ; normally the effort makes your arms numb after a few minutes . -Police are continuing to search 10 properties across the north-west of England in connection with an alleged planned terror attack . -Coach Louis van Gaal has demanded more from his Bayern Munich side despite their recent improvement . -A Department of Environmental Protection biologist warned state officials before the attack that Travis could seriously hurt someone if he felt threatened , noting that he was large and strong . -When the war ended , they danced in the streets . -Can the new generations of workers in Russia , the United States and around the world replicate the mind-set of the people who opened the space frontier ? -1256 : Manchester United striker Wayne Rooney drops to the bench and Carlos Tevez starts , while centre-back Rio Ferdinand is out and Jonny Evans starts in his place . -Observers are warning that such growth cannot continue , with Nationwide suggesting that Northern Ireland will underperform all other regions in 2008 . -The analyst firm also recognized Autonomy as the fastest growing of the leading vendors in the report with 17.6 percent growth from 2007 to 2008 . -BNY Mellon Asset Servicing offers clients worldwide a broad spectrum of specialized asset servicing capabilities , including custody and fund services , securities lending , performance and analytics , and execution services . BNY Mellon Asset Servicing offers its products and services through The Bank of New York Mellon and other subsidiaries of The Bank of New York Mellon Corporation . -And I found it immensely frustrating that one of the few great races of the season , behind Monaco and the sodden chaos of Silverstone , is in the hands of lawyers and could take weeks to unravel . -Cable companies that once rejected the firm 's ads are now having second thoughts , said Noel Biderman , president of Ashley Madison Agency . -President Tamara Mellon follows in the footsteps of Matthew Williamson , Roberto Cavalli and Karl Lagerfeld in the deal with the retail giant . -His father , Felipe , emigrated from Mexico 's Jalisco state in 1951 and within two years mastered English , then was disabled in an industrial accident when John was a baby . -If confirmed , it would be the second missile strike in two days in a tribal region . -Others add so much sugar that their calories content is ultimately not far off what were they were meant to replace . -A number of reports suggested that Britton was quitting the show because she was unhappy at being paid less and having a lower profile than Schofield . -But that view appears far from universal . -That said , " The Four Seasons " has benefited , like other ubiquitous Baroque and Classical works , from the early music world 's constant redefinition of period sound and style . -Madoff is serving a 150-year prison sentence . -That 's the message sent by an international team of scientists who say they 've discovered a protected lunar " lava tube " -- a deep , giant hole -- that might be well suited for a moon colony or a lunar base . -" With the closure of the Laura Ashley factory , we have a valuable opportunity to celebrate the legacy of Laura Ashley , at the same time supporting the economic development of the Carno region . -" Iran 's action , first the expulsion of two diplomats and now the arrest of a number of our locally engaged staff , is unacceptable , unjustified and without foundation , " Prime Minister Gordon Brown told reporters in London . -The jagged and oft-snow-topped blocks of rock that dominate the horizon in every direction make this place feel a bit like Shangri-La : a secret valley , cut off from the world and stranded in happy seclusion . -The suspicion is that Ferguson was not really sure how he would use Cole -- and it has taken Ancelotti some time to assess that , too . -Later that evening , freshly showered and pink-cheeked from the day 's exertions , I enjoy a final blow-out at the Hotel Eden in Les Praz , just on the outskirts of Chamonix . -A barrel of oil vaulted to a record above $ 147 , raising more concerns about inflation and the overall economy . -I am six years divorced now and look forward to new challenges , like meeting a new partner . -What was her favorite thing about Paris ? -But Sir Richard stressed he was not endorsing the policy of any one party . -But he and Bristol split up three months after the birth and Mr Johnston has since been a thorn in Mrs Palin 's side , using a series of interviews with US magazines and television shows to criticise her . -Several Starbucks representatives did not immediately respond to calls and e-mails seeking reaction to that request . -Those who believe Marwan was a double agent loyal to Egypt claim he provided wrong information about the exact hour of the actual 1973 attack . -Another example of this can be seen with my own district nurses . -Such statements involve inherent risks and uncertainties . -Charlotte 's Fresnel lens was moved to another lighthouse and then disappeared . -Not because they are die-hard political pundits or waiting for the truth to emerge from the mouth of the leader of the free world , but because they are playing the six-year-old State of the Union drinking game . -" While we can 't say that this was George Washington 's pipe , we can wonder about it , " Levy said . -He also had his Porsche taken during the raid , among other things , with the car later found burnt-out in West Derby . -The more vivid the view from the bridge , the greater her discomfort . -Is this difficult ? -Mr. Pan also maps the government 's efforts to paper over the horrors of its recent history -- from the Anti-Rightist Campaign begun in the 1950s to the Cultural Revolution ( which began in 1966 and ended a decade later ) to the crushing of the Tiananmen Square protests of 1989 -- while chronicling the brave , sometimes thwarted , efforts of citizens to pay tribute to the victims of those years and to pry a truthful version of the past from the gears of the government 's mammoth propaganda machine . -They will also receive 50 per cent of any sell-on for the German under-21 international who arrived at Upton Park in January , from Brescia , but has failed to make an impact . -It claimed it needed the arms to resist Israel . -Twitter users are becoming increasingly wary about the secret services monitoring Twitter . -The stream of people walking out at last summer 's Proms showed that it was less good on stage . -Students stick fingers in their ears . -This chick is going to make it illegal to be white .. -Not many out-of-state students will be around on Jan . -The European Union bans more than 90 airlines , including six registered in Swaziland , in a move to boost air safety . -Castillo , who was stopped on his way back to California , said he knows federal law requires him to be able to converse in English with an officer but he thought his language skills were good enough to avoid a ticket . -He attempted to describe the confusing politics of Cambodia in the late 1960s and early 70s , as the Vietnam War raged on Cambodia 's eastern border and the Khmer Rouge tried to recruit peasants and intellectuals angry with Sihanouk 's autocratic regime . -There were a couple of notable casting changes this year , with US bass-baritone Alan Titus taking over as the new Hans Sachs and Viennese baritone Adrian Eroed as Beckmesser . -Erdogan contradicted Babacan , telling the state-run Anatolya news agency that economic measures were not yet being implemented . -Daimon Hoyt now faces three charges of soliciting murder , the Las Vegas Review-Journal reported . -A third suspect , Ivory Coast immigrant Rudy Guede , has been convicted and sentenced to 30 years in prison in relation to the student 's death . -" That 's where our primary weight of effort is . -The ship will probably take its cargo to Asia . -One mother said : " It is the nanny state gone too far - generations of my family have grown up on Marmite . -" Israeli security forces will continue to operate in order to ensure freedom of religion and a normal way of life for Palestinians uninvolved in terrorism , " the statement said . -During the regular session , the stock gained $ 2.28 , or 6.2 % , to close at $ 39.15 in a broad market rally . -The film , as far as we can tell , is lacking any real narrative structure -- it 's just a series of shots of Zoller aiming and felling yet another faceless soldier . -It is a difficult operation , " once rescue worker said . -It meant I was approaching the border before Julie had even left the tarmac at Edinburgh airport . -Clin Res Cardiol . -It is the largest renovation in the museum 's century-long history and a transformation of its largest exhibition space , making it as much about the museum 's future as about the ocean 's . -" It is a homing receptor for lymphocytes to get to the gut . -The Cougars last won at the Coliseum in 2000 , 33-27 . . . . -( NYSE : LII ) today reported financial results for the first quarter of 2009 . -And she still sat in her living room , a brooding presence , waiting for something that was never going to happen . -This is one example of the power of partnerships in action . -" I think it is important to pivot as quickly as possible for the superdelegates or others to make a decision as quickly as possible , " Obama said , to give the nominee time to choose a running mate and plan for the party 's convention in August . -10 , 1957 , in Inglewood and graduated from Western High School in Anaheim and Cal State Fullerton . -" We will give them plenty of respect and the Slovakia game was good preparation as they are similar to the Ukraine . -Civil aviation officials in New Delhi accused Continental of gross violation of Indian security rules that prohibit pre-embarkation body checks on certain dignitaries like a former president . -In other markets , US bond yields were steady , while oil prices fell toward $ 51 a barrel , after the International Energy Agency forecast lower global crude demand . -He threw himself into fighting against the deregulation of gas and oil prices , called for tighter regulation of the insurance industry and fought for national health insurance . -" I have ignored my health for years , but recent developments have forced me to re-evaluate my priorities of faith and family . -A third party company took 288 new products and calculated the average price from eBay sales , including postal charges , the auction site said . -14 , the W.H.O. said that cholera had been clinically confirmed in more than 1,055 cases so far in Kurdistan and was suspected in more than 24,500 cases of diarrhea and vomiting . -Watch the full NBC replay of Troy Dumais ' springboard victory at the U.S. Olympic Diving Trials on June 21 . -Wilson allegedly told detectives she then went inside and fell asleep . -The Peak District is a classic British misnomer ; although it holds more than its fair share of cliffs , gorges and high moorland , proper pointy peaks with 360 ° panoramas are in short supply . -But even for Lipsett , who found the training helpful , there 's still an element of resignation when it comes to drunken clubgoers . -Sir Sean Connery has settled a lawsuit with a golf club for an undisclosed sum . -Rarely by a first lieutenant , and virtually never by a captain . -In a recent study , 15-year-olds in the U.S. ranked 21st out of 30 countries in science scores . -To the " STARS " --Come on now , loosen up after all these Great Years of Music and let us ALL enjoy - not just those with BIG BUCKS ! -Its chairman , Sir Ian Kennedy , has pledged to have a new expenses system in place in the next Parliament . -The prince , who wore desert fatigues , also visited the capital , Kabul , " the only royal in living memory " to have done so , according to Clarence House . -She said : " We picked up a T-shirt ... but next to it was this T-shirt with a dagger and blood running down the side of it . -Whether rocking spray-on pencil skirts or fluoro jumpsuits , Scutt has built a reputation for doing things her own way , and her designs have become the epitome of vixen chic . -On Friday , Hugo Chavez , the President of Venezuela and one of Mr Zelaya 's key supporters , said the talks in Costa Rica were dead and that it was " horrible " to see the " usurper " Mr Micheletti being treated with deference by Mr Arias . -Inaugural dresses worn by Helen Taft , Eleanor Roosevelt , Rosalynn Carter and Laura Bush , among others , will also be exhibited . -Each firm raised suspicion because its default rate was at least twice the average of peers in their area . -Hospital and police officials said the two wounded children were 9 and 12 . -Mr Sarkozy came up with the idea of a mini-treaty to revive the key parts of the Constitution while Mrs Merkel drew it up under the German EU presidency last year . -So I felt Uefa was wrong and welcomed the U-turn . -While board members don 't get salaries , some who are ministers get paid for speaking at church events through offerings and honorariums , Swicegood confirmed . -He and Sadeer Saleem were going to join the future bombers Mohammed Siddique Khan and Shehzad Tanweer , who had already left . -From the Carolinas to New England , forecasters called for a second consecutive weekend of choppy seas , dangerous rip currents and heavy rains as Danny was expected to pass to the east of the mainland Saturday . -Three months after the incident , a trip home to Belfast is proving a morale booster but he was not able to be there for the birth of his son Carter . -The remaining non-managerial workers would share no more than $ 1.62 million . -It had been replaced and the machine was being switched back on when the latest problem occurred . -" You have to use careful selection . -Lonnie Lynam , a self-employed carpenter in Pipe Creek , Tex . , specialized in spiral staircases . -Reports suggest Adriano may be considering his footballing future after his fortunes have nosedived . -But the governor offered a different account when spotted in the Atlanta , Georgia , airport after nearly a week out of pocket . -Ahmadinejad said this was rude . -Why is the death of people that have chosen that lifestyle a bad thing . -Days after reports of Mehsud 's death surfaced , Pakistani Interior Minister Rehman Malik told reporters that clashes had erupted in a meeting , or shura , to decide a successor and that one or both of the leading contenders - Hakimullah Mehsud and Waliur Rehman - had been killed or wounded . -Libyan leaders told Asharq al-Awsat that Gadhafi had no issues with Lebanon . -Wilkinson 's name on an England team sheet may still send ticket prices into orbit , but even in his pomp , he was not the all-seeing genius some people made him out to be . -The order states he cannot act in a manner which causes alarm , harassment or distress to any person in England and Wales . -There 's nothing new about SUP , though . -( AP ) Washington Mutual 's holding company is suing federal regulators for billions of dollars , saying the fire sale of the bank 's assets to J.P. Morgan Chase violated its rights . -Remember , when the gov 't gives me money back , that means they didn 't spend it on something frivilous . -Anxiety is also being felt among the businesses that cater to Wall Street and its high-income workers . -The mayor 's office had no comment Thursday . -A wisdom , however , that has not yet made it to the administrator 's office at Columbia . -The Knicks came out shooting well from long range , converting five 3-pointers in the first quarter , when they took a 30-26 lead . -Detective Chief Superintendent Tony Porter , head of the North West Counter-Terrorism Unit , said the raids were part of a continuing operation and that police had acted on intelligence received . -Broader stock indexes also fell . -Yettaw , who was taken to Suu Kyi 's home Thursday by officials to re-enact his visit , told the court Wednesday he had been sent by God to warn Suu Kyi of his premonition that she would be assassinated by terrorists , said Nyan Win , one of Suu Kyi 's lawyers . -There are Eastside Petsitters , Eastside Surfers , Eastside Real Estate and Eastside Studios , and for years , locals came together for an Eastside Art Crawl . -Elsewhere , two schools have collapsed , each trapping nearly 1,000 students and staff . -Guardian home exchange allows you to swap homes and like a local all over the world. wrap. cardiff bay , wales . £ 28-35K plus an enhancement to cover benefits which are normally available to permanent staff. metropolitan housing partnership-1. spirita changes the lives of literally thousands o ... . £ 15000 - £ 19999 per annum + £ 17,361 PA pro rata per hours worked. veterinary laboratories agency. south east england . £ 42,000 - £ 52,865 per annum .. gardens , antique shops , art galleries , heritage farms , working studios , wineries , farmers markets , and an arts center with year round fine art classes , are but ... . wa. park , indoor recreation facility , new performing arts center and restored movie house , and many of our ... charm of their historic heritage. live , work and play ... . oh. cultural activities at your fingertips , theatre , arts , museums , etc. sports enthusiast paradise , for ... is rich with culture and heritage , breathtaking ... . nm . -South Africa 's government has been criticized for its slow reaction to the violence and for not addressing the poverty that is widely blamed for the bloodshed . -Forterraʼs civilian platform , the On-Line Interactive Virtual Environment , is one of many products being used or developed for training first responders to cope with chemical , biological or radiological incidents . -About two dozen students also flew into seats in front of them or onto the floor . -Spector alleges that Shapiro 's work on the case was inadequate and may have led prosecutors to file formal charges against Spector . -Head south for about 150 yards and then turn left ( east ) to follow the contour along Westfield , through a coppice and emerge through a kissing gate on to the Westfield road . -Several bicycle shops rent bikes all year round from $ 15 ( 75 kroner ) a day -- http : / / cykelboersen.dk / en / . -Russia began negotiations for WTO membership in 1993 , but the talks have hit numerous roadblocks over the years and today Russia remains the world 's largest economy still outside the WTO . -Then we started getting closer . -Alice Waters , founder of Berkeley 's Chez Panisse cafe , is known worldwide for her unbending fidelity to locally grown food and organic agriculture . -Martin Brodeur glided past Sean Avery like a man preoccupied with what he was going to do with himself for the next two months until he could open up hockey 's proverbial postseason cottage on the lake . -The database , which has been seen by The Times , raises important data protection concerns . -" We find a loss of weight , loss of waist-to-hip ratio , loss of Body Mass Index -- in significant numbers even though they 're not enormous -- and a drop in blood pressure , " she says . -The wife of a Virginia school administrator was not pleased when a 17-year-old student called her home last week . -So what do this year 's casting decisions mean , if anything ? -The yield , at 13 per cent with a commitment fee of 5 per cent , a 2 per cent exit fee and a 1 per cent fee for undrawn commitments , is a heavy burden for a company which then lends its funds to its small and medium-size business customers at a far lower rate . -But the reform treaty will not come into force unless all of them do so - and it still has to run the gauntlet of a second Irish referendum later this year . -He said Sunday that another important focus of the G8 meeting would be to ensure that emerging nations like China and India sign up to measures to cap the production of greenhouse gases . -Earning A 's in her major subjects , history and international relations , she had fulfilled all the requirements for her bachelor of arts degree , except for physical education . -I had this argument with him awhile back and when I asked him back then about whether he cares if the troops are put in danger during a withdrawal he ran and hid and I expect he will do the same this time . -He does not encourage initiative ; he likes people who do what they are told . -The Menier Chocolate Factory production of the Tony Award-winning show is to open April 18 at a Shubert theater in Manhattan to be announced . -Out here , the economy rests on four fuzzy legs . -Marine reserves have been created off the north-central and south-central portions of the state . -Shareholders balked at an announcement by the National Bureau of Economic Research that the US economy entered recession in December last year . -In the mid Eighties , when she was in her teens , she cooked for her bohemian supper club of " about 20 to 40 people , every Thursday , who all paid a fiver . " -By intervening aggressively , Hank Paulson and his kindred spirits at the Fed haven 't quite ensured a continuation of the status quo -- some reforms will come , and banks and their regulators will tread more gingerly for at least a few years -- but they do seem to have headed off a re-enactment of the New Deal . -The nine foreign nationals were held in Southampton , Hampshire , on Wednesday by officers from the Serious Organised Crime Agency ( Soca ) . -Like corals , they secrete calcium carbonate . -While Americans have gotten much heavier since then , they 've been able to lower their cholesterol with powerful drugs that carry few if any side effects . -The French retailing company Auchan hired Intertek to test toys it stocks ; it discovered the first Mattel lead-paint problem on June 6 . -In " Octopus , " he sings : " So high you go , so low you creep . . . The squeaking door will always squeak . " -They have to rebuild . -I also like everyday vanilla ice cream with the sides of the sandwich rolled in flaked coconut ( do this just after you fill the cookies so the ice cream is still soft enough for the flakes to adhere ) . -The data showed a ratio of 2.9 birth defects per 1,000 live births in Kettleman City during those years . -Still , the Hornets , with Hilton Armstrong starting at center for Chandler ( ankle ) , went toe-to-toe with the Nuggets until Denver 's third-quarter run started the celebration . -ROUEN , France , Feb . -a definitive destination for advertisements from football 's biggest night. vote online for best ads during the live broadcast. capability to share and embed their commercial picks . -If someone allows water to shoot up the nose--say , by doing a somersault in chest-deep water--the amoeba can latch onto the olfactory nerve . -Data show that from Friday through Tuesday night , almost $ 48 billion was moved from prime money funds . -Long before the Kasbah du Toubkal and Richard Branson 's Tamadot opened in the nearby Imlil valley , this was the main crossroads in this part of the High Atlas . -Yusuf spoke to Reuters while seated upright in his hospital bed , where he was watching television and speaking with aides . -The cylinder thermostat should not need to be set higher than 60 ° C / 140 ° F. And dripping taps can waste enough water in a week to fill a bath . -" We remain cautious about trading prospects for the year but our cost cutting measures are having a positive impact and we are currently in line with the board 's expectations , " a statement said . -Meanwhile , Essex 's Ravi Bopara was unable to train at Green Park because of a stomach complaint and left-arm seamer Ryan Sidebottom has discomfort in his lower back . -The IRG boats aggressively approached the U.S. ships during daylight , reportedly getting as close as 200 yards . -The primary goal of this program is to discover and characterize planetary systems and Earth-like planets around other stars . -A 16-year-old boy has been charged with causing criminal damage after almost 30 vehicles were attacked in a tyre slashing spree on Tyneside . -They 're both present from the start . -The Good Schools Guide recommends Belmont Primary School . -Montoya shot Rios and his girlfriend as they slept , then walked out of the rebel camp with Rios ' memory sticks , passport and hand to present to the army . -And , as with any extended foreign travel , it does help to speak at least a few words of the language . -But in a statement issued to the BBC , the MP said he utterly refuted the allegation . -If only dieting were this easy . -It was published on guardian.co.uk at 00.05 GMT on Sunday 28 February 2010. one of the uk 's top architectural practices is loo ... . £ 20,000 - £ 25,000 depending on experience . -The measure now goes to the Senate , where passage is expected . -" I think if someone wants to evacuate , regardless of their economic situation , they 're going to evacuate , " he said . -In early 2003 , he met a collector named Tom Hoyt , who said he was a computer executive anxious to get help in building his collection . -That 's it .. how do you lose ? -The concentration of inorganic arsenic -- considered the more toxic form -- in the rice grains was nearly three times greater under flooded conditions . -But , he adds , the rise in unethical acts today may be more a function of economic uncertainty . -Easy , the set-up of the network is done simply by pushing a button . -In Caia Park , Wrexham , a community-based organisation has been founded to plan for 80 new allotments . -Ross Perot Jr , Perot Systems ' chairman , is expected to be appointed to the Dell board . -" Chelsea is a great team because it has 25 great players , and for me it is impossible to choose the first team . -The Cubs ' offense clicked in the third off starter Jesse Litsch . -Michael Longley won in 2000 and Nobel winner Seamus Heaney added it to his list of awards in 2006 . -A sports club located near the gymnastics venue for the 2012 Games could be forced to close as it is unable to keep up with the rent . -Isner 's win helped him improve on last year 's quarterfinal exit in Auckland , when he reached the main draw through qualifying , and to avenge his narrow , three-set loss to Robredo while representing the United States at last week 's Hopman Cup . -A severe shaved head made the former Neighbours star barely recognisable , but he looked super-sleek in a custom-tailored Burberry tux , dress shirt and swish shoes . -Prowse says Duncan 's charm and generosity now make him regret getting the senior Tory into trouble but that politicians ' attitudes towards their part in the expenses scandal needed to be exposed . -Mr Herbert was very persistent though and he eventually gave him permission to search in the field . -Headley pleaded guilty to conspiracy to bomb public places in India ; conspiracy to murder and maim persons in India ; six counts of aiding and abetting the murder of U.S. citizens in India ; conspiracy to provide material support to terrorism in India ; conspiracy to murder and maim persons in Denmark ; conspiracy to provide material support to terrorism in Denmark ; and conspiracy to provide material support to Lashkar-e-Toiba . -I will come talk to you , ' " Mr. Mohamed said . -Donald has not played since the U.S. Open and will also miss the British Open at Royal Birkdale this week . -How that is best done is what has divided the EU bloc of 27 nations . -He was referring to the frenetic bargaining and deal-making that is apparently taking place in the Zimbabwean capital , Harare , as politicians and the military top brass adjust to the post election political scene . -Thirty-three people lost their lives in the crush . -Griner blocked four shots in the first half ; her long-armed presence seemed to unnerve the Lady Vols . -It 's not known if the suspects were looking for the second safe , Beasley said . -As Elizabeth Palmer reports , at least five demonstrators were killed in this series of clashes with police . -For someone who has been conspicuously stable while closing out matches , rarely letting slip opportunities to win when they present themselves , Nadal 's words superficially contained an element of surprise . -Both have come on the second playoff hole . -The higher fees will make the sale and purchase of a property - the conveyancing process - slightly more expensive . -" But I said very clearly--and I 'd be glad to get a record of what I said--I said the government has to enact reform to prevent the kind of crisis we have , and there was a role for government , and I supported a bipartisan solution , " McCain said . -Ms Blears , the Communities Sectretary , announced she was leaving Cabinet this morning , fuelling the sense of crisis surrounding the Prime Minister after news of Ms Smith 's resignation emerged yesterday . -On Monday , Netanyahu repeated he had no intention of implementing a complete settlement freeze , saying any halt would be temporary , would not extend to east Jerusalem and would exclude some 2,500 units already under construction . -The media center was no more than 50 yards away . -Some Canadian companies have invested in Mexico : Bombardier has factories making aircraft parts and trains , while Scotiabank is Mexico 's seventh-biggest bank . -Many of the 120 students from the housing project have not been to school since March 7 because they fear retaliation after a reputed gang member from ABLA shot and killed another student who lived on a rival gang 's turf . -27 ( UPI ) -- Gil Morgan managed an even-par 72 in the wind Saturday to maintain his two-stroke lead through two rounds of the seniors Turtle Bay Championship . -She said the diagnosis was serious but she was confident she could be in remission quickly and hoped to get back to " my old self very soon . " -A bomb attack killed three security personnel outside an army barracks in Pakistani-administered Kashmir on Wednesday , a police official said . -Levi-Strauss ' death at age 100 was announced in Paris on Tuesday . -There are an estimated four million carriers of the virus in Europe , and between three and four million people in the United States are chronically infected , according to the WHO . -Jenkins said even if terrorists did build a nuclear bomb , it is likely to be a low-level device , perhaps one tenth of a kiloton -- about 10 times the size of the largest truck bomb -- rather than the " standard " assumption of 10 kilotons -- about the size of the bomb the United States dropped on Hiroshima . -The idea for a monumental symbol of American expansion into the West originated in the 1930s but was shelved during World War II . -Taylor Paschal-Placker , 13 , and her best friend Skyla Jade Whitaker , 11 , died after being shot multiple times the afternoon of June 4 , 2008 , less than a half mile from Taylor 's rural Weleetka , Okla . , home . -Do you have a favorite Jenna line ? -Hillary Clinton was somewhere else in China , so couldn 't provide relief from the monotony and static manliness of the delegates . -Theo Walcott and Rooney were to the right and left respectively of Emile Heskey , with Steven Gerrard and Frank Lampard to either side of the holding man , Gareth Barry , in midfield . -Barney Frank , a Democrat and chairman of the House Financial Services Committee , yesterday managed to secure a concession that the US Government will have the authority to seize stakes in banks seeking to benefit from the bailout . -The 6,000-plus candidates have been highly visible and accessible to media in the campaign ( which ends 48 hours before the ballot ) , creating some scepticism . -THE PLACE Everything is just right about Krogbar ( 112 Krog St. ; 404 / 524-1618 ; dinner for two $ 40 ) , a pocket-size wine bar from the owners of Rathbun 's next door : the amber lighting and the chic log-cabin feel ; the careful sourcing behind the anchovies , salumi and cheeses ; and the infectious enthusiasm of sommelier Jon Allen ( he 'll pour you a taste of any of his 50 wines by the glass ) . -Ages 5 and up. batteries ( not included ) . -South Waziristan is part of the lawless tribal region along Pakistan 's border with Afghanistan , and top Taliban and al-Qaida leaders are believed to be hiding there , helping plot attacks on American troops across the border . -The FDA has struggled with the issue of acetaminophen 's safety since at least 1977 , when an agency committee suggested that labels for pain relievers contain a warning that they can damage a patient 's liver . -The consortiumʼs goal to map 50 cancer genomes is 25,000 times larger than the output of the Human Genome Project , Hudson told the Toronto Star . -Note : Adapted from " Doña Tomás : Discovering Authentic Mexican Cooking " by Thomas Schnetz and Dona Savitsky with Mike Wille . -Michael Tamvakis , professor of commodity economics and trade at Cass Business School in London , said it was hard to predict an upturn . -The offer values 100 percent of Opel equity at 515 million euros and assigns the company an enterprise value of 4.45 billion euros as of the end of May . -" Only a small proportion of prison budgets is spent on activity intended to reduce reoffending by prisoners on short sentences , despite the fact that 60 % of such prisoners are reconvicted within a year of release , at an estimated economic and social cost of £ 7bn to £ 10bn a year , " says the NAO report . -A similar confrontation now looms . -Moore was a man of great charm and capability who tended to hide his talent behind a natural diffidence . -Nobody is being enriched through testifying . -BT Vision combines Freeview channels with a library of 3,500 hours of on-demand programming as well as several channel options . -You really don 't , do you ? -Madison 's Impala , a butterscotch two-door with peanut butter-colored interior and 24-inch wheel rims , is expected to make its training camp debut next week . -Pope Benedict will be given the first set of the work , published by the Vatican Secret Archives in collaboration with Italy 's Scrinium cultural foundation , which acted as curator and will have exclusive world distribution rights . -Thirty of the past 35 Gold Cup winners to have run again the same spring failed on their next start . -But the hip-hop heavyweight successfully appealed the ban and received entry clearance from an asylum and immigration tribunal in January . -The senior manager told the inquiry it had partially collapsed less than 24 hours before Ms Hume fell in . -The paid trolls here can cut and paste all day long , but the vast majority of U.S. citizens eagerly await the opportunity to repudiate Obama 's policies . -Scientists have known for a while that having a close relationship with an obese person , whether a friend or a spouse , makes you more likely to become obese . -The enduring erosion of the coastline , caused by bad water management all the way from the upper Mississippi River and the coastal canals , erodes the marshes and leaves coastal Louisiana exposed to the full brunt of future hurricanes . -Tracking her down seemed impossible . -He likened abortion to the Holocaust and the Sept . -Hernandez escaped trouble in the third and fourth innings , then led off the fifth by hitting Byrd . -Hosted by Ryan Seacrest , American Idol remains the most popular show in the United States , with around 26 million viewers tuning in to watch each episode . -Versus the dollar , sterling climbed 0.2 per cent to $ 1.6307 , having jumped as high as $ 1.64 , a three week high . -Government official Rao Iftikhar said four gunmen were killed , including three by army snipers . -First reported by Military.com , the pirates , who have halted all talks with the ship 's owners , are talking to a woman named Michele Lynn Ballarin , instead . -19 , the group had raised almost $ 4.4 million , more than seven times as much as was raised by groups opposing the ballot measure . -The state department will also be giving evidence on Tuesday to present its side of the story . -Early in the month , this planet will be negative fourth magnitude , or very bright . -" China is poised to have more impact on the world over the next 20 years than any other country , " he said . -The damaged link - part of a network of eight similar pieces - is about 2 inches thick and was cracked halfway through . -The Irish News says the scale and ferocity of the rioting has " brought shame " on all those involved . -Correction : " Mugabe and the White African " has not , as we originally wrote , been shown on the BBC . -It will indicate how companies are doing , but also give clues about consumers , simply because when Americans are working , they 're in a better position to feed economic growth . -Provisional dates for a Four Nations tournament involving Scotland , Northern Ireland , Wales and Republic of Ireland have been announced . -Wisely , Mr. Marsh , who based his film on a book Mr. Petit published in 2002 , never alludes to Sept . -Microsoft executives announced that " All You Need Is Love " by The Beatles will be released as a download for Xbox 360 consoles , with proceeds going to humanitarian group Doctors Without Borders . -In the hospital wards , many former lepers are receiving ongoing medical treatment . -He took advantage of cooling track temperatures under clouding skies at Sepang to set the day 's best lap of 1 minute , 35.055 seconds . -" ( With ) this financial meltdown and the constant barrage of negative economic news ... -This article was published on guardian.co.uk at 11.33 GMT on Friday 19 March 2010 . -Yet almost at once the brigade was pinned down in some of the hardest fighting the Army has experienced since the Korean War , and things have gone downhill since . -But Prince couldn 't talk and he couldn 't eat , and he knew something still was wrong . -Open and serve . -The damage assessment training could lead to new construction standards in Haiti that take quakes into consideration . -There is life left in the cleaned-up meaning of hot mess , which has come to mean " disheveled " or " incompetent , " as in " I was a hot mess this morning before I hit the shower . " -No matter what kind of blade does the work , the sculptures have one thing in common . -Meetings are held twice a week and conducted in Creole . -The Senate Judiciary Committee held a hearing on West last week but has not scheduled a vote on his nomination . -Sales during the quarter rose 10 percent to $ 1.31 billion , the company said . -Now I here there are some Added Delegates [ 76 ] that come in later . -At 83 , she recently suffered through a painful bout of shingles , a reactivation of the dormant virus that causes chickenpox . -The trial by the Special Court for Sierra Leone , in The Hague , Netherlands , began last June but adjourned after one day when Taylor fired his lawyer . -At the same time , the unemployment rate increased to a four year high of 4.8 per cent , and the number of jobs on offer fell to a seven-year low with only half as many jobs as applicants . -She knew where practically everything on the menu had come from , or was going ( when I eyed a slice of the " world-famous " carrot cake , she told me that she mailed it to Queen Noor of Jordan whenever she came to Washington , D.C. ) . -more than 5,000 hours of construction . -Samoa 's highlight came with a first-half try from Gavin Williams and the appearance in a record fifth World Cup for Brian Lima , on as a replacement . -He claims that the White House politicised the honour under the Bush regime . -Mike Clasper , former head of BAA , was appointed non-executive senior independent director to replace the outgoing Sir James Crosby , the ex-HBOS chief executive , earlier this week . -Sweet treats from the CD include : " Mostly Ghostly " - a wailing pipe organ introduction ; " Vampire Empire " - a succulent waltz with harpsichord , organ , and kick drums thumping like heartbeats ; and two versions of the traditional " Souling Song " - " All Hallows Version , " which explains the medieval Christian history of All Hallows with cathedral bells and organ , and " Samhain Version , " which explains the pagan origins with drums , organ , and guitars . As Kristen 's singing and compositions range from richly intense to delightfully quirky , this CD will appeal to all ages and types of Trick-or-Treaters . -Turnout at the ballot was 87 per cent , with 57 per cent voting in favour and 43 per cent against . -Clinton matched the mood of the moment after Oklahoma , and saw his approval ratings rise , after his own mid-term election drubbing by Republicans . -The cure is that we stop consuming and start saving , and that 's a recession . -Poor Indian tax payers , do not get any security with this weapons . -Kate Ferranti , a union spokeswoman , said that in those settlements the union has gone far toward its goals of creating national standards for janitors , transforming many jobs from part time to full time and getting many janitors family health coverage . -This poor performance is often blamed on the fact that millions of Americans lack health insurance . -The 1st Infantry Brigade Combat Team from the 10th Mountain Division based in Fort Drum , New York , had been scheduled to relieve another combat brigade in Iraq in January but will not deploy , the Defense Department said in a statement . -If we keep on spending without saving , many of us will be headed straight for personal financial ruin . -Hayden said claims by the European Parliament that at least 1,245 CIA flights transited European airspace or airports are misleading because they implied that most of those flights were rendition flights . -Sheikh Muszaphar reported on his Weblog that the grass around the capsule began burning quite heavily after touchdown . -Videojug is slowly setting about its aim of producing millions of videos that cater for every conceivable problem that people turn to the internet to solve ; unsurprisingly , one of their biggest categories is food and drink , but love and dating also score highly , as do beauty and style . -Currently , Hong Kong 's leader , the chief executive , is picked by an 800-seat committee under the influence of the Communist leadership in Beijing . -Of course , community values aren 't solely the province of people with children . -It may also explain why , in some cases , standard chemotherapy treatment for brain tumours works at first but then loses its ability to beat back tumour growth . -He was described by Martin O 'Neill as " one of the best players in the world , absolutely at the top of his form " on Wednesday . -First in BCS history . -" We intend the same thing to happen with the Twenty20 cricketers coming into the squad . " -Environmental group Greenpeace and animal rights activist group Sea Shepherd said they would track the hunt . -According to a regular participant in last week 's demonstrations , the protesters yesterday hoped to form large groups in side streets before bursting onto the main highway , thwarting attempts to disperse them . -But politics come down to such shallow triggers at times . -" He is thrilled not to have to go shopping , " she said . -Football star Cristiano Ronaldo 's 177mph Porsche 911 convertible is up for sale on Auto Trader . -It seems so unfair . -Although the children try to escape a society built on lying , when they arrive at the animal oasis , they find no single truth : only a seemingly safe environment , with hints of malevolence . -Do consumers suffer ? -" It was a constructive telephone conversation , " Uribe said , adding he needed to be prudent in answering whether Obama indicated he would be open to a vote on the pact this year . -A possible deal between the countries on civilian nuclear power was not mentioned . -Bits of plastic were the most frequently found items , contributing to a 148 % rise in the density of plastic waste since the survey began in 1994 . -Associated Press writer Leila Saralayeva in Bishkek contributed to this report . -Cruise portrays Claus Schenk von Stauffenberg , an aristocratic officer who mounted a failed plot to kill Adolf Hitler in 1944 as Germany was losing the war , and was executed at the Bendlerblock along with his fellow conspirators . -She was told her cancer was terminal in February 2009 and died on 22 March 2009 , aged 27 . -Andrew and I want to meet people who were there - to hear what it felt like , and what the creation of the NHS did for our country . -Consequently most of the extras we used in the film were from the actual school for the Performing Arts . -However Garry Jones , global head of derivatives , said NYSE Liffe would be prepared to sell up to 49 per cent of the equity . -" We 're burning through our reserves right now , so it 's nice to have extra help , " said Sable , who is talking to another local corporate firm this week about taking in subsidized attorneys . -White House press secretary Dana Perino said on Tuesday the JEC was " known for being partisan and political " and that Republican members had not been consulted . -Automakers have been offering substantial discounts on some models and shutting down the plants that make them to keep inventories from growing larger . -While they and their millions of supporters may be powerless to confront the system 's instruments of enforcement , their declarations raise issues that go to the heart of the Islamic Republic , its identity and values , and the legitimacy of those now running it . -Former D.C. first lady Cora Masters Barry 's nonprofit group will get to stay in the recreation center that she was instrumental in building a decade ago , a decision made after negotiations with the city . -Abhisit said there would be no " iron fist " approach to ending the violence and he urged people to prevent rebels from creating rifts between Muslims and the region 's minority Buddhists . -Daewoo Logistics complained it had already invested " not a small amount . " -Look at how people have turned their creativity loose on the iPhone . -The Free Software Foundation 's Coughlan said the difficulties of resolving the issue within the ISO framework illustrated a growing awareness that the implications of such decisions went far beyond the software industry . -West Indies closed their second innings on 96 for one , needing to bat through Wednesday 's final day to secure a draw or reach an improbable victory target of 437 at Providence Stadium . -" The last time I had a conversation with John I had to hold the phone three feet from my head , " says Randy Pullen , the Republican state chairman , whom McCain has repeatedly tried to replace . -Tan said the vessel 's 10 tanks carrying 1,700 tonnes of crude oil remained intact , but fuel had spilled from the engine system , which emergency workers were trying to contain with a floating 500-metre-long barrier . -Automakers were generally higher , particularly truck makers . -Candidates seemed secure on some aspects of the solar system and space but over 20 % of candidates thought the Sun orbited the Earth . -It is the city 's largest gay parade outside Manhattan and it has " a real neighborhood feeling , with people coming out with their folding chairs , their coolers , like you 'd see on Memorial Day , " said City Council Speaker Christine C. Quinn , who is a lesbian and has been one of the parade 's most loyal participants . -Other Hollywood figures such as director Steven Spielberg , executive Jeffrey Katzenberg , and actor Kevin Bacon are also among Madoff 's victims . -Iran today blamed Pakistan as well as the US and Britain for a suicide bombing that killed six of its commanders and 37 others in one of the country 's most unstable provinces . -You can 't even remember the last time you made someone cry . -The broadcasters account for 70 per cent of television viewing . -There is added intrigue over the more prominent of the two films that were withdrawn from the Palm Springs festival : " City of Life and Death " ( also known as " Nanjing ! -AOL president and chief operating officer Ron Grant will leave with Falco after a transitional period of a few weeks , The Washington Post ( NYSE : WPO ) reported Thursday . -Although the absolute risk of dying is still low -- less than 1 % -- such patients also endure longer hospitalizations and other complications and add substantially to the nation 's healthcare costs , researchers from the Duke Clinical Research Institute in Durham , N.C. , reported in the Journal of the American Medical Assn . -( TMS ) . The scholarships are valued at $ 20,000 ( for national winners like Sperry ) or $ 10,000 each , over four years , for study at a four-year college or university starting in the fall of 2009 . -Massive ! -" It feels like we are playing against any other team , " he said . -Britain has complained that the revised law will lead to the withdrawal of certain fungicides and cause sharp reductions in wheat yields . -No-one is talking about hard-core ideologues with links to al-Qaeda . -" I got an idea , none of us can decide who is most in love with her so we all have to make out with her and see , " Robertson joked . -But their gratitude does not extend to support for his Workers ' Party . -Those with the knowledge and hands-on experience with their horses must be ready with the right answers in this important ongoing debate . -Both the video game industry and the music industry are clambering to harness all the revenue making momentum these music games can muster . -Along with several fellow experts Dr Grant spent 36 hours sifting through the documents , comparing them to other pieces Barot was known to have written . -( NYSE : ME ) announced today that Scott Josey , Chairman , President and Chief Executive Officer , will present at the Barclays Capital 2009 CEO Energy / Power Conference in New York City on Thursday , September 10 , 2009 at 3 : 45 p.m. -And with the transfer deadline nearing , the manager is also hoping to strengthen his attacking options with some late additions to the squad . -Roosevelt Principal Kathie Danielson said science teachers " can 't be sure " the pigs came from Roosevelt . -A startup called All inPlay offers online games , including poker , designed to allow play between blind and sighted users . -After exchanging pleasantries with the Taiwanese visitor , Mr Hu said that improving relations between the sides needs efforts from both parties . -Abbas remains out of the country : Amman Monday , Yemen today , Rome later this week . -Does this mean he is losing interest in me ? -" We 've seen it before and we 're seeing it again -- ugly phone calls , misleading mail , misleading TV ads , careless , outrageous comments , " Obama said . -As key partners , the three organizations will create campaign awareness , provide participant encouragement and facilitate communication among agencies . -He said the study , which he was not involved in , pointed to a genetic switch that turns off the body 's defenses against cigarette smoke and leads to lung damage . -Jay-Z and Lil Wayne drew the second-most nominations with seven apiece , while T.I. received six nods . -Insurers and retailers were to blame for the weak performance . -A turquoise Whiting & Davis mesh and Lucite clutch was $ 79 ; the same bag on Buy.com is $ 200 . -" A lot of people just feel uneasy about troops kicking in their doors and driving like hell through the streets , " he says . -He attacked and killed Mr Blair with a carpet knife when his neighbour repeated the allegations . -Trickles of lava also rolled down the 8,000-foot mountain , which towers over the Albay Gulf in the central Philippines , and five new ash explosions shook Mayon 's steep slopes . -If I step outside myself , no . -It sounded like a large animal , or at least a rat . -( AP ) - Republican presidential candidate Mitt Romney on Tuesday said rival Rudy Giuliani " had nothing but praise " when then-first lady Hillary Rodham Clinton proposed a universal health care plan . -A government spokesman said the investigation into the Sept . -First , developing countries such as China and India must play a critical role in any global solution to climate change . -But Rangers took the lead in the 22nd minute when Kris Boyd flicked the ball into the path of Nacho Novo , who rifled it home . -A scrapbook containing what Heritage says are previously unpublished photos of the dead Che are also being auctioned . -Nancy edged Valenciennes 1-0 to climb into 14th place with 15 points , while Valenciennes slipped to last place on nine points . -EDT , valuing it at around 12.2 billion pounds . -Siena is not a team to be dismissed . -In 3 1 / 2 years , however , her quiet existence might change when all the donations sent to her when she was a baby mature into a payment of $ 1 million or more . -In May , Lockton Denver was awarded the Denver Business Journal 's Partners in Philanthropy Award , which recognizes businesses who make significant contributions to the local community . -" A little bit of patience by everyone is appreciated when you 're talking about making investments like this , " Anderson said after signing the contract with Duke at a ceremony at the state Capitol . -Their minimum wage incomes barred her from very basic student aid , like tens of thousands of others . -NEW DELHI ( Reuters ) - India 's monsoon rainfall deficit has widened further , increasing the risk of crop damage , but its impact on the country 's economy was offset by high growth in June industrial output due to buoyant consumer demand . -Mr Tomback said the bill for repairs at Apethorpe , which have involved more than 150,000 man hours , was in fact money well spent given the " invaluable " heritage of the hall . -So far , the search has not yielded any strong links , Orrey said . -Ten years ago , there were 10,000 arcades in the nation , and now the number is close to 3,000 , according to the American Amusement Machine Association . -There are about 25,000 of these , valued at £ 13bn . -NEW YORK ( AP ) -- Gillian Anderson and her boyfriend , Mark Griffiths , are expecting their second child this fall , her manager , Connie Freiberg , said Wednesday . -The government said it would prevent farm trucks carrying protesters from traveling to Bangkok and warned it would use the ISA to censor or close community radio and television channels that incited violence . -Mr Duncan Smith lost the vote the next day . -NEW YORK ( AP ) - Wall Street showed relief early Monday over the government 's plan to bail out Citigroup Inc.--a move it hopes will help address some of the uncertainty hounding the financial sector . -Don 't Read This appeared on a self-publishing website . -Some legal experts say the fact that Murray was trying to get Jackson off propofol , also known as Diprivan , can be interpreted to imply he was aware the drug was harmful to his patient . -They create programmes about exotic wildlife and locations that can be shown anywhere , foreignness being part of their appeal . -The creditors , holding 42 million dollars out of 6.9 billion dollars of Chrysler debt , argue fundamental creditor protections were ignored in a sale forced through by Washington . -Then I remembered what my mother did when she washed my hair as a child ( you are going to love this ) -- that a cup of vinegar mixed in a gallon of water cuts the soap fast . -The results were nearly identical to the year-by-year increases in humidity . -And consultants are already lining up interpreters and forging alliances with community advocates , tenant-association presidents , ministers and others who might help promote their candidates . -Adding fuel to the fire was a decision by the Texas Department of Insurance to adopt a rule allowing title insurance companies to claim a " blanket exception " on their responsibility to determine whether a landowner owns the mineral rights for a piece of property . -The winner will be announced at an awards dinner on 3 September , hosted by Lauren Laverne from The Culture Show . -Furthermore , participation of Afghan civilians is also lacking . -Elana Glatt said they had reluctantly paid for the flowers in advance , with a cashier 's check for $ 27,435.14 . -Will there be an after ? -In her speech Wednesday night , Palin belittled Sen. Barack Obama 's experience as a community organizer in Chicago , Illinois , and remarked that the Democratic presidential nominee and senator had written two memoirs but no major legislation . -Local media reports said the last stumbling block was an Israeli demand that some of the prisoners slated for release be deported to the Gaza Strip or elsewhere after they are freed . -Mr. Obama reached agreement with Mr. Nazarbayev to fly supplies to Afghanistan over Kazakhstan , with no real public criticism of the country . -to use , hardware independent , ZFS based software. information products , database marketing and processing services . -- Turn-up and configuration services are also now available. at http : / / www.nexenta.com or call : ( 877 ) 862-7770 . -A 22-year-old man has been charged with causing her death by dangerous driving and is due to appear in court again in August . -Electricity was knocked out across a wide area of the southern state stretching from Miami up to Daytona Beach , causing traffic jams as signals malfunctioned , and forcing some shops to close before power was restored to most customers by Friday evening . -The woman said the man , who appeared to be homeless , quickly ran from the porch . -As the death toll among British troops rose to 231 , Mr Ainsworth acknowledged that support for the campaign had been " dented " by recent losses . -Bush met on Friday in Peru with Chinese President Hu Jintao for talks focused in part on the global economic turmoil . -INDIANAPOLIS ( AP ) -- The drugmaker Eli Lilly and Co. said Tuesday that Sidney Taurel will retire as chief executive on March 31 after nearly a decade in the post and will step down as chairman of the board at the end of 2008 . -HORNCASTLE , England , June 2 ( UPI ) -- A British man who admitted biting off another man 's ear while dressed in a chicken suit was sentenced to 15 months in jail . -Already cautioned , he unnecessarily caught Lucas 's heels . -According to state regulations , anyone receiving unemployment benefits who works one day and earns less than $ 405 will have his check for the week reduced by 25 % . -Our Sports Bog colleague dug up his 2007 video dispatch from the America 's Polo Cup -- yes , Tareq Salahi 's organization -- and recalls crazy good times hanging out with an exuberant Michaele ( who kept shouting at the crowd , " Hi , beautiful people ! " ) . -The interest rate on one-month sterling in the interbank market rose to 6.715 per cent this week , its highest rate in nine years . -It too was published in real life by Hyperion . -Unveiling the charges against Mr Kilpatrick and Ms Beatty yesterday , Ms Worthy issued a stern lecture on the importance of telling the truth . -Johnson said the number of occupied trailers on the Gulf Coast , which peaked at more than 143,000 after the hurricanes , has dropped to about 34,000 as FEMA rushes to move people into safer housing . -HARARE , Zimbabwe ( AP ) - Zimbabwe 's Election Commission released a handful of results Monday from presidential and legislative elections , announcing an equal number of wins for both parties after a delay that raised tensions amid fears of rigging . -They believe their voices will help avoid similar bloodshed during the next election in 2012 , though early signs suggest that ethnic mistrust is again being stoked by politicians . -2000 - Crackdown on official corruption intensifies , with the execution for bribe taking of a former deputy chairman of the National People 's Congress . -The head of MI5 broke cover last night to defend the service 's foreign intelligence links with countries accused of torturing detainees , saying British lives had been saved as a direct result . -The shares will be offered at £ 14 to UK shareholders and $ 28.29 to Australian shareholders . -A Microsoft spokesman said the company had always maintained it reserves the right to exercise all options but declined to comment specifically on the DealBook report . -But Salim Abdullah , a Sunni lawmaker from the Iraqi Islamic Party , said his group is confident and unafraid of competition by the awakening councils . -Shinichi Ichikawa , equity strategist at Credit Suisse , believes investors should be wary of DPJ campaign pledges that could hurt companies , such as the opposition 's plan for a higher minimum wage . -There is no going back from that . -When robbing an illegal gaming room , take a quick look around to make sure the police aren 't in ... -First up , that if you 're going to attempt them , you have to go the whole hog . -THOMASVILLE , Ala . , July 31 / PRNewswire-FirstCall / -- United Security Bancshares , Inc . -The oil giant 's results were also boosted by higher production volumes , which rose 5 per cent on a year ago to more than 3.6 million barrels a day after the return to service of Thunderhorse , a platform in the Gulf of Mexico . -On top of potentially losing our membership of the Famous Five , there is another threat to Australia 's dignity : the very real prospect that we will lose our other major Olympic title . -Detectives have rejected that claim , and an autopsy found that Lauterbach died of blunt force trauma to the head . -Vessels able to navigate the debris-filled canals are scarce and efforts to import trucks and other vehicles have been hampered by governmental red tape . -The Scottish Football Association believes Uefa 's decision to expand the European Championships to 24 teams will prohibit small nations from acting as hosts . -But the jury -- what jury ? -- is still out . -The U.S. military said an al-Qaida-linked militant connected to his death and a plot to kill other tribal leaders Fallah Khalifa Hiyas Fayyas al-Jumayli , an Iraqi also known as Abu Khamis was seized Saturday during a raid west of Balad , and the search continued for other suspects . -They will use an Opposition Day debate to put the matter to a vote in parliament , and will be hoping to encourage rebel Labour MPs to support them . -Millions of Chinese-made toys have been recalled in recent months . -Jackson 's birthplace of Gary , Indiana , also is planning a memorial service for the deceased singer on July 10 , according to a spokeswoman for the Gary mayor 's office . -Say , if the study points out known offenders like Jose Conseco or David Segui or Jason Giambi -- who has elliptically implied his past steroid use -- there won 't be a whole lot of new information there . -Cramers put the death toll at 21 and said most of the victims were people on the ground . -The group has routinely trained gunmen -- called " fedayeen , " or fighters who volunteer to sacrifice themselves in battle -- to carry out operations in Kashmir and elsewhere in India . -This week in Europeans , " Hans von der Brelie , euronews reporter . -He did have his hands full late with Chase for the Sprint Cup drivers Kasey Kahne and Jeff Gordon , but neither could beat him on a series of late restarts , and Johnson ultimately made it look easy . -Next Article in Opinion ( 6 of 28 ) » A version of this article appeared in print on January 13 , 2010 , on page A30 of the New York edition . -The pressure for a rate cut comes as UK retailers face a crucial test of the strength of consumer demand in the run-up to Christmas . -The Orlando Convention & Visitors Bureau has a Bundles of Free Smiles promotion for hotel stays as well as shopping , dining , attractions , and golf . -Workers from nearby businesses scrambled to rescue the injured before firefighters reached the scene . -Do people really " hate " Clinton , Obama Or even Bush ? -The first time , the leadoff man Shannon Stewart nicked a foul and seemed so shaken that his helmet fell off . -But over five years of observing a group of bonobos the researchers recorded about 10 instances when a group of the apes set out on hunting trips in search of monkeys . -In honor of Madonna 's big day , we asked iReporters to share their stories about turning 50 and what the milestone means to them . -" You don 't have the gigantic gulf of difference you had earlier this decade , " said Philip A. Baggaley , a senior credit analyst with Standard & Poor 's Rating Services . -Japan and North Korea opened talks here on Saturday in a bid to resolve long-standing bilateral disputes and normalise relations , officials said . -The recovery work will begin next Tuesday in the presence of government representatives from France , Britain and Australia after a religious blessing of the site . -The Information Commissioner 's Officer confirmed that it met Google last year to discuss how Street View would be implemented . -A Merseyside Police spokesman confirmed on Tuesday the officer has been sacked . -Of course , in the late-Eighties dance music was the underground , the preserve of a small band of clubbers and DJs in London , Manchester and Liverpool . -" We haven 't always got it right . -Doing so means the items would forever become military property . -Instead , it has taken exposure at the higher level of the Heineken Cup for all these chickens to come home to roost . -Republican Pat McCrory , the mayor of Charlotte , says education was the top issue in the past . -It must ultimately be approved by the County Council . -But now the Senate is moving closer to the House position . -Analysts also are closely watching how government energy policy under the fledgling Obama administration could affect the coal industry . -Galliani said Milan wanted Beckham , one of world 's most famous and marketable athletes , for footballing reasons , not commercial ones . -Uribe will meet with Bush at the White House on Saturday , giving them another chance to urge congressional action . -But Scricca said those differences can be a source of business opportunities . -Often bargain price if you book late afternoon for same day . -We speak to Marc Gander of the Consumer Action Group ( CAG ) and BBC personal finance reporter Ian Pollock . -The first few years as a senator were difficult for Mr. Helms . -" He does not appear to understand how desperate the problem in Zimbabwe is , and the solutions he proposes are too small , " Tsvangirai said in a statement issued on the second day of the talks in Johannesburg . -The leftist party has pledged to block any attempt to privatize the industry . -The International Tennis Federation said it had appealed that decision to the Court of Arbitration for Sport , but for now Gasquet has his tour status . -The CDC is aware of 28 deaths of pregnant women and about 50 of children . -Some trader got there first and the share price has already blasted off . -Abbott is seen by many voters as a climate change skeptic after becoming opposition leader last December on the back of opposing the government 's climate change policy of a carbon emissions trading scheme . -Chances are , folks in Cleveland won 't be talking about the NBA draft Thursday . -Israel plans to free Kantar and four other prisoners Wednesday . -Of the 59,000 defendants facing crown courts in 2007 , around 20,000 " could on the face of it been dealt with by magistrates , " he said . -Spitzer hasn 't been charged in the case , but he apologized and resigned March 12 , shortly after the case became public . -Apparently surprised that some airports long have allowed guns in unsecured areas , Rep. Bennie Thompson , D-Miss . , said the new Georgia legislation represents a significant hole in national security and a threat to travelers . -I learnt that walking is companionable and draws its participants close together , that it oils the tongue and unblocks the spirit , that it challenges and probes and sets puzzles , that of all the outdoor activities it is the one most open to everyone . -With Wednesday 's blizzard compounding the effects of last week 's storm in Washington , the federal government announced another snow day . -NEW YORK , March 31 -- U.S. stocks opened higher this morning , rallying after a steep fall yesterday despite glum news from the housing sector . -BURBANK , Calif . , June 9 / PRNewswire / -- Fire up your rockets and blast off with ZulaWorld.com ( TM ) , an incredible new online destination where kids ages six and older , along with their peers , parents and teachers , can spend hours exploring the Galaxies in fun and challenging ways . From racing around Saturn 's rings while playing the Bumper Spaceships Game , to riding a skateboard at Greater Crater Planetary Park , ZulaWorld.com ( TM ) offers out-of-this-world entertainment that encourages discovery and excitement. www.zulaworld.com. -Some Reverend Wright on steroids might burst into public view . -Tim Ruskell has been pushed out as president and general manager of the Seahawks , leaving the door open for Mike Holmgren 's possible return to Seattle . -It would be difficult and dangerous for the Sunni to relaunch the sectarian civil war in which they were routed a couple of years ago . -He told the programme 's makers : " My most irritating habit is wanting things done properly all the time . -The crew members were arrested after UK Border Agency officers made the discovery . -The lawsuit alleges that three Blackwater guards on a rooftop killed three guards for the Iraqi Media Network and that 20 other Blackwater employees refused to cooperate with Iraqi officials investigating the Feb . -Apple 's Tablet : the future of computing ? 9 Can a Muslim say happy Christmas to his friends ? 3 The Big Question : Has the divide between Britain 's social classes really narrowed ? -A make-up artist helped make disguises for a number of men accused of carrying out the £ 53m Securitas robbery in Kent , the Old Bailey has heard . -In 1990 elections , her National League for Democracy won a landslide victory , but the military annulled the results and has ruled by fiat ever since . -The bank denied reports that it would post huge losses linked to US sub-prime mortgages and filed a statement with the Hong Kong Stock Exchange that said it was " not aware of any reasons for such movement . " -The preliminary hearing is scheduled to resume today . -He said : " With the bin strikes in Leeds and Liverpool , continuing action by our members in the transport sector , the postal workers ' dispute and the attack on firefighters ' jobs and conditions , it is no wonder that there are growing calls for an alternative political option that backs the workers against the bosses and the political elite . -They want to come back here and that is because the business opportunity is here . -I only have to admit that my neck pinches a bit , " he said on his website . -After spending two " hard , long , lonely " years in prison , the two said they were looking forward to spending time with their families and putting this chapter of their lives behind them . -Later , she returned to Indiana to get Lorraine and Johnny Jr . , whom the family called Jay . -" I know this , Man United is the game people really look forward to , a really important one for the supporters , and they can be decisive over the course of the season , " says Torres . -And in the real-life drama of the Israeli Air Force 's raid on a target in Syria last month , there are two particular " dogs " that have not barked . -To talk more about the issue , I called Cindy Pearson , a long-time women 's health activist and executive director of the National Women 's Health Network . -In a hopeful sign for political progress , lawmakers from Iraq 's largest Sunni Arab bloc on Monday ended their boycott of parliament over the " house arrest " of their leader , Adnan al-Dulaimi , who also attended the session . -The visiting restrictions are said to be a precautionary measure to help control the spread of the illness . -" We are in the compassion business and we are seeing families reunited , " said James Winans , development director of the famed Bowery Mission in Manhattan . -" After all this time the thought of actually finishing frightened me to death , " he said . -Tehran has said it has not been notified of any such plans . -I 'd rather have him as CoS than in the House leadership . -Streleski had been a graduate student for 19 years . -In the video , Mr. Azizuddin , wearing a newly grown gray beard , appealed to the Pakistani government to meet the demands of his captors as soon as possible . -( 296 Hanover St. , www.vittoriacaffe.com ) . -On Monday , the state-run al-Sabah daily said Iraq 's Foreign Affairs Ministry had sent a protest note to Tehran . -But the two sides have not been able to agree on the value of the company , nor who would take on the debt . -It 's more likely that the # s are , for example , 50 % approve / 40 % disapprove / 10 % neutral or unsure . -" We will field a team , " BOA chief Simon Clegg told BBC One 's Inside Sport . -A blog called Knife Tricks ( knifetricks.blogspot.com ) noted an increasing propensity toward articles about models , but we are probably still a long way from stippled versions of the Page 3 girls of the British tabloids . -Alternatively , stimulating a different area in the brain may be able to reduce tremors but not affect decision making . -It costs less to create a radio commercial than it does to make a TV commercial and local radio stations can hit more targeted demographics . -Penalties _ Smid , Edm ( tripping ) , 7 : 33 Mottau , NJ ( high-sticking ) , 10 : 18 Tarnstrom , Edm ( high-sticking ) , 14 : 07 . -But then I suppose their attitude was born out of the stupidity of Americans forcing a beautiful country to its knees while ignoring the heinous crimes they inflicted on the American red indians . -Nearly 6,000 copper alloy coins were found buried in two pots in a field at Sully , Vale of Glamorgan by a local metal detector enthusiast in April . -It derailed anyway , the first such derailment in the bullet train 's 40-year history . -A spokesman for the council said it had a duty to take companies to court if it believed they were in breach of regulations , and pointed out that Mr Whalley himself had insisted on the case going to crown court rather than being dealt with by magistrates . • A council is spending £ 400,000 on a training course that includes controversial " neuro-linguistic programming " techniques as practised by television hypnotist Paul McKenna . -For more information on how to vote on the proposed resolution shareholders should contact Georgeson Shareholder Communications Canada Inc . , the Company 's proxy solicitation agent , toll free at 1-866-676-2882 . -The search giant tests Beijing 's appetite for democracy. as long as the new and old customers to buy the corresponding product on this site , both a gift , so stay tuned ! ! 1. sport shoes : Jordan , Nike , adidas , Puma , Gucci , LV , UGG , etc. including women shoes and kids shoes . -About the difference between # 3 and the classic Monty Hall problem . -Compared with a placebo group , women taking flibanserin for 24 weeks reported a statistically significant increase in the number of " satisfying sexual events " they experienced , recorded increased sexual desire in their diaries and had less distress about sex . -However , that doesn 't mean you wouldn 't want to be aware of all your choices -- and have the chance to make the most educated decision about your health care . -Umar Farouk Abdulmutalla , the Nigerian man charged with Friday 's bombing attempt , allegedly told authorities in Detroit that al-Qaida members in Yemen engineered the explosive device found sewn into his underwear , ABC News reported Sunday . -" You find it in coatings boosting the efficiency of auto engines , in protecting electronic devices and making cholesterol-reducing drugs more effective . -The Dow Jones industrials added 181.84 points , or 1.42 percent , to close at 12,980.88 . -The link between family structure and later success in life is established by empirical study . -It appears that no neighborhood is safer than the other . -To participate and ask questions during the conference call , dial the local country telephone number and the confirmation code 4312643 . The toll-free numbers are 888-708-5692 in the United States , 0 800 051 7166 in the United Kingdom , and 800 191 83 in Norway . Other international callers should dial 913-312-0962 ( tolls apply ) . -If our current system goes the way of the dinosaurs its bacause it is a 19th century system trying to meet 21st century problems . -The Post reported that Napper 's documentation as well as the observations of three other employees and a grieving son have sparked an investigation by the Virginia Board of Funeral Directors and Embalmers . -" The risk reductions associated with some whole grain foods and fiber provide general support for the hypothesis that whole grains are better than more refined and sweetened grains for pancreatic cancer prevention , " Chan said . -The trip starts with visits to the Pyramids of Giza and the mysterious Sphinx . -" The jetliner 's computer systems ultimately failed , and the plane broke apart likely in midair as it crashed into the Atlantic " - CBS News , no one knows what happened , so why do you print stuff like this ? -At the targeted sites on Thursday , worshippers said they were relieved that the plot was halted , but they didn 't intend to change their daily routines . -There is no going back , and the manufacturing jobs that have been lost are gone forever . -But her voice was clear . -As he prepared to take office , Kennedy was concerned with not only what he would say , but how he would look , said Stacey Bredhoff , the museum 's curator . -Mr Ban has just returned from a holiday in his home country . -But I think the two of us would have a great time in a car yelling at the idiots cutting us off . -The 5.7 percent decrease was more than economists had expected . -9 Ready to Wear : Is it fair to lambast such an innovative designer in this way ? -Poland gets over 90 % of its electricity from coal . -WASHINGTON - President Barack Obama and Brazilian President Luiz Inacio Lula da Silva discussed the economy , energy and the environment Saturday during their first White House meeting . -Golf No amount of noisy parakeets , even louder jets or sunbathing nudists nearby could distract Louis Oosthuizen from claiming his first European Tour title . -He left his collection to his friend , Paul Kelly who has put it up for auction . -Q. Doesn 't this just hit the very wealthy who can afford it ? -Also , Catholic school girl outfits . -He wrestled his cousins wearing funny masks worn by the locals . -Chrysler is in the process of seeking approval from a U.S. Bankruptcy Court judge to sell itself to a new entity jointly owned by Fiat , the United Auto Workers and the U.S. and Canadian governments . -The first three days were the most difficult , John Westhoff said , especially for his father , who once briefly left the hospital after an operation to grab a sandwich . -Guillermo del Toro has been confirmed as directing The Hobbit and its sequel . -The Planning Board serves as the council 's principal adviser on land-use and community planning , and oversees the county 's parks . -Investigators interviewed a Salvadoran immigrant who has denied any involvement in Levy 's disappearance and killing . -Produced by Universal Pictures , the remake will be set in New York . -The graph in Black 's article looks like the ice is melting in the Arctic at about the same rate as last year , but your graph show a clear lag this year . -It has long been assumed that President Vladimir Putin , whose term of office expires next March , would prefer to remain in power . -" Each hand is a new hand and I 've been really focused on staying in the moment even though my phone is ringing off the hook , the media have been all over me . -That means each dress must be cut separately . -It was set up in 1980 in the Netherlands and in 1995 opened for business in the UK with an office in Bristol , where Mercury Provident , an independent bank that merged with Triodos , was based . -The debate he has called for is needed , but the more urgent matter he unwittingly raises is whether our police chiefs should be elected , and hence whether Sir Ian should be invited to join the debate at all . -" The third-generation iPod shuffle is significantly smaller than a AA battery , holds up to 1,000 songs and is easier to use with all of the controls conveniently located on the earphone cord , " Apple said in a statement . -Nicola Sanders , runner-up to British team-mate Christine Ohuruogu in the 2007 World Championships , continues her comeback from injury . -I assure my fellow readers that Dr. Kass 's sentiments do no represent those of all Christians , just as the views of some who have commented here do not represent the views of all scientists or other concerned individuals . -" People ran out into the streets because of how prolonged the quake was . -1344 : As half time approaches , Mozambique come close to scoring the opening goal . -Unfortunately , the answer to that question is probably nothing . -The project should help protect a small group of homes at the south end of the property that are regularly flooded . -Diplomats in Washington believe that the Iranian leadership is hoping -- possibly in vain -- that the US election will signal a decisive shift in relations with the West . -The tenor of the argument suggested , however , that rather than simply decide the case in favor of the state , a majority of five justices would go further and rule that the challenge to the statute , the strictest voter-identification law in the country , was improperly brought in the first place . -They want to boost the personal income tax rate temporarily by up to 75 percent , pushing the current rate of 3 percent as high as 5.25 percent . -Nigeria , Africa 's most populous nation with 140 million people , is the continent 's largest supplier of peacekeepers and the country has spearheaded or hosted mediation efforts for many of Africa 's hot spots , including Liberia , Sierra Leone and Sudan 's Darfur , where rebels have battled government and allied militia forces for years . -The defense argues Mattis was improperly influenced by a military lawyer , Col. John Ewers , who investigated the killings of 24 Iraqis in Haditha in November 2005 and later became a top aide when the general was deciding who should be charged . -Peter Williams , chief executive of the Quality Assurance Agency ( QAA ) , says the current degree classification system is " arbitrary and unreliable . " -So here is Zuckerman 's exit sign : a necessary book , in the scheme of things , but not a very gratifying one , unless perhaps for its creator . -They say he also didn 't mention his past as a Hitler Youth movement member . -Or Mr. Mailer , who reeled in Joan Didion , Don DeLillo and Gay Talese ? -Harbhajan will miss next month 's triangular one-day series in Bangladesh after being handed a five-game ban . -But a regulation now called the " 90-10 " rule , which requires colleges to collect at least 10 percent of their revenue from nongovernment sources , drove hundreds of shady operations under and lowered the proportion . -Therefore , the minister said , natural disasters would continue to occur . -Raven 's Pass , a three-year-old chestnut ridden by Frankie Dettori , out muscled his rivals in the Santa Anita stretch to win by one-and-three-quarter lengths over Irish raider Henrythenavigator . -With a penchant for Americana , alternative folk , post-rock and dream pop , they have Beach House , the Low Anthem , the Dirty Three and Explosions in the Sky on their roster . -A nine-year-old boy and his 10-year-old sister remain critically ill in hospital after a crash in the Cotswolds , which claimed seven lives . -But the proposed package of middle-class tax cuts and spending on projects like roads and bridges may be a tough sell , even in a Democratic-led Congress . -25 , leading to speculation the court will rule on the results ' validity before then , the report said . -Details of the Montgomery plan are pending , county spokeswoman Esther Bowring said , but if the program mirrors others in use , it would work like this : When drivers arrive at a designated parking space , they would call the phone number on the meter and punch in the meter number , the amount it costs to park and a credit card number . -Family and friends have paid tribute to a heart expert and his GP wife from Barnsley who were killed in a motorbike accident during a holiday in Florida . -The reader is cautioned not to place undue reliance on these forward-looking statements , which are not guarantees of future performance . -They are not voting for the spouse and I doubt -- however many price tags are attached to their high-street clothes -- whether a loving partner will swing a single punter in the polling booth , but my every instinct in 2005 was to put private life on hold and give it my all . -Even the business gloomsters will eventually have to adjust . -It is quite helpful , too , when you need emergency medical care but , of course , the patient has probably paid for that in taxes . -The tweaked bill will return to the House . -But this latest leaflet campaign seems to have touched a particularly raw nerve . -The crash site of Continental Connnection Flight 3407 in Clarence Center , N.Y. Commuter plane crashes into home , 50 people killed . -Champion trainer Paul Nicholls has yet to taste success in the National and his main hopes are Big Fella Thanks , the mount of stable jockey and two-time National victor Ruby Walsh , and Tricky Trickster . -Besides Becker , the Obama administration appointments appear to be mostly technocratic , not ideological . -Pakistan captain Younis Khan said his team were lucky to escape the attack on Sri Lanka 's team bus Tuesday . -Log on to www.hoochymail.com , a free online service that allows couples to e-mail each other customised fantasies . -Brown , who made 32 of 40 field goals in the regular season and playoffs , would not mind getting the tag again , but he wants a lengthy contract . -The campaign attracted nearly 34,700 video entries from almost 200 countries , surpassing all expectations in promoting tourism in the Australian state . -For depression , the drugs are used to augment antidepressant therapy , but Simon said there are several other options , including switching to another antidepressant . -Yesterday , its Bertolli olive oil business was added to that list . -It is marketed as a plant food . -For Storrie , the proof will only come on the debt-repayment dates , when he asks al-Fahim for the money . -She has to watch herself and also watch Bill Clinton to make sure he is in line . -Visitors also learn about visual effects and the company that pioneered digital moviemaking . -That has prompted the Polish government to lead the battle against moves to reform European Union climate laws forcing its coal-burning utilities to pay for permits to continue emitting such large amounts of greenhouse gases . -" England would be a game of massive interest to the fans and while it is not in the bag yet , we are closing in on an agreement . -Mehsud 's aide Kafayat Ullah told The Associated Press a day earlier that Mehsud was killed with one of his two wives Wednesday in his stronghold in the South Waziristan tribal region . -The company 's quarterly profit plunged 64 % to $ 940m . -Dennis Quaid stars as grumpy English professor Lawrence Wetherhold , who has been generally disagreeable to just about everyone since the death of his wife several years earlier . -Martina Navratilova has revealed that she is being treated for breast cancer . -That is why we are working towards a bilateral trade target of £ 60 billion by 2010 . -He also tried to remove J. Edgar Hoover 's name from the FBI building after learning that Hoover had the FBI spy on a former Senate colleague . -Right now , airlines schedule more flights than the runways , terminals and air traffic control system can accommodate . -" That 's how I built my business and that 's the approach I 've brought to a city government that was insular , provincial and married to the conventional , " Bloomberg said , offering a hint of how he might wish to transform Washington . -In the absence of a time-sensitive problem , she said , the commissioners did not have much incentive to push a change through . -No , because it would mean admitting that some teachers need a kick up the backside and some need the sack -- a taboo in the unreal world of the NUT . -The baker , Lobels , said it had exhausted its 4,000-ton reserve of wheat flour and had but two days ' supply left . -" The very idea of competing nation-states that scramble for markets , power and resources will become passé , " he writes , introducing a reasoned plea for one-worldism . -The researchers based the stud book on a review of the handwritten zoo records in Rabat kept from 1969 to 1998 , plus a detailed review of breeding records across zoos worldwide kept from 1974 onwards . -National Express said engineers would be working overnight on Friday . -It could have been 20 minutes or an hour . -China has 24 reactors under construction with total generating capacity of 25.4 gigawatts , he said . -It was NOT meant as a put down to him , but saying that what it represents ( this collection was 50s based ) is a woman who was of HER time , not this one . -While that philosophy still exists to some extent , more commanders in today 's military are trained to be able to spot mental problems , said Brown , who speaks on the issue for the Iraq and Afghanistan Veterans of America . -So civilian suffering and loss of life continues , and the chances of any kind of political settlement recede . -Click here to read more from The New York Times . -Now if I try to sell it I will lose a lot of money , " Li said . -America is now likely to play a major role in Europe 's 2016 ExoMars mission . -Ultimate All-Star Interactive Luncheon -- Step into the kitchen and play sous-chef to an all-star lineup of Wynn Las Vegas and Encore Las Vegas chefs at this incredible interactive luncheon at Wynn | Encore with Paul Bartolotta , Frederic Robert , Alex Stratta and David Walzog . -Longley said she shudders to think what could have happened if there were complications during the birth . -Search-and-rescue operations were to continue at least through the weekend . -Hey , it 's all professional . -Beijing and Shanghai-based gold industry analysts said the country had almost doubled its bullion holdings . -" The policy of the Department of Defense is to treat prisoners humanely , and those who have violated that policy have been investigated and disciplined , " he said . -It will be fun to watch how this develops . -The president also was to visit the Berkeley Plantation in Charles City County , Va . , where it is claimed the first Thanksgiving took place Dec . -Excluding the aforementioned adjustments , on a non-GAAP basis for the three months ended June 30 , 2009 , PAREXEL 's consolidated service revenue would have been $ 268.4 million , operating income would have been $ 35.5 million , taxes would have been $ 10.9 million , net income would have been approximately $ 15.6 million , and earnings per diluted share would have been approximately $ 0.27 . -On the plus side , Sheridan coaxes two genuinely astounding performances out of the child actors Bailee Madison and Mare Winningham , playing Sam 's daughters . -" A motorcyclist riding a green-coloured Kawasaki motorbike and a pillion passenger were involved in a collision with a number of other vehicles , some of which may not be aware and have left the scene , " the spokesman said . -His spokesman Father Michael Canny said he felt " it did not go far enough to condemn violence and people who propose it to be a legitimate means . " -At the time Hunter Biden was receiving consulting payments from MBNA , he also was a Washington lobbyist at a firm he had co-founded . -In a blog post on Thursday , Icahn had downplayed the annual meeting 's importance and said he would not attend . -Like a trainer who promises a washboard stomach in 60 days , Mr. Mount throws out a short Latin inscription -- the epitaph on the tomb of the great Renaissance humanist Leonardo Bruni -- and swears that anyone who makes it through his little books will be able to translate it at the end . -Matt Cullen got the lone goal for Ottawa . -Los Angeles County Fire Captain Mark Savage says Santa Ana winds spurred the blaze to jump a fireline early Monday and burn toward the freeway and populated areas , most of which have already been evacuated . -It was last updated at 10.28 BST on Wednesday 15 July 2009. morgan hunt. a vacancy for an experienced geography lecturer ha ... . £ 24500 - £ 35000 per annum. bedford borough council. bedford . £ 30,546 - £ 33,328 pa + Essential Car User Allowance. badenoch & clark. badenoch & clark is looking to recruit an exec ... . £ 600 - £ 900 per day . -Many of them are working in Kaohsiung County , a mountainous farming region in southern Taiwan . -He has avoided any public scolding of his hosts . -Get Me Out of Here ! experience reminded her of Peter Andre . -Make a couple of calls ! -In a second round which saw one major upset - French favourite Gregory Bauge being pushed into the repechages by diminutive but wily Malaysian Azizulhasni Awang - Britain 's Matthew Crampton also qualified directly for the quarters . -NEW YORK - Madonna 's love letters and erotic answering machine messages to an ex-boyfriend are up for sale in New York City . -On Tuesday , former workers will gather outside 10 Downing Street and the Senedd ahead of a review by actuary Andrew Young to see if financial help for people who lost their pensions could be increased . -" I 'm glad I could do that for my team-mates especially for how hard they 've worked in the last few days , " added the Team Columbia rider . -The breaking of the embargo , a rare agreement in Britain 's usually free-for-all media environment , infuriated the military . -The Town Council has been under fire before . -Last month , a suicide bomber blew up a vehicle outside the main gate of a military base used by North Atlantic Treaty Organization troops near the Kabul airport , killing several Afghans . -Johan Franzen then had a chance to continue the shootout , but his shot was turned aside by Mason , who had 26 saves in regulation and overtime . -But the results are likely to solidify in investors ' minds ( and probably those of regulators and policymakers as well ) the notion that GS is truly different from other markets-oriented banks . -But one leading political analyst doubted Bloomberg will run because he in all likelihood cannot win , noting there is no precedent for a successful independent campaign and suggesting Bloomberg might simply siphon enough votes from the Democrats to hand the election to the Republican nominee . -J Ren Nutr 2007 ; 17 ( 6 ) : 372-380 . -The IPL governing council -- basically Modi in another name -- has increased its number of overseas players to ten per team . -But the Palestinians demanded Israel first commit itself to ceasing all settlement activity , as called for under the long-stalled " road map " peace plan . -Van Commenee said : " It would be silly as a foreigner to come in and dictate and I won 't shut my eyes to the great history of British athletics . -The sanctions , imposed for alleged human rights violations , electoral fraud and suppression of political opposition , include travel bans and the freezing of assets for Mugabe and his allies . -Fifteen out of 30 DJIA stocks were down in midmorning trading , but overall index was up 44.89 points , 0.42 percent , to 10,654.54 . -" I 'm here because this is the only festival in which you can declare your Russianness . -" Birds provide an accurate and easy-to-read environmental barometer , allowing us to see clearly the pressures our current way of life are putting on the world 's biodiversity , " said Mike Rands , chief executive of the alliance of conservation groups . -STAR spokeswoman Jannie Poon said Fox STAR Studios will operate from several locations , with its first unit based in India 's Mumbai , home of the Bollywood industry . -The stimulus will provide some relief in the second and third quarters , but will not follow through to 2009 , it added . -At 317 ¼ p , the shares sit 5 per cent above NAV and offer a 6.4 per cent yield . -Commodities were mixed as the US oil price remained below $ 50 a barrel amid lingering fears about the US car industry , but copper climbed 3.7 per cent to regain $ 4,000 a tonne and gold rose modestly . -" We 've covered most of the aspects of human life , " he said . -I wonder why ? ? ? ? ? ? ? -On Monday , Fame 's share price hit its permitted daily ceiling on Indian bourses , having already tripled this year . -If they get up a head of steam , opponents have trouble bringing them back to within striking distance . -She recently had two shows of her visual art in L.A. galleries , much of it assembled collage-style with disparate images and ideas colliding , as Picasso also did in his Cubist period . -But the Center for Strategic Research , led by former nuclear negotiator Hasan Rowhani , said Ahmadinejad has attempted to downplay the role of his predecessors in developing Iran 's nuclear program , which was started in the 1980s under former Prime Minister Mir Hossein Mousavi , the president 's leading election challenger . -The photo finish in Minnesota 's Senate race came after months of intense campaigning and millions of dollars in ad spending . -So many newcomers for Colorado and the wear and tear became more evident as the season began to grind the kiddie corps down . -Like many people , you 're possibly afraid to share your views on animal experiments , because you don 't want anyone digging up your grandmother 's grave , or setting fire to your house , or stuff like that . -We were even more excited when we went to the site where Dr. Guillotine--yes the very inventor of the grisly machine--had his shop . -Colin Gant , 41 , of Lowestoft , who drives more than 500 miles a week in his job , was forced to take detours to avoid the detection devices . -The good roads take you toward and away from it , making you play a game of topographical peekaboo . -The results are unlikely to be known until next week and will determine the next stage of the investigation . -France and the UK want similar one-off taxes to be adopted across the European Union . -In 2008 , Oxycodone had U.S. sales of approximately U.S. $ 2 billion . -At Chelsea , similar tactics were not needed . -The recommendations by the all-party Parliamentary Football Group would , if implemented , prevent takeovers similar to those of the Glazer family at Manchester United and Tom Hicks and George Gillett at Liverpool . -The unwritten rule is to leave the lavatory as clean as you found it , which means at least wiping out the sink with a paper towel . -" I don 't think I barely resemble myself to myself or to my family , " said Kay Warren . -There are only 1,460 pairs of eight-eyes and 1,461 pairs of three-eyes available globally , per color . -One of the biggest businesses in sport , announced Roger Goodell , league commissioner , was bucking the economic recession . -In a scrappy opening to the game , Chris Baird made the first attempt on goal after eight minutes , striking a low shot from outside the area which Al Habsi got down to his left to save comfortably . -And they understand that freedom really works . -Shannon was last seen on 19 February on her way home from Westmoor Junior School in Dewsbury , West Yorkshire . -Hazelton 's home took 10 weeks to build . -Some of the largest mortgage servicers are scrambling to make the most of this shift . -But in recent months , the group has made a point of branching out beyond music copyrights . -Analysis revealed that both his parents originally came from the Windward Islands in the Caribbean . -Every close line call went against her , including a serve that looked wide but was called good and gave Safina match point . -I think that the only way to rectify the AIG mess is to change its present management and replace them with trustworthy competent people because no one is indispensible . -Sereno also unearthed the Nigersaurus , a plant-eating dinosaur with a huge jaw studded with 500 teeth that lived in the same geologic period , the Cretaceous . -European runner Man of Iron got the upset-filled Championships off to a minor surprise by winning the $ 500,000 Marathon by a nose over 9-year-old Cloudy 's Knight . -Dr Bekinschtein , who began the work as a PhD student in Argentina , said they also found that patients who started to anticipate the puff of air were more likely to show signs of recovery later on , in terms of increased ability to communicate . -But another national teachers union and a group that has supported the law 's goals of holding schools accountable for student progress criticized the proposal . -Three high-profile late withdrawals have , however , taken the shine off some events . -The " right direction " number is up 8 points since February and a remarkable 31 points since October , the month before Obama 's election . -The point is not that the polls were " wrong , " but that people weren 't honest with the pollsters . -" A lot of this has to do with communication , " says Mahesh Rangarajan , a political analyst and professor of history at Delhi University . -You should consider the letter of instruction as a master list , detailing where you keep your financial documents . -His ability to anticipate the location of Karlovic 's serves improved as the game went on . -Moreover , Looking For Eric boasts the most rousing finale of any Loach film . -I put my mother in the car . -Friday and took off his boots , probably so he wouldn 't make any noise while sneaking around , police said . -One of the names they gave out was Paul Farrelly , MP for Newcastle-under-Lyme , who said yesterday that he had neither signed , nor even seen , any letter calling on Mr Brown to resign . -Messages left at a telephone listing for a Karen Robbins in Clinton and with the county public defender 's office were not immediately returned . -The job is a Treasury appointment , although Gordon Brown is taking an interest . -This time , Webb 's throw beat the runner but it could not be held by Tracy , allowing the runners to move up . -A Beijing police spokesman confirmed to AFP an American tourist had been killed and the other two had been injured , but gave no further details . -The premise is that the hospital cannot succeed without a successful city . -Everybody KNOWS ! ! ! -The new subfamily , one of 21 ant subfamilies , is the first new one to be named by scientists since 1967 . -Minnesota legislative leaders have indicated that any stadium-financing assistance would be addressed in 2009 , said William Lester , the executive director of the Metropolitan Sports Facilities Commission , which owns and operates the Metrodome , the longtime home of the Vikings and the Minnesota Twins . -However , nine starters on defense are expected to be back next season , including sophomore cornerback Asher Allen , who had two interceptions , and freshman linebacker Rennie Curran , who had a pair of sacks . -Yinka Saunders , 24 , admitted stealing the cash to pay off £ 40,000 credit-card debts run up by her partner . -From a classified church to a seventies concrete community centre , Norway is developing new ways of renovating buildings saving both money and the planet . -KBR moved to have the claim heard in arbitration as Jonesʼ contract provided . -Nice. if he had a legitimate film career then he wouldn 't have to do those lowly tv ads just for the money . -There were times , White admitted , when he got nervous . -Moscow retains full control of Abkhazia and South Ossetia and says it has the right to establish an buffer zone beyond taking in stretches of the highway linking Tbilisi to Poti . -It 's probably the most powerful development since the Industrial Revolution , but I was a little slow to see it coming . -Neil Hendy , the Bhs design director , left the business to join Marks & Spencer in April last year , and Beverley Churchill , the marketing director , left in October 2006 after only a year in the job . -Obama has a personal interest in both sites . -A visit to Martin O 'Neill 's side , despite their Boxing Day humbling at the Emirates , is hardly the ideal fixture for Benítez , though , given his side 's fragile confidence . -Sophomore Tiana Myers scored 16 points and sophomore Sara Mostafa posted her second straight double-double to lead the Colonials . -By 11 : 30 p.m. , Sen. John McCain , the Republican front-runner , has won seven states ( his home state Arizona , Connecticut , Delaware , Illinois , New Jersey , New York and Oklahoma ) , former Gov. Mike Huckabee has won four states ( his home state of Arkansas , Alabama , Georgia and West Virginia ) and former Gov. Mitt Romney has also won four states ( his home state Massachusetts , Minnesota , North Dakota and Utah ) . -My mother does not even have the strength to come to court . -" I 'm not a performer , " he said . -8 ( UPI ) -- The U.S. game show , " Deal or No Deal , " will go into national syndication next fall , NBC Universal Domestic Television Distribution said Monday . -( 28 ) " Castle , " ABC , 10.96 million viewers . -Asked whether the incident-ridden game might have been his last in charge , Domenech said his only immediate plans were to marry his partner , television journalist Estelle Denis . -I would recommend that you choose your surgeon carefully and do your research on that Doctor . -His accuser , Brian McNamee , brought two photos showing syringes and vials and even a crumpled beer can in a bid to bolster his side of the story . -Pledging to uphold The Lessig Principles as specified in the ccMixter.org RFP , start-up ATM is taking stewardship of the Top-5 Free Music Site . -Bearded & Tschorn : The Times ' coverage ... -NEW YORK ( Reuters Health ) - People with cancers of the head or neck seem to have better survival odds if they have private health insurance , research hints . -Wasps made a bright start and Mark van Gisbergen scored on five minutes . -The baby will be his eighth . -United Nations envoy Ad Melkert said the elections were well-organized and orderly , the Daily Telegraph reported . -Now Dennis , who stood down as team principal in favour of Martin Whitmarsh earlier this year , fears the title is already out of Hamilton 's reach . -It 's the American dream . -Google met Mr Thür in the run-up to launching Street View in Switzerland in September , and was initially given the green light . -And trust me , when this finally evens out years from now , the average American will have amnesia and make the same bad decisions . -He won everything in his career and the only thing lacking was the world record . -" You have to look to where we 're going to be , " Gore urged the Bali delegates . -Browsing through Ibis 's small list is like wandering into a literary cafe filled with a vital spirit of intellectual engagement . -GM has begun a search for a permanent CEO although Whitacre has told directors he is prepared to see the automaker through the next stage of its restructuring and possibly through June when it is due to pay off $ 6.7 billion in U.S. government loans . -PEMBROKE PINES , Fla . , Aug . -organization of its kind in the country. improves the health of Minnesotans by reducing the harm caused by tobacco. with 3 percent of the state 's 1998 tobacco settlement . -The research from the International Institute for Environment and Development ( IIED ) , a non-profit London-based think tank , challenges the common perception in the developed world that waves of refugees will try to move there permanently to escape the impact of global warming . -And going . -Revisions to the back data saw the Q4 year-on-year GDP growth revised to -3.3 per cent from -3.2 per cent . -Bond yields were mixed , with the yield on the two-year Treasury note rising 1 basis point to 0.955 per cent and on the 10-year Treasury note losing 1bp to 2.874 per cent . -He estimated that he could process 5 billion coins annually , separating out 1.2 billion copper pennies . -Rude people who just care about themselves and everyone else be damned . -Descent International had been struggling for months but continued to take deposits as it attempted to finalise a rescue plan . -There have already been a number of protests by haulage companies in Wales and the UK over fuel costs . -" This is the worst tragedy since the start of the Millennium , " said Guido Bertolaso , the head of Italy 's Civil Protection Department . -Associated Press writers Gary D. Robertson , Estes Thompson and Martha Waggoner in Raleigh ; Page Ivey , Susanne M. Schafer and Jim Davenport in Columbia , S.C. ; Bruce Smith in Charleston , S.C. ; and Karen Testa in Boston contributed to this report . -Following domestic and international pressure , the station was reopened . -Football has been a way out for dozens of young men from this sugarcane farming country along Lake Okeechobee , a place so poor that many people can 't afford cars and merely hop rides on pickup trucks around town . -After Walthamstow closes , only Wimbledon in south London and Crayford and Romford , on the city 's fringes , will remain in the capital . -The Leicester forward suffered the injury during England 's defeat by Wales in the opening match of this year 's Six Nations tournament and a consultation with a Swedish specialist has confirmed his worst fears . -China and India have a perfect right to retore original names of their cities and not retain the distorted names the English and other westerners gave who were unable to pronounce Asian names . -" The fact of doing a guy who , when he was 40 , had an affair with a 12-year-old . . . that was the challenge that as an actor you kind of gravitate toward . -The important thing is to get it right , not to get stuck on some specific day or some specific time , " Conrad said . -She opened with an 11-stroke victory in Singapore , defended her Safeway International title with a seven-stroke romp , won the season 's first major by five shots and then led from start to finish last week in Mexico and won by 11 . -The technology would also be available to iPod Touch users , although they would have to buy a microphone and headphones to make calls , PC World reported . -" I don 't usually bet , only the odd pound here and there . -Dominic Grieve , shadow justice secretary , said : " The public will rightly be shocked to learn that not only are the Government failing to deport foreign prisoners but taxpayers are also paying the price for the Government 's incompetence . " -" As interest rates come down , banks will want to lend more to consumers , so you will see a pick-up in loan growth in emerging markets , " he adds . -Much of the damage came when massive storage tanks , stainless steel vats more than 15 feet high , toppled . -A number of the dealerships have already closed . -Oakland is trying to return to normal after a horrendous and atypical bout of black-white racial tension . -The terrier had to be rescued from the jaws of the bull mastiff by both dog owners and was uninjured . -" I don 't think I want to see our party go back to that kind of campaigning , " Romney said in his most pointed rebuttal yet to front-runner McCain 's claim that the former Massachusetts governor favors a timetable for withdrawing troops from Iraq . -Hundreds of research studies have shown that mantras , prayer and meditation techniques can successfully lower blood pressure , help decrease chronic pain , anxiety , digestive problems and ease emotional problems . -Evan Longoria and Carlos Pena had solo home runs for the Rays , who took a 2-1 lead in the best-of-seven series with the four-homer attack . -BUSH : And that 's where we are today . -A colder feeling day with temperatures of eight Celsius . -For the Welsh National Opera ( 1967 ) he sang Don Pasquale and for Glyndebourne Touring Opera ( 1968 ) Dr Dulcamara in L 'Elisir D 'Amore . -In this case hardship was ameliorated by a hope adapted from the biblical dream of Zion , that someday blacks might return to a land from which they were exiled : Ethiopia . -She said she heard male TSA agents snickering as she took out the ring . -Chicago Police commander John Lewin runs the information services department . -Now Cox-Foster says she and colleagues are trying to reproduce CCD 's effects on bee colonies by seeding healthy hives with the agents -- the biomedical equivalent of getting a killer to confess . -Schools also receive 20 cents a lunch to spend on government-issued food . -A 12-year-old girl had returned to Paignton Community and Sports College for two days following a trip to Mexico , unaware she had the virus . -To date , they have preferred breezy slogans . -Obama was asked , twice , whether the Bush administration engaged in torture , which would be illegal . -Department of Homeland Security Secretary Janet Napolitano will announce the nomination Monday with Harding by her side , according to one administration official . -She told officers that Rodriguez had been taken by ambulance to a hospital for a mental evaluation a few days earlier but left without being discharged . -But the item always eventually turned up . -One hedge fund reckons they will provide 70 % of profits in the long run . -He found a conspicuous absence of black comedy shows , so he started plotting how to fill the sketch-comedy void left by Dave Chappelle 's abrupt exit from series TV in early 2006 . -Profile America -- Saturday , November 14th . Each year , November marks National Diabetes Month . The goal is to make the public aware of the serious nature of the disease , and how to detect and control it . When our bodies are unable to maintain a normal blood sugar level , many complications may follow , including kidney failure . The disease is also the leading cause of new cases of blindness . Diabetes in the U.S. is on the rise , and some public health experts even refer to it as an epidemic , fueled by our overweight and inactive lifestyle . Across the country , more than 75,000 people a year die from complications of the disease . You can find these and more facts about America from the U.S. Census Bureau online at http : / / www.census.gov. -Those toxic assets , according to TIME 's calculations , come to around $ 37 billion on U.S. bank books . -Sowell is charged with attempted murder , and two counts each of Rape , Kidnapping and Felonious Assault . -Abortion is legal up to 22 to 24 weeks in Spain , Switzerland and the Netherlands . -His next project , Smith House , Vancouver ( 1966 ) , for the artist Gordon Smith , was a stripped-back cubist structure of glass and timber built into a hillside and strongly-evoking Frank Lloyd Wright 's masterpiece , Falling Water in Pennslyvannia ( 1934 ) . -Ben Bernanke -- for it was he , of course -- has found himself in an even more privileged position to learn such lessons . -Many of them , supporters of the change pointed out , were in traditionally conservative states such as Alabama , Georgia , North Carolina and Tennessee . -BRIC nations represent about 40 percent of the world 's population and 20 percent of global economic output but , as a group , they are just beginning to flex their muscle , after the first summit of the four in Russia last year . -In addition , the company 's former outside lawyer from Mayer Brown , Joseph P. Collins , received a seven-year sentence after a jury convicted him of helping to hide the debts from investors . -But investigators say Abbott had no intentions of staying in jail for long . -He was referring to his greatest asset as a fast bowler : his speed . -Large volumes are stored in cylinders of underground storage facilities . -But she had a double bogey on her 13th hole of the day after she hit her second shot over the green into heavy rough on the 518-yard , par-five No. 4 . -Hey moron .... if we went into Iraq for the oil ..... tell me why all the oil being pumped is still going to the same places it was before we invaded ? -The truck pulled over and the driver and a passenger got out . -When Long entered the room to take orders from the group , she did a visible double-take upon seeing Noonan . -The singer has been in the headlines all year after reports of drug use , rehabilitation treatment and cancelled performances - as well as awards success and a best-selling album . -But many feminists and civil rights leaders cite a long history of separate and unequal education for girls , and argue that segregation will perpetuate damaging stereotypes . -He 's super supportive : He 's got a career and is secure in his work life , so he 's totally supportive of your ambitions . -In the final weeks of the campaign , the Arizona senator struck a more negative tone and , along with vice presidential running mate Gov. Sarah Palin of Alaska , began attacking Obama 's relationships and judgment . -The Kings fell behind 3-0 in the first period against a longtime antagonist . -Tony Hillerman was one of the great novelists of the American South West , using the medium of the modern detective novel to explore traditional Native American beliefs , including notions of witches , ghosts and " skinwalkers " or shape-shifters , and the continued survival of such beliefs and practices within the borders of a modern international superpower . -One good effect of high gasoline prices ? -The yield on the 10-year Treasury bond dropped to 2.919 percent from 3.041 percent Friday and that on the 30-year bond eased to 3.649 percent from 3.722 percent . -It withdrew medical and engineering military units from Afghanistan last year . -But Russo insisted that " a new meeting must take place . " -And then another one . -Osher and Rami were hit in the street . -Detectives said they want to hear from anyone who was in the area at the time . -Sharif , then in his second stint as prime minister , refused to let an airliner carrying his dismissed military chief , Gen. Pervez Musharraf , and 200 other passengers land in Pakistan . -Leave us guys out ! -After Barnes gave the Suns a 113-109 advantage by making two free throws , Houston missed two shots and committed a traveling violation to wipe out its hopes . -Among Chinese developers listed in Hong Kong , China Overseas Land dropped 5.6 percent , Shimao Property fell 7.3 percent and Guangzhou R & F Properties tumbled 11.4 percent . -The uncharitable view would be that , guided by an acute understanding of the nature of commerce , servers are told by restaurant managers to hustle clients through a meal and as many bottles of wine as possible . -It was not possible to confirm casualty figures because of the remote locations of the fighting and the dangers of traveling there . -Industrial demand for silver , used in gizmos such as mobile telephones , is pegged to economic recovery in manufacturing economies such as Japan , the second largest consumer of Mexican silver after the US . -The existing sea route eastwards round the Cape of Good Hope , to India , China and the Spice Islands with their rich resources of pepper , cinnamon and nutmeg , was long and dangerous . -It is just like the tortured media to make anything less than a Dem sweep in next year 's elections a failure on the part of the president and a repudiation of his policies . -Iran is accused of pursuing a weapons program under the guise of peaceful nuclear power and Syria has recently come under fire for allegedly building a covert nuclear facility . -The museum has lost an estimated 15,000 pieces . -While disorganized Republicans and major health-care companies wait for President Obama and Democratic leaders to reveal the details of their plan before criticizing it , Scott is using $ 5 million of his own money and up to $ 15 million more from supporters to try to build resistance to any government-run program . -The forum also wants suicide prevention in drug users to be made a key priority , pointing out that approximately 23 % of drug-related deaths were either intentional or of undetermined intent . -( CBS ) If you lost a bag at the aiport in Phoenix recently - it may have been stolen . -The Met Office spokesman added : " For winds like that you need a showery day . -King has made peace with the possibility that his rivals might go home with his money . -Today they can quietly communicate with each other by text messaging through their cell phones . -The discovery near Alness last Tuesday by a member of the public shocked animal cruelty prevention officers . -Severe weather conditions are adding to the difficulties and police say there is no parking for drivers who are being turned away from the tunnel . -But there had been little publicity of the strike before Tuesday , and morning traffic was typically heavy in the capital as stores and banks opened as usual . -The new rules aim to make it more difficult for potential terrorists to enter the United States from places such as France , Germany , Switzerland , Britain , Belgium , Portugal , Spain , Singapore , New Zealand , Japan and Australia , the government said . -The Board of Regents was set to approve Hauck 's contract in an afternoon meeting , less than one week after he coached the Grizzlies in the Football Championship Subdivision title game . -The Beijing museums to offer free admission include public museums and memorial halls run by municipal or district cultural and heritage protection departments , as well as patriotic educational bases , Xinhua said . -The review could take up to a year and likely would include an examination of the CIA 's detention program and secret prisons , the aide said . -He added that the talks between the two sides would tackle the Sudanese-Ethiopian relations as well as the relations between the Government of Southern Sudan and the Ethiopian government besides the progress of implementation of the Comprehensive Peace Agreement ( CPA ) and the positive developments in this regard in light of the meeting of the Presidency Tuesday night in Khartoum .. -Furthermore , Mr. Dimitroff added , " we are also following closely the current political dialogue regarding the election reforms and the discussions taking place on the formation of the Supreme Commission for Elections and Referendum ( SCER ) . " -" The main drivers were a stronger than expected performance in agrochemicals , as well as cost cutting measures in the chemical segments , " Commerzbank Equity Research analyst Stephan Kippe said in research report . -Nearly 59 percent or 77 individuals reported they never experienced it . -At many of its 18 stops , the relay has been beset by protests against China 's human rights record and a recent crackdown in Tibet following anti-government riots . -" Often when you move anywhere , there is the ' allergic honeymoon , ' which is a respite for a few years , until people become allergic to the new things they 're breathing " and develop runny noses and other actual symptoms . -It is unlikely to crumble like Netscape , the most recent start-up to vie to become a new platform , since hypervisors are much harder to replace than browsers . -Three years later , she not only takes care of her own children , but she also teaches preschool . -The index takes a random sample of species from a taxonomic group to calculate the trends in extinction risk within that group . -SAN FRANCISCO - Yahoo Inc. is setting up a new chain of command amid the turmoil triggered by the embattled Internet icon 's snub of Microsoft Corp. ' s $ 47.5 billion takeover bid . -437 . -" I 'm a man who strives to achieve his expectations , " Kuby said of his modest goal . -It is not clear whether the government could delay access to a lawyer for military detainees within the United States . -If the discount persists IndoChina intends to hold another tender offer next summer - again for 20 per cent . -It was a Sunday , with a ball of sun blazing above and little birds peeping on branches to signal the onrushing spring . -Brook says he acted in self-defense to protect his child . -It can also reduce the perverse incentive for homeowners to go into default to get a better mortgage deal . -But an expert who tracks evangelical Christianity , Larry Eskridge , said few are addressing the subject as directly as Daystar . -MPC Steamship was founded in 2001 and now manages 17 vessels . -The move would be a huge step toward ratifying the deal , which sets out conditions for U.S. military conduct in Iraq as well as a timeline for troops ' withdrawal from the country by the end of 2011 . -Besides having a three-year head start , Gardasil also defends against two more HPV types that cause 90 percent of genital warts , which Cervarix does not target . -There is another key Catholic notion here . -Jason Fogelson , who writes about motorcycling for youmustbetrippin.com , recommends a ramble from New York to New Hope , Pa . , on the Delaware River , where bikers have long congregated on weekends . -A lot of names date and really identify someone as being from a certain generation . -The project would replace a vacant Teledyne building , three low-slung brick office buildings and a large asphalt parking lot . -A top diplomat said Thursday he is heading to Honduras to demand the return of the president toppled at gunpoint -- a mission he said is likely to meet rejection , bringing diplomatic and economic punishment for the impoverished Central American nation . -Investigators probing the loss of Air France flight 447 hope they may yet explain one of the most baffling air crashes of recent times . -But last year the legislature rejected a bill requiring background checks for private sales at gun shows and repealed a law that Mr. Kaine had supported to prohibit anyone from carrying concealed weapons into a club or restaurant where alcohol is served . -Spokeswoman Kendra Barkoff said the Interior Department will follow Congress 's directive and put the new firearms law in effect in late February . -( AP ) - A woman who befriended a Florida lottery winner who later went missing was charged Tuesday with trying to conceal his death , five days after his body was found buried in her backyard . -Everyone would win , including the hard-pressed NHS . -The more than 40,000 Pennsylvania farms located within the watershed discharge 46 percent of the nitrogen and 58 percent of the phosphorus into these waterways and , consequently , farmers today face more stringent water quality requirements . -For tickets and accommodations , this is shaping up to be a particularly challenging Olympics . -In 2001 , he looked set to become one of the decade 's biggest stars . -A visit to the city 's jam-packed Yu Yuan Garden gave her an authentic taste of Shanghainese " xiaolongbao " juicy steam buns and flower tea and the crowding common in the city of more than 20 million people . -The shoes are a reminder to appreciate people 's differences . � � The National Christmas Tree was lit last night on the Ellipse behind the White House . � � More seasonal cheer : There will be a free family concert and singalong Saturday from noon to 2 p.m. at the U.S. Navy Memorial , 701 Pennsylvania Ave . -The property is in Alpine Meadows , a residential neighborhood with an even mix of full-time residents and vacationers , according to the listing agent . -If this is true then all of the government 's targets for reducing our CO2 emissions will make no discernible difference to the planet . -Like the objects she feels compelled to create , Helen is herself posited here as an exotic work of art , one prone to be cherished and misinterpreted . -In another burst of violence , police said at least 30 people died in clashes between Sunni and Shiite Muslims in Parachinar , a town near the Afghan border . -But local officials have not confirmed a link to the scandal . -At a time when so many competent and committed workers are losing their jobs , or are being asked to take a pay cut due to a financial crisis brought about largely by the reckless behaviour of crass bankers like the head of Barclays , the exodus of these people is more than welcome news . -Spain could play on the counter and that was important . -Wexler , a Democrat , has represented parts of Palm Beach and Broward counties since 1997 . -This simply can 't be true . -" I 'd have a little smile on my face , " Matta said . -They hope the scientific meeting , which ends on Friday , will highlight the range of measures needed , such as agreements that covered the entire catchment areas of the wetlands . -Marquette appears to be hitting its stride after struggling past Seton Hall in the opening round . -Another part of AMerican comedy ( or tragedy ) . -With goals by Brad Stuart and Tomas Holmstrom in the first period and another strong defensive effort on Monday night , the Detroit Red Wings beat the visiting Pittsburgh Penguins , 3-0 , to take a 2-0 lead in the Stanley Cup finals . -He had to create his own because the concept of rehabilitating fighting dogs is so unusual . -" I 'm trying to become a better person each and every day , " Woods said . -PITTSBURGH ( Reuters ) - The Pittsburgh Penguins appointed Dan Bylsma as head coach on Tuesday after his successful two-month stint as interim coach . -The proposals were " a jumble of national wish lists " and there should be real " European projects " instead , he added . -Excluding performance-related payments , costs declined £ 1.1m. -Usually , the awards , which were founded in 1996 , recognize individual Web sites , online ads or film and video . -Sri Lanka has many deep-seated problems in need of attention after decades of war against minority Tamil separatists , including resettlement of the estimated 100,000 Tamils still in internment camps , political and legal reform , economic development , and ethnic and political reconciliation . -COPENHAGEN , Denmark - Denmark 's National Gallery says it will exhibit some 100 works by Bob Dylan , including some of his newest acrylic paintings . -And he feels that he has been ' hung out to dry ' over the situation , having suffered a reaction to his knee problem after scoring the winner for Manchester City on Saturday in their friendly with Celtic at Eastlands . -SNP Westminster leader , Angus Robertson , called on the MoD to explain how it was that two subs carrying weapons of mass destruction could possibly have collided " in the middle of the world 's second-largest ocean . " -When an entry-level middle-class home in major markets on either coast lists at $ 500,000 and pays out a 6 percent sales commission , it is easy to see how an agent could get excited . -" Given the circumstances of the last 10 years and our attempts to give assistance in Zimbabwe , which have been thwarted and resisted , it is not possible for us to attend this summit and sit down with President Mugabe , " Brown said in London Tuesday . -Antonucci and others " repeatedly singled out Antonucci 's $ 6.5 million capital investment as evidence that the bank was viable and deserving of TARP funds , " it said . -( CBS ) Tom Engelhardt , co-founder of the American Empire Project , runs the Nation Institute 's TomDispatch.com. -At an event Friday in Barranquilla , Uribe said he would respect the ruling and urged that his successor keep the nation secure . -Property managers are under orders to open up their houses and make them feel unique and lived-in , while visitors will be treated as house guests instead of paying customers . -They are usually low earners , with wages up to £ 25,000 a year , and with either bad or no credit histories . -" Twenty-five percent of the combined white-collar and supplemental workforce " positions will be eliminated , Chrysler spokesman David Elshoff told AFP . -Some 40 tonnes of transformer oil leaked into the river after the explosion , killing fish and raising fears of chemical pollution . -Hunter 's best season was with Philadelphia in 2006-07 when he averaged 6.5 points and 4.8 rebounds starting 41 games . -Rowling , whose mother had multiple sclerosis , is also a patron of MS Scotland . -The case dates back to the late 1970s , when the Justice Department accused Demjanjuk of being a Nazi guard known as " Ivan the Terrible . " -The other two were diagnosed after 52 weeks and 104 weeks of treatment . -Other than the distortion to fit Stewie 's head , the helmet is pretty much the same . -A quick way to cut costs would be to reduce personnel , one analyst suggesting that the total size of the Army , Royal Navy and Royal Air Force will have to fall by more than 30,000 to 142,000 in six years . -Nuremberg nearly soured the start of Munich 's Oktoberfest after Maxim Choupo-Moting evened the score following Ivica Olic 's goal for Bayern . -The BBC 's Mame Less Camara in Dakar says some of those arrested are Mr Senghor 's security guards . -I received a generous gift certificate to Gramercy Tavern , one of my favorite restaurants . -The misdemeanor charge she pleaded to carried a maximum of six months in prison . -The public has almost forgotten the case is still going on . -Corruption at every level of Afghan society has undermined the population 's confidence in the government -- confidence Washington says is sorely needed before the Taliban can be defeated . -Former colleagues say that he is a supporter of a Tobin Tax -- a levy on financial transactions -- which Britain opposed for years until a policy about-turn this summer by Gordon Brown . -But approval ratings have rarely been a good predictor of presidential success . -Sven-Göran Eriksson has reacted angrily to accusations from England players that a " circus " engulfed the team 's appearance in the last World Cup finals in Germany . -A projector is duct-taped to a pedestal , enhancing the transient aura of a museum or art school lecture hall . -Coach K made his first Final Four with Duke in 1986 and hadn 't had this long of a gap between trips . -Bath were hosting Edinburgh in Pool 4 where Ulster are setting the pace after their 23-13 win in Belfast on Saturday over Stade Francais . -The ( Toronto ) Star reported Sunday that about 400 of Canadaʼs 1,600 active tuberculosis cases are in Toronto , a figure experts expect to rise given the cityʼs growing immigrant community . -Scores of students had been among the 500 watching when Jakarta 's mayor unveiled the statue in the park in December . -NEW YORK , March 4 / PRNewswire / -- Pica9 , Inc . , ( www.pica9.com ) , the leading provider of marketing automation solutions for major brands , today announced that it has become an authorized reseller of Google AdWords ( TM ) . -WASHINGTON ( AP ) - Construction of new homes posted the biggest increase in more than two years in April , a rare bit of good news in what has been the worst downturn in housing in more than two decades . -Prior to the FlyDubai and Etihad announcements , Boeing had taken orders for 475 new planes this year , primarily for the short-to-medium-range 737 . -Staple goods , all in higher demand . -Marion scored just four points in 40 minutes of his fifth game since joining Miami . -( AP ) - The NCAA is investigating Arizona 's basketball program for alleged recruiting violations , according to a letter sent to university president Robert Shelton . -The charity responded to a variety of SOS calls , from rescuing children cut off by the tide , to sinking fishing vessels and stricken whales . -These factors can include the genetic makeup of the patient , genetic changes that occur with diseases such as cancer , as well as diet , age , environmental factors . -However , the Commons Transport Committee said it was " not convinced " the project was adequate for the number of trains which could end up using the station and urged the government to look into the issue . -It was the first decline since March 2007 and the biggest drop in more than three years , since inventories fell by 0.3 percent in July 2005 . -Sykora-ML has been delivering solutions to the AS / 400 marketplace for over 20 years . -" Dozens of houses have been destroyed in the past three days by adult elephants entering human settlements to look for their wandering calves , " said the local magistrate , L.S. Changsan . -Hasan Rasouli , one of Mr Khatami 's campaign managers , said the former president announce his official decision to pull out of the race by the Iranian New Year , on March 21 . -And be thankful none of the action will involve any mention of Jonathan Ross or falling house prices . -The Arghandab district where the bombing occurred has long been known as a haven for Taliban forces and their sympathizers . -Ahmadinejad also blamed the West for the unrest following his disputed reelection as a third batch of his opponents was hauled into a courtroom to confess to plotting against the state . -I 've stayed faithful even in Vienna , home of the traditional Kaffeehaus , much to the disgust of an Austrian friend who waited outside in shame . -GM retains a 35 per cent stake and workers will hold 10 per cent . -Let everyone know that this is divine justice . " -WRAP has conducted similar tests of thinner glass and plastic bottles , with equally promising results . -Garrido , eldest son of former Ryder Cup player Antonio , finished strongly on the opening day and played superbly in round two to finish on 15-under-par 129 . -Ryan never had any hesitation about going back to his favorite receiver . -This was true of Rogers 's firm , BGR Holding , which made its name with blue-chip connections in elite Republican circles . -Attorney General Mike Cox " has concerns about the actions that 's he 's read about on the blog and what we 've heard about as far as the trespassing issue with the University of Michigan , " spokesman John Sellek said . -The news isn 't always black and white . -In a 2003 sermon , Mr Wright said blacks should condemn the US . -Its fatwas are nonbinding . -Speaking to the American Israel Public Affairs Committee , Barack Obama endorsed a two-state , Israel-Palestine settlement , and took an implicit poke at President Bush . -Finance Minister Ali Rodriguez said the government is intervening to back deposits in Stanford Bank SA after U.S. fraud charges against other Stanford banking companies provoked a wave of withdrawals by customers in the South American country . -His lifetime record with the Yomiuri Giants ¿ Japan 's equivalent to the esteemed Yankees ¿ was 112-62 with a 3.01 ERA . -The admission has sparked a torrent of protests from Israeli and foreign journalists who claim such a move will jeopardize the objectivity and safety of journalists . -Other participating entertainers included Mariah Carey , Bono , Miley Cyrus and Snoop Dogg . -To quote the authors of your article , " The country seems , if not adrift , becalmed at the moment , even unsure of itself and its identity . " -Lord Turner will also launch a crackdown on the use of offshore , off balance sheet funding vehicles by banks . -A health crisis also loomed a week after the 7.0-magnitude tremor , as aid workers struggled to tend to the homeless and injured amid deteriorating security in Port-au-Prince . -Rooney had to watch from the sidelines as Emile Heskey returned from his own international exile to bring out the best in Owen as England put their qualification bid back on track with vital home wins against Israel and Russia . -( Dan ) Ellis has been named president and chief executive officer and appointed to the board of directors . Concurrently , the company announced that Peter T. Cyrus has resigned from his positions as interim president and chief executive officer , and a member of the company 's board of directors , effective June 9 , 2009 . -We have an immediate gratification culture which has spread to financial services with online sales of car finance , loans , insurance , savings and even mortgages . -" When you run into guys like that , you need to have good pitching , and we didn 't have that , " Tigers manager Jim Leyland said . -The Medicines and Healthcare products Regulatory Agency ( MHRA ) has upheld a complaint over the online advertising of two remedies , Duchy Herbals Echina-Relief Tincture and Duchy Herbals Hyperi-Lift Tincture , which are sold for £ 10 for 50ml in selected Boots and Waitrose stores . -Uddin refused to answer our repeated inquiries about the address . -The free eBook and audiobook downloads are part of an ongoing effort to educate general readers on pulp fiction stories written in the 1920s , 1930s and 1940s . -Three others were on the run , said police , who also detained 13 civilians . -Iranian state television reported that eight people had died in the street violence Sunday , but independent confirmation of the casualty toll was virtually impossible because of curbs on media coverage . -Friday , Ankiel politely declined to specify what they asked or what he told them . -Actually , stormer , you are wrong again . -British cinema was born in a hall on Regent Street in central London in 1896 - when the Lumiere brothers put on the first public show of moving pictures in the UK . -When Jerry Hall told her to open her eyes to the truth about Bailey 's newest model , Catherine , Helvin took a good , hard look . -" In both instances when the small packages were opened , there was a dissemination of smoke and a smell , that 's the best description we have right now , " he said . -All of Pettigrew 's competitive results since January 1997 will be disqualified . -On the 767s and 757s , the BusinessFirst seats recline 156 degrees and measure 21 and 20-inches wide , respectively . -" This really gives information on steps that people can take that 's going to improve their health , " said Fradkin , who had no role in the latest research . -Twenty-seven of the task force 's recommendations , including the express flight paths , could be implemented by next summer , with help from the Federal Aviation Administration , the Port Authority said . -In today 's tough economy , every supply chain participant wants to streamline operations and cut costs while continuing to increase efficiencies . -But it is not necessarily even about being an original songwriter . -Hopefully the audience will maintain . -Italy 's fashion and textile industry employs 800,000 workers in 30,000 companies , the Industry Ministry noted in a statement . -BASKING RIDGE , N.J. , and ATLANTA , July 14 / PRNewswire / -- Verizon Wireless , the nation 's leading wireless provider , and AirSage , a world leader in using wireless signaling data for value added applications such as traffic information , predictive analytics and location services , have entered into an agreement where non-identifying data from Verizon Wireless ' network will be used to help AirSage provide real-time traffic information . -His last pitch was to John Buck , who lined it over the right field fence for a three-run homer to make it 8-2 . -KEITH BOTSFORD has just flown from Costa Rica , where he lives in an angular , steel-covered house overlooking the Caribbean , to Boston , where his wife -- a molecular biologist some 52 years his junior -- lives and works . -The A4 2.0T , ( $ 37,525 , MSRP as tested ) is powered by a 211-hp , 2.0-liter , turbocharged four-cylinder engine that provides 22 mpg overall , on premium fuel , which is good considering the car 's quick acceleration and all wheel drive . -Much of the remainder would be used to maintain those systems . -He made a face ? -We have two other boys , Mitchell , 6 , and Donovan , 10 , and we would have needed a variety of child care arrangements when the baby was born . -Saunders , a Midwest native who , in addition to teaching , runs water conservation workshop for brewers and cheese makers , says she has a " palate for grains " and has loved the taste of beer for as long as she can remember . -But prospects are uncertain in the Senate , where Republicans and coal-state Democrats oppose it . -Liverpool and England player Gerrard , 28 , remains on bail after appearing in court charged with assault and affray last month . -Other airlines with the A380 in their fleet are Emirates , Qantas Airways and Singapore Airlines . -Among men with cancer that had spread beyond the prostate ( metastatic cancer ) , the death rate in the first month was more than triple that in the general population . -Some of the players are duking it out over who owns the patent to commercially clone animals in the first place . -Yet when it comes to scheduling working hours , the U.S. does have more flexibility than Europe . -He asked more questions of himself , and his ability to turn things around , than his interrogators . -" It 's a very , very exciting day for football fans because it 's a ground-breaking move , " he told the BBC . -American University professor Jane Hall told FOXNews.com that the race issue is an unfortunate diversion for Obama , who drew supporters hoping this would not be a campaign about race . -But Gov. Jon Corzine and most of the state legislators are big Rutgers football fans , and that does not make for much of a check on spending . -Abreu added that " it took a long time " to get his normal swing back . -On the other hand , the Chargers have some pleasant memories of Indy , winning there 26-17 two years ago , when the Colts were 13-0 . -Inflation would work the same magic on government debt . -President Hu and Prime Minister Wen are basking in good will from their management of the Olympics , so far widely perceived as a triumph for China . -But he said that troops on each side had not fully pulled back , and that the status of the oil-rich Abiye border region , claimed by each party , had not been resolved . -" I have requested both the ministers to vacate the hotel rooms and go to their respective bhavans ( guest houses ) , " said Pranab Mukherjee , the finance minister . -Joseph under went numerous operations following his accident when he suffered burns over 76 % of his body . -Back home , he plays gigs with his band . -The procedure will take place next week at the Cleveland Clinic . -" Corals face threats ranging from ocean acidification , pollution , and warming to overfishing and starfish outbreaks , " says Dr. Ian Poiner , Chief Executive Officer of the Australian Institute of Marine Science ( AIMS ) , which led the research . -" This is a rare instance where something good can arise from tragedy . -MCCAIN : As a cold political calculation , I could not be more pleased . -On the other hand , if you eat more calories than you burn up , the extra calories are stored in your body as fat -- and you gain weight . -She is a medical school graduate who defends those who preach intolerance of homosexuals . -The probe can also remove a tissue sample which can be removed and examined under a microscope for cancer cells . -Former double world champion Fernando Alonso insisted he has put last season 's last gasp title disappointment behind him as Ferrari prepare for the new Formula One season . -" After several minutes shooting our security guards repulsed the attackers and killed three of them , " the U.N. official told Reuters . -I am not a Hillary hater , but I must admit that I find some facets of her personality grating . -The Security Council , in its first joint formal action since last month 's pro-democracy rallies were crushed , said it " strongly deplores " the repression and called for the release of political prisoners , amid growing concern over the fate of nearly 1,000 demonstrators still being held . -Mr Morse drove the hunt 's 4x4 to the disused airfield and stood in front of it to prevent it taking off . -It has been in business for more than 213 years . -Unlike the formal and intense competition at the Sambadrome , anything goes at the blocos , and they are becoming more and more popular because fans can join the fun . -WASHINGTON ( Reuters ) - The trade deficit narrowed more than expected in March on a record plunge in the value of imports , even as average oil prices surged to a new record , a Commerce Department report on Friday showed . -Indeed Armstrong had hinted to that effect . -Make your request after the wedding . · Anniversary greeting : An anniversary card will be sent to couples who are celebrating their 50th ( or greater ) wedding anniversary . -The comments by the two MPs threaten once again to embarrass Tory leader David Cameron after he was forced last week to reprimand senior frontbencher Alan Duncan for suggesting MPs were living on rations in the wake of the expenses scandal . -IDS said there had also been a number of freezes in the finance sector , media , airlines and road transport . -The events called attention to Beijing 's human rights record in the Himalayan region at a time when Chinese leaders had hoped for a peaceful run-up to the Olympics . -He called her into his office before the start of a season and mused in admiration at how wonderful she was . -The ads cited his criticism of the Haditha incident as well as his comment about " racist " voting tendencies of many western Pennsylvania residents . -To a certain extent , Brown has been undone by events beyond his control that have nearly paralyzed the government in recent months . -" The recovery will be slow and uneven , " he added . -COPENHAGEN ( AP ) - President Barack Obama squeezed two impromptu meetings into his tight schedule and made an animated plea for compromise Friday , making plain his frustration over the difficulty of pushing world leaders to settle on a plan to combat global warming . -" This whole idea of change and disease processes throughout life is something that a medical student should dwell on and hear a lot about and think a lot about , " he said . -Instead , Cliburn did the honors a year later . -This article is somewhat disingenuous about the extent of support Mitt Romney had from so called Conservative leaders . -Associated Press Writer Matt Moore in Berlin contributed to this report . -She prepared me for the interview not by showing me her work but making me a momentary participant . -SANAA ( Reuters ) - A Saudi suspected of belonging to al Qaeda has handed himself in to authorities in Yemen where al Qaeda militancy is on the rise , a government website said on Thursday . -The police investigation of the attempted jewelry store robbery and hostage incident is continuing . -" Yes , I cried , " he says . -" I 'm not convinced we are winning in Afghanistan . -US stocks looked set to show some resilience on Wednesday morning , with buyers being tempted back into the market following two days of selling . -She followed with a series of blockbuster interviews--Mike Tyson and Evander Holyfield , exclusives with singer Whitney Houston and ESPN 's Erin Andrews , and just this week , former Alaska governor , GOP vice presidential candidate Sarah Palin . -Reviewers must complete profiles , listing their e-mail addresses . -Abbey said it had " fully investigated " Mr Bates 's claims but it could not say whether any disciplinary action had been taken . -Privately , local leaders admit it might be a concern if it made a specific commitment to hampering oil and gas exploration , but the general view was that , with the exception of Chavez on the Queen , they have heard it all before . -In von Maltzahn 's approach , one set of nanoparticles lodge in a tumor 's blood vessels and cause local bleeding . The bleeding prompts clotting factors to be produced , which in turn , attracts a second set of nanoparticles that have been programmed to be attracted to the clotting factors and that deliver a cancer drug . The use of the clotting factors dramatically increases the number of targets for the drug-carrying particles . -" Brits going to Australia , New Zealand and Canada are looking for a better lifestyle not to make their fortune , " says Harwood . -Made in Britain , each coat is finished to exceptional quality , with all the original details. east thames group limited. in this varied and challenging role you will provi ... . £ 22,514 - £ 24,246 per Annum. english heritage. bedfordshire . £ 29,500 - £ 33,000 per annum. imperial college. south kensington , london . £ 26,580 - £ 30,360 p.a .. their beautiful waterfront location , the breadth of arts and cultural offerings rivaling those of cities ... reflect their cherished heritage and a famous lineup ... . wi. performing arts. the duties of the heritage chair in ... arts , the john f. kennedy center for the performing arts , and mason 's center for the arts. the ... . va. their beautiful waterfront location , the breadth of arts and cultural offerings rivaling those of cities ... reflect their cherished heritage and a famous lineup ... . wi . -" The funny thing is , one of the groups that was seeing ' Legally Blonde , ' when the band director told his kids they were now going to see ' Spelling Bee , ' the kids were more excited , " he said . -I think there was some disappointment with some of the recent , earlier Saw films that came out before Saw VI . -And for all of the commotion over the Mitchell report that linked about 90 major league players to steroids , a new set of baseball initials , H.G.H. , alias human growth hormone , is about to upstage all the others . -" Peter Galbraith , the highest-ranking American in Afghanistan working for the United Nations Mission there -- recalled today by U.N.Secretary General Ban Ki-moon over disagreements with the head of the mission there , special envoy Kai Eide -- says the Afghan elections recently carried out are the worst the U.N. has overseen . -Yentob took over in 2002 when Massow resigned after criticising the preponderance of conceptual art in the art world . -The bustling commercial and residential area in north London , with its open markets and a scenic canal , is one of the British capital 's top shopping and tourist destinations . -However , the company 's latest big-screen release , " Precious , " a drama about the brutal home life of a teenage girl from Harlem , could be a positive contributor . -It doesn 't get more authentique than that . -CAEN , France - Ugly words on the playground were his first hurtful clue . -22 ) : Your zeal is at a peak but it 's important to hold back . -In Toronto , the snowstorm caused some 600 traffic accidents on Sunday morning alone , the provincial police reprted . -Within hours , he said , a battle erupted ; Ahmed was shot in the leg . -Not because I think the Big Four banks are evil , but because I 'm pretty sure we 'll get a better deal and much better service from one of the small guys . -Damage from the quake may be severe but it is limited to one region . -With any luck , the worms will soon be eating as much as three pounds of food scraps a week , a way to turn kitchen wastes into a rich , dark , earth-smelling soil conditioner you can use in your garden , in flowerpots or even spread over your lawn . -In October , the UN announced that it had been allowed to collaborate with regional authorities to supply relief food , medicine , and veterinary services as well as setting up offices in a key town there . -And the design scheme allowed them to market the 43-story Broadway tower , one of many postwar apartment towers on Broadway , as if it had a coveted Central Park Address , by linking the tower through a rear drive and garden , and its mailing address , to Central Park West . -" Are there lots of errors in the data ? -Despite the comments , Hamas has shown little tolerance for dissent in Gaza . -Stir in turkey , using a wooden spoon to break up the meat . -I guess if he called me he was thinking about a comedy . -After a series of corrections failed to straighten out the glitch , Britt-Crane yielded to the inevitable . -Can you really establish trust by falling backwards and hoping Rick from accounting will , in fact , catch you ? -Gates also made a point Tuesday of saying that the United States is " ready to help resolve disputes over boundaries and hydrocarbons , " a reference to widening tensions between Arabs and Kurds . -Afghanistan , having been largely ignored through the early weeks of the campaign , bounded on to the agenda three weeks ago after more than 70 Afghans were killed in a US air strike called down , in a perverse twist of fate , by a German officer . -She emerged as a heroine from the conflict , during which she led a team of 38 female nurses ministering to wounded soldiers in extremely difficult conditions . -That led them to Mendoza and Tobar-Campos , police said . -The Irish News reports how the former Taoiseach , Albert Reynolds , is behind plans to bring a huge bio-energy project to Londonderry . -Heinz said it now expects earnings of $ 2.54 to $ 2.62 per share for the full fiscal year . -In 2007 , authorities said two Tennessee men set fire to the home of a suspected child pornographer , killing his wife . -Last week there was a modest revival of demand for UK bank shares . -Regulators can fine Microsoft up to 10 percent of yearly global turnover without seeking evidence if the company doesn 't stick to this commitment . -Satyendra Narayan Yadav , from Nararipuri village in Madhubani district , says he is another harassed Act applicant . -Now imagine that a strange phenomenon occurs , in which it is noticed ( but , oddly , not widely reported ) that Americans on the Canadian border seem to stream across into Canada to buy their Big Macs . -Do you worry , though , that being green might just be another trend for musicians and will be forgotten in a few years ? -" That is not a sign that the system worked . -That is , the solution is merely a matter of fully understanding the problem , and not one of needing genuine creativity. re : 58 . -Gesine Schwan , a 65-year-old political scientist backed by the Social Democrats ( SPD ) , said this week that she saw " very , very good chances ( of being elected ) . -He was never arrested or charged , and when he sued the government for ruining his career , a federal judge found " not a scintilla of evidence " linking Hatfill to the mailings . -The global fame and the " odd bit of negative press " in the past week have been too overwhelming for her , judge Piers Morgan told CNN 's " Larry King Live " on Friday . -Let 's go ! " barked the instructor in camouflage fatigues as we shuffled into a cavernous corrugated-tin warehouse in an open field at the Farm . -Eight people have been arrested in last week 's shootings of Byrd and Melanie Billings in their sprawling home near Pensacola , Florida , which they shared with their many children . -Activity | The following two activities are designed so that students find and peruse Times articles on a range of topics related to childhood obesity , nutrition and exercise . -EST , underperforming the German blue-chip DAX .GDAXI , which fell 6.6 percent . -TCU ( 1-1 ) will play the College of Charleston during the evening . -Their captors made off with a sum of cash . -Some in the shrinking community of global warming skeptics and those downplaying the issue , were dubious , however . -The researchers discovered that long before the fly leaps it calculates the location of the threat and comes up with an escape plan . -Continuing that custody while developing a permanency plan , or sending the children ' home ' , will be the subject of subsequent timely hearings . -Nor is Mr. Mukasey ( pronounced mew-KAY-see ) a Washington insider with experience in managing a federal bureaucracy . -There is also Cannibal ! - The Musical , based on the cult film by Trey Parker , the creator of the South Park cartoon series . -The main opposition candidate , Almazbek Atambayev , has dismissed the election as fraudulent and is demanding a rerun . -Thus ensue lectures on genetically modified foods , demonstrations of toxic fumes leaking from mercury-laced fillings and paeans to the benefit of frequent coffee enemas . -True progress would moving away from the courts " deciding " what rights someone has and has not . -For the poll , more than 16,000 people who had just voted were selected at random to fill out questionnaires . -But Fenty 's budget would increase funds per student by $ 175 in both traditional and charter schools , would add $ 2.7 million for summer camps and facility improvements to recreation centers and would add $ 2.4 million for staff and equipment for the new Wilson and upcoming Deanwood pools . -The National Assembly will select a vice president and the entire 32 strong executive Council of State . -I was driven to Loch Fyne in a car stinking of creosote . -The latter , in particular , is simply brilliant ; he makes us chuckle with an eyebrow raised or slight turn of his head . -The two sides are at a stand-off and are apparently determined to take the issue to court . -Nissan 's car sales increased 11 percent , but truck sales were flat . -The new book The Coke Machine : The Dirty Truth Behind The World 's Favorite Soft Drink ( $ 26 / € 19 ) by Michael Blanding , released on September 16 , explains the original Coca-Cola recipe did have small amounts of cocaine present . -New York recently mandated 90 days notice for cuts of 250 or more at one site . -A self-taught painter , Boeke keeps a steady hand crucial for pinstriping and custom painting by abstaining from alcohol and making sure his painting is interruption-free . -Others just want to give their guests or adolescent children privacy . -The audience cheered ; he shouted louder . -Irene from Virginia - we can 't even elect our PM . -" We have now seen all the investment and infrastructure coming together to bring a turnaround in the number of people using bus services . -It is no good coming up with a similar-ish product to todays iPhone ( it 's actually just far more complicated and overwhelming to use it seems ) , but they have to move ahead of it , because Apple doesn 't stand still . -The rolling action is designed to disrupt services while ensuring workers lose only one day 's pay . -With the exceptions of Argentina , Saudi Arabia and South Africa , all of the member countries fall within the list of the top 20 biggest state G.D.P. ' s in the world . -Wicks suggests setting a provisional retirement age for planning purposes -- perhaps 66 , when he would be eligible for a state pension . -One of the many infamous quotes from Seminerio was when he said in the State Assembly if Native Americans didn 't like their life in the US then they could walk back across the Bering Strait to Russia . -The Angels took a lead in the third when rookie shortstop Jed Lowrie went to field a grounder with two outs and could not come up with the ball cleanly , allowing Garret Anderson to move to second and Vladimir Guerrero to reach first safely . -I have no doubt that he cares dearly for these soldiers , but I think he 's way off the mark there . -A Spanish website containing abuse aimed at Lewis Hamilton was condemned yesterday by the FIA , motor sport 's governing body . -Belmont Park opened in 1905 . -That could show up again today when chain-store sales are reported by the International Council of Shopping Centers . -The survey saw more than 400 people interviewed over the phone . -Douglas Carswell , a leading critic of Mr Martin , said a new Speaker was needed with the " moral authority " to pass reforms . -Their members are scattered throughout Russia , Asia and the Western Hemisphere , shunning much of the modern way of life . -CAMBRIDGE , Mass . , March 5 / PRNewswire / -- Cultural Care Au Pair , the leading provider of intercultural childcare in the United States has announced the launch of a brand new program created to address the concerns of American families during this difficult economic time . The Cultural Care " Peace of Mind Guarantee " ( www.culturalcare.com / peaceofmind ) assures that if a host parent loses their job , Cultural Care Au Pair will guarantee a pro-rated refund of all unused program weeks . -Teachers and communities often reach their own arrangements with the Taleban . -Heretofore , my chief anxiety about traveling to France had always been that , at some point during my trip , I would be called upon to pronounce the name of the town Ypres . -The ultimate insult : he was only in charge of the accessories . -They 'd sold their house in Fairfield at a nice profit , so they had money in the bank for the first time . -In a statement , Detective Inspector Ronnie Walker , who led the police inquiry , said : " Jim Ingram hoarded money obsessively and the thought of parting with a large divorce settlement triggered the tragic events of 13 March . -Lynne Spears : Do I promote teen sex ? -Another extraordinary case arrived in court yesterday after O. J. Simpson , the former American football player and murder suspect , burst into a hotel room in Las Vegas last month in an attempt to take sporting memorabilia that , he said , belonged to him . -I don 't see any other front where this is available , " he said . -The intersection will continue to " fail , " highway officials said , but the new lanes should reduce delays there by an average of 15 to 30 percent . -Concerns about China 's plans to limit economic growth and proposed regulatory bank changes from Washington also have pummeled the market . -At one point , as many as 150 students were at the building protesting the tuition increase , Morain said . -1995 - 20th anniversary of the Indonesian invasion marked by protest by 112 East Timorese and sympathisers who enter Russian and Dutch embassies in Jakarta . -The FSA said that as the wine was below 9 % alcohol it could not be legally called a wine and must be labelled a " wine-based drink . " -To watch the video of the lock gates arriving in the Cumberland Basin and to listen to the interview with Matt Ewing , the site manager , click on the link below . -A spike in defaults in the coming months could force the insurers to pay billions of dollars in claims . -The ICSC said that the late Easter had a negative impact on the sales figures , and that a clearer picture of US consumer spending would emerge when looking at March and April 's figures together . -Opponents of his plan said backing away from the larger project would send a negative message amid efforts to borrow against future lottery revenue . -Officials of Disney and Dell declined to comment on specific allegations , but both companies say they carefully monitor factories in China and take action when they find problems or unfair labor practices . -" We have no experience of marketing or customer service . -Mr Brown has struggled to convince the US of the value of a transaction tax . -" We hit a point in growth that we needed people who have done this before , " he said . -Also , more than 1,800 parents were informed and more than 18,000 teenagers directed to " positive " activities . -0943 Peter Robinson is telling jokes . -The glass holding the candles can break during use , presenting a fire hazard , the commission said . -Michael Gilbert , who had been living in Blackburn , Lancashire , was identified through his fingerprints , police said . -" We have been open for 20 years now and have not had an E. coli incident so it 's alarming and distressing that we 've had it now , " she said . -It was enough to frighten his bravest rivals as , dreaming of dancing in London on an Olympic night in 2012 , The World 's Fastest Man sped away into the shimmering distance . -Powerset , a 2-1 / 2-year-old start-up , has licensed natural language processing technology and related machine processing methods developed over three decades at the Xerox PARC research center in Silicon Valley . -He can 't buy his way out of problems as Labour did . -" We were very fortunate that the head of the fire ... was stopped at the backyards of those homes , " said Los Angeles County deputy fire chief John Tripp . -Brain dead today aChangeOfIdeas ? -In adults , researchers have shown an association between metabolic syndrome and a group of diseases called nonalcoholic fatty liver disease ( NAFLD ) , which at its most severe , may progress to irreversible liver damage . -But it doesn 't mean we won 't have a health care bill , " she said . -The publicity material refers to the magazine as " she " and letters from correspondents are signed " Miss " and " Mrs. " -It has chosen an unlikely candidate : Chris Ostrowski , 28 , a manager for John Lewis and the party 's seventh-choice candidate in the Eastern region in the European elections . -Sunderland , which lost its first home league game of the season last month , fielded a weakened team and was beaten , 2-1 , by Notts County , which is struggling in League One , the third tier of the English game . -Also through to the last 16 in early action was German veteran Tommy Haas who defeated France 's Jeremy Chardy 7-5 , 6-3 , 4-6 , 6-4 . -Police Commissioner Raymond Kelly , speaking from Times Square on a Webcast , said his department had many officers in the crowd , both uniformed and plainclothes . -NEW YORK ( Reuters Health ) - China now has more people with diabetes than any other country , a new report shows , making it clear that the nation 's soaring economic growth is taking a toll on public health . -Shiites account for a majority of Iraq 's population , and in the last national elections , in 2005 , Shiite parties ran together under a broad umbrella endorsed by their top religious authority , Ayatollah Ali Sistani , leaving little doubt as to who would emerge the winner . -This is a notion with which Govan himself takes umbrage -- even though part of the reason he took the job in Los Angeles was to take charge of an ambitious three-phase building project ; the first phase alone will cost $ 156 million . -He made the building a co-op again in 1952 . -If you prefer to plan your own activities at each port , Alaska 's official tourism site is a good resource . -" We feel proud but it doesn 't mean we are over-confident , " he said . -More than half of the world 's attempts to land on Mars have ended in failures . -With Libya once more back in the international fold , it is now as easy to sell your music over the net from Benghazi as it is from Brighton , Budapest or Bangalore - and some previously rare tunes are easier to obtain than they have ever been before . -" We are very serious to implement this project since the food crisis is turning serious day by day , " Manjhi , who has eaten rats , told Reuters . -While few cyber crooks are attacking Mac users through Safari and Firefox at the moment , that may change soon if a large number of Windows users migrate to Windows 7 , the successor to Windows Vista , due to be released sometime later this year . -LOS ANGELES , California ( CNN ) -- A 24-year-old gang member was arrested Thursday in connection with a shooting at a Los Angeles bus stop in which eight people were wounded , city officials said . -Oil prices averaged over $ 120 a barrel in the second quarter -- almost double the level in the same period of 2007 -- before rising to a record high above $ 147 / barrel on July 11 . -More important , she said : " There is a huge difference between singing and having a vision for yourself . -Classes explaining scientific method and basic concepts will be included in the induction programme for all Tory MPs after the next election , and sitting members and peers will also be offered the opportunity to attend , The Times has learnt . -Hilton , which is controlled by Blackstone , also owns brands such as Hampton Inn , Hilton Garden Inn and Doubletree . -Many Israeli leaders over the years have requested Pollard 's release , Netanyahu observed in his letter to Obama . -WASHINGTON , June 25 ( UPI ) -- Anderson Hernandez finished off a four-run first with a two-run single Thursday and the Washington Nationals went on to defeat Boston 9-3 . -More than that and runaway global warming becomes much more likely , ice sheets melt and sea levels could rise by several metres . -But Mosley is mostly known for his crime books -- a fact that he blames on marketing . -MONACO ( Reuters ) - Booming global investor interest in an emerging industry to supply clean energy alternatives to fossil fuels , such as wind and solar , has temporarily peaked in the wake of a widespread credit squeeze , a U.N. official said . -But the Red Wings would not surrender their crown without a fight , rookie Ericsson blasting a shot from the point past Fleury with just over six minutes to set up a dramatic finish . -Sony 's shares rose $ 1.15 , or 3.1 percent , to $ 38.17 in midday trading . -It is mothers who pay out a fortune in shops like Lush for daughters whose idea of going green is to buy as many new " green " things as possible , then carry them virtuously home in a paper bag . -Mr. Buffett did not attend the news conference , but said in a statement that he was impressed with Mr. Wang 's record as a manager . -In Diwaniyah , a city south of Baghdad , gunmen killed a policeman in a drive-by shooting , said another police official . -Maybe people find that a little eerie . -The company said the purchase " will be earnings enhancing in 2008 . " -Producer Adonis Shropshire confirmed to E ! that Rihanna , the 21-year-old Grammy-winning singer was indeed back in the studio . -The Tiffen Photo fx 2.0 allows users to enhance images from photos taken or stored on their iPhone or iPod Touch . -Unmarried women went the opposite way , voting for Senator John Kerry by a margin of 62 percent to 37 percent . -I 've often said that it helped when I got married and had an Italian last name . -Arsenal ( 4-1-4-1 ) : L Fabianski -- E Eboué , A Song , M Silvestre , A Traoré -- C Eastmond ( sub : S Watt , 68 ) -- J Wilshere , A Ramsey , T Rosicky , F Mérida -- C Vela . -The win takes United two points clear of Arsenal on 69 points after 31 games . -Net charge-offs rose 6 percent to $ 168 million , and nonperforming assets nearly tripled to $ 1.66 billion . -Even in this traumatic week for him , he has had six endorsements , compared with four for Hillary Clinton . -At the same time , she doesn 't plan on taking her bike for a spin anytime soon . -" They masturbated me , they made me masturbate them , they sodomized me , " he said . -Tamilnet.com said the rebels did not suffer casualties . -PHILADELPHIA - In the battle against black unemployment , places like the Opportunities Industrialization Center are ground zero . -Last month , the two companies sliced their dividends and sold billions of dollars of special stock to raise capital and shore up their finances . -" He was bleeding pretty badly so I asked someone else to sit on the shooter 's ankles and I went into Safeway and asked them for some paper towels and I made a compress . -But after the failed attempt to buy the London International Financial Futures Exchange ( Liffe ) in her first year , and the bids she rebuffed from Deutsche Börse , Euronext , the Macquarie banking group and Nasdaq - twice - she hasn 't always received favourable notices . -U.S. Defense Secretary Robert Gates arrived in Seoul Wednesday for the annual Security Consultative Meeting , which is focused on steps to deter military threats from North Korea . -Hotel Kabuki , 1625 Post Street , San Francisco , 800-533-4567 ; www.jdvhotels.com / kabuki . -He tasked George Mitchell , his envoy to the region , to continue meeting with Israeli and Palestinian officials . -Financial markets are concerned that many Spanish borrowers will not be able to refinance their debts this year at a time when investors are nervous about taking on any European risk . -But provincial authorities identified him as Attaullah Wahab , who served as both the deputy chief and security head of the national police in the province . -Brown 's office gave O 'Keefe and Giles immunity from a state privacy law that prevents unauthorized taping in exchange for a complete set of their videos , the report said . -According to the program , what are the differences between a meteoroid , a meteor and a meteorite ? -Coffee chain retailer Peet 's Coffee & Tea Inc. said Monday it will buy wholesale roaster and distributor Diedrich Coffee Inc. for $ 213 million in a move to enter the rapidly growing single-cup coffee market . -Marcus Gad Binghi Thompson , 30 , of Bowfield Road , Firth Park , Sheffield , appeared at the city 's magistrates ' court charged with murder . -Although the vaccine is destined to be used on girls , it has mainly been tested on adults in the United States , where the vaccination programme started in June last year . -A fight is brewing over the practice of feeding chicken feces and other poultry farm waste to cattle . -Catches taken in a one-day match against South Africa in 2008 . -In his 2007 ad , Brock said the mayor backdated her signature on a document and that he had asked his lawyers to report her for possible criminal investigation . -The result is a kind of Alfie-to-Alfie ( or is that alpha-to-alpha ? ) combat , as these two accomplished actors , playing equally crafty characters , engage in the kind of mirroring that is one of the visual leitmotifs of the film itself . -There was no immediate Palestinian comment . -Yeah , probably , a little , if you judge things so parlously as to measure your life in tenths of a second . -But she had long been afflicted with a genetic heart defect , and the stage was reached when she could neither eat nor speak . -She agrees that newcomers might be taken aback by the full-on rock assault of the group 's live show . -I ignored it and kept on eating . -I find it interesting that the amount of " noise " on this message thread e.g. climate change deniers , increased multifold immediately after my posting # 118 , which debunked much of what " M. " of # 66 said about textbook publishing . -An SFA review panel examined footage of the midfielder 's sending-off for violent conduct in Sunday 's 2-1 Clydesdale Bank Premier League defeat at New Douglas Park and determined it warranted only a yellow card for " adopting a threatening and / or aggressive attitude . " -" You think I 'm bad with all the stuff that goes on in the N.F.L. ? " he said . -Al-Hayat alleged Facebook had come under pressure from Israeli and U.S. lobbyists demanding Palestinian resistance movements such as Hamas be removed . -Scholz , who said Goudreau left the band more than 25 years ago after a three-year stint , objects to the implication that the band and one of its members has endorsed Huckabee 's candidacy . -They presented fans with a slightly different singer than the outrageous legend who was even then awaiting trial for alleged onstage exposure -- podgier and less lizardly than in his previous sex-god image . -On Wednesday , four Republican lawmakers demanded an audit of the $ 787 billion stimulus program following reports of exaggerated or inaccurate accounts of the number of jobs created . -Manning , when he went out , had completed 8 of 10 passes for 173 yards and 2 touchdowns . -To him , she was " My Dear Dusia " and he signed his letters " Br " -- short for brother . -The debate this week will be whether or not he would have survived Chappaquidick -- or the William Kennedy Smith trial -- in the digital age . -But ministers in England are still refusing to drop the £ 6.85 charge . -Shelton was given an interim ban until sentencing on 15 January . -And , arguably , there 's a third Razorback in the race , given the candidacy of Sen. Hillary Clinton , D-N.Y. The state favored President Bush over then-Vice President Al Gore in 2000 and went even more solidly for the sitting president over Sen. John Kerry , D-Mass . , in 2004 . -The company also is less subject to other forms of federal involvement . -Lehtonen was caught leaning to his right , leaving half the net exposed as Vasicek crossed in front of the net for the shot . -They included a AA battery , several tissues , a toenail clipper fashioned into a sharp piece of metal , and a device made of razor blade . -Apps that locate specific shops are ten-a-penny , but what 's the use of getting directions to the nearest supermarket when it has run out of the ingredients you want ? -The Prime Minister also condemned the protest and expressed disappointment that a " tiny minority " of the crowd tried to disrupt the event . -19 when Curran , the Georgia linebacker , recovered one against Arkansas in the Bulldogs ' 52-41 win . -Kellogg products are manufactured in 19 countries and marketed in more than 180 countries . For more information , visit www.kelloggcompany.com. Kellogg Company 's Corporate Responsibility report including its approach , progress and future direction in the marketplace , workplace , environment and community can be found at www.kelloggcompany.com / CR . -Little Obama is 6 months old , born May 3 , weighing 6 pounds 5 ounces . -Announcing the Budget details , the minister said : " Next year , my department intends to spend some £ 245m and disburse over € 300m to farmers and rural dwellers , a very significant investment . -Then perhaps Norman you would like to pay off my share of the national debt ? -Paterson 's microphone sounded like it was turned to " murky punk gig from the late 70s " setting , but this only enhanced the brooding menace of Ramalama and the renegade thrill of the entire performance . -It hardly seems like the power dynamic is skewed toward the passenger right now . -The two-year hiatus ensued , with Lane Fox having to fight every step of the way to recover . -The worst tornado hit the night of Feb . -The rate of home price declines has accelerated on a quarterly basis too . -TechCrunch even posted a video of the Zynga chief executive , Mark Pincus , admitting at a presentation this spring that , " I knew that I wanted to control my destiny , so I knew I needed revenues , right f * * * * * * now . -America wake up ! ! -Some studies say they are too conservative and will not provide the returns future retirees need to finance their golden years . -See how much oil you are actually adding to your food use a spray or measure out oil with a teaspoon instead of pouring it straight from the bottle . -Stack spilled a long range dig from Glen in the final minute but recovered in time , with some assistance from Hogg , to form enough of a barrier to foil the onrushing Driver . -Ali Jerba scored twice as Dons beat Hartlepool United 3-1 . -" This trial separation was agreed to with the goal of ultimately strengthening our marriage , " she said . -The | impact will be most evident in his foreign policy . -The company also said it would no longer obey censorship rules . -The United States is a rich nation . -Kartika Sari Dewi Shukarno says that if a caning is meant to teach a lesson , then it should be in public . -22 ( UPI ) -- A new , minimally invasive surgical instrument to remove varicose veins has been developed by a University of California-Los Angeles researcher . -Geller has even been criticized by other Islamophobes for her extremism and for supporting far-right groups in Europe . -One is that people who live in states with few blacks seem more open to the idea of a president who is not white . -Hatoyama repeated that the party would not raise the politically sensitive consumption tax from its current five percent for next four years . -After a week which saw the UK 's heaviest snowfall for 18 years , closing schools and crippling public transport , the rain has heightened the risk of flooding . -Aren 't travel companies -- particularly online travel agencies -- just victims of this scheme , like us ? -For many shoppers , it paid to wait given the plentiful offerings and good deals . -The first clue was the announcement last year by the American Society of Plastic Surgeons that the number of pectoral implants had tripled . -MiCTA is an association of non-profit organizations , colleges , universities , K-12 schools systems , federal state and local governments . -Ducks : Oct . -His staying power and his consistency , and his approach is unique . -The new bus would have four more seats than a Routemaster and standing room for 30 people , twice as many as its predecessor . -A Balearic Islands Superior Tribunal statement Tuesday says the autopsy shows Gately , 33 , died as a result of a pulmonary edema , or fluid in the lungs . -And it 's taking less time to pick up a truck , which he sees as a sign that there 's less work to keep them on the road--and out of his reposessors ' reach . -Tony Woodley , the leader of Unite , Britain 's biggest trade union told his members to boycott The Sun newspaper after the tabloid abandoned its support for the Labour Party . -Ms. Esfandiari was released from prison on Aug . -He was due to have retired next year . -McClement deflected in a shot from the point by Alex Pietrangelo to tie it with 4 : 25 left . -For Democrats , a break in that fight could allow them to focus on issues that voters say demand attention . -But the 23-year-old has stopped talking to her father , even though they live in the same house . -Typhoon Ketsana hit the Philippines and parts of Vietnam , leaving at least 300 people dead and hundreds of thousands more homeless . -" They physically jumped off , " Police Commissioner Raymond Kelly said . -Next month , Vodafone will increase its minimum call charge for pay-as-you-go customers by 30 per cent , from 15p to 20p , adding to pressure on consumers hit by rising gas and food bills as well as soaring inflation . -One argument is that industry intakes are gender specific . -Rather than being provoked by the attacks , 1st Scots Guards took pains to establish good relations with the local population and press . -While it seems like common sense to pump money into an economy that is pulling the bedcovers over its head , the problem with most social interventions is that they target not robots and machines but human beings -- who regularly respond to interventions in contrarian , paradoxical and unpredictable ways . -In a world that blends flamboyant costumes with decidedly bland personalities , figure skater Christopher Bowman stood out for his irrepressible individuality and unapologetic , undisciplined approach to the sport . -In Europe 's largest economy , Germany , the benchmark DAX index is off slightly more than 20 percent this year , and the CAC-40 in France is down almost 22 percent . -This appraisal of life in a more acidic ocean was if anything conservative , Dr Hall-Spencer said , because it mimicked future ecosystems only partially . -After yesterday 's thrillers with both finalists overcoming opponent fightbacks , this year 's final promises to be a cracker . -The fast-moving developments left liberals in a quandary . -The victims had moved to Palm Beach County from the Brownsville , Texas , area just a few months before they were killed . -In the previous week , 30-year , fixed-rate mortgages averaged 5.29 percent . -His daughter , Tiffany Ward , continues her father 's legacy as executive producer of the feature films " The Adventures of Rocky & Bullwinkle , " Dudley Do-Right " and " George of the Jungle , " and Cartoon Network 's ( sister channel to CNN ) new " George of the Jungle " animated TV series . -Goalies _ Anaheim , Hiller 6-5-1 ( 30 shots-29 saves ) . -She says she tried to talk Boyle into being " more out there " and to stray from India 's Bollywood traditions . -For instance , it 's not just about manufacturing those wind turbines in the cities . -" I showed Gattuso the midfield of the top five teams in Europe and he would not have been a starter in any of those . -" Starting over is not an option , though that 's the battle cry of the congressional wing " of the GOP , Emanuel said . -Now that has failed , they are trying to split the Shia . " -BAD RAP extensively evaluated all 49 dogs from the Vick property and found that 48 of them were safe with people . They selected 10 of the most outgoing and social dogs to bring back to the Bay Area for care , training and re-homing . -Could this be a Harry Potter-like pancultural behemoth ? -The probes were launched in the aftermath of the June 1 crash in the Atlantic Ocean -- when Air France Flight 447 was flying to Paris , France , from Rio de Janeiro , Brazil . -Nicholas is a writer in our Washington bureau . -A mother last night welcomed the decision to extend the prison sentences of the two men responsible for a fatal attack on her son . -During the conference , a policeman asked for their documents and they were subsequently detained , police said . -He was , for many , the ultimate performer . -His resignation after three years on the board of the Cupertino-based company comes after the US government announced in May that it was beginning an antitrust investigation into the interlocking directorships between Apple and Google . -Had it been Obama delivering a message , I 'm sure that King Jong IL would be blasting some more rockets off after a ten minute session with the Obamessiah . -A long bamboo fence divides two fields but every so often Sangram has left open a booby-trapped entrance . -The tonnes of coal and anthracite carved out of the mountains fed the trains and factory furnaces : Hillary Clinton recalled , as a child , marvelling at the Lackawanna River running through the city , black with coal dust , and the burning piles on the horizon . -However recent datapoints are giving us cause to expect some reversal . -We were grown adults . -Down three , minus 800 . -Q : I 've booked a flight during the strike period . -Leah 's Secret was fourth and the favorite , Visit , was never in the mix , finishing sixth . -Billy Tauzin , president of the industry 's trade association , highlighted the gloomy prediction in a June 1 letter to President Obama shortly before striking the deal to cut drug costs by $ 80 billion . -Before that , the final keynote speech comes from Secretary of State Hillary Rodham Clinton . -" I was even more wrong , both personally and professionally , to ask for your help in encouraging others to vote for the film and to comment on another movie . -GM barely won the global sales race with Toyota last year , but Toyota overtook it as the world 's top automaker as measured by global vehicle production in 2007 . -They are angry not only over his long-standing opposition to much modern architecture , but his efforts to block a major steel-and-glass tower project on the site of an old army barracks in the posh Chelsea neighborhood of London . -The Illinois senator has said the war in Afghanistan , where Taliban and al-Qaida-linked militants are resurgent , deserves more troops and more attention as opposed to the conflict in Iraq . -The remarks , published Friday in The Jerusalem Post daily , came just days before President Bush arrives in the region to build on the momentum created at a Mideast peace conference in Annapolis , Md . -Political sketchwriter Ann Treneman of The Times , had been a regular tormentor , her memorable descriptions of Sir Ming ranging from " a wrinkled old turtle " to " looking as if he had just escaped from a care home . " -My wife and I watched the moon rise over the Atlantic Yards site in Brooklyn on Tuesday evening . -Members of the Rashtriya Swayamsevak Sangh are often called chaddi wallahs because their uniform includes baggy khaki shorts . -Initially , it was thought the burial site dated from the Iron Age ( from BC 800 ) to early Roman times ( from AD 43 ) after examining pottery in the pit , later identified as a Roman quarry . -A nunchaku is made up of two wooden sticks joined by a chain or rope . -" The differences in terms of life satisfaction and attitudes towards the future underline the significant inequalities in living conditions and in the experience of daily life for Europeans , " said Eurofound director Jorma Karppinen . -Law enforcement officials say that such sting operations are an extremely effective means of lowering crime rates and stopping the criminally minded before they commit worse offenses . -It was suggested that his brawny physique and square-jawed poses masked a wooden style . -" My son was able to rescue two women ... but unfortunately the second blind rocket fell and claimed his life , " said Sadiq Abbas . -It was centered offshore , about 41 miles southwest of Eureka and 208 miles northwest of Sacramento . -This article was published on guardian.co.uk at 00.47 GMT on Saturday 19 December 2009 . -( AP ) -- The Oakland Raiders informed coach Tom Cable on Tuesday that they will not bring him back as coach next season even though he led them to their best record in eight years this season . -Police have said they do not know the exact reason why Walters attacked him . -That is just one position , " Boycott added . -Nancy Keenan , president of NARAL Pro-Choice America , said the federal ban is " unfair and insulting " and should be repealed . -Lt Col David Wakefield , spokesman for Task Force Helmand , said he would " not be forgotten . " -Overall rent has soared by 22 per cent in just three years , with the average undergraduate paying £ 99 a week in 2009 / 10 . -Zac Efron is more than busy promoting " High School Musical 3 : Senior Year , " but the actor tells Access Hollywood that although he plans on attending USC in the future , he has no plans of heading to Broadway . -MAPLE LEAFS 4 , LIGHTNING 3 Phil Kessel scored at 3 : 33 of overtime to give host Toronto the win . -Industrial production in the eurozone has risen for the fifth consecutive month , sparking hopes that GDP figures to be published on Friday could be higher than originally expected . -If young musicians are to have a chance of enjoying a fruitful career , then we need to establish the principle of artists ' rights throughout the Internet -- and we need to do it now . -First whiff of scandal for Obama ? -The lion 's share , however , will be found by raiding other parts of the overstretched defence budget , with planned cuts that would realise around £ 1.5 billion over three years . -When the House committee met late on July 7 to consider the Financial Services spending bill ( HR 3170 ) , LaTourette offered an amendment to require automakers that have taken government funding , namely General Motors Corp. and Chrysler LLC , to reinstate agreements with auto dealers dropped as part of their restructuring . -Whether it 's how to build something , understand something or create something , we hope you 'll find a book here that interests you . -A new ring of housing , the four- and five-story , Socialist-style apartment compounds , began to envelop the city . -Small said he thinks residents are comfortable with the idea because he 's only gotten a handful of e-mail complaints . -Next season 's play list for the Costa Mesa company , which made its name on the national scene as a launching pad for new work , features four world premieres out of nine main stage selections -- plus two new shows in the theater 's three-play Theatre for Young Audiences series . -During Watergate he scored an exclusive interview with a security guard for the Nixon re-election campaign who had been involved in the break-in at the headquarters of the Democratic National Committee . -The conjuring act has proven harder to pull off this campaign . -The typically bustling Lake Arrowhead resembled a ghost town , with abandoned shops and homes . -Legal experts say ratings agencies may prove harder to sue but that will not stop people trying or stop politicians calling for blood . -Fininvest , of which Mr Berlusconi owns 63.3 % , with the rest owned by his five children , reported net profits on June 18th of € 242m for 2008 and cash holdings of € 729m at year-end . -For example , with acute lymphoblastic leukemia , " we haven 't really added a new drug in 30 to 40 years , " Kamen said . -The announcement came three days after the orca pulled Brancheau into the water by her ponytail in front of park visitors . -So where do you go after interviewing the President ? -Fulham 's profligacy tested the crowd 's patience but they made the breakthrough after 56 minutes , when Dickson Etuhu met Paul Konchesky 's out-swinging corner and directed a header inside the far post . -He said the decline of the dollar , which has shed some 8 percent against the euro this year and hit a record low against a basket of major currencies , has been orderly and added that the exchange rate is set by markets . -Muller is the only man from Luxembourg to play in a Grand Slam tournament in the 40-year Open era--and now the first man or woman from the country to reach the fourth round . -The video clip has since been removed from such major state-linked sites as sina.com but is still a hot viewing item on smaller private video-sharing suppliers and has become the most talked about event in Chinese cyberspace this week . -He also defended the Treasury 's actions after it realised that new Whitehall accounting rules introduced in 2002 to cut costs had actually allowed the MoD to increase spending . -America is truly dead when we agree to pay a freak dog torturer $ 1.6 million to play a kid 's game . -In December a businessman living in Buckinghamshire returned home with his family to find " 3 " masked burglars in their home . -Mr Farage nimbly steered clear of the " spontaneous " demonstration taking place outside the hotel where he was speaking . -Analysts , on average , had expected the same-store sales to rise 1.8 percent , according to Thomson Financial . -The former president doesn 't join his wife on the campaign trail that often . -She died after suffering a brain haemorrhage on 10 September 2007 . -She says she and her husband Sean are focusing on what they have , not on what they might lose . -Isla Palenque , a new island resort community in Panama 's Gulf of Chiriqui , will offer superior service to residents and be a haven for those who decide to retire abroad . -The Pakistan army is currently fighting fierce battles against militants in Bajaur , at the north east extreme of the tribal belt , and Swat , an alpine valley in a more settled region close to the tribal lands . -A death abroad is subject to an inquest if the body is returned to Britain . -The loan comes as President Viktor Yushchenko and Prime Minister Yulia Tymoshenko are locked in a fierce battle over the president 's decision to call fresh elections . -One bore a phone number that rang at a custom ironworks company . -But even if a Conservative government proves to be more ferociously partisan compared with poor old timid New Labour I doubt if it will have cause to eject the Speaker . -The Denali pipeline consortium aims to deliver Alaskan gas to Canada and possibly to markets in the Lower 48 . -Coca cultivation surged by almost 30 percent last year in Colombia , which still provides 90 percent of the cocaine found in the United States . -The six-hitter followed his first career complete game , when he allowed three hits in a 6-3 victory over Atlanta last Wednesday . -Dow Jones industrial average futures rose 15 to 8,464 . -The exchange between Romney and Giuliani set the tone for the night in what is likely to turn into a very bitter battle for the Republican nomination . -They are knowledge intensive , rich in intellectual property and require high-level systems-integration skills . -Detectives and prosecutors stating there was " insufficient evidence " to prosecute . -Aaron Brooks had 19 points for the Rockets , who defeated Portland in six games this past spring in the first round of the playoffs . -Wolfson then read the first two paragraphs of the poll , which explained that in the 20 states where Clinton claimed popular vote victories , she led McCain by 50 percent to 43 percent for the general election . -Iteris is headquartered in Santa Ana , California . -I think Julia changed after moving for a few months and having an affair . -" The outcry from those in need of loans is substantial , " said Lee Sachs , special advisor to Treasury Secretary Timothy F. Geithner . -Seven men and one woman , aged between 18 and 53 , were due to appear at Cheltenham Magistrates Court . -Andre McGee matched his career high with 18 points and Derrick Caracter had 11 points and eight rebounds as Louisville ( 2-0 ) had little trouble with the Tigers ( 0-4 ) . -In April , the bank received a $ 7 billion private-equity investment , which helped build capital reserves . -You walk in off the street -- well , off that whopping new terminal -- straight on to a street , paved , lined with shops , bars , cafés and a pizzeria . -Michigan tourism leaders were also keeping close watch on behalf of Detroit , where casinos bring in $ 1.3 billion a year with the help with thousands of Ohio gamblers . -A central point of the president 's plan is to create a government-sponsored health insurance program that would be an option for all Americans , similar to how Medicare is now an option for Americans over age 65 . -Illinois offenders face similar restrictions , including a ban on dressing in a costume . -Assuming it is a game , of course , rather than , say , an official police procedure or a pitched and genuine battle between rival villages . -This strategy is widely considered to have been a mistake as it led to infighting between liberal and moderate Democrats , creating disillusionment among the public that led to support for healthcare reform declining the longer the debate continued . -Chu , who raised nearly $ 1 million , also had the support of the California Democratic Party and Los Angeles County Federation of Labor , which spent about $ 150,000 campaigning on her behalf . -The coffee table-ization of comics , said Mr. Liang , is partly because of another factor : the consistent growth of trade paperback editions of graphic novels , like Mr. Miller 's " The Dark Knight Returns " or Alan Moore 's " Watchmen , " since the late 1990s . -Instead , trailing by four near the end , they had to go for the try and did so from a close-range lineout won by Jonny Fa 'amatuainu . -To take the temperature of the current crop of albariños in the marketplace , the wine panel recently sampled 25 bottles , 17 from the 2006 vintage and 8 from the 2007 . -" It remains to be seen whether this [ grenade attack ] was caused by hooligans ... or other causes , " said Moldovan Interior Minister Viktor Katan . -In 2002 and 2003 the group ran programmes analysing urine from Afghans . -Joe Biden , in a lot of his speeches , is delivering them in a preacher sort of fashion that tunes really well . -" Thanks Billy , " she says , giving him the thumbs up . -Mr Kadyrov denied any involvement and promised to investigate the killing personally . -But Mr Hague said he had only found out in " the last few months " that the peer , who has donated millions to the Tories , had later renegotiated the deal with government officials . -Indeed , as soon as the motorcade arrived at his downtown hotel , agents began preparing for an imminent departure to an undisclosed dinner location . -The pictures , which capture well-known local scenes , will be on display at the Hadley Gallery . -The filmmakers ' decision to include a four-minute montage tracing the back story of Carl Fredricksen and his wife was a bold stroke that enriched the entire movie . -Overall , family income of blacks in their 30s was $ 35,000 , 58 percent that of comparable whites , a gap that did not surprise researchers . -On the security side , by keeping on George Bush 's first-rate defence secretary , Robert Gates , and ( probably ) by choosing a former general , Jim Jones , as his national security adviser , Mr Obama is showing that he will not let himself be tagged with the " Defeaticrat " label . -My vacations inevitably last an additional week as I return to work , fully jet-lagged . -The firing came just five months before her scheduled retirement . -Mr. Reilly joined Korn / Ferry in June 2001 as Chairman and Chief Executive Officer . -That means it will most likely be obligated to finance a considerably smaller amount as well as tie up less bank capital , though it still could amount to billions of dollars . -" It 's a backdoor tax increase , " said Sen. George Runner ( R-Lancaster ) . -" I 'm a Christian . -The topic of several biographies , Thomas , who was confirmed to the court by a 52-48 margin , has also been described as disinterested because he does not ask questions during oral presentations before the court . -" I 'm used to it blowing in Hawaii , but here it is 30 degrees colder . -The company pitch was that , for a $ 50,000 investment , people could buy homes and not worry about the price because Metro Dream Homes would make their mortgage payments . -The recommendation targets men with a normal reading on the prostate-specific antigen or PSA , test , which is considered the best indicator of the presence of a tumor , because clinical trials of the drug covered only such men . -Students , staff and faculty aim to paralyze the University of California on what is the first day of class on most campuses . -It is not a dish for the chaotic cook or anyone short of time . -An old , yellowed hotel register bears the signatures of Presidents Benjamin Harrison and Grover Cleveland . -The wife of embattled S.C. governor Mark Sanford speaks out for the first time , ... -It was his franchise record 16th RBI of the postseason . -I would have understood completely had he told ( England manager ) Fabio Capello he did not want to play against Egypt . -" The access expansions are a significant step forward , but this legislation will exacerbate the health care costs crisis facing many working families and small businesses , " Karen Ignagni , president of the America 's Health Insurance Plans group , said in a statement . -Yet the drones are unpopular with many Pakistanis , who see them as a violation of their country 's sovereignty -- one reason the United States refuses to officially acknowledge the attacks . -The Penguins managed one shot through the first 15 minutes , but finally turned it around late in the period . -LVG went into administration last Thursday . -The event took place in a hotel ballroom in the heart of Berlin . -1 , 1917 , in Leechburg , Pa . , and graduated from Monmouth College in 1938 . -A few thousand people in Spain 's northern Basque region have police or private bodyguards due to threats by ETA , which is blamed for more than 800 deaths in its long fight for Basque independence . -When the winter rains start , there is little vegetation left in the burn areas to prevent water , rocks , grit and branches from coursing down steep canyons and ravines toward thousands of homes . -However , Mr Donaldson said he should not have claimed for these items . -The Phillies got their first win while sporting a new alternate home uniform , a throwback model from the 1940s minus the customary red pinstripes the team has worn since 1950 . -They don 't care where we are . -I encountered that scam as well with United Healthcare . -Anywhere from 40,000 to 50,000 users are active at any time . -The Shea Stadium boos , which had been reserved for Chipper Jones and Luis Castillo until that point , rang out in full force . -Then Goma became a launching pad for two civil wars , one of which escalated into a regional conflict known as Africa 's First World War . -No one was hurt there , The Associated Press reported . -Further , the new bank holding company model adopted by Goldman Sachs and Morgan Stanley after rival Lehman Brothers was allowed to sink into bankruptcy , comes with regulations that did not apply to these titans when they were investment banks . -Advancing issues outnumbered decliners by about 3 to 1 on the New York Stock Exchange , where volume came to 940.1 million shares . -Homeowners can apply for a grant of up to £ 2,500 to install the technology . -The Europeans can get information and tell the world our demands . -Dallas activist Elizabeth Villafranca praised the News for its courage in giving the immigration debate such importance . -So Neil Gaiman , author of " Coraline " and other dark tales , can get weirded out from time to time . -By the time the team arrived , it was clear that word of the trip had spread like wildfire in the city . -" I honestly believe I can get a lot better . -6 , 2001 , to trigger the plaintiffs ' duty to investigate the alleged fraud . -Thornton later lost the ball to Pistons center Jason Maxiell , leading to an alley-oop dunk by Prince , forcing Saunders to call a timeout . -Most Kenyans cannot get hold of a copy . -The government has already announced plans to add a new course called " economic wellbeing " to PSHE from September 2008 , but it will remain voluntary . -One idea he has floated is a reduction , to 25 percent from 68 percent , in the state 's annual share of the revenue from the 1,500 slot machines that he plans to move from the Monticello Raceway . -The case comes in a state that was at the heart of the U.S. civil rights movement in the 1960s . -He then devoured a parade of oysters , steak , macaroni and cheese , mashed potatoes and several desserts . -After conquering the world 's tallest peak on May 29 , 1953 , Hillary returned to Nepal several times and founded the Himalayan Trust , which has built 27 schools , two hospitals and 12 clinics around Mount Everest . -Others will not be so kind . -" If you just traveled to Mexico and you 're home and feeling well you should get on with your daily life and not worry about it , " Evans said . -Dolsi ( 0-1 ) intentionally walked Torii Hunter before Seay came on . -A delegation including officials from the Chinese Development Bank recently spent a week in Guinea and discussing a deal which could see billions of dollars of Chinese investment in return for mining rights . -Alfred Mann , 87 , has struggled to eat and speak since he was injured in the land mine blast while serving with Royal Army Medical Corps in Italy in 1944 . -The doctor observed that a person suffering from Mr Doran 's symptoms would , in a civilian setting , normally be admitted to a hospital . -Parry has considered leaving Anfield once already this season when he was sounded out about the chief executive 's position at England 's 2018 World Cup bid . -You can look down into the space -- and at its bold graffiti-painted walls . -The woman , said as her " humiliation and terror " mounted she did her best to " switch off , " realising resistance was not only futile but could end in her murder . -A strike ballot is due to be held at Airwave from 24 February until 1 March . -Acting on police advice , the organisers decided to cancel the exhibition of artwork by 11 young ethnic Albanians from the Kosovo capital Pristina , Ljubica Beljanski Ristic told the news agency . -Gossage was anything but soft , and though he was good at his new job he didn 't relish it initially . -Capt. Thomas Ohlhaber of the U.S. Public Health Service -- who serves as deputy director of the Food and Drug Administration 's division of mammography quality and radiation programs -- describes X-rays as one of medicine 's most remarkable achievements . -Lombardi : As far as being able to throw out runners , I 'm not overly concerned about that issue . -" I think our number is either right on or negative equity may be even a little worse , " he said . -The correct figure of around 160,000 who have benefited from " time to pay " has been used by the chancellor , Alistair Darling , and Stephen Timms , the Treasury minister . -But if you show the least sign of nerves , the place gets nasty . -Roach immediately came on to pepper him with short balls and after he got one hook shot away , he fended one straight to short leg to be out for two to leave Australia on 134 for 8 . -But he said 2009 had been a bad year , with poachers killing 12 black rhinos and six white ones . -With public anger over the financial crisis and Wall Street bailouts channeled against the Fed , there is pressure on those backing Mr Bernanke to distance themselves from the central bank in a subsequent vote . -States classified ever larger numbers of young offenders as adults . -So where does this leave Obama 's promise to replace the fist of his White House predecessor and extend an open hand to Iran ? -D 'Arcy had been due to fly to Britain this week to compete in next month 's short-course world championships in Manchester but the Brisbane university student withdrew from the short-course team to consult his lawyers . -Shaken , she then rang her 32 year-old husband , Rob , who is also a vet . -It wasn 't your first effort at a project like this . -It occurs to me that perhaps you can be in fashion and normal after all . -Second-quarter premiums rose 25 percent to $ 3.49 billion , primarily due to a 17 percent increase in average membership and higher per member premium revenues . -Such systems do not seem to lead to the normalisation of unnatural death , and thus a steadily rising number of suicides ; rather , after an initial surge , the number seeking help to die drops and then stays steady . -" We 've got an agreement on an offer that will be made to the government of Iran , " David Miliband , the British foreign secretary , said for the six governments . -It looks for salvation in a revival of what Mr. Gioni calls the " mythopoetic function of art " and in the fantasies of the prophet in the wilderness . -Judging from the noises emanating from some corners of Washington these days , the federal debt has assumed pride of place as the source of national anxiety . -Construction has begun on approximately 60 new homes in a Jewish settlement in Israeli-occupied East Jerusalem , the Israeli campaign group Peace Now says . -" We were nomads , but that is being extinguished by this policy . -Finger pointing and namecalling isn 't becoming of a president . -ONStor , the ONStor logo , Pantera and Bobcat are trademarks of ONStor , Inc. in the U.S. and other countries. be superseded by subsequent documents and is subject to change without notice . -Wickmayer is seeking her second tournament victory in a row after winning last week 's title in Linz , Austria . -The North also appeared to be preparing for more missile tests , including one believed to be capable of reaching the U.S. A U.S. measure imposed on Banco Delta Asia , a bank in the Chinese territory of Macau , in 2005 effectively led to the North being severed from the international financial system , as other institutions voluntarily severed their dealings with the bank and the North . -A spokeswoman said : " Refurbishment of the coping stones and railings was well overdue . -But the last eight wickets fell for 50 runs and Scotland must now beat UAE on Friday , while hoping Afghanistan and Netherlands lose their final games . -That only inspired Ferrer , winner of the Valencia title last month , with the second seed re-breaking and serving out the set to bring on a deciding third . -THC and CBD appear to help people with MS control their bladders . -For the first six weeks , Jan and Sam shuttled back and forth from their home in Surbiton to the neonatal unit at Kingston Hospital . -The front two , facing the street , are carpeted , and one has a walk-in closet . -The rocket that narrowly missed the oncoming Sri Lankan bus had slammed into a parade of shops - reducing one shopfront to cinders . -About 6 hours after Capt. Thurman stopped by the side of the road to help out , his truck was hit by an IED . -The world 's key money markets were in a state of near-seizure for much of yesterday as the shockwaves from Lehman 's closure sparked a panicky scramble by investors to dump shares and risky assets and secure short-term cash to see them through the worsening financial storm. seven years . -England manager Fabio Capello has said Manchester United are not the " war machine " that once ruled the Premier League . -Regarding the private military contractor Blackwater , Bush said he was sorry " that innocent lives were lost , " but declined to comment further until an investigation was complete . -Senator Obama rebutted , point-by-point , each of the sleazy Rove-like attacks on his patriotism and vision . -They 've won three times by 20 points or more . -He increased his volume out of public view , in an intrasquad game and two simulations , and used one of those simulated games to practice how he would pitch to Boston hitters . diff --git a/resources/docs/TUTORIAL_TRAINING_A_MODEL.md b/resources/docs/TUTORIAL_TRAINING_A_MODEL.md index 80cb0e2e43..93bc979508 100644 --- a/resources/docs/TUTORIAL_TRAINING_A_MODEL.md +++ b/resources/docs/TUTORIAL_TRAINING_A_MODEL.md @@ -133,6 +133,13 @@ trainer.train('resources/taggers/example-ner', learning_rate=0.1, mini_batch_size=32, max_epochs=150) + +# 8. plot training curves (optional) +from flair.visual.training_curves import Plotter +plotter = Plotter() +plotter.plot_training_curves('resources/taggers/example-ner/loss.tsv') +plotter.plot_weights('resources/taggers/example-ner/weights.txt') + ``` Alternatively, try using a stacked embedding with charLM and glove, over the full data, for 150 epochs. @@ -201,7 +208,10 @@ word_embeddings = [WordEmbeddings('glove'), CharLMEmbeddings('news-backward')] # 4. init document embedding by passing list of word embeddings -document_embeddings: DocumentLSTMEmbeddings = DocumentLSTMEmbeddings(word_embeddings, hidden_states=512) +document_embeddings: DocumentLSTMEmbeddings = DocumentLSTMEmbeddings(word_embeddings, + hidden_states=512, + reproject_words=True, + reproject_words_dimension=256,) # 5. create the text classifier classifier = TextClassifier(document_embeddings, label_dictionary=label_dict, multi_label=False) @@ -213,7 +223,15 @@ trainer = TextClassifierTrainer(classifier, corpus, label_dict) trainer.train('resources/ag_news/results', learning_rate=0.1, mini_batch_size=32, + anneal_factor=0.5, + patience=5, max_epochs=150) + +# 8. plot training curves (optional) +from flair.visual.training_curves import Plotter +plotter = Plotter() +plotter.plot_training_curves('resources/ag_news/results/loss.tsv') +plotter.plot_weights('resources/ag_news/results/weights.txt') ``` Once the model is trained you can use it to predict the class of new sentences. Just call the `predict` method of the From 24bc299b7e8a566aca52375caae5673efe7b264a Mon Sep 17 00:00:00 2001 From: tabergma Date: Thu, 4 Oct 2018 11:45:23 +0200 Subject: [PATCH 051/113] GH-61: Update test for visuals --- flair/__init__.py | 1 + flair/visual/activations.py | 1 + tests/resources/visual/loss.tsv | 21 + tests/resources/visual/snippet.txt | 10 + tests/resources/visual/weights.txt | 2600 ++++++++++++++++++++++++++++ tests/test_visual.py | 221 +-- 6 files changed, 2714 insertions(+), 140 deletions(-) create mode 100644 tests/resources/visual/loss.tsv create mode 100644 tests/resources/visual/snippet.txt create mode 100644 tests/resources/visual/weights.txt diff --git a/flair/__init__.py b/flair/__init__.py index 98200efa19..a70a3a088d 100644 --- a/flair/__init__.py +++ b/flair/__init__.py @@ -1,6 +1,7 @@ from . import data from . import models from . import visual +from . import trainers import sys import logging diff --git a/flair/visual/activations.py b/flair/visual/activations.py index c7837f3958..33740ac81a 100644 --- a/flair/visual/activations.py +++ b/flair/visual/activations.py @@ -2,6 +2,7 @@ class Highlighter: + def __init__(self): self.color_map = [ "#ff0000", diff --git a/tests/resources/visual/loss.tsv b/tests/resources/visual/loss.tsv new file mode 100644 index 0000000000..9a351e923b --- /dev/null +++ b/tests/resources/visual/loss.tsv @@ -0,0 +1,21 @@ +EPOCH TIMESTAMP TRAIN_LOSS TRAIN_TP TRAIN_TN TRAIN_FP TRAIN_FN TRAIN_PRECISION TRAIN_RECALL TRAIN_F-SCORE TRAIN_ACCURACY DEV_LOSS DEV_TP DEV_TN DEV_FP DEV_FN DEV_PRECISION DEV_RECALL DEV_F-SCORE DEV_ACCURACY TEST_LOSS TEST_TP TEST_TN TEST_FP TEST_FN TEST_PRECISION TEST_RECALL TEST_F-SCORE TEST_ACCURACY +0 10:33:21 _ _ _ _ _ _ _ _ _ _ 1.0 0.0 22.0 23.0 0.043478260869565216 0.041666666666666664 0.0425531914893617 0.021739130434782608 _ 2.0 0.0 44.0 46.0 0.043478260869565216 0.041666666666666664 0.0425531914893617 0.021739130434782608 +1 10:33:24 _ _ _ _ _ _ _ _ _ _ 1.0 0.0 24.0 23.0 0.04 0.041666666666666664 0.04081632653061224 0.020833333333333332 _ 2.0 0.0 48.0 46.0 0.04 0.041666666666666664 0.04081632653061224 0.020833333333333332 +2 10:33:24 _ _ _ _ _ _ _ _ _ _ 3.0 0.0 23.0 21.0 0.11538461538461539 0.125 0.12000000000000001 0.06382978723404255 _ 6.0 0.0 46.0 42.0 0.11538461538461539 0.125 0.12000000000000001 0.06382978723404255 +3 10:33:29 _ _ _ _ _ _ _ _ _ _ 4.0 0.0 22.0 20.0 0.15384615384615385 0.16666666666666666 0.16 0.08695652173913043 _ 8.0 0.0 44.0 40.0 0.15384615384615385 0.16666666666666666 0.16 0.08695652173913043 +4 10:33:33 _ _ _ _ _ _ _ _ _ _ 2.0 0.0 22.0 22.0 0.08333333333333333 0.08333333333333333 0.08333333333333333 0.043478260869565216 _ 4.0 0.0 44.0 44.0 0.08333333333333333 0.08333333333333333 0.08333333333333333 0.043478260869565216 +5 10:33:33 _ _ _ _ _ _ _ _ _ _ 4.0 0.0 22.0 20.0 0.15384615384615385 0.16666666666666666 0.16 0.08695652173913043 _ 8.0 0.0 44.0 40.0 0.15384615384615385 0.16666666666666666 0.16 0.08695652173913043 +6 10:33:37 _ _ _ _ _ _ _ _ _ _ 4.0 0.0 22.0 20.0 0.15384615384615385 0.16666666666666666 0.16 0.08695652173913043 _ 8.0 0.0 44.0 40.0 0.15384615384615385 0.16666666666666666 0.16 0.08695652173913043 +7 10:33:41 _ _ _ _ _ _ _ _ _ _ 5.0 0.0 21.0 19.0 0.19230769230769232 0.20833333333333334 0.2 0.1111111111111111 _ 10.0 0.0 42.0 38.0 0.19230769230769232 0.20833333333333334 0.2 0.1111111111111111 +8 10:33:46 _ _ _ _ _ _ _ _ _ _ 5.0 0.0 21.0 19.0 0.19230769230769232 0.20833333333333334 0.2 0.1111111111111111 _ 10.0 0.0 42.0 38.0 0.19230769230769232 0.20833333333333334 0.2 0.1111111111111111 +9 10:33:50 _ _ _ _ _ _ _ _ _ _ 7.0 0.0 19.0 17.0 0.2692307692307692 0.2916666666666667 0.27999999999999997 0.16279069767441862 _ 14.0 0.0 38.0 34.0 0.2692307692307692 0.2916666666666667 0.27999999999999997 0.16279069767441862 +10 10:33:56 _ _ _ _ _ _ _ _ _ _ 10.0 0.0 16.0 14.0 0.38461538461538464 0.4166666666666667 0.4 0.25 _ 20.0 0.0 32.0 28.0 0.38461538461538464 0.4166666666666667 0.4 0.25 +11 10:34:00 _ _ _ _ _ _ _ _ _ _ 6.0 0.0 20.0 18.0 0.23076923076923078 0.25 0.24000000000000002 0.13636363636363635 _ 12.0 0.0 40.0 36.0 0.23076923076923078 0.25 0.24000000000000002 0.13636363636363635 +12 10:34:01 _ _ _ _ _ _ _ _ _ _ 12.0 0.0 14.0 12.0 0.46153846153846156 0.5 0.48000000000000004 0.3157894736842105 _ 24.0 0.0 28.0 24.0 0.46153846153846156 0.5 0.48000000000000004 0.3157894736842105 +13 10:34:06 _ _ _ _ _ _ _ _ _ _ 15.0 0.0 11.0 9.0 0.5769230769230769 0.625 0.6 0.42857142857142855 _ 30.0 0.0 22.0 18.0 0.5769230769230769 0.625 0.6 0.42857142857142855 +14 10:34:11 _ _ _ _ _ _ _ _ _ _ 14.0 0.0 12.0 10.0 0.5384615384615384 0.5833333333333334 0.5599999999999999 0.3888888888888889 _ 28.0 0.0 24.0 20.0 0.5384615384615384 0.5833333333333334 0.5599999999999999 0.3888888888888889 +15 10:34:11 _ _ _ _ _ _ _ _ _ _ 15.0 0.0 10.0 9.0 0.6 0.625 0.6122448979591836 0.4411764705882353 _ 30.0 0.0 20.0 18.0 0.6 0.625 0.6122448979591836 0.4411764705882353 +16 10:34:15 _ _ _ _ _ _ _ _ _ _ 17.0 0.0 9.0 7.0 0.6538461538461539 0.7083333333333334 0.68 0.5151515151515151 _ 34.0 0.0 18.0 14.0 0.6538461538461539 0.7083333333333334 0.68 0.5151515151515151 +17 10:34:20 _ _ _ _ _ _ _ _ _ _ 13.0 0.0 13.0 11.0 0.5 0.5416666666666666 0.52 0.35135135135135137 _ 26.0 0.0 26.0 22.0 0.5 0.5416666666666666 0.52 0.35135135135135137 +18 10:34:20 _ _ _ _ _ _ _ _ _ _ 17.0 0.0 8.0 7.0 0.68 0.7083333333333334 0.6938775510204083 0.53125 _ 34.0 0.0 16.0 14.0 0.68 0.7083333333333334 0.6938775510204083 0.53125 +19 10:34:24 _ _ _ _ _ _ _ _ _ _ 16.0 0.0 9.0 8.0 0.64 0.6666666666666666 0.6530612244897959 0.48484848484848486 _ 32.0 0.0 18.0 16.0 0.64 0.6666666666666666 0.6530612244897959 0.48484848484848486 diff --git a/tests/resources/visual/snippet.txt b/tests/resources/visual/snippet.txt new file mode 100644 index 0000000000..a87671784e --- /dev/null +++ b/tests/resources/visual/snippet.txt @@ -0,0 +1,10 @@ +The U.S. Centers for Disease Control and Prevention initially advised school systems to close if outbreaks occurred , then reversed itself , saying the apparent mildness of the virus meant most schools and day care centers should stay open , even if they had confirmed cases of swine flu . +When Ms. Winfrey invited Suzanne Somers to share her controversial views about bio-identical hormone treatment on her syndicated show in 2009 , it won Ms. Winfrey a rare dollop of unflattering press , including a Newsweek cover story titled " Crazy Talk : Oprah , Wacky Cures & You . " +Elk calling -- a skill that hunters perfected long ago to lure game with the promise of a little romance -- is now its own sport . +Don 't ! +Fish , ranked 98th in the world , fired 22 aces en route to a 6-3 , 6-7 ( 5 / 7 ) , 7-6 ( 7 / 4 ) win over seventh-seeded Argentinian David Nalbandian . +Why does everything have to become such a big issue ? +AMMAN ( Reuters ) - King Abdullah of Jordan will meet U.S. President Barack Obama in Washington on April 21 to lobby on behalf of Arab states for a stronger U.S. role in Middle East peacemaking , palace officials said on Sunday . +To help keep traffic flowing the Congestion Charge will remain in operation through-out the strike and TfL will be suspending road works on major London roads wherever possible . +If no candidate wins an absolute majority , there will be a runoff between the top two contenders , most likely in mid-October . +Authorities previously served search warrants at Murray 's Las Vegas home and his businesses in Las Vegas and Houston . \ No newline at end of file diff --git a/tests/resources/visual/weights.txt b/tests/resources/visual/weights.txt new file mode 100644 index 0000000000..800937e23d --- /dev/null +++ b/tests/resources/visual/weights.txt @@ -0,0 +1,2600 @@ +0 transitions 0 0.8488653302192688 +0 transitions 1 0.6510334014892578 +0 transitions 2 0.41793251037597656 +0 transitions 3 0.2504116892814636 +0 transitions 4 -0.6688763499259949 +0 transitions 5 1.5638210773468018 +0 transitions 6 -0.3447156846523285 +0 transitions 7 0.9721583127975464 +0 transitions 8 1.1572129726409912 +0 transitions 9 -0.8548757433891296 +0 embedding2nn.weight 0 0.0046721757389605045 +0 embedding2nn.weight 1 -0.0384654626250267 +0 embedding2nn.weight 2 -0.01874154806137085 +0 embedding2nn.weight 3 -0.05152992904186249 +0 embedding2nn.weight 4 -0.012047011405229568 +0 embedding2nn.weight 5 0.07555867731571198 +0 embedding2nn.weight 6 0.064902663230896 +0 embedding2nn.weight 7 -0.0649118423461914 +0 embedding2nn.weight 8 0.005098057445138693 +0 embedding2nn.weight 9 0.08622648566961288 +0 embedding2nn.bias 0 -0.0578378327190876 +0 embedding2nn.bias 1 -0.06655219197273254 +0 embedding2nn.bias 2 -0.040699563920497894 +0 embedding2nn.bias 3 0.022399229928851128 +0 embedding2nn.bias 4 -0.05672124773263931 +0 embedding2nn.bias 5 -0.07599932700395584 +0 embedding2nn.bias 6 -0.03673562780022621 +0 embedding2nn.bias 7 -0.07199881225824356 +0 embedding2nn.bias 8 0.060428667813539505 +0 embedding2nn.bias 9 0.006095290184020996 +0 rnn.weight_ih_l0 0 0.05006050318479538 +0 rnn.weight_ih_l0 1 -0.021499549970030785 +0 rnn.weight_ih_l0 2 -0.0027262847870588303 +0 rnn.weight_ih_l0 3 -0.037493929266929626 +0 rnn.weight_ih_l0 4 -0.005006689112633467 +0 rnn.weight_ih_l0 5 -0.05041030794382095 +0 rnn.weight_ih_l0 6 0.015371788293123245 +0 rnn.weight_ih_l0 7 0.04460011050105095 +0 rnn.weight_ih_l0 8 0.008027112111449242 +0 rnn.weight_ih_l0 9 0.04270688816905022 +0 rnn.weight_hh_l0 0 0.0367879718542099 +0 rnn.weight_hh_l0 1 -0.044259097427129745 +0 rnn.weight_hh_l0 2 0.020844420418143272 +0 rnn.weight_hh_l0 3 0.028410878032445908 +0 rnn.weight_hh_l0 4 0.06242316588759422 +0 rnn.weight_hh_l0 5 0.04843023419380188 +0 rnn.weight_hh_l0 6 0.04352244734764099 +0 rnn.weight_hh_l0 7 -0.05742735415697098 +0 rnn.weight_hh_l0 8 0.06215953081846237 +0 rnn.weight_hh_l0 9 -0.045235682278871536 +0 rnn.bias_ih_l0 0 -0.03819615766406059 +0 rnn.bias_ih_l0 1 -0.023351773619651794 +0 rnn.bias_ih_l0 2 0.058041419833898544 +0 rnn.bias_ih_l0 3 -0.013405791483819485 +0 rnn.bias_ih_l0 4 0.062360282987356186 +0 rnn.bias_ih_l0 5 0.04511547461152077 +0 rnn.bias_ih_l0 6 -0.057125575840473175 +0 rnn.bias_ih_l0 7 -0.0064422497525811195 +0 rnn.bias_ih_l0 8 0.024938471615314484 +0 rnn.bias_ih_l0 9 0.02243085205554962 +0 rnn.bias_hh_l0 0 -0.05880212411284447 +0 rnn.bias_hh_l0 1 -0.04119813069701195 +0 rnn.bias_hh_l0 2 0.02806517668068409 +0 rnn.bias_hh_l0 3 0.04633007571101189 +0 rnn.bias_hh_l0 4 -0.04893298074603081 +0 rnn.bias_hh_l0 5 0.01460266299545765 +0 rnn.bias_hh_l0 6 0.0466526634991169 +0 rnn.bias_hh_l0 7 -0.03780452534556389 +0 rnn.bias_hh_l0 8 -0.028553897514939308 +0 rnn.bias_hh_l0 9 0.005791562609374523 +0 rnn.weight_ih_l0_reverse 0 -0.03489518538117409 +0 rnn.weight_ih_l0_reverse 1 -0.041398193687200546 +0 rnn.weight_ih_l0_reverse 2 0.037643130868673325 +0 rnn.weight_ih_l0_reverse 3 -0.008332296274602413 +0 rnn.weight_ih_l0_reverse 4 -0.05011044815182686 +0 rnn.weight_ih_l0_reverse 5 0.00031758344266563654 +0 rnn.weight_ih_l0_reverse 6 0.012645721435546875 +0 rnn.weight_ih_l0_reverse 7 0.05739935114979744 +0 rnn.weight_ih_l0_reverse 8 0.01909276284277439 +0 rnn.weight_ih_l0_reverse 9 0.05574442073702812 +0 rnn.weight_hh_l0_reverse 0 0.060682252049446106 +0 rnn.weight_hh_l0_reverse 1 0.004405428655445576 +0 rnn.weight_hh_l0_reverse 2 0.03072367049753666 +0 rnn.weight_hh_l0_reverse 3 0.04655856639146805 +0 rnn.weight_hh_l0_reverse 4 -0.017155855894088745 +0 rnn.weight_hh_l0_reverse 5 0.044554755091667175 +0 rnn.weight_hh_l0_reverse 6 0.04368443787097931 +0 rnn.weight_hh_l0_reverse 7 0.024221844971179962 +0 rnn.weight_hh_l0_reverse 8 -0.03680572286248207 +0 rnn.weight_hh_l0_reverse 9 -0.004940468817949295 +0 rnn.bias_ih_l0_reverse 0 0.054941654205322266 +0 rnn.bias_ih_l0_reverse 1 0.001728854957036674 +0 rnn.bias_ih_l0_reverse 2 0.05006153881549835 +0 rnn.bias_ih_l0_reverse 3 -0.03612877428531647 +0 rnn.bias_ih_l0_reverse 4 -0.028129132464528084 +0 rnn.bias_ih_l0_reverse 5 0.048473846167325974 +0 rnn.bias_ih_l0_reverse 6 -0.027244167402386665 +0 rnn.bias_ih_l0_reverse 7 0.050763607025146484 +0 rnn.bias_ih_l0_reverse 8 0.007703111041337252 +0 rnn.bias_ih_l0_reverse 9 0.06227635592222214 +0 rnn.bias_hh_l0_reverse 0 -0.05641821771860123 +0 rnn.bias_hh_l0_reverse 1 -0.03942378982901573 +0 rnn.bias_hh_l0_reverse 2 -0.03047890029847622 +0 rnn.bias_hh_l0_reverse 3 0.022562172263860703 +0 rnn.bias_hh_l0_reverse 4 0.040990155190229416 +0 rnn.bias_hh_l0_reverse 5 0.050864558666944504 +0 rnn.bias_hh_l0_reverse 6 0.04727538302540779 +0 rnn.bias_hh_l0_reverse 7 -0.04192955046892166 +0 rnn.bias_hh_l0_reverse 8 -0.06118109077215195 +0 rnn.bias_hh_l0_reverse 9 0.03684917464852333 +0 linear.weight 0 0.027342354878783226 +0 linear.weight 1 -0.02168887108564377 +0 linear.weight 2 -0.005224860738962889 +0 linear.weight 3 -0.0047921231016516685 +0 linear.weight 4 -0.004396989941596985 +0 linear.weight 5 0.03988152742385864 +0 linear.weight 6 0.03439749404788017 +0 linear.weight 7 -0.04843636602163315 +0 linear.weight 8 -0.0344756618142128 +0 linear.weight 9 0.013581499457359314 +0 linear.bias 0 -0.05875803530216217 +0 linear.bias 1 -0.04567436873912811 +0 linear.bias 2 -0.08127401024103165 +0 linear.bias 3 -0.05690110847353935 +0 linear.bias 4 0.05011451989412308 +0 linear.bias 5 0.02154530957341194 +0 linear.bias 6 0.010653559118509293 +0 linear.bias 7 -0.009388732723891735 +0 linear.bias 8 0.045184966176748276 +0 linear.bias 9 0.006394088733941317 +1 transitions 0 0.8429397344589233 +1 transitions 1 0.6481037139892578 +1 transitions 2 0.4151831865310669 +1 transitions 3 0.24783922731876373 +1 transitions 4 -0.6343594789505005 +1 transitions 5 1.5520538091659546 +1 transitions 6 -0.34661418199539185 +1 transitions 7 0.967778742313385 +1 transitions 8 1.1532642841339111 +1 transitions 9 -0.856695830821991 +1 embedding2nn.weight 0 0.004352289251983166 +1 embedding2nn.weight 1 -0.03984331712126732 +1 embedding2nn.weight 2 -0.018959809094667435 +1 embedding2nn.weight 3 -0.051843058317899704 +1 embedding2nn.weight 4 -0.00913836807012558 +1 embedding2nn.weight 5 0.07535009831190109 +1 embedding2nn.weight 6 0.06516997516155243 +1 embedding2nn.weight 7 -0.0649118423461914 +1 embedding2nn.weight 8 0.004159066826105118 +1 embedding2nn.weight 9 0.08541078120470047 +1 embedding2nn.bias 0 -0.06083976849913597 +1 embedding2nn.bias 1 -0.07378105074167252 +1 embedding2nn.bias 2 -0.04119514673948288 +1 embedding2nn.bias 3 0.01694539003074169 +1 embedding2nn.bias 4 -0.05607350170612335 +1 embedding2nn.bias 5 -0.07934246957302094 +1 embedding2nn.bias 6 -0.03997217118740082 +1 embedding2nn.bias 7 -0.07251674681901932 +1 embedding2nn.bias 8 0.06297324597835541 +1 embedding2nn.bias 9 0.007947472855448723 +1 rnn.weight_ih_l0 0 0.05004507675766945 +1 rnn.weight_ih_l0 1 -0.021402092650532722 +1 rnn.weight_ih_l0 2 -0.002648176159709692 +1 rnn.weight_ih_l0 3 -0.037485986948013306 +1 rnn.weight_ih_l0 4 -0.006147007457911968 +1 rnn.weight_ih_l0 5 -0.050431784242391586 +1 rnn.weight_ih_l0 6 0.014947121031582355 +1 rnn.weight_ih_l0 7 0.04462089017033577 +1 rnn.weight_ih_l0 8 0.00803365744650364 +1 rnn.weight_ih_l0 9 0.04270697757601738 +1 rnn.weight_hh_l0 0 0.03678184375166893 +1 rnn.weight_hh_l0 1 -0.044212061911821365 +1 rnn.weight_hh_l0 2 0.020849518477916718 +1 rnn.weight_hh_l0 3 0.02840355783700943 +1 rnn.weight_hh_l0 4 0.06242426484823227 +1 rnn.weight_hh_l0 5 0.048431675881147385 +1 rnn.weight_hh_l0 6 0.04352192208170891 +1 rnn.weight_hh_l0 7 -0.057442668825387955 +1 rnn.weight_hh_l0 8 0.06215580180287361 +1 rnn.weight_hh_l0 9 -0.04523494839668274 +1 rnn.bias_ih_l0 0 -0.03811132535338402 +1 rnn.bias_ih_l0 1 -0.023278681561350822 +1 rnn.bias_ih_l0 2 0.058053821325302124 +1 rnn.bias_ih_l0 3 -0.012766578234732151 +1 rnn.bias_ih_l0 4 0.06232045218348503 +1 rnn.bias_ih_l0 5 0.04519758000969887 +1 rnn.bias_ih_l0 6 -0.05662884935736656 +1 rnn.bias_ih_l0 7 -0.007610830944031477 +1 rnn.bias_ih_l0 8 0.029823077842593193 +1 rnn.bias_ih_l0 9 0.02243022620677948 +1 rnn.bias_hh_l0 0 -0.05880935117602348 +1 rnn.bias_hh_l0 1 -0.04122747480869293 +1 rnn.bias_hh_l0 2 0.02813774347305298 +1 rnn.bias_hh_l0 3 0.04748278483748436 +1 rnn.bias_hh_l0 4 -0.04899214208126068 +1 rnn.bias_hh_l0 5 0.014561556279659271 +1 rnn.bias_hh_l0 6 0.04664308577775955 +1 rnn.bias_hh_l0 7 -0.037791792303323746 +1 rnn.bias_hh_l0 8 -0.028484992682933807 +1 rnn.bias_hh_l0 9 0.0057825264520943165 +1 rnn.weight_ih_l0_reverse 0 -0.03472187742590904 +1 rnn.weight_ih_l0_reverse 1 -0.041451554745435715 +1 rnn.weight_ih_l0_reverse 2 0.03771577402949333 +1 rnn.weight_ih_l0_reverse 3 -0.00840710662305355 +1 rnn.weight_ih_l0_reverse 4 -0.05013401433825493 +1 rnn.weight_ih_l0_reverse 5 0.0019854239653795958 +1 rnn.weight_ih_l0_reverse 6 0.012566552497446537 +1 rnn.weight_ih_l0_reverse 7 0.057647980749607086 +1 rnn.weight_ih_l0_reverse 8 0.019030306488275528 +1 rnn.weight_ih_l0_reverse 9 0.05582259222865105 +1 rnn.weight_hh_l0_reverse 0 0.060686055570840836 +1 rnn.weight_hh_l0_reverse 1 0.004408028908073902 +1 rnn.weight_hh_l0_reverse 2 0.030724063515663147 +1 rnn.weight_hh_l0_reverse 3 0.046549875289201736 +1 rnn.weight_hh_l0_reverse 4 -0.01715804636478424 +1 rnn.weight_hh_l0_reverse 5 0.044564228504896164 +1 rnn.weight_hh_l0_reverse 6 0.04379921033978462 +1 rnn.weight_hh_l0_reverse 7 0.02409685030579567 +1 rnn.weight_hh_l0_reverse 8 -0.03680695593357086 +1 rnn.weight_hh_l0_reverse 9 -0.004934090189635754 +1 rnn.bias_ih_l0_reverse 0 0.05480990931391716 +1 rnn.bias_ih_l0_reverse 1 0.0017841154476627707 +1 rnn.bias_ih_l0_reverse 2 0.05003868788480759 +1 rnn.bias_ih_l0_reverse 3 -0.0373697355389595 +1 rnn.bias_ih_l0_reverse 4 -0.02811477892100811 +1 rnn.bias_ih_l0_reverse 5 0.04738800972700119 +1 rnn.bias_ih_l0_reverse 6 -0.02145535498857498 +1 rnn.bias_ih_l0_reverse 7 0.05084370821714401 +1 rnn.bias_ih_l0_reverse 8 0.0076798684895038605 +1 rnn.bias_ih_l0_reverse 9 0.06222688779234886 +1 rnn.bias_hh_l0_reverse 0 -0.05403060466051102 +1 rnn.bias_hh_l0_reverse 1 -0.03943980857729912 +1 rnn.bias_hh_l0_reverse 2 -0.030373800545930862 +1 rnn.bias_hh_l0_reverse 3 0.022536205127835274 +1 rnn.bias_hh_l0_reverse 4 0.04096434265375137 +1 rnn.bias_hh_l0_reverse 5 0.050826601684093475 +1 rnn.bias_hh_l0_reverse 6 0.047303859144449234 +1 rnn.bias_hh_l0_reverse 7 -0.04204820841550827 +1 rnn.bias_hh_l0_reverse 8 -0.061128221452236176 +1 rnn.bias_hh_l0_reverse 9 0.0364716574549675 +1 linear.weight 0 0.02497176267206669 +1 linear.weight 1 -0.024556435644626617 +1 linear.weight 2 -0.003516825381666422 +1 linear.weight 3 -0.0011628938373178244 +1 linear.weight 4 -0.004396989941596985 +1 linear.weight 5 0.04080790653824806 +1 linear.weight 6 0.03439749404788017 +1 linear.weight 7 -0.05506732687354088 +1 linear.weight 8 -0.0344756618142128 +1 linear.weight 9 -0.009364835917949677 +1 linear.bias 0 -0.10640990734100342 +1 linear.bias 1 -0.11040365695953369 +1 linear.bias 2 -0.13052998483181 +1 linear.bias 3 -0.10096562653779984 +1 linear.bias 4 0.11376853287220001 +1 linear.bias 5 0.0863620936870575 +1 linear.bias 6 -0.011529647745192051 +1 linear.bias 7 0.001170623698271811 +1 linear.bias 8 0.12470009922981262 +1 linear.bias 9 -0.01945381425321102 +2 transitions 0 0.8396429419517517 +2 transitions 1 0.6453437805175781 +2 transitions 2 0.41226926445961 +2 transitions 3 0.2462739497423172 +2 transitions 4 -0.6015976071357727 +2 transitions 5 1.5452231168746948 +2 transitions 6 -0.3482153117656708 +2 transitions 7 0.9640426635742188 +2 transitions 8 1.150084137916565 +2 transitions 9 -0.8586486577987671 +2 embedding2nn.weight 0 0.0030377760995179415 +2 embedding2nn.weight 1 -0.04059798642992973 +2 embedding2nn.weight 2 -0.0209097471088171 +2 embedding2nn.weight 3 -0.05250842124223709 +2 embedding2nn.weight 4 -0.00869978591799736 +2 embedding2nn.weight 5 0.0745677649974823 +2 embedding2nn.weight 6 0.0646057203412056 +2 embedding2nn.weight 7 -0.06290142238140106 +2 embedding2nn.weight 8 0.0034891690593212843 +2 embedding2nn.weight 9 0.0844922587275505 +2 embedding2nn.bias 0 -0.06527093052864075 +2 embedding2nn.bias 1 -0.07954917848110199 +2 embedding2nn.bias 2 -0.040828876197338104 +2 embedding2nn.bias 3 0.010160430334508419 +2 embedding2nn.bias 4 -0.05733371898531914 +2 embedding2nn.bias 5 -0.07895039767026901 +2 embedding2nn.bias 6 -0.04328151419758797 +2 embedding2nn.bias 7 -0.07324595749378204 +2 embedding2nn.bias 8 0.06588871031999588 +2 embedding2nn.bias 9 0.010090732015669346 +2 rnn.weight_ih_l0 0 0.05007995292544365 +2 rnn.weight_ih_l0 1 -0.021334370598196983 +2 rnn.weight_ih_l0 2 -0.0029359953477978706 +2 rnn.weight_ih_l0 3 -0.03753488510847092 +2 rnn.weight_ih_l0 4 -0.00540031585842371 +2 rnn.weight_ih_l0 5 -0.05064915120601654 +2 rnn.weight_ih_l0 6 0.014499213546514511 +2 rnn.weight_ih_l0 7 0.04458389803767204 +2 rnn.weight_ih_l0 8 0.008055073209106922 +2 rnn.weight_ih_l0 9 0.04273654893040657 +2 rnn.weight_hh_l0 0 0.03678242862224579 +2 rnn.weight_hh_l0 1 -0.044180978089571 +2 rnn.weight_hh_l0 2 0.02084459364414215 +2 rnn.weight_hh_l0 3 0.028380608186125755 +2 rnn.weight_hh_l0 4 0.06241391971707344 +2 rnn.weight_hh_l0 5 0.04844355583190918 +2 rnn.weight_hh_l0 6 0.043525781482458115 +2 rnn.weight_hh_l0 7 -0.05744693800806999 +2 rnn.weight_hh_l0 8 0.06215764209628105 +2 rnn.weight_hh_l0 9 -0.045236095786094666 +2 rnn.bias_ih_l0 0 -0.037942882627248764 +2 rnn.bias_ih_l0 1 -0.023245977237820625 +2 rnn.bias_ih_l0 2 0.058054324239492416 +2 rnn.bias_ih_l0 3 -0.012835018336772919 +2 rnn.bias_ih_l0 4 0.06232196092605591 +2 rnn.bias_ih_l0 5 0.04525524005293846 +2 rnn.bias_ih_l0 6 -0.05629503354430199 +2 rnn.bias_ih_l0 7 -0.008331854827702045 +2 rnn.bias_ih_l0 8 0.0315399207174778 +2 rnn.bias_ih_l0 9 0.02240464836359024 +2 rnn.bias_hh_l0 0 -0.05878742039203644 +2 rnn.bias_hh_l0 1 -0.0412127785384655 +2 rnn.bias_hh_l0 2 0.028317559510469437 +2 rnn.bias_hh_l0 3 0.047680824995040894 +2 rnn.bias_hh_l0 4 -0.049036167562007904 +2 rnn.bias_hh_l0 5 0.014528566040098667 +2 rnn.bias_hh_l0 6 0.04671335592865944 +2 rnn.bias_hh_l0 7 -0.0377902053296566 +2 rnn.bias_hh_l0 8 -0.028247589245438576 +2 rnn.bias_hh_l0 9 0.005904169287532568 +2 rnn.weight_ih_l0_reverse 0 -0.03514368087053299 +2 rnn.weight_ih_l0_reverse 1 -0.04153905808925629 +2 rnn.weight_ih_l0_reverse 2 0.037754908204078674 +2 rnn.weight_ih_l0_reverse 3 -0.00840480625629425 +2 rnn.weight_ih_l0_reverse 4 -0.05019295960664749 +2 rnn.weight_ih_l0_reverse 5 0.003198595019057393 +2 rnn.weight_ih_l0_reverse 6 0.012546161189675331 +2 rnn.weight_ih_l0_reverse 7 0.058054231107234955 +2 rnn.weight_ih_l0_reverse 8 0.018997706472873688 +2 rnn.weight_ih_l0_reverse 9 0.055899884551763535 +2 rnn.weight_hh_l0_reverse 0 0.060674600303173065 +2 rnn.weight_hh_l0_reverse 1 0.004412962589412928 +2 rnn.weight_hh_l0_reverse 2 0.030723560601472855 +2 rnn.weight_hh_l0_reverse 3 0.046552807092666626 +2 rnn.weight_hh_l0_reverse 4 -0.01714085415005684 +2 rnn.weight_hh_l0_reverse 5 0.04456483572721481 +2 rnn.weight_hh_l0_reverse 6 0.04380888491868973 +2 rnn.weight_hh_l0_reverse 7 0.023849567398428917 +2 rnn.weight_hh_l0_reverse 8 -0.036809273064136505 +2 rnn.weight_hh_l0_reverse 9 -0.004944506101310253 +2 rnn.bias_ih_l0_reverse 0 0.05488389730453491 +2 rnn.bias_ih_l0_reverse 1 0.002081377664580941 +2 rnn.bias_ih_l0_reverse 2 0.05005169287323952 +2 rnn.bias_ih_l0_reverse 3 -0.039556436240673065 +2 rnn.bias_ih_l0_reverse 4 -0.02825040929019451 +2 rnn.bias_ih_l0_reverse 5 0.045801691710948944 +2 rnn.bias_ih_l0_reverse 6 -0.019629884511232376 +2 rnn.bias_ih_l0_reverse 7 0.050859078764915466 +2 rnn.bias_ih_l0_reverse 8 0.007846119813621044 +2 rnn.bias_ih_l0_reverse 9 0.06228852644562721 +2 rnn.bias_hh_l0_reverse 0 -0.05106179043650627 +2 rnn.bias_hh_l0_reverse 1 -0.03938395529985428 +2 rnn.bias_hh_l0_reverse 2 -0.030298197641968727 +2 rnn.bias_hh_l0_reverse 3 0.02253197692334652 +2 rnn.bias_hh_l0_reverse 4 0.04106352478265762 +2 rnn.bias_hh_l0_reverse 5 0.05094631761312485 +2 rnn.bias_hh_l0_reverse 6 0.04734167829155922 +2 rnn.bias_hh_l0_reverse 7 -0.04195448383688927 +2 rnn.bias_hh_l0_reverse 8 -0.060668814927339554 +2 rnn.bias_hh_l0_reverse 9 0.036582183092832565 +2 linear.weight 0 0.022625530138611794 +2 linear.weight 1 -0.025964578613638878 +2 linear.weight 2 -0.0025938483886420727 +2 linear.weight 3 0.004720745608210564 +2 linear.weight 4 -0.004396989941596985 +2 linear.weight 5 0.044475506991147995 +2 linear.weight 6 0.03439749404788017 +2 linear.weight 7 -0.054950468242168427 +2 linear.weight 8 -0.0344756618142128 +2 linear.weight 9 -0.03126117214560509 +2 linear.bias 0 -0.14574795961380005 +2 linear.bias 1 -0.15304328501224518 +2 linear.bias 2 -0.15796418488025665 +2 linear.bias 3 -0.12068010866641998 +2 linear.bias 4 0.15247824788093567 +2 linear.bias 5 0.13722170889377594 +2 linear.bias 6 -0.028705665841698647 +2 linear.bias 7 -0.00469314306974411 +2 linear.bias 8 0.1867477148771286 +2 linear.bias 9 -0.03009181283414364 +3 transitions 0 0.8377614617347717 +3 transitions 1 0.642877995967865 +3 transitions 2 0.4094606041908264 +3 transitions 3 0.24537160992622375 +3 transitions 4 -0.5677332282066345 +3 transitions 5 1.5408471822738647 +3 transitions 6 -0.3492555618286133 +3 transitions 7 0.9615576267242432 +3 transitions 8 1.147697925567627 +3 transitions 9 -0.8604885935783386 +3 embedding2nn.weight 0 0.0015024886233732104 +3 embedding2nn.weight 1 -0.0402173288166523 +3 embedding2nn.weight 2 -0.021827835589647293 +3 embedding2nn.weight 3 -0.05288754776120186 +3 embedding2nn.weight 4 -0.006789218168705702 +3 embedding2nn.weight 5 0.07354671508073807 +3 embedding2nn.weight 6 0.0636962279677391 +3 embedding2nn.weight 7 -0.06260253489017487 +3 embedding2nn.weight 8 0.0027799184899777174 +3 embedding2nn.weight 9 0.08251968771219254 +3 embedding2nn.bias 0 -0.06844639033079147 +3 embedding2nn.bias 1 -0.0831136405467987 +3 embedding2nn.bias 2 -0.04258578270673752 +3 embedding2nn.bias 3 0.007461446337401867 +3 embedding2nn.bias 4 -0.059221766889095306 +3 embedding2nn.bias 5 -0.08074060827493668 +3 embedding2nn.bias 6 -0.044897906482219696 +3 embedding2nn.bias 7 -0.0732945054769516 +3 embedding2nn.bias 8 0.06829999387264252 +3 embedding2nn.bias 9 0.010123531334102154 +3 rnn.weight_ih_l0 0 0.05007783696055412 +3 rnn.weight_ih_l0 1 -0.021241649985313416 +3 rnn.weight_ih_l0 2 -0.003155078971758485 +3 rnn.weight_ih_l0 3 -0.03753318265080452 +3 rnn.weight_ih_l0 4 -0.005019878037273884 +3 rnn.weight_ih_l0 5 -0.050967987626791 +3 rnn.weight_ih_l0 6 0.01318141259253025 +3 rnn.weight_ih_l0 7 0.04447569325566292 +3 rnn.weight_ih_l0 8 0.008053520694375038 +3 rnn.weight_ih_l0 9 0.04267042130231857 +3 rnn.weight_hh_l0 0 0.036783840507268906 +3 rnn.weight_hh_l0 1 -0.04419225454330444 +3 rnn.weight_hh_l0 2 0.020847918465733528 +3 rnn.weight_hh_l0 3 0.028359081596136093 +3 rnn.weight_hh_l0 4 0.062398094683885574 +3 rnn.weight_hh_l0 5 0.048456329852342606 +3 rnn.weight_hh_l0 6 0.043528687208890915 +3 rnn.weight_hh_l0 7 -0.05755741894245148 +3 rnn.weight_hh_l0 8 0.06215774640440941 +3 rnn.weight_hh_l0 9 -0.045218564569950104 +3 rnn.bias_ih_l0 0 -0.037491630762815475 +3 rnn.bias_ih_l0 1 -0.023190077394247055 +3 rnn.bias_ih_l0 2 0.058073874562978745 +3 rnn.bias_ih_l0 3 -0.011614486575126648 +3 rnn.bias_ih_l0 4 0.06244097650051117 +3 rnn.bias_ih_l0 5 0.04535349830985069 +3 rnn.bias_ih_l0 6 -0.0560087114572525 +3 rnn.bias_ih_l0 7 -0.00952454749494791 +3 rnn.bias_ih_l0 8 0.03417389467358589 +3 rnn.bias_ih_l0 9 0.022499458864331245 +3 rnn.bias_hh_l0 0 -0.05869104340672493 +3 rnn.bias_hh_l0 1 -0.04112549126148224 +3 rnn.bias_hh_l0 2 0.02892030030488968 +3 rnn.bias_hh_l0 3 0.04537048935890198 +3 rnn.bias_hh_l0 4 -0.04900975152850151 +3 rnn.bias_hh_l0 5 0.014524628408253193 +3 rnn.bias_hh_l0 6 0.04677858576178551 +3 rnn.bias_hh_l0 7 -0.0379541739821434 +3 rnn.bias_hh_l0 8 -0.027773089706897736 +3 rnn.bias_hh_l0 9 0.006079261656850576 +3 rnn.weight_ih_l0_reverse 0 -0.03505350649356842 +3 rnn.weight_ih_l0_reverse 1 -0.04153348132967949 +3 rnn.weight_ih_l0_reverse 2 0.037806689739227295 +3 rnn.weight_ih_l0_reverse 3 -0.008438513614237309 +3 rnn.weight_ih_l0_reverse 4 -0.05030050501227379 +3 rnn.weight_ih_l0_reverse 5 0.0034924300853163004 +3 rnn.weight_ih_l0_reverse 6 0.012512755580246449 +3 rnn.weight_ih_l0_reverse 7 0.05838838219642639 +3 rnn.weight_ih_l0_reverse 8 0.01903502643108368 +3 rnn.weight_ih_l0_reverse 9 0.05587603151798248 +3 rnn.weight_hh_l0_reverse 0 0.06067570298910141 +3 rnn.weight_hh_l0_reverse 1 0.004458143841475248 +3 rnn.weight_hh_l0_reverse 2 0.0307247843593359 +3 rnn.weight_hh_l0_reverse 3 0.04654673486948013 +3 rnn.weight_hh_l0_reverse 4 -0.017143018543720245 +3 rnn.weight_hh_l0_reverse 5 0.04456063732504845 +3 rnn.weight_hh_l0_reverse 6 0.04391610994935036 +3 rnn.weight_hh_l0_reverse 7 0.02385053038597107 +3 rnn.weight_hh_l0_reverse 8 -0.03682764992117882 +3 rnn.weight_hh_l0_reverse 9 -0.004974917974323034 +3 rnn.bias_ih_l0_reverse 0 0.05488606169819832 +3 rnn.bias_ih_l0_reverse 1 0.0022986724507063627 +3 rnn.bias_ih_l0_reverse 2 0.05019591376185417 +3 rnn.bias_ih_l0_reverse 3 -0.04120796173810959 +3 rnn.bias_ih_l0_reverse 4 -0.028222618624567986 +3 rnn.bias_ih_l0_reverse 5 0.04434959217905998 +3 rnn.bias_ih_l0_reverse 6 -0.017659951001405716 +3 rnn.bias_ih_l0_reverse 7 0.050907719880342484 +3 rnn.bias_ih_l0_reverse 8 0.007996148429811 +3 rnn.bias_ih_l0_reverse 9 0.06248819828033447 +3 rnn.bias_hh_l0_reverse 0 -0.050405435264110565 +3 rnn.bias_hh_l0_reverse 1 -0.03937007486820221 +3 rnn.bias_hh_l0_reverse 2 -0.030183209106326103 +3 rnn.bias_hh_l0_reverse 3 0.02256404049694538 +3 rnn.bias_hh_l0_reverse 4 0.04140055924654007 +3 rnn.bias_hh_l0_reverse 5 0.05134500190615654 +3 rnn.bias_hh_l0_reverse 6 0.04735865071415901 +3 rnn.bias_hh_l0_reverse 7 -0.041931238025426865 +3 rnn.bias_hh_l0_reverse 8 -0.06037427857518196 +3 rnn.bias_hh_l0_reverse 9 0.03669888153672218 +3 linear.weight 0 0.021233776584267616 +3 linear.weight 1 -0.025211764499545097 +3 linear.weight 2 -0.0002406096609774977 +3 linear.weight 3 0.008750406093895435 +3 linear.weight 4 -0.004396989941596985 +3 linear.weight 5 0.04374004155397415 +3 linear.weight 6 0.03439749404788017 +3 linear.weight 7 -0.06006752699613571 +3 linear.weight 8 -0.0344756618142128 +3 linear.weight 9 -0.03638610243797302 +3 linear.bias 0 -0.175908163189888 +3 linear.bias 1 -0.1777491271495819 +3 linear.bias 2 -0.17056386172771454 +3 linear.bias 3 -0.12615160644054413 +3 linear.bias 4 0.16674868762493134 +3 linear.bias 5 0.18298226594924927 +3 linear.bias 6 -0.0333949513733387 +3 linear.bias 7 -0.019769437611103058 +3 linear.bias 8 0.2314286082983017 +3 linear.bias 9 -0.02661978267133236 +4 transitions 0 0.8369181156158447 +4 transitions 1 0.6412537097930908 +4 transitions 2 0.4078446626663208 +4 transitions 3 0.24489206075668335 +4 transitions 4 -0.5314639210700989 +4 transitions 5 1.5392723083496094 +4 transitions 6 -0.34971916675567627 +4 transitions 7 0.960340678691864 +4 transitions 8 1.1461122035980225 +4 transitions 9 -0.8617017865180969 +4 embedding2nn.weight 0 0.00030816716025583446 +4 embedding2nn.weight 1 -0.03993997350335121 +4 embedding2nn.weight 2 -0.02384973131120205 +4 embedding2nn.weight 3 -0.053082652390003204 +4 embedding2nn.weight 4 -0.007917249575257301 +4 embedding2nn.weight 5 0.07238371670246124 +4 embedding2nn.weight 6 0.06359467655420303 +4 embedding2nn.weight 7 -0.06322193145751953 +4 embedding2nn.weight 8 0.0034195485059171915 +4 embedding2nn.weight 9 0.0825219601392746 +4 embedding2nn.bias 0 -0.06668394058942795 +4 embedding2nn.bias 1 -0.07933834195137024 +4 embedding2nn.bias 2 -0.04390450567007065 +4 embedding2nn.bias 3 0.012325722724199295 +4 embedding2nn.bias 4 -0.06115618720650673 +4 embedding2nn.bias 5 -0.07854054123163223 +4 embedding2nn.bias 6 -0.04396183788776398 +4 embedding2nn.bias 7 -0.07175154238939285 +4 embedding2nn.bias 8 0.06993695348501205 +4 embedding2nn.bias 9 0.006879141088575125 +4 rnn.weight_ih_l0 0 0.05006180703639984 +4 rnn.weight_ih_l0 1 -0.02113519050180912 +4 rnn.weight_ih_l0 2 -0.00281363888643682 +4 rnn.weight_ih_l0 3 -0.037609368562698364 +4 rnn.weight_ih_l0 4 -0.006017036270350218 +4 rnn.weight_ih_l0 5 -0.050849635154008865 +4 rnn.weight_ih_l0 6 0.013170265592634678 +4 rnn.weight_ih_l0 7 0.04440946877002716 +4 rnn.weight_ih_l0 8 0.007997319102287292 +4 rnn.weight_ih_l0 9 0.04259612038731575 +4 rnn.weight_hh_l0 0 0.03678863123059273 +4 rnn.weight_hh_l0 1 -0.04420548677444458 +4 rnn.weight_hh_l0 2 0.020848054438829422 +4 rnn.weight_hh_l0 3 0.02845110185444355 +4 rnn.weight_hh_l0 4 0.0624028779566288 +4 rnn.weight_hh_l0 5 0.0484623946249485 +4 rnn.weight_hh_l0 6 0.04351995512843132 +4 rnn.weight_hh_l0 7 -0.057517290115356445 +4 rnn.weight_hh_l0 8 0.062149710953235626 +4 rnn.weight_hh_l0 9 -0.04521368071436882 +4 rnn.bias_ih_l0 0 -0.037343140691518784 +4 rnn.bias_ih_l0 1 -0.02307122014462948 +4 rnn.bias_ih_l0 2 0.058150652796030045 +4 rnn.bias_ih_l0 3 -0.01154915988445282 +4 rnn.bias_ih_l0 4 0.06256236135959625 +4 rnn.bias_ih_l0 5 0.045374251902103424 +4 rnn.bias_ih_l0 6 -0.05703049153089523 +4 rnn.bias_ih_l0 7 -0.004285016562789679 +4 rnn.bias_ih_l0 8 0.03521797060966492 +4 rnn.bias_ih_l0 9 0.02257951907813549 +4 rnn.bias_hh_l0 0 -0.05866303667426109 +4 rnn.bias_hh_l0 1 -0.041165582835674286 +4 rnn.bias_hh_l0 2 0.02931917831301689 +4 rnn.bias_hh_l0 3 0.04192119091749191 +4 rnn.bias_hh_l0 4 -0.0490150973200798 +4 rnn.bias_hh_l0 5 0.01448014471679926 +4 rnn.bias_hh_l0 6 0.04671219363808632 +4 rnn.bias_hh_l0 7 -0.03811420500278473 +4 rnn.bias_hh_l0 8 -0.027191581204533577 +4 rnn.bias_hh_l0 9 0.0063538928516209126 +4 rnn.weight_ih_l0_reverse 0 -0.03417428210377693 +4 rnn.weight_ih_l0_reverse 1 -0.04155788570642471 +4 rnn.weight_ih_l0_reverse 2 0.037862882018089294 +4 rnn.weight_ih_l0_reverse 3 -0.008442639373242855 +4 rnn.weight_ih_l0_reverse 4 -0.0504746250808239 +4 rnn.weight_ih_l0_reverse 5 0.003723880974575877 +4 rnn.weight_ih_l0_reverse 6 0.012451756745576859 +4 rnn.weight_ih_l0_reverse 7 0.05838949233293533 +4 rnn.weight_ih_l0_reverse 8 0.019141633063554764 +4 rnn.weight_ih_l0_reverse 9 0.05590435490012169 +4 rnn.weight_hh_l0_reverse 0 0.060660433024168015 +4 rnn.weight_hh_l0_reverse 1 0.004502960946410894 +4 rnn.weight_hh_l0_reverse 2 0.03071594424545765 +4 rnn.weight_hh_l0_reverse 3 0.046542998403310776 +4 rnn.weight_hh_l0_reverse 4 -0.01713048666715622 +4 rnn.weight_hh_l0_reverse 5 0.044552043080329895 +4 rnn.weight_hh_l0_reverse 6 0.044192153960466385 +4 rnn.weight_hh_l0_reverse 7 0.02385127730667591 +4 rnn.weight_hh_l0_reverse 8 -0.036821551620960236 +4 rnn.weight_hh_l0_reverse 9 -0.004993321839720011 +4 rnn.bias_ih_l0_reverse 0 0.054985467344522476 +4 rnn.bias_ih_l0_reverse 1 0.0030942047014832497 +4 rnn.bias_ih_l0_reverse 2 0.050301145762205124 +4 rnn.bias_ih_l0_reverse 3 -0.04308556020259857 +4 rnn.bias_ih_l0_reverse 4 -0.02812294289469719 +4 rnn.bias_ih_l0_reverse 5 0.04637465253472328 +4 rnn.bias_ih_l0_reverse 6 -0.017133213579654694 +4 rnn.bias_ih_l0_reverse 7 0.050925806164741516 +4 rnn.bias_ih_l0_reverse 8 0.00798889808356762 +4 rnn.bias_ih_l0_reverse 9 0.062365807592868805 +4 rnn.bias_hh_l0_reverse 0 -0.04986705258488655 +4 rnn.bias_hh_l0_reverse 1 -0.03939434513449669 +4 rnn.bias_hh_l0_reverse 2 -0.03016609512269497 +4 rnn.bias_hh_l0_reverse 3 0.022670825943350792 +4 rnn.bias_hh_l0_reverse 4 0.04157441481947899 +4 rnn.bias_hh_l0_reverse 5 0.05153859034180641 +4 rnn.bias_hh_l0_reverse 6 0.04737306386232376 +4 rnn.bias_hh_l0_reverse 7 -0.041797664016485214 +4 rnn.bias_hh_l0_reverse 8 -0.060285065323114395 +4 rnn.bias_hh_l0_reverse 9 0.0371006615459919 +4 linear.weight 0 0.017402414232492447 +4 linear.weight 1 -0.026235606521368027 +4 linear.weight 2 0.003336526919156313 +4 linear.weight 3 0.011564305983483791 +4 linear.weight 4 -0.004396989941596985 +4 linear.weight 5 0.03979814797639847 +4 linear.weight 6 0.03439749404788017 +4 linear.weight 7 -0.0708150565624237 +4 linear.weight 8 -0.0344756618142128 +4 linear.weight 9 -0.035784944891929626 +4 linear.bias 0 -0.1938798576593399 +4 linear.bias 1 -0.18923406302928925 +4 linear.bias 2 -0.1597503423690796 +4 linear.bias 3 -0.11419441550970078 +4 linear.bias 4 0.16768109798431396 +4 linear.bias 5 0.22754758596420288 +4 linear.bias 6 -0.02453671023249626 +4 linear.bias 7 -0.03190542757511139 +4 linear.bias 8 0.21018239855766296 +4 linear.bias 9 -0.011959918774664402 +5 transitions 0 0.8345900177955627 +5 transitions 1 0.6393868923187256 +5 transitions 2 0.40576377511024475 +5 transitions 3 0.2438485026359558 +5 transitions 4 -0.4977167844772339 +5 transitions 5 1.5351500511169434 +5 transitions 6 -0.35022708773612976 +5 transitions 7 0.9584410190582275 +5 transitions 8 1.1430385112762451 +5 transitions 9 -0.8632848858833313 +5 embedding2nn.weight 0 -0.0008981279679574072 +5 embedding2nn.weight 1 -0.04000554606318474 +5 embedding2nn.weight 2 -0.02521412819623947 +5 embedding2nn.weight 3 -0.05288767069578171 +5 embedding2nn.weight 4 -0.00537472777068615 +5 embedding2nn.weight 5 0.07107189297676086 +5 embedding2nn.weight 6 0.06368279457092285 +5 embedding2nn.weight 7 -0.06268478184938431 +5 embedding2nn.weight 8 -0.00022472684213425964 +5 embedding2nn.weight 9 0.08107524365186691 +5 embedding2nn.bias 0 -0.07024035602807999 +5 embedding2nn.bias 1 -0.08461042493581772 +5 embedding2nn.bias 2 -0.044548384845256805 +5 embedding2nn.bias 3 0.0052194660529494286 +5 embedding2nn.bias 4 -0.058801088482141495 +5 embedding2nn.bias 5 -0.08103304356336594 +5 embedding2nn.bias 6 -0.04370469972491264 +5 embedding2nn.bias 7 -0.0710923820734024 +5 embedding2nn.bias 8 0.06946933269500732 +5 embedding2nn.bias 9 0.007023623213171959 +5 rnn.weight_ih_l0 0 0.050052862614393234 +5 rnn.weight_ih_l0 1 -0.02094884030520916 +5 rnn.weight_ih_l0 2 -0.003266597166657448 +5 rnn.weight_ih_l0 3 -0.03759641945362091 +5 rnn.weight_ih_l0 4 -0.004717039410024881 +5 rnn.weight_ih_l0 5 -0.051054589450359344 +5 rnn.weight_ih_l0 6 0.012853140942752361 +5 rnn.weight_ih_l0 7 0.04445347934961319 +5 rnn.weight_ih_l0 8 0.008029136806726456 +5 rnn.weight_ih_l0 9 0.04256252944469452 +5 rnn.weight_hh_l0 0 0.03678442910313606 +5 rnn.weight_hh_l0 1 -0.04417414590716362 +5 rnn.weight_hh_l0 2 0.020875295624136925 +5 rnn.weight_hh_l0 3 0.02847294695675373 +5 rnn.weight_hh_l0 4 0.062401797622442245 +5 rnn.weight_hh_l0 5 0.04849345609545708 +5 rnn.weight_hh_l0 6 0.04353126510977745 +5 rnn.weight_hh_l0 7 -0.05747431889176369 +5 rnn.weight_hh_l0 8 0.06214720383286476 +5 rnn.weight_hh_l0 9 -0.045230280607938766 +5 rnn.bias_ih_l0 0 -0.03666481375694275 +5 rnn.bias_ih_l0 1 -0.022991003468632698 +5 rnn.bias_ih_l0 2 0.05835690349340439 +5 rnn.bias_ih_l0 3 -0.011528740637004375 +5 rnn.bias_ih_l0 4 0.06254872679710388 +5 rnn.bias_ih_l0 5 0.04539908468723297 +5 rnn.bias_ih_l0 6 -0.056723255664110184 +5 rnn.bias_ih_l0 7 -0.004166129045188427 +5 rnn.bias_ih_l0 8 0.03658755496144295 +5 rnn.bias_ih_l0 9 0.022606024518609047 +5 rnn.bias_hh_l0 0 -0.05853266268968582 +5 rnn.bias_hh_l0 1 -0.04111050069332123 +5 rnn.bias_hh_l0 2 0.029669586569070816 +5 rnn.bias_hh_l0 3 0.04366026073694229 +5 rnn.bias_hh_l0 4 -0.04895872622728348 +5 rnn.bias_hh_l0 5 0.014456681907176971 +5 rnn.bias_hh_l0 6 0.04672105610370636 +5 rnn.bias_hh_l0 7 -0.03810029476881027 +5 rnn.bias_hh_l0 8 -0.026398509740829468 +5 rnn.bias_hh_l0 9 0.0065668863244354725 +5 rnn.weight_ih_l0_reverse 0 -0.03458048403263092 +5 rnn.weight_ih_l0_reverse 1 -0.04155896604061127 +5 rnn.weight_ih_l0_reverse 2 0.03795471042394638 +5 rnn.weight_ih_l0_reverse 3 -0.008403775282204151 +5 rnn.weight_ih_l0_reverse 4 -0.05062057450413704 +5 rnn.weight_ih_l0_reverse 5 0.004937124904245138 +5 rnn.weight_ih_l0_reverse 6 0.012297193519771099 +5 rnn.weight_ih_l0_reverse 7 0.05867725610733032 +5 rnn.weight_ih_l0_reverse 8 0.019253473728895187 +5 rnn.weight_ih_l0_reverse 9 0.055975887924432755 +5 rnn.weight_hh_l0_reverse 0 0.06067919731140137 +5 rnn.weight_hh_l0_reverse 1 0.00450196024030447 +5 rnn.weight_hh_l0_reverse 2 0.03072289004921913 +5 rnn.weight_hh_l0_reverse 3 0.04648136347532272 +5 rnn.weight_hh_l0_reverse 4 -0.017143897712230682 +5 rnn.weight_hh_l0_reverse 5 0.04450377821922302 +5 rnn.weight_hh_l0_reverse 6 0.04410998895764351 +5 rnn.weight_hh_l0_reverse 7 0.023749396204948425 +5 rnn.weight_hh_l0_reverse 8 -0.036831192672252655 +5 rnn.weight_hh_l0_reverse 9 -0.005067184567451477 +5 rnn.bias_ih_l0_reverse 0 0.05545777454972267 +5 rnn.bias_ih_l0_reverse 1 0.003872453235089779 +5 rnn.bias_ih_l0_reverse 2 0.05038752779364586 +5 rnn.bias_ih_l0_reverse 3 -0.0451122410595417 +5 rnn.bias_ih_l0_reverse 4 -0.027985133230686188 +5 rnn.bias_ih_l0_reverse 5 0.04404008015990257 +5 rnn.bias_ih_l0_reverse 6 -0.017134051769971848 +5 rnn.bias_ih_l0_reverse 7 0.05101094767451286 +5 rnn.bias_ih_l0_reverse 8 0.008059319108724594 +5 rnn.bias_ih_l0_reverse 9 0.062411416321992874 +5 rnn.bias_hh_l0_reverse 0 -0.04942476749420166 +5 rnn.bias_hh_l0_reverse 1 -0.039374370127916336 +5 rnn.bias_hh_l0_reverse 2 -0.02978932484984398 +5 rnn.bias_hh_l0_reverse 3 0.022873684763908386 +5 rnn.bias_hh_l0_reverse 4 0.04195495322346687 +5 rnn.bias_hh_l0_reverse 5 0.05199553817510605 +5 rnn.bias_hh_l0_reverse 6 0.04742788150906563 +5 rnn.bias_hh_l0_reverse 7 -0.04147886484861374 +5 rnn.bias_hh_l0_reverse 8 -0.05977638438344002 +5 rnn.bias_hh_l0_reverse 9 0.03715583309531212 +5 linear.weight 0 0.016329778358340263 +5 linear.weight 1 -0.028602493926882744 +5 linear.weight 2 0.005291108042001724 +5 linear.weight 3 0.014219848439097404 +5 linear.weight 4 -0.004396989941596985 +5 linear.weight 5 0.039819568395614624 +5 linear.weight 6 0.03439749404788017 +5 linear.weight 7 -0.07888923585414886 +5 linear.weight 8 -0.0344756618142128 +5 linear.weight 9 -0.047888077795505524 +5 linear.bias 0 -0.2130245864391327 +5 linear.bias 1 -0.20769238471984863 +5 linear.bias 2 -0.1683242917060852 +5 linear.bias 3 -0.12932421267032623 +5 linear.bias 4 0.1762479841709137 +5 linear.bias 5 0.22429046034812927 +5 linear.bias 6 -0.023120712488889694 +5 linear.bias 7 -0.018547862768173218 +5 linear.bias 8 0.23566743731498718 +5 linear.bias 9 -0.01115999836474657 +6 transitions 0 0.8333225250244141 +6 transitions 1 0.6375167369842529 +6 transitions 2 0.40427154302597046 +6 transitions 3 0.24295832216739655 +6 transitions 4 -0.46157386898994446 +6 transitions 5 1.532928705215454 +6 transitions 6 -0.3507206439971924 +6 transitions 7 0.9569748640060425 +6 transitions 8 1.1404693126678467 +6 transitions 9 -0.8643088936805725 +6 embedding2nn.weight 0 -0.001437824685126543 +6 embedding2nn.weight 1 -0.04035244137048721 +6 embedding2nn.weight 2 -0.026625074446201324 +6 embedding2nn.weight 3 -0.053540054708719254 +6 embedding2nn.weight 4 -0.0021421245764940977 +6 embedding2nn.weight 5 0.07143451273441315 +6 embedding2nn.weight 6 0.06319340318441391 +6 embedding2nn.weight 7 -0.06052876636385918 +6 embedding2nn.weight 8 -0.002460415940731764 +6 embedding2nn.weight 9 0.0771668329834938 +6 embedding2nn.bias 0 -0.07284153997898102 +6 embedding2nn.bias 1 -0.08581642806529999 +6 embedding2nn.bias 2 -0.047455448657274246 +6 embedding2nn.bias 3 0.006939815357327461 +6 embedding2nn.bias 4 -0.059885937720537186 +6 embedding2nn.bias 5 -0.07873901724815369 +6 embedding2nn.bias 6 -0.04324149712920189 +6 embedding2nn.bias 7 -0.07041171938180923 +6 embedding2nn.bias 8 0.07183326780796051 +6 embedding2nn.bias 9 0.006298254709690809 +6 rnn.weight_ih_l0 0 0.05004686862230301 +6 rnn.weight_ih_l0 1 -0.020960897207260132 +6 rnn.weight_ih_l0 2 -0.002933370415121317 +6 rnn.weight_ih_l0 3 -0.037686921656131744 +6 rnn.weight_ih_l0 4 -0.007505069486796856 +6 rnn.weight_ih_l0 5 -0.0507732629776001 +6 rnn.weight_ih_l0 6 0.012985803186893463 +6 rnn.weight_ih_l0 7 0.04444558545947075 +6 rnn.weight_ih_l0 8 0.008012049831449986 +6 rnn.weight_ih_l0 9 0.042564962059259415 +6 rnn.weight_hh_l0 0 0.03678212687373161 +6 rnn.weight_hh_l0 1 -0.04420875012874603 +6 rnn.weight_hh_l0 2 0.020840995013713837 +6 rnn.weight_hh_l0 3 0.02861395664513111 +6 rnn.weight_hh_l0 4 0.06240935996174812 +6 rnn.weight_hh_l0 5 0.04848094284534454 +6 rnn.weight_hh_l0 6 0.043532658368349075 +6 rnn.weight_hh_l0 7 -0.05737599357962608 +6 rnn.weight_hh_l0 8 0.06214286386966705 +6 rnn.weight_hh_l0 9 -0.045286860316991806 +6 rnn.bias_ih_l0 0 -0.0364358127117157 +6 rnn.bias_ih_l0 1 -0.023006493225693703 +6 rnn.bias_ih_l0 2 0.05837696045637131 +6 rnn.bias_ih_l0 3 -0.010079482570290565 +6 rnn.bias_ih_l0 4 0.06280249357223511 +6 rnn.bias_ih_l0 5 0.04502493888139725 +6 rnn.bias_ih_l0 6 -0.057084500789642334 +6 rnn.bias_ih_l0 7 -0.002121312776580453 +6 rnn.bias_ih_l0 8 0.03996369615197182 +6 rnn.bias_ih_l0 9 0.022696761414408684 +6 rnn.bias_hh_l0 0 -0.058452147990465164 +6 rnn.bias_hh_l0 1 -0.041078925132751465 +6 rnn.bias_hh_l0 2 0.030740434303879738 +6 rnn.bias_hh_l0 3 0.03858122229576111 +6 rnn.bias_hh_l0 4 -0.048940882086753845 +6 rnn.bias_hh_l0 5 0.014426474459469318 +6 rnn.bias_hh_l0 6 0.04677058011293411 +6 rnn.bias_hh_l0 7 -0.03817716985940933 +6 rnn.bias_hh_l0 8 -0.024980975314974785 +6 rnn.bias_hh_l0 9 0.0066822730004787445 +6 rnn.weight_ih_l0_reverse 0 -0.03370803967118263 +6 rnn.weight_ih_l0_reverse 1 -0.04155472293496132 +6 rnn.weight_ih_l0_reverse 2 0.03791176527738571 +6 rnn.weight_ih_l0_reverse 3 -0.008331401273608208 +6 rnn.weight_ih_l0_reverse 4 -0.05071483179926872 +6 rnn.weight_ih_l0_reverse 5 0.005389775615185499 +6 rnn.weight_ih_l0_reverse 6 0.012275418266654015 +6 rnn.weight_ih_l0_reverse 7 0.05854691192507744 +6 rnn.weight_ih_l0_reverse 8 0.019317887723445892 +6 rnn.weight_ih_l0_reverse 9 0.05603534355759621 +6 rnn.weight_hh_l0_reverse 0 0.06070033833384514 +6 rnn.weight_hh_l0_reverse 1 0.004456224851310253 +6 rnn.weight_hh_l0_reverse 2 0.030723795294761658 +6 rnn.weight_hh_l0_reverse 3 0.04645000398159027 +6 rnn.weight_hh_l0_reverse 4 -0.01709062233567238 +6 rnn.weight_hh_l0_reverse 5 0.044489357620477676 +6 rnn.weight_hh_l0_reverse 6 0.04503951966762543 +6 rnn.weight_hh_l0_reverse 7 0.02350512333214283 +6 rnn.weight_hh_l0_reverse 8 -0.03683134540915489 +6 rnn.weight_hh_l0_reverse 9 -0.004983511753380299 +6 rnn.bias_ih_l0_reverse 0 0.05556812509894371 +6 rnn.bias_ih_l0_reverse 1 0.004617449827492237 +6 rnn.bias_ih_l0_reverse 2 0.05061577260494232 +6 rnn.bias_ih_l0_reverse 3 -0.04604990780353546 +6 rnn.bias_ih_l0_reverse 4 -0.027985790744423866 +6 rnn.bias_ih_l0_reverse 5 0.04482626914978027 +6 rnn.bias_ih_l0_reverse 6 -0.014943373389542103 +6 rnn.bias_ih_l0_reverse 7 0.051171913743019104 +6 rnn.bias_ih_l0_reverse 8 0.008122265338897705 +6 rnn.bias_ih_l0_reverse 9 0.06248530000448227 +6 rnn.bias_hh_l0_reverse 0 -0.04773293063044548 +6 rnn.bias_hh_l0_reverse 1 -0.039303962141275406 +6 rnn.bias_hh_l0_reverse 2 -0.029796741902828217 +6 rnn.bias_hh_l0_reverse 3 0.02294892445206642 +6 rnn.bias_hh_l0_reverse 4 0.04229755699634552 +6 rnn.bias_hh_l0_reverse 5 0.05243056267499924 +6 rnn.bias_hh_l0_reverse 6 0.04748975485563278 +6 rnn.bias_hh_l0_reverse 7 -0.04147294536232948 +6 rnn.bias_hh_l0_reverse 8 -0.05959070473909378 +6 rnn.bias_hh_l0_reverse 9 0.037460505962371826 +6 linear.weight 0 0.014621207490563393 +6 linear.weight 1 0.001735713449306786 +6 linear.weight 2 0.010105596855282784 +6 linear.weight 3 0.015539637766778469 +6 linear.weight 4 -0.004396989941596985 +6 linear.weight 5 0.03729831427335739 +6 linear.weight 6 0.03439749404788017 +6 linear.weight 7 -0.08489260822534561 +6 linear.weight 8 -0.0344756618142128 +6 linear.weight 9 -0.053921233862638474 +6 linear.bias 0 -0.22992461919784546 +6 linear.bias 1 -0.2220383882522583 +6 linear.bias 2 -0.16681034862995148 +6 linear.bias 3 -0.12003021687269211 +6 linear.bias 4 0.17046596109867096 +6 linear.bias 5 0.25473883748054504 +6 linear.bias 6 -0.01760280132293701 +6 linear.bias 7 -0.017607348039746284 +6 linear.bias 8 0.2242681086063385 +6 linear.bias 9 -0.006245335564017296 +7 transitions 0 0.8314716219902039 +7 transitions 1 0.6358304619789124 +7 transitions 2 0.4023997187614441 +7 transitions 3 0.24194274842739105 +7 transitions 4 -0.42253783345222473 +7 transitions 5 1.529524326324463 +7 transitions 6 -0.35109859704971313 +7 transitions 7 0.9557874202728271 +7 transitions 8 1.1371268033981323 +7 transitions 9 -0.8654578328132629 +7 embedding2nn.weight 0 -0.002741673029959202 +7 embedding2nn.weight 1 -0.04186893627047539 +7 embedding2nn.weight 2 -0.027485448867082596 +7 embedding2nn.weight 3 -0.05436785891652107 +7 embedding2nn.weight 4 -0.0021592313423752785 +7 embedding2nn.weight 5 0.07185593992471695 +7 embedding2nn.weight 6 0.06233169510960579 +7 embedding2nn.weight 7 -0.058755792677402496 +7 embedding2nn.weight 8 -0.006257445085793734 +7 embedding2nn.weight 9 0.07475774735212326 +7 embedding2nn.bias 0 -0.07383165508508682 +7 embedding2nn.bias 1 -0.08824163675308228 +7 embedding2nn.bias 2 -0.048910509794950485 +7 embedding2nn.bias 3 0.0073997629806399345 +7 embedding2nn.bias 4 -0.06296894699335098 +7 embedding2nn.bias 5 -0.08048892021179199 +7 embedding2nn.bias 6 -0.04206109791994095 +7 embedding2nn.bias 7 -0.07163217663764954 +7 embedding2nn.bias 8 0.0753323957324028 +7 embedding2nn.bias 9 0.005559632088989019 +7 rnn.weight_ih_l0 0 0.0500713586807251 +7 rnn.weight_ih_l0 1 -0.020818013697862625 +7 rnn.weight_ih_l0 2 -0.003199223428964615 +7 rnn.weight_ih_l0 3 -0.037678543478250504 +7 rnn.weight_ih_l0 4 -0.006823153700679541 +7 rnn.weight_ih_l0 5 -0.050748493522405624 +7 rnn.weight_ih_l0 6 0.012604172341525555 +7 rnn.weight_ih_l0 7 0.04418130964040756 +7 rnn.weight_ih_l0 8 0.00801620539277792 +7 rnn.weight_ih_l0 9 0.042366255074739456 +7 rnn.weight_hh_l0 0 0.036785684525966644 +7 rnn.weight_hh_l0 1 -0.044331349432468414 +7 rnn.weight_hh_l0 2 0.020847739651799202 +7 rnn.weight_hh_l0 3 0.02862243913114071 +7 rnn.weight_hh_l0 4 0.062385134398937225 +7 rnn.weight_hh_l0 5 0.04849949851632118 +7 rnn.weight_hh_l0 6 0.043539296835660934 +7 rnn.weight_hh_l0 7 -0.05740649998188019 +7 rnn.weight_hh_l0 8 0.0621395967900753 +7 rnn.weight_hh_l0 9 -0.04519569128751755 +7 rnn.bias_ih_l0 0 -0.036376990377902985 +7 rnn.bias_ih_l0 1 -0.022931654006242752 +7 rnn.bias_ih_l0 2 0.05854039639234543 +7 rnn.bias_ih_l0 3 -0.010280696675181389 +7 rnn.bias_ih_l0 4 0.06334265321493149 +7 rnn.bias_ih_l0 5 0.044993702322244644 +7 rnn.bias_ih_l0 6 -0.05683129280805588 +7 rnn.bias_ih_l0 7 -0.0031869469676166773 +7 rnn.bias_ih_l0 8 0.0400380939245224 +7 rnn.bias_ih_l0 9 0.02269228920340538 +7 rnn.bias_hh_l0 0 -0.05823001265525818 +7 rnn.bias_hh_l0 1 -0.04098484665155411 +7 rnn.bias_hh_l0 2 0.03146311268210411 +7 rnn.bias_hh_l0 3 0.03765367344021797 +7 rnn.bias_hh_l0 4 -0.04890597239136696 +7 rnn.bias_hh_l0 5 0.01448608934879303 +7 rnn.bias_hh_l0 6 0.046878017485141754 +7 rnn.bias_hh_l0 7 -0.03816629946231842 +7 rnn.bias_hh_l0 8 -0.02474912814795971 +7 rnn.bias_hh_l0 9 0.006844480521976948 +7 rnn.weight_ih_l0_reverse 0 -0.034386828541755676 +7 rnn.weight_ih_l0_reverse 1 -0.04161050170660019 +7 rnn.weight_ih_l0_reverse 2 0.038000527769327164 +7 rnn.weight_ih_l0_reverse 3 -0.0083384457975626 +7 rnn.weight_ih_l0_reverse 4 -0.05071616172790527 +7 rnn.weight_ih_l0_reverse 5 0.005362505093216896 +7 rnn.weight_ih_l0_reverse 6 0.012143479660153389 +7 rnn.weight_ih_l0_reverse 7 0.05886503681540489 +7 rnn.weight_ih_l0_reverse 8 0.019430242478847504 +7 rnn.weight_ih_l0_reverse 9 0.056042902171611786 +7 rnn.weight_hh_l0_reverse 0 0.060691557824611664 +7 rnn.weight_hh_l0_reverse 1 0.004449358209967613 +7 rnn.weight_hh_l0_reverse 2 0.03072572499513626 +7 rnn.weight_hh_l0_reverse 3 0.04638467729091644 +7 rnn.weight_hh_l0_reverse 4 -0.01704436168074608 +7 rnn.weight_hh_l0_reverse 5 0.04438260942697525 +7 rnn.weight_hh_l0_reverse 6 0.04521745443344116 +7 rnn.weight_hh_l0_reverse 7 0.023524638265371323 +7 rnn.weight_hh_l0_reverse 8 -0.03690077364444733 +7 rnn.weight_hh_l0_reverse 9 -0.00508147943764925 +7 rnn.bias_ih_l0_reverse 0 0.0560230053961277 +7 rnn.bias_ih_l0_reverse 1 0.004524128511548042 +7 rnn.bias_ih_l0_reverse 2 0.05077555403113365 +7 rnn.bias_ih_l0_reverse 3 -0.04637911915779114 +7 rnn.bias_ih_l0_reverse 4 -0.027973929420113564 +7 rnn.bias_ih_l0_reverse 5 0.04334087297320366 +7 rnn.bias_ih_l0_reverse 6 -0.014672169461846352 +7 rnn.bias_ih_l0_reverse 7 0.05139889568090439 +7 rnn.bias_ih_l0_reverse 8 0.00848510954529047 +7 rnn.bias_ih_l0_reverse 9 0.06271844357252121 +7 rnn.bias_hh_l0_reverse 0 -0.04571533203125 +7 rnn.bias_hh_l0_reverse 1 -0.03918629512190819 +7 rnn.bias_hh_l0_reverse 2 -0.029711415991187096 +7 rnn.bias_hh_l0_reverse 3 0.023474983870983124 +7 rnn.bias_hh_l0_reverse 4 0.04236703738570213 +7 rnn.bias_hh_l0_reverse 5 0.052490249276161194 +7 rnn.bias_hh_l0_reverse 6 0.04755803942680359 +7 rnn.bias_hh_l0_reverse 7 -0.04131392762064934 +7 rnn.bias_hh_l0_reverse 8 -0.05953671783208847 +7 rnn.bias_hh_l0_reverse 9 0.03767472133040428 +7 linear.weight 0 0.011929024010896683 +7 linear.weight 1 -0.004612325225025415 +7 linear.weight 2 0.014658228494226933 +7 linear.weight 3 0.01760447584092617 +7 linear.weight 4 -0.004396989941596985 +7 linear.weight 5 0.03187640756368637 +7 linear.weight 6 0.03439749404788017 +7 linear.weight 7 -0.10018330812454224 +7 linear.weight 8 -0.0344756618142128 +7 linear.weight 9 -0.0564078614115715 +7 linear.bias 0 -0.24478688836097717 +7 linear.bias 1 -0.23510320484638214 +7 linear.bias 2 -0.1722325086593628 +7 linear.bias 3 -0.12095522880554199 +7 linear.bias 4 0.14506341516971588 +7 linear.bias 5 0.2730149030685425 +7 linear.bias 6 -0.003548106411471963 +7 linear.bias 7 -0.00793184619396925 +7 linear.bias 8 0.25866854190826416 +7 linear.bias 9 -0.0004999804659746587 +8 transitions 0 0.8306974172592163 +8 transitions 1 0.6352406740188599 +8 transitions 2 0.4017736315727234 +8 transitions 3 0.2414531111717224 +8 transitions 4 -0.3973867893218994 +8 transitions 5 1.5284966230392456 +8 transitions 6 -0.35118716955184937 +8 transitions 7 0.9553868770599365 +8 transitions 8 1.135824203491211 +8 transitions 9 -0.8658986687660217 +8 embedding2nn.weight 0 -0.002658400684595108 +8 embedding2nn.weight 1 -0.041605815291404724 +8 embedding2nn.weight 2 -0.02936047874391079 +8 embedding2nn.weight 3 -0.054486241191625595 +8 embedding2nn.weight 4 -0.003963393624871969 +8 embedding2nn.weight 5 0.07033205032348633 +8 embedding2nn.weight 6 0.06242261826992035 +8 embedding2nn.weight 7 -0.06092071905732155 +8 embedding2nn.weight 8 -0.006188501138240099 +8 embedding2nn.weight 9 0.0748501792550087 +8 embedding2nn.bias 0 -0.0706520825624466 +8 embedding2nn.bias 1 -0.08527529984712601 +8 embedding2nn.bias 2 -0.04900525510311127 +8 embedding2nn.bias 3 0.008354496210813522 +8 embedding2nn.bias 4 -0.06232539564371109 +8 embedding2nn.bias 5 -0.07717977464199066 +8 embedding2nn.bias 6 -0.04155696555972099 +8 embedding2nn.bias 7 -0.0704205259680748 +8 embedding2nn.bias 8 0.07180020958185196 +8 embedding2nn.bias 9 0.00428398884832859 +8 rnn.weight_ih_l0 0 0.050167035311460495 +8 rnn.weight_ih_l0 1 -0.02067406475543976 +8 rnn.weight_ih_l0 2 -0.003145467024296522 +8 rnn.weight_ih_l0 3 -0.037769775837659836 +8 rnn.weight_ih_l0 4 -0.00878524873405695 +8 rnn.weight_ih_l0 5 -0.05061597004532814 +8 rnn.weight_ih_l0 6 0.013142015784978867 +8 rnn.weight_ih_l0 7 0.044087741523981094 +8 rnn.weight_ih_l0 8 0.008021543733775616 +8 rnn.weight_ih_l0 9 0.04219428077340126 +8 rnn.weight_hh_l0 0 0.03678892180323601 +8 rnn.weight_hh_l0 1 -0.044326044619083405 +8 rnn.weight_hh_l0 2 0.020896833389997482 +8 rnn.weight_hh_l0 3 0.02873903326690197 +8 rnn.weight_hh_l0 4 0.06238854303956032 +8 rnn.weight_hh_l0 5 0.048525821417570114 +8 rnn.weight_hh_l0 6 0.043535422533750534 +8 rnn.weight_hh_l0 7 -0.05726883187890053 +8 rnn.weight_hh_l0 8 0.06213516369462013 +8 rnn.weight_hh_l0 9 -0.045381177216768265 +8 rnn.bias_ih_l0 0 -0.03611261025071144 +8 rnn.bias_ih_l0 1 -0.022758934646844864 +8 rnn.bias_ih_l0 2 0.058651894330978394 +8 rnn.bias_ih_l0 3 -0.011053724214434624 +8 rnn.bias_ih_l0 4 0.06317976862192154 +8 rnn.bias_ih_l0 5 0.044812947511672974 +8 rnn.bias_ih_l0 6 -0.057373128831386566 +8 rnn.bias_ih_l0 7 -0.0034624049440026283 +8 rnn.bias_ih_l0 8 0.04061516746878624 +8 rnn.bias_ih_l0 9 0.022708188742399216 +8 rnn.bias_hh_l0 0 -0.05795757472515106 +8 rnn.bias_hh_l0 1 -0.040700513869524 +8 rnn.bias_hh_l0 2 0.03180081024765968 +8 rnn.bias_hh_l0 3 0.03830769658088684 +8 rnn.bias_hh_l0 4 -0.04885735362768173 +8 rnn.bias_hh_l0 5 0.014467589557170868 +8 rnn.bias_hh_l0 6 0.046962395310401917 +8 rnn.bias_hh_l0 7 -0.038217708468437195 +8 rnn.bias_hh_l0 8 -0.024394620209932327 +8 rnn.bias_hh_l0 9 0.00693443464115262 +8 rnn.weight_ih_l0_reverse 0 -0.03388744220137596 +8 rnn.weight_ih_l0_reverse 1 -0.04162043333053589 +8 rnn.weight_ih_l0_reverse 2 0.03810941427946091 +8 rnn.weight_ih_l0_reverse 3 -0.008363459259271622 +8 rnn.weight_ih_l0_reverse 4 -0.05042871832847595 +8 rnn.weight_ih_l0_reverse 5 0.005291346926242113 +8 rnn.weight_ih_l0_reverse 6 0.012128384783864021 +8 rnn.weight_ih_l0_reverse 7 0.05884382128715515 +8 rnn.weight_ih_l0_reverse 8 0.019485075026750565 +8 rnn.weight_ih_l0_reverse 9 0.05614616349339485 +8 rnn.weight_hh_l0_reverse 0 0.06071413680911064 +8 rnn.weight_hh_l0_reverse 1 0.004448592197149992 +8 rnn.weight_hh_l0_reverse 2 0.030730243772268295 +8 rnn.weight_hh_l0_reverse 3 0.04634058102965355 +8 rnn.weight_hh_l0_reverse 4 -0.017006920650601387 +8 rnn.weight_hh_l0_reverse 5 0.044417351484298706 +8 rnn.weight_hh_l0_reverse 6 0.04605976492166519 +8 rnn.weight_hh_l0_reverse 7 0.023532960563898087 +8 rnn.weight_hh_l0_reverse 8 -0.036939069628715515 +8 rnn.weight_hh_l0_reverse 9 -0.005037965252995491 +8 rnn.bias_ih_l0_reverse 0 0.056141771376132965 +8 rnn.bias_ih_l0_reverse 1 0.004731324966996908 +8 rnn.bias_ih_l0_reverse 2 0.05081932246685028 +8 rnn.bias_ih_l0_reverse 3 -0.04737483710050583 +8 rnn.bias_ih_l0_reverse 4 -0.027599219232797623 +8 rnn.bias_ih_l0_reverse 5 0.04577763006091118 +8 rnn.bias_ih_l0_reverse 6 -0.016515931114554405 +8 rnn.bias_ih_l0_reverse 7 0.05148037523031235 +8 rnn.bias_ih_l0_reverse 8 0.008735167793929577 +8 rnn.bias_ih_l0_reverse 9 0.06275147199630737 +8 rnn.bias_hh_l0_reverse 0 -0.045364513993263245 +8 rnn.bias_hh_l0_reverse 1 -0.039138466119766235 +8 rnn.bias_hh_l0_reverse 2 -0.029494939371943474 +8 rnn.bias_hh_l0_reverse 3 0.02236110158264637 +8 rnn.bias_hh_l0_reverse 4 0.04193415865302086 +8 rnn.bias_hh_l0_reverse 5 0.05189061909914017 +8 rnn.bias_hh_l0_reverse 6 0.04768767207860947 +8 rnn.bias_hh_l0_reverse 7 -0.041300032287836075 +8 rnn.bias_hh_l0_reverse 8 -0.059283312410116196 +8 rnn.bias_hh_l0_reverse 9 0.037594519555568695 +8 linear.weight 0 0.011316601186990738 +8 linear.weight 1 0.021020440384745598 +8 linear.weight 2 0.016377786174416542 +8 linear.weight 3 0.019035719335079193 +8 linear.weight 4 -0.004396989941596985 +8 linear.weight 5 0.031815048307180405 +8 linear.weight 6 0.03439749404788017 +8 linear.weight 7 -0.1049303188920021 +8 linear.weight 8 -0.0344756618142128 +8 linear.weight 9 -0.056847263127565384 +8 linear.bias 0 -0.24997462332248688 +8 linear.bias 1 -0.23893411457538605 +8 linear.bias 2 -0.16591817140579224 +8 linear.bias 3 -0.113181471824646 +8 linear.bias 4 0.14956559240818024 +8 linear.bias 5 0.25198662281036377 +8 linear.bias 6 0.009694226086139679 +8 linear.bias 7 0.009202671237289906 +8 linear.bias 8 0.20642933249473572 +8 linear.bias 9 0.00848398543894291 +9 transitions 0 0.8295980095863342 +9 transitions 1 0.634121835231781 +9 transitions 2 0.40087828040122986 +9 transitions 3 0.24067260324954987 +9 transitions 4 -0.37434425950050354 +9 transitions 5 1.5264108180999756 +9 transitions 6 -0.3513907790184021 +9 transitions 7 0.9546195864677429 +9 transitions 8 1.133442759513855 +9 transitions 9 -0.8662157654762268 +9 embedding2nn.weight 0 -0.003226529574021697 +9 embedding2nn.weight 1 -0.04241662099957466 +9 embedding2nn.weight 2 -0.02998422272503376 +9 embedding2nn.weight 3 -0.05457085371017456 +9 embedding2nn.weight 4 -0.00810487475246191 +9 embedding2nn.weight 5 0.07085027545690536 +9 embedding2nn.weight 6 0.06184423714876175 +9 embedding2nn.weight 7 -0.059560392051935196 +9 embedding2nn.weight 8 -0.004042052198201418 +9 embedding2nn.weight 9 0.0749097540974617 +9 embedding2nn.bias 0 -0.06916354596614838 +9 embedding2nn.bias 1 -0.08470739424228668 +9 embedding2nn.bias 2 -0.05170402675867081 +9 embedding2nn.bias 3 0.016141362488269806 +9 embedding2nn.bias 4 -0.0663832351565361 +9 embedding2nn.bias 5 -0.07935860008001328 +9 embedding2nn.bias 6 -0.040224429219961166 +9 embedding2nn.bias 7 -0.07275117933750153 +9 embedding2nn.bias 8 0.07574215531349182 +9 embedding2nn.bias 9 0.003438136773183942 +9 rnn.weight_ih_l0 0 0.05016583204269409 +9 rnn.weight_ih_l0 1 -0.020581109449267387 +9 rnn.weight_ih_l0 2 -0.003794637741521001 +9 rnn.weight_ih_l0 3 -0.03768449276685715 +9 rnn.weight_ih_l0 4 -0.008828981779515743 +9 rnn.weight_ih_l0 5 -0.05062468722462654 +9 rnn.weight_ih_l0 6 0.013453309424221516 +9 rnn.weight_ih_l0 7 0.044108469039201736 +9 rnn.weight_ih_l0 8 0.008029812946915627 +9 rnn.weight_ih_l0 9 0.04184885695576668 +9 rnn.weight_hh_l0 0 0.03678978979587555 +9 rnn.weight_hh_l0 1 -0.04435309022665024 +9 rnn.weight_hh_l0 2 0.020904269069433212 +9 rnn.weight_hh_l0 3 0.028737695887684822 +9 rnn.weight_hh_l0 4 0.06239890679717064 +9 rnn.weight_hh_l0 5 0.04853285476565361 +9 rnn.weight_hh_l0 6 0.04353247582912445 +9 rnn.weight_hh_l0 7 -0.05722283944487572 +9 rnn.weight_hh_l0 8 0.06213177368044853 +9 rnn.weight_hh_l0 9 -0.045282237231731415 +9 rnn.bias_ih_l0 0 -0.035938818007707596 +9 rnn.bias_ih_l0 1 -0.022641737014055252 +9 rnn.bias_ih_l0 2 0.05887354537844658 +9 rnn.bias_ih_l0 3 -0.010753083042800426 +9 rnn.bias_ih_l0 4 0.06384272128343582 +9 rnn.bias_ih_l0 5 0.04495092108845711 +9 rnn.bias_ih_l0 6 -0.057150378823280334 +9 rnn.bias_ih_l0 7 -0.002864059992134571 +9 rnn.bias_ih_l0 8 0.039980076253414154 +9 rnn.bias_ih_l0 9 0.022733395919203758 +9 rnn.bias_hh_l0 0 -0.05784107372164726 +9 rnn.bias_hh_l0 1 -0.040332645177841187 +9 rnn.bias_hh_l0 2 0.032435353845357895 +9 rnn.bias_hh_l0 3 0.03271317109465599 +9 rnn.bias_hh_l0 4 -0.04881548136472702 +9 rnn.bias_hh_l0 5 0.01455310545861721 +9 rnn.bias_hh_l0 6 0.04708769917488098 +9 rnn.bias_hh_l0 7 -0.038269441574811935 +9 rnn.bias_hh_l0 8 -0.023300565779209137 +9 rnn.bias_hh_l0 9 0.006991300266236067 +9 rnn.weight_ih_l0_reverse 0 -0.033645737916231155 +9 rnn.weight_ih_l0_reverse 1 -0.041731178760528564 +9 rnn.weight_ih_l0_reverse 2 0.03816580772399902 +9 rnn.weight_ih_l0_reverse 3 -0.00835490133613348 +9 rnn.weight_ih_l0_reverse 4 -0.05068616569042206 +9 rnn.weight_ih_l0_reverse 5 0.006105470471084118 +9 rnn.weight_ih_l0_reverse 6 0.012031045742332935 +9 rnn.weight_ih_l0_reverse 7 0.05896774306893349 +9 rnn.weight_ih_l0_reverse 8 0.01952262595295906 +9 rnn.weight_ih_l0_reverse 9 0.056230656802654266 +9 rnn.weight_hh_l0_reverse 0 0.06073760613799095 +9 rnn.weight_hh_l0_reverse 1 0.004436945542693138 +9 rnn.weight_hh_l0_reverse 2 0.030738957226276398 +9 rnn.weight_hh_l0_reverse 3 0.046168334782123566 +9 rnn.weight_hh_l0_reverse 4 -0.017019527032971382 +9 rnn.weight_hh_l0_reverse 5 0.04442816227674484 +9 rnn.weight_hh_l0_reverse 6 0.046080343425273895 +9 rnn.weight_hh_l0_reverse 7 0.023532221093773842 +9 rnn.weight_hh_l0_reverse 8 -0.036984995007514954 +9 rnn.weight_hh_l0_reverse 9 -0.004945011809468269 +9 rnn.bias_ih_l0_reverse 0 0.05627807602286339 +9 rnn.bias_ih_l0_reverse 1 0.005156508181244135 +9 rnn.bias_ih_l0_reverse 2 0.05113629251718521 +9 rnn.bias_ih_l0_reverse 3 -0.04618029296398163 +9 rnn.bias_ih_l0_reverse 4 -0.028209103271365166 +9 rnn.bias_ih_l0_reverse 5 0.04840537905693054 +9 rnn.bias_ih_l0_reverse 6 -0.014932319521903992 +9 rnn.bias_ih_l0_reverse 7 0.051543112844228745 +9 rnn.bias_ih_l0_reverse 8 0.009053701534867287 +9 rnn.bias_ih_l0_reverse 9 0.06282217055559158 +9 rnn.bias_hh_l0_reverse 0 -0.04454277455806732 +9 rnn.bias_hh_l0_reverse 1 -0.038822419941425323 +9 rnn.bias_hh_l0_reverse 2 -0.029527034610509872 +9 rnn.bias_hh_l0_reverse 3 0.022535378113389015 +9 rnn.bias_hh_l0_reverse 4 0.041825827211141586 +9 rnn.bias_hh_l0_reverse 5 0.05177896469831467 +9 rnn.bias_hh_l0_reverse 6 0.04772474989295006 +9 rnn.bias_hh_l0_reverse 7 -0.041263334453105927 +9 rnn.bias_hh_l0_reverse 8 -0.059343189001083374 +9 rnn.bias_hh_l0_reverse 9 0.03786426782608032 +9 linear.weight 0 0.010528124868869781 +9 linear.weight 1 0.02018950879573822 +9 linear.weight 2 0.019872920587658882 +9 linear.weight 3 0.020471666008234024 +9 linear.weight 4 -0.004396989941596985 +9 linear.weight 5 0.030787091702222824 +9 linear.weight 6 0.03439749404788017 +9 linear.weight 7 -0.11032320559024811 +9 linear.weight 8 -0.0344756618142128 +9 linear.weight 9 -0.05658744275569916 +9 linear.bias 0 -0.2560339570045471 +9 linear.bias 1 -0.24591846764087677 +9 linear.bias 2 -0.16901877522468567 +9 linear.bias 3 -0.10969474911689758 +9 linear.bias 4 0.11172972619533539 +9 linear.bias 5 0.28585270047187805 +9 linear.bias 6 0.017676884308457375 +9 linear.bias 7 0.00797573197633028 +9 linear.bias 8 0.2429181933403015 +9 linear.bias 9 0.008017689920961857 +10 transitions 0 0.8271072506904602 +10 transitions 1 0.6325253248214722 +10 transitions 2 0.39898625016212463 +10 transitions 3 0.23955149948596954 +10 transitions 4 -0.3505198359489441 +10 transitions 5 1.5234042406082153 +10 transitions 6 -0.3516288995742798 +10 transitions 7 0.9533950090408325 +10 transitions 8 1.130822777748108 +10 transitions 9 -0.8670068383216858 +10 embedding2nn.weight 0 -0.003939580637961626 +10 embedding2nn.weight 1 -0.042516231536865234 +10 embedding2nn.weight 2 -0.030622540041804314 +10 embedding2nn.weight 3 -0.05500829219818115 +10 embedding2nn.weight 4 -0.008292651735246181 +10 embedding2nn.weight 5 0.07001420110464096 +10 embedding2nn.weight 6 0.06200660392642021 +10 embedding2nn.weight 7 -0.05720178410410881 +10 embedding2nn.weight 8 -0.004437539726495743 +10 embedding2nn.weight 9 0.07322648167610168 +10 embedding2nn.bias 0 -0.07355470955371857 +10 embedding2nn.bias 1 -0.09066898375749588 +10 embedding2nn.bias 2 -0.047536060214042664 +10 embedding2nn.bias 3 0.001984847942367196 +10 embedding2nn.bias 4 -0.06026585027575493 +10 embedding2nn.bias 5 -0.08269795030355453 +10 embedding2nn.bias 6 -0.0456366203725338 +10 embedding2nn.bias 7 -0.07042401283979416 +10 embedding2nn.bias 8 0.07455006241798401 +10 embedding2nn.bias 9 0.005324301775544882 +10 rnn.weight_ih_l0 0 0.0501728393137455 +10 rnn.weight_ih_l0 1 -0.02052890509366989 +10 rnn.weight_ih_l0 2 -0.003357616951689124 +10 rnn.weight_ih_l0 3 -0.03775595873594284 +10 rnn.weight_ih_l0 4 -0.008258701302111149 +10 rnn.weight_ih_l0 5 -0.050670966506004333 +10 rnn.weight_ih_l0 6 0.014035218395292759 +10 rnn.weight_ih_l0 7 0.04417090490460396 +10 rnn.weight_ih_l0 8 0.008037913590669632 +10 rnn.weight_ih_l0 9 0.04181291162967682 +10 rnn.weight_hh_l0 0 0.03678828850388527 +10 rnn.weight_hh_l0 1 -0.044456757605075836 +10 rnn.weight_hh_l0 2 0.020965158939361572 +10 rnn.weight_hh_l0 3 0.02872920036315918 +10 rnn.weight_hh_l0 4 0.06239930912852287 +10 rnn.weight_hh_l0 5 0.04851502180099487 +10 rnn.weight_hh_l0 6 0.04355234280228615 +10 rnn.weight_hh_l0 7 -0.057230520993471146 +10 rnn.weight_hh_l0 8 0.062129318714141846 +10 rnn.weight_hh_l0 9 -0.04528060555458069 +10 rnn.bias_ih_l0 0 -0.03604087978601456 +10 rnn.bias_ih_l0 1 -0.022458864375948906 +10 rnn.bias_ih_l0 2 0.058995623141527176 +10 rnn.bias_ih_l0 3 -0.011269906535744667 +10 rnn.bias_ih_l0 4 0.06374242901802063 +10 rnn.bias_ih_l0 5 0.045006025582551956 +10 rnn.bias_ih_l0 6 -0.056925881654024124 +10 rnn.bias_ih_l0 7 -0.0026124208234250546 +10 rnn.bias_ih_l0 8 0.03900991380214691 +10 rnn.bias_ih_l0 9 0.0228604506701231 +10 rnn.bias_hh_l0 0 -0.057633817195892334 +10 rnn.bias_hh_l0 1 -0.040256060659885406 +10 rnn.bias_hh_l0 2 0.03252379223704338 +10 rnn.bias_hh_l0 3 0.03884873166680336 +10 rnn.bias_hh_l0 4 -0.04877498373389244 +10 rnn.bias_hh_l0 5 0.014630609191954136 +10 rnn.bias_hh_l0 6 0.047207217663526535 +10 rnn.bias_hh_l0 7 -0.03820410743355751 +10 rnn.bias_hh_l0 8 -0.02340828999876976 +10 rnn.bias_hh_l0 9 0.00732484320178628 +10 rnn.weight_ih_l0_reverse 0 -0.034041278064250946 +10 rnn.weight_ih_l0_reverse 1 -0.041700538247823715 +10 rnn.weight_ih_l0_reverse 2 0.038402654230594635 +10 rnn.weight_ih_l0_reverse 3 -0.008365726098418236 +10 rnn.weight_ih_l0_reverse 4 -0.05078734830021858 +10 rnn.weight_ih_l0_reverse 5 0.006382322404533625 +10 rnn.weight_ih_l0_reverse 6 0.011930744163691998 +10 rnn.weight_ih_l0_reverse 7 0.05947565287351608 +10 rnn.weight_ih_l0_reverse 8 0.01962187886238098 +10 rnn.weight_ih_l0_reverse 9 0.05627811327576637 +10 rnn.weight_hh_l0_reverse 0 0.060777418315410614 +10 rnn.weight_hh_l0_reverse 1 0.004446720238775015 +10 rnn.weight_hh_l0_reverse 2 0.03073382005095482 +10 rnn.weight_hh_l0_reverse 3 0.04607553780078888 +10 rnn.weight_hh_l0_reverse 4 -0.017007846385240555 +10 rnn.weight_hh_l0_reverse 5 0.04432017728686333 +10 rnn.weight_hh_l0_reverse 6 0.045981187373399734 +10 rnn.weight_hh_l0_reverse 7 0.02333517000079155 +10 rnn.weight_hh_l0_reverse 8 -0.037045855075120926 +10 rnn.weight_hh_l0_reverse 9 -0.005153760313987732 +10 rnn.bias_ih_l0_reverse 0 0.05694439634680748 +10 rnn.bias_ih_l0_reverse 1 0.005488978233188391 +10 rnn.bias_ih_l0_reverse 2 0.05131226032972336 +10 rnn.bias_ih_l0_reverse 3 -0.04720696434378624 +10 rnn.bias_ih_l0_reverse 4 -0.028035378083586693 +10 rnn.bias_ih_l0_reverse 5 0.04346213489770889 +10 rnn.bias_ih_l0_reverse 6 -0.016037654131650925 +10 rnn.bias_ih_l0_reverse 7 0.051937784999608994 +10 rnn.bias_ih_l0_reverse 8 0.00917071383446455 +10 rnn.bias_ih_l0_reverse 9 0.0630049854516983 +10 rnn.bias_hh_l0_reverse 0 -0.0431976281106472 +10 rnn.bias_hh_l0_reverse 1 -0.03887692466378212 +10 rnn.bias_hh_l0_reverse 2 -0.029527228325605392 +10 rnn.bias_hh_l0_reverse 3 0.02251902036368847 +10 rnn.bias_hh_l0_reverse 4 0.04231290519237518 +10 rnn.bias_hh_l0_reverse 5 0.05235288292169571 +10 rnn.bias_hh_l0_reverse 6 0.04779262840747833 +10 rnn.bias_hh_l0_reverse 7 -0.0410226508975029 +10 rnn.bias_hh_l0_reverse 8 -0.05873963609337807 +10 rnn.bias_hh_l0_reverse 9 0.038322776556015015 +10 linear.weight 0 0.008140478283166885 +10 linear.weight 1 0.022060776129364967 +10 linear.weight 2 0.022501256316900253 +10 linear.weight 3 0.022456420585513115 +10 linear.weight 4 -0.004396989941596985 +10 linear.weight 5 0.02950756810605526 +10 linear.weight 6 0.03439749404788017 +10 linear.weight 7 -0.11800111085176468 +10 linear.weight 8 -0.0344756618142128 +10 linear.weight 9 -0.07610425353050232 +10 linear.bias 0 -0.2659739851951599 +10 linear.bias 1 -0.2546931505203247 +10 linear.bias 2 -0.17268817126750946 +10 linear.bias 3 -0.12225279211997986 +10 linear.bias 4 0.1446332335472107 +10 linear.bias 5 0.2679157853126526 +10 linear.bias 6 0.018426906317472458 +10 linear.bias 7 0.016171401366591454 +10 linear.bias 8 0.2339377999305725 +10 linear.bias 9 0.008053528144955635 +11 transitions 0 0.8266475200653076 +11 transitions 1 0.6318554878234863 +11 transitions 2 0.3983372747898102 +11 transitions 3 0.2391747534275055 +11 transitions 4 -0.32878899574279785 +11 transitions 5 1.5224838256835938 +11 transitions 6 -0.3517141044139862 +11 transitions 7 0.9531259536743164 +11 transitions 8 1.1294258832931519 +11 transitions 9 -0.8671793341636658 +11 embedding2nn.weight 0 -0.004237824585288763 +11 embedding2nn.weight 1 -0.04289074242115021 +11 embedding2nn.weight 2 -0.03145270422101021 +11 embedding2nn.weight 3 -0.05537337064743042 +11 embedding2nn.weight 4 -0.007861139252781868 +11 embedding2nn.weight 5 0.06953925639390945 +11 embedding2nn.weight 6 0.061975330114364624 +11 embedding2nn.weight 7 -0.053329721093177795 +11 embedding2nn.weight 8 -0.0031920243054628372 +11 embedding2nn.weight 9 0.07313809543848038 +11 embedding2nn.bias 0 -0.07259893417358398 +11 embedding2nn.bias 1 -0.08851931244134903 +11 embedding2nn.bias 2 -0.05061279237270355 +11 embedding2nn.bias 3 0.014974812977015972 +11 embedding2nn.bias 4 -0.0667651891708374 +11 embedding2nn.bias 5 -0.08313687145709991 +11 embedding2nn.bias 6 -0.04226723313331604 +11 embedding2nn.bias 7 -0.07150962948799133 +11 embedding2nn.bias 8 0.07777910679578781 +11 embedding2nn.bias 9 0.002697673626244068 +11 rnn.weight_ih_l0 0 0.05019538104534149 +11 rnn.weight_ih_l0 1 -0.02054627053439617 +11 rnn.weight_ih_l0 2 -0.003652901155874133 +11 rnn.weight_ih_l0 3 -0.03776795044541359 +11 rnn.weight_ih_l0 4 -0.0074971821159124374 +11 rnn.weight_ih_l0 5 -0.050880271941423416 +11 rnn.weight_ih_l0 6 0.013056306168437004 +11 rnn.weight_ih_l0 7 0.04411405324935913 +11 rnn.weight_ih_l0 8 0.008107065223157406 +11 rnn.weight_ih_l0 9 0.041764020919799805 +11 rnn.weight_hh_l0 0 0.03678809478878975 +11 rnn.weight_hh_l0 1 -0.04448381066322327 +11 rnn.weight_hh_l0 2 0.021007638424634933 +11 rnn.weight_hh_l0 3 0.028737954795360565 +11 rnn.weight_hh_l0 4 0.062408044934272766 +11 rnn.weight_hh_l0 5 0.04850563779473305 +11 rnn.weight_hh_l0 6 0.04349667206406593 +11 rnn.weight_hh_l0 7 -0.05718547850847244 +11 rnn.weight_hh_l0 8 0.06212906911969185 +11 rnn.weight_hh_l0 9 -0.045294586569070816 +11 rnn.bias_ih_l0 0 -0.035992275923490524 +11 rnn.bias_ih_l0 1 -0.022364890202879906 +11 rnn.bias_ih_l0 2 0.05923185870051384 +11 rnn.bias_ih_l0 3 -0.01104915700852871 +11 rnn.bias_ih_l0 4 0.06421729922294617 +11 rnn.bias_ih_l0 5 0.045163463801145554 +11 rnn.bias_ih_l0 6 -0.05669806897640228 +11 rnn.bias_ih_l0 7 -0.003828678745776415 +11 rnn.bias_ih_l0 8 0.04019007086753845 +11 rnn.bias_ih_l0 9 0.023002251982688904 +11 rnn.bias_hh_l0 0 -0.05743897706270218 +11 rnn.bias_hh_l0 1 -0.039971206337213516 +11 rnn.bias_hh_l0 2 0.03300367295742035 +11 rnn.bias_hh_l0 3 0.033329691737890244 +11 rnn.bias_hh_l0 4 -0.04859189689159393 +11 rnn.bias_hh_l0 5 0.014760753139853477 +11 rnn.bias_hh_l0 6 0.047296639531850815 +11 rnn.bias_hh_l0 7 -0.038251716643571854 +11 rnn.bias_hh_l0 8 -0.022972380742430687 +11 rnn.bias_hh_l0 9 0.0074598719365894794 +11 rnn.weight_ih_l0_reverse 0 -0.033858686685562134 +11 rnn.weight_ih_l0_reverse 1 -0.041825294494628906 +11 rnn.weight_ih_l0_reverse 2 0.03838995471596718 +11 rnn.weight_ih_l0_reverse 3 -0.00835186243057251 +11 rnn.weight_ih_l0_reverse 4 -0.050815071910619736 +11 rnn.weight_ih_l0_reverse 5 0.006282699294388294 +11 rnn.weight_ih_l0_reverse 6 0.011908694170415401 +11 rnn.weight_ih_l0_reverse 7 0.059574417769908905 +11 rnn.weight_ih_l0_reverse 8 0.01962186023592949 +11 rnn.weight_ih_l0_reverse 9 0.05642043799161911 +11 rnn.weight_hh_l0_reverse 0 0.0607714019715786 +11 rnn.weight_hh_l0_reverse 1 0.004444134421646595 +11 rnn.weight_hh_l0_reverse 2 0.030740533024072647 +11 rnn.weight_hh_l0_reverse 3 0.0459444597363472 +11 rnn.weight_hh_l0_reverse 4 -0.016971787437796593 +11 rnn.weight_hh_l0_reverse 5 0.044352781027555466 +11 rnn.weight_hh_l0_reverse 6 0.0471823588013649 +11 rnn.weight_hh_l0_reverse 7 0.023567721247673035 +11 rnn.weight_hh_l0_reverse 8 -0.03715653344988823 +11 rnn.weight_hh_l0_reverse 9 -0.005095353815704584 +11 rnn.bias_ih_l0_reverse 0 0.05716545507311821 +11 rnn.bias_ih_l0_reverse 1 0.005456894636154175 +11 rnn.bias_ih_l0_reverse 2 0.05145065858960152 +11 rnn.bias_ih_l0_reverse 3 -0.04786013066768646 +11 rnn.bias_ih_l0_reverse 4 -0.02824936993420124 +11 rnn.bias_ih_l0_reverse 5 0.04694202169775963 +11 rnn.bias_ih_l0_reverse 6 -0.015155462548136711 +11 rnn.bias_ih_l0_reverse 7 0.05198277533054352 +11 rnn.bias_ih_l0_reverse 8 0.009434408508241177 +11 rnn.bias_ih_l0_reverse 9 0.06315065920352936 +11 rnn.bias_hh_l0_reverse 0 -0.04356657341122627 +11 rnn.bias_hh_l0_reverse 1 -0.038811661303043365 +11 rnn.bias_hh_l0_reverse 2 -0.029447348788380623 +11 rnn.bias_hh_l0_reverse 3 0.022682301700115204 +11 rnn.bias_hh_l0_reverse 4 0.04226786643266678 +11 rnn.bias_hh_l0_reverse 5 0.05233455076813698 +11 rnn.bias_hh_l0_reverse 6 0.04787302762269974 +11 rnn.bias_hh_l0_reverse 7 -0.04085933417081833 +11 rnn.bias_hh_l0_reverse 8 -0.05937042832374573 +11 rnn.bias_hh_l0_reverse 9 0.038315534591674805 +11 linear.weight 0 0.007413036655634642 +11 linear.weight 1 0.01377498172223568 +11 linear.weight 2 0.023981239646673203 +11 linear.weight 3 0.02291802503168583 +11 linear.weight 4 -0.004396989941596985 +11 linear.weight 5 0.02837974578142166 +11 linear.weight 6 0.03439749404788017 +11 linear.weight 7 -0.12750402092933655 +11 linear.weight 8 -0.0344756618142128 +11 linear.weight 9 -0.07359231263399124 +11 linear.bias 0 -0.26965534687042236 +11 linear.bias 1 -0.2579105496406555 +11 linear.bias 2 -0.16923297941684723 +11 linear.bias 3 -0.11284077167510986 +11 linear.bias 4 0.11079636216163635 +11 linear.bias 5 0.28156644105911255 +11 linear.bias 6 0.030766554176807404 +11 linear.bias 7 0.02423185668885708 +11 linear.bias 8 0.24624019861221313 +11 linear.bias 9 0.013933123089373112 +12 transitions 0 0.8257455229759216 +12 transitions 1 0.631404459476471 +12 transitions 2 0.39783975481987 +12 transitions 3 0.2385529726743698 +12 transitions 4 -0.3130399286746979 +12 transitions 5 1.521733045578003 +12 transitions 6 -0.3518028259277344 +12 transitions 7 0.9527362585067749 +12 transitions 8 1.128018856048584 +12 transitions 9 -0.8674675822257996 +12 embedding2nn.weight 0 -0.004358003847301006 +12 embedding2nn.weight 1 -0.043119415640830994 +12 embedding2nn.weight 2 -0.03117641992866993 +12 embedding2nn.weight 3 -0.05629103258252144 +12 embedding2nn.weight 4 -0.009048174135386944 +12 embedding2nn.weight 5 0.06800323724746704 +12 embedding2nn.weight 6 0.062038667500019073 +12 embedding2nn.weight 7 -0.0539393275976181 +12 embedding2nn.weight 8 -0.0010014568688347936 +12 embedding2nn.weight 9 0.07214179635047913 +12 embedding2nn.bias 0 -0.07010301947593689 +12 embedding2nn.bias 1 -0.08561374992132187 +12 embedding2nn.bias 2 -0.05171223729848862 +12 embedding2nn.bias 3 0.010595484636723995 +12 embedding2nn.bias 4 -0.06287101656198502 +12 embedding2nn.bias 5 -0.07797432690858841 +12 embedding2nn.bias 6 -0.042812004685401917 +12 embedding2nn.bias 7 -0.07078107446432114 +12 embedding2nn.bias 8 0.07457388937473297 +12 embedding2nn.bias 9 0.004418426193296909 +12 rnn.weight_ih_l0 0 0.05024053156375885 +12 rnn.weight_ih_l0 1 -0.020550290122628212 +12 rnn.weight_ih_l0 2 -0.0035139243118464947 +12 rnn.weight_ih_l0 3 -0.03803465515375137 +12 rnn.weight_ih_l0 4 -0.00848633423447609 +12 rnn.weight_ih_l0 5 -0.050577469170093536 +12 rnn.weight_ih_l0 6 0.014568239450454712 +12 rnn.weight_ih_l0 7 0.04411189258098602 +12 rnn.weight_ih_l0 8 0.008177605457603931 +12 rnn.weight_ih_l0 9 0.04213279113173485 +12 rnn.weight_hh_l0 0 0.036796554923057556 +12 rnn.weight_hh_l0 1 -0.0445571132004261 +12 rnn.weight_hh_l0 2 0.02108924835920334 +12 rnn.weight_hh_l0 3 0.02873290702700615 +12 rnn.weight_hh_l0 4 0.0624033585190773 +12 rnn.weight_hh_l0 5 0.048504702746868134 +12 rnn.weight_hh_l0 6 0.04350599646568298 +12 rnn.weight_hh_l0 7 -0.057176705449819565 +12 rnn.weight_hh_l0 8 0.062126368284225464 +12 rnn.weight_hh_l0 9 -0.04535174369812012 +12 rnn.bias_ih_l0 0 -0.03594021499156952 +12 rnn.bias_ih_l0 1 -0.0222276970744133 +12 rnn.bias_ih_l0 2 0.059276171028614044 +12 rnn.bias_ih_l0 3 -0.01184697076678276 +12 rnn.bias_ih_l0 4 0.06377437710762024 +12 rnn.bias_ih_l0 5 0.044875193387269974 +12 rnn.bias_ih_l0 6 -0.056807465851306915 +12 rnn.bias_ih_l0 7 -0.0025992353912442923 +12 rnn.bias_ih_l0 8 0.040241166949272156 +12 rnn.bias_ih_l0 9 0.023038072511553764 +12 rnn.bias_hh_l0 0 -0.05720148980617523 +12 rnn.bias_hh_l0 1 -0.040252793580293655 +12 rnn.bias_hh_l0 2 0.03320582956075668 +12 rnn.bias_hh_l0 3 0.034038323909044266 +12 rnn.bias_hh_l0 4 -0.04843994975090027 +12 rnn.bias_hh_l0 5 0.01479409821331501 +12 rnn.bias_hh_l0 6 0.04728718474507332 +12 rnn.bias_hh_l0 7 -0.03820818290114403 +12 rnn.bias_hh_l0 8 -0.02313070371747017 +12 rnn.bias_hh_l0 9 0.00755421444773674 +12 rnn.weight_ih_l0_reverse 0 -0.034030407667160034 +12 rnn.weight_ih_l0_reverse 1 -0.04185754433274269 +12 rnn.weight_ih_l0_reverse 2 0.038600314408540726 +12 rnn.weight_ih_l0_reverse 3 -0.008429179899394512 +12 rnn.weight_ih_l0_reverse 4 -0.05098070949316025 +12 rnn.weight_ih_l0_reverse 5 0.006394646130502224 +12 rnn.weight_ih_l0_reverse 6 0.011825259774923325 +12 rnn.weight_ih_l0_reverse 7 0.05988221615552902 +12 rnn.weight_ih_l0_reverse 8 0.01975031942129135 +12 rnn.weight_ih_l0_reverse 9 0.05651141330599785 +12 rnn.weight_hh_l0_reverse 0 0.06078888475894928 +12 rnn.weight_hh_l0_reverse 1 0.004438673611730337 +12 rnn.weight_hh_l0_reverse 2 0.030757199972867966 +12 rnn.weight_hh_l0_reverse 3 0.04587488994002342 +12 rnn.weight_hh_l0_reverse 4 -0.01695425808429718 +12 rnn.weight_hh_l0_reverse 5 0.044258963316679 +12 rnn.weight_hh_l0_reverse 6 0.04767967388033867 +12 rnn.weight_hh_l0_reverse 7 0.023376846686005592 +12 rnn.weight_hh_l0_reverse 8 -0.037154123187065125 +12 rnn.weight_hh_l0_reverse 9 -0.005116089712828398 +12 rnn.bias_ih_l0_reverse 0 0.05736585706472397 +12 rnn.bias_ih_l0_reverse 1 0.0056584682315588 +12 rnn.bias_ih_l0_reverse 2 0.051328130066394806 +12 rnn.bias_ih_l0_reverse 3 -0.04827475920319557 +12 rnn.bias_ih_l0_reverse 4 -0.027815449982881546 +12 rnn.bias_ih_l0_reverse 5 0.047006119042634964 +12 rnn.bias_ih_l0_reverse 6 -0.015887465327978134 +12 rnn.bias_ih_l0_reverse 7 0.05231034755706787 +12 rnn.bias_ih_l0_reverse 8 0.00948526430875063 +12 rnn.bias_ih_l0_reverse 9 0.06283844262361526 +12 rnn.bias_hh_l0_reverse 0 -0.04530096799135208 +12 rnn.bias_hh_l0_reverse 1 -0.038901012390851974 +12 rnn.bias_hh_l0_reverse 2 -0.029465679079294205 +12 rnn.bias_hh_l0_reverse 3 0.02223336137831211 +12 rnn.bias_hh_l0_reverse 4 0.042473241686820984 +12 rnn.bias_hh_l0_reverse 5 0.05254216492176056 +12 rnn.bias_hh_l0_reverse 6 0.04782799258828163 +12 rnn.bias_hh_l0_reverse 7 -0.04051556438207626 +12 rnn.bias_hh_l0_reverse 8 -0.05912478640675545 +12 rnn.bias_hh_l0_reverse 9 0.03847954049706459 +12 linear.weight 0 0.007329669315367937 +12 linear.weight 1 0.031095389276742935 +12 linear.weight 2 0.02757696621119976 +12 linear.weight 3 0.023257339373230934 +12 linear.weight 4 -0.004396989941596985 +12 linear.weight 5 0.028021864593029022 +12 linear.weight 6 0.03439749404788017 +12 linear.weight 7 -0.13071709871292114 +12 linear.weight 8 -0.0344756618142128 +12 linear.weight 9 -0.07892180234193802 +12 linear.bias 0 -0.2728925943374634 +12 linear.bias 1 -0.26029065251350403 +12 linear.bias 2 -0.164958655834198 +12 linear.bias 3 -0.10891222208738327 +12 linear.bias 4 0.13004285097122192 +12 linear.bias 5 0.2801709473133087 +12 linear.bias 6 0.03410575911402702 +12 linear.bias 7 0.019406039267778397 +12 linear.bias 8 0.18564710021018982 +12 linear.bias 9 0.015357955358922482 +13 transitions 0 0.824190080165863 +13 transitions 1 0.6300852298736572 +13 transitions 2 0.3960183560848236 +13 transitions 3 0.23784060776233673 +13 transitions 4 -0.2870699465274811 +13 transitions 5 1.519013524055481 +13 transitions 6 -0.35195112228393555 +13 transitions 7 0.9520976543426514 +13 transitions 8 1.1257140636444092 +13 transitions 9 -0.8677855730056763 +13 embedding2nn.weight 0 -0.003931653220206499 +13 embedding2nn.weight 1 -0.0428188219666481 +13 embedding2nn.weight 2 -0.03181719407439232 +13 embedding2nn.weight 3 -0.056721560657024384 +13 embedding2nn.weight 4 -0.008171620778739452 +13 embedding2nn.weight 5 0.06817183643579483 +13 embedding2nn.weight 6 0.06200543791055679 +13 embedding2nn.weight 7 -0.05131428316235542 +13 embedding2nn.weight 8 -0.005599370226264 +13 embedding2nn.weight 9 0.07295440137386322 +13 embedding2nn.bias 0 -0.07274292409420013 +13 embedding2nn.bias 1 -0.08809075504541397 +13 embedding2nn.bias 2 -0.05001591891050339 +13 embedding2nn.bias 3 0.01401620451360941 +13 embedding2nn.bias 4 -0.06868983060121536 +13 embedding2nn.bias 5 -0.08440590649843216 +13 embedding2nn.bias 6 -0.04140985384583473 +13 embedding2nn.bias 7 -0.07253146171569824 +13 embedding2nn.bias 8 0.07798691093921661 +13 embedding2nn.bias 9 0.00225074146874249 +13 rnn.weight_ih_l0 0 0.050254937261343 +13 rnn.weight_ih_l0 1 -0.02038601227104664 +13 rnn.weight_ih_l0 2 -0.0032915817573666573 +13 rnn.weight_ih_l0 3 -0.03823826089501381 +13 rnn.weight_ih_l0 4 -0.006632636766880751 +13 rnn.weight_ih_l0 5 -0.050623491406440735 +13 rnn.weight_ih_l0 6 0.01405639760196209 +13 rnn.weight_ih_l0 7 0.04393826052546501 +13 rnn.weight_ih_l0 8 0.008196022361516953 +13 rnn.weight_ih_l0 9 0.0421111062169075 +13 rnn.weight_hh_l0 0 0.036832842975854874 +13 rnn.weight_hh_l0 1 -0.044538941234350204 +13 rnn.weight_hh_l0 2 0.021109934896230698 +13 rnn.weight_hh_l0 3 0.02874833345413208 +13 rnn.weight_hh_l0 4 0.06242218613624573 +13 rnn.weight_hh_l0 5 0.04848082736134529 +13 rnn.weight_hh_l0 6 0.04350436478853226 +13 rnn.weight_hh_l0 7 -0.05721340700984001 +13 rnn.weight_hh_l0 8 0.06212252378463745 +13 rnn.weight_hh_l0 9 -0.04532548785209656 +13 rnn.bias_ih_l0 0 -0.03535417467355728 +13 rnn.bias_ih_l0 1 -0.022168204188346863 +13 rnn.bias_ih_l0 2 0.059812113642692566 +13 rnn.bias_ih_l0 3 -0.011670036241412163 +13 rnn.bias_ih_l0 4 0.06447169929742813 +13 rnn.bias_ih_l0 5 0.045060329139232635 +13 rnn.bias_ih_l0 6 -0.05660080537199974 +13 rnn.bias_ih_l0 7 -0.005770258605480194 +13 rnn.bias_ih_l0 8 0.03986314684152603 +13 rnn.bias_ih_l0 9 0.023226777091622353 +13 rnn.bias_hh_l0 0 -0.056900754570961 +13 rnn.bias_hh_l0 1 -0.03972102701663971 +13 rnn.bias_hh_l0 2 0.03483043983578682 +13 rnn.bias_hh_l0 3 0.03348603844642639 +13 rnn.bias_hh_l0 4 -0.04813942685723305 +13 rnn.bias_hh_l0 5 0.015002160333096981 +13 rnn.bias_hh_l0 6 0.04742111638188362 +13 rnn.bias_hh_l0 7 -0.03815978765487671 +13 rnn.bias_hh_l0 8 -0.023227598518133163 +13 rnn.bias_hh_l0 9 0.007940421812236309 +13 rnn.weight_ih_l0_reverse 0 -0.03376830741763115 +13 rnn.weight_ih_l0_reverse 1 -0.04197243973612785 +13 rnn.weight_ih_l0_reverse 2 0.03889210894703865 +13 rnn.weight_ih_l0_reverse 3 -0.008453810587525368 +13 rnn.weight_ih_l0_reverse 4 -0.05074869096279144 +13 rnn.weight_ih_l0_reverse 5 0.006496469955891371 +13 rnn.weight_ih_l0_reverse 6 0.011819759383797646 +13 rnn.weight_ih_l0_reverse 7 0.060216642916202545 +13 rnn.weight_ih_l0_reverse 8 0.019731618463993073 +13 rnn.weight_ih_l0_reverse 9 0.0567062646150589 +13 rnn.weight_hh_l0_reverse 0 0.060816358774900436 +13 rnn.weight_hh_l0_reverse 1 0.004426890984177589 +13 rnn.weight_hh_l0_reverse 2 0.030765200033783913 +13 rnn.weight_hh_l0_reverse 3 0.04570603370666504 +13 rnn.weight_hh_l0_reverse 4 -0.016927670687437057 +13 rnn.weight_hh_l0_reverse 5 0.04420078918337822 +13 rnn.weight_hh_l0_reverse 6 0.04822644591331482 +13 rnn.weight_hh_l0_reverse 7 0.023585183545947075 +13 rnn.weight_hh_l0_reverse 8 -0.03718762844800949 +13 rnn.weight_hh_l0_reverse 9 -0.005171533208340406 +13 rnn.bias_ih_l0_reverse 0 0.057585686445236206 +13 rnn.bias_ih_l0_reverse 1 0.005609831307083368 +13 rnn.bias_ih_l0_reverse 2 0.051706377416849136 +13 rnn.bias_ih_l0_reverse 3 -0.048476751893758774 +13 rnn.bias_ih_l0_reverse 4 -0.02781234309077263 +13 rnn.bias_ih_l0_reverse 5 0.04832405969500542 +13 rnn.bias_ih_l0_reverse 6 -0.015519311651587486 +13 rnn.bias_ih_l0_reverse 7 0.05248453468084335 +13 rnn.bias_ih_l0_reverse 8 0.009869952686131 +13 rnn.bias_ih_l0_reverse 9 0.06282953172922134 +13 rnn.bias_hh_l0_reverse 0 -0.04577593132853508 +13 rnn.bias_hh_l0_reverse 1 -0.03891745209693909 +13 rnn.bias_hh_l0_reverse 2 -0.0294955987483263 +13 rnn.bias_hh_l0_reverse 3 0.02245115488767624 +13 rnn.bias_hh_l0_reverse 4 0.04252496734261513 +13 rnn.bias_hh_l0_reverse 5 0.05260394513607025 +13 rnn.bias_hh_l0_reverse 6 0.04794766753911972 +13 rnn.bias_hh_l0_reverse 7 -0.04037955030798912 +13 rnn.bias_hh_l0_reverse 8 -0.05949733406305313 +13 rnn.bias_hh_l0_reverse 9 0.038683295249938965 +13 linear.weight 0 0.007077266927808523 +13 linear.weight 1 0.01879849098622799 +13 linear.weight 2 0.03002571314573288 +13 linear.weight 3 0.023644983768463135 +13 linear.weight 4 -0.004396989941596985 +13 linear.weight 5 0.026535825803875923 +13 linear.weight 6 0.03439749404788017 +13 linear.weight 7 -0.13205856084823608 +13 linear.weight 8 -0.0344756618142128 +13 linear.weight 9 -0.07718458026647568 +13 linear.bias 0 -0.2793393135070801 +13 linear.bias 1 -0.26641130447387695 +13 linear.bias 2 -0.16698475182056427 +13 linear.bias 3 -0.11430428177118301 +13 linear.bias 4 0.114806167781353 +13 linear.bias 5 0.27826425433158875 +13 linear.bias 6 0.04451955109834671 +13 linear.bias 7 0.028284363448619843 +13 linear.bias 8 0.23093804717063904 +13 linear.bias 9 0.019625648856163025 +14 transitions 0 0.8216243982315063 +14 transitions 1 0.6285150051116943 +14 transitions 2 0.3933996558189392 +14 transitions 3 0.2368573397397995 +14 transitions 4 -0.25771328806877136 +14 transitions 5 1.5155563354492188 +14 transitions 6 -0.35212770104408264 +14 transitions 7 0.9510644674301147 +14 transitions 8 1.1233128309249878 +14 transitions 9 -0.8682103753089905 +14 embedding2nn.weight 0 -0.0059739211574196815 +14 embedding2nn.weight 1 -0.04385605454444885 +14 embedding2nn.weight 2 -0.03373902291059494 +14 embedding2nn.weight 3 -0.05680086091160774 +14 embedding2nn.weight 4 -0.003997976891696453 +14 embedding2nn.weight 5 0.06810314208269119 +14 embedding2nn.weight 6 0.06241893023252487 +14 embedding2nn.weight 7 -0.0445188544690609 +14 embedding2nn.weight 8 -0.013734666630625725 +14 embedding2nn.weight 9 0.07229366898536682 +14 embedding2nn.bias 0 -0.07807187736034393 +14 embedding2nn.bias 1 -0.09585173428058624 +14 embedding2nn.bias 2 -0.04711904004216194 +14 embedding2nn.bias 3 0.0005061440169811249 +14 embedding2nn.bias 4 -0.06478245556354523 +14 embedding2nn.bias 5 -0.08992713689804077 +14 embedding2nn.bias 6 -0.04171469435095787 +14 embedding2nn.bias 7 -0.06889936327934265 +14 embedding2nn.bias 8 0.07738233357667923 +14 embedding2nn.bias 9 0.0026715050917118788 +14 rnn.weight_ih_l0 0 0.05026600509881973 +14 rnn.weight_ih_l0 1 -0.020386043936014175 +14 rnn.weight_ih_l0 2 -0.0031253439374268055 +14 rnn.weight_ih_l0 3 -0.03843719884753227 +14 rnn.weight_ih_l0 4 -0.0069564273580908775 +14 rnn.weight_ih_l0 5 -0.050454169511795044 +14 rnn.weight_ih_l0 6 0.013554485514760017 +14 rnn.weight_ih_l0 7 0.04390985518693924 +14 rnn.weight_ih_l0 8 0.008291550911962986 +14 rnn.weight_ih_l0 9 0.04215870425105095 +14 rnn.weight_hh_l0 0 0.03684189170598984 +14 rnn.weight_hh_l0 1 -0.044473547488451004 +14 rnn.weight_hh_l0 2 0.02114655077457428 +14 rnn.weight_hh_l0 3 0.028753910213708878 +14 rnn.weight_hh_l0 4 0.06244289129972458 +14 rnn.weight_hh_l0 5 0.048455823212862015 +14 rnn.weight_hh_l0 6 0.04350871592760086 +14 rnn.weight_hh_l0 7 -0.057130731642246246 +14 rnn.weight_hh_l0 8 0.06212082877755165 +14 rnn.weight_hh_l0 9 -0.04532123729586601 +14 rnn.bias_ih_l0 0 -0.03493567928671837 +14 rnn.bias_ih_l0 1 -0.02164875715970993 +14 rnn.bias_ih_l0 2 0.06037981063127518 +14 rnn.bias_ih_l0 3 -0.010883759707212448 +14 rnn.bias_ih_l0 4 0.06470293551683426 +14 rnn.bias_ih_l0 5 0.0454886294901371 +14 rnn.bias_ih_l0 6 -0.05615789070725441 +14 rnn.bias_ih_l0 7 -0.00471928995102644 +14 rnn.bias_ih_l0 8 0.03937532380223274 +14 rnn.bias_ih_l0 9 0.02347138151526451 +14 rnn.bias_hh_l0 0 -0.05673433467745781 +14 rnn.bias_hh_l0 1 -0.03939957916736603 +14 rnn.bias_hh_l0 2 0.035961978137493134 +14 rnn.bias_hh_l0 3 0.03538438305258751 +14 rnn.bias_hh_l0 4 -0.04782579094171524 +14 rnn.bias_hh_l0 5 0.01515323668718338 +14 rnn.bias_hh_l0 6 0.04753796383738518 +14 rnn.bias_hh_l0 7 -0.03822655975818634 +14 rnn.bias_hh_l0 8 -0.02360505796968937 +14 rnn.bias_hh_l0 9 0.008379651233553886 +14 rnn.weight_ih_l0_reverse 0 -0.034102924168109894 +14 rnn.weight_ih_l0_reverse 1 -0.042053502053022385 +14 rnn.weight_ih_l0_reverse 2 0.03919902443885803 +14 rnn.weight_ih_l0_reverse 3 -0.008550469763576984 +14 rnn.weight_ih_l0_reverse 4 -0.05096902325749397 +14 rnn.weight_ih_l0_reverse 5 0.007153776474297047 +14 rnn.weight_ih_l0_reverse 6 0.011807397939264774 +14 rnn.weight_ih_l0_reverse 7 0.06038431450724602 +14 rnn.weight_ih_l0_reverse 8 0.019886991009116173 +14 rnn.weight_ih_l0_reverse 9 0.056895364075899124 +14 rnn.weight_hh_l0_reverse 0 0.06092303618788719 +14 rnn.weight_hh_l0_reverse 1 0.004440034739673138 +14 rnn.weight_hh_l0_reverse 2 0.030779890716075897 +14 rnn.weight_hh_l0_reverse 3 0.04555046185851097 +14 rnn.weight_hh_l0_reverse 4 -0.016899632290005684 +14 rnn.weight_hh_l0_reverse 5 0.044090185314416885 +14 rnn.weight_hh_l0_reverse 6 0.04824286699295044 +14 rnn.weight_hh_l0_reverse 7 0.023683980107307434 +14 rnn.weight_hh_l0_reverse 8 -0.03735045716166496 +14 rnn.weight_hh_l0_reverse 9 -0.00525254663079977 +14 rnn.bias_ih_l0_reverse 0 0.05860627442598343 +14 rnn.bias_ih_l0_reverse 1 0.005896052811294794 +14 rnn.bias_ih_l0_reverse 2 0.052245981991291046 +14 rnn.bias_ih_l0_reverse 3 -0.049143824726343155 +14 rnn.bias_ih_l0_reverse 4 -0.027723588049411774 +14 rnn.bias_ih_l0_reverse 5 0.04457731172442436 +14 rnn.bias_ih_l0_reverse 6 -0.016766540706157684 +14 rnn.bias_ih_l0_reverse 7 0.05261726304888725 +14 rnn.bias_ih_l0_reverse 8 0.009920155629515648 +14 rnn.bias_ih_l0_reverse 9 0.06298186630010605 +14 rnn.bias_hh_l0_reverse 0 -0.043898288160562515 +14 rnn.bias_hh_l0_reverse 1 -0.038902074098587036 +14 rnn.bias_hh_l0_reverse 2 -0.02933465503156185 +14 rnn.bias_hh_l0_reverse 3 0.022457865998148918 +14 rnn.bias_hh_l0_reverse 4 0.0432104654610157 +14 rnn.bias_hh_l0_reverse 5 0.05352029204368591 +14 rnn.bias_hh_l0_reverse 6 0.048116814345121384 +14 rnn.bias_hh_l0_reverse 7 -0.040280308574438095 +14 rnn.bias_hh_l0_reverse 8 -0.05850490927696228 +14 rnn.bias_hh_l0_reverse 9 0.03927866742014885 +14 linear.weight 0 0.005779434461146593 +14 linear.weight 1 0.022312723100185394 +14 linear.weight 2 0.0356004498898983 +14 linear.weight 3 0.025063442066311836 +14 linear.weight 4 -0.004396989941596985 +14 linear.weight 5 0.02296966314315796 +14 linear.weight 6 0.03439749404788017 +14 linear.weight 7 -0.13205856084823608 +14 linear.weight 8 -0.0344756618142128 +14 linear.weight 9 -0.09196941554546356 +14 linear.bias 0 -0.28764966130256653 +14 linear.bias 1 -0.2739725112915039 +14 linear.bias 2 -0.16876380145549774 +14 linear.bias 3 -0.12179511785507202 +14 linear.bias 4 0.12910065054893494 +14 linear.bias 5 0.25679585337638855 +14 linear.bias 6 0.05203200876712799 +14 linear.bias 7 0.04083017259836197 +14 linear.bias 8 0.23956312239170074 +14 linear.bias 9 0.025068508461117744 +15 transitions 0 0.8214089870452881 +15 transitions 1 0.6282925009727478 +15 transitions 2 0.39324015378952026 +15 transitions 3 0.23658540844917297 +15 transitions 4 -0.24301652610301971 +15 transitions 5 1.5153141021728516 +15 transitions 6 -0.3521777391433716 +15 transitions 7 0.9508687257766724 +15 transitions 8 1.1223620176315308 +15 transitions 9 -0.8682916760444641 +15 embedding2nn.weight 0 -0.0059788706712424755 +15 embedding2nn.weight 1 -0.0438445545732975 +15 embedding2nn.weight 2 -0.0344034805893898 +15 embedding2nn.weight 3 -0.05670171231031418 +15 embedding2nn.weight 4 -0.012964229099452496 +15 embedding2nn.weight 5 0.06688647717237473 +15 embedding2nn.weight 6 0.06270143389701843 +15 embedding2nn.weight 7 -0.047995880246162415 +15 embedding2nn.weight 8 -0.011401516385376453 +15 embedding2nn.weight 9 0.07010933756828308 +15 embedding2nn.bias 0 -0.0713803619146347 +15 embedding2nn.bias 1 -0.08802444487810135 +15 embedding2nn.bias 2 -0.05220814794301987 +15 embedding2nn.bias 3 0.010576571337878704 +15 embedding2nn.bias 4 -0.06647782027721405 +15 embedding2nn.bias 5 -0.08343641459941864 +15 embedding2nn.bias 6 -0.03940269723534584 +15 embedding2nn.bias 7 -0.06896982342004776 +15 embedding2nn.bias 8 0.07692500948905945 +15 embedding2nn.bias 9 0.0028787855990231037 +15 rnn.weight_ih_l0 0 0.05034330487251282 +15 rnn.weight_ih_l0 1 -0.020357292145490646 +15 rnn.weight_ih_l0 2 -0.0032693909015506506 +15 rnn.weight_ih_l0 3 -0.03872701898217201 +15 rnn.weight_ih_l0 4 -0.008558331988751888 +15 rnn.weight_ih_l0 5 -0.05042484402656555 +15 rnn.weight_ih_l0 6 0.013713615946471691 +15 rnn.weight_ih_l0 7 0.04378729313611984 +15 rnn.weight_ih_l0 8 0.00830957293510437 +15 rnn.weight_ih_l0 9 0.042035166174173355 +15 rnn.weight_hh_l0 0 0.03686100244522095 +15 rnn.weight_hh_l0 1 -0.04443295672535896 +15 rnn.weight_hh_l0 2 0.021209733560681343 +15 rnn.weight_hh_l0 3 0.02877218648791313 +15 rnn.weight_hh_l0 4 0.06244749203324318 +15 rnn.weight_hh_l0 5 0.04844977334141731 +15 rnn.weight_hh_l0 6 0.04351644963026047 +15 rnn.weight_hh_l0 7 -0.057043083012104034 +15 rnn.weight_hh_l0 8 0.06211899593472481 +15 rnn.weight_hh_l0 9 -0.04541594535112381 +15 rnn.bias_ih_l0 0 -0.034693218767642975 +15 rnn.bias_ih_l0 1 -0.021392274647951126 +15 rnn.bias_ih_l0 2 0.06057819724082947 +15 rnn.bias_ih_l0 3 -0.011450828984379768 +15 rnn.bias_ih_l0 4 0.06518431007862091 +15 rnn.bias_ih_l0 5 0.045080289244651794 +15 rnn.bias_ih_l0 6 -0.05629217252135277 +15 rnn.bias_ih_l0 7 -0.003222996136173606 +15 rnn.bias_ih_l0 8 0.03966138884425163 +15 rnn.bias_ih_l0 9 0.02352595143020153 +15 rnn.bias_hh_l0 0 -0.05650565028190613 +15 rnn.bias_hh_l0 1 -0.03903939574956894 +15 rnn.bias_hh_l0 2 0.03637376427650452 +15 rnn.bias_hh_l0 3 0.036522671580314636 +15 rnn.bias_hh_l0 4 -0.04759831726551056 +15 rnn.bias_hh_l0 5 0.015284299850463867 +15 rnn.bias_hh_l0 6 0.04758567735552788 +15 rnn.bias_hh_l0 7 -0.03826595097780228 +15 rnn.bias_hh_l0 8 -0.02265365608036518 +15 rnn.bias_hh_l0 9 0.008620001375675201 +15 rnn.weight_ih_l0_reverse 0 -0.03362007811665535 +15 rnn.weight_ih_l0_reverse 1 -0.042048729956150055 +15 rnn.weight_ih_l0_reverse 2 0.03923213109374046 +15 rnn.weight_ih_l0_reverse 3 -0.008544377982616425 +15 rnn.weight_ih_l0_reverse 4 -0.05064305290579796 +15 rnn.weight_ih_l0_reverse 5 0.006817522458732128 +15 rnn.weight_ih_l0_reverse 6 0.011739728972315788 +15 rnn.weight_ih_l0_reverse 7 0.06019292026758194 +15 rnn.weight_ih_l0_reverse 8 0.019931048154830933 +15 rnn.weight_ih_l0_reverse 9 0.056870635598897934 +15 rnn.weight_hh_l0_reverse 0 0.06091591343283653 +15 rnn.weight_hh_l0_reverse 1 0.004451114218682051 +15 rnn.weight_hh_l0_reverse 2 0.030800599604845047 +15 rnn.weight_hh_l0_reverse 3 0.045403216034173965 +15 rnn.weight_hh_l0_reverse 4 -0.016823792830109596 +15 rnn.weight_hh_l0_reverse 5 0.04417017847299576 +15 rnn.weight_hh_l0_reverse 6 0.04836050420999527 +15 rnn.weight_hh_l0_reverse 7 0.023607127368450165 +15 rnn.weight_hh_l0_reverse 8 -0.03741675242781639 +15 rnn.weight_hh_l0_reverse 9 -0.005058954935520887 +15 rnn.bias_ih_l0_reverse 0 0.058338794857263565 +15 rnn.bias_ih_l0_reverse 1 0.005959671456366777 +15 rnn.bias_ih_l0_reverse 2 0.051666244864463806 +15 rnn.bias_ih_l0_reverse 3 -0.04898853972554207 +15 rnn.bias_ih_l0_reverse 4 -0.02760329656302929 +15 rnn.bias_ih_l0_reverse 5 0.048754025250673294 +15 rnn.bias_ih_l0_reverse 6 -0.016294505447149277 +15 rnn.bias_ih_l0_reverse 7 0.052583031356334686 +15 rnn.bias_ih_l0_reverse 8 0.009871255606412888 +15 rnn.bias_ih_l0_reverse 9 0.06290200352668762 +15 rnn.bias_hh_l0_reverse 0 -0.04466983303427696 +15 rnn.bias_hh_l0_reverse 1 -0.03875858709216118 +15 rnn.bias_hh_l0_reverse 2 -0.029074164107441902 +15 rnn.bias_hh_l0_reverse 3 0.022517023608088493 +15 rnn.bias_hh_l0_reverse 4 0.04285811632871628 +15 rnn.bias_hh_l0_reverse 5 0.05309007689356804 +15 rnn.bias_hh_l0_reverse 6 0.04813633859157562 +15 rnn.bias_hh_l0_reverse 7 -0.0403120331466198 +15 rnn.bias_hh_l0_reverse 8 -0.058415576815605164 +15 rnn.bias_hh_l0_reverse 9 0.0391431599855423 +15 linear.weight 0 0.005779434461146593 +15 linear.weight 1 0.021293630823493004 +15 linear.weight 2 0.0413128100335598 +15 linear.weight 3 0.025209901854395866 +15 linear.weight 4 -0.004396989941596985 +15 linear.weight 5 0.022867420688271523 +15 linear.weight 6 0.03439749404788017 +15 linear.weight 7 -0.13205856084823608 +15 linear.weight 8 -0.0344756618142128 +15 linear.weight 9 -0.08545565605163574 +15 linear.bias 0 -0.2890442907810211 +15 linear.bias 1 -0.27501723170280457 +15 linear.bias 2 -0.1607954353094101 +15 linear.bias 3 -0.11186253279447556 +15 linear.bias 4 0.10623281449079514 +15 linear.bias 5 0.2651507556438446 +15 linear.bias 6 0.05907547101378441 +15 linear.bias 7 0.037460409104824066 +15 linear.bias 8 0.20733366906642914 +15 linear.bias 9 0.02757040597498417 +16 transitions 0 0.819715678691864 +16 transitions 1 0.6267836689949036 +16 transitions 2 0.3911966383457184 +16 transitions 3 0.2355320304632187 +16 transitions 4 -0.21408140659332275 +16 transitions 5 1.5126069784164429 +16 transitions 6 -0.3523148000240326 +16 transitions 7 0.9499322175979614 +16 transitions 8 1.1192219257354736 +16 transitions 9 -0.8685082197189331 +16 embedding2nn.weight 0 -0.006100327242165804 +16 embedding2nn.weight 1 -0.04534785822033882 +16 embedding2nn.weight 2 -0.035174667835235596 +16 embedding2nn.weight 3 -0.056097544729709625 +16 embedding2nn.weight 4 -0.010469280183315277 +16 embedding2nn.weight 5 0.06642412394285202 +16 embedding2nn.weight 6 0.06273350119590759 +16 embedding2nn.weight 7 -0.04142111539840698 +16 embedding2nn.weight 8 -0.013386673294007778 +16 embedding2nn.weight 9 0.06907793879508972 +16 embedding2nn.bias 0 -0.07437953352928162 +16 embedding2nn.bias 1 -0.09158015251159668 +16 embedding2nn.bias 2 -0.048436976969242096 +16 embedding2nn.bias 3 0.012162389233708382 +16 embedding2nn.bias 4 -0.0714663565158844 +16 embedding2nn.bias 5 -0.08549769967794418 +16 embedding2nn.bias 6 -0.037552282214164734 +16 embedding2nn.bias 7 -0.06847827881574631 +16 embedding2nn.bias 8 0.07842326164245605 +16 embedding2nn.bias 9 -0.00022615394846070558 +16 rnn.weight_ih_l0 0 0.050425563007593155 +16 rnn.weight_ih_l0 1 -0.020188651978969574 +16 rnn.weight_ih_l0 2 -0.0031728073954582214 +16 rnn.weight_ih_l0 3 -0.0388287752866745 +16 rnn.weight_ih_l0 4 -0.009603733196854591 +16 rnn.weight_ih_l0 5 -0.05018468573689461 +16 rnn.weight_ih_l0 6 0.012503954581916332 +16 rnn.weight_ih_l0 7 0.043554384261369705 +16 rnn.weight_ih_l0 8 0.008525367826223373 +16 rnn.weight_ih_l0 9 0.0420033298432827 +16 rnn.weight_hh_l0 0 0.03692684695124626 +16 rnn.weight_hh_l0 1 -0.04440389946103096 +16 rnn.weight_hh_l0 2 0.021258793771266937 +16 rnn.weight_hh_l0 3 0.028814133256673813 +16 rnn.weight_hh_l0 4 0.06248440966010094 +16 rnn.weight_hh_l0 5 0.04845535010099411 +16 rnn.weight_hh_l0 6 0.043498918414115906 +16 rnn.weight_hh_l0 7 -0.05691322684288025 +16 rnn.weight_hh_l0 8 0.06211375817656517 +16 rnn.weight_hh_l0 9 -0.045114997774362564 +16 rnn.bias_ih_l0 0 -0.034147653728723526 +16 rnn.bias_ih_l0 1 -0.020987695083022118 +16 rnn.bias_ih_l0 2 0.06116851791739464 +16 rnn.bias_ih_l0 3 -0.011085809208452702 +16 rnn.bias_ih_l0 4 0.0653405487537384 +16 rnn.bias_ih_l0 5 0.045415036380290985 +16 rnn.bias_ih_l0 6 -0.055954571813344955 +16 rnn.bias_ih_l0 7 -0.00022985068790148944 +16 rnn.bias_ih_l0 8 0.039133910089731216 +16 rnn.bias_ih_l0 9 0.02357126958668232 +16 rnn.bias_hh_l0 0 -0.056010134518146515 +16 rnn.bias_hh_l0 1 -0.03845217451453209 +16 rnn.bias_hh_l0 2 0.037328675389289856 +16 rnn.bias_hh_l0 3 0.034389834851026535 +16 rnn.bias_hh_l0 4 -0.047362424433231354 +16 rnn.bias_hh_l0 5 0.015473026782274246 +16 rnn.bias_hh_l0 6 0.047679394483566284 +16 rnn.bias_hh_l0 7 -0.038253266364336014 +16 rnn.bias_hh_l0 8 -0.022131256759166718 +16 rnn.bias_hh_l0 9 0.009084910154342651 +16 rnn.weight_ih_l0_reverse 0 -0.033635225147008896 +16 rnn.weight_ih_l0_reverse 1 -0.04215126857161522 +16 rnn.weight_ih_l0_reverse 2 0.039788879454135895 +16 rnn.weight_ih_l0_reverse 3 -0.008656592108309269 +16 rnn.weight_ih_l0_reverse 4 -0.05110003799200058 +16 rnn.weight_ih_l0_reverse 5 0.007261468097567558 +16 rnn.weight_ih_l0_reverse 6 0.011624349281191826 +16 rnn.weight_ih_l0_reverse 7 0.061028920114040375 +16 rnn.weight_ih_l0_reverse 8 0.020098460838198662 +16 rnn.weight_ih_l0_reverse 9 0.057008761912584305 +16 rnn.weight_hh_l0_reverse 0 0.06098819151520729 +16 rnn.weight_hh_l0_reverse 1 0.0044708591885864735 +16 rnn.weight_hh_l0_reverse 2 0.030835764482617378 +16 rnn.weight_hh_l0_reverse 3 0.04504567012190819 +16 rnn.weight_hh_l0_reverse 4 -0.01683793216943741 +16 rnn.weight_hh_l0_reverse 5 0.04403064027428627 +16 rnn.weight_hh_l0_reverse 6 0.04872389882802963 +16 rnn.weight_hh_l0_reverse 7 0.023703403770923615 +16 rnn.weight_hh_l0_reverse 8 -0.03740508854389191 +16 rnn.weight_hh_l0_reverse 9 -0.005029088817536831 +16 rnn.bias_ih_l0_reverse 0 0.058561570942401886 +16 rnn.bias_ih_l0_reverse 1 0.006156252231448889 +16 rnn.bias_ih_l0_reverse 2 0.052167147397994995 +16 rnn.bias_ih_l0_reverse 3 -0.0500989630818367 +16 rnn.bias_ih_l0_reverse 4 -0.027442174032330513 +16 rnn.bias_ih_l0_reverse 5 0.046747978776693344 +16 rnn.bias_ih_l0_reverse 6 -0.016020147129893303 +16 rnn.bias_ih_l0_reverse 7 0.05276082828640938 +16 rnn.bias_ih_l0_reverse 8 0.00996160414069891 +16 rnn.bias_ih_l0_reverse 9 0.06329207122325897 +16 rnn.bias_hh_l0_reverse 0 -0.044790033251047134 +16 rnn.bias_hh_l0_reverse 1 -0.03881954029202461 +16 rnn.bias_hh_l0_reverse 2 -0.02889503538608551 +16 rnn.bias_hh_l0_reverse 3 0.02293528988957405 +16 rnn.bias_hh_l0_reverse 4 0.04335286468267441 +16 rnn.bias_hh_l0_reverse 5 0.05369555205106735 +16 rnn.bias_hh_l0_reverse 6 0.0482267290353775 +16 rnn.bias_hh_l0_reverse 7 -0.039860401302576065 +16 rnn.bias_hh_l0_reverse 8 -0.058096010237932205 +16 rnn.bias_hh_l0_reverse 9 0.03916342183947563 +16 linear.weight 0 0.005530455615371466 +16 linear.weight 1 0.011884394101798534 +16 linear.weight 2 0.047947149723768234 +16 linear.weight 3 0.025812489911913872 +16 linear.weight 4 -0.004396989941596985 +16 linear.weight 5 0.021931249648332596 +16 linear.weight 6 0.03439749404788017 +16 linear.weight 7 -0.13205856084823608 +16 linear.weight 8 -0.0344756618142128 +16 linear.weight 9 -0.07970447838306427 +16 linear.bias 0 -0.2938591539859772 +16 linear.bias 1 -0.28011205792427063 +16 linear.bias 2 -0.1597052663564682 +16 linear.bias 3 -0.10922181606292725 +16 linear.bias 4 0.11279323697090149 +16 linear.bias 5 0.25698378682136536 +16 linear.bias 6 0.06428508460521698 +16 linear.bias 7 0.042781051248311996 +16 linear.bias 8 0.22828111052513123 +16 linear.bias 9 0.02862384542822838 +17 transitions 0 0.8185954093933105 +17 transitions 1 0.6262224316596985 +17 transitions 2 0.3902488946914673 +17 transitions 3 0.23497481644153595 +17 transitions 4 -0.19715972244739532 +17 transitions 5 1.511116623878479 +17 transitions 6 -0.3524032533168793 +17 transitions 7 0.9492687582969666 +17 transitions 8 1.1177743673324585 +17 transitions 9 -0.868635892868042 +17 embedding2nn.weight 0 -0.0072334143333137035 +17 embedding2nn.weight 1 -0.045321423560380936 +17 embedding2nn.weight 2 -0.0357089601457119 +17 embedding2nn.weight 3 -0.05604789778590202 +17 embedding2nn.weight 4 -0.010177446529269218 +17 embedding2nn.weight 5 0.06332453340291977 +17 embedding2nn.weight 6 0.06285908073186874 +17 embedding2nn.weight 7 -0.043786704540252686 +17 embedding2nn.weight 8 -0.012126276269555092 +17 embedding2nn.weight 9 0.06761594861745834 +17 embedding2nn.bias 0 -0.07444357872009277 +17 embedding2nn.bias 1 -0.09105461090803146 +17 embedding2nn.bias 2 -0.04917953908443451 +17 embedding2nn.bias 3 0.0006674448377452791 +17 embedding2nn.bias 4 -0.0606129914522171 +17 embedding2nn.bias 5 -0.08060452342033386 +17 embedding2nn.bias 6 -0.041200168430805206 +17 embedding2nn.bias 7 -0.06783442944288254 +17 embedding2nn.bias 8 0.07348281145095825 +17 embedding2nn.bias 9 0.006125360261648893 +17 rnn.weight_ih_l0 0 0.050498075783252716 +17 rnn.weight_ih_l0 1 -0.020156962797045708 +17 rnn.weight_ih_l0 2 -0.0031560154166072607 +17 rnn.weight_ih_l0 3 -0.03943122178316116 +17 rnn.weight_ih_l0 4 -0.009727063588798046 +17 rnn.weight_ih_l0 5 -0.049854181706905365 +17 rnn.weight_ih_l0 6 0.014274327084422112 +17 rnn.weight_ih_l0 7 0.04354093596339226 +17 rnn.weight_ih_l0 8 0.008573782630264759 +17 rnn.weight_ih_l0 9 0.042030807584524155 +17 rnn.weight_hh_l0 0 0.03696851059794426 +17 rnn.weight_hh_l0 1 -0.04438430443406105 +17 rnn.weight_hh_l0 2 0.021306488662958145 +17 rnn.weight_hh_l0 3 0.028809312731027603 +17 rnn.weight_hh_l0 4 0.062482405453920364 +17 rnn.weight_hh_l0 5 0.048431482166051865 +17 rnn.weight_hh_l0 6 0.04349259287118912 +17 rnn.weight_hh_l0 7 -0.056867845356464386 +17 rnn.weight_hh_l0 8 0.06211043521761894 +17 rnn.weight_hh_l0 9 -0.04506908357143402 +17 rnn.bias_ih_l0 0 -0.03378807753324509 +17 rnn.bias_ih_l0 1 -0.020910531282424927 +17 rnn.bias_ih_l0 2 0.06119101867079735 +17 rnn.bias_ih_l0 3 -0.011318860575556755 +17 rnn.bias_ih_l0 4 0.0649750679731369 +17 rnn.bias_ih_l0 5 0.04551047831773758 +17 rnn.bias_ih_l0 6 -0.05585867539048195 +17 rnn.bias_ih_l0 7 -0.0032031594309955835 +17 rnn.bias_ih_l0 8 0.03926560655236244 +17 rnn.bias_ih_l0 9 0.02354389801621437 +17 rnn.bias_hh_l0 0 -0.055942386388778687 +17 rnn.bias_hh_l0 1 -0.03881499171257019 +17 rnn.bias_hh_l0 2 0.037498895078897476 +17 rnn.bias_hh_l0 3 0.034690357744693756 +17 rnn.bias_hh_l0 4 -0.04730157554149628 +17 rnn.bias_hh_l0 5 0.015691954642534256 +17 rnn.bias_hh_l0 6 0.04775334522128105 +17 rnn.bias_hh_l0 7 -0.03823952004313469 +17 rnn.bias_hh_l0 8 -0.021916579455137253 +17 rnn.bias_hh_l0 9 0.009304128587245941 +17 rnn.weight_ih_l0_reverse 0 -0.033962078392505646 +17 rnn.weight_ih_l0_reverse 1 -0.04216044768691063 +17 rnn.weight_ih_l0_reverse 2 0.03988882526755333 +17 rnn.weight_ih_l0_reverse 3 -0.008749661967158318 +17 rnn.weight_ih_l0_reverse 4 -0.050957247614860535 +17 rnn.weight_ih_l0_reverse 5 0.00735511677339673 +17 rnn.weight_ih_l0_reverse 6 0.011562192812561989 +17 rnn.weight_ih_l0_reverse 7 0.06112326309084892 +17 rnn.weight_ih_l0_reverse 8 0.02016875520348549 +17 rnn.weight_ih_l0_reverse 9 0.057037003338336945 +17 rnn.weight_hh_l0_reverse 0 0.0610613115131855 +17 rnn.weight_hh_l0_reverse 1 0.004452341701835394 +17 rnn.weight_hh_l0_reverse 2 0.03085404448211193 +17 rnn.weight_hh_l0_reverse 3 0.044903334230184555 +17 rnn.weight_hh_l0_reverse 4 -0.01680116355419159 +17 rnn.weight_hh_l0_reverse 5 0.043959688395261765 +17 rnn.weight_hh_l0_reverse 6 0.04829198494553566 +17 rnn.weight_hh_l0_reverse 7 0.023473862558603287 +17 rnn.weight_hh_l0_reverse 8 -0.03745732456445694 +17 rnn.weight_hh_l0_reverse 9 -0.005126554518938065 +17 rnn.bias_ih_l0_reverse 0 0.05882933363318443 +17 rnn.bias_ih_l0_reverse 1 0.006211595609784126 +17 rnn.bias_ih_l0_reverse 2 0.0521550327539444 +17 rnn.bias_ih_l0_reverse 3 -0.04909061640501022 +17 rnn.bias_ih_l0_reverse 4 -0.02731849066913128 +17 rnn.bias_ih_l0_reverse 5 0.04631249979138374 +17 rnn.bias_ih_l0_reverse 6 -0.01687450520694256 +17 rnn.bias_ih_l0_reverse 7 0.0532132163643837 +17 rnn.bias_ih_l0_reverse 8 0.010010503232479095 +17 rnn.bias_ih_l0_reverse 9 0.0634106770157814 +17 rnn.bias_hh_l0_reverse 0 -0.045563168823719025 +17 rnn.bias_hh_l0_reverse 1 -0.03915426880121231 +17 rnn.bias_hh_l0_reverse 2 -0.02890247106552124 +17 rnn.bias_hh_l0_reverse 3 0.022785328328609467 +17 rnn.bias_hh_l0_reverse 4 0.04351726919412613 +17 rnn.bias_hh_l0_reverse 5 0.05388083681464195 +17 rnn.bias_hh_l0_reverse 6 0.048258885741233826 +17 rnn.bias_hh_l0_reverse 7 -0.0397026501595974 +17 rnn.bias_hh_l0_reverse 8 -0.057951364666223526 +17 rnn.bias_hh_l0_reverse 9 0.039420727640390396 +17 linear.weight 0 0.004963491577655077 +17 linear.weight 1 0.0304242093116045 +17 linear.weight 2 0.050723958760499954 +17 linear.weight 3 0.026012910529971123 +17 linear.weight 4 -0.004396989941596985 +17 linear.weight 5 0.022635551169514656 +17 linear.weight 6 0.03439749404788017 +17 linear.weight 7 -0.13269060850143433 +17 linear.weight 8 -0.0344756618142128 +17 linear.weight 9 -0.08839801698923111 +17 linear.bias 0 -0.29630470275878906 +17 linear.bias 1 -0.2824646830558777 +17 linear.bias 2 -0.15610240399837494 +17 linear.bias 3 -0.10796662420034409 +17 linear.bias 4 0.11431429535150528 +17 linear.bias 5 0.24448969960212708 +17 linear.bias 6 0.06484142690896988 +17 linear.bias 7 0.03757862001657486 +17 linear.bias 8 0.19745227694511414 +17 linear.bias 9 0.029141193255782127 +18 transitions 0 0.8183441162109375 +18 transitions 1 0.6259543299674988 +18 transitions 2 0.3899310231208801 +18 transitions 3 0.2348252236843109 +18 transitions 4 -0.18789289891719818 +18 transitions 5 1.5105262994766235 +18 transitions 6 -0.3524238169193268 +18 transitions 7 0.949140727519989 +18 transitions 8 1.117303490638733 +18 transitions 9 -0.8686699867248535 +18 embedding2nn.weight 0 -0.007060606498271227 +18 embedding2nn.weight 1 -0.04517776891589165 +18 embedding2nn.weight 2 -0.036172837018966675 +18 embedding2nn.weight 3 -0.056124038994312286 +18 embedding2nn.weight 4 -0.012216655537486076 +18 embedding2nn.weight 5 0.06338880211114883 +18 embedding2nn.weight 6 0.06292803585529327 +18 embedding2nn.weight 7 -0.043625276535749435 +18 embedding2nn.weight 8 -0.010732834227383137 +18 embedding2nn.weight 9 0.06794615834951401 +18 embedding2nn.bias 0 -0.07264532148838043 +18 embedding2nn.bias 1 -0.08709122985601425 +18 embedding2nn.bias 2 -0.050269290804862976 +18 embedding2nn.bias 3 0.018262097612023354 +18 embedding2nn.bias 4 -0.07105758041143417 +18 embedding2nn.bias 5 -0.08205357939004898 +18 embedding2nn.bias 6 -0.0364084355533123 +18 embedding2nn.bias 7 -0.06812173128128052 +18 embedding2nn.bias 8 0.07741501927375793 +18 embedding2nn.bias 9 0.0022671527694910765 +18 rnn.weight_ih_l0 0 0.05051656439900398 +18 rnn.weight_ih_l0 1 -0.0201554112136364 +18 rnn.weight_ih_l0 2 -0.0033559994772076607 +18 rnn.weight_ih_l0 3 -0.03931436315178871 +18 rnn.weight_ih_l0 4 -0.009129504673182964 +18 rnn.weight_ih_l0 5 -0.04985971376299858 +18 rnn.weight_ih_l0 6 0.013577830046415329 +18 rnn.weight_ih_l0 7 0.04351363331079483 +18 rnn.weight_ih_l0 8 0.008673764765262604 +18 rnn.weight_ih_l0 9 0.041967280209064484 +18 rnn.weight_hh_l0 0 0.036978352814912796 +18 rnn.weight_hh_l0 1 -0.04437405243515968 +18 rnn.weight_hh_l0 2 0.021317634731531143 +18 rnn.weight_hh_l0 3 0.02883428893983364 +18 rnn.weight_hh_l0 4 0.062483128160238266 +18 rnn.weight_hh_l0 5 0.04842843487858772 +18 rnn.weight_hh_l0 6 0.04349273815751076 +18 rnn.weight_hh_l0 7 -0.0568540133535862 +18 rnn.weight_hh_l0 8 0.06211017444729805 +18 rnn.weight_hh_l0 9 -0.045013800263404846 +18 rnn.bias_ih_l0 0 -0.03400171548128128 +18 rnn.bias_ih_l0 1 -0.020910127088427544 +18 rnn.bias_ih_l0 2 0.0614447258412838 +18 rnn.bias_ih_l0 3 -0.010167527943849564 +18 rnn.bias_ih_l0 4 0.0651196837425232 +18 rnn.bias_ih_l0 5 0.04553261771798134 +18 rnn.bias_ih_l0 6 -0.05567705258727074 +18 rnn.bias_ih_l0 7 -0.0013641240075230598 +18 rnn.bias_ih_l0 8 0.039389997720718384 +18 rnn.bias_ih_l0 9 0.023518463596701622 +18 rnn.bias_hh_l0 0 -0.05579974129796028 +18 rnn.bias_hh_l0 1 -0.03843792900443077 +18 rnn.bias_hh_l0 2 0.03798957169055939 +18 rnn.bias_hh_l0 3 0.026570122689008713 +18 rnn.bias_hh_l0 4 -0.047077201306819916 +18 rnn.bias_hh_l0 5 0.015758005902171135 +18 rnn.bias_hh_l0 6 0.04778636619448662 +18 rnn.bias_hh_l0 7 -0.0383865088224411 +18 rnn.bias_hh_l0 8 -0.0212198868393898 +18 rnn.bias_hh_l0 9 0.009265616536140442 +18 rnn.weight_ih_l0_reverse 0 -0.03336202725768089 +18 rnn.weight_ih_l0_reverse 1 -0.042176660150289536 +18 rnn.weight_ih_l0_reverse 2 0.03977521136403084 +18 rnn.weight_ih_l0_reverse 3 -0.008732298389077187 +18 rnn.weight_ih_l0_reverse 4 -0.05093865841627121 +18 rnn.weight_ih_l0_reverse 5 0.007107497192919254 +18 rnn.weight_ih_l0_reverse 6 0.011528599075973034 +18 rnn.weight_ih_l0_reverse 7 0.061110250651836395 +18 rnn.weight_ih_l0_reverse 8 0.019982462748885155 +18 rnn.weight_ih_l0_reverse 9 0.05705813691020012 +18 rnn.weight_hh_l0_reverse 0 0.061060480773448944 +18 rnn.weight_hh_l0_reverse 1 0.004449218511581421 +18 rnn.weight_hh_l0_reverse 2 0.030859466642141342 +18 rnn.weight_hh_l0_reverse 3 0.044829003512859344 +18 rnn.weight_hh_l0_reverse 4 -0.016761861741542816 +18 rnn.weight_hh_l0_reverse 5 0.04402604699134827 +18 rnn.weight_hh_l0_reverse 6 0.048533450812101364 +18 rnn.weight_hh_l0_reverse 7 0.02361427992582321 +18 rnn.weight_hh_l0_reverse 8 -0.03753717616200447 +18 rnn.weight_hh_l0_reverse 9 -0.0050187367014586926 +18 rnn.bias_ih_l0_reverse 0 0.058809928596019745 +18 rnn.bias_ih_l0_reverse 1 0.006413079798221588 +18 rnn.bias_ih_l0_reverse 2 0.05250826105475426 +18 rnn.bias_ih_l0_reverse 3 -0.05051659047603607 +18 rnn.bias_ih_l0_reverse 4 -0.02753208763897419 +18 rnn.bias_ih_l0_reverse 5 0.046896226704120636 +18 rnn.bias_ih_l0_reverse 6 -0.015892401337623596 +18 rnn.bias_ih_l0_reverse 7 0.052903976291418076 +18 rnn.bias_ih_l0_reverse 8 0.010161679238080978 +18 rnn.bias_ih_l0_reverse 9 0.06357771903276443 +18 rnn.bias_hh_l0_reverse 0 -0.0461445115506649 +18 rnn.bias_hh_l0_reverse 1 -0.039113666862249374 +18 rnn.bias_hh_l0_reverse 2 -0.02881297655403614 +18 rnn.bias_hh_l0_reverse 3 0.022817861288785934 +18 rnn.bias_hh_l0_reverse 4 0.0433785542845726 +18 rnn.bias_hh_l0_reverse 5 0.05371687188744545 +18 rnn.bias_hh_l0_reverse 6 0.04837727174162865 +18 rnn.bias_hh_l0_reverse 7 -0.04016033932566643 +18 rnn.bias_hh_l0_reverse 8 -0.058653783053159714 +18 rnn.bias_hh_l0_reverse 9 0.03913908824324608 +18 linear.weight 0 0.00479766633361578 +18 linear.weight 1 0.02637237310409546 +18 linear.weight 2 0.05069718509912491 +18 linear.weight 3 0.026042168959975243 +18 linear.weight 4 -0.004396989941596985 +18 linear.weight 5 0.02257634699344635 +18 linear.weight 6 0.03439749404788017 +18 linear.weight 7 -0.13418060541152954 +18 linear.weight 8 -0.0344756618142128 +18 linear.weight 9 -0.07971733063459396 +18 linear.bias 0 -0.29716357588768005 +18 linear.bias 1 -0.2832487225532532 +18 linear.bias 2 -0.1538861244916916 +18 linear.bias 3 -0.10345316678285599 +18 linear.bias 4 0.10352876782417297 +18 linear.bias 5 0.2574295699596405 +18 linear.bias 6 0.07048293203115463 +18 linear.bias 7 0.04203685000538826 +18 linear.bias 8 0.21062171459197998 +18 linear.bias 9 0.03194578364491463 +19 transitions 0 0.8158766031265259 +19 transitions 1 0.6248497366905212 +19 transitions 2 0.3870950937271118 +19 transitions 3 0.2341669797897339 +19 transitions 4 -0.16244225203990936 +19 transitions 5 1.5060665607452393 +19 transitions 6 -0.3525603413581848 +19 transitions 7 0.9480622410774231 +19 transitions 8 1.115427851676941 +19 transitions 9 -0.8688962459564209 +19 embedding2nn.weight 0 -0.0077723185531795025 +19 embedding2nn.weight 1 -0.046502865850925446 +19 embedding2nn.weight 2 -0.03789488598704338 +19 embedding2nn.weight 3 -0.05645715445280075 +19 embedding2nn.weight 4 -0.008285919204354286 +19 embedding2nn.weight 5 0.062144868075847626 +19 embedding2nn.weight 6 0.06285717338323593 +19 embedding2nn.weight 7 -0.03494725003838539 +19 embedding2nn.weight 8 -0.01531227771192789 +19 embedding2nn.weight 9 0.06824876368045807 +19 embedding2nn.bias 0 -0.07609594613313675 +19 embedding2nn.bias 1 -0.09345712512731552 +19 embedding2nn.bias 2 -0.04270146042108536 +19 embedding2nn.bias 3 0.0006376192905008793 +19 embedding2nn.bias 4 -0.06342048943042755 +19 embedding2nn.bias 5 -0.08827486634254456 +19 embedding2nn.bias 6 -0.041349146515131 +19 embedding2nn.bias 7 -0.06765807420015335 +19 embedding2nn.bias 8 0.07408146560192108 +19 embedding2nn.bias 9 0.00452579976990819 +19 rnn.weight_ih_l0 0 0.050594512373209 +19 rnn.weight_ih_l0 1 -0.02016350068151951 +19 rnn.weight_ih_l0 2 -0.0031676706857979298 +19 rnn.weight_ih_l0 3 -0.039644982665777206 +19 rnn.weight_ih_l0 4 -0.0094218160957098 +19 rnn.weight_ih_l0 5 -0.04953043535351753 +19 rnn.weight_ih_l0 6 0.013386703096330166 +19 rnn.weight_ih_l0 7 0.04342036321759224 +19 rnn.weight_ih_l0 8 0.008816220797598362 +19 rnn.weight_ih_l0 9 0.04181557521224022 +19 rnn.weight_hh_l0 0 0.037028733640909195 +19 rnn.weight_hh_l0 1 -0.04437616094946861 +19 rnn.weight_hh_l0 2 0.021430199965834618 +19 rnn.weight_hh_l0 3 0.028835635632276535 +19 rnn.weight_hh_l0 4 0.06247827783226967 +19 rnn.weight_hh_l0 5 0.04841994866728783 +19 rnn.weight_hh_l0 6 0.04349634796380997 +19 rnn.weight_hh_l0 7 -0.0567576065659523 +19 rnn.weight_hh_l0 8 0.06210705637931824 +19 rnn.weight_hh_l0 9 -0.04501841217279434 +19 rnn.bias_ih_l0 0 -0.03280783072113991 +19 rnn.bias_ih_l0 1 -0.02050754800438881 +19 rnn.bias_ih_l0 2 0.06162342056632042 +19 rnn.bias_ih_l0 3 -0.010499394498765469 +19 rnn.bias_ih_l0 4 0.06480387598276138 +19 rnn.bias_ih_l0 5 0.045747097581624985 +19 rnn.bias_ih_l0 6 -0.05547764524817467 +19 rnn.bias_ih_l0 7 -0.0007803377811796963 +19 rnn.bias_ih_l0 8 0.03863464295864105 +19 rnn.bias_ih_l0 9 0.0235279630869627 +19 rnn.bias_hh_l0 0 -0.055571138858795166 +19 rnn.bias_hh_l0 1 -0.038328006863594055 +19 rnn.bias_hh_l0 2 0.03940620645880699 +19 rnn.bias_hh_l0 3 0.03339524567127228 +19 rnn.bias_hh_l0 4 -0.04696952551603317 +19 rnn.bias_hh_l0 5 0.016145071014761925 +19 rnn.bias_hh_l0 6 0.04795444384217262 +19 rnn.bias_hh_l0 7 -0.03838137537240982 +19 rnn.bias_hh_l0 8 -0.021201729774475098 +19 rnn.bias_hh_l0 9 0.009537811391055584 +19 rnn.weight_ih_l0_reverse 0 -0.034214768558740616 +19 rnn.weight_ih_l0_reverse 1 -0.04215731471776962 +19 rnn.weight_ih_l0_reverse 2 0.03982832655310631 +19 rnn.weight_ih_l0_reverse 3 -0.008798502385616302 +19 rnn.weight_ih_l0_reverse 4 -0.051179978996515274 +19 rnn.weight_ih_l0_reverse 5 0.006559276953339577 +19 rnn.weight_ih_l0_reverse 6 0.011357705108821392 +19 rnn.weight_ih_l0_reverse 7 0.06158386170864105 +19 rnn.weight_ih_l0_reverse 8 0.020039834082126617 +19 rnn.weight_ih_l0_reverse 9 0.057177964597940445 +19 rnn.weight_hh_l0_reverse 0 0.06116056814789772 +19 rnn.weight_hh_l0_reverse 1 0.004429491236805916 +19 rnn.weight_hh_l0_reverse 2 0.030873550102114677 +19 rnn.weight_hh_l0_reverse 3 0.04479429870843887 +19 rnn.weight_hh_l0_reverse 4 -0.01676160842180252 +19 rnn.weight_hh_l0_reverse 5 0.0438743531703949 +19 rnn.weight_hh_l0_reverse 6 0.0483517199754715 +19 rnn.weight_hh_l0_reverse 7 0.023493926972150803 +19 rnn.weight_hh_l0_reverse 8 -0.03759157285094261 +19 rnn.weight_hh_l0_reverse 9 -0.005126998294144869 +19 rnn.bias_ih_l0_reverse 0 0.059200551360845566 +19 rnn.bias_ih_l0_reverse 1 0.0065492079593241215 +19 rnn.bias_ih_l0_reverse 2 0.052582550793886185 +19 rnn.bias_ih_l0_reverse 3 -0.04941976070404053 +19 rnn.bias_ih_l0_reverse 4 -0.027343714609742165 +19 rnn.bias_ih_l0_reverse 5 0.043595775961875916 +19 rnn.bias_ih_l0_reverse 6 -0.018276002258062363 +19 rnn.bias_ih_l0_reverse 7 0.05352816358208656 +19 rnn.bias_ih_l0_reverse 8 0.010170046240091324 +19 rnn.bias_ih_l0_reverse 9 0.06371207535266876 +19 rnn.bias_hh_l0_reverse 0 -0.04534933343529701 +19 rnn.bias_hh_l0_reverse 1 -0.039251405745744705 +19 rnn.bias_hh_l0_reverse 2 -0.028610270470380783 +19 rnn.bias_hh_l0_reverse 3 0.023225834593176842 +19 rnn.bias_hh_l0_reverse 4 0.04387157782912254 +19 rnn.bias_hh_l0_reverse 5 0.0542881116271019 +19 rnn.bias_hh_l0_reverse 6 0.048447154462337494 +19 rnn.bias_hh_l0_reverse 7 -0.04035099595785141 +19 rnn.bias_hh_l0_reverse 8 -0.05784037709236145 +19 rnn.bias_hh_l0_reverse 9 0.03951377049088478 +19 linear.weight 0 0.003820854239165783 +19 linear.weight 1 0.027973085641860962 +19 linear.weight 2 0.0516531839966774 +19 linear.weight 3 0.026774905622005463 +19 linear.weight 4 -0.004396989941596985 +19 linear.weight 5 0.021971002221107483 +19 linear.weight 6 0.03439749404788017 +19 linear.weight 7 -0.13418060541152954 +19 linear.weight 8 -0.0344756618142128 +19 linear.weight 9 -0.0890987291932106 +19 linear.bias 0 -0.30146682262420654 +19 linear.bias 1 -0.2875022888183594 +19 linear.bias 2 -0.15730184316635132 +19 linear.bias 3 -0.11112113296985626 +19 linear.bias 4 0.1143278107047081 +19 linear.bias 5 0.2329164445400238 +19 linear.bias 6 0.07406847178936005 +19 linear.bias 7 0.04407217726111412 +19 linear.bias 8 0.2095656543970108 +19 linear.bias 9 0.03721478208899498 diff --git a/tests/test_visual.py b/tests/test_visual.py index 4e51d4d2e4..6c86ad60da 100644 --- a/tests/test_visual.py +++ b/tests/test_visual.py @@ -1,200 +1,141 @@ +import os +import shutil + +import pytest + from flair.visual import * from flair.data import Sentence from flair.embeddings import CharLMEmbeddings, StackedEmbeddings -import unittest import numpy +from flair.visual.training_curves import Plotter -class Test(unittest.TestCase): - def test_prepare(self): - with open('resources/data/snippet.txt') as f: - sentences = [x for x in f.read().split('\n') if x] - - sentences = [Sentence(x) for x in sentences[:100]] - - charlm_embedding_forward = CharLMEmbeddings('news-forward') - charlm_embedding_backward = CharLMEmbeddings('news-backward') - - embeddings = StackedEmbeddings( - [charlm_embedding_backward, charlm_embedding_forward] - ) - - X = prepare_word_embeddings(embeddings, sentences) - contexts = word_contexts(sentences) - - numpy.save('resources/data/embeddings', X) - - with open('resources/data/contexts.txt', 'w') as f: - f.write('\n'.join(contexts)) - - def test_tSNE(self): - - X = numpy.load('resources/data/embeddings.npy') - trans_ = tSNE() - reduced = trans_.fit(X) - - numpy.save('resources/data/tsne', reduced) - - def test__prepare_char(self): - - with open('resources/data/snippet.txt') as f: - sentences = [x for x in f.read().split('\n') if x] - - sentences = [Sentence(x) for x in sentences[:100]] - - embeddings = CharLMEmbeddings('news-forward') - - X_forward = prepare_char_embeddings(embeddings, sentences) - - embeddings = CharLMEmbeddings('news-backward') - - X_backward = prepare_char_embeddings(embeddings, sentences) - - X = numpy.concatenate([X_forward, X_backward], axis=1) - - numpy.save('resources/data/char_embeddings', X) - - def test_tSNE_char(self): - - X = numpy.load('resources/data/char_embeddings.npy') - trans_ = tSNE() - reduced = trans_.fit(X) - - numpy.save('resources/data/char_tsne', reduced) - - def test_prepare_char_uni(self): - - with open('resources/data/snippet.txt') as f: - sentences = [x for x in f.read().split('\n') if x] - - sentences = [Sentence(x) for x in sentences[:100]] - - embeddings = CharLMEmbeddings('news-forward') - - X = prepare_char_embeddings(embeddings, sentences) - - numpy.save('resources/data/uni_embeddings', X) - - def test_tSNE_char_uni(self): - - X = numpy.load('resources/data/uni_embeddings.npy') - trans_ = tSNE() - reduced = trans_.fit(X) - - numpy.save('resources/data/uni_tsne', reduced) - def test_char_contexts(self): +@pytest.mark.skip(reason='Skipping test by default due to long execution time.') +def test_benchmark(): + import time - with open('resources/data/snippet.txt') as f: - sentences = [x for x in f.read().split('\n') if x] + with open('./resources/visual/snippet.txt') as f: + sentences = [x for x in f.read().split('\n') if x] - sentences = [Sentence(x) for x in sentences[:100]] + sentences = [Sentence(x) for x in sentences[:10]] - contexts = char_contexts(sentences) + charlm_embedding_forward = CharLMEmbeddings('news-forward') + charlm_embedding_backward = CharLMEmbeddings('news-backward') - with open('resources/data/char_contexts.txt', 'w') as f: - f.write('\n'.join(contexts)) + embeddings = StackedEmbeddings( + [charlm_embedding_backward, charlm_embedding_forward] + ) - def test_benchmark(self): + tic = time.time() - import time + prepare_word_embeddings(embeddings, sentences) - with open('resources/data/snippet.txt') as f: - sentences = [x for x in f.read().split('\n') if x] + current_elaped = time.time() - tic - sentences = [Sentence(x) for x in sentences[:10]] + print('current implementation: {} sec/ sentence'.format(current_elaped / 10)) + embeddings_f = CharLMEmbeddings('news-forward') + embeddings_b = CharLMEmbeddings('news-backward') - charlm_embedding_forward = CharLMEmbeddings('news-forward') - charlm_embedding_backward = CharLMEmbeddings('news-backward') + tic = time.time() - embeddings = StackedEmbeddings( - [charlm_embedding_backward, charlm_embedding_forward] - ) + prepare_char_embeddings(embeddings_f, sentences) + prepare_char_embeddings(embeddings_b, sentences) - tic = time.time() + current_elaped = time.time() - tic - prepare_word_embeddings(embeddings, sentences) + print('pytorch implementation: {} sec/ sentence'.format(current_elaped / 10)) - current_elaped = time.time() - tic - print('current implementation: {} sec/ sentence'.format(current_elaped / 10)) +@pytest.mark.skipif("TRAVIS" in os.environ and os.environ["TRAVIS"] == "true", reason="Skipping this test on Travis CI.") +def test_show_word_embeddings(): - embeddings_f = CharLMEmbeddings('news-forward') - embeddings_b = CharLMEmbeddings('news-backward') + with open('./resources/visual/snippet.txt') as f: + sentences = [x for x in f.read().split('\n') if x] - tic = time.time() + sentences = [Sentence(x) for x in sentences] - prepare_char_embeddings(embeddings_f, sentences) - prepare_char_embeddings(embeddings_b, sentences) + charlm_embedding_forward = CharLMEmbeddings('news-forward') + charlm_embedding_backward = CharLMEmbeddings('news-backward') - current_elaped = time.time() - tic + embeddings = StackedEmbeddings([charlm_embedding_backward, charlm_embedding_forward]) - print('pytorch implementation: {} sec/ sentence'.format(current_elaped / 10)) + X = prepare_word_embeddings(embeddings, sentences) + contexts = word_contexts(sentences) + trans_ = tSNE() + reduced = trans_.fit(X) -class Test_show(unittest.TestCase): - def test_word(self): + show(reduced, contexts) - reduced = numpy.load('resources/data/tsne.npy') - with open('resources/data/contexts.txt') as f: - contexts = f.read().split('\n') +@pytest.mark.skipif("TRAVIS" in os.environ and os.environ["TRAVIS"] == "true", reason="Skipping this test on Travis CI.") +def test_show_char_embeddings(): - show(reduced, contexts) + with open('./resources/visual/snippet.txt') as f: + sentences = [x for x in f.read().split('\n') if x] - def test_char(self): + sentences = [Sentence(x) for x in sentences] - reduced = numpy.load('resources/data/char_tsne.npy') + embeddings = CharLMEmbeddings('news-forward') - with open('resources/data/char_contexts.txt') as f: - contexts = f.read().split('\n') + X_forward = prepare_char_embeddings(embeddings, sentences) - show(reduced, contexts) + embeddings = CharLMEmbeddings('news-backward') - def test_uni_sentence(self): + X_backward = prepare_char_embeddings(embeddings, sentences) - reduced = numpy.load('resources/data/uni_tsne.npy') + X = numpy.concatenate([X_forward, X_backward], axis=1) - with open('resources/data/snippet.txt') as f: - sentences = [x for x in f.read().split('\n') if x] + contexts = char_contexts(sentences) - l = len(sentences[0]) + trans_ = tSNE() + reduced = trans_.fit(X) - with open('resources/data/char_contexts.txt') as f: - contexts = f.read().split('\n') + show(reduced, contexts) - show(reduced[:l], contexts[:l]) - def test_uni(self): +@pytest.mark.skipif("TRAVIS" in os.environ and os.environ["TRAVIS"] == "true", reason="Skipping this test on Travis CI.") +def test_show_uni_sentence_embeddings(): - reduced = numpy.load('resources/data/uni_tsne.npy') + with open('./resources/visual/snippet.txt') as f: + sentences = [x for x in f.read().split('\n') if x] - with open('resources/data/char_contexts.txt') as f: - contexts = f.read().split('\n') + sentences = [Sentence(x) for x in sentences] - show(reduced, contexts) + embeddings = CharLMEmbeddings('news-forward') + X = prepare_char_embeddings(embeddings, sentences) -class TestHighlighter(unittest.TestCase): - def test(self): + trans_ = tSNE() + reduced = trans_.fit(X) - i = numpy.random.choice(2048) + l = len(sentences[0]) - with open('resources/data/snippet.txt') as f: - sentences = [x for x in f.read().split('\n') if x] + contexts = char_contexts(sentences) - embeddings = CharLMEmbeddings('news-forward') + show(reduced[:l], contexts[:l]) - features = embeddings.lm.get_representation(sentences[0]).squeeze() - Highlighter().highlight_selection(features, sentences[0], n=1000) +def test_highlighter(): + with open('./resources/visual/snippet.txt') as f: + sentences = [x for x in f.read().split('\n') if x] + embeddings = CharLMEmbeddings('news-forward') + features = embeddings.lm.get_representation(sentences[0]).squeeze() -if __name__ == '__main__': - unittest.main() + Highlighter().highlight_selection(features, sentences[0], n=1000, file_='./resources/visual/data/highligh.html') + shutil.rmtree('./resources/visual/data') +def test_plotting_training_curves_and_weights(): + plotter = Plotter() + plotter.plot_training_curves('./resources/visual/loss.tsv') + plotter.plot_weights('./resources/visual/weights.txt') + # clean up directory + os.remove('./resources/visual/weights.png') + os.remove('./resources/visual/training.png') \ No newline at end of file From 24823108f4882388c0bc387de320b92c996e1736 Mon Sep 17 00:00:00 2001 From: tabergma Date: Thu, 4 Oct 2018 11:51:07 +0200 Subject: [PATCH 052/113] GH-61: Update .gitignore --- .gitignore | 5 +---- flair/visual/training_curves.py | 5 ++--- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index 4d3de671f2..7ca63573f7 100644 --- a/.gitignore +++ b/.gitignore @@ -103,7 +103,4 @@ venv.bak/ /site # mypy -.mypy_cache/ - -# data directory -resources/data +.mypy_cache/ \ No newline at end of file diff --git a/flair/visual/training_curves.py b/flair/visual/training_curves.py index 734f92b3f2..e1e4ae3468 100644 --- a/flair/visual/training_curves.py +++ b/flair/visual/training_curves.py @@ -6,7 +6,6 @@ import matplotlib import math -import sys matplotlib.use('Agg') @@ -32,8 +31,8 @@ class Plotter(object): """ - Plots training curves (loss, f-score, and accuracy) and training weights over time. - Input files are the output files loss.tsv and weights.txt from training either a sequence label tagger or text + Plots training parameters (loss, f-score, and accuracy) and training weights over time. + Input files are the output files 'loss.tsv' and 'weights.txt' from training either a sequence tagger or text classification model. """ From d86d4712b5f462dac69a2097d2596ab2913509b2 Mon Sep 17 00:00:00 2001 From: tabergma Date: Thu, 4 Oct 2018 12:08:10 +0200 Subject: [PATCH 053/113] GH-61: Save embedding visualization. --- flair/visual/__init__.py | 2 +- flair/visual/manifold.py | 6 ++---- tests/test_visual.py | 20 +++++++++++++++----- 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/flair/visual/__init__.py b/flair/visual/__init__.py index 4b35bff9f1..25affcab43 100644 --- a/flair/visual/__init__.py +++ b/flair/visual/__init__.py @@ -1,2 +1,2 @@ -from .manifold import tSNE, show, prepare_word_embeddings, prepare_char_embeddings, word_contexts, char_contexts +from .manifold import tSNE, visualize, prepare_word_embeddings, prepare_char_embeddings, word_contexts, char_contexts from .activations import Highlighter \ No newline at end of file diff --git a/flair/visual/manifold.py b/flair/visual/manifold.py index 9ea486c52d..6da7b23cec 100644 --- a/flair/visual/manifold.py +++ b/flair/visual/manifold.py @@ -38,7 +38,6 @@ def word_contexts(sentences): def prepare_char_embeddings(embeddings, sentences): - X = [] print('computing embeddings') @@ -55,7 +54,6 @@ def prepare_char_embeddings(embeddings, sentences): def char_contexts(sentences): - contexts = [] for sentence in sentences: @@ -89,7 +87,7 @@ def __init__(self): TSNE(n_components=2, verbose=1, perplexity=40, n_iter=300) -def show(X, contexts): +def visualize(X, contexts, file): import matplotlib.pyplot import mpld3 @@ -113,4 +111,4 @@ def show(X, contexts): mpld3.plugins.connect(fig, tooltip) - mpld3.show() + mpld3.save_html(fig, file) diff --git a/tests/test_visual.py b/tests/test_visual.py index 6c86ad60da..44f8ca0671 100644 --- a/tests/test_visual.py +++ b/tests/test_visual.py @@ -67,7 +67,10 @@ def test_show_word_embeddings(): trans_ = tSNE() reduced = trans_.fit(X) - show(reduced, contexts) + visualize(reduced, contexts, './resources/visual/sentence_embeddings.html') + + # clean up directory + os.remove('./resources/visual/sentence_embeddings.html') @pytest.mark.skipif("TRAVIS" in os.environ and os.environ["TRAVIS"] == "true", reason="Skipping this test on Travis CI.") @@ -93,7 +96,10 @@ def test_show_char_embeddings(): trans_ = tSNE() reduced = trans_.fit(X) - show(reduced, contexts) + visualize(reduced, contexts, './resources/visual/char_embeddings.html') + + # clean up directory + os.remove('./resources/visual/char_embeddings.html') @pytest.mark.skipif("TRAVIS" in os.environ and os.environ["TRAVIS"] == "true", reason="Skipping this test on Travis CI.") @@ -115,7 +121,10 @@ def test_show_uni_sentence_embeddings(): contexts = char_contexts(sentences) - show(reduced[:l], contexts[:l]) + visualize(reduced[:l], contexts[:l], './resources/visual/uni_sentence_embeddings.html') + + # clean up directory + os.remove('./resources/visual/uni_sentence_embeddings.html') def test_highlighter(): @@ -126,9 +135,10 @@ def test_highlighter(): features = embeddings.lm.get_representation(sentences[0]).squeeze() - Highlighter().highlight_selection(features, sentences[0], n=1000, file_='./resources/visual/data/highligh.html') + Highlighter().highlight_selection(features, sentences[0], n=1000, file_='./resources/visual/highligh.html') - shutil.rmtree('./resources/visual/data') + # clean up directory + os.remove('./resources/visual/highligh.html') def test_plotting_training_curves_and_weights(): From fa4285a046853045fbf0ed7404b534cb35a8a317 Mon Sep 17 00:00:00 2001 From: tabergma Date: Thu, 4 Oct 2018 14:00:11 +0200 Subject: [PATCH 054/113] GH-61: Wrap embedding visualiation into class. Fix requirement issues. --- flair/visual/__init__.py | 2 +- flair/visual/activations.py | 2 +- flair/visual/manifold.py | 154 ++++++++++++++++++++---------------- requirements.txt | 11 +-- tests/test_visual.py | 85 +++++--------------- 5 files changed, 114 insertions(+), 140 deletions(-) diff --git a/flair/visual/__init__.py b/flair/visual/__init__.py index 25affcab43..5027694df3 100644 --- a/flair/visual/__init__.py +++ b/flair/visual/__init__.py @@ -1,2 +1,2 @@ -from .manifold import tSNE, visualize, prepare_word_embeddings, prepare_char_embeddings, word_contexts, char_contexts +from .manifold import Visualizer from .activations import Highlighter \ No newline at end of file diff --git a/flair/visual/activations.py b/flair/visual/activations.py index 33740ac81a..39c876d0f2 100644 --- a/flair/visual/activations.py +++ b/flair/visual/activations.py @@ -1,7 +1,7 @@ import numpy -class Highlighter: +class Highlighter(object): def __init__(self): self.color_map = [ diff --git a/flair/visual/manifold.py b/flair/visual/manifold.py index 6da7b23cec..bd80d7dc3f 100644 --- a/flair/visual/manifold.py +++ b/flair/visual/manifold.py @@ -3,112 +3,132 @@ import numpy -def prepare_word_embeddings(embeddings, sentences): - X = [] +class _Transform: + def __init__(self): + pass - print('computing embeddings') - for sentence in tqdm.tqdm(sentences): - embeddings.embed(sentence) + def fit(self, X): + return self.transform.fit_transform(X) - for i, token in enumerate(sentence): - X.append(token.embedding.detach().numpy()[None, :]) - X = numpy.concatenate(X, 0) +class tSNE(_Transform): + def __init__(self): + super().__init__() - return X + self.transform = \ + TSNE(n_components=2, verbose=1, perplexity=40, n_iter=300) -def word_contexts(sentences): - contexts = [] +class Visualizer(object): - for sentence in sentences: + def visualize_word_emeddings(self, embeddings, sentences, output_file): + X = self.prepare_word_embeddings(embeddings, sentences) + contexts = self.word_contexts(sentences) - strs = [x.text for x in sentence.tokens] + trans_ = tSNE() + reduced = trans_.fit(X) - for i, token in enumerate(strs): - prop = ' {token} '.format( - token=token) + self.visualize(reduced, contexts, output_file) - prop = ' '.join(strs[max(i - 4, 0):i]) + prop - prop = prop + ' '.join(strs[i + 1:min(len(strs), i + 5)]) + def visualize_char_emeddings(self, embeddings, sentences, output_file): + X = self.prepare_char_embeddings(embeddings, sentences) + contexts = self.char_contexts(sentences) - contexts.append('

' + prop + '

') + trans_ = tSNE() + reduced = trans_.fit(X) - return contexts + self.visualize(reduced, contexts, output_file) + @staticmethod + def prepare_word_embeddings(embeddings, sentences): + X = [] -def prepare_char_embeddings(embeddings, sentences): - X = [] + print('computing embeddings') + for sentence in tqdm.tqdm(sentences): + embeddings.embed(sentence) - print('computing embeddings') - for sentence in tqdm.tqdm(sentences): + for i, token in enumerate(sentence): + X.append(token.embedding.detach().numpy()[None, :]) - sentence = ' '.join([x.text for x in sentence]) + X = numpy.concatenate(X, 0) - hidden = embeddings.lm.get_representation([sentence]) - X.append(hidden.squeeze().detach().numpy()) + return X - X = numpy.concatenate(X, 0) + @staticmethod + def word_contexts(sentences): + contexts = [] - return X + for sentence in sentences: + strs = [x.text for x in sentence.tokens] -def char_contexts(sentences): - contexts = [] + for i, token in enumerate(strs): + prop = ' {token} '.format( + token=token) - for sentence in sentences: - sentence = ' '.join([token.text for token in sentence]) + prop = ' '.join(strs[max(i - 4, 0):i]) + prop + prop = prop + ' '.join(strs[i + 1:min(len(strs), i + 5)]) - for i, char in enumerate(sentence): + contexts.append('

' + prop + '

') - context = '{}'.format(char) - context = ''.join(sentence[max(i - 30, 0):i]) + context - context = context + ''.join(sentence[i + 1:min(len(sentence), i + 30)]) + return contexts - contexts.append(context) + @staticmethod + def prepare_char_embeddings(embeddings, sentences): + X = [] - return contexts + print('computing embeddings') + for sentence in tqdm.tqdm(sentences): + sentence = ' '.join([x.text for x in sentence]) -class _Transform: - def __init__(self): - pass + hidden = embeddings.lm.get_representation([sentence]) + X.append(hidden.squeeze().detach().numpy()) - def fit(self, X): - return self.transform.fit_transform(X) + X = numpy.concatenate(X, 0) + return X -class tSNE(_Transform): - def __init__(self): + @staticmethod + def char_contexts(sentences): + contexts = [] - super().__init__() + for sentence in sentences: + sentence = ' '.join([token.text for token in sentence]) - self.transform = \ - TSNE(n_components=2, verbose=1, perplexity=40, n_iter=300) + for i, char in enumerate(sentence): + + context = '{}'.format(char) + context = ''.join(sentence[max(i - 30, 0):i]) + context + context = context + ''.join(sentence[i + 1:min(len(sentence), i + 30)]) + + contexts.append(context) + return contexts -def visualize(X, contexts, file): - import matplotlib.pyplot - import mpld3 + @staticmethod + def visualize(X, contexts, file): + import matplotlib.pyplot + import mpld3 - fig, ax = matplotlib.pyplot.subplots() + fig, ax = matplotlib.pyplot.subplots() - ax.grid(True, alpha=0.3) + ax.grid(True, alpha=0.3) - points = ax.plot(X[:, 0], X[:, 1], 'o', color='b', - mec='k', ms=5, mew=1, alpha=.6) + points = ax.plot(X[:, 0], X[:, 1], 'o', color='b', + mec='k', ms=5, mew=1, alpha=.6) - ax.set_xlabel('x') - ax.set_ylabel('y') - ax.set_title('Hover mouse to reveal context', size=20) + ax.set_xlabel('x') + ax.set_ylabel('y') + ax.set_title('Hover mouse to reveal context', size=20) - tooltip = mpld3.plugins.PointHTMLTooltip( - points[0], - contexts, - voffset=10, - hoffset=10 - ) + tooltip = mpld3.plugins.PointHTMLTooltip( + points[0], + contexts, + voffset=10, + hoffset=10 + ) - mpld3.plugins.connect(fig, tooltip) + mpld3.plugins.connect(fig, tooltip) - mpld3.save_html(fig, file) + mpld3.save_html(fig, file) diff --git a/requirements.txt b/requirements.txt index 350fc679cb..3adfd24aab 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,8 +3,9 @@ awscli==1.14.32 gensim==3.4.0 typing==3.6.4 pytest==3.6.4 -tqdm -segtok -matplotlib -mpld3 -sklearn \ No newline at end of file +tqdm==4.26.0 +segtok==1.5.7 +matplotlib==3.0.0 +mpld3==0.3 +jinja2==2.10 +sklearn==0.0 \ No newline at end of file diff --git a/tests/test_visual.py b/tests/test_visual.py index 44f8ca0671..46c3b1eb86 100644 --- a/tests/test_visual.py +++ b/tests/test_visual.py @@ -8,48 +8,32 @@ from flair.embeddings import CharLMEmbeddings, StackedEmbeddings import numpy +from flair.visual.manifold import Visualizer, tSNE from flair.visual.training_curves import Plotter -@pytest.mark.skip(reason='Skipping test by default due to long execution time.') -def test_benchmark(): - import time +@pytest.mark.skipif("TRAVIS" in os.environ and os.environ["TRAVIS"] == "true", reason="Skipping this test on Travis CI.") +def test_visualize_word_emeddings(): with open('./resources/visual/snippet.txt') as f: sentences = [x for x in f.read().split('\n') if x] - sentences = [Sentence(x) for x in sentences[:10]] + sentences = [Sentence(x) for x in sentences] charlm_embedding_forward = CharLMEmbeddings('news-forward') charlm_embedding_backward = CharLMEmbeddings('news-backward') - embeddings = StackedEmbeddings( - [charlm_embedding_backward, charlm_embedding_forward] - ) - - tic = time.time() - - prepare_word_embeddings(embeddings, sentences) - - current_elaped = time.time() - tic - - print('current implementation: {} sec/ sentence'.format(current_elaped / 10)) - - embeddings_f = CharLMEmbeddings('news-forward') - embeddings_b = CharLMEmbeddings('news-backward') - - tic = time.time() - - prepare_char_embeddings(embeddings_f, sentences) - prepare_char_embeddings(embeddings_b, sentences) + embeddings = StackedEmbeddings([charlm_embedding_backward, charlm_embedding_forward]) - current_elaped = time.time() - tic + visualizer = Visualizer() + visualizer.visualize_word_emeddings(embeddings, sentences, './resources/visual/sentence_embeddings.html') - print('pytorch implementation: {} sec/ sentence'.format(current_elaped / 10)) + # clean up directory + os.remove('./resources/visual/sentence_embeddings.html') @pytest.mark.skipif("TRAVIS" in os.environ and os.environ["TRAVIS"] == "true", reason="Skipping this test on Travis CI.") -def test_show_word_embeddings(): +def test_visualize_word_emeddings(): with open('./resources/visual/snippet.txt') as f: sentences = [x for x in f.read().split('\n') if x] @@ -57,24 +41,16 @@ def test_show_word_embeddings(): sentences = [Sentence(x) for x in sentences] charlm_embedding_forward = CharLMEmbeddings('news-forward') - charlm_embedding_backward = CharLMEmbeddings('news-backward') - - embeddings = StackedEmbeddings([charlm_embedding_backward, charlm_embedding_forward]) - X = prepare_word_embeddings(embeddings, sentences) - contexts = word_contexts(sentences) - - trans_ = tSNE() - reduced = trans_.fit(X) - - visualize(reduced, contexts, './resources/visual/sentence_embeddings.html') + visualizer = Visualizer() + visualizer.visualize_char_emeddings(charlm_embedding_forward, sentences, './resources/visual/sentence_embeddings.html') # clean up directory os.remove('./resources/visual/sentence_embeddings.html') @pytest.mark.skipif("TRAVIS" in os.environ and os.environ["TRAVIS"] == "true", reason="Skipping this test on Travis CI.") -def test_show_char_embeddings(): +def test_visualize(): with open('./resources/visual/snippet.txt') as f: sentences = [x for x in f.read().split('\n') if x] @@ -83,50 +59,27 @@ def test_show_char_embeddings(): embeddings = CharLMEmbeddings('news-forward') - X_forward = prepare_char_embeddings(embeddings, sentences) + visualizer = Visualizer() + + X_forward = visualizer.prepare_char_embeddings(embeddings, sentences) embeddings = CharLMEmbeddings('news-backward') - X_backward = prepare_char_embeddings(embeddings, sentences) + X_backward = visualizer.prepare_char_embeddings(embeddings, sentences) X = numpy.concatenate([X_forward, X_backward], axis=1) - contexts = char_contexts(sentences) + contexts = visualizer.char_contexts(sentences) trans_ = tSNE() reduced = trans_.fit(X) - visualize(reduced, contexts, './resources/visual/char_embeddings.html') + visualizer.visualize(reduced, contexts, './resources/visual/char_embeddings.html') # clean up directory os.remove('./resources/visual/char_embeddings.html') -@pytest.mark.skipif("TRAVIS" in os.environ and os.environ["TRAVIS"] == "true", reason="Skipping this test on Travis CI.") -def test_show_uni_sentence_embeddings(): - - with open('./resources/visual/snippet.txt') as f: - sentences = [x for x in f.read().split('\n') if x] - - sentences = [Sentence(x) for x in sentences] - - embeddings = CharLMEmbeddings('news-forward') - - X = prepare_char_embeddings(embeddings, sentences) - - trans_ = tSNE() - reduced = trans_.fit(X) - - l = len(sentences[0]) - - contexts = char_contexts(sentences) - - visualize(reduced[:l], contexts[:l], './resources/visual/uni_sentence_embeddings.html') - - # clean up directory - os.remove('./resources/visual/uni_sentence_embeddings.html') - - def test_highlighter(): with open('./resources/visual/snippet.txt') as f: sentences = [x for x in f.read().split('\n') if x] From 5ca09ff99be13ee9b5f66f68bc8e138dfcf2bca3 Mon Sep 17 00:00:00 2001 From: tabergma Date: Thu, 4 Oct 2018 14:06:16 +0200 Subject: [PATCH 055/113] GH-61: Add missing dependency. --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3adfd24aab..e96a4ac386 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,4 +8,5 @@ segtok==1.5.7 matplotlib==3.0.0 mpld3==0.3 jinja2==2.10 -sklearn==0.0 \ No newline at end of file +sklearn==0.0 +google-compute-engine==2.8.3 From 05a68fdb43a5750d5569abdbf72fe5af5279e41c Mon Sep 17 00:00:00 2001 From: tabergma Date: Thu, 4 Oct 2018 14:12:17 +0200 Subject: [PATCH 056/113] GH-61: Revert last change. --- flair/embeddings.py | 2 ++ flair/models/language_model.py | 2 ++ requirements.txt | 1 - 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/flair/embeddings.py b/flair/embeddings.py index 0fc16c0356..02c2aa18cd 100644 --- a/flair/embeddings.py +++ b/flair/embeddings.py @@ -669,6 +669,8 @@ def embed(self, sentences: Union[List[Sentence], Sentence]): packed = torch.nn.utils.rnn.pack_padded_sequence(sentence_tensor, lengths) + self.rnn.flatten_parameters() + lstm_out, hidden = self.rnn(packed) outputs, output_lengths = torch.nn.utils.rnn.pad_packed_sequence(lstm_out) diff --git a/flair/models/language_model.py b/flair/models/language_model.py index 1516145818..e5851f3e46 100644 --- a/flair/models/language_model.py +++ b/flair/models/language_model.py @@ -66,6 +66,8 @@ def forward(self, input, hidden, ordered_sequence_lengths=None): encoded = self.encoder(input) emb = self.drop(encoded) + self.rnn.flatten_parameters() + output, hidden = self.rnn(emb, hidden) if self.proj is not None: diff --git a/requirements.txt b/requirements.txt index e96a4ac386..5aa24415ef 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,4 +9,3 @@ matplotlib==3.0.0 mpld3==0.3 jinja2==2.10 sklearn==0.0 -google-compute-engine==2.8.3 From 36fc3cf537754b2e2f72240554f9f7f7ba5e8da4 Mon Sep 17 00:00:00 2001 From: tabergma Date: Thu, 4 Oct 2018 14:30:12 +0200 Subject: [PATCH 057/113] GH-61: Set sudo to false in travis.yml --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index fd66eb67dd..72cfdc70fe 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,7 @@ env: - TRAVIS=true language: python +sudo: false python: - "3.6" install: @@ -9,5 +10,5 @@ before_script: cd tests script: - pytest branches: - only: + only: - master From 5aebd44511f41282100acfa47c93e78673469a00 Mon Sep 17 00:00:00 2001 From: tabergma Date: Thu, 4 Oct 2018 14:36:32 +0200 Subject: [PATCH 058/113] GH-61: Update travis.yml --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 72cfdc70fe..410eb7c04f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,11 +1,10 @@ env: - TRAVIS=true language: python -sudo: false python: - "3.6" install: - - pip install -r requirements.txt -q + - pip install -r requirements.txt before_script: cd tests script: - pytest From 8003e9e8e4d5601c4d19fe8e849c0979ee28b0b4 Mon Sep 17 00:00:00 2001 From: tabergma Date: Thu, 4 Oct 2018 14:45:42 +0200 Subject: [PATCH 059/113] GH-61: build everything --- .travis.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 410eb7c04f..c17a8fe0bc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,10 +4,7 @@ language: python python: - "3.6" install: - - pip install -r requirements.txt + - pip install -r requirements.txt -q before_script: cd tests script: - - pytest -branches: - only: - - master + - pytest \ No newline at end of file From bdd34f92dedb0c4f3e8796d3e0798fee6dad0727 Mon Sep 17 00:00:00 2001 From: tabergma Date: Thu, 4 Oct 2018 14:49:48 +0200 Subject: [PATCH 060/113] GH-61: Add sudo:false in travis.yml --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index c17a8fe0bc..4063862c02 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,7 @@ env: - TRAVIS=true language: python +sudo: false python: - "3.6" install: From 677f7273f3d408c07b49f06339e92ddd4793eff3 Mon Sep 17 00:00:00 2001 From: tabergma Date: Thu, 4 Oct 2018 14:53:31 +0200 Subject: [PATCH 061/113] GH-61: Remove flaten parameters --- flair/embeddings.py | 2 -- flair/models/language_model.py | 2 -- 2 files changed, 4 deletions(-) diff --git a/flair/embeddings.py b/flair/embeddings.py index 02c2aa18cd..0fc16c0356 100644 --- a/flair/embeddings.py +++ b/flair/embeddings.py @@ -669,8 +669,6 @@ def embed(self, sentences: Union[List[Sentence], Sentence]): packed = torch.nn.utils.rnn.pack_padded_sequence(sentence_tensor, lengths) - self.rnn.flatten_parameters() - lstm_out, hidden = self.rnn(packed) outputs, output_lengths = torch.nn.utils.rnn.pad_packed_sequence(lstm_out) diff --git a/flair/models/language_model.py b/flair/models/language_model.py index e5851f3e46..1516145818 100644 --- a/flair/models/language_model.py +++ b/flair/models/language_model.py @@ -66,8 +66,6 @@ def forward(self, input, hidden, ordered_sequence_lengths=None): encoded = self.encoder(input) emb = self.drop(encoded) - self.rnn.flatten_parameters() - output, hidden = self.rnn(emb, hidden) if self.proj is not None: From 509f4e95fe1fd3c0aa646dcb79d5d099789e87e0 Mon Sep 17 00:00:00 2001 From: tabergma Date: Thu, 4 Oct 2018 17:07:51 +0200 Subject: [PATCH 062/113] GH-108: Add flatten parameters --- flair/embeddings.py | 2 ++ flair/models/language_model.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/flair/embeddings.py b/flair/embeddings.py index 0fc16c0356..02c2aa18cd 100644 --- a/flair/embeddings.py +++ b/flair/embeddings.py @@ -669,6 +669,8 @@ def embed(self, sentences: Union[List[Sentence], Sentence]): packed = torch.nn.utils.rnn.pack_padded_sequence(sentence_tensor, lengths) + self.rnn.flatten_parameters() + lstm_out, hidden = self.rnn(packed) outputs, output_lengths = torch.nn.utils.rnn.pad_packed_sequence(lstm_out) diff --git a/flair/models/language_model.py b/flair/models/language_model.py index 1516145818..e5851f3e46 100644 --- a/flair/models/language_model.py +++ b/flair/models/language_model.py @@ -66,6 +66,8 @@ def forward(self, input, hidden, ordered_sequence_lengths=None): encoded = self.encoder(input) emb = self.drop(encoded) + self.rnn.flatten_parameters() + output, hidden = self.rnn(emb, hidden) if self.proj is not None: From 7684ca25cf0858e87047b5ece6fc18dc7c32e313 Mon Sep 17 00:00:00 2001 From: tabergma Date: Fri, 5 Oct 2018 09:25:45 +0200 Subject: [PATCH 063/113] GH-108: refactor eval method in text classifier --- flair/trainers/text_classification_trainer.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/flair/trainers/text_classification_trainer.py b/flair/trainers/text_classification_trainer.py index 3e739dae68..9657f640be 100644 --- a/flair/trainers/text_classification_trainer.py +++ b/flair/trainers/text_classification_trainer.py @@ -11,10 +11,13 @@ from flair.training_utils import convert_labels_to_one_hot, calculate_micro_avg_metric, init_output_file, \ clear_embeddings, calculate_class_metrics, WeightExtractor, Metric + MICRO_AVG_METRIC = 'MICRO_AVG' + log = logging.getLogger(__name__) + class TextClassifierTrainer: """ Training class to train and evaluate a text classification model. @@ -216,16 +219,13 @@ def evaluate(self, sentences: List[Sentence], eval_class_metrics: bool = False, labels = self.model.obtain_labels(scores) loss = self.model.calculate_loss(scores, batch) - eval_loss += loss - - y_true.extend([sentence.get_label_names() for sentence in batch]) - y_pred.extend([[label.value for label in sent_labels] for sent_labels in labels]) - if not embeddings_in_memory: clear_embeddings(batch) - y_true = convert_labels_to_one_hot(y_true, self.label_dict) - y_pred = convert_labels_to_one_hot(y_pred, self.label_dict) + eval_loss += loss + + y_pred.extend(convert_labels_to_one_hot([[label.value for label in sent_labels] for sent_labels in labels], self.label_dict)) + y_true.extend(convert_labels_to_one_hot([sentence.get_label_names() for sentence in batch], self.label_dict)) metrics = [calculate_micro_avg_metric(y_true, y_pred, self.label_dict)] if eval_class_metrics: From c6211f293b5e36076785f4fac95a69c8bc36b6d0 Mon Sep 17 00:00:00 2001 From: tabergma Date: Fri, 5 Oct 2018 10:36:11 +0200 Subject: [PATCH 064/113] GH-108: Use torch.no_grad in evaluation method --- flair/trainers/text_classification_trainer.py | 41 ++++++++++--------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/flair/trainers/text_classification_trainer.py b/flair/trainers/text_classification_trainer.py index 9657f640be..db1708ba03 100644 --- a/flair/trainers/text_classification_trainer.py +++ b/flair/trainers/text_classification_trainer.py @@ -206,33 +206,34 @@ def evaluate(self, sentences: List[Sentence], eval_class_metrics: bool = False, :param mini_batch_size: the mini batch size to use :return: list of metrics, and the loss """ - eval_loss = 0 + with torch.no_grad(): + eval_loss = 0 - batches = [sentences[x:x + mini_batch_size] for x in - range(0, len(sentences), mini_batch_size)] + batches = [sentences[x:x + mini_batch_size] for x in + range(0, len(sentences), mini_batch_size)] - y_pred = [] - y_true = [] + y_pred = [] + y_true = [] - for batch in batches: - scores = self.model.forward(batch) - labels = self.model.obtain_labels(scores) - loss = self.model.calculate_loss(scores, batch) + for batch in batches: + scores = self.model.forward(batch) + labels = self.model.obtain_labels(scores) + loss = self.model.calculate_loss(scores, batch) - if not embeddings_in_memory: - clear_embeddings(batch) + if not embeddings_in_memory: + clear_embeddings(batch) - eval_loss += loss + eval_loss += loss - y_pred.extend(convert_labels_to_one_hot([[label.value for label in sent_labels] for sent_labels in labels], self.label_dict)) - y_true.extend(convert_labels_to_one_hot([sentence.get_label_names() for sentence in batch], self.label_dict)) + y_pred.extend(convert_labels_to_one_hot([[label.value for label in sent_labels] for sent_labels in labels], self.label_dict)) + y_true.extend(convert_labels_to_one_hot([sentence.get_label_names() for sentence in batch], self.label_dict)) - metrics = [calculate_micro_avg_metric(y_true, y_pred, self.label_dict)] - if eval_class_metrics: - metrics.extend(calculate_class_metrics(y_true, y_pred, self.label_dict)) + metrics = [calculate_micro_avg_metric(y_true, y_pred, self.label_dict)] + if eval_class_metrics: + metrics.extend(calculate_class_metrics(y_true, y_pred, self.label_dict)) - eval_loss /= len(sentences) + eval_loss /= len(sentences) - metrics_dict = {metric.name: metric for metric in metrics} + metrics_dict = {metric.name: metric for metric in metrics} - return metrics_dict, eval_loss + return metrics_dict, eval_loss From 7cd174623d2a8180e414528b3df3641fef642ad0 Mon Sep 17 00:00:00 2001 From: aakbik Date: Wed, 10 Oct 2018 21:57:36 +0200 Subject: [PATCH 065/113] GH-137: switch to train only at beginning of epoch --- flair/models/text_classification_model.py | 2 ++ flair/trainers/text_classification_trainer.py | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/flair/models/text_classification_model.py b/flair/models/text_classification_model.py index 51323542f9..e85e19cc3f 100644 --- a/flair/models/text_classification_model.py +++ b/flair/models/text_classification_model.py @@ -64,6 +64,8 @@ def save(self, model_file: str): Saves the current model to the provided file. :param model_file: the model file """ + self.eval() + model_state = { 'state_dict': self.state_dict(), 'document_embeddings': self.document_embeddings, diff --git a/flair/trainers/text_classification_trainer.py b/flair/trainers/text_classification_trainer.py index db1708ba03..979badc59c 100644 --- a/flair/trainers/text_classification_trainer.py +++ b/flair/trainers/text_classification_trainer.py @@ -142,8 +142,6 @@ def train(self, f.write('{}\t{:%H:%M:%S}\t{}\t{}\t{}\t{}\t{}\t{}\n'.format( epoch, datetime.datetime.now(), train_loss, train_metric_str, dev_loss, dev_metric_str, '_', Metric.to_empty_tsv())) - self.model.train() - # anneal against train loss if training with dev, otherwise anneal against dev score scheduler.step(current_loss) if train_with_dev else scheduler.step(dev_metric.f_score()) From 45fd3353981d903f9006dc4156ae99f0aba7af9a Mon Sep 17 00:00:00 2001 From: aakbik Date: Wed, 10 Oct 2018 21:58:45 +0200 Subject: [PATCH 066/113] GH-137: add option to anneal with restarts --- flair/trainers/sequence_tagger_trainer.py | 47 +++++++++++++++-------- 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/flair/trainers/sequence_tagger_trainer.py b/flair/trainers/sequence_tagger_trainer.py index 96e38ffd74..7f8c0886fe 100644 --- a/flair/trainers/sequence_tagger_trainer.py +++ b/flair/trainers/sequence_tagger_trainer.py @@ -31,6 +31,7 @@ def train(self, embeddings_in_memory: bool = True, checkpoint: bool = False, save_final_model: bool = True, + anneal_with_restarts: bool = False, ): evaluation_method = 'F1' @@ -47,8 +48,7 @@ def train(self, optimizer = torch.optim.SGD(self.model.parameters(), lr=learning_rate) anneal_mode = 'min' if train_with_dev else 'max' - scheduler: ReduceLROnPlateau = ReduceLROnPlateau(optimizer, factor=anneal_factor, patience=patience, - mode=anneal_mode) + scheduler = ReduceLROnPlateau(optimizer, factor=anneal_factor, patience=patience, mode=anneal_mode, verbose=True) train_data = self.corpus.train @@ -59,8 +59,25 @@ def train(self, # At any point you can hit Ctrl + C to break out of training early. try: - for epoch in range(max_epochs): - log.info('-' * 100) + previous_learning_rate = learning_rate + for epoch in range(0, max_epochs): + + bad_epochs = scheduler.num_bad_epochs + for group in optimizer.param_groups: + learning_rate = group['lr'] + + if learning_rate != previous_learning_rate and anneal_with_restarts: + print('resetting to best model') + self.model.load_from_file(base_path + "/best-model.pt") + # restart optimizer and scheduler + optimizer = torch.optim.SGD(self.model.parameters(), lr=learning_rate) + scheduler = ReduceLROnPlateau(optimizer, factor=anneal_factor, patience=patience, mode=anneal_mode) + + previous_learning_rate = learning_rate + + if learning_rate < 0.001: + print('learning rate too small - quitting training!') + break if not self.test_mode: random.shuffle(train_data) @@ -72,9 +89,6 @@ def train(self, seen_sentences = 0 modulo = max(1, int(len(batches) / 10)) - for group in optimizer.param_groups: - learning_rate = group['lr'] - for batch_no, batch in enumerate(batches): batch: List[Sentence] = batch @@ -113,27 +127,30 @@ def train(self, dev_score = dev_metric = None if not train_with_dev: dev_score, dev_metric = self.evaluate(self.corpus.dev, base_path, - evaluation_method=evaluation_method, - embeddings_in_memory=embeddings_in_memory) + evaluation_method=evaluation_method, + embeddings_in_memory=embeddings_in_memory) test_score, test_metric = self.evaluate(self.corpus.test, base_path, - evaluation_method=evaluation_method, - embeddings_in_memory=embeddings_in_memory) + evaluation_method=evaluation_method, + embeddings_in_memory=embeddings_in_memory) # anneal against train loss if training with dev, otherwise anneal against dev score scheduler.step(current_loss) if train_with_dev else scheduler.step(dev_score) - log.info("EPOCH {0}: lr {1:.4f} - bad epochs {2}".format(epoch + 1, learning_rate, scheduler.num_bad_epochs)) + log.info("EPOCH {0}: lr {1:.4f} - bad epochs {2}".format(epoch + 1, learning_rate, bad_epochs)) if not train_with_dev: log.info("{0:<4}: f-score {1:.4f} - acc {2:.4f} - tp {3} - fp {4} - fn {5} - tn {6}".format( - 'DEV', dev_metric.f_score(), dev_metric.accuracy(), dev_metric._tp, dev_metric._fp, dev_metric._fn, dev_metric._tn)) + 'DEV', dev_metric.f_score(), dev_metric.accuracy(), dev_metric._tp, dev_metric._fp, + dev_metric._fn, dev_metric._tn)) log.info("{0:<4}: f-score {1:.4f} - acc {2:.4f} - tp {3} - fp {4} - fn {5} - tn {6}".format( - 'TEST', test_metric.f_score(), test_metric.accuracy(), test_metric._tp, test_metric._fp, test_metric._fn, test_metric._tn)) + 'TEST', test_metric.f_score(), test_metric.accuracy(), test_metric._tp, test_metric._fp, + test_metric._fn, test_metric._tn)) with open(loss_txt, 'a') as f: dev_metric_str = dev_metric.to_tsv() if dev_metric is not None else Metric.to_empty_tsv() f.write('{}\t{:%H:%M:%S}\t{}\t{}\t{}\t{}\t{}\t{}\n'.format( - epoch, datetime.datetime.now(), '_', Metric.to_empty_tsv(), '_', dev_metric_str, '_', test_metric.to_tsv())) + epoch, datetime.datetime.now(), '_', Metric.to_empty_tsv(), '_', dev_metric_str, '_', + test_metric.to_tsv())) # if we use dev data, remember best model based on dev evaluation score if not train_with_dev and dev_score == scheduler.best: From 7776ef1cc2a9058dc01d4c7d17fbb525cb0b5b1b Mon Sep 17 00:00:00 2001 From: aakbik Date: Wed, 10 Oct 2018 22:01:48 +0200 Subject: [PATCH 067/113] GH-137: add caching option to CharLMEmbeddings --- flair/data.py | 10 ++-- flair/embeddings.py | 59 ++++++++++++++++++++--- flair/models/text_classification_model.py | 2 - requirements.txt | 1 + 4 files changed, 58 insertions(+), 14 deletions(-) diff --git a/flair/data.py b/flair/data.py index 704b5c17d1..262357dc29 100644 --- a/flair/data.py +++ b/flair/data.py @@ -169,10 +169,10 @@ def get_head(self): return self.sentence.get_token(self.head_id) def __str__(self) -> str: - return 'Token: %d %s' % (self.idx, self.text) if self.idx is not None else 'Token: %s' % (self.text) + return 'Token: {} {}'.format(self.idx, self.text) if self.idx is not None else 'Token: {}'.format(self.text) def __repr__(self) -> str: - return 'Token: %d %s' % (self.idx, self.text) if self.idx is not None else 'Token: %s' % (self.text) + return 'Token: {} {}'.format(self.idx, self.text) if self.idx is not None else 'Token: {}'.format(self.text) def set_embedding(self, name: str, vector: torch.autograd.Variable): self._embeddings[name] = vector.cpu() @@ -480,20 +480,20 @@ def __iter__(self): return iter(self.tokens) def __repr__(self): - return 'Sentence: "' + ' '.join([t.text for t in self.tokens]) + '" - %d Tokens' % len(self) + return 'Sentence: "{}" - {} Tokens'.format(' '.join([t.text for t in self.tokens]), len(self)) def __copy__(self): s = Sentence() for token in self.tokens: nt = Token(token.text) for tag_type in token.tags: - nt.add_tag(tag_type, token.get_tag(tag_type)) + nt.add_tag(tag_type, token.get_tag(tag_type).value, token.get_tag(tag_type).score) s.add_token(nt) return s def __str__(self) -> str: - return 'Sentence: "' + ' '.join([t.text for t in self.tokens]) + '" - %d Tokens' % len(self) + return 'Sentence: "{}" - {} Tokens'.format(' '.join([t.text for t in self.tokens]), len(self)) def __len__(self) -> int: return len(self.tokens) diff --git a/flair/embeddings.py b/flair/embeddings.py index 02c2aa18cd..71d25a4dbb 100644 --- a/flair/embeddings.py +++ b/flair/embeddings.py @@ -1,7 +1,7 @@ import os import re from abc import abstractmethod -from typing import List, Union, Dict +from typing import List, Union, Dict, Tuple import gensim import numpy as np @@ -94,11 +94,11 @@ def __init__(self, embeddings: List[TokenEmbeddings], detach: bool = True): # IMPORTANT: add embeddings as torch modules for i, embedding in enumerate(embeddings): - self.add_module('list_embedding_%s' % str(i), embedding) + self.add_module('list_embedding_{}'.format(i), embedding) - self.detach = detach - self.name = 'Stack' - self.static_embeddings = True + self.detach: bool = detach + self.name: str = 'Stack' + self.static_embeddings: bool = True self.__embedding_type: str = embeddings[0].embedding_type @@ -111,6 +111,7 @@ def embed(self, sentences: Union[Sentence, List[Sentence]], static_embeddings: b if type(sentences) is Sentence: sentences = [sentences] + # default case: do not use cache for embedding in self.embeddings: embedding.embed(sentences) @@ -355,7 +356,7 @@ def _add_embeddings_internal(self, sentences: List[Sentence]): class CharLMEmbeddings(TokenEmbeddings): """Contextual string embeddings of words, as proposed in Akbik et al., 2018.""" - def __init__(self, model, detach: bool = True): + def __init__(self, model, detach: bool = True, use_cache: bool = True): """ Contextual string embeddings of words, as proposed in Akbik et al., 2018. @@ -419,17 +420,56 @@ def __init__(self, model, detach: bool = True): self.is_forward_lm: bool = self.lm.is_forward_lm + # caching variables + self.use_cache: bool = use_cache + self.cache_added: int = 0 + self.cache = None + dummy_sentence: Sentence = Sentence() dummy_sentence.add_token(Token('hello')) embedded_dummy = self.embed(dummy_sentence) self.__embedding_length: int = len(embedded_dummy[0].get_token(1).get_embedding()) + def __getstate__(self): + # Copy the object's state from self.__dict__ which contains + # all our instance attributes. Always use the dict.copy() + # method to avoid modifying the original state. + state = self.__dict__.copy() + # Remove the unpicklable entries. + state['cache'] = None + return state + @property def embedding_length(self) -> int: return self.__embedding_length def _add_embeddings_internal(self, sentences: List[Sentence]) -> List[Sentence]: - # get text sentences + + # if cache is used, try setting embeddings from cache first + if self.use_cache: + + # lazy initialization of cache + if not self.cache: + from sqlitedict import SqliteDict + self.cache = SqliteDict('{}-tmp-cache.sqllite'.format(self.name), autocommit=True) + + # try populating embeddings from cache + all_embeddings_retrieved_from_cache: bool = True + for sentence in sentences: + key = sentence.to_tokenized_string() + embeddings = self.cache.get(key) + + if not embeddings: + all_embeddings_retrieved_from_cache = False + break + else: + for token, embedding in zip(sentence, embeddings): + token.set_embedding(self.name, torch.FloatTensor(embedding)) + + if all_embeddings_retrieved_from_cache: + return sentences + + # if this is not possible, use LM to generate embedding. First, get text sentences text_sentences = [sentence.to_tokenized_string() for sentence in sentences] longest_character_sequence_in_batch: int = len(max(text_sentences, key=len)) @@ -479,6 +519,11 @@ def _add_embeddings_internal(self, sentences: List[Sentence]) -> List[Sentence]: token.set_embedding(self.name, embedding) + if self.use_cache: + for sentence in sentences: + self.cache[sentence.to_tokenized_string()] = [token._embeddings[self.name].tolist() for token in sentence] + self.cache_added += 1 + return sentences diff --git a/flair/models/text_classification_model.py b/flair/models/text_classification_model.py index e85e19cc3f..51323542f9 100644 --- a/flair/models/text_classification_model.py +++ b/flair/models/text_classification_model.py @@ -64,8 +64,6 @@ def save(self, model_file: str): Saves the current model to the provided file. :param model_file: the model file """ - self.eval() - model_state = { 'state_dict': self.state_dict(), 'document_embeddings': self.document_embeddings, diff --git a/requirements.txt b/requirements.txt index 5aa24415ef..58bf5a5eae 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,3 +9,4 @@ matplotlib==3.0.0 mpld3==0.3 jinja2==2.10 sklearn==0.0 +sqlitedict \ No newline at end of file From 4c6ab4f2afac5d3e76d55de564f725b01a271fc9 Mon Sep 17 00:00:00 2001 From: aakbik Date: Thu, 11 Oct 2018 10:45:17 +0200 Subject: [PATCH 068/113] GH-137: add caching option to CharLMEmbeddings --- flair/embeddings.py | 4 +++- flair/trainers/sequence_tagger_trainer.py | 12 +++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/flair/embeddings.py b/flair/embeddings.py index 71d25a4dbb..05d3efacbe 100644 --- a/flair/embeddings.py +++ b/flair/embeddings.py @@ -111,7 +111,6 @@ def embed(self, sentences: Union[Sentence, List[Sentence]], static_embeddings: b if type(sentences) is Sentence: sentences = [sentences] - # default case: do not use cache for embedding in self.embeddings: embedding.embed(sentences) @@ -368,6 +367,9 @@ def __init__(self, model, detach: bool = True, use_cache: bool = True): arg2 : detach if set to false, the gradient will propagate into the language model. this dramatically slows down training and often leads to worse results, so not recommended. + arg3 : use_cache + if set to false, will not write embeddings to file for later retrieval. this saves disk space but will + not allow re-use of once computed embeddings that do not fit into memory """ super().__init__() diff --git a/flair/trainers/sequence_tagger_trainer.py b/flair/trainers/sequence_tagger_trainer.py index 7f8c0886fe..6b78c0dec8 100644 --- a/flair/trainers/sequence_tagger_trainer.py +++ b/flair/trainers/sequence_tagger_trainer.py @@ -47,6 +47,7 @@ def train(self, optimizer = torch.optim.SGD(self.model.parameters(), lr=learning_rate) + # annealing scheduler anneal_mode = 'min' if train_with_dev else 'max' scheduler = ReduceLROnPlateau(optimizer, factor=anneal_factor, patience=patience, mode=anneal_mode, verbose=True) @@ -60,23 +61,23 @@ def train(self, try: previous_learning_rate = learning_rate + for epoch in range(0, max_epochs): bad_epochs = scheduler.num_bad_epochs for group in optimizer.param_groups: learning_rate = group['lr'] + # reload last best model if annealing with restarts is enabled if learning_rate != previous_learning_rate and anneal_with_restarts: - print('resetting to best model') + log.info('resetting to best model') self.model.load_from_file(base_path + "/best-model.pt") - # restart optimizer and scheduler - optimizer = torch.optim.SGD(self.model.parameters(), lr=learning_rate) - scheduler = ReduceLROnPlateau(optimizer, factor=anneal_factor, patience=patience, mode=anneal_mode) previous_learning_rate = learning_rate + # stop training if learning rate becomes too small if learning_rate < 0.001: - print('learning rate too small - quitting training!') + log.info('learning rate too small - quitting training!') break if not self.test_mode: random.shuffle(train_data) @@ -137,6 +138,7 @@ def train(self, # anneal against train loss if training with dev, otherwise anneal against dev score scheduler.step(current_loss) if train_with_dev else scheduler.step(dev_score) + # logging info log.info("EPOCH {0}: lr {1:.4f} - bad epochs {2}".format(epoch + 1, learning_rate, bad_epochs)) if not train_with_dev: log.info("{0:<4}: f-score {1:.4f} - acc {2:.4f} - tp {3} - fp {4} - fn {5} - tn {6}".format( From 7d2543aa3907e4e47325c64bf98f4312065fe698 Mon Sep 17 00:00:00 2001 From: aakbik Date: Thu, 11 Oct 2018 10:48:51 +0200 Subject: [PATCH 069/113] GH-137: add caching option to CharLMEmbeddings --- flair/embeddings.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/flair/embeddings.py b/flair/embeddings.py index 05d3efacbe..ea87fbdcb9 100644 --- a/flair/embeddings.py +++ b/flair/embeddings.py @@ -424,7 +424,6 @@ def __init__(self, model, detach: bool = True, use_cache: bool = True): # caching variables self.use_cache: bool = use_cache - self.cache_added: int = 0 self.cache = None dummy_sentence: Sentence = Sentence() @@ -524,7 +523,6 @@ def _add_embeddings_internal(self, sentences: List[Sentence]) -> List[Sentence]: if self.use_cache: for sentence in sentences: self.cache[sentence.to_tokenized_string()] = [token._embeddings[self.name].tolist() for token in sentence] - self.cache_added += 1 return sentences From 821ba52f1d023f0b57198fcfe5265be435d0b4b1 Mon Sep 17 00:00:00 2001 From: aakbik Date: Thu, 11 Oct 2018 10:57:28 +0200 Subject: [PATCH 070/113] GH-137: ignore deletion errors on test class to handle sqllite journalling --- tests/test_language_model_trainer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_language_model_trainer.py b/tests/test_language_model_trainer.py index fad940179b..b052e9a338 100644 --- a/tests/test_language_model_trainer.py +++ b/tests/test_language_model_trainer.py @@ -30,6 +30,6 @@ def test_training(): print(sentence[1].embedding.size()) # clean up results directory - shutil.rmtree('./results') + shutil.rmtree('./results', ignore_errors=True) From 8d362877197d01a66d35a823d30146bf47fdf146 Mon Sep 17 00:00:00 2001 From: tabergma Date: Thu, 11 Oct 2018 11:05:11 +0200 Subject: [PATCH 071/113] GH-54: Add method to convert sentence to dict. --- flair/data.py | 92 ++++++++++++++++++++++++++++++++++++++++++---- tests/test_data.py | 55 ++++++++++++++++++++++++++- 2 files changed, 138 insertions(+), 9 deletions(-) diff --git a/flair/data.py b/flair/data.py index 704b5c17d1..129e138715 100644 --- a/flair/data.py +++ b/flair/data.py @@ -129,6 +129,12 @@ def score(self, score): else: self._score = 1.0 + def to_dict(self): + return { + 'value': self.value, + 'confidence': self.score + } + def __str__(self): return "{} ({})".format(self._value, self._score) @@ -147,12 +153,16 @@ def __init__(self, idx: int = None, head_id: int = None, whitespace_after: bool = True, + start_position: int = None ): self.text: str = text self.idx: int = idx self.head_id: int = head_id self.whitespace_after: bool = whitespace_after + self.start_pos = start_position + self.end_pos = start_position + len(text) if start_position is not None else None + self.sentence: Sentence = None self._embeddings: Dict = {} self.tags: Dict[str, Label] = {} @@ -168,12 +178,6 @@ def get_tag(self, tag_type: str) -> Label: def get_head(self): return self.sentence.get_token(self.head_id) - def __str__(self) -> str: - return 'Token: %d %s' % (self.idx, self.text) if self.idx is not None else 'Token: %s' % (self.text) - - def __repr__(self) -> str: - return 'Token: %d %s' % (self.idx, self.text) if self.idx is not None else 'Token: %s' % (self.text) - def set_embedding(self, name: str, vector: torch.autograd.Variable): self._embeddings[name] = vector.cpu() @@ -189,10 +193,24 @@ def get_embedding(self) -> torch.FloatTensor: return torch.FloatTensor() + @property + def start_position(self) -> int: + return self.start_pos + + @property + def end_position(self) -> int: + return self.end_pos + @property def embedding(self): return self.get_embedding() + def __str__(self) -> str: + return 'Token: %d %s' % (self.idx, self.text) if self.idx is not None else 'Token: %s' % (self.text) + + def __repr__(self) -> str: + return 'Token: %d %s' % (self.idx, self.text) if self.idx is not None else 'Token: %s' % (self.text) + class Span: """ @@ -203,11 +221,39 @@ def __init__(self, tokens: List[Token], tag: str = None, score=1.): self.tokens = tokens self.tag = tag self.score = score + self.start_pos = None + self.end_pos = None + + if tokens: + self.start_pos = tokens[0].start_position + self.end_pos = tokens[len(tokens) - 1].end_position @property def text(self) -> str: return ' '.join([t.text for t in self.tokens]) + def to_original_text(self) -> str: + str = '' + pos = self.tokens[0].start_pos + for t in self.tokens: + while t.start_pos != pos: + str += ' ' + pos += 1 + + str += t.text + pos += len(t.text) + + return str + + def to_dict(self): + return { + 'text': self.to_original_text(), + 'start_pos': self.start_pos, + 'end_pos': self.end_pos, + 'type': self.tag, + 'confidence': self.score + } + def __str__(self) -> str: ids = ','.join([str(t.idx) for t in self.tokens]) return '{}-span [{}]: "{}"'.format(self.tag, ids, self.text) \ @@ -254,7 +300,7 @@ def __init__(self, text: str = None, use_tokenizer: bool = False, labels: Union[ last_word_offset = -1 last_token = None for word in tokens: - token = Token(word) + token = Token(word, start_position=index(word, running_offset)) self.add_token(token) try: word_offset = index(word, running_offset) @@ -270,10 +316,12 @@ def __init__(self, text: str = None, use_tokenizer: bool = False, labels: Union[ # otherwise assumes whitespace tokenized text else: # add each word in tokenized string as Token object to Sentence + offset = 0 for word in text.split(' '): if word: - token = Token(word) + token = Token(word, start_position=text.index(word, offset)) self.add_token(token) + offset += len(word) + 1 def get_token(self, token_id: int) -> Token: for token in self.tokens: @@ -473,6 +521,34 @@ def infer_space_after(self): last_token = token return self + def to_original_text(self) -> str: + str = '' + pos = 0 + for t in self.tokens: + while t.start_pos != pos: + str += ' ' + pos += 1 + + str += t.text + pos += len(t.text) + + return str + + def to_dict(self, tag_type: str = None): + labels = [] + entities = [] + + if tag_type: + entities = [span.to_dict() for span in self.get_spans(tag_type)] + if self.labels: + labels = [l.to_dict() for l in self.labels] + + return { + 'text': self.to_original_text(), + 'labels': labels, + 'entities': entities + } + def __getitem__(self, idx: int) -> Token: return self.tokens[idx] diff --git a/tests/test_data.py b/tests/test_data.py index e4f0a30660..5478d6b329 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -448,4 +448,57 @@ def test_spans(): assert (1 == len(spans)) spans: List[Span] = sentence.get_spans('ner', min_score=0.99) - assert (0 == len(spans)) \ No newline at end of file + assert (0 == len(spans)) + + +def test_token_position_in_sentence(): + sentence = Sentence("I love Berlin .") + + assert(0 == sentence.tokens[0].start_position) + assert(1 == sentence.tokens[0].end_position) + assert(2 == sentence.tokens[1].start_position) + assert(6 == sentence.tokens[1].end_position) + assert(7 == sentence.tokens[2].start_position) + assert(13 == sentence.tokens[2].end_position) + + sentence = Sentence(" I love Berlin.", use_tokenizer=True) + + assert(1 == sentence.tokens[0].start_position) + assert(2 == sentence.tokens[0].end_position) + assert(3 == sentence.tokens[1].start_position) + assert(7 == sentence.tokens[1].end_position) + assert(9 == sentence.tokens[2].start_position) + assert(15 == sentence.tokens[2].end_position) + + +def test_sentence_to_dict(): + sentence = Sentence('Zalando Research is located in Berlin, the capital of Germany.', labels=['business'], use_tokenizer=True) + + # bioes tags + sentence[0].add_tag('ner', 'B-ORG') + sentence[1].add_tag('ner', 'E-ORG') + sentence[5].add_tag('ner', 'S-LOC') + sentence[10].add_tag('ner', 'S-LOC') + + dict = sentence.to_dict('ner') + + assert ('Zalando Research is located in Berlin, the capital of Germany.' == dict['text']) + assert ('Zalando Research' == dict['entities'][0]['text']) + assert ('Berlin' == dict['entities'][1]['text']) + assert ('Germany' == dict['entities'][2]['text']) + assert (1 == len(dict['labels'])) + + sentence = Sentence('Facebook, Inc. is a company, and Google is one as well.', use_tokenizer=True) + + # bioes tags + sentence[0].add_tag('ner', 'B-ORG') + sentence[1].add_tag('ner', 'I-ORG') + sentence[2].add_tag('ner', 'E-ORG') + sentence[8].add_tag('ner', 'S-ORG') + + dict = sentence.to_dict('ner') + + assert ('Facebook, Inc. is a company, and Google is one as well.' == dict['text']) + assert ('Facebook, Inc.' == dict['entities'][0]['text']) + assert ('Google' == dict['entities'][1]['text']) + assert (0 == len(dict['labels'])) \ No newline at end of file From 6a673368e155ef90b4e7763e28497d2c4d36af74 Mon Sep 17 00:00:00 2001 From: tabergma Date: Thu, 11 Oct 2018 09:57:57 +0200 Subject: [PATCH 072/113] GH-139: Adapt defualt params for text classifier. --- flair/trainers/text_classification_trainer.py | 46 +++++++++++-------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/flair/trainers/text_classification_trainer.py b/flair/trainers/text_classification_trainer.py index 979badc59c..f847472d46 100644 --- a/flair/trainers/text_classification_trainer.py +++ b/flair/trainers/text_classification_trainer.py @@ -11,10 +11,8 @@ from flair.training_utils import convert_labels_to_one_hot, calculate_micro_avg_metric, init_output_file, \ clear_embeddings, calculate_class_metrics, WeightExtractor, Metric - MICRO_AVG_METRIC = 'MICRO_AVG' - log = logging.getLogger(__name__) @@ -23,7 +21,8 @@ class TextClassifierTrainer: Training class to train and evaluate a text classification model. """ - def __init__(self, model: TextClassifier, corpus: TaggedCorpus, label_dict: Dictionary, test_mode: bool = False) -> None: + def __init__(self, model: TextClassifier, corpus: TaggedCorpus, label_dict: Dictionary, + test_mode: bool = False) -> None: self.model: TextClassifier = model self.corpus: TaggedCorpus = corpus self.label_dict: Dictionary = label_dict @@ -33,20 +32,20 @@ def train(self, base_path: str, learning_rate: float = 0.1, mini_batch_size: int = 32, - eval_mini_batch_size: int = 8, - max_epochs: int = 100, + max_epochs: int = 50, anneal_factor: float = 0.5, - patience: int = 2, + patience: int = 5, save_model: bool = True, - embeddings_in_memory: bool = True, + embeddings_in_memory: bool = False, train_with_dev: bool = False, - eval_on_train: bool = False): + eval_on_train: bool = True): """ Trains the model using the training data of the corpus. + :param patience: number of 'bad' epochs before learning rate gets decreased + :param anneal_factor: learning rate will be decreased by this factor :param base_path: the directory to which any results should be written to :param learning_rate: the learning rate :param mini_batch_size: the mini batch size - :param eval_mini_batch_size: the mini batch size for evaluation :param max_epochs: the maximum number of epochs to train :param save_model: boolean value indicating, whether the model should be saved or not :param embeddings_in_memory: boolean value indicating, if embeddings should be kept in memory or not @@ -122,7 +121,8 @@ def train(self, self.model.eval() log.info('-' * 100) - log.info("EPOCH {0}: lr {1:.4f} - bad epochs {2}".format(epoch + 1, learning_rate, scheduler.num_bad_epochs)) + log.info( + "EPOCH {0}: lr {1:.4f} - bad epochs {2}".format(epoch + 1, learning_rate, scheduler.num_bad_epochs)) dev_metric = train_metric = None dev_loss = '_' @@ -130,17 +130,18 @@ def train(self, if eval_on_train: train_metric, train_loss = self._calculate_evaluation_results_for( - 'TRAIN', self.corpus.train, embeddings_in_memory, eval_mini_batch_size) + 'TRAIN', self.corpus.train, embeddings_in_memory, mini_batch_size) if not train_with_dev: dev_metric, dev_loss = self._calculate_evaluation_results_for( - 'DEV', self.corpus.dev, embeddings_in_memory, eval_mini_batch_size) + 'DEV', self.corpus.dev, embeddings_in_memory, mini_batch_size) with open(loss_txt, 'a') as f: train_metric_str = train_metric.to_tsv() if train_metric is not None else Metric.to_empty_tsv() dev_metric_str = dev_metric.to_tsv() if dev_metric is not None else Metric.to_empty_tsv() f.write('{}\t{:%H:%M:%S}\t{}\t{}\t{}\t{}\t{}\t{}\n'.format( - epoch, datetime.datetime.now(), train_loss, train_metric_str, dev_loss, dev_metric_str, '_', Metric.to_empty_tsv())) + epoch, datetime.datetime.now(), train_loss, train_metric_str, dev_loss, dev_metric_str, '_', + Metric.to_empty_tsv())) # anneal against train loss if training with dev, otherwise anneal against dev score scheduler.step(current_loss) if train_with_dev else scheduler.step(dev_metric.f_score()) @@ -149,8 +150,8 @@ def train(self, current_score = dev_metric.f_score() if not train_with_dev else train_metric.f_score() if current_score > best_score: - best_score = current_score - is_best_model_so_far = True + best_score = current_score + is_best_model_so_far = True if is_best_model_so_far: if save_model: @@ -186,7 +187,7 @@ def train(self, def _calculate_evaluation_results_for(self, dataset_name, dataset, embeddings_in_memory, mini_batch_size): metrics, loss = self.evaluate(dataset, mini_batch_size=mini_batch_size, - embeddings_in_memory=embeddings_in_memory) + embeddings_in_memory=embeddings_in_memory) f_score = metrics[MICRO_AVG_METRIC].f_score() acc = metrics[MICRO_AVG_METRIC].accuracy() @@ -196,12 +197,14 @@ def _calculate_evaluation_results_for(self, dataset_name, dataset, embeddings_in return metrics[MICRO_AVG_METRIC], loss - def evaluate(self, sentences: List[Sentence], eval_class_metrics: bool = False, mini_batch_size: int = 16, - embeddings_in_memory: bool = True) -> (dict, float): + def evaluate(self, sentences: List[Sentence], eval_class_metrics: bool = False, mini_batch_size: int = 32, + embeddings_in_memory: bool = False) -> (dict, float): """ Evaluates the model with the given list of sentences. :param sentences: the list of sentences + :param eval_class_metrics: boolean indicating whether to print class metrics or not :param mini_batch_size: the mini batch size to use + :param embeddings_in_memory: boolean value indicating, if embeddings should be kept in memory or not :return: list of metrics, and the loss """ with torch.no_grad(): @@ -223,8 +226,11 @@ def evaluate(self, sentences: List[Sentence], eval_class_metrics: bool = False, eval_loss += loss - y_pred.extend(convert_labels_to_one_hot([[label.value for label in sent_labels] for sent_labels in labels], self.label_dict)) - y_true.extend(convert_labels_to_one_hot([sentence.get_label_names() for sentence in batch], self.label_dict)) + y_pred.extend( + convert_labels_to_one_hot([[label.value for label in sent_labels] for sent_labels in labels], + self.label_dict)) + y_true.extend( + convert_labels_to_one_hot([sentence.get_label_names() for sentence in batch], self.label_dict)) metrics = [calculate_micro_avg_metric(y_true, y_pred, self.label_dict)] if eval_class_metrics: From 362dec0e9dfd0036fd3027f76a37c7d2f8ac7954 Mon Sep 17 00:00:00 2001 From: aakbik Date: Thu, 11 Oct 2018 13:48:32 +0200 Subject: [PATCH 073/113] GH-89: consistent notation for epoch/split --- flair/trainers/language_model_trainer.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/flair/trainers/language_model_trainer.py b/flair/trainers/language_model_trainer.py index d3e2023597..b3ef298f56 100644 --- a/flair/trainers/language_model_trainer.py +++ b/flair/trainers/language_model_trainer.py @@ -169,9 +169,13 @@ def train(self, anneal_factor: float = 0.25, patience: int = 10, clip=0.25, - max_epochs: int = 10000): + max_epochs: int = 1000): + + number_of_splits: int = len(self.corpus.train_files) + + # an epoch has a number, so calculate total max splits bby multiplying max_epochs with number_of_splits + max_splits: int = number_of_splits * max_epochs - number_of_splits = len(self.corpus.train_files) val_data = self._batchify(self.corpus.valid, mini_batch_size) os.makedirs(base_path, exist_ok=True) @@ -186,7 +190,7 @@ def train(self, scheduler: ReduceLROnPlateau = ReduceLROnPlateau(optimizer, verbose=True, factor=anneal_factor, patience=patience) - for split in range(1, max_epochs + 1): + for split in range(1, max_splits + 1): log.info('Split %d' % split + '\t - ({:%H:%M:%S})'.format(datetime.datetime.now())) From 8d88db884f5625a580d543368b328ceec8266dd9 Mon Sep 17 00:00:00 2001 From: tabergma Date: Thu, 11 Oct 2018 14:06:38 +0200 Subject: [PATCH 074/113] GH-82: Set default warning settings. --- flair/__init__.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/flair/__init__.py b/flair/__init__.py index a70a3a088d..09e1736a8d 100644 --- a/flair/__init__.py +++ b/flair/__init__.py @@ -5,10 +5,16 @@ import sys import logging +import warnings logger = logging.getLogger(__name__) FORMAT = '%(asctime)-15s %(message)s' logging.basicConfig(level=logging.WARNING, format=FORMAT, stream=sys.stdout) -logging.getLogger('flair').setLevel(logging.INFO) \ No newline at end of file +logging.getLogger('flair').setLevel(logging.INFO) + +warnings.filterwarnings("ignore", category=DeprecationWarning) +warnings.filterwarnings("ignore", category=ImportWarning) +warnings.filterwarnings("ignore", category=PendingDeprecationWarning) +warnings.filterwarnings("ignore", category=ResourceWarning) \ No newline at end of file From 3c5bf1afb05246c3e813fd4862a5f273850590bb Mon Sep 17 00:00:00 2001 From: aakbik Date: Thu, 11 Oct 2018 14:17:57 +0200 Subject: [PATCH 075/113] GH-139: Polish character language model embeddings --- README.md | 1 + flair/embeddings.py | 10 ++++++++++ resources/docs/TUTORIAL_WORD_EMBEDDING.md | 2 ++ 3 files changed, 13 insertions(+) diff --git a/README.md b/README.md index 23f374bc6c..5c4e545e18 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,7 @@ Flair outperforms the previous best methods on a range of NLP tasks: | Emerging Entity Detection (English) | WNUT-17 | **50.20** (F1) | *45.55 [(Aguilar et al., 2018)](http://aclweb.org/anthology/N18-1127.pdf)* | | Named Entity Recognition (German) | Conll-03 | **88.32** (F1) | *78.76 [(Lample et al., 2016)](https://arxiv.org/abs/1603.01360)* | | Named Entity Recognition (German) | Germeval | **84.65** (F1) | *79.08 [(Hänig et al, 2014)](http://asv.informatik.uni-leipzig.de/publication/file/300/GermEval2014_ExB.pdf)*| +| Named Entity Recognition (Polish) | PolEval-2018 | **86.6** (F1) [(Borchmann et al., 2018)](https://github.com/applicaai/poleval-2018) | *85.1 [(PolDeepNer)](https://github.com/CLARIN-PL/PolDeepNer/)*| | Part-of-Speech tagging | WSJ | **97.85** | *97.64 [(Choi, 2016)](https://www.aclweb.org/anthology/N16-1031)*| | Chunking | Conll-2000 | **96.72** (F1) | *96.36 [(Peters et al., 2017)](https://arxiv.org/pdf/1705.00108.pdf)* diff --git a/flair/embeddings.py b/flair/embeddings.py index ea87fbdcb9..63935015bd 100644 --- a/flair/embeddings.py +++ b/flair/embeddings.py @@ -413,6 +413,16 @@ def __init__(self, model, detach: bool = True, use_cache: bool = True): base_path = 'https://s3.eu-central-1.amazonaws.com/alan-nlp/resources/embeddings/lm-mix-german-backward-v0.2rc.pt' model = cached_path(base_path, cache_dir='embeddings') + # common crawl Polish forward + if model.lower() == 'polish-forward': + base_path = 'https://s3.eu-central-1.amazonaws.com/alan-nlp/resources/embeddings/lm-polish-forward-v0.2.pt' + model = cached_path(base_path, cache_dir='embeddings') + + # common crawl Polish backward + if model.lower() == 'polish-backward': + base_path = 'https://s3.eu-central-1.amazonaws.com/alan-nlp/resources/embeddings/lm-polish-backward-v0.2.pt' + model = cached_path(base_path, cache_dir='embeddings') + self.name = model self.static_embeddings = detach diff --git a/resources/docs/TUTORIAL_WORD_EMBEDDING.md b/resources/docs/TUTORIAL_WORD_EMBEDDING.md index 4e03875d07..f6f7cf1907 100644 --- a/resources/docs/TUTORIAL_WORD_EMBEDDING.md +++ b/resources/docs/TUTORIAL_WORD_EMBEDDING.md @@ -108,6 +108,8 @@ Currently, the following contextual string embeddings are provided (more coming) | 'mix-backward' | English | Backward LM embeddings over mixed corpus (Web, Wikipedia, Subtitles) | | 'german-forward' | German | Forward LM embeddings over mixed corpus (Web, Wikipedia, Subtitles) | | 'german-backward' | German | Backward LM embeddings over mixed corpus (Web, Wikipedia, Subtitles) | +| 'polish-forward' | Polish | Forward LM embeddings over web crawls (Polish part of CommonCrawl) | +| 'polish-backward' | Polish | Backward LM embeddings over web crawls (Polish part of CommonCrawl) | So, if you want to load embeddings from the English news backward LM model, instantiate the method as follows: From 73184449e6757884cb751c28f917e1a6bb4711a5 Mon Sep 17 00:00:00 2001 From: aakbik Date: Thu, 11 Oct 2018 14:20:28 +0200 Subject: [PATCH 076/113] GH-139: Polish character language model embeddings --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5c4e545e18..897efa0f4c 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ Now at [version 0.2.0](https://github.com/zalandoresearch/flair/releases)! Flair outperforms the previous best methods on a range of NLP tasks: -| Task | Dataset | Our Result | Previous best | +| Task | Dataset | Flair | Previous best | | ------------------------------- | ----------- | ---------------- | ------------- | | Named Entity Recognition (English) | Conll-03 | **93.09** (F1) | *92.22 [(Peters et al., 2018)](https://arxiv.org/pdf/1802.05365.pdf)* | | Named Entity Recognition (English) | Ontonotes | **89.71** (F1) | *86.28 [(Chiu et al., 2016)](https://arxiv.org/pdf/1511.08308.pdf)* | From 51f68efa66ca476061959f64c99c0d1771009d96 Mon Sep 17 00:00:00 2001 From: aakbik Date: Thu, 11 Oct 2018 14:28:49 +0200 Subject: [PATCH 077/113] GH-138: linked better setup instructions on Ubuntu 16.04 --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 897efa0f4c..1a326292ab 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ Now at [version 0.2.0](https://github.com/zalandoresearch/flair/releases)! Flair outperforms the previous best methods on a range of NLP tasks: -| Task | Dataset | Flair | Previous best | +| Task | Dataset | F | Previous best | | ------------------------------- | ----------- | ---------------- | ------------- | | Named Entity Recognition (English) | Conll-03 | **93.09** (F1) | *92.22 [(Peters et al., 2018)](https://arxiv.org/pdf/1802.05365.pdf)* | | Named Entity Recognition (English) | Ontonotes | **89.71** (F1) | *86.28 [(Chiu et al., 2016)](https://arxiv.org/pdf/1511.08308.pdf)* | @@ -56,7 +56,7 @@ Alan Akbik, Duncan Blythe and Roland Vollgraf. ### Requirements and Installation The project is based on PyTorch 0.4+ and Python 3.6+, because methods signatures and type hints are beautiful. -If you do not have Python 3.6, install it first. [Here is how for Ubuntu 16.04](http://ubuntuhandbook.org/index.php/2017/07/install-python-3-6-1-in-ubuntu-16-04-lts/). +If you do not have Python 3.6, install it first. [Here is how for Ubuntu 16.04](https://vsupalov.com/developing-with-python3-6-on-ubuntu-16-04/). Then, in your favorite virtual environment, simply do: ``` From d81d644f5f38df0d409cb781e2b788c9054c55e1 Mon Sep 17 00:00:00 2001 From: aakbik Date: Thu, 11 Oct 2018 14:29:43 +0200 Subject: [PATCH 078/113] GH-138: linked better setup instructions on Ubuntu 16.04 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1a326292ab..da51193247 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ Now at [version 0.2.0](https://github.com/zalandoresearch/flair/releases)! Flair outperforms the previous best methods on a range of NLP tasks: -| Task | Dataset | F | Previous best | +| Task | Dataset | Flair | Previous best | | ------------------------------- | ----------- | ---------------- | ------------- | | Named Entity Recognition (English) | Conll-03 | **93.09** (F1) | *92.22 [(Peters et al., 2018)](https://arxiv.org/pdf/1802.05365.pdf)* | | Named Entity Recognition (English) | Ontonotes | **89.71** (F1) | *86.28 [(Chiu et al., 2016)](https://arxiv.org/pdf/1511.08308.pdf)* | From a5f7e554e9b4fee7cf2774dd733b7678cc6230b6 Mon Sep 17 00:00:00 2001 From: aakbik Date: Thu, 11 Oct 2018 14:36:15 +0200 Subject: [PATCH 079/113] GH-89: log epoch and split count --- flair/trainers/language_model_trainer.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/flair/trainers/language_model_trainer.py b/flair/trainers/language_model_trainer.py index b3ef298f56..c5ebfe75f6 100644 --- a/flair/trainers/language_model_trainer.py +++ b/flair/trainers/language_model_trainer.py @@ -192,6 +192,10 @@ def train(self, for split in range(1, max_splits + 1): + # after pass over all splits, increment epoch count + if (split - 1) % number_of_splits == 0: + epoch += 1 + log.info('Split %d' % split + '\t - ({:%H:%M:%S})'.format(datetime.datetime.now())) for group in optimizer.param_groups: From 53e2d73fc5406e5c3d798e056203dc179e25a483 Mon Sep 17 00:00:00 2001 From: aakbik Date: Thu, 11 Oct 2018 14:47:27 +0200 Subject: [PATCH 080/113] GH-139: save last best model --- flair/trainers/text_classification_trainer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flair/trainers/text_classification_trainer.py b/flair/trainers/text_classification_trainer.py index f847472d46..e6dc9dad6b 100644 --- a/flair/trainers/text_classification_trainer.py +++ b/flair/trainers/text_classification_trainer.py @@ -149,7 +149,7 @@ def train(self, is_best_model_so_far: bool = False current_score = dev_metric.f_score() if not train_with_dev else train_metric.f_score() - if current_score > best_score: + if current_score >= best_score: best_score = current_score is_best_model_so_far = True From d9c629dfc569a3e2533f99d7b2f0334d150a3781 Mon Sep 17 00:00:00 2001 From: aakbik Date: Thu, 11 Oct 2018 14:52:14 +0200 Subject: [PATCH 081/113] GH-139: documentation --- README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/README.md b/README.md index da51193247..de007a1f1c 100644 --- a/README.md +++ b/README.md @@ -18,9 +18,7 @@ Flair is: recognition (NER), part-of-speech tagging (PoS), frame sense disambiguation, chunking and classification to your text. * **A text embedding library.** Flair has simple interfaces that allow you to use and combine different word and -document embeddings. In particular, you can try out our proposed -**[contextual string embeddings](https://drive.google.com/file/d/17yVpFA7MmXaQFTe-HDpZuqw9fJlmzg56/view?usp=sharing)** -to build your own state-of-the-art NLP methods. +document embeddings, including our proposed **[contextual string embeddings](https://drive.google.com/file/d/17yVpFA7MmXaQFTe-HDpZuqw9fJlmzg56/view?usp=sharing)**. * **A Pytorch NLP framework.** Our framework builds directly on [Pytorch](https://pytorch.org/), making it easy to train your own models and experiment with new approaches using Flair embeddings and classes. From 5854699823cae8f215862fd0e610357e2ba2aac9 Mon Sep 17 00:00:00 2001 From: aakbik Date: Thu, 11 Oct 2018 14:54:32 +0200 Subject: [PATCH 082/113] GH-139: documentation --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index de007a1f1c..65f4fc728e 100644 --- a/README.md +++ b/README.md @@ -14,8 +14,9 @@ A very simple framework for **state-of-the-art NLP**. Developed by [Zalando Rese Flair is: -* **A powerful syntactic-semantic tagger / classifier.** Flair allows you to apply our state-of-the-art models for named entity -recognition (NER), part-of-speech tagging (PoS), frame sense disambiguation, chunking and classification to your text. +* **A powerful NLP library.** Flair allows you to apply our state-of-the-art natural language processing (NLP) +models to your text, such as named entity recognition (NER), part-of-speech tagging (PoS), + frame sense disambiguation, chunking and classification. * **A text embedding library.** Flair has simple interfaces that allow you to use and combine different word and document embeddings, including our proposed **[contextual string embeddings](https://drive.google.com/file/d/17yVpFA7MmXaQFTe-HDpZuqw9fJlmzg56/view?usp=sharing)**. From 28f906a8e94f4b203f23d6fce8ba7193bd9edccb Mon Sep 17 00:00:00 2001 From: aakbik Date: Thu, 11 Oct 2018 14:58:30 +0200 Subject: [PATCH 083/113] GH-139: documentation --- README.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 65f4fc728e..21a374c8dc 100644 --- a/README.md +++ b/README.md @@ -30,16 +30,16 @@ Now at [version 0.2.0](https://github.com/zalandoresearch/flair/releases)! Flair outperforms the previous best methods on a range of NLP tasks: -| Task | Dataset | Flair | Previous best | -| ------------------------------- | ----------- | ---------------- | ------------- | -| Named Entity Recognition (English) | Conll-03 | **93.09** (F1) | *92.22 [(Peters et al., 2018)](https://arxiv.org/pdf/1802.05365.pdf)* | -| Named Entity Recognition (English) | Ontonotes | **89.71** (F1) | *86.28 [(Chiu et al., 2016)](https://arxiv.org/pdf/1511.08308.pdf)* | -| Emerging Entity Detection (English) | WNUT-17 | **50.20** (F1) | *45.55 [(Aguilar et al., 2018)](http://aclweb.org/anthology/N18-1127.pdf)* | -| Named Entity Recognition (German) | Conll-03 | **88.32** (F1) | *78.76 [(Lample et al., 2016)](https://arxiv.org/abs/1603.01360)* | -| Named Entity Recognition (German) | Germeval | **84.65** (F1) | *79.08 [(Hänig et al, 2014)](http://asv.informatik.uni-leipzig.de/publication/file/300/GermEval2014_ExB.pdf)*| -| Named Entity Recognition (Polish) | PolEval-2018 | **86.6** (F1) [(Borchmann et al., 2018)](https://github.com/applicaai/poleval-2018) | *85.1 [(PolDeepNer)](https://github.com/CLARIN-PL/PolDeepNer/)*| -| Part-of-Speech tagging | WSJ | **97.85** | *97.64 [(Choi, 2016)](https://www.aclweb.org/anthology/N16-1031)*| -| Chunking | Conll-2000 | **96.72** (F1) | *96.36 [(Peters et al., 2017)](https://arxiv.org/pdf/1705.00108.pdf)* +| Task | Language | Dataset | Flair | Previous best | +| ------------------------------- | --- | ----------- | ---------------- | ------------- | +| Named Entity Recognition |English | Conll-03 | **93.09** (F1) | *92.22 [(Peters et al., 2018)](https://arxiv.org/pdf/1802.05365.pdf)* | +| Named Entity Recognition |English | Ontonotes | **89.71** (F1) | *86.28 [(Chiu et al., 2016)](https://arxiv.org/pdf/1511.08308.pdf)* | +| Emerging Entity Detection | English | WNUT-17 | **50.20** (F1) | *45.55 [(Aguilar et al., 2018)](http://aclweb.org/anthology/N18-1127.pdf)* | +| Part-of-Speech tagging |English| WSJ | **97.85** | *97.64 [(Choi, 2016)](https://www.aclweb.org/anthology/N16-1031)*| +| Chunking |English| Conll-2000 | **96.72** (F1) | *96.36 [(Peters et al., 2017)](https://arxiv.org/pdf/1705.00108.pdf)* +| Named Entity Recognition | German | Conll-03 | **88.32** (F1) | *78.76 [(Lample et al., 2016)](https://arxiv.org/abs/1603.01360)* | +| Named Entity Recognition |German | Germeval | **84.65** (F1) | *79.08 [(Hänig et al, 2014)](http://asv.informatik.uni-leipzig.de/publication/file/300/GermEval2014_ExB.pdf)*| +| Named Entity Recognition |Polish | PolEval-2018 | **86.6** (F1) [(Borchmann et al., 2018)](https://github.com/applicaai/poleval-2018) | *85.1 [(PolDeepNer)](https://github.com/CLARIN-PL/PolDeepNer/)*| From 317a283f1d9deb053829dd43d91a1ab04b376583 Mon Sep 17 00:00:00 2001 From: aakbik Date: Thu, 11 Oct 2018 15:02:11 +0200 Subject: [PATCH 084/113] GH-139: documentation --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 21a374c8dc..27736515a9 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ Flair outperforms the previous best methods on a range of NLP tasks: | Chunking |English| Conll-2000 | **96.72** (F1) | *96.36 [(Peters et al., 2017)](https://arxiv.org/pdf/1705.00108.pdf)* | Named Entity Recognition | German | Conll-03 | **88.32** (F1) | *78.76 [(Lample et al., 2016)](https://arxiv.org/abs/1603.01360)* | | Named Entity Recognition |German | Germeval | **84.65** (F1) | *79.08 [(Hänig et al, 2014)](http://asv.informatik.uni-leipzig.de/publication/file/300/GermEval2014_ExB.pdf)*| -| Named Entity Recognition |Polish | PolEval-2018 | **86.6** (F1) [(Borchmann et al., 2018)](https://github.com/applicaai/poleval-2018) | *85.1 [(PolDeepNer)](https://github.com/CLARIN-PL/PolDeepNer/)*| +| Named Entity Recognition |Polish | PolEval-2018 | **86.6** (F1)
[(Borchmann et al., 2018)](https://github.com/applicaai/poleval-2018) | *85.1 [(PolDeepNer)](https://github.com/CLARIN-PL/PolDeepNer/)*| From 3a6f81c2d8f10f5dea8e4996e46c976ff26daa97 Mon Sep 17 00:00:00 2001 From: aakbik Date: Thu, 11 Oct 2018 15:06:27 +0200 Subject: [PATCH 085/113] GH-139: documentation --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 27736515a9..b5cab84668 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ Flair is: * **A powerful NLP library.** Flair allows you to apply our state-of-the-art natural language processing (NLP) models to your text, such as named entity recognition (NER), part-of-speech tagging (PoS), - frame sense disambiguation, chunking and classification. + sense disambiguation and classification. * **A text embedding library.** Flair has simple interfaces that allow you to use and combine different word and document embeddings, including our proposed **[contextual string embeddings](https://drive.google.com/file/d/17yVpFA7MmXaQFTe-HDpZuqw9fJlmzg56/view?usp=sharing)**. From eb2d601c11dc19cb192f43e8695d11bd275bfbfa Mon Sep 17 00:00:00 2001 From: aakbik Date: Thu, 11 Oct 2018 15:25:27 +0200 Subject: [PATCH 086/113] GH-139: documentation --- flair/embeddings.py | 4 +++ resources/docs/TUTORIAL_TAGGING.md | 40 ++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/flair/embeddings.py b/flair/embeddings.py index 63935015bd..866570727f 100644 --- a/flair/embeddings.py +++ b/flair/embeddings.py @@ -456,6 +456,10 @@ def embedding_length(self) -> int: def _add_embeddings_internal(self, sentences: List[Sentence]) -> List[Sentence]: + # by default, use_cache is false (for older pre-trained models TODO: remove in version 0.4) + if 'cache' not in self.__dict__: + self.use_cache = False + # if cache is used, try setting embeddings from cache first if self.use_cache: diff --git a/resources/docs/TUTORIAL_TAGGING.md b/resources/docs/TUTORIAL_TAGGING.md index cc4389710a..70ec542a8c 100644 --- a/resources/docs/TUTORIAL_TAGGING.md +++ b/resources/docs/TUTORIAL_TAGGING.md @@ -32,6 +32,46 @@ This should print: George Washington went to Washington . ``` +## Getting Annotated Spans + +Many sequence labeling methods annotate spans that consist of multiple words, +such as "George Washington" in our example sentence. + +You can directly get such spans a tagged sentence like this: + +```python +for entity in sentence.get_spans('ner'): + print(entity) +``` + +This should print: +```console +PER-span [1,2]: "George Washington" +LOC-span [5]: "Washington" +``` + +Which indicates that "George Washington" is a person (PER) and "Washington" is +a location (LOC). Each such `Span` has a text, a tag value, its position +in the sentence and "score" that indicates how confident the tagger is that the prediction is correct. + +You can also get additional information, such as the position offsets of +each entity in the sentence by calling: + +```python +print(sentence.to_dict(tag_type='ner')) +``` + +This should print: +```console +{'text': 'George Washington went to Washington .', + 'entities': [ + {'text': 'George Washington', 'start_pos': 0, 'end_pos': 17, 'type': 'PER', 'confidence': 0.9999}, + {'text': 'Washington', 'start_pos': 26, 'end_pos': 36, 'type': 'LOC', 'confidence': 0.9988} + ]} +``` + +## List of Pre-Trained Models + You chose which pre-trained model you load by passing the appropriate string to the `load()` method of the `SequenceTagger` class. Currently, the following pre-trained models are provided: From 541711ba05fd892a88449694e5e8039a5cc8804a Mon Sep 17 00:00:00 2001 From: aakbik Date: Thu, 11 Oct 2018 15:26:21 +0200 Subject: [PATCH 087/113] GH-139: documentation --- resources/docs/TUTORIAL_TAGGING.md | 1 - 1 file changed, 1 deletion(-) diff --git a/resources/docs/TUTORIAL_TAGGING.md b/resources/docs/TUTORIAL_TAGGING.md index 70ec542a8c..8c10acfc1b 100644 --- a/resources/docs/TUTORIAL_TAGGING.md +++ b/resources/docs/TUTORIAL_TAGGING.md @@ -53,7 +53,6 @@ LOC-span [5]: "Washington" Which indicates that "George Washington" is a person (PER) and "Washington" is a location (LOC). Each such `Span` has a text, a tag value, its position in the sentence and "score" that indicates how confident the tagger is that the prediction is correct. - You can also get additional information, such as the position offsets of each entity in the sentence by calling: From 724b3e5ad4fa3a8caa7074ee441908d492f282aa Mon Sep 17 00:00:00 2001 From: aakbik Date: Thu, 11 Oct 2018 15:26:48 +0200 Subject: [PATCH 088/113] GH-139: documentation --- resources/docs/TUTORIAL_TAGGING.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/resources/docs/TUTORIAL_TAGGING.md b/resources/docs/TUTORIAL_TAGGING.md index 8c10acfc1b..d922011b4c 100644 --- a/resources/docs/TUTORIAL_TAGGING.md +++ b/resources/docs/TUTORIAL_TAGGING.md @@ -64,8 +64,8 @@ This should print: ```console {'text': 'George Washington went to Washington .', 'entities': [ - {'text': 'George Washington', 'start_pos': 0, 'end_pos': 17, 'type': 'PER', 'confidence': 0.9999}, - {'text': 'Washington', 'start_pos': 26, 'end_pos': 36, 'type': 'LOC', 'confidence': 0.9988} + {'text': 'George Washington', 'start_pos': 0, 'end_pos': 17, 'type': 'PER', 'confidence': 0.999}, + {'text': 'Washington', 'start_pos': 26, 'end_pos': 36, 'type': 'LOC', 'confidence': 0.998} ]} ``` From be20dbc66150e1503e9c28c90558ce3f0eb93abb Mon Sep 17 00:00:00 2001 From: aakbik Date: Thu, 11 Oct 2018 15:28:11 +0200 Subject: [PATCH 089/113] GH-139: documentation --- resources/docs/TUTORIAL_TAGGING.md | 1 - 1 file changed, 1 deletion(-) diff --git a/resources/docs/TUTORIAL_TAGGING.md b/resources/docs/TUTORIAL_TAGGING.md index d922011b4c..b83504fa4c 100644 --- a/resources/docs/TUTORIAL_TAGGING.md +++ b/resources/docs/TUTORIAL_TAGGING.md @@ -36,7 +36,6 @@ George Washington went to Washington . Many sequence labeling methods annotate spans that consist of multiple words, such as "George Washington" in our example sentence. - You can directly get such spans a tagged sentence like this: ```python From 5b0e09454d57a13459c52c84535272975f8655bd Mon Sep 17 00:00:00 2001 From: aakbik Date: Thu, 11 Oct 2018 15:34:03 +0200 Subject: [PATCH 090/113] GH-139: documentation --- resources/docs/TUTORIAL_TAGGING.md | 2 +- resources/docs/TUTORIAL_WORD_EMBEDDING.md | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/resources/docs/TUTORIAL_TAGGING.md b/resources/docs/TUTORIAL_TAGGING.md index b83504fa4c..204e94a6d3 100644 --- a/resources/docs/TUTORIAL_TAGGING.md +++ b/resources/docs/TUTORIAL_TAGGING.md @@ -70,7 +70,7 @@ This should print: ## List of Pre-Trained Models -You chose which pre-trained model you load by passing the appropriate +You choose which pre-trained model you load by passing the appropriate string to the `load()` method of the `SequenceTagger` class. Currently, the following pre-trained models are provided: diff --git a/resources/docs/TUTORIAL_WORD_EMBEDDING.md b/resources/docs/TUTORIAL_WORD_EMBEDDING.md index f6f7cf1907..5eb2f3943e 100644 --- a/resources/docs/TUTORIAL_WORD_EMBEDDING.md +++ b/resources/docs/TUTORIAL_WORD_EMBEDDING.md @@ -11,7 +11,7 @@ All word embedding classes inherit from the `TokenEmbeddings` class and implemen call to embed your text. This means that for most users of Flair, the complexity of different embeddings remains hidden behind this interface. Simply instantiate the embedding class you require and call `embed()` to embed your text. -All embeddings produced with our methods are pytorch vectors, so they can be immediately used for training and +All embeddings produced with our methods are Pytorch vectors, so they can be immediately used for training and fine-tuning. ## Classic Word Embeddings @@ -28,9 +28,8 @@ from flair.embeddings import WordEmbeddings # init embedding glove_embedding = WordEmbeddings('glove') ``` -Now, create an example sentence and call the embedding's `embed()` method. You always pass a list of sentences to -this method since some embedding types make use of batching to increase speed. So if you only have one sentence, -pass a list containing only one sentence: +Now, create an example sentence and call the embedding's `embed()` method. You can also pass a list of sentences to +this method since some embedding types make use of batching to increase speed. ```python # create sentence. From b312eb632f1c896c2c1f6df8650cb2dc637801d6 Mon Sep 17 00:00:00 2001 From: aakbik Date: Thu, 11 Oct 2018 15:37:16 +0200 Subject: [PATCH 091/113] GH-139: documentation --- resources/docs/TUTORIAL_WORD_EMBEDDING.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/resources/docs/TUTORIAL_WORD_EMBEDDING.md b/resources/docs/TUTORIAL_WORD_EMBEDDING.md index 5eb2f3943e..70a8eb6546 100644 --- a/resources/docs/TUTORIAL_WORD_EMBEDDING.md +++ b/resources/docs/TUTORIAL_WORD_EMBEDDING.md @@ -56,7 +56,7 @@ are provided (more coming): | 'en-numberbatch' (or 'numberbatch') | English |[Numberbatch](https://github.com/commonsense/conceptnet-numberbatch) embeddings | | 'en-extvec' (or 'extvec') | English |Komnios embeddings | | 'en-crawl' (or 'crawl') | English | FastText embeddings over Web crawls | -| 'en-news' (or 'news') |E nglish | FastText embeddings over news and wikipedia data | +| 'en-news' (or 'news') |English | FastText embeddings over news and wikipedia data | | 'en-twitter' (or 'twitter') | English | GloVe embeddings computed over twitter data | | 'de-fasttext' | German |German FastText embeddings | | 'de-numberbatch' | German | German Numberbatch embeddings | @@ -167,7 +167,8 @@ Now instantiate the `StackedEmbeddings` class and pass it a list containing thes from flair.embeddings import StackedEmbeddings # now create the StackedEmbedding object that combines all embeddings -stacked_embeddings = StackedEmbeddings(embeddings=[glove_embedding, charlm_embedding_forward, charlm_embedding_backward]) +stacked_embeddings = StackedEmbeddings( + embeddings=[glove_embedding, charlm_embedding_forward, charlm_embedding_backward]) ``` That's it! Now just use this embedding like all the other embeddings, i.e. call the `embed()` method over your sentences. From 0098c3f06e191edfa10826d2644277a0ec4e67e3 Mon Sep 17 00:00:00 2001 From: aakbik Date: Thu, 11 Oct 2018 15:40:24 +0200 Subject: [PATCH 092/113] GH-139: documentation --- resources/docs/TUTORIAL_TEXT_EMBEDDINGS.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/resources/docs/TUTORIAL_TEXT_EMBEDDINGS.md b/resources/docs/TUTORIAL_TEXT_EMBEDDINGS.md index bc8f4d7928..551f71c41b 100644 --- a/resources/docs/TUTORIAL_TEXT_EMBEDDINGS.md +++ b/resources/docs/TUTORIAL_TEXT_EMBEDDINGS.md @@ -25,8 +25,8 @@ Currently, we have two different methods to obtain a document embedding from a l The first method calculates the mean over all word embeddings in a document. The resulting embedding is taken as document embedding. -To create a mean document embedding simply create any number of `TokenEmbeddings` first. -Afterwards, initiate the `DocumentMeanEmbeddings` and pass a list containing the created `WordEmbeddings`. +To create a mean document embedding simply create any number of `TokenEmbeddings` first and put them in a list. +Afterwards, initiate the `DocumentMeanEmbeddings` with this list of `TokenEmbeddings`. So, if you want to create a document embedding using GloVe embeddings together with CharLMEmbeddings, use the following code: @@ -91,7 +91,9 @@ print(sentence.get_embedding()) The embedding dimensionality depends on the number of hidden states you are using and whether the LSTM is bidirectional or not. -There are a number of hyperparameters of the LSTM you can tune to improve learning: +Note that while MEAN embeddings are immediately meaningful, LSTM embeddings need to be tuned on the downstream task. +This happens automatically in Flair if you train a new model with these embeddings. There are a number of hyperparameters of +the LSTM you can tune to improve learning: ```text :param hidden_states: the number of hidden states in the lstm @@ -104,9 +106,6 @@ layer before putting them into the lstm or not representation of the lstm to be used as final document embedding or not ``` -Note that while MEAN embeddings are immediately meaningful, LSTM embeddings need to be tuned on the downstream task. -This happens automatically in Flair if you train a new model with these embeddings. - ## Next From 67afc01f3d8f55478f4bfc5affffdd6e2bf72713 Mon Sep 17 00:00:00 2001 From: tabergma Date: Thu, 11 Oct 2018 16:21:09 +0200 Subject: [PATCH 093/113] GH-139: Add possibility to set cache directory. --- flair/embeddings.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/flair/embeddings.py b/flair/embeddings.py index 866570727f..efb3774a71 100644 --- a/flair/embeddings.py +++ b/flair/embeddings.py @@ -355,7 +355,7 @@ def _add_embeddings_internal(self, sentences: List[Sentence]): class CharLMEmbeddings(TokenEmbeddings): """Contextual string embeddings of words, as proposed in Akbik et al., 2018.""" - def __init__(self, model, detach: bool = True, use_cache: bool = True): + def __init__(self, model, detach: bool = True, use_cache: bool = True, cache_directory: str = None): """ Contextual string embeddings of words, as proposed in Akbik et al., 2018. @@ -370,6 +370,9 @@ def __init__(self, model, detach: bool = True, use_cache: bool = True): arg3 : use_cache if set to false, will not write embeddings to file for later retrieval. this saves disk space but will not allow re-use of once computed embeddings that do not fit into memory + arg3 : cache_directory + if cache_directory is not set, the cache will be written to ~/.flair/embeddings. otherwise the cache + is written to the provided directory. """ super().__init__() @@ -435,6 +438,7 @@ def __init__(self, model, detach: bool = True, use_cache: bool = True): # caching variables self.use_cache: bool = use_cache self.cache = None + self.cache_directory: str = cache_directory dummy_sentence: Sentence = Sentence() dummy_sentence.add_token(Token('hello')) @@ -465,8 +469,11 @@ def _add_embeddings_internal(self, sentences: List[Sentence]) -> List[Sentence]: # lazy initialization of cache if not self.cache: + cache_path = '{}-tmp-cache.sqllite'.format(self.name) if self.cache_directory is None else os.path.join( + self.cache_directory, '{}-tmp-cache.sqllite'.format(os.path.basename(self.name))) + from sqlitedict import SqliteDict - self.cache = SqliteDict('{}-tmp-cache.sqllite'.format(self.name), autocommit=True) + self.cache = SqliteDict(cache_path, autocommit=True) # try populating embeddings from cache all_embeddings_retrieved_from_cache: bool = True From 4e5f9dd6ff8f737602a60d6f047ad4d9ea309708 Mon Sep 17 00:00:00 2001 From: tabergma Date: Thu, 11 Oct 2018 16:21:09 +0200 Subject: [PATCH 094/113] GH-139: Add possibility to set cache directory. --- flair/embeddings.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/flair/embeddings.py b/flair/embeddings.py index 866570727f..06cf2d46a2 100644 --- a/flair/embeddings.py +++ b/flair/embeddings.py @@ -355,7 +355,7 @@ def _add_embeddings_internal(self, sentences: List[Sentence]): class CharLMEmbeddings(TokenEmbeddings): """Contextual string embeddings of words, as proposed in Akbik et al., 2018.""" - def __init__(self, model, detach: bool = True, use_cache: bool = True): + def __init__(self, model, detach: bool = True, use_cache: bool = True, cache_directory: str = None): """ Contextual string embeddings of words, as proposed in Akbik et al., 2018. @@ -370,6 +370,9 @@ def __init__(self, model, detach: bool = True, use_cache: bool = True): arg3 : use_cache if set to false, will not write embeddings to file for later retrieval. this saves disk space but will not allow re-use of once computed embeddings that do not fit into memory + arg3 : cache_directory + if cache_directory is not set, the cache will be written to ~/.flair/embeddings. otherwise the cache + is written to the provided directory. """ super().__init__() @@ -435,6 +438,7 @@ def __init__(self, model, detach: bool = True, use_cache: bool = True): # caching variables self.use_cache: bool = use_cache self.cache = None + self.cache_directory: str = cache_directory dummy_sentence: Sentence = Sentence() dummy_sentence.add_token(Token('hello')) @@ -459,14 +463,18 @@ def _add_embeddings_internal(self, sentences: List[Sentence]) -> List[Sentence]: # by default, use_cache is false (for older pre-trained models TODO: remove in version 0.4) if 'cache' not in self.__dict__: self.use_cache = False + self.cache_directory = None # if cache is used, try setting embeddings from cache first if self.use_cache: # lazy initialization of cache if not self.cache: + cache_path = '{}-tmp-cache.sqllite'.format(self.name) if self.cache_directory is None else os.path.join( + self.cache_directory, '{}-tmp-cache.sqllite'.format(os.path.basename(self.name))) + from sqlitedict import SqliteDict - self.cache = SqliteDict('{}-tmp-cache.sqllite'.format(self.name), autocommit=True) + self.cache = SqliteDict(cache_path, autocommit=True) # try populating embeddings from cache all_embeddings_retrieved_from_cache: bool = True From 9e956ee356667636b226b378ddbb8078176eff44 Mon Sep 17 00:00:00 2001 From: tabergma Date: Thu, 11 Oct 2018 16:33:06 +0200 Subject: [PATCH 095/113] GH-139: Fix bug in plotter. --- flair/visual/training_curves.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/flair/visual/training_curves.py b/flair/visual/training_curves.py index e1e4ae3468..ca0b2af496 100644 --- a/flair/visual/training_curves.py +++ b/flair/visual/training_curves.py @@ -61,15 +61,15 @@ def _extract_evaluation_data(file_name) -> dict: next(tsvin, None) for row in tsvin: - if row[TRAIN_LOSS] != '_': training_curves['train']['loss'].append(row[TRAIN_LOSS]) - if row[TRAIN_F_SCORE] != '_': training_curves['train']['f_score'].append(row[TRAIN_F_SCORE]) - if row[TRAIN_ACCURACY] != '_': training_curves['train']['acc'].append(row[TRAIN_ACCURACY]) - if row[DEV_LOSS] != '_': training_curves['dev']['loss'].append(row[DEV_LOSS]) - if row[DEV_F_SCORE] != '_': training_curves['dev']['f_score'].append(row[DEV_F_SCORE]) - if row[DEV_ACCURACY] != '_': training_curves['dev']['acc'].append(row[DEV_ACCURACY]) - if row[TEST_LOSS] != '_': training_curves['test']['loss'].append(row[TEST_LOSS]) - if row[TEST_F_SCORE] != '_': training_curves['test']['f_score'].append(row[TEST_F_SCORE]) - if row[TEST_ACCURACY] != '_': training_curves['test']['acc'].append(row[TEST_ACCURACY]) + if row[TRAIN_LOSS] != '_': training_curves['train']['loss'].append(float(row[TRAIN_LOSS])) + if row[TRAIN_F_SCORE] != '_': training_curves['train']['f_score'].append(float(row[TRAIN_F_SCORE])) + if row[TRAIN_ACCURACY] != '_': training_curves['train']['acc'].append(float(row[TRAIN_ACCURACY])) + if row[DEV_LOSS] != '_': training_curves['dev']['loss'].append(float(row[DEV_LOSS])) + if row[DEV_F_SCORE] != '_': training_curves['dev']['f_score'].append(float(row[DEV_F_SCORE])) + if row[DEV_ACCURACY] != '_': training_curves['dev']['acc'].append(float(row[DEV_ACCURACY])) + if row[TEST_LOSS] != '_': training_curves['test']['loss'].append(float(row[TEST_LOSS])) + if row[TEST_F_SCORE] != '_': training_curves['test']['f_score'].append(float(row[TEST_F_SCORE])) + if row[TEST_ACCURACY] != '_': training_curves['test']['acc'].append(float(row[TEST_ACCURACY])) return training_curves From d257b30e596f5fd63b4b1cd9f91b66229ac501ae Mon Sep 17 00:00:00 2001 From: aakbik Date: Thu, 11 Oct 2018 17:04:07 +0200 Subject: [PATCH 096/113] GH-139: documentation --- resources/docs/TUTORIAL_TRAINING_A_MODEL.md | 45 ++++++++++++++++----- 1 file changed, 36 insertions(+), 9 deletions(-) diff --git a/resources/docs/TUTORIAL_TRAINING_A_MODEL.md b/resources/docs/TUTORIAL_TRAINING_A_MODEL.md index 93bc979508..4371a3f99b 100644 --- a/resources/docs/TUTORIAL_TRAINING_A_MODEL.md +++ b/resources/docs/TUTORIAL_TRAINING_A_MODEL.md @@ -3,6 +3,9 @@ This part of the tutorial shows how you can train your own sequence labeling and text classification models using state-of-the-art word embeddings. +For this tutorial, we assume that you're familiar with the [base types](/resources/docs/TUTORIAL_BASICS.md) of this +library and how [word embeddings](/resources/docs/TUTORIAL_WORD_EMBEDDING.md) work. + ## Reading an Evaluation Dataset Flair provides helper methods to read common NLP datasets, such as the CoNLL-03 and CoNLL-2000 evaluation datasets, @@ -29,21 +32,45 @@ corpus: TaggedCorpus = NLPTaskDataFetcher.fetch_data(NLPTask.CONLL_03) This gives you a `TaggedCorpus` object that contains the data. However, this only works if the relative folder structure perfectly matches the presets. If not - or you are using -a different dataset, you can still use the inbuilt functions to read different CoNLL formats: +a different dataset, you can still use the inbuilt functions to read different CoNLL formats. + + +## Reading Any Column-Formatted Dataset + +Most sequence labeling datasets in NLP use some sort of column format in which each line is a word and each column is +one level of linguistic annotation. See for instance this sentence: + +```console +George N B-PER +Washington N I-PER +went V O +to P O +Washington N B-LOC +``` + +The first column is the word itself, the second coarse PoS tags, and the third BIO-annotated NER tags. To read such a dataset, +define the column structure as a dictionary and use a helper method. ```python -# use your own data path -data_folder = 'path/to/your/data' -# get training, test and dev data -sentences_train = NLPTaskDataFetcher.read_conll_sequence_labeling_data(data_folder + '/eng.train') -sentences_dev = NLPTaskDataFetcher.read_conll_sequence_labeling_data(data_folder + '/eng.testa') -sentences_test = NLPTaskDataFetcher.read_conll_sequence_labeling_data(data_folder + '/eng.testb') +# define columns +columns = {0: 'text', 1: 'pos', 2: 'np'} + +# this is the folder in which train, test and dev files reside +data_folder = '/path/to/data/folder' -# return corpus -return TaggedCorpus(sentences_train, sentences_dev, sentences_test) +# retrieve corpus using column format, data folder and the names of the train, dev and test files +corpus: TaggedCorpus = NLPTaskDataFetcher.fetch_column_corpus(data_folder, columns, + train_file='train.txt', + test_file='test.txt', + dev_file='dev.txt') ``` +This again gives you a `TaggedCorpus` object that contains the data. + + +## The TaggedCorpus Object + The `TaggedCorpus` contains a bunch of useful helper functions. For instance, you can downsample the data by calling `downsample()` and passing a ratio. So, if you normally get a corpus like this: From 179862ec6cfd935651ac995cb6f4874b32d63c45 Mon Sep 17 00:00:00 2001 From: aakbik Date: Thu, 11 Oct 2018 17:19:00 +0200 Subject: [PATCH 097/113] GH-139: documentation --- resources/docs/TUTORIAL_TRAINING_A_MODEL.md | 53 +++++++++------------ 1 file changed, 23 insertions(+), 30 deletions(-) diff --git a/resources/docs/TUTORIAL_TRAINING_A_MODEL.md b/resources/docs/TUTORIAL_TRAINING_A_MODEL.md index 4371a3f99b..3eea7dbd3c 100644 --- a/resources/docs/TUTORIAL_TRAINING_A_MODEL.md +++ b/resources/docs/TUTORIAL_TRAINING_A_MODEL.md @@ -6,36 +6,8 @@ classification models using state-of-the-art word embeddings. For this tutorial, we assume that you're familiar with the [base types](/resources/docs/TUTORIAL_BASICS.md) of this library and how [word embeddings](/resources/docs/TUTORIAL_WORD_EMBEDDING.md) work. -## Reading an Evaluation Dataset -Flair provides helper methods to read common NLP datasets, such as the CoNLL-03 and CoNLL-2000 evaluation datasets, -and the CoNLL-U format. These might be interesting to you if you want to train your own sequence labelers. - -All helper methods for reading data are bundled in the `NLPTaskDataFetcher` class. One option for you is to follow -the instructions for putting the training data in the appropriate folder structure, and use the prepared functions. -For instance, if you want to use the CoNLL-03 data, get it from the task web site and place train, test and dev data -in `/resources/tasks/conll_03/` as follows: - -``` -/resources/tasks/conll_03/eng.testa -/resources/tasks/conll_03/eng.testb -/resources/tasks/conll_03/eng.train -``` - -This allows the `NLPTaskDataFetcher` class to read the data into our data structures. Use the `NLPTask` enum to select -the dataset, as follows: - -```python -corpus: TaggedCorpus = NLPTaskDataFetcher.fetch_data(NLPTask.CONLL_03) -``` - -This gives you a `TaggedCorpus` object that contains the data. - -However, this only works if the relative folder structure perfectly matches the presets. If not - or you are using -a different dataset, you can still use the inbuilt functions to read different CoNLL formats. - - -## Reading Any Column-Formatted Dataset +## Reading A Column-Formatted Dataset Most sequence labeling datasets in NLP use some sort of column format in which each line is a word and each column is one level of linguistic annotation. See for instance this sentence: @@ -66,7 +38,28 @@ corpus: TaggedCorpus = NLPTaskDataFetcher.fetch_column_corpus(data_folder, colum dev_file='dev.txt') ``` -This again gives you a `TaggedCorpus` object that contains the data. +This gives you a `TaggedCorpus` object that contains the train, dev and test splits, each as a list of `Sentence`. +So, to check how many sentences there are in the training split, do + +```python +len(corpus.train) +``` + +You can also access a sentence and check out annotations. Lets assume that the first sentence in the training split is +the example sentence from above, then executing these commands + +```python +print(corpus.train[0].to_tagged_string('pos')) +print(corpus.train[0].to_tagged_string('ner')) +``` + +will print the sentence with different layers of annotation: + +```console +George Washington went to

Washington + +George Washington went to Washington . +``` ## The TaggedCorpus Object From 5bcd5d06ad768aa06f639b410354b43b884ebf9f Mon Sep 17 00:00:00 2001 From: aakbik Date: Thu, 11 Oct 2018 17:23:46 +0200 Subject: [PATCH 098/113] GH-139: documentation --- resources/docs/TUTORIAL_TRAINING_A_MODEL.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/resources/docs/TUTORIAL_TRAINING_A_MODEL.md b/resources/docs/TUTORIAL_TRAINING_A_MODEL.md index 3eea7dbd3c..d1f0202b44 100644 --- a/resources/docs/TUTORIAL_TRAINING_A_MODEL.md +++ b/resources/docs/TUTORIAL_TRAINING_A_MODEL.md @@ -146,7 +146,7 @@ tagger: SequenceTagger = SequenceTagger(hidden_size=256, # 6. initialize trainer from flair.trainers import SequenceTaggerTrainer -trainer: SequenceTaggerTrainer = SequenceTaggerTrainer(tagger, corpus, test_mode=True) +trainer: SequenceTaggerTrainer = SequenceTaggerTrainer(tagger, corpus) # 7. start training trainer.train('resources/taggers/example-ner', @@ -175,16 +175,16 @@ The AGNews corpus can be downloaded [here](https://www.di.unipi.it/~gulli/AG_cor ### Preparing the data -You need to convert the data into the following format so that the `NLPTaskDataFetcher` can read it: +We use the [FastText format](https://fasttext.cc/docs/en/supervised-tutorial.html) for text classification data, in which +each line in the file represents a text document. A document can have one or multiple labels that are defined at the beginning of the line starting with the prefix +`__label__`. This looks like this: + ```bash __label__ __label__ __label__ ``` -Here, each line in the file contains a textual document. A document can have one or multiple labels that are defined at the beginning of the line starting with the prefix -`__label__`. (If your text classification data files have a different format, feel free to add new methods to the `NLPTaskDataFetcher`.) - Point the `NLPTaskDataFetcher` to this file to convert each line to a `Sentence` object annotated with the labels. It returns a list of `Sentence`. ```python From eb534662cd020915b56f00f17c8dd0901dd4099b Mon Sep 17 00:00:00 2001 From: aakbik Date: Thu, 11 Oct 2018 17:31:55 +0200 Subject: [PATCH 099/113] GH-139: documentation --- resources/docs/TUTORIAL_TRAINING_A_MODEL.md | 30 ++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/resources/docs/TUTORIAL_TRAINING_A_MODEL.md b/resources/docs/TUTORIAL_TRAINING_A_MODEL.md index d1f0202b44..c499e0432d 100644 --- a/resources/docs/TUTORIAL_TRAINING_A_MODEL.md +++ b/resources/docs/TUTORIAL_TRAINING_A_MODEL.md @@ -257,16 +257,40 @@ plotter.plot_weights('resources/ag_news/results/weights.txt') Once the model is trained you can use it to predict the class of new sentences. Just call the `predict` method of the model. -``` +```python sentences = model.predict(Sentence('France is the current world cup winner.')) ``` The predict method adds the class labels directly to the sentences. Each label has a name and a confidence value. -``` +```python for sentence in sentences: print(sentence.labels) ``` + +## Plotting Training Curves and Weights + +Flair includes a helper method to plot training curves and weights in the neural network. +Both the `SequenceTaggerTrainer` and `TextClassificationTrainer` automatically generate a `loss.tsv` and a `weights.txt` +file in the result folder. + +After training, simple point the plotter to these files: + +```python +from flair.visual.training_curves import Plotter +plotter = Plotter() +plotter.plot_training_curves('resources/ag_news/results/loss.tsv') +plotter.plot_weights('resources/ag_news/results/weights.txt') +``` + +This generates PNG plots in the result folder. + + +## Training on Large Data Sets + + + + ## Next -You can now look into [training your own embeddings](/resources/docs/TUTORIAL_TRAINING_LM_EMBEDDINGS.md). +You can now look into [training your own embeddings](/resources/docs/TUTORIAL_TRAINING_LM_EMBEDDINGS.md). \ No newline at end of file From fad959113866df8be3d5cce8352cbcdc6ed25139 Mon Sep 17 00:00:00 2001 From: tabergma Date: Fri, 12 Oct 2018 14:51:04 +0200 Subject: [PATCH 100/113] GH-139: Update version of sklearn. --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 58bf5a5eae..c8a5d9e580 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,5 +8,5 @@ segtok==1.5.7 matplotlib==3.0.0 mpld3==0.3 jinja2==2.10 -sklearn==0.0 +sklearn sqlitedict \ No newline at end of file From bfb990582ff975fdeb12a29e3134a0e3d4bb31c8 Mon Sep 17 00:00:00 2001 From: tabergma Date: Fri, 12 Oct 2018 14:54:34 +0200 Subject: [PATCH 101/113] GH-139: Round metric values. --- flair/training_utils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/flair/training_utils.py b/flair/training_utils.py index 8ed03040a6..397d8f5c79 100644 --- a/flair/training_utils.py +++ b/flair/training_utils.py @@ -34,22 +34,22 @@ def fn(self): def precision(self): if self._tp + self._fp > 0: - return self._tp / (self._tp + self._fp) + return round(self._tp / (self._tp + self._fp), 4) return 0.0 def recall(self): if self._tp + self._fn > 0: - return self._tp / (self._tp + self._fn) + return round(self._tp / (self._tp + self._fn)) return 0.0 def f_score(self): if self.precision() + self.recall() > 0: - return 2 * (self.precision() * self.recall()) / (self.precision() + self.recall()) + return round(2 * (self.precision() * self.recall()) / (self.precision() + self.recall())) return 0.0 def accuracy(self): if self._tp + self._tn + self._fp + self._fn > 0: - return (self._tp + self._tn) / (self._tp + self._tn + self._fp + self._fn) + return round((self._tp + self._tn) / (self._tp + self._tn + self._fp + self._fn)) return 0.0 def to_tsv(self): From e2448f3b8f1822d20f8676a27e503c7147ecf6d5 Mon Sep 17 00:00:00 2001 From: aakbik Date: Fri, 12 Oct 2018 16:05:07 +0200 Subject: [PATCH 102/113] GH-139: documentation --- resources/docs/EXPERIMENTS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/docs/EXPERIMENTS.md b/resources/docs/EXPERIMENTS.md index 8f8bf60607..f2113f0547 100644 --- a/resources/docs/EXPERIMENTS.md +++ b/resources/docs/EXPERIMENTS.md @@ -39,7 +39,7 @@ The full code to get a state-of-the-art model for English NER is as follows: ```python from flair.data import TaggedCorpus from flair.data_fetcher import NLPTaskDataFetcher, NLPTask -from flair.embeddings import TokenEmbeddings, WordEmbeddings, StackedEmbeddings, CharLMEmbeddings, CharacterEmbeddings +from flair.embeddings import TokenEmbeddings, WordEmbeddings, StackedEmbeddings, CharLMEmbeddings from typing import List import torch From 8993ee6d9565ca868a685a09d0a57b1165531500 Mon Sep 17 00:00:00 2001 From: aakbik Date: Fri, 12 Oct 2018 16:10:13 +0200 Subject: [PATCH 103/113] GH-139: documentation --- resources/docs/EXPERIMENTS.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/resources/docs/EXPERIMENTS.md b/resources/docs/EXPERIMENTS.md index f2113f0547..a1bf09ca72 100644 --- a/resources/docs/EXPERIMENTS.md +++ b/resources/docs/EXPERIMENTS.md @@ -81,9 +81,13 @@ tagger: SequenceTagger = SequenceTagger(hidden_size=256, # initialize trainer from flair.trainers import SequenceTaggerTrainer -trainer: SequenceTaggerTrainer = SequenceTaggerTrainer(tagger, corpus, test_mode=False) +trainer: SequenceTaggerTrainer = SequenceTaggerTrainer(tagger, corpus) -trainer.train('resources/taggers/example-ner', learning_rate=0.1, mini_batch_size=32, max_epochs=150) +trainer.train('resources/taggers/example-ner', + learning_rate=0.1, + mini_batch_size=32, + max_epochs=150, + embeddings_in_memory=True) ``` ## Ontonotes Named Entity Recognition (English) From eb01a90712b9414cfb41921fe589af00ddf02c44 Mon Sep 17 00:00:00 2001 From: aakbik Date: Fri, 12 Oct 2018 16:11:29 +0200 Subject: [PATCH 104/113] GH-139: documentation --- resources/docs/EXPERIMENTS.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/resources/docs/EXPERIMENTS.md b/resources/docs/EXPERIMENTS.md index a1bf09ca72..2bf7522c2a 100644 --- a/resources/docs/EXPERIMENTS.md +++ b/resources/docs/EXPERIMENTS.md @@ -58,13 +58,13 @@ print(tag_dictionary.idx2item) embedding_types: List[TokenEmbeddings] = [ # GloVe embeddings - WordEmbeddings('glove') - , + WordEmbeddings('glove'), + # contextual string embeddings, forward - CharLMEmbeddings('news-forward') - , + CharLMEmbeddings('news-forward'), + # contextual string embeddings, backward - CharLMEmbeddings('news-backward') + CharLMEmbeddings('news-backward'), ] embeddings: StackedEmbeddings = StackedEmbeddings(embeddings=embedding_types) From ee091bb111b2ec179682db10d91b8071bf9c56ac Mon Sep 17 00:00:00 2001 From: aakbik Date: Fri, 12 Oct 2018 16:15:05 +0200 Subject: [PATCH 105/113] GH-139: documentation --- resources/docs/EXPERIMENTS.md | 95 ++++++++++++++++++++++------------- 1 file changed, 59 insertions(+), 36 deletions(-) diff --git a/resources/docs/EXPERIMENTS.md b/resources/docs/EXPERIMENTS.md index 2bf7522c2a..4ee1e30b07 100644 --- a/resources/docs/EXPERIMENTS.md +++ b/resources/docs/EXPERIMENTS.md @@ -59,7 +59,7 @@ embedding_types: List[TokenEmbeddings] = [ # GloVe embeddings WordEmbeddings('glove'), - + # contextual string embeddings, forward CharLMEmbeddings('news-forward'), @@ -111,7 +111,7 @@ FastText embeddings (they work better on this dataset). The full code then is as ```python from flair.data import TaggedCorpus from flair.data_fetcher import NLPTaskDataFetcher, NLPTask -from flair.embeddings import TokenEmbeddings, WordEmbeddings, StackedEmbeddings, CharLMEmbeddings, CharacterEmbeddings +from flair.embeddings import TokenEmbeddings, WordEmbeddings, StackedEmbeddings, CharLMEmbeddings from typing import List import torch @@ -129,11 +129,11 @@ print(tag_dictionary.idx2item) # initialize embeddings embedding_types: List[TokenEmbeddings] = [ - WordEmbeddings('crawl') - , - CharLMEmbeddings('news-forward') - , - CharLMEmbeddings('news-backward') + WordEmbeddings('crawl'), + + CharLMEmbeddings('news-forward'), + + CharLMEmbeddings('news-backward'), ] embeddings: StackedEmbeddings = StackedEmbeddings(embeddings=embedding_types) @@ -152,7 +152,12 @@ from flair.trainers import SequenceTaggerTrainer trainer: SequenceTaggerTrainer = SequenceTaggerTrainer(tagger, corpus, test_mode=False) -trainer.train('resources/taggers/example-ner', learning_rate=0.1, mini_batch_size=32, max_epochs=150) +trainer.train('resources/taggers/example-ner', + learning_rate=0.1, + mini_batch_size=32, + max_epochs=150, + # it's a big dataset so maybe set embeddings_in_memory to False + embeddings_in_memory=False) ``` ## CoNLL-03 Named Entity Recognition (German) @@ -173,7 +178,7 @@ FastText word embeddings and German contextual string embeddings. The full code ```python from flair.data import TaggedCorpus from flair.data_fetcher import NLPTaskDataFetcher, NLPTask -from flair.embeddings import TokenEmbeddings, WordEmbeddings, StackedEmbeddings, CharLMEmbeddings, CharacterEmbeddings +from flair.embeddings import TokenEmbeddings, WordEmbeddings, StackedEmbeddings, CharLMEmbeddings from typing import List import torch @@ -191,11 +196,11 @@ print(tag_dictionary.idx2item) # initialize embeddings embedding_types: List[TokenEmbeddings] = [ - WordEmbeddings('de-fasttext') - , - CharLMEmbeddings('german-forward') - , - CharLMEmbeddings('german-backward') + WordEmbeddings('de-fasttext'), + + CharLMEmbeddings('german-forward'), + + CharLMEmbeddings('german-backward'), ] embeddings: StackedEmbeddings = StackedEmbeddings(embeddings=embedding_types) @@ -214,7 +219,11 @@ from flair.trainers import SequenceTaggerTrainer trainer: SequenceTaggerTrainer = SequenceTaggerTrainer(tagger, corpus, test_mode=False) -trainer.train('resources/taggers/example-ner', learning_rate=0.1, mini_batch_size=32, max_epochs=150) +trainer.train('resources/taggers/example-ner', + learning_rate=0.1, + mini_batch_size=32, + max_epochs=150, + embeddings_in_memory=True) ``` ## Germeval Named Entity Recognition (German) @@ -235,7 +244,7 @@ Once you have the data, reproduce our experiments exactly like for the German Co ```python from flair.data import TaggedCorpus from flair.data_fetcher import NLPTaskDataFetcher, NLPTask -from flair.embeddings import TokenEmbeddings, WordEmbeddings, StackedEmbeddings, CharLMEmbeddings, CharacterEmbeddings +from flair.embeddings import TokenEmbeddings, WordEmbeddings, StackedEmbeddings, CharLMEmbeddings from typing import List import torch @@ -253,11 +262,11 @@ print(tag_dictionary.idx2item) # initialize embeddings embedding_types: List[TokenEmbeddings] = [ - WordEmbeddings('de-fasttext') - , - CharLMEmbeddings('german-forward') - , - CharLMEmbeddings('german-backward') + WordEmbeddings('de-fasttext'), + + CharLMEmbeddings('german-forward'), + + CharLMEmbeddings('german-backward'), ] embeddings: StackedEmbeddings = StackedEmbeddings(embeddings=embedding_types) @@ -276,7 +285,11 @@ from flair.trainers import SequenceTaggerTrainer trainer: SequenceTaggerTrainer = SequenceTaggerTrainer(tagger, corpus, test_mode=False) -trainer.train('resources/taggers/example-ner', learning_rate=0.1, mini_batch_size=32, max_epochs=150) +trainer.train('resources/taggers/example-ner', + learning_rate=0.1, + mini_batch_size=32, + max_epochs=150, + embeddings_in_memory=True) ``` ## Penn Treebank Part-of-Speech Tagging (English) @@ -298,7 +311,7 @@ so the algorithm knows that POS tags and not NER are to be predicted from this d ```python from flair.data import TaggedCorpus from flair.data_fetcher import NLPTaskDataFetcher, NLPTask -from flair.embeddings import TokenEmbeddings, WordEmbeddings, StackedEmbeddings, CharLMEmbeddings, CharacterEmbeddings +from flair.embeddings import TokenEmbeddings, WordEmbeddings, StackedEmbeddings, CharLMEmbeddings from typing import List import torch @@ -316,11 +329,11 @@ print(tag_dictionary.idx2item) # initialize embeddings embedding_types: List[TokenEmbeddings] = [ - WordEmbeddings('extvec') - , - CharLMEmbeddings('news-forward') - , - CharLMEmbeddings('news-backward') + WordEmbeddings('extvec'), + + CharLMEmbeddings('news-forward'), + + CharLMEmbeddings('news-backward'), ] embeddings: StackedEmbeddings = StackedEmbeddings(embeddings=embedding_types) @@ -338,7 +351,12 @@ from flair.trainers import SequenceTaggerTrainer trainer: SequenceTaggerTrainer = SequenceTaggerTrainer(tagger, corpus, test_mode=False) -trainer.train('resources/taggers/example-ner', learning_rate=0.1, mini_batch_size=32, max_epochs=150) +trainer.train('resources/taggers/example-ner', + learning_rate=0.1, + mini_batch_size=32, + max_epochs=150, + # its a bit dataset, so maybe set embeddings_in_memory=False + embeddings_in_memory=True) ``` ## CoNLL-2000 Noun Phrase Chunking (English) @@ -360,7 +378,7 @@ so the algorithm knows that chunking tags and not NER are to be predicted from t ```python from flair.data import TaggedCorpus from flair.data_fetcher import NLPTaskDataFetcher, NLPTask -from flair.embeddings import TokenEmbeddings, WordEmbeddings, StackedEmbeddings, CharLMEmbeddings, CharacterEmbeddings +from flair.embeddings import TokenEmbeddings, WordEmbeddings, StackedEmbeddings, CharLMEmbeddings from typing import List import torch @@ -378,11 +396,11 @@ print(tag_dictionary.idx2item) # initialize embeddings embedding_types: List[TokenEmbeddings] = [ - WordEmbeddings('extvec') - , - CharLMEmbeddings('news-forward') - , - CharLMEmbeddings('news-backward') + WordEmbeddings('extvec'), + + CharLMEmbeddings('news-forward'), + + CharLMEmbeddings('news-backward'), ] embeddings: StackedEmbeddings = StackedEmbeddings(embeddings=embedding_types) @@ -401,5 +419,10 @@ from flair.trainers import SequenceTaggerTrainer trainer: SequenceTaggerTrainer = SequenceTaggerTrainer(tagger, corpus, test_mode=False) -trainer.train('resources/taggers/example-ner', learning_rate=0.1, mini_batch_size=32, max_epochs=150) +trainer.train('resources/taggers/example-ner', + learning_rate=0.1, + mini_batch_size=32, + max_epochs=150, + # its a bit dataset, so maybe set embeddings_in_memory=False + embeddings_in_memory=True) ``` \ No newline at end of file From 0b020cb32467c10e50254abd420675b13d38ff1c Mon Sep 17 00:00:00 2001 From: aakbik Date: Fri, 12 Oct 2018 16:15:51 +0200 Subject: [PATCH 106/113] GH-139: documentation --- resources/docs/EXPERIMENTS.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/resources/docs/EXPERIMENTS.md b/resources/docs/EXPERIMENTS.md index 4ee1e30b07..5a953a9d8b 100644 --- a/resources/docs/EXPERIMENTS.md +++ b/resources/docs/EXPERIMENTS.md @@ -397,7 +397,7 @@ print(tag_dictionary.idx2item) embedding_types: List[TokenEmbeddings] = [ WordEmbeddings('extvec'), - + CharLMEmbeddings('news-forward'), CharLMEmbeddings('news-backward'), @@ -423,6 +423,5 @@ trainer.train('resources/taggers/example-ner', learning_rate=0.1, mini_batch_size=32, max_epochs=150, - # its a bit dataset, so maybe set embeddings_in_memory=False embeddings_in_memory=True) ``` \ No newline at end of file From fa95008b1b6378f48c5ee557a91a3a0f5bc800d2 Mon Sep 17 00:00:00 2001 From: aakbik Date: Fri, 12 Oct 2018 16:33:48 +0200 Subject: [PATCH 107/113] GH-139: scalability documentation --- resources/docs/TUTORIAL_TRAINING_A_MODEL.md | 32 +++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/resources/docs/TUTORIAL_TRAINING_A_MODEL.md b/resources/docs/TUTORIAL_TRAINING_A_MODEL.md index c499e0432d..6ed351f0bb 100644 --- a/resources/docs/TUTORIAL_TRAINING_A_MODEL.md +++ b/resources/docs/TUTORIAL_TRAINING_A_MODEL.md @@ -201,7 +201,7 @@ sentences: List[Sentence] = NLPTaskDataFetcher.read_text_classification_file(dat ### Training the classifier To train a model, you need to create three files in this way: A train, dev and test file. After you converted the data you can use the -`NLPTask.AG_NEWS` to read the data, if the data files are located in the following folder structure: +`NLPTaskDataFetcher` to read the data, if the data files are located in the following folder structure: ``` /resources/tasks/ag_news/train.txt /resources/tasks/ag_news/dev.txt @@ -286,8 +286,36 @@ plotter.plot_weights('resources/ag_news/results/weights.txt') This generates PNG plots in the result folder. -## Training on Large Data Sets +## Scalability: Training on Large Data Sets +The main thing to consider when using `CharLMEmbeddings` (which you should) is that they are +somewhat costly to generate for large training data sets. Depending on your setup, you can +set options to optimize training time. There are three questions to ask: + +1. Do you have a GPU? + +`CharLMEmbeddings` are generated using Pytorch RNNs and are thus optimized for GPUs. If you have one, +you can set large mini-batch sizes to make use of batching. If not, you may want to use smaller language models. +For English, we package '-fast' variants of our embeddings, loadable like this: `CharLMEmbeddings('news-forward-fast')`. + +Regardless, all computed embeddings get materialized to disk upon first computation. This means that if you rerun an +experiment on the same dataset, they will be retrieved from disk instead of re-computed, potentially saving a lot +of time. + +2. Do embeddings for the entire dataset fit into memory? + +In the best-case scenario, all embeddings for the dataset fit into your regular memory, which greatly increases +training speed. If this is not the case, you must set the flag `embeddings_in_memory=False` in the trainer to +avoid memory problems. With the flag, embeddings are either (a) recomputed at each epoch or (b) +retrieved from disk (where they are materialized by default). The second option is the default and is typically +much faster. + +3. Do you have a fast hard drive? + +You benefit the most from the default behavior of storing computed embeddings on disk for later retrieval +if your disk is large and fast. If you either do not have a lot of disk space, or a really slow hard drive, +you should disable this option. You can do this when instantiating the embeddings by setting `use_cache=False`, + i.e. `CharLMEmbeddings('news-forward-fast', use_cache=False')` From 4d72868edd78985973ed55a7780ee6a3f031ece5 Mon Sep 17 00:00:00 2001 From: aakbik Date: Fri, 12 Oct 2018 16:36:17 +0200 Subject: [PATCH 108/113] GH-139: scalability documentation --- resources/docs/TUTORIAL_TRAINING_A_MODEL.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/resources/docs/TUTORIAL_TRAINING_A_MODEL.md b/resources/docs/TUTORIAL_TRAINING_A_MODEL.md index 6ed351f0bb..9564840f05 100644 --- a/resources/docs/TUTORIAL_TRAINING_A_MODEL.md +++ b/resources/docs/TUTORIAL_TRAINING_A_MODEL.md @@ -296,7 +296,7 @@ set options to optimize training time. There are three questions to ask: `CharLMEmbeddings` are generated using Pytorch RNNs and are thus optimized for GPUs. If you have one, you can set large mini-batch sizes to make use of batching. If not, you may want to use smaller language models. -For English, we package '-fast' variants of our embeddings, loadable like this: `CharLMEmbeddings('news-forward-fast')`. +For English, we package 'fast' variants of our embeddings, loadable like this: `CharLMEmbeddings('news-forward-fast')`. Regardless, all computed embeddings get materialized to disk upon first computation. This means that if you rerun an experiment on the same dataset, they will be retrieved from disk instead of re-computed, potentially saving a lot @@ -305,7 +305,8 @@ of time. 2. Do embeddings for the entire dataset fit into memory? In the best-case scenario, all embeddings for the dataset fit into your regular memory, which greatly increases -training speed. If this is not the case, you must set the flag `embeddings_in_memory=False` in the trainer to +training speed. If this is not the case, you must set the flag `embeddings_in_memory=False` in the respective trainer + (i.e. `SequenceTaggerTrainer` or `TextClassifierTrainer`) to avoid memory problems. With the flag, embeddings are either (a) recomputed at each epoch or (b) retrieved from disk (where they are materialized by default). The second option is the default and is typically much faster. From 226351b0bd9d41d5479006af26051d3fc2a952e0 Mon Sep 17 00:00:00 2001 From: aakbik Date: Fri, 12 Oct 2018 16:37:08 +0200 Subject: [PATCH 109/113] GH-139: scalability documentation --- resources/docs/TUTORIAL_TRAINING_A_MODEL.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/resources/docs/TUTORIAL_TRAINING_A_MODEL.md b/resources/docs/TUTORIAL_TRAINING_A_MODEL.md index 9564840f05..87889e2bd5 100644 --- a/resources/docs/TUTORIAL_TRAINING_A_MODEL.md +++ b/resources/docs/TUTORIAL_TRAINING_A_MODEL.md @@ -315,8 +315,8 @@ much faster. You benefit the most from the default behavior of storing computed embeddings on disk for later retrieval if your disk is large and fast. If you either do not have a lot of disk space, or a really slow hard drive, -you should disable this option. You can do this when instantiating the embeddings by setting `use_cache=False`, - i.e. `CharLMEmbeddings('news-forward-fast', use_cache=False')` +you should disable this option. You can do this when instantiating the embeddings by setting `use_cache=False`. So +instantiate like this: `CharLMEmbeddings('news-forward-fast', use_cache=False')` From 3cdd9806af04fa4f33791a629b93c1e7f28de3c2 Mon Sep 17 00:00:00 2001 From: aakbik Date: Sat, 13 Oct 2018 14:16:51 +0200 Subject: [PATCH 110/113] GH-139: fix rounding error in Metric class --- flair/training_utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flair/training_utils.py b/flair/training_utils.py index 397d8f5c79..b84fe22103 100644 --- a/flair/training_utils.py +++ b/flair/training_utils.py @@ -39,17 +39,17 @@ def precision(self): def recall(self): if self._tp + self._fn > 0: - return round(self._tp / (self._tp + self._fn)) + return round(self._tp / (self._tp + self._fn), 4) return 0.0 def f_score(self): if self.precision() + self.recall() > 0: - return round(2 * (self.precision() * self.recall()) / (self.precision() + self.recall())) + return round(2 * (self.precision() * self.recall()) / (self.precision() + self.recall()), 4) return 0.0 def accuracy(self): if self._tp + self._tn + self._fp + self._fn > 0: - return round((self._tp + self._tn) / (self._tp + self._tn + self._fp + self._fn)) + return round((self._tp + self._tn) / (self._tp + self._tn + self._fp + self._fn), 4) return 0.0 def to_tsv(self): From beffa4a32947d0a7a0afbb431bb65e201e4ac757 Mon Sep 17 00:00:00 2001 From: aakbik Date: Sat, 13 Oct 2018 20:49:20 +0200 Subject: [PATCH 111/113] GH-139: allow for one-character tags --- flair/data.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/flair/data.py b/flair/data.py index a2d3ab9455..66684426f6 100644 --- a/flair/data.py +++ b/flair/data.py @@ -351,7 +351,8 @@ def get_spans(self, tag_type: str, min_score=-1) -> List[Span]: tag_value = tag.value # non-set tags are OUT tags - if len(tag_value) < 2: tag_value = 'O-' + if tag_value == '' or tag_value == 'O': + tag_value = 'O-' # anything that is not a BIOES tag is a SINGLE tag if tag_value[0:2] not in ['B-', 'I-', 'O-', 'E-', 'S-']: From e7de71d014a9f86f6061f0ca0ed3efc441067ca8 Mon Sep 17 00:00:00 2001 From: tabergma Date: Mon, 15 Oct 2018 09:06:10 +0200 Subject: [PATCH 112/113] GH-139: Make finding word offset more robust. --- flair/data.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/flair/data.py b/flair/data.py index 66684426f6..5a6913e811 100644 --- a/flair/data.py +++ b/flair/data.py @@ -300,14 +300,19 @@ def __init__(self, text: str = None, use_tokenizer: bool = False, labels: Union[ last_word_offset = -1 last_token = None for word in tokens: - token = Token(word, start_position=index(word, running_offset)) - self.add_token(token) try: word_offset = index(word, running_offset) + start_position = word_offset except: word_offset = last_word_offset + 1 + start_position = running_offset + 1 if running_offset > 0 else running_offset + + token = Token(word, start_position=start_position) + self.add_token(token) + if word_offset - 1 == last_word_offset and last_token is not None: last_token.whitespace_after = False + word_len = len(word) running_offset = word_offset + word_len last_word_offset = running_offset - 1 @@ -319,7 +324,12 @@ def __init__(self, text: str = None, use_tokenizer: bool = False, labels: Union[ offset = 0 for word in text.split(' '): if word: - token = Token(word, start_position=text.index(word, offset)) + try: + word_offset = text.index(word, offset) + except: + word_offset = offset + + token = Token(word, start_position=word_offset) self.add_token(token) offset += len(word) + 1 From bfa441b4f9be006bac1be77d45cd5093bc9be0c9 Mon Sep 17 00:00:00 2001 From: aakbik Date: Tue, 16 Oct 2018 15:21:59 +0200 Subject: [PATCH 113/113] GH-139: update setup and requirements for release 0.3 --- requirements.txt | 2 +- setup.py | 11 ++++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index c8a5d9e580..49eceb683c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,4 +9,4 @@ matplotlib==3.0.0 mpld3==0.3 jinja2==2.10 sklearn -sqlitedict \ No newline at end of file +sqlitedict==1.6.0 \ No newline at end of file diff --git a/setup.py b/setup.py index 1a337088b6..b86fbc47bc 100644 --- a/setup.py +++ b/setup.py @@ -2,9 +2,9 @@ setup( name='flair', - version='0.2.0.post1', + version='0.3.0', description='A very simple framework for state-of-the-art NLP', - long_description=open("README.md",encoding='utf-8').read(), + long_description=open("README.md", encoding='utf-8').read(), long_description_content_type="text/markdown", author='Alan Akbik', author_email='alan.akbik@zalando.de', @@ -17,7 +17,12 @@ 'gensim==3.4.0', 'typing==3.6.4', 'tqdm==4.23.4', - 'segtok==1.5.6' + 'segtok==1.5.6', + 'matplotlib==3.0.0', + 'mpld3==0.3', + 'jinja2==2.10', + 'sklearn', + 'sqlitedict==1.6.0', ], include_package_data=True, python_requires='>=3.6',