Fix tokenization (fixes facebookresearch#926) (facebookresearch#929)

Summary:
Fixes facebookresearch#926
Pull Request resolved: facebookresearch#929

Differential Revision: D16560281

Pulled By: myleott

fbshipit-source-id: 751051bcdbf25207315bb05f5bee0235d21be627

fairseq/models/roberta/hub_interface.py

@@ -36,8 +36,8 @@ def device(self):
     def encode(self, sentence: str, *addl_sentences) -> torch.LongTensor:
         bpe_sentence = '<s> ' + self.bpe.encode(sentence) + ' </s>'
         for s in addl_sentences:
-            bpe_sentence += ' </s> ' + self.bpe.encode(s)
-        tokens = self.task.source_dictionary.encode_line(bpe_sentence, append_eos=True)
+            bpe_sentence += ' </s> ' + self.bpe.encode(s) + ' </s>'
+        tokens = self.task.source_dictionary.encode_line(bpe_sentence, append_eos=False)
         return tokens.long()
 
     def extract_features(self, tokens: torch.LongTensor, return_all_hiddens=False) -> torch.Tensor: