From 1974b67227dd28885bf95f50b040283dae710f90 Mon Sep 17 00:00:00 2001 From: Enliven26 <16521443@mahasiswa.itb.ac.id> Date: Sun, 25 Feb 2024 00:27:07 +0700 Subject: [PATCH 1/9] feat: id_sentiment_analysis dataloader --- .../id_sentiment_analysis.py | 154 ++++++++++++++++++ 1 file changed, 154 insertions(+) create mode 100644 seacrowd/sea_datasets/id_sentiment_analysis/id_sentiment_analysis.py diff --git a/seacrowd/sea_datasets/id_sentiment_analysis/id_sentiment_analysis.py b/seacrowd/sea_datasets/id_sentiment_analysis/id_sentiment_analysis.py new file mode 100644 index 000000000..3430206d7 --- /dev/null +++ b/seacrowd/sea_datasets/id_sentiment_analysis/id_sentiment_analysis.py @@ -0,0 +1,154 @@ +# coding=utf-8 +# Copyright 2022 The HuggingFace Datasets Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Dict, List, Tuple + +import datasets +import pandas as pd + +from seacrowd.utils import schemas +from seacrowd.utils.configs import SEACrowdConfig +from seacrowd.utils.constants import TASK_TO_SCHEMA, Licenses, Tasks + +_CITATION = """\ +""" + +_DATASETNAME = "id_sentiment_analysis" + +_DESCRIPTION = """\ +This dataset consists of 10806 labeled Indonesian tweets with their corresponding sentiment analysis: positive, negative, and neutral, up to 2019. +This dataset was developed in Cloud Experience Research Group, Gadjah Mada University. +There is no further explanation of the dataset. Contributor found this dataset after skimming through "Sentiment analysis of Indonesian datasets based on a hybrid deep-learning strategy" (Lin CH and Nuha U, 2023). +""" + +_HOMEPAGE = "https://github.com/ridife/dataset-idsa/blob/master/Indonesian%20Sentiment%20Twitter%20Dataset%20Labeled.csv" + +_LANGUAGES = ["ind"] # We follow ISO639-3 language code (https://iso639-3.sil.org/code_tables/639/data) + +_LICENSE = Licenses.CC_BY_NC_4_0.value + +_LOCAL = False + +_URLS = { + _DATASETNAME: "https://raw.githubusercontent.com/ridife/dataset-idsa/master/Indonesian%20Sentiment%20Twitter%20Dataset%20Labeled.csv", +} + +_SUPPORTED_TASKS = [Tasks.SENTIMENT_ANALYSIS] +_SUPPORTED_SCHEMA_STRINGS = [f"seacrowd_{str(TASK_TO_SCHEMA[task]).lower()}" for task in _SUPPORTED_TASKS] + +_SOURCE_VERSION = "1.0.0" + +_SEACROWD_VERSION = "1.0.0" + + +class IdSentimentAnalysis(datasets.GeneratorBasedBuilder): + """This dataset consists of 10806 labeled Indonesian tweets with their corresponding sentiment analysis: positive, negative, and neutral, up to 2019.""" + + SOURCE_VERSION = datasets.Version(_SOURCE_VERSION) + SEACROWD_VERSION = datasets.Version(_SEACROWD_VERSION) + + BUILDER_CONFIGS = [ + SEACrowdConfig( + name=f"{_DATASETNAME}_source", + version=SOURCE_VERSION, + description=f"{_DATASETNAME} source schema", + schema="source", + subset_id=f"{_DATASETNAME}", + ), + ] + + seacrowd_schema_config: list[SEACrowdConfig] = [] + + for seacrowd_schema in _SUPPORTED_SCHEMA_STRINGS: + + seacrowd_schema_config.append( + SEACrowdConfig( + name=f"{_DATASETNAME}_{seacrowd_schema}", + version=SEACROWD_VERSION, + description=f"{_DATASETNAME} {seacrowd_schema} schema", + schema=f"{seacrowd_schema}", + subset_id=f"{_DATASETNAME}", + ) + ) + + BUILDER_CONFIGS.extend(seacrowd_schema_config) + + DEFAULT_CONFIG_NAME = f"{_DATASETNAME}_source" + + def _info(self) -> datasets.DatasetInfo: + + if self.config.schema == "source": + features = datasets.Features( + { + "sentimen": datasets.Value("int32"), + "Tweet": datasets.Value("string"), + } + ) + + elif self.config.schema == f"seacrowd_{str(TASK_TO_SCHEMA[Tasks.SENTIMENT_ANALYSIS]).lower()}": + features = schemas.text_features(label_names=[1, -1, 0]) + + else: + raise ValueError(f"Invalid config: {self.config.name}") + + return datasets.DatasetInfo( + description=_DESCRIPTION, + features=features, + homepage=_HOMEPAGE, + license=_LICENSE, + citation=_CITATION, + ) + + def _split_generators(self, dl_manager: datasets.DownloadManager) -> List[datasets.SplitGenerator]: + """Returns SplitGenerators.""" + + path = dl_manager.download_and_extract(_URLS[_DATASETNAME]) + + print(path) + + return [ + datasets.SplitGenerator( + name=datasets.Split.TRAIN, + gen_kwargs={ + "path": path, + }, + ), + ] + + def _generate_examples(self, path: str) -> Tuple[int, Dict]: + """Yields examples as (key, example) tuples.""" + + idx = 0 + + if self.config.schema == "source": + df = pd.read_csv(path, delimiter="\t") + + for _, row in df.iterrows(): + yield idx, row.to_dict() + idx += 1 + + elif self.config.schema == f"seacrowd_{str(TASK_TO_SCHEMA[Tasks.SENTIMENT_ANALYSIS]).lower()}": + df = pd.read_csv(path, delimiter="\t") + + df["id"] = df.index + df.rename(columns={"sentimen": "label"}, inplace=True) + df.rename(columns={"Tweet": "text"}, inplace=True) + + for _, row in df.iterrows(): + yield idx, row.to_dict() + idx += 1 + + else: + raise ValueError(f"Invalid config: {self.config.name}") From 502cf991d53901313ecd2baf4975c9a1fab892ff Mon Sep 17 00:00:00 2001 From: Enliven26 <16521443@mahasiswa.itb.ac.id> Date: Sun, 25 Feb 2024 02:23:57 +0700 Subject: [PATCH 2/9] refactor: remove print --- .../sea_datasets/id_sentiment_analysis/id_sentiment_analysis.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/seacrowd/sea_datasets/id_sentiment_analysis/id_sentiment_analysis.py b/seacrowd/sea_datasets/id_sentiment_analysis/id_sentiment_analysis.py index 3430206d7..6d2cdd1ff 100644 --- a/seacrowd/sea_datasets/id_sentiment_analysis/id_sentiment_analysis.py +++ b/seacrowd/sea_datasets/id_sentiment_analysis/id_sentiment_analysis.py @@ -116,8 +116,6 @@ def _split_generators(self, dl_manager: datasets.DownloadManager) -> List[datase path = dl_manager.download_and_extract(_URLS[_DATASETNAME]) - print(path) - return [ datasets.SplitGenerator( name=datasets.Split.TRAIN, From 985b0036a7b34e087fd2ca09c82b0bb2c76085cc Mon Sep 17 00:00:00 2001 From: Enliven26 <16521443@mahasiswa.itb.ac.id> Date: Tue, 27 Feb 2024 11:22:38 +0700 Subject: [PATCH 3/9] Create __init__.py --- seacrowd/sea_datasets/id_sentiment_analysis/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 seacrowd/sea_datasets/id_sentiment_analysis/__init__.py diff --git a/seacrowd/sea_datasets/id_sentiment_analysis/__init__.py b/seacrowd/sea_datasets/id_sentiment_analysis/__init__.py new file mode 100644 index 000000000..e69de29bb From cba226810d0e6113f21ca418706301e8ea064766 Mon Sep 17 00:00:00 2001 From: Enliven26 <16521443@mahasiswa.itb.ac.id> Date: Mon, 4 Mar 2024 15:40:45 +0700 Subject: [PATCH 4/9] refactor: citation --- .../id_sentiment_analysis/id_sentiment_analysis.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/seacrowd/sea_datasets/id_sentiment_analysis/id_sentiment_analysis.py b/seacrowd/sea_datasets/id_sentiment_analysis/id_sentiment_analysis.py index 6d2cdd1ff..510eef477 100644 --- a/seacrowd/sea_datasets/id_sentiment_analysis/id_sentiment_analysis.py +++ b/seacrowd/sea_datasets/id_sentiment_analysis/id_sentiment_analysis.py @@ -23,6 +23,14 @@ from seacrowd.utils.constants import TASK_TO_SCHEMA, Licenses, Tasks _CITATION = """\ +@misc{ridife2019idsa, + author = {Fe, Ridi}, + title = {Indonesia Sentiment Analysis Dataset}, + year = {2019}, + publisher = {GitHub}, + journal = {GitHub repository}, + howpublished = {\url{https://github.com/ridife/dataset-idsa}} +} """ _DATASETNAME = "id_sentiment_analysis" From 246ef7337ee0bf2f9276965fc55a891d25f36a02 Mon Sep 17 00:00:00 2001 From: Enliven26 <16521443@mahasiswa.itb.ac.id> Date: Thu, 7 Mar 2024 09:11:35 +0700 Subject: [PATCH 5/9] fix: change lincense to unknown --- .../sea_datasets/id_sentiment_analysis/id_sentiment_analysis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seacrowd/sea_datasets/id_sentiment_analysis/id_sentiment_analysis.py b/seacrowd/sea_datasets/id_sentiment_analysis/id_sentiment_analysis.py index 510eef477..aa8029d05 100644 --- a/seacrowd/sea_datasets/id_sentiment_analysis/id_sentiment_analysis.py +++ b/seacrowd/sea_datasets/id_sentiment_analysis/id_sentiment_analysis.py @@ -45,7 +45,7 @@ _LANGUAGES = ["ind"] # We follow ISO639-3 language code (https://iso639-3.sil.org/code_tables/639/data) -_LICENSE = Licenses.CC_BY_NC_4_0.value +_LICENSE = Licenses.UNKNOWN.value _LOCAL = False From d1eb72f15dae5182f420f92d92ad8270c1449bbe Mon Sep 17 00:00:00 2001 From: Enliven26 <16521443@mahasiswa.itb.ac.id> Date: Thu, 7 Mar 2024 09:14:12 +0700 Subject: [PATCH 6/9] fix: minor errors --- .../id_sentiment_analysis/id_sentiment_analysis.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/seacrowd/sea_datasets/id_sentiment_analysis/id_sentiment_analysis.py b/seacrowd/sea_datasets/id_sentiment_analysis/id_sentiment_analysis.py index aa8029d05..95ce04bc0 100644 --- a/seacrowd/sea_datasets/id_sentiment_analysis/id_sentiment_analysis.py +++ b/seacrowd/sea_datasets/id_sentiment_analysis/id_sentiment_analysis.py @@ -29,7 +29,7 @@ year = {2019}, publisher = {GitHub}, journal = {GitHub repository}, - howpublished = {\url{https://github.com/ridife/dataset-idsa}} + howpublished = {\\url{https://github.com/ridife/dataset-idsa}} } """ @@ -77,7 +77,7 @@ class IdSentimentAnalysis(datasets.GeneratorBasedBuilder): ), ] - seacrowd_schema_config: list[SEACrowdConfig] = [] + seacrowd_schema_config: List[SEACrowdConfig] = [] for seacrowd_schema in _SUPPORTED_SCHEMA_STRINGS: From 19822679c155e82a2292e9edc73add12f91812e2 Mon Sep 17 00:00:00 2001 From: Johanes Lee <89065724+Enliven26@users.noreply.github.com> Date: Sat, 9 Mar 2024 19:55:00 +0700 Subject: [PATCH 7/9] refactor: feature naming Co-authored-by: Salsabil Maulana Akbar --- .../sea_datasets/id_sentiment_analysis/id_sentiment_analysis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seacrowd/sea_datasets/id_sentiment_analysis/id_sentiment_analysis.py b/seacrowd/sea_datasets/id_sentiment_analysis/id_sentiment_analysis.py index 95ce04bc0..f1f4e6916 100644 --- a/seacrowd/sea_datasets/id_sentiment_analysis/id_sentiment_analysis.py +++ b/seacrowd/sea_datasets/id_sentiment_analysis/id_sentiment_analysis.py @@ -101,7 +101,7 @@ def _info(self) -> datasets.DatasetInfo: features = datasets.Features( { "sentimen": datasets.Value("int32"), - "Tweet": datasets.Value("string"), + "tweet": datasets.Value("string"), } ) From bbb0c506c691be8b2a6b7a803e469b2306aa6aee Mon Sep 17 00:00:00 2001 From: Enliven26 <16521443@mahasiswa.itb.ac.id> Date: Sun, 10 Mar 2024 12:12:44 +0700 Subject: [PATCH 8/9] fix: homepage url --- .../sea_datasets/id_sentiment_analysis/id_sentiment_analysis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seacrowd/sea_datasets/id_sentiment_analysis/id_sentiment_analysis.py b/seacrowd/sea_datasets/id_sentiment_analysis/id_sentiment_analysis.py index 95ce04bc0..c851507bd 100644 --- a/seacrowd/sea_datasets/id_sentiment_analysis/id_sentiment_analysis.py +++ b/seacrowd/sea_datasets/id_sentiment_analysis/id_sentiment_analysis.py @@ -41,7 +41,7 @@ There is no further explanation of the dataset. Contributor found this dataset after skimming through "Sentiment analysis of Indonesian datasets based on a hybrid deep-learning strategy" (Lin CH and Nuha U, 2023). """ -_HOMEPAGE = "https://github.com/ridife/dataset-idsa/blob/master/Indonesian%20Sentiment%20Twitter%20Dataset%20Labeled.csv" +_HOMEPAGE = "https://ridi.staff.ugm.ac.id/2019/03/06/indonesia-sentiment-analysis-dataset/" _LANGUAGES = ["ind"] # We follow ISO639-3 language code (https://iso639-3.sil.org/code_tables/639/data) From 2dbb3fe9f831cc9f2b8192d65ab47f7754f49cfa Mon Sep 17 00:00:00 2001 From: Enliven26 <16521443@mahasiswa.itb.ac.id> Date: Mon, 11 Mar 2024 11:08:29 +0700 Subject: [PATCH 9/9] fix: lowercase feature name --- .../sea_datasets/id_sentiment_analysis/id_sentiment_analysis.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/seacrowd/sea_datasets/id_sentiment_analysis/id_sentiment_analysis.py b/seacrowd/sea_datasets/id_sentiment_analysis/id_sentiment_analysis.py index 799dbdeda..626eb9646 100644 --- a/seacrowd/sea_datasets/id_sentiment_analysis/id_sentiment_analysis.py +++ b/seacrowd/sea_datasets/id_sentiment_analysis/id_sentiment_analysis.py @@ -141,6 +141,8 @@ def _generate_examples(self, path: str) -> Tuple[int, Dict]: if self.config.schema == "source": df = pd.read_csv(path, delimiter="\t") + df.rename(columns={"Tweet": "tweet"}, inplace=True) + for _, row in df.iterrows(): yield idx, row.to_dict() idx += 1