generate.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'system_message': 'You are reviewing the paper titled Learning SO(3)-Invariant Correspondence via Point-wise Local Shape Transform. The keywords are Point cloud understanding, 3D dense correspondence, SO(3)-invariance, Part label transfer. You will read this paper and write a review for it.', 'prompt': 'The abstract is：\\n\\nEstablishing accurate dense 3D correspondences between diverse shapes stands as a pivotal challenge with profound implications for computer vision and robotics. However, existing self-supervised methods for this problem assume perfect input shape alignment, restricting their real-world applicability. In this work, we introduce a novel self-supervised SO(3)-invariant 3D correspondence learner, dubbed LSTNet, that learns to establish dense correspondences between shapes even under challenging intra-class variations. Specifically, LSTNet learns to dynamically formulate an SO(3)-invariant local shape transform for each point, which maps the SO(3)-equivariant global shape descriptor of the input shape to a local shape descriptor. These local shape descriptors are provided as inputs to our decoder to facilitate point cloud self- and cross-reconstruction. Our proposed self-supervised training pipeline encourages semantically corresponding points from different shape instances to be mapped to similar local shape descriptors, enabling LSTNet to establish the dense point-wise correspondences. LSTNet demonstrates state-of-the-art performances on 3D semantic keypoint transfer and part segmentation label transfer given arbitrarily rotated point cloud pairs, outperforming existing methods by significant margins.\\n\\nAfter reading this abstract, please write your review for it.', 'response': 'summary: This work proposes LSTNet, a self-supervised method to establish reliable 3D dense correspondences irrespective of the input point clouds’ rotational orientation.\\n\\nSpecifically, LSTNet learns to formulate SO(3)-invariant local shape transform for each point in a dynamic, input-dependent manner. Each point-wise local shape transform can map the SO(3)-equivariant global shape descriptor of the input shape to a local shape descriptor, which is passed to the decoder to reconstruct the shape and pose of the input point cloud. \\n\\nThe proposed self-supervised training pipeline encourages semantically corresponding points from different shape instances to be mapped to similar local shape descriptors, enabling LSTNet to establish dense point-wise correspondences via nearest point pairs between cross-reconstructed point clouds.\\n\\nstrengths: The self- and cross-reconstruction training strategy is simple yet effective. \\n\\nLSTNet demonstrates state-of-the-art performance on 3D semantic matching when evaluated on the KeypointNet dataset and part segmentation label transfer when evaluated on the ShapeNet dataset.\\n\\nweaknesses: The performance of aligned shape pairs under the setting of I/I shows that other methods, such as CPAE, are much better than LSTNet.\\n\\nquestions: The reason why other methods are much better than LSTNet under the setting of I/I should be clarified.\\n\\nLack of limitations.\\n\\nsoundness: 3 good\\n\\npresentation: 2 fair\\n\\ncontribution: 3 good\\n\\nrating: 6: marginally above the acceptance threshold\\n\\nconfidence: 2: You are willing to defend your assessment, but it is quite likely that you did not understand the central parts of the submission or that you are unfamiliar with some pieces of related work. Math/other details were not carefully checked.\\n\\n'}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c56bcf2864aa45edb8b0d4f3e6da8ca6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from argparse import Namespace\n",
    "\n",
    "from transformers import AutoTokenizer, AutoModelForCausalLM, AutoConfig, PreTrainedModel, GenerationConfig, PreTrainedTokenizer\n",
    "from peft import LoraConfig, get_peft_model, PeftModel\n",
    "from torch.utils.data import Dataset, DataLoader, DistributedSampler\n",
    "\n",
    "\n",
    "from openreviewer.arguments import get_args\n",
    "from openreviewer.dataset import InstructionTuningDataset\n",
    "from openreviewer.utils import vicuna_sample_processor, print_rank, broadcast_model, move_dict_to_device, save_checkpoint, openreviewer_data_preprocessor\n",
    "from openreviewer.scheduler import CosineWarmUpScheduler\n",
    "from openreviewer.common import freeze_ffn_target_moudles, lora_target_modules\n",
    "\n",
    "from main import get_dataset\n",
    "\n",
    "model_path = \"/root/autodl-tmp/model/vicuna-7b-v1.5-16k\"\n",
    "save_path = \"/root/autodl-tmp/checkpoints/1204\"\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_path)\n",
    "args = Namespace(\n",
    "    dataset_type=\"InstructionTuningDataset\",\n",
    "    data_path=\"/root/autodl-tmp/data/iclr2024/1204.jsonl\",\n",
    "    model_type=\"vicuna\",\n",
    "    max_length=6144,\n",
    "    max_prompt_length=4096,\n",
    ")\n",
    "\n",
    "dataset = get_dataset(args, tokenizer, preprocessor=openreviewer_data_preprocessor)\n",
    "\n",
    "print(dataset[0])\n",
    "\n",
    "model = AutoModelForCausalLM.from_pretrained(model_path)\n",
    "model = PeftModel.from_pretrained(model, save_path)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "===\n",
      "<s> You are reviewing the paper titled Learning SO(3)-Invariant Correspondence via Point-wise Local Shape Transform. The keywords are Point cloud understanding, 3D dense correspondence, SO(3)-invariance, Part label transfer. You will read this paper and write a review for it. USER: The abstract is：\n",
      "\n",
      "Establishing accurate dense 3D correspondences between diverse shapes stands as a pivotal challenge with profound implications for computer vision and robotics. However, existing self-supervised methods for this problem assume perfect input shape alignment, restricting their real-world applicability. In this work, we introduce a novel self-supervised SO(3)-invariant 3D correspondence learner, dubbed LSTNet, that learns to establish dense correspondences between shapes even under challenging intra-class variations. Specifically, LSTNet learns to dynamically formulate an SO(3)-invariant local shape transform for each point, which maps the SO(3)-equivariant global shape descriptor of the input shape to a local shape descriptor. These local shape descriptors are provided as inputs to our decoder to facilitate point cloud self- and cross-reconstruction. Our proposed self-supervised training pipeline encourages semantically corresponding points from different shape instances to be mapped to similar local shape descriptors, enabling LSTNet to establish the dense point-wise correspondences. LSTNet demonstrates state-of-the-art performances on 3D semantic keypoint transfer and part segmentation label transfer given arbitrarily rotated point cloud pairs, outperforming existing methods by significant margins.\n",
      "\n",
      "After reading this abstract, please write your review for it. ASSISTANT:\n",
      "---\n",
      " summary: This paper addresses dense 3D correspondence establishment in point cloud reconstruction and point cloud part label transfer.  The authors propose a method that is grounded in point local shape descriptor transformation. The resulting descriptor is then used in a decoder to establish correspondence between shapes. This method is trained with a contrastive self-supervised method similar to previous work.\n",
      "\n",
      "strengths: + It seems the authors have achieved substantial experimental evaluation as benchmark performance is provided and outperforms prior works.\n",
      "\n",
      "+ A good presentation of the proposed idea behind LSTNet.\n",
      "\n",
      "weaknesses: Originality: As far as I am aware, this seems to be the first work that leverages point local shape representation for 3D correspondence establishment. A couple of additional points:\n",
      " - While LSTNet is a novel method, it does rely on the local shape descriptor transformation idea, which is from Ling-Xiao Lu (2022). The connection to Ling-Xiao Lu (2022) could be clarified a bit more.\n",
      " - It would be beneficial to further elaborate how LSTNet differs from a method such as PointSplat (2022).\n",
      "\n",
      "Quality: The empirical evaluation is thorough and the technical writing is clear. However, one point of confusion is that the author's claim that their method outperforms all existing 3D correspondential methods seems to not be supported well. Could the author provide a comparison that demonstrates this? For example, how does the method compare to other related methods such as Key Point Differentiable Surface Kinect (KDSK)?\n",
      "\n",
      "In terms of presentation, clarity, and soundness, the quality overall has good scores, but not quite excellent ones.\n",
      "\n",
      "Significance: As previously mentioned, the method has good soundness. However, in terms of significance, the novelty seems somewhat weak.\n",
      "\n",
      "Clarity: 4\n",
      "\n",
      "Presentation: 4\n",
      "\n",
      "Soundness: 4 (weak but still good)\n",
      "\n",
      "### Questions:\n",
      "1. In section 2.1 [2], the authors claim that the proposed framework can be trained with the Siamese Moment Matching loss, since it is rotation-free. However, in table 1, all of the prior correspondence estimation methods have been trained on Siamese Moment Matching loss, and thus the statement seems incorrect. Can the authors clarify this?\n",
      "2. In section 2.2, can the authors explain a bit more about why the loss in Table 1 is rotation (and translation) invariant in the context of LSTNet?  \n",
      "\n",
      "It is also very helpful if the authors can provide more details around their decoder training pipeline, for example the loss and how you get the correspondence in the decoder network. This could be a little bit more elaborate than in section 2.3 and in section 2.4 (see my minor comments)\n",
      "\n",
      "### Minor questions & comments:\n",
      "\n",
      "1. In section 2.1 [2], the authors claim that the proposed framework can be trained with the Siamese Moment Matching loss, since it is rotation-free. However, in table 1, all of the prior correspondence estimation methods have been trained on Siamese Moment Matching loss, and thus the statement seems incorrect. Can the authors clarify this? It looks like the loss in Table 1 could cause issues with LSTNet's rotation-free loss function that you claim.\n",
      "2. Would the authors clarify what the method can do when the point cloud data is not aligned in terms of global shape? Is this a limitation to the method? This is one possible area that can be improved. Is alignment only considered during the training phase?\n",
      "3. In section 2.3, could the authors briefly explain what \"corresponding points\" mean? It seems like they are the actual points that can be used for inference after training. It would be useful if they can quantify what is a \"corresponding point\", for example some distance distance thresholds or the point label in some way.\n",
      "4. In section 3.1 [3], is it not also true that given a matching pair of initial points, the global transformation parameters (rotation and translation) to transform the second point to be aligned with the first one would be needed. This makes it hard to achieve the goal of \"SO(3) invariant correspondences\". \n",
      "5. In section 5.2 [4], could the authors clarify what \"SO(3) consistent\" means? How does this align with SO(3) invariant correspondences and why is it critical?\n",
      "6. In eq. (3), the equation itself introduces some issues.\n",
      "* The term (1-cos(theta)) / (1\n",
      "\n",
      "===\n",
      "<s> You are reviewing the paper titled Learning SO(3)-Invariant Correspondence via Point-wise Local Shape Transform. The keywords are Point cloud understanding, 3D dense correspondence, SO(3)-invariance, Part label transfer. You will read this paper and write a review for it. USER: The abstract is：\n",
      "\n",
      "Establishing accurate dense 3D correspondences between diverse shapes stands as a pivotal challenge with profound implications for computer vision and robotics. However, existing self-supervised methods for this problem assume perfect input shape alignment, restricting their real-world applicability. In this work, we introduce a novel self-supervised SO(3)-invariant 3D correspondence learner, dubbed LSTNet, that learns to establish dense correspondences between shapes even under challenging intra-class variations. Specifically, LSTNet learns to dynamically formulate an SO(3)-invariant local shape transform for each point, which maps the SO(3)-equivariant global shape descriptor of the input shape to a local shape descriptor. These local shape descriptors are provided as inputs to our decoder to facilitate point cloud self- and cross-reconstruction. Our proposed self-supervised training pipeline encourages semantically corresponding points from different shape instances to be mapped to similar local shape descriptors, enabling LSTNet to establish the dense point-wise correspondences. LSTNet demonstrates state-of-the-art performances on 3D semantic keypoint transfer and part segmentation label transfer given arbitrarily rotated point cloud pairs, outperforming existing methods by significant margins.\n",
      "\n",
      "After reading this abstract, please write your review for it. ASSISTANT:\n",
      "---\n",
      " summary: The paper tackles the problem of self-supervised learning for dense point correspondence by establishing correspondences between an input and a transformed version of that point cloud. The authors proposed learning a local shape transform that can be applied element-wise to the input point cloud. To do so, they first learn a shape transform on the shape manifold and then use a network to learn how to decompose a point cloud into its canonical and transformation components. The network is then used to learn point-wise mappings between shape components of corresponding points across different shapes.\n",
      "\n",
      "strengths: - The paper is easy to follow and it makes sense.\n",
      "- Experiments are well motivated and performed.\n",
      "\n",
      "weaknesses: - The authors didn’t compare the proposed method with recent self-supervised methods for learning dense point correspondence (see https://arxiv.org/pdf/2301.12638.pdf). It is unclear how the method performed compared to the baselines.\n",
      "- The method relies on point cloud and point-wise correspondence models to generate both canonical and local representations. The authors could compare it with methods that do not use such models in training e.g. https://papr2023.isi.edu/docs/Bayesian_Few_Points_of_Information_in_Smooth_Applications_of_PointNetLK-v3_on_PCSemantics.pdf\n",
      "- The authors did use the canonical/local representation to train the network, but did not compare it with the results of training with the canonical representation. They also did not compare the performance of the network without it, which would provide more insights into how important the addition of a canonical representation.\n",
      "\n",
      "questions: - Some clarifications would be appreciated as it was not always clear what was meant by the various components.\n",
      "- What was the performance of the method when tested for dense point correspondence?\n",
      "- I have a few concerns on the way points are reconstructed. For example, in Table 1 when using point correspondence, the reconstructed point clouds have relatively low mean point distance (0.07) compared to the actual (0.32). The reconstructed point cloud distances are also relatively low (0.12). In these experiments, the network is training with a fixed $w$, $h$, $t$, and $f$ which implies that there are only 4 parameters in the network.\n",
      "\n",
      "soundness: 2 fair\n",
      "\n",
      "presentation: 2 fair\n",
      "\n",
      "contribution: 2 fair\n",
      "\n",
      "rating: 3: reject, not good enough\n",
      "\n",
      "confidence: 5: You are absolutely certain about your assessment. You are very familiar with the related work and checked the math/other details carefully.\n",
      "\n",
      "</s>\n",
      "\n",
      "===\n",
      "<s> You are reviewing the paper titled Learning SO(3)-Invariant Correspondence via Point-wise Local Shape Transform. The keywords are Point cloud understanding, 3D dense correspondence, SO(3)-invariance, Part label transfer. You will read this paper and write a review for it. USER: The abstract is：\n",
      "\n",
      "Establishing accurate dense 3D correspondences between diverse shapes stands as a pivotal challenge with profound implications for computer vision and robotics. However, existing self-supervised methods for this problem assume perfect input shape alignment, restricting their real-world applicability. In this work, we introduce a novel self-supervised SO(3)-invariant 3D correspondence learner, dubbed LSTNet, that learns to establish dense correspondences between shapes even under challenging intra-class variations. Specifically, LSTNet learns to dynamically formulate an SO(3)-invariant local shape transform for each point, which maps the SO(3)-equivariant global shape descriptor of the input shape to a local shape descriptor. These local shape descriptors are provided as inputs to our decoder to facilitate point cloud self- and cross-reconstruction. Our proposed self-supervised training pipeline encourages semantically corresponding points from different shape instances to be mapped to similar local shape descriptors, enabling LSTNet to establish the dense point-wise correspondences. LSTNet demonstrates state-of-the-art performances on 3D semantic keypoint transfer and part segmentation label transfer given arbitrarily rotated point cloud pairs, outperforming existing methods by significant margins.\n",
      "\n",
      "After reading this abstract, please write your review for it. ASSISTANT:\n",
      "---\n",
      " summary: This paper presents a self-supervised method to establish dense 3D correspondence. It trains a decoder that can do a \"local shape transform\" for each point in a pointcloud, and a corresponding encoder that can take the encoder output to generate dense correspondence. In this way, this method is robust to intra-class variations, which is an important challenge addressed in the paper.\n",
      "\n",
      "strengths: The paper introduces a novel self-supervised method to establish dense 3D correspondence.\n",
      "\n",
      "weaknesses: I do not see the contributions from this paper. This method is similar to [1].\n",
      "\n",
      "[1] Liu, Yifan, et al. \"SO(3)-invariant 3D correspondence learning.\" Proceedings of the IEEE/CVF conference on Computer Vision and Pattern Recognition. 2021.\n",
      "\n",
      "questions: Please see the weaknesses.\n",
      "\n",
      "soundness: 2 fair\n",
      "\n",
      "presentation: 2 fair\n",
      "\n",
      "contribution: 2 fair\n",
      "\n",
      "rating: 3: reject, not good enough\n",
      "\n",
      "confidence: 4: You are confident in your assessment, but not absolutely certain. It is unlikely, but not impossible, that you did not understand some parts of the submission or that you are unfamiliar with some pieces of related work.\n",
      "\n",
      "</s>\n",
      "\n",
      "===\n",
      "<s> You are reviewing the paper titled Learning SO(3)-Invariant Correspondence via Point-wise Local Shape Transform. The keywords are Point cloud understanding, 3D dense correspondence, SO(3)-invariance, Part label transfer. You will read this paper and write a review for it. USER: The abstract is：\n",
      "\n",
      "Establishing accurate dense 3D correspondences between diverse shapes stands as a pivotal challenge with profound implications for computer vision and robotics. However, existing self-supervised methods for this problem assume perfect input shape alignment, restricting their real-world applicability. In this work, we introduce a novel self-supervised SO(3)-invariant 3D correspondence learner, dubbed LSTNet, that learns to establish dense correspondences between shapes even under challenging intra-class variations. Specifically, LSTNet learns to dynamically formulate an SO(3)-invariant local shape transform for each point, which maps the SO(3)-equivariant global shape descriptor of the input shape to a local shape descriptor. These local shape descriptors are provided as inputs to our decoder to facilitate point cloud self- and cross-reconstruction. Our proposed self-supervised training pipeline encourages semantically corresponding points from different shape instances to be mapped to similar local shape descriptors, enabling LSTNet to establish the dense point-wise correspondences. LSTNet demonstrates state-of-the-art performances on 3D semantic keypoint transfer and part segmentation label transfer given arbitrarily rotated point cloud pairs, outperforming existing methods by significant margins.\n",
      "\n",
      "After reading this abstract, please write your review for it. ASSISTANT:\n",
      "---\n",
      " summary: This study proposes a self-supervised training framework for point cloud segmentation tasks, which incorporates self-supervised training for both point cloud registration and segmentation. The proposed framework is experimented using synthetic datasets, and the results demonstrated superior performance compared to existing SO3-invariant registration methods.\n",
      "\n",
      "strengths: 1. The proposed training framework is formulated as a self-supervised framework, which addresses the weaknesses of traditional supervised methods.\n",
      "2. The proposed method has substantial performance improvement as compared to the baseline SO3-invariance registration method in the synthetic datasets.\n",
      "\n",
      "weaknesses: 1. While the experimental dataset is synthetic, this study has limitations in real-world dataset usage, and it has not been empirically evaluated based on real-world datasets.\n",
      "2. This study does not provide an in-depth illustration of the implementation or the specific point of the proposed method.\n",
      "\n",
      "questions: This framework has been formulated as a self-supervised framework, it is unclear as to how exactly this self-supervised method works. Additionally, it seems like this study has not conducted extensive research on the proposed method, and it needs to provide a further elaboration in the description and illustration of the proposed method.\n",
      "\n",
      "soundness: 2 fair\n",
      "\n",
      "presentation: 3 good\n",
      "\n",
      "contribution: 3 good\n",
      "\n",
      "rating: 6: marginally above the acceptance threshold\n",
      "\n",
      "confidence: 4: You are confident in your assessment, but not absolutely certain. It is unlikely, but not impossible, that you did not understand some parts of the submission or that you are unfamiliar with some pieces of related work.\n",
      "\n",
      "</s>\n",
      "\n",
      "===\n",
      "<s> You are reviewing the paper titled Learning SO(3)-Invariant Correspondence via Point-wise Local Shape Transform. The keywords are Point cloud understanding, 3D dense correspondence, SO(3)-invariance, Part label transfer. You will read this paper and write a review for it. USER: The abstract is：\n",
      "\n",
      "Establishing accurate dense 3D correspondences between diverse shapes stands as a pivotal challenge with profound implications for computer vision and robotics. However, existing self-supervised methods for this problem assume perfect input shape alignment, restricting their real-world applicability. In this work, we introduce a novel self-supervised SO(3)-invariant 3D correspondence learner, dubbed LSTNet, that learns to establish dense correspondences between shapes even under challenging intra-class variations. Specifically, LSTNet learns to dynamically formulate an SO(3)-invariant local shape transform for each point, which maps the SO(3)-equivariant global shape descriptor of the input shape to a local shape descriptor. These local shape descriptors are provided as inputs to our decoder to facilitate point cloud self- and cross-reconstruction. Our proposed self-supervised training pipeline encourages semantically corresponding points from different shape instances to be mapped to similar local shape descriptors, enabling LSTNet to establish the dense point-wise correspondences. LSTNet demonstrates state-of-the-art performances on 3D semantic keypoint transfer and part segmentation label transfer given arbitrarily rotated point cloud pairs, outperforming existing methods by significant margins.\n",
      "\n",
      "After reading this abstract, please write your review for it. ASSISTANT:\n",
      "---\n",
      " summary: The paper introduces an SO(3)-invariant point-wise correspondence based on a point local shape transform by 3D-2D point projection to make learning possible, and proposes a SO(3)-invariant 3D correspondence generator with two decoders via the backbone PWO3-R2T.  An SO(3)-equivariant loss introduces 3D-2D PWM (Projection-Weighted-Maxnorm/L1 distance) to ensure the point-wise correspondences between the predicted local feature.\n",
      "\n",
      "strengths: 1. The paper proposed a novel formulation for SO(3)-invariant point-wise correspondences and point local shape transforms.  Theoretically, SO(3)-invariant 3D correspondence is a highly challenging problem and the motivation is theoretically motivated. In addition, the proposed method outperforms state-of-the-art methods in 3D semantic keypoint transfer and part segmentation label transfer datasets.\n",
      "\n",
      "2. The motivation of SO(3)-invariance point local shape transform is theoretically highly sound.\n",
      "\n",
      "weaknesses: 1. The experimental results seem not convincing on the 43D dataset. The performance of the baselines is too high that the improvements of the method based on existing points-only methods are less significant\n",
      "\n",
      "2. A potential solution for the limitation of 3D point-wise correspondence could be provided such as leveraging 3D-2D part-based shape reconstruction \n",
      "\n",
      "3. The details of the training process and architecture are less clear. The paper states: \"In practice we use a U-Net-like backbone instead of a point cloud backbone to leverage part segmentation labels and shape representation, which provides a dense 3D representation to guide point-wise correspondences during training and test\", but the architecture structure is not introduced which makes it confusing.\n",
      "\n",
      "questions: My main motivation was questioned by the authors. The SO(3)-invariance is introduced as a technical challenge for the state-of-the-art methods to deal with them. A theoretical proof on the limitations of traditional correspondence methods on the SO(3)-invariance is needed\n",
      "\n",
      "soundness: 3 good\n",
      "\n",
      "presentation: 3 good\n",
      "\n",
      "contribution: 2 fair\n",
      "\n",
      "rating: 5: marginally below the acceptance threshold\n",
      "\n",
      "confidence: 4: You are confident in your assessment, but not absolutely certain. It is unlikely, but not impossible, that you did not understand some parts of the submission or that you are unfamiliar with some pieces of related work.\n",
      "\n",
      "</s>\n",
      "\n",
      "===\n",
      "<s> You are reviewing the paper titled Breaking Physical and Linguistic Borders: Multilingual Federated Prompt Tuning for Low-Resource Languages. The keywords are Multilingual Federated Learning, Natural Language Processing. You will read this paper and write a review for it. USER: The abstract is：\n",
      "\n",
      "Pretrained large language models (LLMs) have emerged as a cornerstone in modern natural language processing, with their utility expanding to various applications and languages. However, the fine-tuning of multilingual LLMs, particularly for low-resource languages, is fraught with challenges steming from data-sharing restrictions (the physical border) and from the inherent linguistic differences (the linguistic border). These barriers hinder users of various languages, especially those in low-resource regions, from fully benefiting from the advantages of LLMs.\n",
      "\n",
      "To overcome these challenges, we propose the Federated Prompt Tuning Paradigm for Multilingual Scenarios, which leverages parameter-efficient fine-tuning in a manner that preserves user privacy. We have designed a comprehensive set of experiments and introduced the concept of \"language distance\" to highlight the several strengths of this paradigm. Even under computational constraints, our method not only bolsters data efficiency but also facilitates mutual enhancements across languages, particularly benefiting low-resource ones. Compared to traditional local crosslingual transfer tuning methods, our approach achieves a 6.9\\% higher accuracy, reduces the training parameters by over 99\\%, and demonstrates stronger cross-lingual generalization. Such findings underscore the potential of our approach to promote social equality, ensure user privacy, and champion linguistic diversity.\n",
      "\n",
      "After reading this abstract, please write your review for it. ASSISTANT:\n",
      "---\n",
      " summary: This paper proposes a multilingual federated prompt tuning (FL-PT) method that leverages the prompt diversity to promote cross-lingual transfer to low-resource languages. Experimental results on multiple datasets show the effectiveness of the proposed method.\n",
      "\n",
      "strengths: The paper proposes an innovative idea in a timely and relevant topic. The overall contribution of this paper is significant for ML community. I believe this paper will be a classic model in the area of natural language processing.\n",
      "\n",
      "weaknesses: The paper needs to be improved in terms of clarity, presentation, and formalization. The paper also needs to work on further technical issues related to the original version of the paper.  I have also included several comments below.\n",
      "\n",
      "questions: - In Section 3.2, there is no formal definition of prompt distance. What does it mean? What is the meaning to be minimized or maximized in Section 4.2.2?\n",
      "- In Section 4.2.2, why the objective function is not linear? I believe that it is more intuitive to use the quadratic function as $Q(p_i) = \\omega_i^{(N)} L(m; \\theta) + \\sum_{i=1}^{N} \\omega_i^{(i)} L(p_i; \\theta)$. With this formulation, we can further study the global/local contributions of fine-tuning to each language model and user. I see that the authors have shown that the local contribution can be improved by the prompt distance. However, I think that the global contribution (parameter $Q$ of prompts) still needs studying.\n",
      "- In Section 4, I don’t see how the authors justify the hypothesis, i.e., the prompt diversity is proportional to the performance improvement of language models.\n",
      "- Section 6.1: this is not my area/topic, but in a general sense, it is interesting to see how the authors use model size, prompt diversity, and cross-lingual generalization to compare their method to the original. However, without further discussing how much the contribution is for each of these components, it is difficult to evaluate their technical contribution. I think the authors need to work on this by comparing the contribution of each one of these components to the result (in terms of both technical and empirical merits).\n",
      "- In Table 1, how did the authors obtain the number of prompts per language? is there a threshold to decide whether a prompt belongs to a language model?\n",
      "- In Table 1, the number of languages is 10 for both LLM1 and LLM2. However, for the other models, the number of languages is 255, 232, 210 for LLM1, 255, 110 for LLM2, respectively. How did the authors decide which number should be used for each model and why?\n",
      "- Acknowledgments: it seems that not all the authors are listed. Please add the missing authors.\n",
      "- Figure 1: the title and subtitle are not aligned.\n",
      "- Reference the related work well.\n",
      "- Table 1: I see one error in the definition of LLMs in the main text. However, the caption doesn’t show the correct definition. Please clarify this.\n",
      "\n",
      "soundness: 3 good\n",
      "\n",
      "presentation: 1 poor\n",
      "\n",
      "contribution: 3 good\n",
      "\n",
      "rating: 6: marginally above the acceptance threshold\n",
      "\n",
      "confidence: 3: You are fairly confident in your assessment. It is possible that you did not understand some parts of the submission or that you are unfamiliar with some pieces of related work. Math/other details were not carefully checked.\n",
      "\n",
      "</s>\n",
      "\n",
      "===\n",
      "<s> You are reviewing the paper titled Breaking Physical and Linguistic Borders: Multilingual Federated Prompt Tuning for Low-Resource Languages. The keywords are Multilingual Federated Learning, Natural Language Processing. You will read this paper and write a review for it. USER: The abstract is：\n",
      "\n",
      "Pretrained large language models (LLMs) have emerged as a cornerstone in modern natural language processing, with their utility expanding to various applications and languages. However, the fine-tuning of multilingual LLMs, particularly for low-resource languages, is fraught with challenges steming from data-sharing restrictions (the physical border) and from the inherent linguistic differences (the linguistic border). These barriers hinder users of various languages, especially those in low-resource regions, from fully benefiting from the advantages of LLMs.\n",
      "\n",
      "To overcome these challenges, we propose the Federated Prompt Tuning Paradigm for Multilingual Scenarios, which leverages parameter-efficient fine-tuning in a manner that preserves user privacy. We have designed a comprehensive set of experiments and introduced the concept of \"language distance\" to highlight the several strengths of this paradigm. Even under computational constraints, our method not only bolsters data efficiency but also facilitates mutual enhancements across languages, particularly benefiting low-resource ones. Compared to traditional local crosslingual transfer tuning methods, our approach achieves a 6.9\\% higher accuracy, reduces the training parameters by over 99\\%, and demonstrates stronger cross-lingual generalization. Such findings underscore the potential of our approach to promote social equality, ensure user privacy, and champion linguistic diversity.\n",
      "\n",
      "After reading this abstract, please write your review for it. ASSISTANT:\n",
      "---\n",
      " summary: This paper proposes a method to leverage cross-lingual knowledge distillation for large/medium-sized LLMs such as BERT or XLMR through a \"training-from-scratch\" procedure. The key idea is to start from a small LLM using a different set of source (high-res) parameters, using a different subset of sources (low-res), and target it on a given test set. The proposed pipeline is compared to other transfer tuning and fine-tuning baselines on a number of cross-lingual benchmarks.\n",
      "\n",
      "strengths: - the paper is well-written, and the experiments seem to be well-motivated and carried out with proper caution, in particular, cautioning the reader about the limitations of the cross-lingual task as it is defined by BERT-large/medium models\n",
      "- the approach is also compared to other state-of-the-art cross-lingual transfer-tuning methods, and seems to perform competitively.\n",
      "- the authors also provide some analysis on the nature of different languages, and the different levels of generality that models exhibit\n",
      "\n",
      "weaknesses: - although the experiments are not new, and to some extent are part and parcel of the broader cross-lingual literature, the idea is certainly novel and relevant.\n",
      "- despite the fact that this might be a good paper, it does not really push the envelope of cross-lingual transfer learning for multilingual LLMs, and is certainly not a strong enough contribution to be published at ICML 2023 (though I guess for a good paper from NeurIPS 2023 it might merit an acceptance, but I would then wonder where the paper from NIPS 2023 resides?). While I am an enthusiast of research on language modeling and natural language processing, I can clearly see how my field needs more methodologies with the aim of pushing the state of the art, and is not limited to some single-aspect research work.\n",
      "\n",
      "questions:  - what is the exact target distribution over all test sets used in the experiments?\n",
      "- the authors seem to be using a large number of different languages in the cross-lingual experiments, have they any idea of the overall effect of the proposed method on language-level performance (and not on small-scaled language instances inside these larger data splits)?\n",
      "\n",
      "Please consider my questions as my feedback based on my knowledge of the field, and not as an indication that the questions themselves are flawed. I also acknowledge that a good paper should contain more than 5000 words and may require many more experiments than published here for it to be considered good enough.\n",
      "\n",
      "soundness: 2 fair\n",
      "\n",
      "presentation: 3 good\n",
      "\n",
      "contribution: 1 poor\n",
      "\n",
      "rating: 3: reject, not good enough\n",
      "\n",
      "confidence: 5: You are absolutely certain about your assessment. You are very familiar with the related work and checked the math/other details carefully.\n",
      "\n",
      "</s>\n",
      "\n",
      "===\n",
      "<s> You are reviewing the paper titled Breaking Physical and Linguistic Borders: Multilingual Federated Prompt Tuning for Low-Resource Languages. The keywords are Multilingual Federated Learning, Natural Language Processing. You will read this paper and write a review for it. USER: The abstract is：\n",
      "\n",
      "Pretrained large language models (LLMs) have emerged as a cornerstone in modern natural language processing, with their utility expanding to various applications and languages. However, the fine-tuning of multilingual LLMs, particularly for low-resource languages, is fraught with challenges steming from data-sharing restrictions (the physical border) and from the inherent linguistic differences (the linguistic border). These barriers hinder users of various languages, especially those in low-resource regions, from fully benefiting from the advantages of LLMs.\n",
      "\n",
      "To overcome these challenges, we propose the Federated Prompt Tuning Paradigm for Multilingual Scenarios, which leverages parameter-efficient fine-tuning in a manner that preserves user privacy. We have designed a comprehensive set of experiments and introduced the concept of \"language distance\" to highlight the several strengths of this paradigm. Even under computational constraints, our method not only bolsters data efficiency but also facilitates mutual enhancements across languages, particularly benefiting low-resource ones. Compared to traditional local crosslingual transfer tuning methods, our approach achieves a 6.9\\% higher accuracy, reduces the training parameters by over 99\\%, and demonstrates stronger cross-lingual generalization. Such findings underscore the potential of our approach to promote social equality, ensure user privacy, and champion linguistic diversity.\n",
      "\n",
      "After reading this abstract, please write your review for it. ASSISTANT:\n",
      "---\n",
      " summary: In this paper, the authors propose a federated optimization method for fine-tuning large-scale multilingual models across languages with distinct linguistic and geographical borders, without the use of shared parameters and without violating existing data-sharing restrictions. The proposed method achieves a 1.9x acceleration in training time with computational constraints, while maintaining a 64-time reduction in parameter size and a 6.9% improvement in model accuracy relative to existing crosslingual transfer tuning techniques.\n",
      "\n",
      "strengths: 1.This paper is very important to the topic and makes practical contribution. \n",
      "\n",
      "2. The description can explain the problem well and this paper is easy to follow. \n",
      "\n",
      "3. Experiments results are reliable and convincing.\n",
      "\n",
      "weaknesses: 1. This paper is very important for the task of multilingual language model fine-tuning and the topic of federated machine learning. However, I am afraid that this paper is not quite good enough to be accepted by ICLR, especially for the following aspects:\n",
      "     \n",
      "    The authors claim in the introduction that \"Existing cross-lingual models suffer from several limitations due to limitations in data size and the need to maintain privacy during training; these issues can make it extremely difficult for low-resource language communities to fully benefit from the utility of LLMs.\" The authors also mention a few ways to tackle these limitations, such as parameter-efficient transfer, incremental fine-tuning, and federated optimization. However, if so, it is not clear whether these methods can meet the objective of this paper, i.e., federating machine learning and reducing language barriers. In addition, this paper mentions several works for transfer model by prompt tuning, such as LPM, DT, DPP-LTM.  Moreover, this paper mentions TPT, LLMF, and LLM. However, to make the readers better get the idea of the proposed method, it is necessary to explain the experimental setup of these works too; otherwise, the paper is not strong enough.\n",
      "\n",
      "    In section 4.1, the authors explain a very good definition of data-partition strategy (which is better than the methods in this paper). It also explains the methodology of the proposed method so that the readers can understand clearly. However, it is a pity that the authors leave out the evaluation of these methods and other works in section 5.2. Therefore the paper is not strong enough since the authors do not compare the proposed method with these works. \n",
      "\n",
      "2. The writing in this paper is limited: the paper lacks some examples to help readers understand the objective of the paper more concretely, and it is hard to follow its arguments since most details are missing in the paper. I expect readers to have a strong background in machine learning to read this paper.\n",
      "\n",
      "3. The authors mention that the paper is to \"mitigate the potential effects of data partition\", but this paper is not good enough to demonstrate this to the readers since the experimental setup is missing.\n",
      "\n",
      "questions: See the weaknesses\n",
      "\n",
      "soundness: 2 fair\n",
      "\n",
      "presentation: 2 fair\n",
      "\n",
      "contribution: 2 fair\n",
      "\n",
      "rating: 6: marginally above the acceptance threshold\n",
      "\n",
      "confidence: 3: You are fairly confident in your assessment. It is possible that you did not understand some parts of the submission or that you are unfamiliar with some pieces of related work. Math/other details were not carefully checked.\n",
      "\n",
      "</s>\n",
      "\n",
      "===\n",
      "<s> You are reviewing the paper titled Breaking Physical and Linguistic Borders: Multilingual Federated Prompt Tuning for Low-Resource Languages. The keywords are Multilingual Federated Learning, Natural Language Processing. You will read this paper and write a review for it. USER: The abstract is：\n",
      "\n",
      "Pretrained large language models (LLMs) have emerged as a cornerstone in modern natural language processing, with their utility expanding to various applications and languages. However, the fine-tuning of multilingual LLMs, particularly for low-resource languages, is fraught with challenges steming from data-sharing restrictions (the physical border) and from the inherent linguistic differences (the linguistic border). These barriers hinder users of various languages, especially those in low-resource regions, from fully benefiting from the advantages of LLMs.\n",
      "\n",
      "To overcome these challenges, we propose the Federated Prompt Tuning Paradigm for Multilingual Scenarios, which leverages parameter-efficient fine-tuning in a manner that preserves user privacy. We have designed a comprehensive set of experiments and introduced the concept of \"language distance\" to highlight the several strengths of this paradigm. Even under computational constraints, our method not only bolsters data efficiency but also facilitates mutual enhancements across languages, particularly benefiting low-resource ones. Compared to traditional local crosslingual transfer tuning methods, our approach achieves a 6.9\\% higher accuracy, reduces the training parameters by over 99\\%, and demonstrates stronger cross-lingual generalization. Such findings underscore the potential of our approach to promote social equality, ensure user privacy, and champion linguistic diversity.\n",
      "\n",
      "After reading this abstract, please write your review for it. ASSISTANT:\n",
      "---\n",
      " summary: This paper presents a novel paradigm for Multilingual Federated Prompt Tuning (MFPT) to address the challenges of data-sharing restrictions and linguistic differences in low-resource language applications of pre-trained NLP models. The proposed method leverages the concept of “language distance”, which quantifies the perceived similarity between different languages, and utilizes a fine-tuning approach with reduced training parameters. The experimental results demonstrate significant performance gains and lower computational costs compared to traditional local crosslingual transfer tuning methods, paving the way for greater data efficiency, cross-lingual enhancements, and greater linguistic diversity.\n",
      "\n",
      "strengths: The paper is well-written with an engaging introduction and a comprehensive overview of existing federated learning methods. The proposed approach leverages language distance to facilitate multi-lingual federated learning and offers a refined version of FPTS-X for low resource languages. The paper demonstrates a strong performance improvement over traditional local crosslingual transfer tuning methods.\n",
      "\n",
      "weaknesses: 1. The paper highlights that the proposed method enables “data efficiency”, “computational efficiency” and promotes “linguistic diversity” but I don’t see how such features are realized in terms of quantitative analyses in the main paper. The experimental results should be reorganized and better explicated in the main paper, in order to be comprehensive and convincing.\n",
      "\n",
      "2. The evaluation metrics used in the experiment should be properly introduced. It is not clear where did the data come from and what is the dataset, so the effectiveness of the approach in promoting data efficiency is uncertain.\n",
      "\n",
      "3. Figure 1 should be replaced since in this era of large models, “BIG” simply means large scale. We see many works like “HUGBERT\" have already explored the “big scale\". It would be better to see whether the experimental model has some specific properties on the scale for MFPT, e.g. whether it contains a lot of small words, long sentences, or if it has good translation ability to transfer across languages.\n",
      "\n",
      "4. The paper is good at demonstrating the effectiveness of the experimental evaluation method, it can be improved with details or a more comprehensive explanation or references of those evaluation methods.\n",
      "\n",
      "questions: 1. How is the language distance actually defined here? What is the underlying theory or how is it calculated?\n",
      "2. It seems to me that many tasks such as classification or translation may not depend on the specific scale of the model. Is the evaluation of the experimental models focused on the large LLM models in this submission?\n",
      "3. How is the data distributed across the different languages in the experiment? Is the sample efficiency the same across all the languages in the experiment or the sample efficiency varies significantly for each language?\n",
      "\n",
      "soundness: 2 fair\n",
      "\n",
      "presentation: 2 fair\n",
      "\n",
      "contribution: 2 fair\n",
      "\n",
      "rating: 5: marginally below the acceptance threshold\n",
      "\n",
      "confidence: 4: You are confident in your assessment, but not absolutely certain. It is unlikely, but not impossible, that you did not understand some parts of the submission or that you are unfamiliar with some pieces of related work.\n",
      "\n",
      "</s>\n",
      "\n",
      "===\n",
      "<s> You are reviewing the paper titled Stochastic Adversarial Networks for Multi-Domain Text Classification. The keywords are Multi-domain text classification, Adversarial training, Multivariate gaussian distribution. You will read this paper and write a review for it. USER: The abstract is：\n",
      "\n",
      "Adversarial training has played a pivotal role in the significant advancements of multi-domain text classification (MDTC). Recent MDTC methods often adopt the shared-private paradigm, wherein a shared feature extractor captures domain-invariant knowledge, while private feature extractors per domain extract domain-dependent knowledge. These approaches have demonstrated state-of-the-art performance. However, a major challenge remains: the exponential increase in model parameters as new domains emerge. To address this challenge, we propose the Stochastic Adversarial Network (SAN), which models multiple domain-specific feature extractors as a multivariate Gaussian distribution rather than weight vectors. With SAN, we can sample as many domain-specific feature extractors as necessary without drastically increasing the number of model parameters. Consequently, the model size of SAN remains comparable to having a single domain-specific feature extractor when data from multiple domains. Additionally, we incorporate domain label smoothing and robust pseudo-label regularization techniques to enhance the stability of the adversarial training and improve feature discriminability, respectively. The evaluations conducted on two prominent MDTC benchmarks validate the competitiveness of our proposed SAN method against state-of-the-art approaches.\n",
      "\n",
      "After reading this abstract, please write your review for it. ASSISTANT:\n",
      "---\n",
      " summary: In this paper, the author proposes the stochastic adversarial network (SAN for short) to solve the adversarial training problem. Based on SAN, the model is trained by alternating adversarial training of domain-specific feature extractors generated randomly by this model. The proposed GMM model can be trained with a relatively small parameter set but sufficient for training the model. Contritionally, the GMM model is usually trained for one-dimensional random variable in previous works. But the proposed work assumes SAN can achieve an uniform distribution of all the feature extractors randomly sampled.\n",
      "\n",
      "strengths: 1. This paper investigates the application of SAN (Stochastic Adversarial Networks) in multi-domain text classification tasks. Moreover, compared with the previous methods, the authors use GMM (Gaussian Multi-modal) to approximate these domains.\n",
      "\n",
      "2. This paper is solid and the proposed method has novel and meaningful ideas.\n",
      "\n",
      "3. This paper is the first to explore the use of SAN in multi-domain text classification tasks. \n",
      "\n",
      "4. Evaluation results show that the proposed method is effective.\n",
      "\n",
      "weaknesses: 1. This paper does not provide the proposed method in a clear way. This makes readers hard to understand the proposed idea.\n",
      "\n",
      "2. To my idea, the author should provide the loss for the GMM model to show the proposed stochastic adversarial network (SAN) can be learned with GMM model.\n",
      "\n",
      "3. Missing the technical details of the SAN, e.g., how to make a distribution of domains and the loss for the SAN model.\n",
      "\n",
      "questions: 1. In the second last paragraph, the authors write the proposed method SAN is easy to learn and fast to train. Please give more details on how to train SAN. \n",
      "\n",
      "2. How does the author learn the distribution in which a sampled data $d$ follows in SAN?\n",
      "\n",
      "3. How to calculate the loss for SAN in the training process?\n",
      "\n",
      "4. What is the difference between the method in Fig 1 and the proposed SAN?\n",
      "\n",
      "soundness: 2 fair\n",
      "\n",
      "presentation: 2 fair\n",
      "\n",
      "contribution: 2 fair\n",
      "\n",
      "rating: 3: reject, not good enough\n",
      "\n",
      "confidence: 5: You are absolutely certain about your assessment. You are very familiar with the related work and checked the math/other details carefully.\n",
      "\n",
      "</s>\n",
      "\n",
      "===\n",
      "<s> You are reviewing the paper titled Stochastic Adversarial Networks for Multi-Domain Text Classification. The keywords are Multi-domain text classification, Adversarial training, Multivariate gaussian distribution. You will read this paper and write a review for it. USER: The abstract is：\n",
      "\n",
      "Adversarial training has played a pivotal role in the significant advancements of multi-domain text classification (MDTC). Recent MDTC methods often adopt the shared-private paradigm, wherein a shared feature extractor captures domain-invariant knowledge, while private feature extractors per domain extract domain-dependent knowledge. These approaches have demonstrated state-of-the-art performance. However, a major challenge remains: the exponential increase in model parameters as new domains emerge. To address this challenge, we propose the Stochastic Adversarial Network (SAN), which models multiple domain-specific feature extractors as a multivariate Gaussian distribution rather than weight vectors. With SAN, we can sample as many domain-specific feature extractors as necessary without drastically increasing the number of model parameters. Consequently, the model size of SAN remains comparable to having a single domain-specific feature extractor when data from multiple domains. Additionally, we incorporate domain label smoothing and robust pseudo-label regularization techniques to enhance the stability of the adversarial training and improve feature discriminability, respectively. The evaluations conducted on two prominent MDTC benchmarks validate the competitiveness of our proposed SAN method against state-of-the-art approaches.\n",
      "\n",
      "After reading this abstract, please write your review for it. ASSISTANT:\n",
      "---\n",
      " summary: The paper proposes the stochastic adversarial networks (SAN) which model multiple domain-specific feature extractors as multivariate Gaussian samples, allowing for adaptable and scalable text classification models as more domains emerge. SAN demonstrates superior performance on two MDTC benchmarks.\n",
      "\n",
      "strengths: 1. SAN provides a simple and flexible solution to address the model size issues when training models in the multiproduct settings\n",
      "2. The paper is well-written and clearly presented\n",
      "\n",
      "weaknesses: 1. While SAN demonstrates good results on MSR-CCR and SQuAD, I think it can get worse results on the real application settings. For example, the paper uses the GPT-3 fine-tuning and fine-tuning on the SQuAD dataset on the GLue tasks. However, SAN still needs the extra fine-tuning on each tasks on these datasets. This is not practical for the real scenarios.\n",
      "2. In Table 7, \"SAN 1000 (D-1000) # domain-specific classes\" is incorrect, I think SAN 1000 is the domain-shared setting, it should be shown as \"SAN 1000 (D-2) # domain-specific classes\"\n",
      "\n",
      "[1] Yu, Xiaoyu, et al. \"Adversarial training with stochastic networks for zero-shot and fine-tuned machine learning tasks.\" AAAI 2022: Proceedings of the Thirty-Seventh Conference on Artificial Intelligence. 2022, pp. 5810-5819.\n",
      "\n",
      "questions: 1. The experiment results on the GLue-1K benchmark shows that the fine training on each task is required in the SAN, and results of the GLue-2K benchmark also shows the fine-tuning is required on each task. I think this could limit the usage of the SAN. Therefore, I suggest the researchers to provide more analysis and details of the experiments conducted on the benchmarks, especially for the real application scenarios.\n",
      "2. In SAN, the feature extractor is sampled from the multivariate Gaussian distribution, when samples the feature extractors, if they sample from only one location and never sample from other locations during the training, it seems that the feature extractor is not robust to other latent feature distributions which are not included in the training, right?\n",
      "3. As the training algorithm, the adversarial training algorithm with the multivariate Gaussian distribution is a bit specific. Therefore, I suggest the authors could include more comparison algorithms for the adversarial training in the related work section\n",
      "\n",
      "soundness: 2 fair\n",
      "\n",
      "presentation: 3 good\n",
      "\n",
      "contribution: 3 good\n",
      "\n",
      "rating: 6: marginally above the acceptance threshold\n",
      "\n",
      "confidence: 3: You are fairly confident in your assessment. It is possible that you did not understand some parts of the submission or that you are unfamiliar with some pieces of related work. Math/other details were not carefully checked.\n",
      "\n",
      "</s>\n",
      "\n",
      "===\n",
      "<s> You are reviewing the paper titled Stochastic Adversarial Networks for Multi-Domain Text Classification. The keywords are Multi-domain text classification, Adversarial training, Multivariate gaussian distribution. You will read this paper and write a review for it. USER: The abstract is：\n",
      "\n",
      "Adversarial training has played a pivotal role in the significant advancements of multi-domain text classification (MDTC). Recent MDTC methods often adopt the shared-private paradigm, wherein a shared feature extractor captures domain-invariant knowledge, while private feature extractors per domain extract domain-dependent knowledge. These approaches have demonstrated state-of-the-art performance. However, a major challenge remains: the exponential increase in model parameters as new domains emerge. To address this challenge, we propose the Stochastic Adversarial Network (SAN), which models multiple domain-specific feature extractors as a multivariate Gaussian distribution rather than weight vectors. With SAN, we can sample as many domain-specific feature extractors as necessary without drastically increasing the number of model parameters. Consequently, the model size of SAN remains comparable to having a single domain-specific feature extractor when data from multiple domains. Additionally, we incorporate domain label smoothing and robust pseudo-label regularization techniques to enhance the stability of the adversarial training and improve feature discriminability, respectively. The evaluations conducted on two prominent MDTC benchmarks validate the competitiveness of our proposed SAN method against state-of-the-art approaches.\n",
      "\n",
      "After reading this abstract, please write your review for it. ASSISTANT:\n",
      "---\n",
      " summary: The authors focus on stochastic adversarial networks, which model multiple domain-specific feature extractors as a multivariate Gaussian distribution rather than vector weights. This method leads to a lower model size as multiple domains emerge, compared to state-of-the-art method. For adversarial training, the authors use domain label smoothing and domain-robust pseudo-label normalization. The proposed method shows a significant advantage over the state-of-the-art approach.\n",
      "\n",
      "strengths: The paper is well-written and easy to read. The motivation of each component is clear and justified.\n",
      "\n",
      "The performance on the dataset is solid.\n",
      "\n",
      "weaknesses: One weakness is the computational cost for Stochastic adversarial networks: it is not clear to me which specific components of the algorithm requires high computational cost. I think that some explanation would make the paper more concrete and would help to understand how the proposed method is actually used and whether it is more efficient than traditional adversarial training.\n",
      "\n",
      "SAN uses an idea similar to domain adaptive adversarial training: instead of a single feature extractor for each domain, multiple extractors are trained using sampled normal distributions. While the authors did some empirical justification and ablation studies, it is not clear to me whether domain-unaware extractors are actually better than domain-aware extractors and whether sampling is actually beneficial. I think that a thorough empirical study comparing the proposed method with domain-aware extractors is necessary to prove the proposed method’s effectiveness.\n",
      "\n",
      "questions: Please see the weaknesses. It is also necessary to give more explanation or theoretical guarantee for the proposed method.\n",
      "\n",
      "soundness: 3 good\n",
      "\n",
      "presentation: 3 good\n",
      "\n",
      "contribution: 3 good\n",
      "\n",
      "rating: 6: marginally above the acceptance threshold\n",
      "\n",
      "confidence: 4: You are confident in your assessment, but not absolutely certain. It is unlikely, but not impossible, that you did not understand some parts of the submission or that you are unfamiliar with some pieces of related work.\n",
      "\n",
      "</s>\n",
      "\n",
      "===\n",
      "<s> You are reviewing the paper titled On the generalization capacity of neural networks during generic multimodal reasoning. The keywords are compositional generalization, compositionality, representation learning, out-of-distribution generalization, multimodal reasoning. You will read this paper and write a review for it. USER: The abstract is：\n",
      "\n",
      "The advent of the Transformer has led to the development of large language models (LLM), which appear to demonstrate human-like capabilities. To assess the generality of this class of models and a variety of other base neural network architectures to multimodal domains, we evaluated and compared their capacity for multimodal generalization. We introduce a multimodal question-answer benchmark to evaluate three specific types of out-of-distribution (OOD) generalization performance: distractor generalization (generalization in the presence of distractors), systematic compositional generalization (generalization to new task permutations), and productive compositional generalization (generalization to more complex tasks with deeper dependencies). While we found that most architectures faired poorly on most forms of generalization (e.g., RNNs and standard Transformers), models that leveraged cross-attention mechanisms between input domains, such as the Perceiver, fared better. Our positive results demonstrate that for multimodal distractor and systematic generalization, cross-attention is an important mechanism to integrate multiple sources of information. On the other hand, all architectures failed in productive generalization, suggesting fundamental limitations of existing architectures for specific types of multimodal OOD generalization.\n",
      "These results demonstrate the strengths and limitations of specific architectural components underlying modern neural models for multimodal reasoning. Finally, we provide *Generic COG* (gCOG), a configurable benchmark with several multimodal generalization splits, for future studies to explore.\n",
      "\n",
      "After reading this abstract, please write your review for it. ASSISTANT:\n",
      "---\n",
      " summary: This paper introduces a new benchmark called gCOG for evaluating distractor generalization, systematic compositional generalization, and productive compositional generalization in multimodal domains. gCOG includes several splits to cover different tasks with varying complexity and domain. This benchmark can thus be used to evaluate generalization performance of existing architectures and study their strengths and limitations in multimodal reasoning tasks. The results show that models that use cross-attention mechanisms between input domains perform better in some forms of generalization, but none are capable of productive compositional generalization, which is the most challenging problem of the benchmark. The authors therefore suggest that more research is needed to investigate the limitations of existing architectures and explore alternative approaches to achieving productive composition.\n",
      "\n",
      "strengths: 1. The paper introduces a comprehensive benchmark with different multimodal tasks that can be divided into several splits, covering different problems from simple ones to the most intricate one.\n",
      "2. The methodology used to evaluate the models on the benchmark is straightforward and the analysis results are very informatively and easy to follow.\n",
      "\n",
      "weaknesses: 1. While I think cross-attention in Transformer architecture is the most intuitive way to perform multimodal reasoning in my opinion, I was wondering if there are other ways that achieve similar or better performance in the benchmark. In particular, is there a significant difference in the model performance between models using cross-attention and those that utilize other types of attention mechanisms such as the Perceiver? Or are the results just attributed to the architecture?\n",
      "\n",
      "questions: 1. In your experimental section, you mention several models that you tested on gCOG, what are the settings in these models used in the testing? i.e., did you randomly sample questions from the split? What is the distribution of questions you used for the various models in testing?\n",
      "\n",
      "soundness: 3 good\n",
      "\n",
      "presentation: 4 excellent\n",
      "\n",
      "contribution: 3 good\n",
      "\n",
      "rating: 6: marginally above the acceptance threshold\n",
      "\n",
      "confidence: 3: You are fairly confident in your assessment. It is possible that you did not understand some pieces of the submission or that you are unfamiliar with some pieces of related work. Math/other details were not carefully checked.\n",
      "\n",
      "</s>\n",
      "\n",
      "===\n",
      "<s> You are reviewing the paper titled On the generalization capacity of neural networks during generic multimodal reasoning. The keywords are compositional generalization, compositionality, representation learning, out-of-distribution generalization, multimodal reasoning. You will read this paper and write a review for it. USER: The abstract is：\n",
      "\n",
      "The advent of the Transformer has led to the development of large language models (LLM), which appear to demonstrate human-like capabilities. To assess the generality of this class of models and a variety of other base neural network architectures to multimodal domains, we evaluated and compared their capacity for multimodal generalization. We introduce a multimodal question-answer benchmark to evaluate three specific types of out-of-distribution (OOD) generalization performance: distractor generalization (generalization in the presence of distractors), systematic compositional generalization (generalization to new task permutations), and productive compositional generalization (generalization to more complex tasks with deeper dependencies). While we found that most architectures faired poorly on most forms of generalization (e.g., RNNs and standard Transformers), models that leveraged cross-attention mechanisms between input domains, such as the Perceiver, fared better. Our positive results demonstrate that for multimodal distractor and systematic generalization, cross-attention is an important mechanism to integrate multiple sources of information. On the other hand, all architectures failed in productive generalization, suggesting fundamental limitations of existing architectures for specific types of multimodal OOD generalization.\n",
      "These results demonstrate the strengths and limitations of specific architectural components underlying modern neural models for multimodal reasoning. Finally, we provide *Generic COG* (gCOG), a configurable benchmark with several multimodal generalization splits, for future studies to explore.\n",
      "\n",
      "After reading this abstract, please write your review for it. ASSISTANT:\n",
      "---\n",
      " summary: The authors conducted experiments and analyses which examine whether different NN architectures can integrate information from different sources of multimodal data and how this affects their out-of-distribution generalization capabilities.  Specifically they investigated several different NN architectures including the Perceiver, RoBERTa, and T5 across three different types of OOD generalization tasks: distractor (DG), systematic compositional (SCOG), and productive compositional (PCOG).  Their analyses reveal that the perceptors and T5 performed better in DG than other architectures, and that no architecture performed consistently across all generalization types.\n",
      "\n",
      "strengths: - I found the work to be a reasonable and sophisticated first step into the domain of multimodal reasoning for NNs.  The author’s focus on the generalization capacities of different architectures is a useful perspective and the work could be a building block for more complex work to follow.\n",
      "- The paper is clearly written and easy to follow.\n",
      "\n",
      "weaknesses: - I’m not quite clear on how the definitions of each type of generalization are chosen; why not have one more than the number of architectures?\n",
      "- The performance of the 3 architectures are plotted, but no real discussion of which one is better in a certain setup for a given dataset.\n",
      "- It’s not really clear to me how good these performances for the PCOG and SCOG tasks actually are — one has a fixed set of questions and answers so all architectures perform “well” — and the other has more complex questions, but it is not totally clear how good these complex questions are. A more detailed explanation in the appendix would be helpful.\n",
      "- Some of the numbers seem to be a little small — the number of images is quite small for example; if you’re looking for generalization to out of distribution tasks I would have expected more data. Are the out of distribution questions also chosen randomly, or do you have a method for choosing them? This is important because randomness is potentially another issue with using out of distribution data.\n",
      "\n",
      "questions: - How do you choose the number of generalization types?\n",
      "- Do you have any data on how good you need to be at each task before generalization is even a useful thing?\n",
      "- How well do you all know DD; I’m unclear on how you’re defining that here.\n",
      "\n",
      "soundness: 3 good\n",
      "\n",
      "presentation: 3 good\n",
      "\n",
      "contribution: 3 good\n",
      "\n",
      "rating: 5: marginally below the acceptance threshold\n",
      "\n",
      "confidence: 3: You are fairly confident in your assessment. It is possible that you did not understand some parts of the submission or that you are unfamiliar with some pieces of related work. Math/other details were not carefully checked.\n",
      "\n",
      "</s>\n",
      "\n",
      "===\n",
      "<s> You are reviewing the paper titled On the generalization capacity of neural networks during generic multimodal reasoning. The keywords are compositional generalization, compositionality, representation learning, out-of-distribution generalization, multimodal reasoning. You will read this paper and write a review for it. USER: The abstract is：\n",
      "\n",
      "The advent of the Transformer has led to the development of large language models (LLM), which appear to demonstrate human-like capabilities. To assess the generality of this class of models and a variety of other base neural network architectures to multimodal domains, we evaluated and compared their capacity for multimodal generalization. We introduce a multimodal question-answer benchmark to evaluate three specific types of out-of-distribution (OOD) generalization performance: distractor generalization (generalization in the presence of distractors), systematic compositional generalization (generalization to new task permutations), and productive compositional generalization (generalization to more complex tasks with deeper dependencies). While we found that most architectures faired poorly on most forms of generalization (e.g., RNNs and standard Transformers), models that leveraged cross-attention mechanisms between input domains, such as the Perceiver, fared better. Our positive results demonstrate that for multimodal distractor and systematic generalization, cross-attention is an important mechanism to integrate multiple sources of information. On the other hand, all architectures failed in productive generalization, suggesting fundamental limitations of existing architectures for specific types of multimodal OOD generalization.\n",
      "These results demonstrate the strengths and limitations of specific architectural components underlying modern neural models for multimodal reasoning. Finally, we provide *Generic COG* (gCOG), a configurable benchmark with several multimodal generalization splits, for future studies to explore.\n",
      "\n",
      "After reading this abstract, please write your review for it. ASSISTANT:\n",
      "---\n",
      " summary: This paper presents a generalizable benchmark that evaluates several neural architectures, including Transformer and Perceiver, in multimodal question answering. The benchmark focuses on three different aspects of out-of sample generalization: distractor generalization, systematic compositional generalization and productive compositional generalization. The authors find that while modern deep model architecture shows robust performance in out of distribution problem on distractor generalization and systematic compositional generalization, this is not so for productive generalization. Therefore the authors argue that for productive generalization multimodality is important and one needs to model it. The paper suggests that more work is required in the area of multimodality to produce generalizable models.\n",
      "\n",
      "strengths: This paper presents a comprehensive generalizable benchmark for evaluating multimodal reasoning models.\n",
      "More work is required in the area of multimodality to produce generalizable models, highlighting its importance.\n",
      "\n",
      "weaknesses: I believe that this paper would benefit from a careful reading of the related state of the art, that includes work related to multimodal reasoning and out-of sample generalization. The paper includes work that is unrelated at this point and therefore the work is not complete. The generalizable benchmark that is proposed also raises a number of doubts. It is difficult to understand the importance and relevance of the different splits and therefore the relevance of the presented results.\n",
      "\n",
      "questions: In the introduction page 3, the authors talk about \"generalizable models, which exhibit consistent performance across different domains\". I understand that the models are generalizable for different types of out of sample generalization problems but what is the relevance to this question?\n",
      "In the introduction the authors also talk about \"more complex functions for the downstream task\". How does this relate to the definition of the downstream task that needs to be defined?\n",
      "Is the task always multimodal or are there multimodal-monomodal settings?\n",
      "Is the data from LENA and MML?\n",
      "\n",
      "soundness: 2 fair\n",
      "\n",
      "presentation: 3 good\n",
      "\n",
      "contribution: 3 good\n",
      "\n",
      "rating: 6: marginally above the acceptance threshold\n",
      "\n",
      "confidence: 4: You are confident in your assessment, but not absolutely certain. It is unlikely, but not impossible, that you did not understand some parts of the submission or that you are unfamiliar with some pieces of related work.\n",
      "\n",
      "</s>\n",
      "\n",
      "===\n",
      "<s> You are reviewing the paper titled CLIP meets Model Zoo Experts: Pseudo-Supervision for Visual Enhancement. The keywords are Contrastive Learning, CLIP, Distillation, Dense prediction, 3D Understanding, Task-specific experts. You will read this paper and write a review for it. USER: The abstract is：\n",
      "\n",
      "Contrastive language image pretraining (CLIP) is a standard method for training vision-language models. While CLIP is scalable, promptable, and robust to distribution shifts on image classification tasks, it lacks object localization capabilities.  This paper studies the following question: Can we augment CLIP training with task-specific vision models from model zoos to improve its visual representations? Towards this end, we leverage open-source task-specific vision models to generate pseudo-labels for an uncurated and noisy image-text dataset. Subsequently, we train CLIP models on these pseudo-labels in addition to the contrastive training on image and text pairs. This simple setup shows substantial improvements of up to 16.3\\% across different vision tasks, including segmentation, detection, depth estimation, and surface normal estimation. Importantly, these enhancements are achieved without compromising CLIP's existing capabilities, including its proficiency in promptable zero-shot classification.\n",
      "\n",
      "After reading this abstract, please write your review for it. ASSISTANT:\n",
      "---\n",
      " summary: This paper studies if it's possible to enhance an CLIP model by adding a model that has localization capabilities. They use several model zoo models and add pseudo-labeling to CLIP to train CLIP. They compare with CLIP's performance on VL-50k (zero shot/text only) and also show that the additional training data improves CLIPs capability in zero shot classification. \n",
      "\n",
      "strengths: The paper is well written.\n",
      "\n",
      "weaknesses: * They claim that localization capabilities are important. Which means the quality of the model zoo models is important, otherwise CLIP performs well in many of the tasks. However they only study 4 models (from Model Zoo) and report improved performance without any comparison to other model zoo models (or just one from MSCOCO).\n",
      "\n",
      "* Zero-shot performance of CLIP is very poor. It is almost always on par with CLIP itself. The authors then claim that CLIP+model zoo is better than CLIP. However, no such improvement appears in Table 2 at all on 0-shot 60 and 120. Also, since you would expect a high zero-shot performance from CLIP already, I think this experiment does not justify the claim that localization is important.\n",
      "\n",
      "\n",
      "* The method is simple. However, the paper claims that \"localization capabilities may enable models to learn strong representations that capture task-specific semantics ... \" But the only evidence provided is that after the CLIP+model zoo is trained on the pseudo-labels, the task results are  \"consistently ... better than\" the unenhanced CLIP. While CLIP is better, the fact a model zoo model is better than CLIP + Pseudo-labeling + Localization model zoo is a bit harder to believe.\n",
      "\n",
      "\n",
      "* In Table 1, the only model showing improvement is R-FPN, which would be very different from the model zoo model that is actually used to generate pseudo-labels. The author's claim that CLIP+model zoo surpasses CLIP on every single task, which requires zero assumption from the reader and seems way too strong to me.\n",
      "\n",
      "\n",
      "* The Figure 1 is hard to read.\n",
      "\n",
      "questions: Please see Weakness above.\n",
      "\n",
      "soundness: 3 good\n",
      "\n",
      "presentation: 2 fair\n",
      "\n",
      "contribution: 2 fair\n",
      "\n",
      "rating: 3: reject, not good enough\n",
      "\n",
      "confidence: 2: You are fairly confident in your assessment. It is possible that you did not understand some parts of the submission or that you are unfamiliar with some pieces of related work. Math/other details were not carefully checked.\n",
      "\n",
      "</s>\n",
      "\n",
      "===\n",
      "<s> You are reviewing the paper titled CLIP meets Model Zoo Experts: Pseudo-Supervision for Visual Enhancement. The keywords are Contrastive Learning, CLIP, Distillation, Dense prediction, 3D Understanding, Task-specific experts. You will read this paper and write a review for it. USER: The abstract is：\n",
      "\n",
      "Contrastive language image pretraining (CLIP) is a standard method for training vision-language models. While CLIP is scalable, promptable, and robust to distribution shifts on image classification tasks, it lacks object localization capabilities.  This paper studies the following question: Can we augment CLIP training with task-specific vision models from model zoos to improve its visual representations? Towards this end, we leverage open-source task-specific vision models to generate pseudo-labels for an uncurated and noisy image-text dataset. Subsequently, we train CLIP models on these pseudo-labels in addition to the contrastive training on image and text pairs. This simple setup shows substantial improvements of up to 16.3\\% across different vision tasks, including segmentation, detection, depth estimation, and surface normal estimation. Importantly, these enhancements are achieved without compromising CLIP's existing capabilities, including its proficiency in promptable zero-shot classification.\n",
      "\n",
      "After reading this abstract, please write your review for it. ASSISTANT:\n",
      "---\n",
      " summary: This paper proposes to train a CLIP model with a pretrained task-specific model as a \"helper\". In doing so, the paper claims that the trained CLIP model achieves improvement of up to 16.3% in segmentation.\n",
      "\n",
      "strengths: ### Use of a publicly available model: \n",
      "The paper leverages a publicly available model, viz, DenseNet 121 (D121) to augment CLIP. \n",
      "\n",
      "## Advantages of using \"model zoo\" models\n",
      "1. Since this work leverages a publicly available pre-trained model, future research will not need to retrain the model.\n",
      "\n",
      "2. Leveraging the pre-trained model for an auxiliary task of generating pseudo-labels for a dataset has not been studied before.\n",
      "\n",
      "weaknesses: The primary shortcoming of this paper is in the methodological and technical deficiencies. While training a visual model with a CLIP model seems like a viable approach, the author uses a pre-trained model for this purpose without any significant contribution. In addition, the proposed method contains some fundamental errors. \n",
      "\n",
      "## Methodological issues\n",
      "1. The approach of training with a pre-trained model is based on the assumption that a pre-trained model will bring improvements. If the pre-trained model is a task-specific supervision model, it can provide supervision of the task. On the other hand, if it is a fine-tuned model for a specific task, can it provide the necessary visual information for the pre-trained CLIP model? Furthermore, in the proposed algorithm, what is the purpose of training the pre-trained model with a CLIP model? How does a pre-trained CLIP based model help with a pre-trained model for a specific task? These are the issues that need to be clarified.\n",
      "\n",
      "2. The reason the the proposed model is being used for training is mentioned as \"the pre-trained model can provide a representation of the entire object, reducing inaccuracies due to the noisy dataset.\" Is it necessary to train the pre-trained model with the CLIP model? If this is the case, why is the pre-trained model not trained with the visual model only? If this is not the reason for training the pre-trained model, then why not train it with a CLIP-based model (CLIP-DenseNet)? This is essential, and the purpose needs to be clarified. \n",
      "\n",
      "3. In addition, the proposed method contains some fundamental errors. Is the proposed pre-trained model, \"task zoo image\", really helpful for visual enhancement? What are the advantages of using this pre-trained model in the proposed method? Instead of focusing on these issues, the author uses an extensive number of pre-trained models, with most of the proposed models consisting of one-shot image-text learning. Furthermore, the proposed model has not been evaluated and can lead to overfitting in the pre-training process. \n",
      "\n",
      "## Technical issues\n",
      "1. First, on the data collection side, the proposed method does not provide a clear explanation regarding why it is proposed to collect the data using the approach where \"the pre-trained model can provide a representation of the entire object, reducing the inaccuracies due to noisy datasets.\" The reason needs to be clarified. \n",
      "\n",
      "2. The proposed method needs to address the error in collecting the data. What if the image and the caption do not have the same representation? Moreover, is it possible for the pre-trained model to collect the data? \n",
      "\n",
      "3. The proposed method's approach to training the pre-trained visual model with a CLIP model is unclear. \n",
      "\n",
      "4. The proposed method has not provided explanations regarding the pre-training process. The reason for training with CLIP is not clear. What is the role of CLIP in visual enhancement? The author needs to provide an explanation. \n",
      "\n",
      "5. Finally, in conclusion, the proposed method needs a significant improvement. A detailed explanation regarding each of the proposed method's components is also necessary to provide a clear understanding of the algorithm.\n",
      "\n",
      "questions: How valuable are the proposed methods that augment CLIP with a task-specific method based on extensive training on the task zoo image dataset? Is the pre-trained model (for task zoo or any other) also used as a reference point of the standard model?\n",
      "\n",
      "soundness: 1 poor\n",
      "\n",
      "presentation: 2 fair\n",
      "\n",
      "contribution: 2 fair\n",
      "\n",
      "rating: 3: reject, not good enough\n",
      "\n",
      "confidence: 4: You are confident in your assessment, but not absolutely certain. It is unlikely, but not impossible, that you\n",
      "\n"
     ]
    }
   ],
   "source": [
    "generation_config = GenerationConfig(\n",
    "    do_sample=True,\n",
    "    temerature=0.7,\n",
    "    max_new_tokens=1024\n",
    ")\n",
    "\n",
    "args.batch_size = 1\n",
    "args.num_workers = 1\n",
    "\n",
    "dp_rank=0\n",
    "dp_world_size = 1\n",
    "\n",
    "sampler = DistributedSampler(\n",
    "    dataset,\n",
    "    shuffle=False,\n",
    "    drop_last=False,\n",
    "    rank=dp_rank,\n",
    "    num_replicas=dp_world_size\n",
    ")\n",
    "\n",
    "dataloader = DataLoader(\n",
    "    dataset,\n",
    "    sampler=sampler,\n",
    "    batch_size=args.batch_size,\n",
    "    num_workers=args.num_workers,\n",
    "    collate_fn=dataset.collate\n",
    ")\n",
    "\n",
    "model.eval()\n",
    "model.cuda()\n",
    "\n",
    "results = []\n",
    "\n",
    "for num_samples, (input_batch, gen_batch, other_batch) in enumerate(dataloader):\n",
    "    move_dict_to_device(gen_batch, model.device)\n",
    "    output_ids = model.generate(**gen_batch, generation_config=generation_config)\n",
    "    inputs = tokenizer.batch_decode(gen_batch['input_ids'])\n",
    "    outputs = tokenizer.batch_decode(output_ids[:, gen_batch['input_ids'].size(-1):])\n",
    "    for input_str, output_str in zip(inputs, outputs):\n",
    "        print('===')\n",
    "        print(input_str)\n",
    "        print('---')\n",
    "        print(output_str)\n",
    "        print()\n",
    "    results.append(\n",
    "        {\n",
    "            \"prompt\": input_str,\n",
    "            \"response\": output_str\n",
    "        }\n",
    "    )\n",
    "    if num_samples == 16:\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "with open('results.json', 'w', encoding='utf-8') as f:\n",
    "    json.dump(results, f, indent=2, ensure_ascii=False)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "vllm",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}