From a68b71ea732bd3502aec6dddbc3a9027138ab8c9 Mon Sep 17 00:00:00 2001 From: "Justin W. Hart" Date: Wed, 6 Feb 2019 00:01:56 -0600 Subject: [PATCH 1/4] Add new Hand Me That task, which is a spinoff from prior I Want This task. --- Rulebook.tex | 3 ++ scoresheets/HandMeThat.tex | 18 ++++++++++++ tasks/HandMeThat.tex | 60 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 81 insertions(+) create mode 100644 scoresheets/HandMeThat.tex create mode 100644 tasks/HandMeThat.tex diff --git a/Rulebook.tex b/Rulebook.tex index 73bed1a5..3fbd73b2 100644 --- a/Rulebook.tex +++ b/Rulebook.tex @@ -85,6 +85,9 @@ \chapter{Tests in Stage II} \newpage \input{tasks/SetTable} +\newpage +\input{tasks/HandMeThat} + \newpage \input{tests/Finals} diff --git a/scoresheets/HandMeThat.tex b/scoresheets/HandMeThat.tex new file mode 100644 index 00000000..d478e961 --- /dev/null +++ b/scoresheets/HandMeThat.tex @@ -0,0 +1,18 @@ + +The maximum time for this test is 15 minutes. + +\begin{scorelist} + \scoreheading{Communicating} + \scoreitem[5]{500}{Correctly determine an item on the first try} + \penaltyitem[5]{-200}{Correctly determine an item on the second try} + \penaltyitem[5]{-400}{Correctly determine an item on the third or a subsequent try} + \penaltyitem[5]{-400}{Correctly determine an item after the referee has picked it up or touched it} + + % No longer necessary, computes automatically + % \setTotalScore{1000} +\end{scorelist} + + +% Local Variables: +% TeX-master: "Rulebook" +% End: diff --git a/tasks/HandMeThat.tex b/tasks/HandMeThat.tex new file mode 100644 index 00000000..2e1700d1 --- /dev/null +++ b/tasks/HandMeThat.tex @@ -0,0 +1,60 @@ +\section{Hand Me That [Party Host]} +A guest at the party speaks English, but with only a limited vocabulary. The robot will assist them in obtaining things that they gesture for. + +%\subsection{Focus} +%Joint attention is a well-studied and important task in Human-Robot Interaction. The goal of this task is to really challenge the teams to perform a hard HRI task. + +\subsection{Main Goal} +Five collections of items are set out. The referee will choose a random item from each collection to gesture to or describe to the robot. The robot must tell the referee its name, point to, or touch it. The robot can prompt the user with requests like \textit{point to it} or \textit{describe it to me}. + +\noindent\textbf{Reward:} 2500pts\\ + + +% %% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% +% Setup +% +% %% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\subsection{Setup} +\begin{enumerate} + \item \textbf{Object Collections:} The test area should collections of objects on various surfaces (2-5). Difficulty of understanding the point gesture will be regulated by how far apart the objects in each collection are ($100cm - 1m$). Difficulty of describing the object is difficult to quantify, and should be considered random. + + \item \textbf{Random Object Cards:} The referee will carry a set of cards with random objects, arranged by difficulty. The job of the referee is to get the robot to say the name of, or touch, the object that they gesture to or describe, at the robot's discretion. + +\end{enumerate} + + +% %% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% +% Setup +% +% %% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\subsection{Procedure} +\begin{enumerate} + \item \textbf{Pick an object} The robot will ask the referee what they want. The referee will walk near to the object that they want, and gesture to it. The robot can ask clarifying questions to help identify the object. The referee will not say the name of the object, or its color, but anything else is fair game. + \item \textbf{Repeat} Repeat up to 5 times for the maximum score. + + +\end{enumerate} + + +% %% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% +% Additional Rules +% +% %% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\subsection{Additional rules and remarks} +\begin{enumerate} + \item \textbf{Keep going:} The robot should keep trying to determine the referred to object until they score or run out of time. + \item \textbf{Pass:} The robot may say pass to try the next object. + \item \textbf{Touch:} The robot may ask the referee may pick up or touch the object by saying \textit{pick up} or \textit{touch}. If the robot does this, identifying the object is worth only 100 points. + \item \textbf{Oops:} Incorrect guesses reduce the value of the correct guess by 200 points, each, but cannot make the value of the correct guess go below 100 points. + +\end{enumerate} +\newpage +\subsection{Score sheet} +\input{scoresheets/HandMeThat.tex} + +% Local Variables: +% TeX-master: "Rulebook" +% End: From 7f74ed140abf9a6e3df54f0a99ef304be54a0ccc Mon Sep 17 00:00:00 2001 From: "Justin W. Hart" Date: Mon, 11 Feb 2019 23:41:08 -0600 Subject: [PATCH 2/4] Added Mauricio's suggested rules. --- tasks/HandMeThat.tex | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tasks/HandMeThat.tex b/tasks/HandMeThat.tex index 2e1700d1..9984439e 100644 --- a/tasks/HandMeThat.tex +++ b/tasks/HandMeThat.tex @@ -49,6 +49,8 @@ \subsection{Additional rules and remarks} \item \textbf{Pass:} The robot may say pass to try the next object. \item \textbf{Touch:} The robot may ask the referee may pick up or touch the object by saying \textit{pick up} or \textit{touch}. If the robot does this, identifying the object is worth only 100 points. \item \textbf{Oops:} Incorrect guesses reduce the value of the correct guess by 200 points, each, but cannot make the value of the correct guess go below 100 points. +\item\textbf{Colors and categories:} Asking for the color or category of a pointed object applies a penalty of 400 points for that particular object. +\item\textbf{Uneducated operator:} The referee may instruct the operator to answer \emph{I don't understand} or \emph{I don't know} if the robot asks complex questions or is attempting blind guessing. \end{enumerate} \newpage From a4cb9b22558635bd07852886a8a68e46c632634b Mon Sep 17 00:00:00 2001 From: "Justin W. Hart" Date: Mon, 11 Feb 2019 23:48:33 -0600 Subject: [PATCH 3/4] Fixes based on github converstaion. --- scoresheets/HandMeThat.tex | 12 +++++++----- tasks/HandMeThat.tex | 17 ++++++++++------- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/scoresheets/HandMeThat.tex b/scoresheets/HandMeThat.tex index d478e961..fe0e0465 100644 --- a/scoresheets/HandMeThat.tex +++ b/scoresheets/HandMeThat.tex @@ -1,12 +1,14 @@ -The maximum time for this test is 15 minutes. +The maximum time for this test is 10 minutes. \begin{scorelist} \scoreheading{Communicating} - \scoreitem[5]{500}{Correctly determine an item on the first try} - \penaltyitem[5]{-200}{Correctly determine an item on the second try} - \penaltyitem[5]{-400}{Correctly determine an item on the third or a subsequent try} - \penaltyitem[5]{-400}{Correctly determine an item after the referee has picked it up or touched it} + \scoreitem[5]{500}{Correctly determine an item on the first attempt} + \penaltyitem[5]{-200}{Correctly determine an item on the second attempt} + \penaltyitem[5]{-400}{Correctly determine an item on a subsequent attempt} + \penaltyitem[5]{-150}{Asking 1 clarifying question.} + \penaltyitem[5]{-300}{Asking 2 clarifying questions.} + \penaltyitem[5]{-450}{Asking 3 clarifying questions.} % No longer necessary, computes automatically % \setTotalScore{1000} diff --git a/tasks/HandMeThat.tex b/tasks/HandMeThat.tex index 9984439e..2ce82743 100644 --- a/tasks/HandMeThat.tex +++ b/tasks/HandMeThat.tex @@ -5,7 +5,7 @@ \section{Hand Me That [Party Host]} %Joint attention is a well-studied and important task in Human-Robot Interaction. The goal of this task is to really challenge the teams to perform a hard HRI task. \subsection{Main Goal} -Five collections of items are set out. The referee will choose a random item from each collection to gesture to or describe to the robot. The robot must tell the referee its name, point to, or touch it. The robot can prompt the user with requests like \textit{point to it} or \textit{describe it to me}. +The robot identifies (touching or naming) the object at which the operator is pointing at. \noindent\textbf{Reward:} 2500pts\\ @@ -17,21 +17,25 @@ \subsection{Main Goal} % %% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% \subsection{Setup} \begin{enumerate} - \item \textbf{Object Collections:} The test area should collections of objects on various surfaces (2-5). Difficulty of understanding the point gesture will be regulated by how far apart the objects in each collection are ($100cm - 1m$). Difficulty of describing the object is difficult to quantify, and should be considered random. - - \item \textbf{Random Object Cards:} The referee will carry a set of cards with random objects, arranged by difficulty. The job of the referee is to get the robot to say the name of, or touch, the object that they gesture to or describe, at the robot's discretion. +\item \textbf{Location:} This takes place in a room in the arena. +\item \textbf{Starting position:} The robot and the operator stand in a predefined location announced beforehand % (OC instructions: announce this 2 hours before the test). +\item \textbf{Objects:} Groups of 2--5 objects are placed at random locations on the room (avg distance $50cm -- 150cm$ from starting position). +\item \textbf{Deck:} The referee has a deck of objects to request, one per group, sorted by distance. \end{enumerate} % %% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % -% Setup +% Procedure % % %% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% \subsection{Procedure} \begin{enumerate} - \item \textbf{Pick an object} The robot will ask the referee what they want. The referee will walk near to the object that they want, and gesture to it. The robot can ask clarifying questions to help identify the object. The referee will not say the name of the object, or its color, but anything else is fair game. + \item \textbf{Pick an object} The robot will ask the operator: \emph{what do you need?}. % We rule out natural language interaction +Then the operator walks near to the object and points at it. +The robot can ask as many questions as necessary. +The operator does not know the name of the object. \item \textbf{Repeat} Repeat up to 5 times for the maximum score. @@ -47,7 +51,6 @@ \subsection{Additional rules and remarks} \begin{enumerate} \item \textbf{Keep going:} The robot should keep trying to determine the referred to object until they score or run out of time. \item \textbf{Pass:} The robot may say pass to try the next object. - \item \textbf{Touch:} The robot may ask the referee may pick up or touch the object by saying \textit{pick up} or \textit{touch}. If the robot does this, identifying the object is worth only 100 points. \item \textbf{Oops:} Incorrect guesses reduce the value of the correct guess by 200 points, each, but cannot make the value of the correct guess go below 100 points. \item\textbf{Colors and categories:} Asking for the color or category of a pointed object applies a penalty of 400 points for that particular object. \item\textbf{Uneducated operator:} The referee may instruct the operator to answer \emph{I don't understand} or \emph{I don't know} if the robot asks complex questions or is attempting blind guessing. From 782c4ab7cca11d82cd554252373a592c89e209b1 Mon Sep 17 00:00:00 2001 From: "Justin W. Hart" Date: Thu, 21 Feb 2019 00:20:24 -0600 Subject: [PATCH 4/4] Fixes per @johaq's suggestions. --- scoresheets/HandMeThat.tex | 3 ++- tasks/HandMeThat.tex | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/scoresheets/HandMeThat.tex b/scoresheets/HandMeThat.tex index fe0e0465..104aaa44 100644 --- a/scoresheets/HandMeThat.tex +++ b/scoresheets/HandMeThat.tex @@ -5,7 +5,8 @@ \scoreheading{Communicating} \scoreitem[5]{500}{Correctly determine an item on the first attempt} \penaltyitem[5]{-200}{Correctly determine an item on the second attempt} - \penaltyitem[5]{-400}{Correctly determine an item on a subsequent attempt} + \penaltyitem[5]{-400}{Correctly determine an item on the third or fourth attempt} + \penaltyitem[5]{-500}{Correctly determine an item on a subsequent attempt} \penaltyitem[5]{-150}{Asking 1 clarifying question.} \penaltyitem[5]{-300}{Asking 2 clarifying questions.} \penaltyitem[5]{-450}{Asking 3 clarifying questions.} diff --git a/tasks/HandMeThat.tex b/tasks/HandMeThat.tex index 2ce82743..c8ab04ca 100644 --- a/tasks/HandMeThat.tex +++ b/tasks/HandMeThat.tex @@ -51,7 +51,7 @@ \subsection{Additional rules and remarks} \begin{enumerate} \item \textbf{Keep going:} The robot should keep trying to determine the referred to object until they score or run out of time. \item \textbf{Pass:} The robot may say pass to try the next object. - \item \textbf{Oops:} Incorrect guesses reduce the value of the correct guess by 200 points, each, but cannot make the value of the correct guess go below 100 points. + \item \textbf{Oops:} Incorrect guesses reduce the value of the correct guess by 200 points, each, the first two times. Guessing correctly on the third or fourth attempt is worth 100 points. After the fourth guess is worth no points. \item\textbf{Colors and categories:} Asking for the color or category of a pointed object applies a penalty of 400 points for that particular object. \item\textbf{Uneducated operator:} The referee may instruct the operator to answer \emph{I don't understand} or \emph{I don't know} if the robot asks complex questions or is attempting blind guessing.