diff --git a/index.js b/index.js
index 92b8bde..f65dd74 100644
--- a/index.js
+++ b/index.js
@@ -35,6 +35,13 @@ module.exports = app => {
 
   app.on('pull_request.closed', async context => {
     const params = context.issue()
+
+    const thisPR = await context.github.pullRequests.get(params)
+    if (!thisPR.data.merged) {
+      app.log('PR is not merged, but closed')
+      return
+    }
+
     const comments  = await context.github.issues.listComments(params)
 
     // Obtain all targets
@@ -51,12 +58,7 @@ module.exports = app => {
 
     if (targets.length === 0) {
       app.log('Nothing to backport')
-      return
-    }
-
-    const thisPR = await context.github.pullRequests.get(params)
-    if (!thisPR.data.merged) {
-      app.log('PR is not merged, but closed')
+      pr.askForBackports(thisPR.data.number)
       return
     }
 
@@ -64,7 +66,7 @@ module.exports = app => {
 
     app.log(targets)
     const success = await backport(context, context.issue.number, targets)
-    
+
     if (success) {
       pr.removeBackportRequestLabel(context)
     }
diff --git a/lib/pr.js b/lib/pr.js
index 9b691db..3e8b319 100644
--- a/lib/pr.js
+++ b/lib/pr.js
@@ -58,6 +58,14 @@ module.exports = {
         )
     },
 
+    askForBackports: async function(context, target, prId) {
+        return context.github.issues.createComment(
+            context.issue({
+                body: 'Should this be backported?'
+            })
+        )
+    },
+
     getReviewers: async function(context) {
         const reviewers1 = await context.github.pullRequests.listReviewRequests(context.issue())
         const reviewers2 = await context.github.pullRequests.listReviews(context.issue())
@@ -76,4 +84,4 @@ module.exports = {
 
         return context.github.pullRequests.createReviewRequest(params);
     }
-}
\ No newline at end of file
+}