avoid expensive queries that do not change certain results (#53)

spatie · Oct 22, 2024 · 31763e5 · 31763e5
1 parent 053c9df
commit 31763e5
Showing 1 changed file with 5 additions and 3 deletions.
diff --git a/src/Robots.php b/src/Robots.php
@@ -41,15 +41,17 @@ public function mayIndex(string $url, ?string $userAgent = null): bool
 
         $robotsTxt = $this->robotsTxt ?? RobotsTxt::create($this->createRobotsUrl($url));
 
+        if (! $robotsTxt->allows($url, $userAgent)) {
+            return false;
+        }
+
         $content = @file_get_contents($url);
 
         if ($content === false) {
             throw new InvalidArgumentException("Could not read url `{$url}`");
         }
 
-        return
-            $robotsTxt->allows($url, $userAgent)
-            && RobotsMeta::create($content)->mayIndex()
+        return RobotsMeta::create($content)->mayIndex()
             && RobotsHeaders::create($http_response_header ?? [])->mayIndex();
     }