Merge pull request #1 from 0x-pankaj/fix/robots-txt-parsing

fix(crawler): improve robots.txt parsing accuracy

Author: sharafdin

src/crawler/robots.ts

@@ -113,8 +113,10 @@ export class RobotsParser {
     this.sitemaps.set(host, sitemaps);
   }
 
+
   /**
    * Check if a URL is allowed by robots.txt
+   * Follows Google's spec: longest matching pattern wins
    */
   isAllowed(url: string, userAgent = '*'): boolean {
     try {
@@ -136,28 +138,48 @@ export class RobotsParser {
         return true; // No matching rules = allow
       }
 
-      // Check rules (more specific rules take precedence)
+      // Collect ALL matching patterns from all rules
+      const matches: { pattern: string; isAllow: boolean }[] = [];
+
       for (const rule of matchingRules) {
-        // Check disallow first
+        // Collect matching allow patterns
+        for (const allow of rule.allow) {
+          if (this.pathMatches(path, allow)) {
+            matches.push({ pattern: allow, isAllow: true });
+          }
+        }
+        // Collect matching disallow patterns
         for (const disallow of rule.disallow) {
           if (this.pathMatches(path, disallow)) {
-            // Check if there's a more specific allow
-            for (const allow of rule.allow) {
-              if (this.pathMatches(path, allow) && allow.length > disallow.length) {
-                return true;
-              }
-            }
-            return false;
+            matches.push({ pattern: disallow, isAllow: false });
           }
         }
       }
 
-      return true;
+      // No matching patterns = allow
+      if (matches.length === 0) {
+        return true;
+      }
+
+      // Find the longest matching pattern (Google's spec: most specific wins)
+      // If tied, allow wins (per spec: "allow" takes precedence on equal length)
+      const longest = matches.reduce((best, current) => {
+        if (current.pattern.length > best.pattern.length) {
+          return current;
+        }
+        if (current.pattern.length === best.pattern.length && current.isAllow) {
+          return current; // Allow wins on tie
+        }
+        return best;
+      });
+
+      return longest.isAllow;
     } catch {
       return true;
     }
   }
 
+
   /**
    * Get crawl delay for a host
    */
@@ -191,17 +213,23 @@ export class RobotsParser {
   }
 
   private pathMatches(path: string, pattern: string): boolean {
-    // Simple pattern matching (supports * and $ wildcards)
+    // Robots.txt pattern matching (supports * and $ wildcards)
+    // Per Google spec: * matches any sequence, $ means end-of-URL
     if (pattern === '/') return true;
 
+    // Check if pattern ends with $ (exact match anchor)
+    const hasEndAnchor = pattern.endsWith('$');
+    const patternToConvert = hasEndAnchor ? pattern.slice(0, -1) : pattern;
+
     // Convert pattern to regex
-    let regex = pattern
+    // Escape special regex chars EXCEPT * (we handle it separately)
+    let regex = patternToConvert
       .replace(/[.+?^${}()|[\]\\]/g, '\\$&') // Escape special chars
       .replace(/\*/g, '.*'); // * becomes .*
 
-    // $ at end means exact match
-    if (regex.endsWith('\\$')) {
-      regex = regex.slice(0, -2) + '$';
+    // Add end anchor if pattern ended with $
+    if (hasEndAnchor) {
+      regex += '$';
     }
 
     try {