test_firecrawl.py

#!/usr/bin/env python3
"""
FireCrawl API Test Script
Simple script to test the FireCrawl API connection
"""

import os
import json
import requests
from dotenv import load_dotenv

# Load environment variables
load_dotenv()

# Get API key
FIRECRAWL_API_KEY = os.environ.get("FIRECRAWL_API_KEY")
if not FIRECRAWL_API_KEY:
    print("ERROR: FIRECRAWL_API_KEY not found in environment variables")
    exit(1)

print(f"FireCrawl API Key: {FIRECRAWL_API_KEY[:5]}... (redacted)")

# Test different API endpoint variations
API_ENDPOINTS = [
    "https://api.firecrawl.dev/v1/scrape",
    "https://firecrawl.dev/api/v1/scrape",
    "https://api.firecrawl.dev/api/v1/scrape",
    "https://firecrawl.dev/v1/scrape",
    # Additional endpoints to try
    "https://api.firecrawl.dev/v1/chrome",
    "https://api.firecrawl.dev/chrome",
    "https://api.firecrawl.dev/chrome/store"
]

# Test payload - simplified to bare minimum
payload = {
    "url": "https://news.ycombinator.com/"
}

headers = {
    "Content-Type": "application/json",
    "Authorization": f"Bearer {FIRECRAWL_API_KEY}"
}

# Try each endpoint
for endpoint in API_ENDPOINTS:
    print("\n" + "="*50)
    print(f"Testing endpoint: {endpoint}")
    print("="*50)
    
    try:
        print(f"Sending request to {endpoint}...")
        response = requests.post(
            endpoint,
            headers=headers,
            json=payload,
            timeout=10  # 10 second timeout
        )
        
        print(f"Response status code: {response.status_code}")
        print(f"Response headers: {dict(response.headers)}")
        
        if response.status_code == 200:
            print("SUCCESS! API call succeeded")
            print("\nResponse preview:")
            
            # Try to parse the response
            try:
                data = response.json()
                print(json.dumps(data, indent=2)[:500] + "...")
                
                # Save the response to a file
                with open(f"firecrawl_response_{endpoint.split('/')[-2]}.json", "w") as f:
                    json.dump(data, f, indent=2)
                print(f"Full response saved to firecrawl_response_{endpoint.split('/')[-2]}.json")
                
                # Exit on first success
                print("\nTest successful! First working endpoint found.")
                break
            except json.JSONDecodeError:
                print("Failed to parse JSON response:")
                print(response.text[:1000] + "...")
        else:
            print(f"ERROR: Received status code {response.status_code}")
            print("Response body:")
            print(response.text[:1000] + "...")
    except Exception as e:
        print(f"ERROR: Failed to connect to {endpoint}")
        print(f"Error details: {str(e)}")

print("\nAPI testing complete")