How We Classify 16,024+ AI Tools
from a 102M-Domain Corpus

#!/usr/bin/env python3
"""AI Tool Classification Pipeline — feature extraction and category prediction."""

import json
from dataclasses import dataclass
from typing import Dict, List, Tuple

@dataclass
class DomainFeatures:
    domain: str
    title: str
    meta_description: str
    headings: List[str]
    body_text: str
    has_chat_interface: bool
    has_code_editor: bool
    has_file_upload: bool
    has_image_canvas: bool
    ai_sdk_references: List[str]
    api_docs_present: bool

CATEGORY_LABELS = [
    "AI Chatbots", "AI Code Assistants",
    "AI Image Generators", "AI Video Tools",
    "AI Audio/Music", "AI Writing Tools",
    "AI Search Engines", "AI Data Analytics",
    "AI Marketing Tools", "AI Design Tools",
    "AI Education", "AI Healthcare",
    "AI Finance", "AI Legal",
    "AI HR/Recruiting", "AI Customer Service",
    "AI Productivity", "AI Developer Tools",
]

def extract_features(domain: str, html: str, headers: dict) -> DomainFeatures:
    """Extract classification features from raw HTTP response."""
    from html.parser import HTMLParser
    title = extract_tag_content(html, "title")
    meta_desc = extract_meta(html, "description")
    headings = extract_all_headings(html)
    body = extract_visible_text(html, max_tokens=2000)

    return DomainFeatures(
        domain=domain,
        title=title,
        meta_description=meta_desc,
        headings=headings,
        body_text=body,
        has_chat_interface="chat" in html.lower() and "textarea" in html.lower(),
        has_code_editor=any(s in html for s in ["monaco", "codemirror", "ace-editor"]),
        has_file_upload='type="file"' in html or "dropzone" in html.lower(),
        has_image_canvas="<canvas" in html and "generate" in html.lower(),
        ai_sdk_references=detect_ai_sdks(html),
        api_docs_present=any(p in html for p in ["/api", "/docs", "swagger"]),
    )

def classify(features: DomainFeatures) -> List[Tuple[str, float]]:
    """Run the classification model and return ranked predictions."""
    feature_vector = vectorize(features)
    probabilities = model.predict(feature_vector)  # transformer output
    ranked = sorted(
        zip(CATEGORY_LABELS, probabilities),
        key=lambda x: x[1], reverse=True
    )
    return [(label, round(prob, 4)) for label, prob in ranked if prob > 0.05]

# Example: classify a discovered domain
features = extract_features("example-ai-code.dev", raw_html, resp_headers)
predictions = classify(features)
print(f"Domain:  {features.domain}")
print(f"Title:   {features.title}")
for label, score in predictions[:3]:
    print(f"  {label:30s}  confidence: {score:.4f}")
# Output:
#   AI Code Assistants              confidence: 0.9230
#   AI Developer Tools               confidence: 0.1840
#   AI Productivity                   confidence: 0.0620

#!/usr/bin/env python3
"""Confidence scoring and tier assignment for classified domains."""

from dataclasses import dataclass
from enum import Enum
from typing import Optional

class ReviewTier(Enum):
    AUTO_PUBLISH = "auto_publish"
    MANUAL_REVIEW = "manual_review"
    BATCH_HOLD = "batch_hold"
    REJECTED = "rejected"

CONFIDENCE_THRESHOLDS = {
    "auto_publish":  0.85,   # ≥0.85 → publish automatically
    "manual_review": 0.65,   # 0.65–0.85 → human review queue
    "batch_hold":    0.40,   # 0.40–0.65 → weekly batch review
    "reject":        0.40,   # <0.40 → discard as non-AI
}

@dataclass
class ClassificationResult:
    domain: str
    primary_category: str
    confidence: float
    secondary_category: Optional[str]
    secondary_confidence: Optional[float]
    tier: ReviewTier

def assign_review_tier(confidence: float) -> ReviewTier:
    """Map confidence score to processing tier."""
    if confidence >= CONFIDENCE_THRESHOLDS["auto_publish"]:
        return ReviewTier.AUTO_PUBLISH
    elif confidence >= CONFIDENCE_THRESHOLDS["manual_review"]:
        return ReviewTier.MANUAL_REVIEW
    elif confidence >= CONFIDENCE_THRESHOLDS["batch_hold"]:
        return ReviewTier.BATCH_HOLD
    else:
        return ReviewTier.REJECTED

def process_classification(domain: str, predictions: list) -> ClassificationResult:
    """Produce final classification with tier assignment."""
    primary_label, primary_conf = predictions[0]
    secondary = predictions[1] if len(predictions) > 1 and predictions[1][1] > 0.25 else None
    return ClassificationResult(
        domain=domain,
        primary_category=primary_label,
        confidence=primary_conf,
        secondary_category=secondary[0] if secondary else None,
        secondary_confidence=secondary[1] if secondary else None,
        tier=assign_review_tier(primary_conf),
    )

# Example: process classification output
predictions = [("AI Code Assistants", 0.923), ("AI Developer Tools", 0.184)]
result = process_classification("example-ai-code.dev", predictions)
print(f"Domain:     {result.domain}")
print(f"Category:   {result.primary_category} ({result.confidence:.1%})")
print(f"Tier:       {result.tier.value}")
# Output: Tier: auto_publish (confidence 92.3% exceeds 85% threshold)