Query DSL & Relevance

GET /products/_search
{
  "query": {
    "bool": {
      "must": [
        {
          "match": {
            "title": "wireless headphones"
          }
        }
      ],
      "filter": [
        {
          "term": { "status": "active" }
        },
        {
          "range": {
            "price": { "gte": 50, "lte": 200 }
          }
        },
        {
          "term": { "category": "electronics" }
        }
      ]
    }
  }
}

# What happens:
#   "must" → QUERY context
#     - "wireless headphones" is analyzed and scored
#     - Documents with both terms score higher
#     - Documents with terms in the title field score higher
#     - This determines the _score (ranking)
#
#   "filter" → FILTER context
#     - status = "active" → yes/no, no scoring
#     - price between 50-200 → yes/no, no scoring
#     - category = "electronics" → yes/no, no scoring
#     - These are cached as bitsets for fast repeat queries
#     - They narrow results WITHOUT affecting relevance ranking

# Basic match — analyzes input, finds documents containing any term
GET /products/_search
{
  "query": {
    "match": {
      "description": {
        "query": "lightweight running shoes",
        "operator": "and"
      }
    }
  }
}

# operator: "or" (default) — match ANY term (lightweight OR running OR shoes)
# operator: "and" — match ALL terms (lightweight AND running AND shoes)
# minimum_should_match: "75%" — at least 75% of terms must match

# What happens internally:
#   1. "lightweight running shoes" → analyzer → ["lightweight", "run", "shoe"]
#   2. Search inverted index for each token
#   3. Score documents based on BM25 (TF, IDF, field length)
#   4. Return ranked results

# match_phrase — terms must appear in exact order (with optional slop)
GET /articles/_search
{
  "query": {
    "match_phrase": {
      "body": {
        "query": "distributed systems design",
        "slop": 2
      }
    }
  }
}

# slop: 0 (default) — exact phrase, no words between
# slop: 1 — one word can be between/moved
# slop: 2 — two positional moves allowed

# Examples with slop:
#   "distributed systems design"     → matches (exact)
#   "distributed computing systems design" → matches with slop >= 1
#   "systems distributed design"     → matches with slop >= 2 (two swaps)

# How it works:
#   Uses positional data stored in the inverted index
#   Checks that token positions are within 'slop' distance of each other
#   More expensive than match — requires position lookups

# multi_match — search the same query across multiple fields
GET /products/_search
{
  "query": {
    "multi_match": {
      "query": "noise cancelling headphones",
      "fields": ["title^3", "description", "tags^2"],
      "type": "best_fields",
      "tie_breaker": 0.3
    }
  }
}

# Types:
#   best_fields (default) — score from the BEST matching field
#     Use when: fields compete (title vs description for same content)
#     tie_breaker: 0-1, how much other fields contribute (0.3 = 30%)
#
#   most_fields — SUM scores from all matching fields
#     Use when: same text indexed with different analyzers
#     (e.g., english analyzer + standard analyzer on same content)
#
#   cross_fields — treats all fields as ONE big field
#     Use when: a concept spans fields (first_name + last_name)
#     "John Smith" → "John" in first_name AND "Smith" in last_name
#
# Field boosting: "title^3" means title matches are 3x more important
# This is the most common way to tune relevance without function_score

# term — exact match (use on keyword fields ONLY)
{ "term": { "status": "active" } }

# terms — match any value in a list
{ "terms": { "category": ["electronics", "accessories", "audio"] } }

# range — numeric or date range
{
  "range": {
    "price": { "gte": 50, "lte": 200 }
  }
}
{
  "range": {
    "created_at": {
      "gte": "now-7d/d",
      "lte": "now/d",
      "format": "strict_date_optional_time"
    }
  }
}

# exists — field has a value
{ "exists": { "field": "discount_price" } }

# prefix — starts with (on keyword fields)
{ "prefix": { "sku": "PROD-2024" } }

# wildcard — pattern matching (* = any chars, ? = single char)
{ "wildcard": { "sku": "PROD-*-BLUE" } }

# fuzzy — edit distance (typo tolerance)
{
  "fuzzy": {
    "name": {
      "value": "headphnes",
      "fuzziness": "AUTO"
    }
  }
}
# fuzziness AUTO: 0-2 chars → exact, 3-5 chars → 1 edit, 6+ chars → 2 edits

GET /products/_search
{
  "query": {
    "bool": {
      "must": [
        {
          "multi_match": {
            "query": "wireless noise cancelling headphones",
            "fields": ["title^3", "description", "brand^2"],
            "type": "best_fields"
          }
        }
      ],
      "filter": [
        { "term": { "status": "active" } },
        { "term": { "in_stock": true } },
        { "range": { "price": { "gte": 50, "lte": 300 } } },
        { "terms": { "category": ["headphones", "earbuds"] } }
      ],
      "should": [
        { "term": { "is_prime_eligible": true } },
        { "range": { "rating": { "gte": 4.0 } } },
        { "term": { "brand": "Sony" } }
      ],
      "must_not": [
        { "term": { "condition": "refurbished" } },
        { "range": { "price": { "lt": 10 } } }
      ],
      "minimum_should_match": 0
    }
  }
}

# How this works:
#   1. FILTER (fast, cached, no scoring):
#      - Only active, in-stock products
#      - Price between $50-$300
#      - Category is headphones or earbuds
#      - NOT refurbished, NOT suspiciously cheap
#
#   2. MUST (scored):
#      - Full-text search across title (3x boost), description, brand (2x)
#      - This determines the base relevance score
#
#   3. SHOULD (score boosters):
#      - Prime eligible? Bonus points
#      - Rating >= 4.0? Bonus points
#      - Brand is Sony? Bonus points
#      - minimum_should_match: 0 means none required, just bonus
#
#   Result: Filtered set, ranked by text relevance + business signals

BM25 Score for a term in a document:

  score(term, doc) = IDF(term) * (TF(term, doc) * (k1 + 1))
                     ─────────────────────────────────────────
                     TF(term, doc) + k1 * (1 - b + b * |doc| / avgDL)

Three components:

1. IDF — Inverse Document Frequency
   "How rare is this term across ALL documents?"
   - Common words (the, is, a) → low IDF → low score contribution
   - Rare words (elasticsearch, kubernetes) → high IDF → high score
   - Formula: log(1 + (N - n + 0.5) / (n + 0.5))
     where N = total docs, n = docs containing term

2. TF — Term Frequency (with saturation)
   "How often does this term appear in THIS document?"
   - TF-IDF: score grows linearly (10 occurrences = 10x score) — BAD
   - BM25: score saturates (diminishing returns after ~5-10 occurrences)
   - k1 controls saturation speed (default 1.2)
     k1 = 0: TF is ignored entirely
     k1 = large: TF matters more (slower saturation)

3. Field Length Normalization
   "How long is this document's field?"
   - Short fields with the term score higher than long fields
   - "wireless" in a 3-word title scores higher than in a 500-word description
   - b controls normalization strength (default 0.75)
     b = 0: no length normalization
     b = 1: full normalization (short fields heavily favored)

Example:
  Query: "elasticsearch"
  Doc A: title = "Elasticsearch Guide" (2 words, term appears 1x)
  Doc B: title = "The Complete Guide to Elasticsearch and Search" (7 words, 1x)
  Doc C: description = "...elasticsearch...elasticsearch..." (500 words, 5x)

  Ranking: A > B > C
  - A: short field + term present = highest score
  - B: longer field, same TF = lower score
  - C: high TF but saturates + very long field = lowest score

GET /products/_search
{
  "query": {
    "function_score": {
      "query": {
        "bool": {
          "must": {
            "multi_match": {
              "query": "wireless headphones",
              "fields": ["title^3", "description"]
            }
          },
          "filter": [
            { "term": { "status": "active" } }
          ]
        }
      },
      "functions": [
        {
          "field_value_factor": {
            "field": "sales_count",
            "factor": 1.2,
            "modifier": "log1p",
            "missing": 1
          },
          "weight": 2
        },
        {
          "gauss": {
            "created_at": {
              "origin": "now",
              "scale": "30d",
              "offset": "7d",
              "decay": 0.5
            }
          },
          "weight": 1.5
        },
        {
          "filter": { "term": { "is_sponsored": true } },
          "weight": 5
        },
        {
          "script_score": {
            "script": {
              "source": "Math.log(2 + doc['review_count'].value) * doc['avg_rating'].value"
            }
          },
          "weight": 1
        }
      ],
      "score_mode": "sum",
      "boost_mode": "multiply",
      "max_boost": 50
    }
  }
}

# Functions explained:
#
# 1. field_value_factor — popularity signal
#    Uses sales_count field. log1p modifier prevents huge values from dominating.
#    Product with 10000 sales doesn't get 10000x boost (log1p(10000) ≈ 9.2)
#
# 2. gauss decay — recency signal
#    Documents from the last 7 days get full score (offset).
#    Score decays to 50% at 30 days (scale + decay).
#    Old documents still match but rank lower.
#
# 3. filter + weight — sponsored boost
#    Sponsored products get a flat 5x weight boost.
#    Only applies to docs matching the filter.
#
# 4. script_score — custom formula
#    Combines review_count and avg_rating into a quality signal.
#    Full flexibility but slower (runs per document).
#
# score_mode: how function scores combine (sum, multiply, avg, max, min)
# boost_mode: how combined function score merges with query score
#   multiply (default): final = query_score * function_score
#   sum: final = query_score + function_score
#   replace: final = function_score (ignores BM25 entirely)

GET /articles/_search
{
  "query": {
    "match": { "body": "distributed consensus algorithms" }
  },
  "_source": ["title", "author", "published_at", "url"],
  "highlight": {
    "fields": {
      "body": {
        "fragment_size": 150,
        "number_of_fragments": 3,
        "pre_tags": ["<mark>"],
        "post_tags": ["</mark>"]
      }
    }
  },
  "sort": [
    { "_score": "desc" },
    { "published_at": "desc" }
  ],
  "from": 0,
  "size": 20
}

# _source: only return these fields (reduces network payload)
# highlight: wraps matched terms in <mark> tags for UI display
# sort: primary by score, secondary by date (tiebreaker)
# from/size: offset-based pagination (page 1 = from:0, page 2 = from:20)

# ─────────────────────────────────────────────────────────
# search_after — cursor-based pagination (for deep pages)
# ─────────────────────────────────────────────────────────
GET /articles/_search
{
  "query": { "match": { "body": "distributed systems" } },
  "sort": [
    { "_score": "desc" },
    { "_id": "asc" }
  ],
  "size": 20,
  "search_after": [0.85, "doc_id_12345"]
}

# search_after uses the sort values from the last result of the previous page
# No deep pagination cost — always O(size) regardless of page number
# Requires a unique tiebreaker field (_id) in sort

# ─────────────────────────────────────────────────────────
# explain API — debug why a document scored the way it did
# ─────────────────────────────────────────────────────────
GET /products/_explain/product_123
{
  "query": {
    "match": { "title": "wireless headphones" }
  }
}
# Returns detailed breakdown: IDF, TF, field length norm, boosts
# Essential for debugging relevance issues