[
  {
    "path": "components/01-consistent-hashing.md",
    "sha256": "6f825d66cce8b75309918e17f2edb7d51bfd8585210cc46cd0618a2247215957",
    "lines": 519,
    "words": 7387,
    "headings": [
      {
        "line": 1,
        "title": "Consistent Hashing — Primitives Deep Dive"
      },
      {
        "line": 3,
        "title": "1. What It Is & Why It Exists"
      },
      {
        "line": 5,
        "title": "The Core Problem"
      },
      {
        "line": 18,
        "title": "Concrete Proof: The Modulus Resharding Breakdown ($N=4 \\to 5$)"
      },
      {
        "line": 36,
        "title": "The First-Principles Solution: The Hash Ring"
      },
      {
        "line": 72,
        "title": "2. Core Mechanics & Algorithmic Architecture"
      },
      {
        "line": 74,
        "title": "Virtual Nodes (Vnodes) & Mathematical Uniformity"
      },
      {
        "line": 78,
        "title": "The Mathematical Distribution Principle"
      },
      {
        "line": 92,
        "title": "Step-by-Step Deterministic Token Generation & Routing"
      },
      {
        "line": 96,
        "title": "Step 1: Deterministic String Salting"
      },
      {
        "line": 104,
        "title": "Step 2: Non-Cryptographic Hash Placement"
      },
      {
        "line": 112,
        "title": "Step 3: In-Memory Binary Search Tree Mapping"
      },
      {
        "line": 124,
        "title": "Step 4: Clockwise Binary Search Routing"
      },
      {
        "line": 131,
        "title": "Concrete Student Walkthrough: Tracing 3 Incoming Keys"
      },
      {
        "line": 157,
        "title": "Supporting Heterogeneous Hardware Fleets"
      },
      {
        "line": 181,
        "title": "3. Data Migration & Anti-Entropy Mechanisms (Re-Shuffling in Production)"
      },
      {
        "line": 185,
        "title": "1. Bounded Token Range Streaming"
      },
      {
        "line": 191,
        "title": "2. Anti-Entropy Reconciliation via Merkle Trees"
      },
      {
        "line": 245,
        "title": "Step-by-Step Anti-Entropy Tree Walkdown:"
      },
      {
        "line": 253,
        "title": "3. Transient Fault Handling: Hinted Handoffs"
      },
      {
        "line": 258,
        "title": "4. Quorum Read Repair"
      },
      {
        "line": 265,
        "title": "4. Client-Side vs. Proxy-Mediated Routing Topologies"
      },
      {
        "line": 291,
        "title": "Trade-Off Matrix"
      },
      {
        "line": 304,
        "title": "5. Real-World Engine Comparison"
      },
      {
        "line": 320,
        "title": "6. Critical Edge Cases & Distributed Failure Modes"
      },
      {
        "line": 322,
        "title": "1. Cascading Node Failure (The Domino Effect)"
      },
      {
        "line": 350,
        "title": "Concrete Student Walkthrough: Domino Collapse vs. Vnode Dispersion"
      },
      {
        "line": 360,
        "title": "2. Network Partitions & Split-Ring Views (Split-Brain)"
      },
      {
        "line": 364,
        "title": "3. Hotspot Keys (\"The Justin Bieber Problem\")"
      },
      {
        "line": 372,
        "title": "7. Production Pitfalls & Anti-Patterns (The \"Gotchas\")"
      },
      {
        "line": 374,
        "title": "Anti-Pattern 1: Employing Cryptographic Hashes (SHA-256) for Routing"
      },
      {
        "line": 379,
        "title": "Anti-Pattern 2: Under-Provisioning Virtual Nodes ($V < 50$)"
      },
      {
        "line": 384,
        "title": "Anti-Pattern 3: Ungraceful Ring Topology Mutation (Abrupt Cut-Overs)"
      },
      {
        "line": 389,
        "title": "8. AWS Cloud Service Implementation & Production Patterns"
      },
      {
        "line": 391,
        "title": "1. Amazon DynamoDB Partition Hash Ring"
      },
      {
        "line": 411,
        "title": "2. Amazon OpenSearch Service: Fixed Routing Shards & Split Index API"
      },
      {
        "line": 417,
        "title": "3. Amazon Kinesis Data Streams: 128-Bit MD5 Shard Hash Ranges"
      },
      {
        "line": 423,
        "title": "4. Amazon ElastiCache for Redis & Memcached"
      },
      {
        "line": 427,
        "title": "5. AWS CloudFront Origin Shield (Not a Hash Ring)"
      },
      {
        "line": 432,
        "title": "9. Production Sizing Matrix & Operational Runbook"
      },
      {
        "line": 434,
        "title": "Sizing Virtual Nodes Formula"
      },
      {
        "line": 445,
        "title": "Operational Runbook: Adding & Removing Nodes Safely"
      },
      {
        "line": 453,
        "title": "10. Production Diagnostics: Telemetry Signatures & Incident Response Playbook"
      },
      {
        "line": 457,
        "title": "1. Telemetry Signatures & Root-Cause Playbooks"
      },
      {
        "line": 459,
        "title": "Diagnostic 1: Localized Throttling Amid Low Global Utilization (DynamoDB)"
      },
      {
        "line": 464,
        "title": "Diagnostic 2: Persistent Redirection Storms (Redis MOVED / ASK Errors)"
      },
      {
        "line": 469,
        "title": "Diagnostic 3: Kinesis Hot Shard Write Rejection (`WriteProvisionedThroughputExceeded`)"
      },
      {
        "line": 474,
        "title": "Diagnostic 4: OpenSearch Routing Skew & Thread Pool Queue Rejection (`es_rejected_execution_exception`)"
      },
      {
        "line": 479,
        "title": "Diagnostic 5: Cache Miss Storms & Origin Shield Spikes (CloudFront CDN)"
      },
      {
        "line": 484,
        "title": "Diagnostic 6: Skewed Shard CPU & Memory Utilization Metrics (Fleet Imbalance)"
      },
      {
        "line": 491,
        "title": "2. Manual Configuration vs. Automated AWS Back-End Management"
      },
      {
        "line": 504,
        "title": "3. Common Production Errors & Anti-Patterns"
      }
    ],
    "fences": [
      {
        "start": 50,
        "end": 66,
        "language": "mermaid"
      },
      {
        "start": 140,
        "end": 153,
        "language": "mermaid"
      },
      {
        "start": 163,
        "end": 177,
        "language": "mermaid"
      },
      {
        "start": 197,
        "end": 241,
        "language": "mermaid"
      },
      {
        "start": 269,
        "end": 287,
        "language": "mermaid"
      },
      {
        "start": 326,
        "end": 346,
        "language": "mermaid"
      },
      {
        "start": 400,
        "end": 407,
        "language": "mermaid"
      }
    ],
    "links": [
      "../loop-primitives/08-sharding-hot-keys-and-rebalancing.md",
      "#operational-runbook-adding-removing-nodes-safely",
      "../loop-primitives/08-sharding-hot-keys-and-rebalancing.md",
      "#the-core-problem",
      "../loop-primitives/08-sharding-hot-keys-and-rebalancing.md",
      "../loop-primitives/08-sharding-hot-keys-and-rebalancing.md",
      "#1-amazon-dynamodb-partition-hash-ring",
      "#2-amazon-opensearch-service-fixed-routing-shards-split-index-api",
      "#3-amazon-kinesis-data-streams-128-bit-md5-shard-hash-ranges",
      "#4-amazon-elasticache-for-redis-memcached",
      "#4-amazon-elasticache-for-redis-memcached",
      "#5-aws-cloudfront-origin-shield-not-a-hash-ring"
    ],
    "lessons": [
      12,
      13,
      14,
      17
    ]
  },
  {
    "path": "components/02-distributed-rate-limiting.md",
    "sha256": "90ac0e57ab2351027a5f6c6772ac399cdefa2bf3557a4fa3e8098c7129f7be6a",
    "lines": 559,
    "words": 6541,
    "headings": [
      {
        "line": 1,
        "title": "Distributed Rate Limiting — Primitives Deep Dive"
      },
      {
        "line": 3,
        "title": "1. What It Is & Why It Exists"
      },
      {
        "line": 5,
        "title": "The Core Problem"
      },
      {
        "line": 11,
        "title": "The Breakdown: The Naive Counter Breakdown & Boundary Cliff"
      },
      {
        "line": 13,
        "title": "1. The Distributed Drift Cliff (Uncoordinated Local State)"
      },
      {
        "line": 16,
        "title": "2. The Fixed-Window Boundary Spike (The 2× Burst Cliff)"
      },
      {
        "line": 19,
        "title": "Concrete Proof: Naive Modulo vs. Fixed-Window vs. Distributed Control"
      },
      {
        "line": 30,
        "title": "The First-Principles Solution: Distributed Admission Control"
      },
      {
        "line": 61,
        "title": "2. Core Mechanics & Algorithmic Architecture"
      },
      {
        "line": 63,
        "title": "Mathematical Formulations of Core Rate Limiting Algorithms"
      },
      {
        "line": 65,
        "title": "1. Token Bucket Algorithm"
      },
      {
        "line": 82,
        "title": "2. Leaky Bucket Algorithm"
      },
      {
        "line": 97,
        "title": "3. Sliding Window Counter Algorithm (Weighted Moving Average)"
      },
      {
        "line": 109,
        "title": "Step-by-Step Deterministic Token Generation & Routing Pipeline"
      },
      {
        "line": 136,
        "title": "In-Memory State Representation in Redis"
      },
      {
        "line": 148,
        "title": "State-Transition Trace: Token Bucket Under Burst & Depletion"
      },
      {
        "line": 162,
        "title": "Heterogeneous Tenant Fleet Sizing & Tiered Rules"
      },
      {
        "line": 182,
        "title": "3. Data Migration, Anti-Entropy & Consistency Protocols"
      },
      {
        "line": 186,
        "title": "1. Multi-Tier L1/L2 Rate Limiting (Batch Reconciliation)"
      },
      {
        "line": 194,
        "title": "2. Multi-Region Active-Active Anti-Entropy"
      },
      {
        "line": 219,
        "title": "Multi-Region Strategies:"
      },
      {
        "line": 223,
        "title": "3. Graceful Draining, Token Lease Protocol & Cluster Resharding"
      },
      {
        "line": 229,
        "title": "4. Client-Side vs. Proxy-Mediated Routing Topologies"
      },
      {
        "line": 251,
        "title": "Trade-Off Matrix"
      },
      {
        "line": 264,
        "title": "5. Real-World Distributed Engine Comparison"
      },
      {
        "line": 276,
        "title": "6. Critical Edge Cases & Distributed Failure Modes"
      },
      {
        "line": 278,
        "title": "1. Cascading Retry Storms (The Retry Avalanche)"
      },
      {
        "line": 305,
        "title": "Concrete Proof: Fixed Retry vs. Full Jitter"
      },
      {
        "line": 314,
        "title": "2. Distributed Race Conditions (The Check-Then-Act Flaw)"
      },
      {
        "line": 320,
        "title": "3. Rate Limiter Outages & The Fail-Open vs. Fail-Closed Dilemma"
      },
      {
        "line": 328,
        "title": "7. Production Pitfalls & Anti-Patterns (The \"Gotchas\")"
      },
      {
        "line": 330,
        "title": "Anti-Pattern 1: IP-Only Rate Limiting (The NAT Gateway Pitfall)"
      },
      {
        "line": 335,
        "title": "Anti-Pattern 2: Hotspotting Single Redis Keys Under High-Throughput Ingress"
      },
      {
        "line": 343,
        "title": "Anti-Pattern 3: Cross-Slot Redis Cluster Lua Failures"
      },
      {
        "line": 353,
        "title": "8. AWS Cloud Service Implementation & Production Patterns"
      },
      {
        "line": 368,
        "title": "1. AWS WAF (Edge Rate-Based Rules)"
      },
      {
        "line": 372,
        "title": "2. Amazon API Gateway Usage Plans & Method Throttling"
      },
      {
        "line": 378,
        "title": "3. Amazon ElastiCache for Redis (Sliding Window & Token Bucket Lua Script)"
      },
      {
        "line": 429,
        "title": "4. Amazon AppSync (GraphQL Directive Throttling)"
      },
      {
        "line": 432,
        "title": "5. Amazon SQS with Lambda Concurrency (Egress Leaky Flow)"
      },
      {
        "line": 437,
        "title": "9. Production Sizing Matrix & Operational Runbook"
      },
      {
        "line": 439,
        "title": "Mathematical Memory Sizing Formula"
      },
      {
        "line": 452,
        "title": "Production Sizing Matrix"
      },
      {
        "line": 464,
        "title": "Operational Runbook: 4-Phase Safe Production Rollout"
      },
      {
        "line": 488,
        "title": "10. Production Diagnostics: Telemetry Signatures & Incident Response Playbook"
      },
      {
        "line": 490,
        "title": "1. Six Production Diagnostic Playbooks"
      },
      {
        "line": 492,
        "title": "Diagnostic 1: Sudden 429 Throttling Storm with Low Downstream Backend CPU"
      },
      {
        "line": 500,
        "title": "Diagnostic 2: Redis Latency Spikes & Gateway Connection Timeouts (`RedisLatency > 5ms`)"
      },
      {
        "line": 508,
        "title": "Diagnostic 3: Cross-Slot Command Rejection (`CROSSSLOT Keys in request don't hash to the same slot`)"
      },
      {
        "line": 516,
        "title": "Diagnostic 4: Amazon API Gateway Regional Throttling (`429 Too Many Requests`)"
      },
      {
        "line": 524,
        "title": "Diagnostic 5: Microservice Retry Cascade (Ingress 429 + Downstream 504 Gateway Timeout)"
      },
      {
        "line": 532,
        "title": "Diagnostic 6: WAF Rate-Based False Positive Spike on Enterprise SSO"
      },
      {
        "line": 541,
        "title": "2. Manual Configuration vs. Automated AWS Management"
      },
      {
        "line": 553,
        "title": "3. Common Production Anti-Patterns Quick Reference"
      }
    ],
    "fences": [
      {
        "start": 34,
        "end": 55,
        "language": "mermaid"
      },
      {
        "start": 111,
        "end": 132,
        "language": "mermaid"
      },
      {
        "start": 166,
        "end": 176,
        "language": "mermaid"
      },
      {
        "start": 197,
        "end": 215,
        "language": "mermaid"
      },
      {
        "start": 231,
        "end": 247,
        "language": "mermaid"
      },
      {
        "start": 284,
        "end": 301,
        "language": "mermaid"
      },
      {
        "start": 355,
        "end": 364,
        "language": "mermaid"
      },
      {
        "start": 381,
        "end": 427,
        "language": "lua"
      },
      {
        "start": 466,
        "end": 471,
        "language": "mermaid"
      }
    ],
    "links": [
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md"
    ],
    "lessons": [
      8,
      9
    ]
  },
  {
    "path": "components/03-bloom-filters-and-counting-filters.md",
    "sha256": "05f3bb18dfd4fbb01b255d9ea9b5d7d9d8560fcd412bf26ad57783a70a72f97c",
    "lines": 589,
    "words": 8785,
    "headings": [
      {
        "line": 1,
        "title": "Bloom Filters & Counting Filters — Primitives Deep Dive"
      },
      {
        "line": 3,
        "title": "1. What It Is & Why It Exists"
      },
      {
        "line": 5,
        "title": "The Core Problem"
      },
      {
        "line": 12,
        "title": "The Breakdown: The Negative Lookup Penalty & Hash Table Bloat"
      },
      {
        "line": 14,
        "title": "Concrete Proof: In-Memory Set vs. Raw Disk Scan vs. Bloom Filter"
      },
      {
        "line": 29,
        "title": "The First-Principles Solution: Probabilistic Set Membership"
      },
      {
        "line": 59,
        "title": "2. Core Mechanics & Algorithmic Architecture"
      },
      {
        "line": 61,
        "title": "A Tiny Worked Example: Tracing Insertions and Queries Bit by Bit"
      },
      {
        "line": 100,
        "title": "Mathematical Foundations of Bloom Filters"
      },
      {
        "line": 104,
        "title": "1. Bit Setting Probability"
      },
      {
        "line": 117,
        "title": "2. False Positive Probability ($p$)"
      },
      {
        "line": 122,
        "title": "3. Optimal Number of Hash Functions ($k^*$)"
      },
      {
        "line": 129,
        "title": "4. Required Bit Array Sizing Formula ($m$)"
      },
      {
        "line": 143,
        "title": "The Kirsch-Mitzenmacher Optimization (Two-Hash Generation)"
      },
      {
        "line": 153,
        "title": "In-Memory State Representation & Bit Storage"
      },
      {
        "line": 166,
        "title": "State-Transition Trace: Probabilistic Membership & Collision Dynamics"
      },
      {
        "line": 180,
        "title": "Filter Variants: Beyond the Standard Bloom Filter"
      },
      {
        "line": 204,
        "title": "Comparison of Advanced Probabilistic Filters"
      },
      {
        "line": 218,
        "title": "State-of-the-Art Alternatives: XOR & Ribbon Filters"
      },
      {
        "line": 230,
        "title": "3. Data Migration, Anti-Entropy & Consistency Protocols"
      },
      {
        "line": 234,
        "title": "1. LSM-Tree SSTable Lifecycle & Regeneration"
      },
      {
        "line": 262,
        "title": "2. Distributed Anti-Entropy: Summary Cache & Cache Digests"
      },
      {
        "line": 267,
        "title": "3. Distributed Bit-Vector Delta Sync & Atomic Double-Buffering"
      },
      {
        "line": 273,
        "title": "4. Client-Side vs. Proxy-Mediated Routing Topologies"
      },
      {
        "line": 302,
        "title": "Trade-Off Matrix"
      },
      {
        "line": 315,
        "title": "5. Real-World Distributed Engine Comparison"
      },
      {
        "line": 327,
        "title": "6. Critical Edge Cases & Distributed Failure Modes"
      },
      {
        "line": 329,
        "title": "1. Filter Saturation & The False Positive Avalanche"
      },
      {
        "line": 357,
        "title": "2. The Deletion Impossibility & Bit Corruption"
      },
      {
        "line": 364,
        "title": "3. Cache Penetration DDoS Attacks"
      },
      {
        "line": 370,
        "title": "4. Time-Windowed (Rotating) Filters for Streaming Deduplication"
      },
      {
        "line": 406,
        "title": "7. Production Pitfalls & Anti-Patterns (The \"Gotchas\")"
      },
      {
        "line": 408,
        "title": "Anti-Pattern 1: Sizing Filters Without Hard Capacity Caps"
      },
      {
        "line": 413,
        "title": "Anti-Pattern 2: Employing Cryptographic Hashes (SHA-256) for Filter Coordinates"
      },
      {
        "line": 418,
        "title": "Anti-Pattern 3: Using Standard Bloom Filters for Range Queries"
      },
      {
        "line": 425,
        "title": "8. AWS Cloud Service Implementation & Production Patterns"
      },
      {
        "line": 438,
        "title": "1. Amazon DynamoDB: No Published Partition Filters"
      },
      {
        "line": 442,
        "title": "Edge Case (a design exercise): Filter Recalculation During a Partition Split"
      },
      {
        "line": 448,
        "title": "2. Amazon ElastiCache for Valkey Bloom Filters (RedisBloom-Compatible Commands)"
      },
      {
        "line": 462,
        "title": "3. Amazon OpenSearch Service (Where Lucene Deliberately Does *Not* Use a Bloom Filter)"
      },
      {
        "line": 466,
        "title": "4. AWS Lambda@Edge & CloudFront Token Revocation"
      },
      {
        "line": 472,
        "title": "9. Production Sizing Matrix & Operational Runbook"
      },
      {
        "line": 474,
        "title": "Mathematical Memory Sizing Formula"
      },
      {
        "line": 479,
        "title": "Interactive Capacity Sizing Calculator"
      },
      {
        "line": 485,
        "title": "Production Sizing Matrix"
      },
      {
        "line": 498,
        "title": "Operational Runbook: Managing Filter Lifecycles"
      },
      {
        "line": 520,
        "title": "10. Production Diagnostics: Telemetry Signatures & Incident Response Playbook"
      },
      {
        "line": 522,
        "title": "1. Six Production Diagnostic Playbooks"
      },
      {
        "line": 524,
        "title": "Diagnostic 1: Sudden Spike in SSD Read Latency & Disk IOPS Schedulers (`DiskReadLatency > 50ms`)"
      },
      {
        "line": 531,
        "title": "Diagnostic 2: Cache Penetration Attack Under Low Cache Hit Ratio (`CacheHitRate < 10%`)"
      },
      {
        "line": 539,
        "title": "Diagnostic 3: RedisBloom Out-Of-Memory Error (`OOM command not allowed`)"
      },
      {
        "line": 547,
        "title": "Diagnostic 4: Rising Error Rate in Ad Click Deduplication"
      },
      {
        "line": 554,
        "title": "Diagnostic 5: JVM Off-Heap Native Memory Exhaustion in Apache Cassandra"
      },
      {
        "line": 562,
        "title": "Diagnostic 6: Cuckoo Filter Table Full Error"
      },
      {
        "line": 571,
        "title": "2. Manual Configuration vs. Automated Cloud Management"
      },
      {
        "line": 583,
        "title": "3. Common Production Anti-Patterns Quick Reference"
      }
    ],
    "fences": [
      {
        "start": 36,
        "end": 53,
        "language": "mermaid"
      },
      {
        "start": 69,
        "end": 71,
        "language": "text"
      },
      {
        "start": 77,
        "end": 79,
        "language": "text"
      },
      {
        "start": 83,
        "end": 85,
        "language": "text"
      },
      {
        "start": 184,
        "end": 200,
        "language": "mermaid"
      },
      {
        "start": 240,
        "end": 260,
        "language": "mermaid"
      },
      {
        "start": 275,
        "end": 298,
        "language": "mermaid"
      },
      {
        "start": 336,
        "end": 351,
        "language": "mermaid"
      },
      {
        "start": 381,
        "end": 400,
        "language": "mermaid"
      },
      {
        "start": 427,
        "end": 434,
        "language": "mermaid"
      },
      {
        "start": 450,
        "end": 460,
        "language": "bash"
      },
      {
        "start": 482,
        "end": 483,
        "language": "bloom-sizing-calculator"
      },
      {
        "start": 500,
        "end": 505,
        "language": "mermaid"
      }
    ],
    "links": [],
    "lessons": [
      5
    ]
  },
  {
    "path": "components/04-distributed-cache-patterns-and-eviction.md",
    "sha256": "514df93d721703d53a9491fda3aefd4e9edfaa46da85d18e0f9f64ee7378c458",
    "lines": 542,
    "words": 6685,
    "headings": [
      {
        "line": 1,
        "title": "Distributed Caching Patterns & Eviction — Primitives Deep Dive"
      },
      {
        "line": 3,
        "title": "1. What It Is & Why It Exists"
      },
      {
        "line": 5,
        "title": "The Core Problem"
      },
      {
        "line": 13,
        "title": "The Breakdown: The Unshielded Database Collapse"
      },
      {
        "line": 15,
        "title": "Concrete Proof: Unshielded Database vs. Distributed Cache"
      },
      {
        "line": 33,
        "title": "The First-Principles Solution: In-Memory Caching"
      },
      {
        "line": 57,
        "title": "2. Core Mechanics & Caching Design Patterns"
      },
      {
        "line": 59,
        "title": "Caching Design Patterns Deep-Dive"
      },
      {
        "line": 61,
        "title": "1. Cache-Aside (Lazy Loading)"
      },
      {
        "line": 68,
        "title": "2. Write-Through"
      },
      {
        "line": 74,
        "title": "3. Write-Behind (Write-Back)"
      },
      {
        "line": 80,
        "title": "4. Refresh-Ahead (Proactive Caching)"
      },
      {
        "line": 110,
        "title": "Formal Mathematical Formulations"
      },
      {
        "line": 112,
        "title": "1. Effective Access Time (EAT)"
      },
      {
        "line": 126,
        "title": "2. Probabilistic Early Expiration (The XFetch Algorithm)"
      },
      {
        "line": 140,
        "title": "Eviction Policies & In-Memory Data Structures"
      },
      {
        "line": 159,
        "title": "In-Memory Node Layout"
      },
      {
        "line": 173,
        "title": "State-Transition Trace: LRU Doubly-Linked List Eviction"
      },
      {
        "line": 186,
        "title": "Eviction Algorithm Comparison Matrix"
      },
      {
        "line": 198,
        "title": "3. Data Migration, Anti-Entropy & Consistency Protocols"
      },
      {
        "line": 217,
        "title": "The Invalidation Rule: Never Update, Always Invalidate"
      },
      {
        "line": 222,
        "title": "Asynchronous Anti-Entropy via Change Data Capture (CDC)"
      },
      {
        "line": 239,
        "title": "3. Online Cluster Resharding, Anti-Entropy & Graceful Draining"
      },
      {
        "line": 250,
        "title": "4. Client-Side vs. Proxy-Mediated Routing Topologies"
      },
      {
        "line": 273,
        "title": "Trade-Off Matrix"
      },
      {
        "line": 286,
        "title": "5. Real-World Distributed Engine Comparison"
      },
      {
        "line": 298,
        "title": "6. Critical Edge Cases & Distributed Failure Modes"
      },
      {
        "line": 300,
        "title": "1. Cache Stampede (The Thundering Herd)"
      },
      {
        "line": 334,
        "title": "2. Cache Avalanche (Synchronized TTL Collapse)"
      },
      {
        "line": 365,
        "title": "3. Cache Penetration (Non-Existent Keys Attack)"
      },
      {
        "line": 373,
        "title": "7. Production Pitfalls & Anti-Patterns (The \"Gotchas\")"
      },
      {
        "line": 375,
        "title": "Anti-Pattern 1: Executing Blocking $O(N)$ Commands in Production (`KEYS *`)"
      },
      {
        "line": 380,
        "title": "Anti-Pattern 2: Storing Megabyte-Sized Payloads in Single Keys"
      },
      {
        "line": 385,
        "title": "Anti-Pattern 3: Ignoring the Memory Fragmentation Ratio (`mem_fragmentation_ratio > 1.5`)"
      },
      {
        "line": 398,
        "title": "8. AWS Cloud Service Implementation & Production Patterns"
      },
      {
        "line": 413,
        "title": "1. Amazon ElastiCache for Redis Multi-AZ Cluster"
      },
      {
        "line": 417,
        "title": "2. Amazon DynamoDB Accelerator (DAX)"
      },
      {
        "line": 421,
        "title": "3. Amazon MemoryDB for Redis"
      },
      {
        "line": 425,
        "title": "4. Amazon CloudFront with Origin Shield"
      },
      {
        "line": 431,
        "title": "9. Production Sizing Matrix & Operational Runbook"
      },
      {
        "line": 433,
        "title": "Sizing: measure the working set"
      },
      {
        "line": 436,
        "title": "Production Sizing Matrix"
      },
      {
        "line": 449,
        "title": "Operational Runbook: 4-Phase Safe Deployment & Migration"
      },
      {
        "line": 471,
        "title": "10. Production Diagnostics: Telemetry Signatures & Incident Response Playbook"
      },
      {
        "line": 473,
        "title": "1. Six Production Diagnostic Playbooks"
      },
      {
        "line": 475,
        "title": "Diagnostic 1: Cache Stampede Alert (`EngineCPUUtilization = 100%` & DB Connection Exhaustion)"
      },
      {
        "line": 483,
        "title": "Diagnostic 2: Sudden Drop in Cache Hit Ratio (`CacheHitRate < 70%`)"
      },
      {
        "line": 491,
        "title": "Diagnostic 3: High Eviction Velocity (`Evictions > 20,000/sec`)"
      },
      {
        "line": 499,
        "title": "Diagnostic 4: Replication Lag & Primary-Replica Disconnection (`ReplicationLag > 5s`)"
      },
      {
        "line": 506,
        "title": "Diagnostic 5: Excessive Memory Fragmentation (`mem_fragmentation_ratio > 1.8`)"
      },
      {
        "line": 514,
        "title": "Diagnostic 6: Network Interface Bandwidth Saturation (`NetworkBandwidthOutAllowanceExceeded`)"
      },
      {
        "line": 524,
        "title": "2. Manual Configuration vs. Automated Cloud Management"
      },
      {
        "line": 536,
        "title": "3. Common Production Anti-Patterns Quick Reference"
      }
    ],
    "fences": [
      {
        "start": 37,
        "end": 51,
        "language": "mermaid"
      },
      {
        "start": 86,
        "end": 104,
        "language": "mermaid"
      },
      {
        "start": 144,
        "end": 155,
        "language": "mermaid"
      },
      {
        "start": 202,
        "end": 213,
        "language": "mermaid"
      },
      {
        "start": 225,
        "end": 233,
        "language": "mermaid"
      },
      {
        "start": 252,
        "end": 269,
        "language": "mermaid"
      },
      {
        "start": 307,
        "end": 330,
        "language": "mermaid"
      },
      {
        "start": 340,
        "end": 359,
        "language": "mermaid"
      },
      {
        "start": 389,
        "end": 394,
        "language": "conf"
      },
      {
        "start": 400,
        "end": 409,
        "language": "mermaid"
      },
      {
        "start": 451,
        "end": 456,
        "language": "mermaid"
      }
    ],
    "links": [
      "#1-cache-stampede-the-thundering-herd"
    ],
    "lessons": [
      10,
      11
    ]
  },
  {
    "path": "components/05-message-queues-vs-event-streams.md",
    "sha256": "d001738a5efa7735fc68939873ec3a243c25c42f35d6c461dc26a673b46ceff2",
    "lines": 458,
    "words": 5514,
    "headings": [
      {
        "line": 1,
        "title": "Message Queues vs. Event Streams — Primitives Deep Dive"
      },
      {
        "line": 3,
        "title": "1. What It Is & Why It Exists"
      },
      {
        "line": 5,
        "title": "The Core Problem"
      },
      {
        "line": 11,
        "title": "The Breakdown: Synchronous Cascade vs. Queue vs. Stream"
      },
      {
        "line": 13,
        "title": "Concrete Proof: Synchronous Call vs. Message Queue vs. Event Stream"
      },
      {
        "line": 31,
        "title": "The First-Principles Solution: Asynchronous Decoupling"
      },
      {
        "line": 66,
        "title": "2. Core Mechanics & Algorithmic Architecture"
      },
      {
        "line": 68,
        "title": "Formal Mathematical Formulations"
      },
      {
        "line": 70,
        "title": "1. Consumer Lag & Backlog Formulation"
      },
      {
        "line": 82,
        "title": "2. Consumer Fleet Autoscaling Rule"
      },
      {
        "line": 89,
        "title": "3. Mathematical Idempotency Invariant"
      },
      {
        "line": 98,
        "title": "Step-by-Step Processing Pipelines"
      },
      {
        "line": 124,
        "title": "In-Memory & On-Disk State Structures"
      },
      {
        "line": 126,
        "title": "Kafka Log Segment Architecture"
      },
      {
        "line": 158,
        "title": "Scenario Execution Matrix: Task Queues vs. Event Streams"
      },
      {
        "line": 168,
        "title": "3. Data Migration, Anti-Entropy & Consistency Protocols"
      },
      {
        "line": 191,
        "title": "1. Log Compaction"
      },
      {
        "line": 195,
        "title": "2. In-Sync Replicas (ISR) & Quorum Replication"
      },
      {
        "line": 200,
        "title": "3. Consumer Rebalance Protocols, Graceful Draining & Dynamic Re-partitioning"
      },
      {
        "line": 207,
        "title": "4. Client-Side vs. Proxy-Mediated Routing Topologies"
      },
      {
        "line": 230,
        "title": "Trade-Off Matrix"
      },
      {
        "line": 243,
        "title": "5. Real-World Distributed Engine Comparison"
      },
      {
        "line": 255,
        "title": "6. Critical Edge Cases & Distributed Failure Modes"
      },
      {
        "line": 257,
        "title": "1. Poison Pill Messages & The Infinite Redrive Loop"
      },
      {
        "line": 286,
        "title": "2. Kafka Consumer Group Rebalance Storms (Stop-The-World Freezes)"
      },
      {
        "line": 295,
        "title": "3. The Celebrity Partition Hotspot (\"The Justin Bieber Shard\")"
      },
      {
        "line": 303,
        "title": "7. Production Pitfalls & Anti-Patterns (The \"Gotchas\")"
      },
      {
        "line": 305,
        "title": "Anti-Pattern 1: Premature Offset Committing"
      },
      {
        "line": 310,
        "title": "Anti-Pattern 2: Under-Partitioning Streaming Topics"
      },
      {
        "line": 315,
        "title": "Anti-Pattern 3: Treating Message Queues as Long-Term Datastores"
      },
      {
        "line": 322,
        "title": "8. AWS Cloud Service Implementation & Production Patterns"
      },
      {
        "line": 336,
        "title": "1. Amazon SQS (Standard & FIFO)"
      },
      {
        "line": 340,
        "title": "2. Amazon Kinesis Data Streams"
      },
      {
        "line": 343,
        "title": "3. Amazon MSK (Managed Streaming for Apache Kafka)"
      },
      {
        "line": 346,
        "title": "4. Amazon EventBridge"
      },
      {
        "line": 351,
        "title": "9. Production Sizing Matrix & Operational Runbook"
      },
      {
        "line": 353,
        "title": "Mathematical Sizing Formulas"
      },
      {
        "line": 355,
        "title": "Partition Sizing Formula"
      },
      {
        "line": 360,
        "title": "Disk Storage Sizing Formula"
      },
      {
        "line": 363,
        "title": "Production Sizing Matrix"
      },
      {
        "line": 374,
        "title": "Operational Runbook: 4-Phase Safe Deployment"
      },
      {
        "line": 390,
        "title": "10. Production Diagnostics: Telemetry Signatures & Incident Response Playbook"
      },
      {
        "line": 392,
        "title": "1. Six Production Diagnostic Playbooks"
      },
      {
        "line": 394,
        "title": "Diagnostic 1: Runaway Consumer Lag (`ConsumerLag > 100,000`)"
      },
      {
        "line": 402,
        "title": "Diagnostic 2: SQS Oldest Message Spiking (`ApproximateAgeOfOldestMessage > 300s`)"
      },
      {
        "line": 410,
        "title": "Diagnostic 3: Kafka Rebalance Storm (`rebalance-rate-per-hour > 300`, i.e. 5/min)"
      },
      {
        "line": 418,
        "title": "Diagnostic 4: Kinesis Shard Write Throttling (`WriteProvisionedThroughputExceeded`)"
      },
      {
        "line": 425,
        "title": "Diagnostic 5: Poison Pill Infinite Redrive in SQS"
      },
      {
        "line": 432,
        "title": "Diagnostic 6: Kafka Broker Storage Saturation (`DiskSpaceUtilization > 85%`)"
      },
      {
        "line": 441,
        "title": "2. Manual Configuration vs. Automated Cloud Management"
      },
      {
        "line": 452,
        "title": "3. Common Production Anti-Patterns Quick Reference"
      }
    ],
    "fences": [
      {
        "start": 37,
        "end": 60,
        "language": "mermaid"
      },
      {
        "start": 100,
        "end": 120,
        "language": "mermaid"
      },
      {
        "start": 135,
        "end": 152,
        "language": "mermaid"
      },
      {
        "start": 170,
        "end": 187,
        "language": "mermaid"
      },
      {
        "start": 209,
        "end": 226,
        "language": "mermaid"
      },
      {
        "start": 262,
        "end": 280,
        "language": "mermaid"
      },
      {
        "start": 324,
        "end": 332,
        "language": "mermaid"
      },
      {
        "start": 376,
        "end": 381,
        "language": "mermaid"
      }
    ],
    "links": [
      "../loop-primitives/05-change-streams-and-transactional-outbox.md",
      "../loop-primitives/12-queues-and-delivery-semantics.md",
      "../loop-primitives/12-queues-and-delivery-semantics.md"
    ],
    "lessons": [
      19,
      21
    ]
  },
  {
    "path": "components/06-distributed-locks-and-leases.md",
    "sha256": "33737713eb03cbe8a31a3afa2bdf8670df3d7a46bd26ad4c2b1cb531592df071",
    "lines": 466,
    "words": 5501,
    "headings": [
      {
        "line": 1,
        "title": "Distributed Locks & Leases — Primitives Deep Dive"
      },
      {
        "line": 3,
        "title": "1. What It Is & Why It Exists"
      },
      {
        "line": 5,
        "title": "The Core Problem"
      },
      {
        "line": 11,
        "title": "The Breakdown: The Unfenced Lock & The GC Pause Cliff"
      },
      {
        "line": 13,
        "title": "The Fatal Breakdown of Naive Distributed Locks"
      },
      {
        "line": 22,
        "title": "Concrete Proof: Naive Lock vs. Lease vs. Fencing Token"
      },
      {
        "line": 34,
        "title": "The First-Principles Solution: Leases with Fencing Tokens"
      },
      {
        "line": 67,
        "title": "2. Core Mechanics & Algorithmic Architecture"
      },
      {
        "line": 69,
        "title": "Formal Mathematical Formulations"
      },
      {
        "line": 71,
        "title": "1. Fencing Token Validation Invariant"
      },
      {
        "line": 81,
        "title": "2. Lease Duration & Drift Safety Formula"
      },
      {
        "line": 90,
        "title": "3. Heartbeat Renewal Condition"
      },
      {
        "line": 99,
        "title": "Step-by-Step Deterministic Lifecycle Pipeline"
      },
      {
        "line": 124,
        "title": "In-Memory & Distributed State Structures"
      },
      {
        "line": 135,
        "title": "Scenario Execution Matrix: Distributed Leases & Fencing Tokens"
      },
      {
        "line": 148,
        "title": "3. Data Migration, Anti-Entropy & Consistency Protocols"
      },
      {
        "line": 150,
        "title": "Consensus-Backed Replicated State Machines (etcd / ZooKeeper)"
      },
      {
        "line": 173,
        "title": "The Redis Failover Anomaly (Why Redis is an AP Lock)"
      },
      {
        "line": 183,
        "title": "3. Graceful Draining, Cooperative Cancellation & Lease Abandonment"
      },
      {
        "line": 189,
        "title": "4. Client-Side vs. Proxy-Mediated Routing Topologies"
      },
      {
        "line": 209,
        "title": "Trade-Off Matrix"
      },
      {
        "line": 222,
        "title": "5. Real-World Distributed Engine Comparison"
      },
      {
        "line": 234,
        "title": "6. Critical Edge Cases & Distributed Failure Modes"
      },
      {
        "line": 236,
        "title": "1. The Redlock NTP Clock Skew Vulnerability"
      },
      {
        "line": 256,
        "title": "2. The Zombie Worker (Heartbeat Thread Starvation)"
      },
      {
        "line": 266,
        "title": "3. Distributed Deadlock via Lock Inversion"
      },
      {
        "line": 274,
        "title": "7. Production Pitfalls & Anti-Patterns (The \"Gotchas\")"
      },
      {
        "line": 276,
        "title": "Anti-Pattern 1: Holding Distributed Locks Across Multi-Minute Workflows"
      },
      {
        "line": 281,
        "title": "Anti-Pattern 2: Unconditional Lock Deletion (`DEL lock_key`)"
      },
      {
        "line": 286,
        "title": "Anti-Pattern 3: Missing Jitter on Lock Acquisition Retries"
      },
      {
        "line": 294,
        "title": "8. AWS Cloud Service Implementation & Production Patterns"
      },
      {
        "line": 306,
        "title": "1. Amazon DynamoDB Lock Client (`aws-dynamodb-lock-client`)"
      },
      {
        "line": 348,
        "title": "2. Amazon Aurora / RDS Optimistic Locking"
      },
      {
        "line": 356,
        "title": "3. Amazon S3 Object Lock (WORM Storage)"
      },
      {
        "line": 361,
        "title": "9. Production Sizing Matrix & Operational Runbook"
      },
      {
        "line": 363,
        "title": "Mathematical Lease Sizing Formula"
      },
      {
        "line": 371,
        "title": "Production Sizing Matrix"
      },
      {
        "line": 382,
        "title": "Operational Runbook: 4-Phase Safe Deployment"
      },
      {
        "line": 398,
        "title": "10. Production Diagnostics: Telemetry Signatures & Incident Response Playbook"
      },
      {
        "line": 400,
        "title": "1. Six Production Diagnostic Playbooks"
      },
      {
        "line": 402,
        "title": "Diagnostic 1: Sudden Spike in Lock Acquisition Timeouts (`LockAcquireTimeoutException`)"
      },
      {
        "line": 411,
        "title": "Diagnostic 2: High Volume of Fencing Token Rejections at Storage"
      },
      {
        "line": 419,
        "title": "Diagnostic 3: Heartbeat Miss Alarms (`HeartbeatMissCount > 2`)"
      },
      {
        "line": 426,
        "title": "Diagnostic 4: DynamoDB Lock Table Provisioned Throughput Throttling"
      },
      {
        "line": 433,
        "title": "Diagnostic 5: Redis Dual-Owner Collision After Failover"
      },
      {
        "line": 440,
        "title": "Diagnostic 6: etcd / ZooKeeper Session Expirations and Leader Churn"
      },
      {
        "line": 449,
        "title": "2. Manual Configuration vs. Automated Cloud Management"
      },
      {
        "line": 460,
        "title": "3. Common Production Anti-Patterns Quick Reference"
      }
    ],
    "fences": [
      {
        "start": 40,
        "end": 61,
        "language": "mermaid"
      },
      {
        "start": 101,
        "end": 120,
        "language": "mermaid"
      },
      {
        "start": 155,
        "end": 169,
        "language": "mermaid"
      },
      {
        "start": 191,
        "end": 205,
        "language": "mermaid"
      },
      {
        "start": 241,
        "end": 250,
        "language": "mermaid"
      },
      {
        "start": 296,
        "end": 302,
        "language": "mermaid"
      },
      {
        "start": 384,
        "end": 389,
        "language": "mermaid"
      }
    ],
    "links": [
      "../loop-primitives/03-leases-fencing-tokens-and-distributed-locks.md",
      "../loop-primitives/03-leases-fencing-tokens-and-distributed-locks.md",
      "../loop-primitives/03-leases-fencing-tokens-and-distributed-locks.md"
    ],
    "lessons": [
      16
    ]
  },
  {
    "path": "components/07-write-ahead-log-and-lsm-trees.md",
    "sha256": "739f7bc039b086a6733033a19315c27fc5792de5e47bd642f40fa1c975fcdd54",
    "lines": 478,
    "words": 6293,
    "headings": [
      {
        "line": 1,
        "title": "Write-Ahead Log (WAL) & LSM-Trees — Primitives Deep Dive"
      },
      {
        "line": 3,
        "title": "1. What It Is & Why It Exists"
      },
      {
        "line": 5,
        "title": "The Core Problem: The B+ Tree In-Place Update Penalty"
      },
      {
        "line": 15,
        "title": "The Breakdown: The In-Place Page Rewrite Cliff"
      },
      {
        "line": 17,
        "title": "Concrete Proof: B+ Tree Random Page Rewrites vs. LSM-Tree Sequential Appends"
      },
      {
        "line": 31,
        "title": "The First-Principles Solution: Append-Only LSM-Trees & WAL"
      },
      {
        "line": 65,
        "title": "2. Core Mechanics & Algorithmic Architecture"
      },
      {
        "line": 67,
        "title": "The Fundamental Distributed Storage Triangle: R-W-S Amplification"
      },
      {
        "line": 92,
        "title": "Leveled Compaction Sizing Mathematics"
      },
      {
        "line": 115,
        "title": "Step-by-Step Deterministic Write & Read Pipelines"
      },
      {
        "line": 155,
        "title": "In-Memory & On-Disk State Structures"
      },
      {
        "line": 157,
        "title": "1. In-Memory SkipList Architecture"
      },
      {
        "line": 160,
        "title": "2. Immutable SSTable File Anatomy"
      },
      {
        "line": 187,
        "title": "Scenario Execution Matrix: LSM Read, Write, and Compaction Dynamics"
      },
      {
        "line": 197,
        "title": "3. Data Migration, Anti-Entropy & Consistency Protocols"
      },
      {
        "line": 199,
        "title": "1. Compaction Strategies Compared"
      },
      {
        "line": 226,
        "title": "2. WAL Group Commit & Fsync Durability Protocols"
      },
      {
        "line": 232,
        "title": "3. Distributed Anti-Entropy, Merkle Trees & Crash Recovery"
      },
      {
        "line": 241,
        "title": "4. Client-Side vs. Proxy-Mediated Routing Topologies"
      },
      {
        "line": 262,
        "title": "Trade-Off Matrix"
      },
      {
        "line": 274,
        "title": "5. Real-World Distributed Engine Comparison"
      },
      {
        "line": 286,
        "title": "6. Critical Edge Cases & Distributed Failure Modes"
      },
      {
        "line": 288,
        "title": "1. Compaction Write Stalls (The Backpressure Saturation Freeze)"
      },
      {
        "line": 314,
        "title": "2. Tombstone Saturation & Range Scan Meltdown"
      },
      {
        "line": 321,
        "title": "3. Partial Page Writes & WAL Corruption on Power Failure"
      },
      {
        "line": 327,
        "title": "7. Production Pitfalls & Anti-Patterns (The \"Gotchas\")"
      },
      {
        "line": 329,
        "title": "Anti-Pattern 1: Omitting Bloom Filters on LSM Tables"
      },
      {
        "line": 334,
        "title": "Anti-Pattern 2: Storing Megabyte-Sized Values Directly in MemTables"
      },
      {
        "line": 339,
        "title": "Anti-Pattern 3: Synchronous `fsync` on Every Mutation"
      },
      {
        "line": 346,
        "title": "8. AWS Cloud Service Implementation & Production Patterns"
      },
      {
        "line": 359,
        "title": "1. Amazon DynamoDB Storage Node Partitions"
      },
      {
        "line": 362,
        "title": "2. Amazon Aurora: \"The Log is the Database\""
      },
      {
        "line": 365,
        "title": "3. Amazon OpenSearch Service (Lucene Translog)"
      },
      {
        "line": 368,
        "title": "4. Amazon EBS `io2 Block Express` for Self-Managed RocksDB / Cassandra"
      },
      {
        "line": 373,
        "title": "9. Production Sizing Matrix & Operational Runbook"
      },
      {
        "line": 375,
        "title": "Mathematical Sizing Formulas"
      },
      {
        "line": 377,
        "title": "MemTable Memory Sizing"
      },
      {
        "line": 380,
        "title": "Compaction Disk Space Headroom"
      },
      {
        "line": 384,
        "title": "Production Tuning Matrix (RocksDB)"
      },
      {
        "line": 395,
        "title": "Operational Runbook: 4-Phase Lifecycle Management"
      },
      {
        "line": 411,
        "title": "10. Production Diagnostics: Telemetry Signatures & Incident Response Playbook"
      },
      {
        "line": 413,
        "title": "1. Six Production Diagnostic Playbooks"
      },
      {
        "line": 415,
        "title": "Diagnostic 1: Compaction Write Stall (`rocksdb.stall.micros > 0`)"
      },
      {
        "line": 423,
        "title": "Diagnostic 2: Tombstone Scan Degradation & Query Timeouts"
      },
      {
        "line": 431,
        "title": "Diagnostic 3: Excessive WAL Fsync Latency (`rocksdb.wal.file.sync.micros > 20,000`)"
      },
      {
        "line": 438,
        "title": "Diagnostic 4: Disk Out-Of-Space Failure During Compaction"
      },
      {
        "line": 445,
        "title": "Diagnostic 5: Low Bloom Filter Hit Rate (useful ÷ checked $< 80\\%$)"
      },
      {
        "line": 452,
        "title": "Diagnostic 6: MemTable Flush Queue Backpressure (`rocksdb.mem-table-flush-pending = 1`, `rocksdb.num-immutable-mem-table` growing)"
      },
      {
        "line": 461,
        "title": "2. Manual Configuration vs. Automated Cloud Management"
      },
      {
        "line": 472,
        "title": "3. Common Production Anti-Patterns Quick Reference"
      }
    ],
    "fences": [
      {
        "start": 39,
        "end": 59,
        "language": "mermaid"
      },
      {
        "start": 77,
        "end": 88,
        "language": "mermaid"
      },
      {
        "start": 117,
        "end": 151,
        "language": "mermaid"
      },
      {
        "start": 170,
        "end": 181,
        "language": "mermaid"
      },
      {
        "start": 201,
        "end": 214,
        "language": "mermaid"
      },
      {
        "start": 243,
        "end": 258,
        "language": "mermaid"
      },
      {
        "start": 293,
        "end": 308,
        "language": "mermaid"
      },
      {
        "start": 348,
        "end": 355,
        "language": "mermaid"
      },
      {
        "start": 397,
        "end": 402,
        "language": "mermaid"
      }
    ],
    "links": [
      "../loop-primitives/01-write-ahead-log-fsync-and-group-commit.md",
      "../loop-primitives/01-write-ahead-log-fsync-and-group-commit.md",
      "../loop-primitives/02-lsm-trees-and-compaction.md",
      "../loop-primitives/02-lsm-trees-and-compaction.md",
      "../loop-primitives/02-lsm-trees-and-compaction.md",
      "../loop-primitives/02-lsm-trees-and-compaction.md"
    ],
    "lessons": [
      2,
      3,
      4,
      22
    ]
  },
  {
    "path": "components/08-database-sharding-and-partition-keys.md",
    "sha256": "0b289fca4b05b1e777aa57ab52a5a80856a6a8dad52e69ce61030aefa1a5fb58",
    "lines": 491,
    "words": 5516,
    "headings": [
      {
        "line": 1,
        "title": "Database Sharding & Partition Keys — Primitives Deep Dive"
      },
      {
        "line": 3,
        "title": "1. What It Is & Why It Exists"
      },
      {
        "line": 5,
        "title": "The Core Problem: The Vertical Scaling Wall"
      },
      {
        "line": 11,
        "title": "The Breakdown: Monolithic Database vs. Shared-Nothing Sharding"
      },
      {
        "line": 13,
        "title": "Concrete Proof: The Vertical Scaling Wall vs. Horizontal Sharding"
      },
      {
        "line": 27,
        "title": "The First-Principles Solution: Shared-Nothing Horizontal Partitioning"
      },
      {
        "line": 55,
        "title": "2. Core Mechanics & Algorithmic Architecture"
      },
      {
        "line": 57,
        "title": "Sharding Strategies Compared"
      },
      {
        "line": 69,
        "title": "1. Hash-Based Sharding"
      },
      {
        "line": 75,
        "title": "2. Range-Based Sharding"
      },
      {
        "line": 81,
        "title": "3. Directory-Based (Lookup Table) Sharding"
      },
      {
        "line": 89,
        "title": "Formal Mathematical Sizing Formulations"
      },
      {
        "line": 91,
        "title": "1. Minimum Cluster Shard Sizing Formula"
      },
      {
        "line": 101,
        "title": "2. Load Imbalance Factor (Coefficient of Variation)"
      },
      {
        "line": 112,
        "title": "Step-by-Step Deterministic Routing Pipeline"
      },
      {
        "line": 144,
        "title": "In-Memory Shard Mapping Representation"
      },
      {
        "line": 157,
        "title": "Query Execution & Routing Trace Matrix"
      },
      {
        "line": 169,
        "title": "3. Data Migration, Anti-Entropy & Consistency Protocols"
      },
      {
        "line": 171,
        "title": "1. Online Live Resharding Protocol (Zero-Downtime Cluster Expansion)"
      },
      {
        "line": 207,
        "title": "2. Cross-Shard Transactions: Two-Phase Commit (2PC) vs. Entity Co-Location"
      },
      {
        "line": 214,
        "title": "3. Shard Draining Lifecycles & Anti-Entropy Resharding Verification"
      },
      {
        "line": 224,
        "title": "4. Client-Side vs. Proxy-Mediated Routing Topologies"
      },
      {
        "line": 249,
        "title": "Trade-Off Matrix"
      },
      {
        "line": 262,
        "title": "5. Real-World Distributed Engine Comparison"
      },
      {
        "line": 274,
        "title": "6. Critical Edge Cases & Distributed Failure Modes"
      },
      {
        "line": 276,
        "title": "1. Scatter-Gather Query Latency Tail Explosion"
      },
      {
        "line": 309,
        "title": "2. Distributed Cross-Shard Deadlocks"
      },
      {
        "line": 316,
        "title": "3. Monotonic Range-Sharding Hotspots"
      },
      {
        "line": 323,
        "title": "7. Production Pitfalls & Anti-Patterns (The \"Gotchas\")"
      },
      {
        "line": 325,
        "title": "Anti-Pattern 1: Sharding on Low-Cardinality Keys (`country_code`)"
      },
      {
        "line": 330,
        "title": "Anti-Pattern 2: Cross-Shard Joins and Foreign Key Constraints"
      },
      {
        "line": 335,
        "title": "Anti-Pattern 3: Unbounded Partition Growth in Single-Table Designs"
      },
      {
        "line": 342,
        "title": "8. AWS Cloud Service Implementation & Production Patterns"
      },
      {
        "line": 355,
        "title": "1. Amazon DynamoDB Single-Table Design"
      },
      {
        "line": 373,
        "title": "2. Amazon Aurora Limitless Database"
      },
      {
        "line": 376,
        "title": "3. Amazon OpenSearch Service Routing Optimization"
      },
      {
        "line": 385,
        "title": "9. Production Sizing Matrix & Operational Runbook"
      },
      {
        "line": 387,
        "title": "DynamoDB Physical Partition Limits"
      },
      {
        "line": 392,
        "title": "Minimum Partition Calculation"
      },
      {
        "line": 397,
        "title": "Production Sizing Matrix"
      },
      {
        "line": 408,
        "title": "Operational Runbook: 4-Phase Safe Shard Migration"
      },
      {
        "line": 424,
        "title": "10. Production Diagnostics: Telemetry Signatures & Incident Response Playbook"
      },
      {
        "line": 426,
        "title": "1. Six Production Diagnostic Playbooks"
      },
      {
        "line": 428,
        "title": "Diagnostic 1: DynamoDB ProvisionedThroughputExceededException Amid Low Overall Utilization"
      },
      {
        "line": 436,
        "title": "Diagnostic 2: Scatter-Gather Query Storm (`CrossShardQueryRate > 10%`)"
      },
      {
        "line": 444,
        "title": "Diagnostic 3: Cross-Shard 2PC Latency Spikes (`DistributedTransactionDuration > 1,000ms`)"
      },
      {
        "line": 451,
        "title": "Diagnostic 4: Severe Shard Storage Skew (`MaxShardDiskUtilization > 85%` vs. Fleet Average of $30\\%$)"
      },
      {
        "line": 458,
        "title": "Diagnostic 5: VTGate / Proxy Connection Pool Exhaustion"
      },
      {
        "line": 465,
        "title": "Diagnostic 6: Stale Client-Side Routing Cache Mismatch"
      },
      {
        "line": 474,
        "title": "2. Manual Configuration vs. Automated Cloud Management"
      },
      {
        "line": 485,
        "title": "3. Common Production Anti-Patterns Quick Reference"
      }
    ],
    "fences": [
      {
        "start": 34,
        "end": 49,
        "language": "mermaid"
      },
      {
        "start": 59,
        "end": 67,
        "language": "mermaid"
      },
      {
        "start": 114,
        "end": 140,
        "language": "mermaid"
      },
      {
        "start": 174,
        "end": 203,
        "language": "mermaid"
      },
      {
        "start": 226,
        "end": 245,
        "language": "mermaid"
      },
      {
        "start": 287,
        "end": 303,
        "language": "mermaid"
      },
      {
        "start": 344,
        "end": 351,
        "language": "mermaid"
      },
      {
        "start": 410,
        "end": 415,
        "language": "mermaid"
      }
    ],
    "links": [
      "../loop-primitives/08-sharding-hot-keys-and-rebalancing.md",
      "../loop-primitives/08-sharding-hot-keys-and-rebalancing.md",
      "../loop-primitives/08-sharding-hot-keys-and-rebalancing.md"
    ],
    "lessons": [
      12,
      13,
      17
    ]
  },
  {
    "path": "components/09-distributed-consensus-raft-and-paxos.md",
    "sha256": "2abf35839218f25867166ec182805d2264968550c0fed18f919a54275638c8d6",
    "lines": 454,
    "words": 5631,
    "headings": [
      {
        "line": 1,
        "title": "Distributed Consensus (Raft & Paxos) — Primitives Deep Dive"
      },
      {
        "line": 3,
        "title": "1. What It Is & Why It Exists"
      },
      {
        "line": 5,
        "title": "The Core Problem: Agreement Across Unreliable Networks"
      },
      {
        "line": 10,
        "title": "The FLP Impossibility Theorem (Fischer, Lynch, Paterson, 1985)"
      },
      {
        "line": 15,
        "title": "The Breakdown: Asynchronous Replication vs. 2PC vs. Quorum Consensus"
      },
      {
        "line": 17,
        "title": "Concrete Proof: The Split-Brain Blackout vs. Quorum Safety"
      },
      {
        "line": 28,
        "title": "The First-Principles Solution: Quorum-Based Replicated State Machines"
      },
      {
        "line": 67,
        "title": "2. Core Mechanics & Algorithmic Architecture"
      },
      {
        "line": 69,
        "title": "The Raft Algorithm: 3 Decomposed Sub-Mechanisms"
      },
      {
        "line": 85,
        "title": "1. Leader Election & Mathematical Quorum"
      },
      {
        "line": 98,
        "title": "2. Log Replication & The Log Matching Property"
      },
      {
        "line": 103,
        "title": "3. Linearizable Reads (Avoiding Stale Reads on Partitioned Leaders)"
      },
      {
        "line": 110,
        "title": "Step-by-Step Deterministic Write & Replication Pipeline"
      },
      {
        "line": 141,
        "title": "In-Memory & Distributed State Structures"
      },
      {
        "line": 157,
        "title": "Consensus Scenario & Quorum State Matrix"
      },
      {
        "line": 169,
        "title": "3. Data Migration, Anti-Entropy & Consistency Protocols"
      },
      {
        "line": 171,
        "title": "1. Log Compaction & Snapshotting"
      },
      {
        "line": 192,
        "title": "2. Dynamic Cluster Membership Changes (Joint Consensus)"
      },
      {
        "line": 198,
        "title": "3. Accelerated Log Conflict Resolution & Graceful Draining"
      },
      {
        "line": 208,
        "title": "4. Client-Side vs. Proxy-Mediated Routing Topologies"
      },
      {
        "line": 230,
        "title": "Trade-Off Matrix"
      },
      {
        "line": 242,
        "title": "5. Real-World Distributed Engine Comparison"
      },
      {
        "line": 254,
        "title": "6. Critical Edge Cases & Distributed Failure Modes"
      },
      {
        "line": 256,
        "title": "1. The Split-Vote Deadlock (Election Storm)"
      },
      {
        "line": 280,
        "title": "2. Asymmetric Partitions & The Pre-Vote Protocol"
      },
      {
        "line": 290,
        "title": "3. Disk Fsync Latency Spikes Breaking Leader Leases"
      },
      {
        "line": 296,
        "title": "7. Production Pitfalls & Anti-Patterns (The \"Gotchas\")"
      },
      {
        "line": 298,
        "title": "Anti-Pattern 1: Deploying Even-Numbered Node Counts ($N=4$ or $N=6$)"
      },
      {
        "line": 303,
        "title": "Anti-Pattern 2: Storing Large Payloads Directly in the Consensus Log"
      },
      {
        "line": 308,
        "title": "Anti-Pattern 3: Omitting Pre-Vote in Geographically Distributed Clusters"
      },
      {
        "line": 315,
        "title": "8. AWS Cloud Service Implementation & Production Patterns"
      },
      {
        "line": 328,
        "title": "1. Amazon EKS Managed etcd Architecture"
      },
      {
        "line": 331,
        "title": "2. Amazon Aurora 6-Way Storage Quorum"
      },
      {
        "line": 337,
        "title": "3. Amazon DynamoDB Partition Paxos Groups"
      },
      {
        "line": 342,
        "title": "9. Production Sizing Matrix & Operational Runbook"
      },
      {
        "line": 344,
        "title": "Mathematical Sizing Formulas"
      },
      {
        "line": 348,
        "title": "Production Sizing Matrix"
      },
      {
        "line": 359,
        "title": "Operational Runbook: 4-Phase Safe Maintenance Lifecycle"
      },
      {
        "line": 388,
        "title": "10. Production Diagnostics: Telemetry Signatures & Incident Response Playbook"
      },
      {
        "line": 390,
        "title": "1. Six Production Diagnostic Playbooks"
      },
      {
        "line": 392,
        "title": "Diagnostic 1: Total Loss of Cluster Leader (`etcd_server_has_leader == 0`)"
      },
      {
        "line": 399,
        "title": "Diagnostic 2: Rapid Leader Election Churn (`leader_changes_seen_total > 3 in 10min`)"
      },
      {
        "line": 407,
        "title": "Diagnostic 3: High WAL Fsync Latency Alert (p99 `wal_fsync_duration_seconds` above etcd's 10 ms guideline)"
      },
      {
        "line": 414,
        "title": "Diagnostic 4: Slow Follower Snapshot Catch-Up Alert"
      },
      {
        "line": 421,
        "title": "Diagnostic 5: Pre-Vote Term Storm Spike"
      },
      {
        "line": 428,
        "title": "Diagnostic 6: Aurora Storage Quorum Write Failure"
      },
      {
        "line": 437,
        "title": "2. Manual Configuration vs. Automated Cloud Management"
      },
      {
        "line": 448,
        "title": "3. Common Production Anti-Patterns Quick Reference"
      }
    ],
    "fences": [
      {
        "start": 35,
        "end": 61,
        "language": "mermaid"
      },
      {
        "start": 73,
        "end": 81,
        "language": "mermaid"
      },
      {
        "start": 112,
        "end": 137,
        "language": "mermaid"
      },
      {
        "start": 177,
        "end": 188,
        "language": "mermaid"
      },
      {
        "start": 210,
        "end": 226,
        "language": "mermaid"
      },
      {
        "start": 260,
        "end": 274,
        "language": "mermaid"
      },
      {
        "start": 317,
        "end": 324,
        "language": "mermaid"
      },
      {
        "start": 361,
        "end": 366,
        "language": "mermaid"
      }
    ],
    "links": [
      "../loop-primitives/06-replication-quorums-and-read-your-writes.md",
      "../loop-primitives/06-replication-quorums-and-read-your-writes.md",
      "../loop-primitives/03-leases-fencing-tokens-and-distributed-locks.md",
      "../loop-primitives/09-multi-region-failover.md"
    ],
    "lessons": [
      14,
      15
    ]
  },
  {
    "path": "components/10-two-phase-commit-and-saga-orchestration.md",
    "sha256": "ab25e39e9b2e279a6d0be7f6e5d4275f99d5c2c4679072cb5f2c451e28c908f6",
    "lines": 845,
    "words": 8298,
    "headings": [
      {
        "line": 1,
        "title": "Two-Phase Commit (2PC) & Saga Orchestration — Primitives Deep Dive"
      },
      {
        "line": 3,
        "title": "1. What It Is & Why It Exists"
      },
      {
        "line": 5,
        "title": "The Core Problem: Multi-Database ACID Transactions in Microservices"
      },
      {
        "line": 35,
        "title": "The Breakdown: The Naive Dual-Write & Coordinator Freeze Cliff"
      },
      {
        "line": 52,
        "title": "The First-Principles Solution"
      },
      {
        "line": 87,
        "title": "2. Core Mechanics & Algorithmic Architecture"
      },
      {
        "line": 89,
        "title": "Mathematical Formulation of 2PC and Sagas"
      },
      {
        "line": 91,
        "title": "1. Two-Phase Commit Safety & Blocking Vulnerability"
      },
      {
        "line": 100,
        "title": "2. The Saga Pattern & The Compensation Idempotency Invariant"
      },
      {
        "line": 118,
        "title": "Step-by-Step Deterministic Execution Pipeline"
      },
      {
        "line": 155,
        "title": "In-Memory & Persistent State Structures"
      },
      {
        "line": 170,
        "title": "Scenario Execution & Rollback Matrix"
      },
      {
        "line": 180,
        "title": "Heterogeneous Fleet Sizing & SLA Allocation"
      },
      {
        "line": 192,
        "title": "3. Data Migration, Anti-Entropy & Consistency Protocols"
      },
      {
        "line": 194,
        "title": "The Dual-Write Vulnerability & Transactional Outbox Pattern"
      },
      {
        "line": 207,
        "title": "The Senior SWE Solution: The Transactional Outbox Pattern with CDC"
      },
      {
        "line": 244,
        "title": "Forward Recovery vs. Backward Recovery Strategies"
      },
      {
        "line": 276,
        "title": "Anti-Entropy State Reconciliation"
      },
      {
        "line": 283,
        "title": "In-Flight Saga Workflow Versioning & Worker Graceful Draining"
      },
      {
        "line": 291,
        "title": "4. Client-Side vs. Proxy-Mediated Routing Topologies"
      },
      {
        "line": 326,
        "title": "6-Dimension Architectural Trade-Off Matrix"
      },
      {
        "line": 339,
        "title": "5. Real-World Distributed Engine Comparison"
      },
      {
        "line": 360,
        "title": "6. Critical Edge Cases & Distributed Failure Modes"
      },
      {
        "line": 362,
        "title": "1. The 2PC Indoubt Freeze: Mathematical Proof of Connection Pool Exhaustion"
      },
      {
        "line": 405,
        "title": "2. Saga Isolation Anomalies (The Lack of ACID Isolation)"
      },
      {
        "line": 408,
        "title": "A. The Lost Update Anomaly"
      },
      {
        "line": 414,
        "title": "B. The Dirty Read Anomaly"
      },
      {
        "line": 418,
        "title": "C. The Fuzzy (Non-Repeatable) Read Anomaly"
      },
      {
        "line": 424,
        "title": "3. Compensation Failure (\"The Zombie Saga\")"
      },
      {
        "line": 433,
        "title": "7. Production Pitfalls & Anti-Patterns (The \"Gotchas\")"
      },
      {
        "line": 435,
        "title": "1. Firing Irreversible Physical Actions in Early Steps"
      },
      {
        "line": 443,
        "title": "2. Missing Idempotency Key Storage in Compensating Endpoints"
      },
      {
        "line": 450,
        "title": "3. The Choreographed Event Loop (\"The Kafka Death Spiral\")"
      },
      {
        "line": 455,
        "title": "4. Attempting 2PC Over External Public REST APIs"
      },
      {
        "line": 462,
        "title": "8. AWS Cloud Service Implementation & Production Patterns"
      },
      {
        "line": 489,
        "title": "1. AWS Step Functions: Standard vs. Express Workflows"
      },
      {
        "line": 493,
        "title": "2. Amazon DynamoDB `TransactWriteItems`"
      },
      {
        "line": 508,
        "title": "3. Amazon EventBridge & SQS FIFO"
      },
      {
        "line": 514,
        "title": "Production AWS Step Functions State Machine (ASL JSON)"
      },
      {
        "line": 665,
        "title": "9. Production Sizing Matrix & Operational Runbook"
      },
      {
        "line": 667,
        "title": "Mathematical Sizing Formulas"
      },
      {
        "line": 669,
        "title": "1. Total Workflow Timeout Bound ($T_{\\text{max}}$)"
      },
      {
        "line": 676,
        "title": "2. History Event Storage Consumption"
      },
      {
        "line": 683,
        "title": "Sizing Matrix: Workload Archetypes vs. Coordination Engine"
      },
      {
        "line": 694,
        "title": "Phased Operational Rollout Runbook"
      },
      {
        "line": 710,
        "title": "Phase 1: Outbox & Semantic Lock Provisioning"
      },
      {
        "line": 715,
        "title": "Phase 2: Dual Execution & Shadow Verification"
      },
      {
        "line": 720,
        "title": "Phase 3: Canary Traffic Cutover & DLQ Alarms"
      },
      {
        "line": 725,
        "title": "Phase 4: Chaos Injection & Anti-Entropy Verification"
      },
      {
        "line": 734,
        "title": "10. Production Diagnostics: Telemetry Signatures & Incident Response Playbook"
      },
      {
        "line": 736,
        "title": "Diagnostic Playbook 1: `ExecutionsFailed` Alarm Spike (> 0.5%)"
      },
      {
        "line": 746,
        "title": "Diagnostic Playbook 2: `ExecutionThrottled` on Step Functions API Rate Limits"
      },
      {
        "line": 756,
        "title": "Diagnostic Playbook 3: `CompensationFailureRate` > 0 (P1 Critical Disaster)"
      },
      {
        "line": 774,
        "title": "Diagnostic Playbook 4: Database Connection Pool Exhaustion from 2PC Locks"
      },
      {
        "line": 794,
        "title": "Diagnostic Playbook 5: Outbox CDC Lag Spike (`DebeziumLag` > 10,000 Records)"
      },
      {
        "line": 811,
        "title": "Diagnostic Playbook 6: SQS FIFO Dead-Letter Queue Accumulation"
      },
      {
        "line": 827,
        "title": "Shared-Responsibility Matrix: Distributed Transaction Topologies"
      },
      {
        "line": 839,
        "title": "Common Production Anti-Patterns Quick-Reference"
      }
    ],
    "fences": [
      {
        "start": 15,
        "end": 29,
        "language": "mermaid"
      },
      {
        "start": 58,
        "end": 81,
        "language": "mermaid"
      },
      {
        "start": 120,
        "end": 149,
        "language": "mermaid"
      },
      {
        "start": 197,
        "end": 203,
        "language": "python"
      },
      {
        "start": 210,
        "end": 227,
        "language": "mermaid"
      },
      {
        "start": 247,
        "end": 269,
        "language": "mermaid"
      },
      {
        "start": 295,
        "end": 322,
        "language": "mermaid"
      },
      {
        "start": 375,
        "end": 399,
        "language": "mermaid"
      },
      {
        "start": 466,
        "end": 485,
        "language": "mermaid"
      },
      {
        "start": 516,
        "end": 661,
        "language": "json"
      },
      {
        "start": 696,
        "end": 706,
        "language": "mermaid"
      }
    ],
    "links": [
      "../loop-primitives/10-retries-timeouts-backpressure-and-load-shedding.md"
    ],
    "lessons": [
      18,
      23,
      24
    ]
  },
  {
    "path": "components/11-geospatial-geohash-quadtree-s2.md",
    "sha256": "928cd931f8f1a1eb92ad632c4430e65e347500ddff7081c3479712e459904079",
    "lines": 161,
    "words": 1777,
    "headings": [
      {
        "line": 1,
        "title": "Geospatial Indexing (Geohash, Quadtree, S2 & H3) — Primitives Deep Dive"
      },
      {
        "line": 3,
        "title": "1. What It Is & Why It Exists"
      },
      {
        "line": 5,
        "title": "The Core Problem: 2D Coordinates on 1D Database Indexes"
      },
      {
        "line": 15,
        "title": "The First-Principles Solution: Space-Filling Curves"
      },
      {
        "line": 32,
        "title": "2. Core Mechanics & Algorithmic Architectures"
      },
      {
        "line": 34,
        "title": "1. Geohash (Z-Order Curve Interleaving)"
      },
      {
        "line": 40,
        "title": "2. Google S2 (Hilbert Curve on Cube Projection)"
      },
      {
        "line": 45,
        "title": "3. Uber H3 (Hexagonal Hierarchical Spatial Index)"
      },
      {
        "line": 61,
        "title": "3. Comprehensive Spatial Engine Comparison Matrix"
      },
      {
        "line": 73,
        "title": "4. Mathematical Foundations: The Haversine Distance Formula"
      },
      {
        "line": 83,
        "title": "5. Critical Edge Cases & Distributed Failure Modes"
      },
      {
        "line": 85,
        "title": "1. The Geohash Boundary Edge Miss (The \"Fence Paradox\")"
      },
      {
        "line": 103,
        "title": "2. The Antimeridian / 180th Meridian Wrap-Around"
      },
      {
        "line": 107,
        "title": "3. Dynamic QuadTree Rebalancing Bottleneck"
      },
      {
        "line": 113,
        "title": "6. Production Pitfalls & Anti-Patterns (The \"Gotchas\")"
      },
      {
        "line": 121,
        "title": "7. AWS Cloud Service Implementation & Production Patterns"
      },
      {
        "line": 123,
        "title": "1. Amazon Aurora PostgreSQL PostGIS Spatial Query"
      },
      {
        "line": 136,
        "title": "2. Amazon ElastiCache Redis Geospatial Commands"
      },
      {
        "line": 150,
        "title": "8. Production Sizing Formulas & Operational Runbook"
      },
      {
        "line": 152,
        "title": "Sizing In-Memory Geospatial Driver Fleet (Redis GEO)"
      },
      {
        "line": 158,
        "title": "Critical Operational Alarms"
      }
    ],
    "fences": [
      {
        "start": 10,
        "end": 12,
        "language": "sql"
      },
      {
        "start": 18,
        "end": 26,
        "language": "mermaid"
      },
      {
        "start": 51,
        "end": 57,
        "language": "mermaid"
      },
      {
        "start": 91,
        "end": 99,
        "language": "mermaid"
      },
      {
        "start": 124,
        "end": 134,
        "language": "sql"
      },
      {
        "start": 139,
        "end": 146,
        "language": "bash"
      }
    ],
    "links": [
      "../interview-loops/05-location-and-geospatial/01-proximity-service.md"
    ],
    "lessons": [
      33
    ]
  },
  {
    "path": "components/12-change-data-capture-and-outbox-pattern.md",
    "sha256": "92962af4a2eb7d7345ee7386b4582a432c86e2565a398f5f6958f05f700eab2d",
    "lines": 166,
    "words": 1885,
    "headings": [
      {
        "line": 1,
        "title": "Change Data Capture (CDC) & Outbox Pattern — Primitives Deep Dive"
      },
      {
        "line": 3,
        "title": "1. What It Is & Why It Exists"
      },
      {
        "line": 5,
        "title": "The Core Problem: The Dual-Write Desynchronization Trap"
      },
      {
        "line": 30,
        "title": "The First-Principles Solution: Transactional Outbox & Log-Based CDC"
      },
      {
        "line": 35,
        "title": "2. Core Mechanics: Transactional Outbox vs. Log-Based CDC"
      },
      {
        "line": 60,
        "title": "Outbox Table Schema & DDL"
      },
      {
        "line": 86,
        "title": "3. Comparison Matrix: CDC Extraction Mechanisms"
      },
      {
        "line": 96,
        "title": "4. Critical Edge Cases & Distributed Failure Modes"
      },
      {
        "line": 98,
        "title": "1. PostgreSQL WAL Disk Exhaustion (`pg_wal` Replication Slot Trap)"
      },
      {
        "line": 103,
        "title": "2. Out-of-Order Event Processing on Partitioned Streams"
      },
      {
        "line": 110,
        "title": "3. Downstream Poison Pill Batches in Lambda / Kinesis"
      },
      {
        "line": 117,
        "title": "5. Production Pitfalls & Anti-Patterns (The \"Gotchas\")"
      },
      {
        "line": 125,
        "title": "6. AWS Cloud Service Implementation & Production Patterns"
      },
      {
        "line": 127,
        "title": "1. Amazon DynamoDB Streams + AWS Lambda"
      },
      {
        "line": 140,
        "title": "2. AWS Database Migration Service (DMS) CDC"
      },
      {
        "line": 145,
        "title": "7. Production Tuning & Sizing Runbook"
      },
      {
        "line": 147,
        "title": "AWS Lambda Event Source Mapping Configuration"
      },
      {
        "line": 163,
        "title": "Critical Operational Alarms"
      }
    ],
    "fences": [
      {
        "start": 8,
        "end": 23,
        "language": "mermaid"
      },
      {
        "start": 37,
        "end": 56,
        "language": "mermaid"
      },
      {
        "start": 61,
        "end": 82,
        "language": "sql"
      },
      {
        "start": 130,
        "end": 136,
        "language": "mermaid"
      },
      {
        "start": 148,
        "end": 161,
        "language": "yaml"
      }
    ],
    "links": [
      "../loop-primitives/05-change-streams-and-transactional-outbox.md"
    ],
    "lessons": [
      20
    ]
  },
  {
    "path": "components/13-trie-data-structure-and-inverted-index.md",
    "sha256": "536f2ea2e1fb0fea92c7f1fafd735a1e14a42a0b8308008abace33bf7bf1990d",
    "lines": 149,
    "words": 1362,
    "headings": [
      {
        "line": 1,
        "title": "Trie & Inverted Index — Primitives Deep Dive"
      },
      {
        "line": 3,
        "title": "1. What It Is & Why It Exists"
      },
      {
        "line": 5,
        "title": "The Core Problem: Prefix Matching & Arbitrary Full-Text Search"
      },
      {
        "line": 10,
        "title": "The First-Principles Solution: Specialized Information Retrieval Structures"
      },
      {
        "line": 39,
        "title": "2. Mathematical Foundations & Algorithmic Mechanics"
      },
      {
        "line": 41,
        "title": "1. The Okapi BM25 Ranking Formula"
      },
      {
        "line": 52,
        "title": "2. Posting List Intersection Time Complexity"
      },
      {
        "line": 60,
        "title": "3. Real-World Engine Comparison Matrix"
      },
      {
        "line": 71,
        "title": "4. Critical Edge Cases & Distributed Failure Modes"
      },
      {
        "line": 73,
        "title": "1. High Memory Consumption in Standard Tries"
      },
      {
        "line": 78,
        "title": "2. High Frequency Terms / Stop Word Posting List Explosion"
      },
      {
        "line": 83,
        "title": "3. Top-K Ranking Explosion in Tries"
      },
      {
        "line": 90,
        "title": "5. Production Pitfalls & Anti-Patterns (The \"Gotchas\")"
      },
      {
        "line": 98,
        "title": "6. AWS Cloud Service Implementation & Production Patterns"
      },
      {
        "line": 100,
        "title": "Amazon OpenSearch Completion Suggester"
      },
      {
        "line": 138,
        "title": "7. Production Sizing Formulas & Operational Runbook"
      },
      {
        "line": 140,
        "title": "In-Memory Trie Sizing Formula (with Top-K Cache)"
      },
      {
        "line": 146,
        "title": "Critical Operational Alarms"
      }
    ],
    "fences": [
      {
        "start": 14,
        "end": 33,
        "language": "mermaid"
      },
      {
        "start": 101,
        "end": 118,
        "language": "json"
      },
      {
        "start": 120,
        "end": 134,
        "language": "json"
      }
    ],
    "links": [
      "../interview-loops/03-storage-and-search/02-search-autocomplete-system.md"
    ],
    "lessons": [
      32
    ]
  },
  {
    "path": "components/14-api-gateway-and-reverse-proxy.md",
    "sha256": "1913fc3f09948f64a78fefb59afd9935bfba0af3ccdfca32ce64a5eb69539150",
    "lines": 139,
    "words": 1442,
    "headings": [
      {
        "line": 1,
        "title": "API Gateway & Reverse Proxy — Primitives Deep Dive"
      },
      {
        "line": 3,
        "title": "1. What It Is & Why It Exists"
      },
      {
        "line": 5,
        "title": "The Core Problem: Direct Client-to-Microservice Chaos"
      },
      {
        "line": 11,
        "title": "The First-Principles Solution: Reverse Proxy & API Gateway"
      },
      {
        "line": 38,
        "title": "2. Core Mechanics: Layer 4 vs. Layer 7 Ingress"
      },
      {
        "line": 53,
        "title": "Comprehensive Comparison Matrix"
      },
      {
        "line": 65,
        "title": "3. Resilience Patterns: Circuit Breakers & Adaptive Concurrency Limiting"
      },
      {
        "line": 67,
        "title": "1. The Circuit Breaker State Machine"
      },
      {
        "line": 83,
        "title": "2. Adaptive Concurrency Limiting (Vegas Algorithm)"
      },
      {
        "line": 90,
        "title": "4. Critical Edge Cases & Distributed Failure Modes"
      },
      {
        "line": 92,
        "title": "1. Gateway Connection Pool Exhaustion (The \"Slow Backend\" Outage)"
      },
      {
        "line": 97,
        "title": "2. JWT Verification Latency Spikes at the Edge"
      },
      {
        "line": 103,
        "title": "5. Production Pitfalls & Anti-Patterns (The \"Gotchas\")"
      },
      {
        "line": 111,
        "title": "6. AWS Cloud Service Implementation & Production Patterns"
      },
      {
        "line": 113,
        "title": "Amazon API Gateway with Lambda Authorizer & CloudFront Edge"
      },
      {
        "line": 128,
        "title": "7. Production Sizing Formulas & Operational Runbook"
      },
      {
        "line": 130,
        "title": "Upstream Connection Pool Sizing Formula"
      },
      {
        "line": 135,
        "title": "Critical Operational Alarms"
      }
    ],
    "fences": [
      {
        "start": 14,
        "end": 32,
        "language": "mermaid"
      },
      {
        "start": 40,
        "end": 51,
        "language": "mermaid"
      },
      {
        "start": 70,
        "end": 77,
        "language": "mermaid"
      },
      {
        "start": 114,
        "end": 122,
        "language": "mermaid"
      }
    ],
    "links": [
      "../loop-primitives/10-retries-timeouts-backpressure-and-load-shedding.md"
    ],
    "lessons": [
      7,
      29
    ]
  },
  {
    "path": "components/15-distributed-unique-id-generators.md",
    "sha256": "bbee8ad32fb8c5e22416d9d2bf08b7a57f164df211dd5af28521762598b5cbac",
    "lines": 179,
    "words": 1650,
    "headings": [
      {
        "line": 1,
        "title": "Distributed Unique ID Generators — Primitives Deep Dive"
      },
      {
        "line": 3,
        "title": "1. What It Is & Why It Exists"
      },
      {
        "line": 5,
        "title": "The Core Problem: Why Centralized DB Auto-Increment & UUIDv4 Fail"
      },
      {
        "line": 10,
        "title": "The First-Principles Solution: 64-Bit K-Ordered IDs (Twitter Snowflake & UUIDv7)"
      },
      {
        "line": 34,
        "title": "2. Core Mechanics: Snowflake vs. UUIDv4 vs. UUIDv7 vs. ULID"
      },
      {
        "line": 45,
        "title": "Comprehensive Comparison Matrix"
      },
      {
        "line": 57,
        "title": "3. Mathematical Bit-Shift Assembly Implementation"
      },
      {
        "line": 122,
        "title": "4. Critical Edge Cases & Distributed Failure Modes"
      },
      {
        "line": 124,
        "title": "1. The Clock Skew & NTP Step-Backward Disaster"
      },
      {
        "line": 132,
        "title": "2. Worker ID Collision & Multi-Pod Auto-Scaling"
      },
      {
        "line": 156,
        "title": "5. Production Pitfalls & Anti-Patterns (The \"Gotchas\")"
      },
      {
        "line": 163,
        "title": "6. AWS Cloud Service Implementation & Production Patterns"
      },
      {
        "line": 170,
        "title": "7. Production Sizing Formulas & Operational Runbook"
      },
      {
        "line": 172,
        "title": "Snowflake Capacity Bounds"
      },
      {
        "line": 176,
        "title": "Critical Operational Alarms"
      }
    ],
    "fences": [
      {
        "start": 13,
        "end": 28,
        "language": "mermaid"
      },
      {
        "start": 59,
        "end": 118,
        "language": "typescript"
      },
      {
        "start": 138,
        "end": 152,
        "language": "mermaid"
      }
    ],
    "links": [
      "../interview-loops/01-core-infrastructure/02-distributed-unique-id-generator.md"
    ],
    "lessons": [
      27
    ]
  },
  {
    "path": "components/16-gossip-protocol-and-failure-detection.md",
    "sha256": "ae93d58827f7d73b00ab0c864d601ec85f66fe1a5ca95e51f458fb3a83e55b81",
    "lines": 154,
    "words": 1851,
    "headings": [
      {
        "line": 1,
        "title": "Gossip Protocol & Failure Detection — Primitives Deep Dive"
      },
      {
        "line": 3,
        "title": "1. What It Is & Why It Exists"
      },
      {
        "line": 5,
        "title": "The Core Problem: Decentralized Health & Topology at Massive Scale"
      },
      {
        "line": 11,
        "title": "The First-Principles Solution: The Gossip Protocol (SWIM & Epidemic Spread)"
      },
      {
        "line": 30,
        "title": "2. Core Mechanics: The SWIM Protocol & Mathematical Dissemination Bounds"
      },
      {
        "line": 32,
        "title": "1. Mathematical Infection Bounds"
      },
      {
        "line": 40,
        "title": "2. SWIM (Scalable Weakly-Consistent Infection-Style Process Group Membership)"
      },
      {
        "line": 69,
        "title": "3. Suspicion Mechanism with Incarnation Numbers"
      },
      {
        "line": 76,
        "title": "3. Continuous $\\Phi$-Accrual Failure Detector (Cassandra / Akka)"
      },
      {
        "line": 98,
        "title": "4. Comprehensive Engine Comparison Matrix"
      },
      {
        "line": 109,
        "title": "5. Critical Edge Cases & Distributed Failure Modes"
      },
      {
        "line": 111,
        "title": "1. Asymmetric Network Partitions (One-Way Traffic Drops)"
      },
      {
        "line": 116,
        "title": "2. Tombstone Propagation & Phantom Rejoin"
      },
      {
        "line": 122,
        "title": "6. Production Pitfalls & Anti-Patterns (The \"Gotchas\")"
      },
      {
        "line": 130,
        "title": "7. AWS Cloud Service Implementation & Production Patterns"
      },
      {
        "line": 137,
        "title": "8. Production Tuning & Sizing Runbook"
      },
      {
        "line": 139,
        "title": "Cassandra Gossip Configuration (`cassandra.yaml`)"
      },
      {
        "line": 151,
        "title": "Critical Operational Alarms"
      }
    ],
    "fences": [
      {
        "start": 14,
        "end": 24,
        "language": "mermaid"
      },
      {
        "start": 43,
        "end": 67,
        "language": "mermaid"
      },
      {
        "start": 84,
        "end": 92,
        "language": "mermaid"
      },
      {
        "start": 140,
        "end": 149,
        "language": "yaml"
      }
    ],
    "links": [
      "../loop-primitives/06-replication-quorums-and-read-your-writes.md"
    ],
    "lessons": [
      28
    ]
  },
  {
    "path": "components/17-vector-databases-and-approximate-nn.md",
    "sha256": "974b8cd55481c9968deb2cd1206e23916ffd17edad376452d8353f2879e70015",
    "lines": 161,
    "words": 1699,
    "headings": [
      {
        "line": 1,
        "title": "Vector Databases & Approximate Nearest Neighbor (ANN) — Primitives Deep Dive"
      },
      {
        "line": 3,
        "title": "1. What It Is & Why It Exists"
      },
      {
        "line": 5,
        "title": "The Core Problem: High-Dimensional Semantic Search at Scale"
      },
      {
        "line": 12,
        "title": "The First-Principles Solution: Approximate Nearest Neighbor (ANN)"
      },
      {
        "line": 31,
        "title": "2. Mathematical Foundations: Distance Metrics & Algorithms"
      },
      {
        "line": 33,
        "title": "1. Vector Distance Formulations"
      },
      {
        "line": 43,
        "title": "3. ANN Indexing Architectures: HNSW vs. IVF-PQ"
      },
      {
        "line": 65,
        "title": "Comprehensive Comparison Matrix"
      },
      {
        "line": 76,
        "title": "4. Critical Edge Cases & Distributed Failure Modes"
      },
      {
        "line": 78,
        "title": "1. The Post-Filtering Zero-Result Trap"
      },
      {
        "line": 96,
        "title": "2. High-Dimensional Memory Exhaustion (The \"RAM Wall\")"
      },
      {
        "line": 105,
        "title": "5. Production Pitfalls & Anti-Patterns (The \"Gotchas\")"
      },
      {
        "line": 113,
        "title": "6. AWS Cloud Service Implementation & Production Patterns"
      },
      {
        "line": 115,
        "title": "1. Amazon Aurora PostgreSQL with `pgvector` & HNSW"
      },
      {
        "line": 145,
        "title": "2. Amazon OpenSearch Service k-NN"
      },
      {
        "line": 150,
        "title": "7. Production Sizing Formulas & Operational Runbook"
      },
      {
        "line": 152,
        "title": "HNSW Memory Sizing Formula"
      },
      {
        "line": 157,
        "title": "Critical Operational Alarms"
      }
    ],
    "fences": [
      {
        "start": 15,
        "end": 25,
        "language": "mermaid"
      },
      {
        "start": 45,
        "end": 61,
        "language": "mermaid"
      },
      {
        "start": 83,
        "end": 92,
        "language": "mermaid"
      },
      {
        "start": 116,
        "end": 143,
        "language": "sql"
      }
    ],
    "links": [],
    "lessons": [
      34
    ]
  },
  {
    "path": "components/18-websocket-sse-and-long-polling.md",
    "sha256": "23f575daa086a5f335f39eb28f82216736becd97282ca5a2c73bdafe497d9287",
    "lines": 117,
    "words": 1376,
    "headings": [
      {
        "line": 1,
        "title": "WebSocket, Server-Sent Events (SSE) & Long Polling — Primitives Deep Dive"
      },
      {
        "line": 3,
        "title": "1. What It Is & Why It Exists"
      },
      {
        "line": 5,
        "title": "The Core Problem: Real-Time Server-to-Client Push"
      },
      {
        "line": 10,
        "title": "The First-Principles Solution: Persistent Streaming Protocols"
      },
      {
        "line": 24,
        "title": "2. Comprehensive Protocol Comparison Matrix"
      },
      {
        "line": 35,
        "title": "3. Distributed WebSocket Gateway Architecture & Routing Backplane"
      },
      {
        "line": 50,
        "title": "DynamoDB Connection Registry Schema"
      },
      {
        "line": 56,
        "title": "4. Critical Edge Cases & Distributed Failure Modes"
      },
      {
        "line": 58,
        "title": "1. The Reconnection Storm (Thundering Herd Outage)"
      },
      {
        "line": 64,
        "title": "2. Half-Open TCP Sockets (Silent Zombie Connections)"
      },
      {
        "line": 71,
        "title": "5. Production Pitfalls & Anti-Patterns (The \"Gotchas\")"
      },
      {
        "line": 79,
        "title": "6. AWS Cloud Service Implementation & Production Patterns"
      },
      {
        "line": 81,
        "title": "Amazon API Gateway WebSocket API (Serverless Connection Management)"
      },
      {
        "line": 100,
        "title": "7. Production Tuning & Sizing Runbook"
      },
      {
        "line": 102,
        "title": "Linux Kernel OS Socket Tuning (`/etc/sysctl.conf`)"
      },
      {
        "line": 113,
        "title": "Critical Operational Alarms"
      }
    ],
    "fences": [
      {
        "start": 13,
        "end": 20,
        "language": "mermaid"
      },
      {
        "start": 39,
        "end": 46,
        "language": "mermaid"
      },
      {
        "start": 86,
        "end": 94,
        "language": "mermaid"
      },
      {
        "start": 103,
        "end": 111,
        "language": "ini"
      }
    ],
    "links": [],
    "lessons": [
      30
    ]
  },
  {
    "path": "components/19-api-protocols-http1-http2-http3-quic-grpc.md",
    "sha256": "a209a6ffbbdbb008455dcf1f211aa32efb296bd063252e090389a668b7bbce80",
    "lines": 180,
    "words": 1775,
    "headings": [
      {
        "line": 1,
        "title": "API Protocols — HTTP/1.1 vs. HTTP/2 vs. HTTP/3 (QUIC) vs. gRPC"
      },
      {
        "line": 3,
        "title": "1. What It Is & Why It Exists"
      },
      {
        "line": 5,
        "title": "The Core Problem: Network Inefficiency & Head-of-Line Blocking"
      },
      {
        "line": 33,
        "title": "2. Core Mechanics & Wire Protocol Comparison"
      },
      {
        "line": 66,
        "title": "Comprehensive Protocol Comparison Matrix"
      },
      {
        "line": 82,
        "title": "3. Mathematical & Network Performance Formulations"
      },
      {
        "line": 84,
        "title": "1. Connection Establishment Latency Derivation"
      },
      {
        "line": 94,
        "title": "2. Protobuf Serialization vs. JSON Payload Efficiency"
      },
      {
        "line": 104,
        "title": "4. Production gRPC Service Architecture"
      },
      {
        "line": 132,
        "title": "5. Critical Edge Cases & Distributed Failure Modes"
      },
      {
        "line": 134,
        "title": "1. The HTTP/2 Large File Stream Starvation"
      },
      {
        "line": 139,
        "title": "2. QUIC UDP Blocking in Enterprise Corporate Firewalls"
      },
      {
        "line": 143,
        "title": "3. gRPC Client Load Balancing Pitfall (Sticky TCP Connection)"
      },
      {
        "line": 149,
        "title": "6. AWS Reference Architecture"
      },
      {
        "line": 173,
        "title": "7. Interview Delivery Framework"
      }
    ],
    "fences": [
      {
        "start": 12,
        "end": 29,
        "language": "mermaid"
      },
      {
        "start": 35,
        "end": 62,
        "language": "mermaid"
      },
      {
        "start": 106,
        "end": 128,
        "language": "mermaid"
      },
      {
        "start": 151,
        "end": 167,
        "language": "mermaid"
      }
    ],
    "links": [],
    "lessons": [
      29
    ]
  },
  {
    "path": "components/20-oauth2-oidc-and-distributed-token-authentication.md",
    "sha256": "629fdaf34858998253321b348727f48f855a4d4daf3d60cffb582eaf0a575cb4",
    "lines": 161,
    "words": 1674,
    "headings": [
      {
        "line": 1,
        "title": "OAuth 2.0, OIDC & Distributed Token Authentication"
      },
      {
        "line": 3,
        "title": "1. What It Is & Why It Exists"
      },
      {
        "line": 5,
        "title": "The Core Problem: Delegated Authorization vs. Authentication"
      },
      {
        "line": 30,
        "title": "2. OAuth 2.0 Authorization Code Flow with PKCE"
      },
      {
        "line": 58,
        "title": "3. Token Formats & Cryptographic Comparison"
      },
      {
        "line": 60,
        "title": "Comprehensive Comparison Matrix"
      },
      {
        "line": 71,
        "title": "4. Distributed Token Revocation Architecture"
      },
      {
        "line": 101,
        "title": "Revocation Formulas & False Positive Sizing"
      },
      {
        "line": 109,
        "title": "5. Critical Edge Cases & Security Failure Modes"
      },
      {
        "line": 111,
        "title": "1. The `alg: none` and Key Confusion Attack"
      },
      {
        "line": 116,
        "title": "2. JWKS Endpoint Outage Latency Cascade"
      },
      {
        "line": 120,
        "title": "3. Refresh Token Replay Attacks"
      },
      {
        "line": 126,
        "title": "6. AWS Reference Architecture"
      },
      {
        "line": 154,
        "title": "7. Interview Delivery Framework"
      }
    ],
    "fences": [
      {
        "start": 10,
        "end": 26,
        "language": "mermaid"
      },
      {
        "start": 34,
        "end": 54,
        "language": "mermaid"
      },
      {
        "start": 75,
        "end": 97,
        "language": "mermaid"
      },
      {
        "start": 128,
        "end": 148,
        "language": "mermaid"
      }
    ],
    "links": [],
    "lessons": [
      31
    ]
  },
  {
    "path": "components/21-database-isolation-levels-acid-and-concurrency-anomalies.md",
    "sha256": "dffc95a8bf5a13dcc82ac1c69b3e86558bb3dcdce28338ffc6ffc074936e5ec1",
    "lines": 192,
    "words": 1989,
    "headings": [
      {
        "line": 1,
        "title": "Database Isolation Levels, ACID & Concurrency Anomalies"
      },
      {
        "line": 3,
        "title": "1. What It Is & Why It Exists"
      },
      {
        "line": 5,
        "title": "The Core Problem: Concurrency vs. Consistency Trade-off"
      },
      {
        "line": 34,
        "title": "2. Concurrency Anomalies & ANSI SQL Lattice"
      },
      {
        "line": 59,
        "title": "Comprehensive Isolation Levels Matrix"
      },
      {
        "line": 71,
        "title": "3. Deep Dive: The On-Call Doctor Write Skew Anomaly"
      },
      {
        "line": 105,
        "title": "Why Repeatable Read / Snapshot Isolation Fails"
      },
      {
        "line": 108,
        "title": "The Solutions:"
      },
      {
        "line": 118,
        "title": "4. MVCC Mechanics: Multi-Version Concurrency Control"
      },
      {
        "line": 141,
        "title": "Key MVCC Properties:"
      },
      {
        "line": 148,
        "title": "5. Critical Edge Cases & Distributed Pitfalls"
      },
      {
        "line": 150,
        "title": "1. The PostgreSQL Deadlock Storm"
      },
      {
        "line": 155,
        "title": "2. Lock Escalation & Giant Deletes"
      },
      {
        "line": 162,
        "title": "6. AWS Reference Architecture"
      },
      {
        "line": 184,
        "title": "7. Interview Delivery Framework"
      }
    ],
    "fences": [
      {
        "start": 13,
        "end": 30,
        "language": "mermaid"
      },
      {
        "start": 36,
        "end": 55,
        "language": "mermaid"
      },
      {
        "start": 75,
        "end": 103,
        "language": "mermaid"
      },
      {
        "start": 120,
        "end": 137,
        "language": "mermaid"
      },
      {
        "start": 164,
        "end": 178,
        "language": "mermaid"
      }
    ],
    "links": [
      "../loop-primitives/04-idempotency-and-effectively-once-processing.md",
      "../loop-primitives/10-retries-timeouts-backpressure-and-load-shedding.md"
    ],
    "lessons": [
      1
    ]
  },
  {
    "path": "components/22-circuit-breaker-bulkhead-and-fault-tolerance-patterns.md",
    "sha256": "6a2872e8bc916750a43c6af14fc8e04591abe463cd80807dcc91c0a9724d9281",
    "lines": 218,
    "words": 3565,
    "headings": [
      {
        "line": 1,
        "title": "Circuit Breaker, Bulkhead & Fault Tolerance Patterns"
      },
      {
        "line": 3,
        "title": "1. What It Is & Why It Exists"
      },
      {
        "line": 5,
        "title": "The Core Problem: Microservice Cascading Outages"
      },
      {
        "line": 26,
        "title": "2. Core Resilience Patterns"
      },
      {
        "line": 46,
        "title": "3. Circuit Breaker State Machine & Mechanics"
      },
      {
        "line": 73,
        "title": "Sliding Window Mathematical Formulation"
      },
      {
        "line": 81,
        "title": "4. Bulkhead Pattern: Thread Pool & Semaphore Isolation"
      },
      {
        "line": 121,
        "title": "5. Exponential Backoff with Full Jitter"
      },
      {
        "line": 140,
        "title": "6. Deadline & Context Propagation"
      },
      {
        "line": 166,
        "title": "7. Critical Edge Cases & Distributed Failure Modes"
      },
      {
        "line": 168,
        "title": "1. The Circuit Breaker Half-Open Thundering Herd"
      },
      {
        "line": 172,
        "title": "2. Dual-Write Cascade on Retries"
      },
      {
        "line": 178,
        "title": "8. AWS Reference Architecture"
      },
      {
        "line": 211,
        "title": "9. Interview Delivery Framework"
      }
    ],
    "fences": [
      {
        "start": 11,
        "end": 20,
        "language": "mermaid"
      },
      {
        "start": 28,
        "end": 40,
        "language": "mermaid"
      },
      {
        "start": 48,
        "end": 69,
        "language": "mermaid"
      },
      {
        "start": 83,
        "end": 111,
        "language": "mermaid"
      },
      {
        "start": 127,
        "end": 134,
        "language": "mermaid"
      },
      {
        "start": 142,
        "end": 160,
        "language": "mermaid"
      },
      {
        "start": 180,
        "end": 205,
        "language": "mermaid"
      }
    ],
    "links": [
      "../loop-primitives/10-retries-timeouts-backpressure-and-load-shedding.md",
      "../loop-primitives/10-retries-timeouts-backpressure-and-load-shedding.md",
      "../loop-primitives/10-retries-timeouts-backpressure-and-load-shedding.md",
      "../loop-primitives/10-retries-timeouts-backpressure-and-load-shedding.md"
    ],
    "lessons": [
      6,
      7
    ]
  },
  {
    "path": "components/23-event-sourcing-and-cqrs-architecture.md",
    "sha256": "6d7732a744a6fc4ae8b5215e1f9069393bfe6a8987a59cf09446021f23d81104",
    "lines": 167,
    "words": 1589,
    "headings": [
      {
        "line": 1,
        "title": "Event Sourcing & CQRS Architecture"
      },
      {
        "line": 3,
        "title": "1. What It Is & Why It Exists"
      },
      {
        "line": 5,
        "title": "The Core Problem: State Mutation Loss & Query Contention"
      },
      {
        "line": 30,
        "title": "2. CQRS: Command Query Responsibility Segregation"
      },
      {
        "line": 67,
        "title": "3. Event Store Mechanics & Aggregate Snapshotting"
      },
      {
        "line": 69,
        "title": "1. The Append-Only Event Log"
      },
      {
        "line": 80,
        "title": "2. Snapshotting Optimization for Long-Lived Aggregates"
      },
      {
        "line": 103,
        "title": "4. Comprehensive Comparison Matrix"
      },
      {
        "line": 117,
        "title": "5. Critical Edge Cases & Distributed Failure Modes"
      },
      {
        "line": 119,
        "title": "1. Concurrent Write Collisions & Optimistic Lock Failure"
      },
      {
        "line": 123,
        "title": "2. Out-of-Order Projection Consumer Processing"
      },
      {
        "line": 127,
        "title": "3. Event Schema Evolution (Upcasting)"
      },
      {
        "line": 133,
        "title": "6. AWS Reference Architecture"
      },
      {
        "line": 161,
        "title": "7. Interview Delivery Framework"
      }
    ],
    "fences": [
      {
        "start": 11,
        "end": 24,
        "language": "mermaid"
      },
      {
        "start": 32,
        "end": 63,
        "language": "mermaid"
      },
      {
        "start": 71,
        "end": 78,
        "language": "json"
      },
      {
        "start": 86,
        "end": 97,
        "language": "mermaid"
      },
      {
        "start": 135,
        "end": 155,
        "language": "mermaid"
      }
    ],
    "links": [
      "../loop-primitives/04-idempotency-and-effectively-once-processing.md",
      "../loop-primitives/06-replication-quorums-and-read-your-writes.md"
    ],
    "lessons": [
      26
    ]
  },
  {
    "path": "components/24-cloud-disaster-recovery-and-multi-region-active-active.md",
    "sha256": "88d9f5289e13c7ef6f2f466d301610d2c47aaab6a594fae0b113bac69a0637ad",
    "lines": 251,
    "words": 3890,
    "headings": [
      {
        "line": 1,
        "title": "Cloud Disaster Recovery & Multi-Region Active-Active"
      },
      {
        "line": 3,
        "title": "1. What It Is & Why It Exists"
      },
      {
        "line": 5,
        "title": "The Core Problem: Regional Catastrophes & Data Loss"
      },
      {
        "line": 25,
        "title": "2. The 4 Cloud Disaster Recovery Strategies"
      },
      {
        "line": 48,
        "title": "Comprehensive DR Comparison Matrix"
      },
      {
        "line": 59,
        "title": "3. Active-Passive Disaster Recovery: Pilot Light and Warm Standby"
      },
      {
        "line": 61,
        "title": "What active-passive means"
      },
      {
        "line": 69,
        "title": "Pilot Light"
      },
      {
        "line": 78,
        "title": "Warm Standby"
      },
      {
        "line": 108,
        "title": "Pilot Light and Warm Standby, step by step"
      },
      {
        "line": 130,
        "title": "Choosing between them, and the hot standby in between"
      },
      {
        "line": 140,
        "title": "4. Multi-Region Active-Active Data Synchronization & CRDTs"
      },
      {
        "line": 168,
        "title": "Conflict-Free Replicated Data Types (CRDTs)"
      },
      {
        "line": 175,
        "title": "5. Global Traffic Ingress: Route 53 & Global Accelerator"
      },
      {
        "line": 201,
        "title": "6. Critical Edge Cases & Distributed Failure Modes"
      },
      {
        "line": 203,
        "title": "1. Cross-Region Write Skew and Split-Brain Partitioning"
      },
      {
        "line": 208,
        "title": "2. Cascading Regional Failover Overload (The \"Herd of Elephants\")"
      },
      {
        "line": 214,
        "title": "7. AWS Reference Architecture"
      },
      {
        "line": 242,
        "title": "8. Interview Delivery Framework"
      }
    ],
    "fences": [
      {
        "start": 11,
        "end": 19,
        "language": "mermaid"
      },
      {
        "start": 27,
        "end": 46,
        "language": "mermaid"
      },
      {
        "start": 87,
        "end": 104,
        "language": "mermaid"
      },
      {
        "start": 144,
        "end": 166,
        "language": "mermaid"
      },
      {
        "start": 177,
        "end": 195,
        "language": "mermaid"
      },
      {
        "start": 216,
        "end": 236,
        "language": "mermaid"
      }
    ],
    "links": [
      "../loop-primitives/09-multi-region-failover.md",
      "../loop-primitives/09-multi-region-failover.md",
      "../loop-primitives/09-multi-region-failover.md",
      "../loop-primitives/09-multi-region-failover.md",
      "../loop-primitives/03-leases-fencing-tokens-and-distributed-locks.md",
      "../loop-primitives/09-multi-region-failover.md"
    ],
    "lessons": [
      25
    ],
    "discovery_sha256": "88d2a8183643c47e2394c38f3d100f5deddefdc96221690f6e5b66a2ac48fb8a",
    "reviewed_commit": "0a6e4d67a2d97c9f2295ac7f9a512b3d984ccf25"
  },
  {
    "path": "components/25-bot-defense-sybil-resistance-and-registration-abuse.md",
    "sha256": "f8ea3c71a127740f64eb8ade5e1e2c2ff961e07fb6f93a6acf29d9ddf73edcdc",
    "lines": 202,
    "words": 2013,
    "headings": [
      {
        "line": 1,
        "title": "Bot Defense, Sybil Resistance & Registration Abuse Mitigation — Primitives Deep Dive"
      },
      {
        "line": 3,
        "title": "1. What It Is & Why It Exists"
      },
      {
        "line": 5,
        "title": "The Attack Vector: Account Creation Abuse & Sybil Attacks"
      },
      {
        "line": 42,
        "title": "The Economic Model of Bot Defense"
      },
      {
        "line": 55,
        "title": "2. Ingress Defense Pipeline: Sequence Architecture"
      },
      {
        "line": 114,
        "title": "3. Technology & Mechanism Comparison Matrix"
      },
      {
        "line": 128,
        "title": "4. AWS Production Architecture"
      },
      {
        "line": 165,
        "title": "Key AWS Implementation Details"
      },
      {
        "line": 178,
        "title": "5. Failure Modes, Pitfalls & Production Trade-offs"
      },
      {
        "line": 180,
        "title": "1. The False-Positive Trap: University & Corporate NATs"
      },
      {
        "line": 184,
        "title": "2. The Email Bombing & SES Bounce Rate Risk"
      },
      {
        "line": 191,
        "title": "3. Accessibility & Cognitive Load (WCAG 2.2 Compliance)"
      },
      {
        "line": 197,
        "title": "6. System Design Interview Cheat Sheet"
      }
    ],
    "fences": [
      {
        "start": 10,
        "end": 38,
        "language": "mermaid"
      },
      {
        "start": 57,
        "end": 110,
        "language": "mermaid"
      },
      {
        "start": 130,
        "end": 161,
        "language": "mermaid"
      }
    ],
    "links": [
      "./26-honeypot-fields-and-canary-traps.md",
      "./27-disposable-email-and-domain-blocklists.md",
      "./28-cloudflare-turnstile-managed-challenges.md",
      "./29-google-recaptcha-v2-and-enterprise-risk-analysis.md",
      "./30-hashcash-and-client-side-proof-of-work.md",
      "../interview-loops/08-real-world-case-studies/06-bot-defense-and-sybil-resistant-registration-architecture.md"
    ],
    "lessons": [
      35
    ]
  },
  {
    "path": "components/26-honeypot-fields-and-canary-traps.md",
    "sha256": "fcfd7e131510288df0250669163b734b6499a88fdac896fc282e1c8b9dc0d7e6",
    "lines": 232,
    "words": 1773,
    "headings": [
      {
        "line": 1,
        "title": "Honeypot Fields & Canary Traps — Primitives Deep Dive"
      },
      {
        "line": 3,
        "title": "1. What It Is & Why It Exists"
      },
      {
        "line": 5,
        "title": "The Problem: Indiscriminate Automated Form Submission"
      },
      {
        "line": 37,
        "title": "The Zero-Friction UX Advantage"
      },
      {
        "line": 45,
        "title": "2. Core Mechanics & Mathematical / Algorithmic Foundation"
      },
      {
        "line": 47,
        "title": "A. CSS Off-Screen Positioning vs Visual Concealment"
      },
      {
        "line": 57,
        "title": "B. Screen Reader & Accessibility Safeguards (WCAG 2.2)"
      },
      {
        "line": 63,
        "title": "C. Time-to-Submit Analysis (Timing Traps)"
      },
      {
        "line": 82,
        "title": "3. Implementation Patterns & Production Code"
      },
      {
        "line": 84,
        "title": "Client-Side React / Next.js Component"
      },
      {
        "line": 128,
        "title": "Server-Side Silent Honeytrap Validation (Node.js / ASP.NET Core)"
      },
      {
        "line": 156,
        "title": "4. Production Engineering, Scaling & AWS Cloud Architecture"
      },
      {
        "line": 189,
        "title": "1. AWS WAF Edge Inspection"
      },
      {
        "line": 194,
        "title": "2. Canary Database & Delayed Blacklisting"
      },
      {
        "line": 202,
        "title": "5. Security Pitfalls, Failure Modes & Bypass Mitigations"
      },
      {
        "line": 204,
        "title": "1. The Browser Autofill False Positive"
      },
      {
        "line": 209,
        "title": "2. Headless Browser Style Inspection (`getComputedStyle`)"
      },
      {
        "line": 219,
        "title": "6. System Design Interview Blueprint & Trade-Off Matrix"
      },
      {
        "line": 229,
        "title": "Key Interview Talking Points"
      }
    ],
    "fences": [
      {
        "start": 8,
        "end": 33,
        "language": "mermaid"
      },
      {
        "start": 85,
        "end": 126,
        "language": "tsx"
      },
      {
        "start": 129,
        "end": 152,
        "language": "typescript"
      },
      {
        "start": 158,
        "end": 185,
        "language": "mermaid"
      }
    ],
    "links": [
      "../interview-loops/08-real-world-case-studies/06-bot-defense-and-sybil-resistant-registration-architecture.md"
    ],
    "lessons": [
      35
    ]
  },
  {
    "path": "components/27-disposable-email-and-domain-blocklists.md",
    "sha256": "a3e0c05365b6da3f15e5f872602f2c44f17f90ef518e4af1ba03e05648db7441",
    "lines": 260,
    "words": 1934,
    "headings": [
      {
        "line": 1,
        "title": "Disposable Email & Domain Blocklists — Primitives Deep Dive"
      },
      {
        "line": 3,
        "title": "1. What It Is & Why It Exists"
      },
      {
        "line": 5,
        "title": "The Attack Vector: Disposable Burner Inboxes"
      },
      {
        "line": 38,
        "title": "The Systemic Impact: SES Reputation Destruction"
      },
      {
        "line": 46,
        "title": "2. Core Mechanics & Algorithmic Foundation"
      },
      {
        "line": 48,
        "title": "A. Fast Set Membership: In-Memory Set vs Bloom Filter"
      },
      {
        "line": 62,
        "title": "B. Pre-flight DNS MX Record Resolution"
      },
      {
        "line": 93,
        "title": "3. Implementation Patterns & Production Code"
      },
      {
        "line": 95,
        "title": "High-Performance TypeScript Domain Validator"
      },
      {
        "line": 171,
        "title": "4. Production Engineering, Scaling & AWS Cloud Architecture"
      },
      {
        "line": 211,
        "title": "1. Automated Blocklist Synchronization Pipeline"
      },
      {
        "line": 217,
        "title": "2. Amazon SES Sender Reputation Safeguard"
      },
      {
        "line": 223,
        "title": "5. Failure Modes, Pitfalls & Production Trade-offs"
      },
      {
        "line": 225,
        "title": "1. Custom / Self-Hosted Catch-All Domains"
      },
      {
        "line": 229,
        "title": "2. The Plus-Addressing Exploit (`user+alias@gmail.com`)"
      },
      {
        "line": 248,
        "title": "6. System Design Interview Blueprint & Trade-Off Matrix"
      },
      {
        "line": 257,
        "title": "Key Interview Talking Points"
      }
    ],
    "fences": [
      {
        "start": 10,
        "end": 34,
        "language": "mermaid"
      },
      {
        "start": 65,
        "end": 89,
        "language": "mermaid"
      },
      {
        "start": 96,
        "end": 167,
        "language": "typescript"
      },
      {
        "start": 173,
        "end": 207,
        "language": "mermaid"
      }
    ],
    "links": [],
    "lessons": [
      35
    ]
  },
  {
    "path": "components/28-cloudflare-turnstile-managed-challenges.md",
    "sha256": "7173c139673e31212bf41d66f9b03283eee529586498269381ec52576fdf1cd4",
    "lines": 272,
    "words": 1909,
    "headings": [
      {
        "line": 1,
        "title": "Cloudflare Turnstile & Managed Challenges — Primitives Deep Dive"
      },
      {
        "line": 3,
        "title": "1. What It Is & Why It Exists"
      },
      {
        "line": 5,
        "title": "The Death of Visual CAPTCHAs"
      },
      {
        "line": 31,
        "title": "2. Core Mechanics & Cryptographic / Telemetry Engine"
      },
      {
        "line": 33,
        "title": "A. Non-Interactive Browser Telemetry Checks"
      },
      {
        "line": 40,
        "title": "B. Private Access Tokens (PATs) & IETF Privacy Pass"
      },
      {
        "line": 73,
        "title": "3. Implementation Patterns & Production Code"
      },
      {
        "line": 75,
        "title": "Client-Side React / Next.js Component"
      },
      {
        "line": 142,
        "title": "Server-Side Token Verification Endpoint"
      },
      {
        "line": 198,
        "title": "4. Production Engineering, Scaling & AWS Cloud Architecture"
      },
      {
        "line": 228,
        "title": "1. Token Replay Defense (Anti-Double-Spend)"
      },
      {
        "line": 237,
        "title": "2. Failure Handling & Circuit Breaking"
      },
      {
        "line": 244,
        "title": "5. Security Pitfalls, Failure Modes & Bypass Mitigations"
      },
      {
        "line": 246,
        "title": "1. The Headless Stealth Bypass (`puppeteer-extra-plugin-stealth`)"
      },
      {
        "line": 250,
        "title": "2. Token Stealing via Clickjacking"
      },
      {
        "line": 256,
        "title": "6. Privacy & Regulatory Compliance (GDPR vs Google reCAPTCHA)"
      },
      {
        "line": 267,
        "title": "7. System Design Interview Blueprint & Trade-Off Matrix"
      },
      {
        "line": 269,
        "title": "Key Interview Talking Points"
      }
    ],
    "fences": [
      {
        "start": 12,
        "end": 25,
        "language": "mermaid"
      },
      {
        "start": 43,
        "end": 66,
        "language": "mermaid"
      },
      {
        "start": 76,
        "end": 140,
        "language": "tsx"
      },
      {
        "start": 143,
        "end": 194,
        "language": "typescript"
      },
      {
        "start": 200,
        "end": 224,
        "language": "mermaid"
      }
    ],
    "links": [
      "../interview-loops/08-real-world-case-studies/06-bot-defense-and-sybil-resistant-registration-architecture.md"
    ],
    "lessons": [
      36
    ]
  },
  {
    "path": "components/29-google-recaptcha-v2-and-enterprise-risk-analysis.md",
    "sha256": "0270001d07a94e4fd868a9a4ade5787e63ee6393f9f7ff4cbb7e16cc349184d8",
    "lines": 263,
    "words": 2045,
    "headings": [
      {
        "line": 1,
        "title": "Google reCAPTCHA v2 & Enterprise Risk Analysis — Primitives Deep Dive"
      },
      {
        "line": 3,
        "title": "1. What It Is & Why It Exists"
      },
      {
        "line": 5,
        "title": "The Evolution of Turing Tests on the Web"
      },
      {
        "line": 25,
        "title": "2. Core Mechanics & Algorithmic Foundation"
      },
      {
        "line": 27,
        "title": "A. Kinematic Mouse Trajectory & Entropy Analysis"
      },
      {
        "line": 50,
        "title": "B. The Image Grid Ground-Truth Engine"
      },
      {
        "line": 57,
        "title": "3. Implementation Patterns & Production Code"
      },
      {
        "line": 59,
        "title": "Client-Side React / Next.js Integration"
      },
      {
        "line": 119,
        "title": "Server-Side Siteverify Protocol (Node.js)"
      },
      {
        "line": 161,
        "title": "4. The Adversarial Arms Race: Why reCAPTCHA v2 Is Failing"
      },
      {
        "line": 191,
        "title": "The Economic Equation of the Attack"
      },
      {
        "line": 198,
        "title": "5. Production Engineering, Scaling & AWS Cloud Architecture"
      },
      {
        "line": 226,
        "title": "High-Throughput Token Caching Pattern"
      },
      {
        "line": 236,
        "title": "6. Accessibility & Regulatory Pitfalls (WCAG 2.2 & GDPR)"
      },
      {
        "line": 238,
        "title": "A. WCAG 2.2 (Non-text Content and Accessible Authentication)"
      },
      {
        "line": 243,
        "title": "B. GDPR & ePrivacy Directive (EU Cookie Laws)"
      },
      {
        "line": 250,
        "title": "7. System Design Interview Blueprint & Trade-Off Matrix"
      },
      {
        "line": 260,
        "title": "Key Interview Takeaways"
      }
    ],
    "fences": [
      {
        "start": 8,
        "end": 15,
        "language": "mermaid"
      },
      {
        "start": 32,
        "end": 46,
        "language": "mermaid"
      },
      {
        "start": 60,
        "end": 117,
        "language": "tsx"
      },
      {
        "start": 120,
        "end": 157,
        "language": "typescript"
      },
      {
        "start": 165,
        "end": 187,
        "language": "mermaid"
      },
      {
        "start": 200,
        "end": 222,
        "language": "mermaid"
      }
    ],
    "links": [
      "../interview-loops/08-real-world-case-studies/06-bot-defense-and-sybil-resistant-registration-architecture.md"
    ],
    "lessons": [
      36
    ]
  },
  {
    "path": "components/30-hashcash-and-client-side-proof-of-work.md",
    "sha256": "1b39856909ecaaaac7a0c45e43494cd174c0102c7dff209d06aad4934241f213",
    "lines": 264,
    "words": 2190,
    "headings": [
      {
        "line": 1,
        "title": "Hashcash & Client-Side Proof of Work — Primitives Deep Dive"
      },
      {
        "line": 3,
        "title": "1. What It Is & Why It Exists"
      },
      {
        "line": 5,
        "title": "The Asymmetric Cost Paradigm"
      },
      {
        "line": 25,
        "title": "Complete Decentralization & Zero Third-Party Reliance"
      },
      {
        "line": 33,
        "title": "2. Core Mechanics & Mathematical / Algorithmic Foundation"
      },
      {
        "line": 35,
        "title": "A. Partial Preimage Collision (SHA-256)"
      },
      {
        "line": 66,
        "title": "B. Algorithmic Complexity Asymmetry"
      },
      {
        "line": 76,
        "title": "C. Dynamic Difficulty Adjustment ($d$)"
      },
      {
        "line": 91,
        "title": "3. Implementation Patterns & Production Code"
      },
      {
        "line": 93,
        "title": "Client-Side Non-Blocking Web Worker Solver (TypeScript)"
      },
      {
        "line": 132,
        "title": "Server-Side Stateless Verifier (Node.js / Express)"
      },
      {
        "line": 193,
        "title": "4. Production Engineering, Scaling & AWS Cloud Architecture"
      },
      {
        "line": 225,
        "title": "Zero-Database Stateless Challenge Issuance"
      },
      {
        "line": 232,
        "title": "5. Security Pitfalls, Failure Modes & Mitigations"
      },
      {
        "line": 234,
        "title": "1. Hardware Asymmetry: Mobile CPU vs GPU / ASIC"
      },
      {
        "line": 238,
        "title": "2. Mobile Battery Drain & Thermal Throttling"
      },
      {
        "line": 244,
        "title": "6. Real-World Case Studies: Who Uses Hashcash Today?"
      },
      {
        "line": 252,
        "title": "7. System Design Interview Blueprint & Trade-Off Matrix"
      },
      {
        "line": 261,
        "title": "Key Interview Talking Points"
      }
    ],
    "fences": [
      {
        "start": 10,
        "end": 21,
        "language": "mermaid"
      },
      {
        "start": 40,
        "end": 64,
        "language": "mermaid"
      },
      {
        "start": 96,
        "end": 130,
        "language": "typescript"
      },
      {
        "start": 133,
        "end": 189,
        "language": "typescript"
      },
      {
        "start": 195,
        "end": 221,
        "language": "mermaid"
      }
    ],
    "links": [
      "../interview-loops/08-real-world-case-studies/06-bot-defense-and-sybil-resistant-registration-architecture.md"
    ],
    "lessons": [
      37
    ]
  },
  {
    "path": "loop-primitives/01-write-ahead-log-fsync-and-group-commit.md",
    "sha256": "16d87d2392032b27b77eaf8f5bdcf573f578414ab61aae7b35d6116f8c970b34",
    "lines": 1288,
    "words": 19541,
    "headings": [
      {
        "line": 1,
        "title": "Write-Ahead Log, fsync & Group Commit"
      },
      {
        "line": 4,
        "title": "Part 0. Start here"
      },
      {
        "line": 6,
        "title": "The problem: 100,000 acknowledged writes a second"
      },
      {
        "line": 33,
        "title": "The big picture"
      },
      {
        "line": 52,
        "title": "What you'll be able to do after this page"
      },
      {
        "line": 70,
        "title": "Part 1. What \"acknowledged\" has to mean"
      },
      {
        "line": 74,
        "title": "The WAL rule"
      },
      {
        "line": 83,
        "title": "Four ways to fail"
      },
      {
        "line": 94,
        "title": "The durability chain"
      },
      {
        "line": 123,
        "title": "The tiny store, now watching the log"
      },
      {
        "line": 148,
        "title": "Part 2. What fsync really guarantees"
      },
      {
        "line": 152,
        "title": "The calls"
      },
      {
        "line": 165,
        "title": "The device's own cache"
      },
      {
        "line": 182,
        "title": "Creating and growing the log file"
      },
      {
        "line": 190,
        "title": "Trace: the first write"
      },
      {
        "line": 214,
        "title": "What a sync costs"
      },
      {
        "line": 260,
        "title": "Part 3. When fsync fails"
      },
      {
        "line": 264,
        "title": "What a failed sync means"
      },
      {
        "line": 273,
        "title": "Trace: the log's sync fails (branch E1)"
      },
      {
        "line": 286,
        "title": "Trace: the flush's sync fails (branch E2)"
      },
      {
        "line": 321,
        "title": "The rule"
      },
      {
        "line": 338,
        "title": "Part 4. The log record and the append"
      },
      {
        "line": 342,
        "title": "What a record holds"
      },
      {
        "line": 360,
        "title": "Trace: `wal-1` after write 4"
      },
      {
        "line": 373,
        "title": "The append"
      },
      {
        "line": 411,
        "title": "Part 5. Group commit"
      },
      {
        "line": 415,
        "title": "One sync, many writers"
      },
      {
        "line": 419,
        "title": "The algorithm"
      },
      {
        "line": 470,
        "title": "Trace: 24 writes, 14 syncs"
      },
      {
        "line": 557,
        "title": "The math"
      },
      {
        "line": 583,
        "title": "The window variant"
      },
      {
        "line": 612,
        "title": "Part 6. Durability modes: what \"acknowledged\" means"
      },
      {
        "line": 616,
        "title": "The ladder"
      },
      {
        "line": 629,
        "title": "Trace: one power loss, four modes"
      },
      {
        "line": 642,
        "title": "How engines name it"
      },
      {
        "line": 657,
        "title": "Losing writes vs corrupting data"
      },
      {
        "line": 692,
        "title": "Part 7. Power loss and torn writes"
      },
      {
        "line": 696,
        "title": "What a torn write is"
      },
      {
        "line": 702,
        "title": "Trace: power fails during B14 (branch P)"
      },
      {
        "line": 725,
        "title": "The recovery scan"
      },
      {
        "line": 774,
        "title": "Writes that survived without their OK"
      },
      {
        "line": 800,
        "title": "Part 8. Segments, rotation and truncation"
      },
      {
        "line": 804,
        "title": "Why segments"
      },
      {
        "line": 812,
        "title": "When a segment may go"
      },
      {
        "line": 833,
        "title": "Trace: after write 24, with and without a stuck reader (branch T)"
      },
      {
        "line": 848,
        "title": "The log as a change stream"
      },
      {
        "line": 877,
        "title": "Part 9. The WAL in B-tree engines"
      },
      {
        "line": 881,
        "title": "Log first in a page-based engine"
      },
      {
        "line": 887,
        "title": "pageLSN makes redo safe to repeat"
      },
      {
        "line": 907,
        "title": "Checkpoints bound replay"
      },
      {
        "line": 911,
        "title": "Torn pages"
      },
      {
        "line": 940,
        "title": "Trace: the tiny store on two pages (branch B)"
      },
      {
        "line": 986,
        "title": "SQLite's WAL mode"
      },
      {
        "line": 1006,
        "title": "Part 10. Replication as a durability level"
      },
      {
        "line": 1010,
        "title": "What each promise means"
      },
      {
        "line": 1023,
        "title": "The latency rule"
      },
      {
        "line": 1027,
        "title": "Trace: three replicas take writes 21 to 24 (branch R)"
      },
      {
        "line": 1053,
        "title": "When the leader is cut off, when everyone loses power"
      },
      {
        "line": 1060,
        "title": "Local sync vs quorum, on equal terms"
      },
      {
        "line": 1086,
        "title": "One acknowledged write through every layer"
      },
      {
        "line": 1135,
        "title": "Part 11. On AWS"
      },
      {
        "line": 1139,
        "title": "Managed services that use a log"
      },
      {
        "line": 1154,
        "title": "Running it yourself on EC2"
      },
      {
        "line": 1166,
        "title": "Not the same thing"
      },
      {
        "line": 1184,
        "title": "Part 12. What you've learned"
      },
      {
        "line": 1186,
        "title": "Back to the 100,000 writes a second"
      },
      {
        "line": 1197,
        "title": "The cheat card"
      },
      {
        "line": 1228,
        "title": "Think-first drills"
      },
      {
        "line": 1264,
        "title": "Interview questions"
      },
      {
        "line": 1283,
        "title": "Where to go next"
      }
    ],
    "fences": [
      {
        "start": 37,
        "end": 48,
        "language": "mermaid"
      },
      {
        "start": 98,
        "end": 119,
        "language": "mermaid"
      },
      {
        "start": 169,
        "end": 176,
        "language": "mermaid"
      },
      {
        "start": 194,
        "end": 210,
        "language": "mermaid"
      },
      {
        "start": 242,
        "end": 246,
        "language": "bash"
      },
      {
        "start": 290,
        "end": 305,
        "language": "mermaid"
      },
      {
        "start": 375,
        "end": 387,
        "language": "pseudocode"
      },
      {
        "start": 421,
        "end": 442,
        "language": "pseudocode"
      },
      {
        "start": 450,
        "end": 466,
        "language": "mermaid"
      },
      {
        "start": 505,
        "end": 553,
        "language": "mermaid"
      },
      {
        "start": 706,
        "end": 721,
        "language": "mermaid"
      },
      {
        "start": 727,
        "end": 748,
        "language": "pseudocode"
      },
      {
        "start": 752,
        "end": 761,
        "language": "mermaid"
      },
      {
        "start": 821,
        "end": 829,
        "language": "mermaid"
      },
      {
        "start": 891,
        "end": 903,
        "language": "pseudocode"
      },
      {
        "start": 915,
        "end": 934,
        "language": "mermaid"
      },
      {
        "start": 944,
        "end": 958,
        "language": "mermaid"
      },
      {
        "start": 1037,
        "end": 1049,
        "language": "mermaid"
      },
      {
        "start": 1088,
        "end": 1101,
        "language": "mermaid"
      }
    ],
    "links": [
      "../loop-primitives/02-lsm-trees-and-compaction.md",
      "../interview-loops/01-core-infrastructure/01-distributed-key-value-store.md",
      "../interview-loops/01-core-infrastructure/01-distributed-key-value-store.md",
      "../interview-loops/01-core-infrastructure/03-distributed-message-queue.md",
      "../interview-loops/04-financial-and-transactional/03-stock-exchange-matching-engine.md",
      "../loop-primitives/02-lsm-trees-and-compaction.md",
      "04-idempotency-and-effectively-once-processing.md",
      "../interview-loops/01-core-infrastructure/03-distributed-message-queue.md",
      "../interview-loops/06-media-and-streaming/04-distributed-email-service.md",
      "../interview-loops/01-core-infrastructure/03-distributed-message-queue.md",
      "04-idempotency-and-effectively-once-processing.md",
      "../interview-loops/01-core-infrastructure/01-distributed-key-value-store.md",
      "../interview-loops/04-financial-and-transactional/05-transactional-outbox-and-event-driven-ledger.md",
      "../interview-loops/04-financial-and-transactional/01-payment-processing-system.md",
      "05-change-streams-and-transactional-outbox.md",
      "../interview-loops/07-mobile-and-offline/01-mobile-news-feed-offline-first.md",
      "../interview-loops/07-mobile-and-offline/02-mobile-chat-architecture.md",
      "06-replication-quorums-and-read-your-writes.md",
      "../components/09-distributed-consensus-raft-and-paxos.md",
      "../interview-loops/03-storage-and-search/04-metrics-monitoring-and-alerting-system.md",
      "04-idempotency-and-effectively-once-processing.md",
      "../loop-primitives/02-lsm-trees-and-compaction.md",
      "04-idempotency-and-effectively-once-processing.md",
      "05-change-streams-and-transactional-outbox.md",
      "06-replication-quorums-and-read-your-writes.md",
      "../components/09-distributed-consensus-raft-and-paxos.md",
      "../components/12-change-data-capture-and-outbox-pattern.md",
      "../components/21-database-isolation-levels-acid-and-concurrency-anomalies.md",
      "../interview-loops/01-core-infrastructure/01-distributed-key-value-store.md",
      "../interview-loops/01-core-infrastructure/03-distributed-message-queue.md",
      "../interview-loops/04-financial-and-transactional/03-stock-exchange-matching-engine.md",
      "../interview-loops/03-storage-and-search/01-s3-like-object-storage.md",
      "../interview-loops/03-storage-and-search/04-metrics-monitoring-and-alerting-system.md",
      "../interview-loops/07-mobile-and-offline/01-mobile-news-feed-offline-first.md",
      "../interview-loops/07-mobile-and-offline/02-mobile-chat-architecture.md",
      "../interview-loops/04-financial-and-transactional/05-transactional-outbox-and-event-driven-ledger.md",
      "../interview-loops/04-financial-and-transactional/01-payment-processing-system.md"
    ],
    "lessons": [
      2,
      3
    ]
  },
  {
    "path": "loop-primitives/02-lsm-trees-and-compaction.md",
    "sha256": "803af8b403315e6db5f4c3e78d74f1616ec7cfa107c82405ce3f76fa6295b937",
    "lines": 1270,
    "words": 17456,
    "headings": [
      {
        "line": 1,
        "title": "LSM-Trees & Compaction"
      },
      {
        "line": 4,
        "title": "Part 0. Start here"
      },
      {
        "line": 6,
        "title": "The problem: 100,000 writes a second on one cloud disk"
      },
      {
        "line": 29,
        "title": "The big picture"
      },
      {
        "line": 48,
        "title": "What you'll be able to do after this page"
      },
      {
        "line": 66,
        "title": "Part 1. The core mental model"
      },
      {
        "line": 70,
        "title": "The three pieces"
      },
      {
        "line": 99,
        "title": "Three rules the engine never breaks"
      },
      {
        "line": 107,
        "title": "The tiny store we follow"
      },
      {
        "line": 143,
        "title": "Part 2. Writes, step by step"
      },
      {
        "line": 147,
        "title": "What we need from the log"
      },
      {
        "line": 156,
        "title": "Versions and tombstones"
      },
      {
        "line": 175,
        "title": "The MemTable is a skip list"
      },
      {
        "line": 192,
        "title": "The write path"
      },
      {
        "line": 226,
        "title": "Trace: writes 1 to 4 fill the first MemTable"
      },
      {
        "line": 247,
        "title": "Part 3. Flushing the MemTable to disk"
      },
      {
        "line": 251,
        "title": "What a flush writes: the SSTable"
      },
      {
        "line": 272,
        "title": "The safe order"
      },
      {
        "line": 326,
        "title": "Trace: F1 and F2 land in Level 0"
      },
      {
        "line": 379,
        "title": "Part 4. Reads, step by step"
      },
      {
        "line": 383,
        "title": "The search path"
      },
      {
        "line": 428,
        "title": "Bloom filters skip files that lack the key"
      },
      {
        "line": 436,
        "title": "Trace: four kinds of read"
      },
      {
        "line": 471,
        "title": "Range scans"
      },
      {
        "line": 510,
        "title": "Part 5. Compaction: merging SSTables"
      },
      {
        "line": 514,
        "title": "The merge is a k-way merge of sorted files"
      },
      {
        "line": 543,
        "title": "Trace: the first compaction"
      },
      {
        "line": 580,
        "title": "Trace: writes 10 to 16 push data down to Level 2"
      },
      {
        "line": 608,
        "title": "Trace: writes 17 to 24 finish a delete"
      },
      {
        "line": 638,
        "title": "Worked example: a tombstone that must stay"
      },
      {
        "line": 655,
        "title": "Leveled vs size-tiered, in one table"
      },
      {
        "line": 665,
        "title": "Why writes stall"
      },
      {
        "line": 725,
        "title": "Part 6. How files are tracked and levels are assigned"
      },
      {
        "line": 729,
        "title": "The manifest is the directory of levels"
      },
      {
        "line": 752,
        "title": "How a file gets its level"
      },
      {
        "line": 774,
        "title": "The in-memory file metadata routes every read"
      },
      {
        "line": 810,
        "title": "Part 7. B-tree vs LSM tree"
      },
      {
        "line": 849,
        "title": "Part 8. End to end: from your code to the disk"
      },
      {
        "line": 853,
        "title": "The layers"
      },
      {
        "line": 882,
        "title": "The files in the data directory"
      },
      {
        "line": 896,
        "title": "A write, traced down to the device"
      },
      {
        "line": 924,
        "title": "A read, traced down to the device"
      },
      {
        "line": 961,
        "title": "Who does what"
      },
      {
        "line": 982,
        "title": "Part 9. Restarts and the boot sequence"
      },
      {
        "line": 986,
        "title": "The boot sequence"
      },
      {
        "line": 1021,
        "title": "Trace: a crash after write 22"
      },
      {
        "line": 1034,
        "title": "Why this is fast"
      },
      {
        "line": 1040,
        "title": "A crash in the middle of a compaction"
      },
      {
        "line": 1060,
        "title": "Part 10. Corruption: detecting and recovering"
      },
      {
        "line": 1064,
        "title": "Checksums on everything"
      },
      {
        "line": 1073,
        "title": "A damaged WAL"
      },
      {
        "line": 1089,
        "title": "A damaged SSTable"
      },
      {
        "line": 1095,
        "title": "Getting the data back"
      },
      {
        "line": 1131,
        "title": "Part 11. On AWS"
      },
      {
        "line": 1135,
        "title": "Managed services that use an LSM tree"
      },
      {
        "line": 1142,
        "title": "Running it yourself on EC2"
      },
      {
        "line": 1155,
        "title": "Look-alikes that are not an LSM tree"
      },
      {
        "line": 1174,
        "title": "Part 12. What you've learned"
      },
      {
        "line": 1176,
        "title": "Back to the 100,000 writes a second"
      },
      {
        "line": 1187,
        "title": "The cheat card"
      },
      {
        "line": 1218,
        "title": "Think-first drills"
      },
      {
        "line": 1254,
        "title": "Interview questions"
      },
      {
        "line": 1265,
        "title": "Where to go next"
      }
    ],
    "fences": [
      {
        "start": 33,
        "end": 44,
        "language": "mermaid"
      },
      {
        "start": 80,
        "end": 95,
        "language": "mermaid"
      },
      {
        "start": 181,
        "end": 188,
        "language": "mermaid"
      },
      {
        "start": 194,
        "end": 205,
        "language": "pseudocode"
      },
      {
        "start": 209,
        "end": 222,
        "language": "mermaid"
      },
      {
        "start": 262,
        "end": 268,
        "language": "mermaid"
      },
      {
        "start": 274,
        "end": 285,
        "language": "pseudocode"
      },
      {
        "start": 291,
        "end": 301,
        "language": "mermaid"
      },
      {
        "start": 330,
        "end": 345,
        "language": "mermaid"
      },
      {
        "start": 351,
        "end": 367,
        "language": "mermaid"
      },
      {
        "start": 385,
        "end": 396,
        "language": "pseudocode"
      },
      {
        "start": 400,
        "end": 410,
        "language": "pseudocode"
      },
      {
        "start": 414,
        "end": 424,
        "language": "mermaid"
      },
      {
        "start": 440,
        "end": 456,
        "language": "mermaid"
      },
      {
        "start": 475,
        "end": 494,
        "language": "pseudocode"
      },
      {
        "start": 521,
        "end": 539,
        "language": "pseudocode"
      },
      {
        "start": 560,
        "end": 576,
        "language": "mermaid"
      },
      {
        "start": 587,
        "end": 604,
        "language": "mermaid"
      },
      {
        "start": 616,
        "end": 634,
        "language": "mermaid"
      },
      {
        "start": 669,
        "end": 679,
        "language": "mermaid"
      },
      {
        "start": 705,
        "end": 711,
        "language": "mermaid"
      },
      {
        "start": 760,
        "end": 770,
        "language": "mermaid"
      },
      {
        "start": 855,
        "end": 878,
        "language": "mermaid"
      },
      {
        "start": 898,
        "end": 915,
        "language": "mermaid"
      },
      {
        "start": 928,
        "end": 949,
        "language": "mermaid"
      },
      {
        "start": 988,
        "end": 1004,
        "language": "pseudocode"
      },
      {
        "start": 1008,
        "end": 1017,
        "language": "mermaid"
      },
      {
        "start": 1109,
        "end": 1119,
        "language": "mermaid"
      }
    ],
    "links": [
      "../interview-loops/01-core-infrastructure/01-distributed-key-value-store.md",
      "../interview-loops/03-storage-and-search/01-s3-like-object-storage.md",
      "01-write-ahead-log-fsync-and-group-commit.md",
      "../components/03-bloom-filters-and-counting-filters.md",
      "../situations/10-wal-lsm-compaction-storm.md",
      "../situations/10-wal-lsm-compaction-storm.md",
      "../situations/10-wal-lsm-compaction-storm.md",
      "01-write-ahead-log-fsync-and-group-commit.md",
      "01-write-ahead-log-fsync-and-group-commit.md",
      "07-event-time-watermarks-and-checkpoints.md",
      "01-write-ahead-log-fsync-and-group-commit.md",
      "01-write-ahead-log-fsync-and-group-commit.md",
      "01-write-ahead-log-fsync-and-group-commit.md",
      "../components/03-bloom-filters-and-counting-filters.md",
      "../situations/10-wal-lsm-compaction-storm.md",
      "../interview-loops/01-core-infrastructure/01-distributed-key-value-store.md",
      "../interview-loops/03-storage-and-search/01-s3-like-object-storage.md"
    ],
    "lessons": [
      3,
      4
    ]
  },
  {
    "path": "loop-primitives/03-leases-fencing-tokens-and-distributed-locks.md",
    "sha256": "dbd5fa657076f39892729cffac5f4111c4981d75b28720243a0ef153e6b6eb2d",
    "lines": 1436,
    "words": 21971,
    "headings": [
      {
        "line": 1,
        "title": "Leases, Fencing Tokens & Distributed Locks"
      },
      {
        "line": 4,
        "title": "Part 0. Start here"
      },
      {
        "line": 6,
        "title": "The problem: a job that must run exactly once"
      },
      {
        "line": 28,
        "title": "The big picture"
      },
      {
        "line": 51,
        "title": "What you'll be able to do after this page"
      },
      {
        "line": 69,
        "title": "Part 1. The core mental model"
      },
      {
        "line": 73,
        "title": "Lock, lease, fence"
      },
      {
        "line": 91,
        "title": "Three rules the design never breaks"
      },
      {
        "line": 99,
        "title": "Not every \"lease\" is this lease"
      },
      {
        "line": 103,
        "title": "The example we follow"
      },
      {
        "line": 132,
        "title": "Part 2. Acquire, renew, release: the happy path"
      },
      {
        "line": 136,
        "title": "Acquire"
      },
      {
        "line": 153,
        "title": "What the lock service keeps, and where"
      },
      {
        "line": 164,
        "title": "Renew and the holder's stop time"
      },
      {
        "line": 226,
        "title": "Release"
      },
      {
        "line": 257,
        "title": "Snapshot S1: A holds the job"
      },
      {
        "line": 284,
        "title": "Part 3. Time: whose clock decides?"
      },
      {
        "line": 288,
        "title": "Two clocks in every machine"
      },
      {
        "line": 295,
        "title": "Two different clock errors"
      },
      {
        "line": 304,
        "title": "What the margin covers"
      },
      {
        "line": 317,
        "title": "The storage's clock must not decide"
      },
      {
        "line": 321,
        "title": "Trace: B's wall clock steps back 3 s"
      },
      {
        "line": 364,
        "title": "Part 4. The pause, the zombie and the fencing token"
      },
      {
        "line": 368,
        "title": "Trace: A pauses, B takes over"
      },
      {
        "line": 406,
        "title": "Where the token is checked"
      },
      {
        "line": 424,
        "title": "Raise the fence first"
      },
      {
        "line": 448,
        "title": "Snapshots S2 and S3: the zombie is stopped"
      },
      {
        "line": 484,
        "title": "The same fence on S3"
      },
      {
        "line": 501,
        "title": "The same rule in other systems"
      },
      {
        "line": 505,
        "title": "What fencing does not protect"
      },
      {
        "line": 517,
        "title": "Part 5. Conditional writes are the check"
      },
      {
        "line": 521,
        "title": "DynamoDB"
      },
      {
        "line": 569,
        "title": "SQL"
      },
      {
        "line": 586,
        "title": "S3"
      },
      {
        "line": 598,
        "title": "What each store can compare"
      },
      {
        "line": 608,
        "title": "A lease built on a DynamoDB item"
      },
      {
        "line": 643,
        "title": "Trace: D replaces B, and D's clock is 3 s fast"
      },
      {
        "line": 684,
        "title": "TTL is not a deadline, and it resets epochs"
      },
      {
        "line": 701,
        "title": "Part 6. Crashes and orphaned locks"
      },
      {
        "line": 705,
        "title": "Trace: B dies holding the job"
      },
      {
        "line": 729,
        "title": "Two kinds of lock, two kinds of cleanup"
      },
      {
        "line": 743,
        "title": "The sweeper"
      },
      {
        "line": 763,
        "title": "How soon can someone take over?"
      },
      {
        "line": 783,
        "title": "A sweeper that limits mass re-queues"
      },
      {
        "line": 793,
        "title": "Snapshot S4: the orphan"
      },
      {
        "line": 821,
        "title": "Part 7. Leader election and single writers"
      },
      {
        "line": 825,
        "title": "A leader is a lease holder"
      },
      {
        "line": 846,
        "title": "An election has no orphan"
      },
      {
        "line": 868,
        "title": "etcd, ZooKeeper and Kubernetes compared"
      },
      {
        "line": 880,
        "title": "Single writers across a failover"
      },
      {
        "line": 945,
        "title": "Part 8. When the lock service itself fails"
      },
      {
        "line": 949,
        "title": "Where the grant lives"
      },
      {
        "line": 965,
        "title": "Trace: the leader dies while C holds the job"
      },
      {
        "line": 993,
        "title": "What a minority can't do"
      },
      {
        "line": 997,
        "title": "What if the lock service is Redis?"
      },
      {
        "line": 1034,
        "title": "Redlock and the argument about it"
      },
      {
        "line": 1081,
        "title": "Across Regions"
      },
      {
        "line": 1091,
        "title": "Restores"
      },
      {
        "line": 1103,
        "title": "Part 9. Do you need a lock at all?"
      },
      {
        "line": 1107,
        "title": "When the database is enough"
      },
      {
        "line": 1113,
        "title": "Four ways to let one worker win without a lock service"
      },
      {
        "line": 1122,
        "title": "How much to lock"
      },
      {
        "line": 1128,
        "title": "The same job three ways"
      },
      {
        "line": 1138,
        "title": "An equal-terms comparison"
      },
      {
        "line": 1171,
        "title": "Part 10. End to end: from your code to the disk"
      },
      {
        "line": 1175,
        "title": "The layers"
      },
      {
        "line": 1195,
        "title": "An acquire, traced down"
      },
      {
        "line": 1220,
        "title": "A keep-alive, traced down"
      },
      {
        "line": 1224,
        "title": "A fenced write, traced down"
      },
      {
        "line": 1241,
        "title": "Who does what"
      },
      {
        "line": 1261,
        "title": "Part 11. On AWS"
      },
      {
        "line": 1265,
        "title": "Managed services that use it"
      },
      {
        "line": 1276,
        "title": "Running it yourself"
      },
      {
        "line": 1285,
        "title": "Look-alikes that are not this mechanism"
      },
      {
        "line": 1306,
        "title": "Part 12. What you've learned"
      },
      {
        "line": 1308,
        "title": "Back to the billing job"
      },
      {
        "line": 1318,
        "title": "The whole story, event by event"
      },
      {
        "line": 1347,
        "title": "The cheat card"
      },
      {
        "line": 1381,
        "title": "Think-first drills"
      },
      {
        "line": 1417,
        "title": "Interview questions"
      },
      {
        "line": 1428,
        "title": "Where to go next"
      }
    ],
    "fences": [
      {
        "start": 30,
        "end": 47,
        "language": "mermaid"
      },
      {
        "start": 81,
        "end": 87,
        "language": "mermaid"
      },
      {
        "start": 138,
        "end": 147,
        "language": "pseudocode"
      },
      {
        "start": 166,
        "end": 182,
        "language": "pseudocode"
      },
      {
        "start": 186,
        "end": 201,
        "language": "mermaid"
      },
      {
        "start": 230,
        "end": 236,
        "language": "pseudocode"
      },
      {
        "start": 242,
        "end": 253,
        "language": "mermaid"
      },
      {
        "start": 259,
        "end": 272,
        "language": "mermaid"
      },
      {
        "start": 344,
        "end": 352,
        "language": "mermaid"
      },
      {
        "start": 370,
        "end": 391,
        "language": "mermaid"
      },
      {
        "start": 410,
        "end": 418,
        "language": "pseudocode"
      },
      {
        "start": 450,
        "end": 463,
        "language": "mermaid"
      },
      {
        "start": 467,
        "end": 480,
        "language": "mermaid"
      },
      {
        "start": 528,
        "end": 550,
        "language": "json"
      },
      {
        "start": 554,
        "end": 565,
        "language": "json"
      },
      {
        "start": 573,
        "end": 582,
        "language": "sql"
      },
      {
        "start": 590,
        "end": 596,
        "language": "http"
      },
      {
        "start": 622,
        "end": 637,
        "language": "json"
      },
      {
        "start": 667,
        "end": 680,
        "language": "mermaid"
      },
      {
        "start": 717,
        "end": 725,
        "language": "mermaid"
      },
      {
        "start": 745,
        "end": 757,
        "language": "pseudocode"
      },
      {
        "start": 795,
        "end": 809,
        "language": "mermaid"
      },
      {
        "start": 829,
        "end": 840,
        "language": "pseudocode"
      },
      {
        "start": 857,
        "end": 864,
        "language": "mermaid"
      },
      {
        "start": 884,
        "end": 891,
        "language": "pseudocode"
      },
      {
        "start": 897,
        "end": 908,
        "language": "mermaid"
      },
      {
        "start": 953,
        "end": 961,
        "language": "mermaid"
      },
      {
        "start": 977,
        "end": 989,
        "language": "mermaid"
      },
      {
        "start": 1001,
        "end": 1018,
        "language": "mermaid"
      },
      {
        "start": 1038,
        "end": 1046,
        "language": "pseudocode"
      },
      {
        "start": 1054,
        "end": 1069,
        "language": "mermaid"
      },
      {
        "start": 1177,
        "end": 1191,
        "language": "mermaid"
      },
      {
        "start": 1199,
        "end": 1216,
        "language": "mermaid"
      },
      {
        "start": 1226,
        "end": 1237,
        "language": "mermaid"
      }
    ],
    "links": [
      "../interview-loops/01-core-infrastructure/02-distributed-unique-id-generator.md",
      "../interview-loops/03-storage-and-search/05-distributed-job-scheduler-and-task-queue.md",
      "../interview-loops/01-core-infrastructure/03-distributed-message-queue.md",
      "../interview-loops/05-location-and-geospatial/04-ride-sharing-dispatch-service.md",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md",
      "../interview-loops/01-core-infrastructure/05-url-shortener.md",
      "../situations/09-distributed-lock-fencing-token.md",
      "../interview-loops/04-financial-and-transactional/03-stock-exchange-matching-engine.md",
      "04-idempotency-and-effectively-once-processing.md",
      "../situations/09-distributed-lock-fencing-token.md",
      "../interview-loops/01-core-infrastructure/02-distributed-unique-id-generator.md",
      "../interview-loops/03-storage-and-search/05-distributed-job-scheduler-and-task-queue.md",
      "../interview-loops/04-financial-and-transactional/01-payment-processing-system.md",
      "../interview-loops/06-media-and-streaming/03-ad-click-event-aggregation.md",
      "../interview-loops/04-financial-and-transactional/05-transactional-outbox-and-event-driven-ledger.md",
      "../interview-loops/04-financial-and-transactional/03-stock-exchange-matching-engine.md",
      "../interview-loops/04-financial-and-transactional/01-payment-processing-system.md",
      "../interview-loops/04-financial-and-transactional/04-hotel-reservation-system.md",
      "../interview-loops/06-media-and-streaming/04-distributed-email-service.md",
      "../interview-loops/04-financial-and-transactional/02-digital-wallet-system.md",
      "../interview-loops/06-media-and-streaming/02-google-drive-file-sync-and-storage.md",
      "../interview-loops/05-location-and-geospatial/02-google-maps-and-routing-engine.md",
      "06-replication-quorums-and-read-your-writes.md",
      "../components/09-distributed-consensus-raft-and-paxos.md",
      "../situations/11-raft-consensus-split-brain.md",
      "09-multi-region-failover.md",
      "../interview-loops/04-financial-and-transactional/04-hotel-reservation-system.md",
      "../interview-loops/05-location-and-geospatial/04-ride-sharing-dispatch-service.md",
      "04-idempotency-and-effectively-once-processing.md",
      "../interview-loops/04-financial-and-transactional/05-transactional-outbox-and-event-driven-ledger.md",
      "../interview-loops/07-mobile-and-offline/01-mobile-news-feed-offline-first.md",
      "../situations/09-distributed-lock-fencing-token.md",
      "../components/21-database-isolation-levels-acid-and-concurrency-anomalies.md",
      "01-write-ahead-log-fsync-and-group-commit.md",
      "04-idempotency-and-effectively-once-processing.md",
      "06-replication-quorums-and-read-your-writes.md",
      "../components/09-distributed-consensus-raft-and-paxos.md",
      "09-multi-region-failover.md",
      "../components/24-cloud-disaster-recovery-and-multi-region-active-active.md",
      "01-write-ahead-log-fsync-and-group-commit.md",
      "../components/15-distributed-unique-id-generators.md",
      "../situations/09-distributed-lock-fencing-token.md",
      "../situations/11-raft-consensus-split-brain.md",
      "../interview-loops/01-core-infrastructure/02-distributed-unique-id-generator.md",
      "../interview-loops/03-storage-and-search/05-distributed-job-scheduler-and-task-queue.md",
      "../interview-loops/01-core-infrastructure/03-distributed-message-queue.md",
      "../interview-loops/05-location-and-geospatial/04-ride-sharing-dispatch-service.md"
    ],
    "lessons": [
      16
    ]
  },
  {
    "path": "loop-primitives/04-idempotency-and-effectively-once-processing.md",
    "sha256": "720eb0f88b7ac14d3750a5b560dc72695777cc1837ccc31ad49a4a96211fc824",
    "lines": 1687,
    "words": 26006,
    "headings": [
      {
        "line": 1,
        "title": "Idempotency & Effectively-Once Processing"
      },
      {
        "line": 4,
        "title": "Part 0. Start here"
      },
      {
        "line": 6,
        "title": "A buyer taps once and is charged twice"
      },
      {
        "line": 12,
        "title": "One tap, eight systems"
      },
      {
        "line": 54,
        "title": "The big picture"
      },
      {
        "line": 71,
        "title": "Each system remembers a repeat for a different length of time"
      },
      {
        "line": 86,
        "title": "What you'll be able to do after this page"
      },
      {
        "line": 105,
        "title": "Part 1. The core mental model"
      },
      {
        "line": 109,
        "title": "Three outcomes of a request"
      },
      {
        "line": 145,
        "title": "An error reply that hides a success"
      },
      {
        "line": 156,
        "title": "Which HTTP methods are already safe to repeat"
      },
      {
        "line": 162,
        "title": "Downstream windows"
      },
      {
        "line": 169,
        "title": "At-most-once, at-least-once, effectively-once"
      },
      {
        "line": 179,
        "title": "Three rules the design never breaks"
      },
      {
        "line": 187,
        "title": "Not this dedup"
      },
      {
        "line": 196,
        "title": "The example we follow"
      },
      {
        "line": 230,
        "title": "Part 2. Idempotency keys"
      },
      {
        "line": 234,
        "title": "Who makes the key, and when"
      },
      {
        "line": 257,
        "title": "What the key covers"
      },
      {
        "line": 266,
        "title": "Binding the key to the request's meaning"
      },
      {
        "line": 291,
        "title": "Derive what others must re-create; store what you send"
      },
      {
        "line": 318,
        "title": "Trace: the tap and the claim"
      },
      {
        "line": 345,
        "title": "Part 3. The key record and the durable guard: claim, record, run, answer"
      },
      {
        "line": 349,
        "title": "Record the intent before the side effect"
      },
      {
        "line": 423,
        "title": "The states of a key record"
      },
      {
        "line": 438,
        "title": "What gets stored: final answers and pointers"
      },
      {
        "line": 448,
        "title": "Trace: a concurrent duplicate, a lost reply and a retry"
      },
      {
        "line": 502,
        "title": "Every answer to a repeat"
      },
      {
        "line": 513,
        "title": "A crashed or paused owner"
      },
      {
        "line": 533,
        "title": "One store or two, and what if the key store is down"
      },
      {
        "line": 543,
        "title": "Finish what the first attempt started"
      },
      {
        "line": 555,
        "title": "Part 4. Time: how long to remember, and the late original"
      },
      {
        "line": 559,
        "title": "When a late original can hurt"
      },
      {
        "line": 563,
        "title": "The retry horizon"
      },
      {
        "line": 583,
        "title": "Trace: the buyer comes back"
      },
      {
        "line": 627,
        "title": "The deadline"
      },
      {
        "line": 674,
        "title": "Resolve, don't just look"
      },
      {
        "line": 701,
        "title": "When a compensation times out"
      },
      {
        "line": 713,
        "title": "Part 5. Idempotent by design: absolute writes and conditions on the durable record"
      },
      {
        "line": 717,
        "title": "Write the result, not the change"
      },
      {
        "line": 742,
        "title": "State transitions, and the late webhook"
      },
      {
        "line": 753,
        "title": "Versions from one counter"
      },
      {
        "line": 768,
        "title": "Condition on the durable record"
      },
      {
        "line": 794,
        "title": "Part 6. Queues and logs: dedup windows, redelivery and the inbox"
      },
      {
        "line": 798,
        "title": "Two places repeats are born"
      },
      {
        "line": 818,
        "title": "The FIFO dedup window, and what keeps the ID stable"
      },
      {
        "line": 830,
        "title": "Trace: the relay crashes twice"
      },
      {
        "line": 858,
        "title": "The idempotent producer, and why a restart defeats it"
      },
      {
        "line": 871,
        "title": "Transactions in the log"
      },
      {
        "line": 882,
        "title": "The consumer's two guards"
      },
      {
        "line": 908,
        "title": "Trace: the worker dies after acting"
      },
      {
        "line": 959,
        "title": "A re-send carries the original ID"
      },
      {
        "line": 990,
        "title": "Part 7. Side effects you can't take back: notifications"
      },
      {
        "line": 994,
        "title": "Why providers can't help"
      },
      {
        "line": 1005,
        "title": "Intent, send, mark"
      },
      {
        "line": 1044,
        "title": "Logical duplicates and windows"
      },
      {
        "line": 1053,
        "title": "Email: the send record, and choosing your failure"
      },
      {
        "line": 1062,
        "title": "Third parties with no key"
      },
      {
        "line": 1066,
        "title": "A detected duplicate must still send the notification"
      },
      {
        "line": 1078,
        "title": "Part 8. Streams: exactly-once state plus idempotent sinks"
      },
      {
        "line": 1084,
        "title": "Checkpoints make state exactly-once"
      },
      {
        "line": 1094,
        "title": "Trace: the job restores"
      },
      {
        "line": 1142,
        "title": "Idempotent sinks vs two-phase-commit sinks"
      },
      {
        "line": 1186,
        "title": "Dedup state, event time and the acceptance horizon"
      },
      {
        "line": 1210,
        "title": "Part 9. Across Regions"
      },
      {
        "line": 1214,
        "title": "What replication does to a claim"
      },
      {
        "line": 1225,
        "title": "Trace: two Regions claim the same key (branch F2)"
      },
      {
        "line": 1263,
        "title": "Trace: a Region fails in the middle of replication (branch F)"
      },
      {
        "line": 1311,
        "title": "Where a claim can live"
      },
      {
        "line": 1334,
        "title": "Part 10. Do you need it? Compared on equal terms"
      },
      {
        "line": 1338,
        "title": "The same payment under seven approaches"
      },
      {
        "line": 1354,
        "title": "The comparison, on equal terms"
      },
      {
        "line": 1381,
        "title": "What a saga gives up"
      },
      {
        "line": 1395,
        "title": "Part 11. End to end: from the tap to the notification"
      },
      {
        "line": 1399,
        "title": "Every place a repeat is born"
      },
      {
        "line": 1417,
        "title": "The hop table"
      },
      {
        "line": 1431,
        "title": "One tap, one sequence"
      },
      {
        "line": 1472,
        "title": "Part 12. On AWS"
      },
      {
        "line": 1476,
        "title": "Managed services that use it"
      },
      {
        "line": 1491,
        "title": "Running it yourself"
      },
      {
        "line": 1498,
        "title": "Shared limits"
      },
      {
        "line": 1507,
        "title": "Look-alikes that are not this mechanism"
      },
      {
        "line": 1528,
        "title": "Part 13. What you've learned"
      },
      {
        "line": 1530,
        "title": "Back to the tap"
      },
      {
        "line": 1543,
        "title": "The whole story, event by event"
      },
      {
        "line": 1581,
        "title": "The cheat card"
      },
      {
        "line": 1617,
        "title": "Think-first drills"
      },
      {
        "line": 1661,
        "title": "Interview questions"
      },
      {
        "line": 1672,
        "title": "Where to go next"
      }
    ],
    "fences": [
      {
        "start": 56,
        "end": 67,
        "language": "mermaid"
      },
      {
        "start": 123,
        "end": 141,
        "language": "mermaid"
      },
      {
        "start": 270,
        "end": 283,
        "language": "pseudocode"
      },
      {
        "start": 299,
        "end": 312,
        "language": "mermaid"
      },
      {
        "start": 327,
        "end": 335,
        "language": "http"
      },
      {
        "start": 351,
        "end": 386,
        "language": "pseudocode"
      },
      {
        "start": 392,
        "end": 405,
        "language": "json"
      },
      {
        "start": 409,
        "end": 421,
        "language": "sql"
      },
      {
        "start": 425,
        "end": 434,
        "language": "mermaid"
      },
      {
        "start": 462,
        "end": 477,
        "language": "mermaid"
      },
      {
        "start": 594,
        "end": 609,
        "language": "mermaid"
      },
      {
        "start": 643,
        "end": 670,
        "language": "mermaid"
      },
      {
        "start": 678,
        "end": 693,
        "language": "pseudocode"
      },
      {
        "start": 774,
        "end": 780,
        "language": "mermaid"
      },
      {
        "start": 800,
        "end": 814,
        "language": "mermaid"
      },
      {
        "start": 839,
        "end": 854,
        "language": "mermaid"
      },
      {
        "start": 891,
        "end": 904,
        "language": "sql"
      },
      {
        "start": 917,
        "end": 934,
        "language": "mermaid"
      },
      {
        "start": 940,
        "end": 955,
        "language": "mermaid"
      },
      {
        "start": 1007,
        "end": 1017,
        "language": "pseudocode"
      },
      {
        "start": 1104,
        "end": 1120,
        "language": "mermaid"
      },
      {
        "start": 1126,
        "end": 1138,
        "language": "mermaid"
      },
      {
        "start": 1171,
        "end": 1182,
        "language": "pseudocode"
      },
      {
        "start": 1229,
        "end": 1245,
        "language": "mermaid"
      },
      {
        "start": 1277,
        "end": 1291,
        "language": "mermaid"
      },
      {
        "start": 1401,
        "end": 1413,
        "language": "mermaid"
      },
      {
        "start": 1435,
        "end": 1460,
        "language": "mermaid"
      }
    ],
    "links": [
      "../interview-loops/04-financial-and-transactional/01-payment-processing-system.md",
      "../interview-loops/04-financial-and-transactional/01-payment-processing-system.md",
      "../interview-loops/07-mobile-and-offline/03-mobile-stock-trading-app.md",
      "../interview-loops/06-media-and-streaming/03-ad-click-event-aggregation.md",
      "../interview-loops/02-social-and-realtime/04-notification-system.md",
      "01-write-ahead-log-fsync-and-group-commit.md",
      "../interview-loops/01-core-infrastructure/01-distributed-key-value-store.md",
      "../interview-loops/06-media-and-streaming/02-google-drive-file-sync-and-storage.md",
      "../interview-loops/03-storage-and-search/03-web-crawler-at-scale.md",
      "05-change-streams-and-transactional-outbox.md",
      "05-change-streams-and-transactional-outbox.md",
      "01-write-ahead-log-fsync-and-group-commit.md",
      "03-leases-fencing-tokens-and-distributed-locks.md",
      "03-leases-fencing-tokens-and-distributed-locks.md",
      "../interview-loops/07-mobile-and-offline/01-mobile-news-feed-offline-first.md",
      "03-leases-fencing-tokens-and-distributed-locks.md",
      "../situations/12-saga-orchestration-compensating-failure.md",
      "../interview-loops/07-mobile-and-offline/01-mobile-news-feed-offline-first.md",
      "../interview-loops/06-media-and-streaming/04-distributed-email-service.md",
      "05-change-streams-and-transactional-outbox.md",
      "05-change-streams-and-transactional-outbox.md",
      "../interview-loops/02-social-and-realtime/04-notification-system.md",
      "../interview-loops/06-media-and-streaming/04-distributed-email-service.md",
      "../interview-loops/06-media-and-streaming/01-youtube-video-streaming-platform.md",
      "07-event-time-watermarks-and-checkpoints.md",
      "09-multi-region-failover.md",
      "06-replication-quorums-and-read-your-writes.md",
      "09-multi-region-failover.md",
      "../components/24-cloud-disaster-recovery-and-multi-region-active-active.md",
      "../components/21-database-isolation-levels-acid-and-concurrency-anomalies.md",
      "03-leases-fencing-tokens-and-distributed-locks.md",
      "../situations/12-saga-orchestration-compensating-failure.md",
      "../components/10-two-phase-commit-and-saga-orchestration.md",
      "01-write-ahead-log-fsync-and-group-commit.md",
      "03-leases-fencing-tokens-and-distributed-locks.md",
      "05-change-streams-and-transactional-outbox.md",
      "../components/12-change-data-capture-and-outbox-pattern.md",
      "06-replication-quorums-and-read-your-writes.md",
      "07-event-time-watermarks-and-checkpoints.md",
      "09-multi-region-failover.md",
      "../components/24-cloud-disaster-recovery-and-multi-region-active-active.md",
      "../components/05-message-queues-vs-event-streams.md",
      "../components/10-two-phase-commit-and-saga-orchestration.md",
      "../situations/12-saga-orchestration-compensating-failure.md",
      "../interview-loops/04-financial-and-transactional/01-payment-processing-system.md",
      "../interview-loops/04-financial-and-transactional/02-digital-wallet-system.md",
      "../interview-loops/04-financial-and-transactional/04-hotel-reservation-system.md",
      "../interview-loops/04-financial-and-transactional/05-transactional-outbox-and-event-driven-ledger.md",
      "../interview-loops/04-financial-and-transactional/03-stock-exchange-matching-engine.md",
      "../interview-loops/01-core-infrastructure/03-distributed-message-queue.md",
      "../interview-loops/01-core-infrastructure/01-distributed-key-value-store.md",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md",
      "../interview-loops/01-core-infrastructure/05-url-shortener.md",
      "../interview-loops/02-social-and-realtime/02-chat-and-instant-messaging-system.md",
      "../interview-loops/02-social-and-realtime/03-realtime-gaming-leaderboard.md",
      "../interview-loops/02-social-and-realtime/04-notification-system.md",
      "../interview-loops/03-storage-and-search/05-distributed-job-scheduler-and-task-queue.md",
      "../interview-loops/03-storage-and-search/01-s3-like-object-storage.md",
      "../interview-loops/03-storage-and-search/03-web-crawler-at-scale.md",
      "../interview-loops/05-location-and-geospatial/01-proximity-service.md",
      "../interview-loops/05-location-and-geospatial/03-nearby-friends-location-service.md",
      "../interview-loops/05-location-and-geospatial/04-ride-sharing-dispatch-service.md",
      "../interview-loops/06-media-and-streaming/01-youtube-video-streaming-platform.md",
      "../interview-loops/06-media-and-streaming/02-google-drive-file-sync-and-storage.md",
      "../interview-loops/06-media-and-streaming/03-ad-click-event-aggregation.md",
      "../interview-loops/06-media-and-streaming/04-distributed-email-service.md",
      "../interview-loops/07-mobile-and-offline/01-mobile-news-feed-offline-first.md",
      "../interview-loops/07-mobile-and-offline/02-mobile-chat-architecture.md",
      "../interview-loops/07-mobile-and-offline/03-mobile-stock-trading-app.md"
    ],
    "lessons": [
      18,
      21
    ]
  },
  {
    "path": "loop-primitives/05-change-streams-and-transactional-outbox.md",
    "sha256": "ce1ab3c3e222627ce75acb2791c7405cbb19ad849c316f0705da42930c267779",
    "lines": 1721,
    "words": 25218,
    "headings": [
      {
        "line": 1,
        "title": "Change Streams & the Transactional Outbox"
      },
      {
        "line": 4,
        "title": "Part 0. Start here"
      },
      {
        "line": 6,
        "title": "The problem: one change, four systems"
      },
      {
        "line": 39,
        "title": "The big picture"
      },
      {
        "line": 60,
        "title": "What you'll be able to do after this page"
      },
      {
        "line": 78,
        "title": "Part 1. Why writing twice fails"
      },
      {
        "line": 82,
        "title": "Trace: the pod dies after the index write"
      },
      {
        "line": 118,
        "title": "Three ways two writes disagree"
      },
      {
        "line": 138,
        "title": "Snapshot S1: four systems, four answers"
      },
      {
        "line": 161,
        "title": "One write, then followers"
      },
      {
        "line": 167,
        "title": "Three rules the design never breaks"
      },
      {
        "line": 175,
        "title": "The example we follow"
      },
      {
        "line": 208,
        "title": "Part 2. The outbox: one transaction"
      },
      {
        "line": 212,
        "title": "One transaction"
      },
      {
        "line": 262,
        "title": "The table and the envelope"
      },
      {
        "line": 275,
        "title": "What it guarantees, and what it doesn't"
      },
      {
        "line": 284,
        "title": "Events the producer designs, not rows that changed"
      },
      {
        "line": 298,
        "title": "Part 3. The relay and at-least-once delivery"
      },
      {
        "line": 302,
        "title": "Trace: the relay dies after publishing"
      },
      {
        "line": 352,
        "title": "The relay loop"
      },
      {
        "line": 378,
        "title": "Consumers apply by version"
      },
      {
        "line": 408,
        "title": "A new owner replays the old owner's work"
      },
      {
        "line": 452,
        "title": "The cursor trap: numbers are not commit order"
      },
      {
        "line": 488,
        "title": "Waking the relay"
      },
      {
        "line": 497,
        "title": "When the broker is down"
      },
      {
        "line": 515,
        "title": "Part 4. Order per aggregate"
      },
      {
        "line": 519,
        "title": "Trace: two relays split one order"
      },
      {
        "line": 531,
        "title": "Every hop must keep order"
      },
      {
        "line": 553,
        "title": "Gaps are checked number by number"
      },
      {
        "line": 577,
        "title": "Adding partitions"
      },
      {
        "line": 618,
        "title": "One transaction, several events"
      },
      {
        "line": 630,
        "title": "Epochs: when the writer itself changes"
      },
      {
        "line": 653,
        "title": "Part 5. Reading the database's own log"
      },
      {
        "line": 657,
        "title": "Why read the log"
      },
      {
        "line": 671,
        "title": "The outbox table or the business tables?"
      },
      {
        "line": 686,
        "title": "PostgreSQL: the replication slot"
      },
      {
        "line": 746,
        "title": "When the slot stalls"
      },
      {
        "line": 761,
        "title": "MySQL: the binlog is purged by time (text only)"
      },
      {
        "line": 775,
        "title": "Cutting over from the relay (event 19)"
      },
      {
        "line": 799,
        "title": "End to end through the layers"
      },
      {
        "line": 843,
        "title": "Part 6. Poison records and repair"
      },
      {
        "line": 847,
        "title": "Trace: `o9`'s bad address"
      },
      {
        "line": 888,
        "title": "The error boundary"
      },
      {
        "line": 942,
        "title": "Part 7. Refilling what the stream can't give you: republish and bootstrap"
      },
      {
        "line": 946,
        "title": "Trace: the slot is invalidated"
      },
      {
        "line": 967,
        "title": "Where gaps come from"
      },
      {
        "line": 977,
        "title": "Detecting a gap"
      },
      {
        "line": 1011,
        "title": "Republish: refill from the stored source, then resume"
      },
      {
        "line": 1076,
        "title": "Trace: the index is down for 30 hours"
      },
      {
        "line": 1148,
        "title": "The tools the source gives you"
      },
      {
        "line": 1170,
        "title": "Part 8. Schema changes and deletes"
      },
      {
        "line": 1174,
        "title": "Trace: old consumer code reads the new `currency` field"
      },
      {
        "line": 1186,
        "title": "Compatibility in plain words"
      },
      {
        "line": 1249,
        "title": "A delete must travel"
      },
      {
        "line": 1259,
        "title": "Trace: `o7` is erased, and a repair comes too late"
      },
      {
        "line": 1308,
        "title": "Part 9. The same story on DynamoDB"
      },
      {
        "line": 1324,
        "title": "DynamoDB Streams in one table"
      },
      {
        "line": 1338,
        "title": "Trace: the poison batch, bisected"
      },
      {
        "line": 1380,
        "title": "Trace: a shard splits"
      },
      {
        "line": 1397,
        "title": "Lambda and Pipes settings"
      },
      {
        "line": 1412,
        "title": "Rebuilding inside 24 hours"
      },
      {
        "line": 1426,
        "title": "Kinesis Data Streams for DynamoDB"
      },
      {
        "line": 1451,
        "title": "Global tables"
      },
      {
        "line": 1455,
        "title": "Do you need an outbox on DynamoDB?"
      },
      {
        "line": 1467,
        "title": "Part 10. Choosing"
      },
      {
        "line": 1471,
        "title": "The same change, three ways"
      },
      {
        "line": 1481,
        "title": "On equal terms"
      },
      {
        "line": 1527,
        "title": "Part 11. On AWS"
      },
      {
        "line": 1531,
        "title": "Managed services that use it"
      },
      {
        "line": 1544,
        "title": "Running it yourself"
      },
      {
        "line": 1551,
        "title": "Look-alikes that are not this mechanism"
      },
      {
        "line": 1571,
        "title": "Part 12. What you've learned"
      },
      {
        "line": 1573,
        "title": "Back to the 432 a day"
      },
      {
        "line": 1585,
        "title": "The whole story, event by event"
      },
      {
        "line": 1621,
        "title": "The cheat card"
      },
      {
        "line": 1658,
        "title": "Think-first drills"
      },
      {
        "line": 1702,
        "title": "Interview questions"
      },
      {
        "line": 1713,
        "title": "Where to go next"
      }
    ],
    "fences": [
      {
        "start": 41,
        "end": 56,
        "language": "mermaid"
      },
      {
        "start": 86,
        "end": 97,
        "language": "mermaid"
      },
      {
        "start": 103,
        "end": 114,
        "language": "mermaid"
      },
      {
        "start": 140,
        "end": 157,
        "language": "mermaid"
      },
      {
        "start": 216,
        "end": 231,
        "language": "sql"
      },
      {
        "start": 235,
        "end": 248,
        "language": "sql"
      },
      {
        "start": 308,
        "end": 325,
        "language": "mermaid"
      },
      {
        "start": 354,
        "end": 372,
        "language": "pseudocode"
      },
      {
        "start": 382,
        "end": 396,
        "language": "pseudocode"
      },
      {
        "start": 422,
        "end": 439,
        "language": "pseudocode"
      },
      {
        "start": 456,
        "end": 471,
        "language": "mermaid"
      },
      {
        "start": 533,
        "end": 539,
        "language": "mermaid"
      },
      {
        "start": 557,
        "end": 573,
        "language": "pseudocode"
      },
      {
        "start": 581,
        "end": 594,
        "language": "mermaid"
      },
      {
        "start": 690,
        "end": 692,
        "language": "sql"
      },
      {
        "start": 696,
        "end": 708,
        "language": "mermaid"
      },
      {
        "start": 726,
        "end": 730,
        "language": "sql"
      },
      {
        "start": 779,
        "end": 793,
        "language": "mermaid"
      },
      {
        "start": 803,
        "end": 815,
        "language": "mermaid"
      },
      {
        "start": 851,
        "end": 863,
        "language": "mermaid"
      },
      {
        "start": 911,
        "end": 922,
        "language": "pseudocode"
      },
      {
        "start": 979,
        "end": 990,
        "language": "pseudocode"
      },
      {
        "start": 998,
        "end": 1007,
        "language": "mermaid"
      },
      {
        "start": 1013,
        "end": 1022,
        "language": "pseudocode"
      },
      {
        "start": 1028,
        "end": 1042,
        "language": "mermaid"
      },
      {
        "start": 1086,
        "end": 1102,
        "language": "mermaid"
      },
      {
        "start": 1222,
        "end": 1233,
        "language": "pseudocode"
      },
      {
        "start": 1237,
        "end": 1245,
        "language": "mermaid"
      },
      {
        "start": 1342,
        "end": 1360,
        "language": "mermaid"
      },
      {
        "start": 1366,
        "end": 1376,
        "language": "json"
      },
      {
        "start": 1384,
        "end": 1391,
        "language": "mermaid"
      }
    ],
    "links": [
      "../interview-loops/04-financial-and-transactional/05-transactional-outbox-and-event-driven-ledger.md",
      "../interview-loops/06-media-and-streaming/04-distributed-email-service.md",
      "../interview-loops/05-location-and-geospatial/04-ride-sharing-dispatch-service.md",
      "../interview-loops/06-media-and-streaming/03-ad-click-event-aggregation.md",
      "../interview-loops/02-social-and-realtime/03-realtime-gaming-leaderboard.md",
      "../interview-loops/06-media-and-streaming/02-google-drive-file-sync-and-storage.md",
      "../situations/14-cdc-outbox-dual-write-drift.md",
      "04-idempotency-and-effectively-once-processing.md",
      "03-leases-fencing-tokens-and-distributed-locks.md",
      "04-idempotency-and-effectively-once-processing.md",
      "../interview-loops/07-mobile-and-offline/02-mobile-chat-architecture.md",
      "03-leases-fencing-tokens-and-distributed-locks.md",
      "04-idempotency-and-effectively-once-processing.md",
      "../interview-loops/04-financial-and-transactional/01-payment-processing-system.md",
      "../interview-loops/04-financial-and-transactional/05-transactional-outbox-and-event-driven-ledger.md",
      "../interview-loops/07-mobile-and-offline/01-mobile-news-feed-offline-first.md",
      "../interview-loops/04-financial-and-transactional/02-digital-wallet-system.md",
      "../interview-loops/04-financial-and-transactional/05-transactional-outbox-and-event-driven-ledger.md",
      "../interview-loops/04-financial-and-transactional/01-payment-processing-system.md",
      "03-leases-fencing-tokens-and-distributed-locks.md",
      "../interview-loops/04-financial-and-transactional/05-transactional-outbox-and-event-driven-ledger.md",
      "../situations/14-cdc-outbox-dual-write-drift.md",
      "01-write-ahead-log-fsync-and-group-commit.md",
      "../interview-loops/04-financial-and-transactional/05-transactional-outbox-and-event-driven-ledger.md",
      "04-idempotency-and-effectively-once-processing.md",
      "01-write-ahead-log-fsync-and-group-commit.md",
      "04-idempotency-and-effectively-once-processing.md",
      "../interview-loops/07-mobile-and-offline/01-mobile-news-feed-offline-first.md",
      "../interview-loops/06-media-and-streaming/03-ad-click-event-aggregation.md",
      "../interview-loops/07-mobile-and-offline/01-mobile-news-feed-offline-first.md",
      "../interview-loops/04-financial-and-transactional/05-transactional-outbox-and-event-driven-ledger.md",
      "07-event-time-watermarks-and-checkpoints.md",
      "../interview-loops/05-location-and-geospatial/02-google-maps-and-routing-engine.md",
      "../interview-loops/02-social-and-realtime/03-realtime-gaming-leaderboard.md",
      "../components/23-event-sourcing-and-cqrs-architecture.md",
      "../components/10-two-phase-commit-and-saga-orchestration.md",
      "../components/10-two-phase-commit-and-saga-orchestration.md",
      "03-leases-fencing-tokens-and-distributed-locks.md",
      "01-write-ahead-log-fsync-and-group-commit.md",
      "03-leases-fencing-tokens-and-distributed-locks.md",
      "04-idempotency-and-effectively-once-processing.md",
      "06-replication-quorums-and-read-your-writes.md",
      "08-sharding-hot-keys-and-rebalancing.md",
      "09-multi-region-failover.md",
      "../components/05-message-queues-vs-event-streams.md",
      "../components/21-database-isolation-levels-acid-and-concurrency-anomalies.md",
      "../components/10-two-phase-commit-and-saga-orchestration.md",
      "../components/23-event-sourcing-and-cqrs-architecture.md",
      "../situations/14-cdc-outbox-dual-write-drift.md",
      "../interview-loops/04-financial-and-transactional/05-transactional-outbox-and-event-driven-ledger.md",
      "../interview-loops/04-financial-and-transactional/01-payment-processing-system.md",
      "../interview-loops/04-financial-and-transactional/02-digital-wallet-system.md",
      "../interview-loops/04-financial-and-transactional/04-hotel-reservation-system.md",
      "../interview-loops/04-financial-and-transactional/03-stock-exchange-matching-engine.md",
      "../interview-loops/05-location-and-geospatial/01-proximity-service.md",
      "../interview-loops/05-location-and-geospatial/04-ride-sharing-dispatch-service.md",
      "../interview-loops/05-location-and-geospatial/02-google-maps-and-routing-engine.md",
      "../interview-loops/05-location-and-geospatial/03-nearby-friends-location-service.md",
      "../interview-loops/06-media-and-streaming/04-distributed-email-service.md",
      "../interview-loops/06-media-and-streaming/01-youtube-video-streaming-platform.md",
      "../interview-loops/06-media-and-streaming/02-google-drive-file-sync-and-storage.md",
      "../interview-loops/06-media-and-streaming/03-ad-click-event-aggregation.md",
      "../interview-loops/07-mobile-and-offline/03-mobile-stock-trading-app.md",
      "../interview-loops/07-mobile-and-offline/01-mobile-news-feed-offline-first.md",
      "../interview-loops/02-social-and-realtime/01-news-feed-system.md",
      "../interview-loops/02-social-and-realtime/03-realtime-gaming-leaderboard.md",
      "../interview-loops/02-social-and-realtime/02-chat-and-instant-messaging-system.md",
      "../interview-loops/01-core-infrastructure/05-url-shortener.md",
      "../interview-loops/03-storage-and-search/01-s3-like-object-storage.md",
      "../interview-loops/03-storage-and-search/03-web-crawler-at-scale.md",
      "../interview-loops/01-core-infrastructure/03-distributed-message-queue.md",
      "../interview-loops/02-social-and-realtime/04-notification-system.md",
      "../interview-loops/07-mobile-and-offline/02-mobile-chat-architecture.md"
    ],
    "lessons": [
      20
    ]
  },
  {
    "path": "loop-primitives/06-replication-quorums-and-read-your-writes.md",
    "sha256": "eaf0e925ec3229150edced2ded9c4a71102f8dc8f930634bb2c5ebebadb90c64",
    "lines": 1372,
    "words": 24112,
    "headings": [
      {
        "line": 1,
        "title": "Replication, Quorums & Read-Your-Writes"
      },
      {
        "line": 4,
        "title": "Part 0. Start here"
      },
      {
        "line": 6,
        "title": "The problem: 500 lost orders and a stale checkout page"
      },
      {
        "line": 42,
        "title": "The big picture"
      },
      {
        "line": 67,
        "title": "What you'll be able to do after this page"
      },
      {
        "line": 85,
        "title": "Part 1. What a replica keeps"
      },
      {
        "line": 89,
        "title": "One log, three copies"
      },
      {
        "line": 95,
        "title": "Stored, committed, applied"
      },
      {
        "line": 129,
        "title": "Three families"
      },
      {
        "line": 141,
        "title": "What the Write-Ahead Log loop primitive already covers"
      },
      {
        "line": 145,
        "title": "The example we follow"
      },
      {
        "line": 174,
        "title": "Part 2. When to say OK: sync, async, semi-sync"
      },
      {
        "line": 178,
        "title": "Trace: A dies at 1.2 ms"
      },
      {
        "line": 196,
        "title": "When followers are down"
      },
      {
        "line": 219,
        "title": "PostgreSQL's cancelled wait"
      },
      {
        "line": 230,
        "title": "How many writes an asynchronous failover can lose"
      },
      {
        "line": 248,
        "title": "Part 3. Replication lag and stale reads"
      },
      {
        "line": 252,
        "title": "Trace: the stale checkout"
      },
      {
        "line": 297,
        "title": "Shipping vs applying"
      },
      {
        "line": 306,
        "title": "Where lag comes from"
      },
      {
        "line": 317,
        "title": "Measuring it and detecting a stuck replica"
      },
      {
        "line": 323,
        "title": "Reads that go backwards"
      },
      {
        "line": 335,
        "title": "Part 4. Read-your-writes and monotonic reads"
      },
      {
        "line": 339,
        "title": "The two promises"
      },
      {
        "line": 350,
        "title": "The mechanism: carry the position"
      },
      {
        "line": 380,
        "title": "The fixes, on equal terms"
      },
      {
        "line": 417,
        "title": "Engines and services"
      },
      {
        "line": 428,
        "title": "The read half, end to end through the layers"
      },
      {
        "line": 465,
        "title": "Part 5. Quorums: N, W and R"
      },
      {
        "line": 469,
        "title": "The leaderless replay"
      },
      {
        "line": 505,
        "title": "The overlap rule"
      },
      {
        "line": 527,
        "title": "Where the overlap stops"
      },
      {
        "line": 537,
        "title": "Consistency levels, strongest first"
      },
      {
        "line": 551,
        "title": "Sloppy quorums and hinted handoff, briefly"
      },
      {
        "line": 587,
        "title": "Part 6. The leader crashes: elections"
      },
      {
        "line": 591,
        "title": "Trace: A dies after sending index 3 to only one follower"
      },
      {
        "line": 608,
        "title": "Terms, votes, and why votes are written down"
      },
      {
        "line": 645,
        "title": "Trace: who wins"
      },
      {
        "line": 690,
        "title": "The no-op and the commitment rule"
      },
      {
        "line": 727,
        "title": "The write that committed anyway"
      },
      {
        "line": 744,
        "title": "Part 7. A network partition and two leaders"
      },
      {
        "line": 748,
        "title": "Trace: C keeps writing"
      },
      {
        "line": 778,
        "title": "What each side can do"
      },
      {
        "line": 789,
        "title": "Terms fence the old leader"
      },
      {
        "line": 793,
        "title": "Reads on a leader that may be deposed"
      },
      {
        "line": 830,
        "title": "When the link returns"
      },
      {
        "line": 863,
        "title": "3 vs 4 vs 5 replicas"
      },
      {
        "line": 873,
        "title": "When a database fails over"
      },
      {
        "line": 905,
        "title": "Part 8. Catching up: log or snapshot"
      },
      {
        "line": 909,
        "title": "Trace: B returns to a compacted log"
      },
      {
        "line": 940,
        "title": "Log or snapshot"
      },
      {
        "line": 978,
        "title": "Part 9. Repair, deletes and what replication can't protect"
      },
      {
        "line": 982,
        "title": "Trace: Alice's erased address comes back (Q7)"
      },
      {
        "line": 994,
        "title": "Read repair and anti-entropy"
      },
      {
        "line": 1025,
        "title": "A delete is a versioned write"
      },
      {
        "line": 1031,
        "title": "Which copy may a repair read?"
      },
      {
        "line": 1037,
        "title": "Backups keep deleted data"
      },
      {
        "line": 1050,
        "title": "One bad update everywhere"
      },
      {
        "line": 1086,
        "title": "Part 10. Choosing"
      },
      {
        "line": 1090,
        "title": "The same write, three families"
      },
      {
        "line": 1100,
        "title": "Families at equal durability"
      },
      {
        "line": 1115,
        "title": "Sync, semi-sync and async on equal terms"
      },
      {
        "line": 1126,
        "title": "Conflicts when many can write"
      },
      {
        "line": 1138,
        "title": "Alice in two Regions"
      },
      {
        "line": 1173,
        "title": "Part 11. On AWS"
      },
      {
        "line": 1177,
        "title": "Managed services that use it"
      },
      {
        "line": 1190,
        "title": "Running it yourself"
      },
      {
        "line": 1197,
        "title": "Look-alikes that are not this mechanism"
      },
      {
        "line": 1217,
        "title": "Part 12. What you've learned"
      },
      {
        "line": 1219,
        "title": "Back to the 500 lost orders and the stale checkout"
      },
      {
        "line": 1227,
        "title": "The whole story, event by event"
      },
      {
        "line": 1270,
        "title": "The cheat card"
      },
      {
        "line": 1307,
        "title": "Think-first drills"
      },
      {
        "line": 1352,
        "title": "Interview questions"
      },
      {
        "line": 1363,
        "title": "Where to go next"
      }
    ],
    "fences": [
      {
        "start": 44,
        "end": 63,
        "language": "mermaid"
      },
      {
        "start": 107,
        "end": 125,
        "language": "mermaid"
      },
      {
        "start": 223,
        "end": 226,
        "language": "text"
      },
      {
        "start": 266,
        "end": 281,
        "language": "mermaid"
      },
      {
        "start": 354,
        "end": 376,
        "language": "pseudocode"
      },
      {
        "start": 432,
        "end": 444,
        "language": "mermaid"
      },
      {
        "start": 481,
        "end": 501,
        "language": "mermaid"
      },
      {
        "start": 617,
        "end": 639,
        "language": "pseudocode"
      },
      {
        "start": 668,
        "end": 686,
        "language": "mermaid"
      },
      {
        "start": 694,
        "end": 702,
        "language": "pseudocode"
      },
      {
        "start": 762,
        "end": 774,
        "language": "mermaid"
      },
      {
        "start": 803,
        "end": 818,
        "language": "pseudocode"
      },
      {
        "start": 834,
        "end": 843,
        "language": "mermaid"
      },
      {
        "start": 920,
        "end": 928,
        "language": "mermaid"
      },
      {
        "start": 1000,
        "end": 1019,
        "language": "mermaid"
      }
    ],
    "links": [
      "../interview-loops/04-financial-and-transactional/02-digital-wallet-system.md",
      "../interview-loops/01-core-infrastructure/03-distributed-message-queue.md",
      "../interview-loops/01-core-infrastructure/01-distributed-key-value-store.md",
      "../interview-loops/01-core-infrastructure/05-url-shortener.md",
      "../interview-loops/03-storage-and-search/01-s3-like-object-storage.md",
      "01-write-ahead-log-fsync-and-group-commit.md",
      "01-write-ahead-log-fsync-and-group-commit.md",
      "01-write-ahead-log-fsync-and-group-commit.md",
      "01-write-ahead-log-fsync-and-group-commit.md",
      "04-idempotency-and-effectively-once-processing.md",
      "01-write-ahead-log-fsync-and-group-commit.md",
      "01-write-ahead-log-fsync-and-group-commit.md",
      "../interview-loops/07-mobile-and-offline/04-mobile-paging-library-architecture.md",
      "../components/21-database-isolation-levels-acid-and-concurrency-anomalies.md",
      "../interview-loops/04-financial-and-transactional/02-digital-wallet-system.md",
      "../interview-loops/04-financial-and-transactional/01-payment-processing-system.md",
      "../interview-loops/05-location-and-geospatial/03-nearby-friends-location-service.md",
      "01-write-ahead-log-fsync-and-group-commit.md",
      "../interview-loops/01-core-infrastructure/01-distributed-key-value-store.md",
      "../interview-loops/01-core-infrastructure/01-distributed-key-value-store.md",
      "04-idempotency-and-effectively-once-processing.md",
      "01-write-ahead-log-fsync-and-group-commit.md",
      "../situations/11-raft-consensus-split-brain.md",
      "03-leases-fencing-tokens-and-distributed-locks.md",
      "03-leases-fencing-tokens-and-distributed-locks.md",
      "03-leases-fencing-tokens-and-distributed-locks.md",
      "../interview-loops/01-core-infrastructure/03-distributed-message-queue.md",
      "../situations/11-raft-consensus-split-brain.md",
      "03-leases-fencing-tokens-and-distributed-locks.md",
      "09-multi-region-failover.md",
      "05-change-streams-and-transactional-outbox.md",
      "01-write-ahead-log-fsync-and-group-commit.md",
      "../interview-loops/01-core-infrastructure/01-distributed-key-value-store.md",
      "05-change-streams-and-transactional-outbox.md",
      "../interview-loops/06-media-and-streaming/01-youtube-video-streaming-platform.md",
      "../interview-loops/06-media-and-streaming/02-google-drive-file-sync-and-storage.md",
      "01-write-ahead-log-fsync-and-group-commit.md",
      "03-leases-fencing-tokens-and-distributed-locks.md",
      "../interview-loops/01-core-infrastructure/01-distributed-key-value-store.md",
      "09-multi-region-failover.md",
      "../situations/05-replication-multi-region-consistency.md",
      "01-write-ahead-log-fsync-and-group-commit.md",
      "01-write-ahead-log-fsync-and-group-commit.md",
      "01-write-ahead-log-fsync-and-group-commit.md",
      "05-change-streams-and-transactional-outbox.md",
      "01-write-ahead-log-fsync-and-group-commit.md",
      "03-leases-fencing-tokens-and-distributed-locks.md",
      "04-idempotency-and-effectively-once-processing.md",
      "05-change-streams-and-transactional-outbox.md",
      "08-sharding-hot-keys-and-rebalancing.md",
      "09-multi-region-failover.md",
      "../components/09-distributed-consensus-raft-and-paxos.md",
      "../components/21-database-isolation-levels-acid-and-concurrency-anomalies.md",
      "../situations/11-raft-consensus-split-brain.md",
      "../interview-loops/01-core-infrastructure/01-distributed-key-value-store.md",
      "../interview-loops/01-core-infrastructure/03-distributed-message-queue.md",
      "../interview-loops/04-financial-and-transactional/02-digital-wallet-system.md",
      "../interview-loops/04-financial-and-transactional/01-payment-processing-system.md",
      "../interview-loops/04-financial-and-transactional/03-stock-exchange-matching-engine.md",
      "../interview-loops/01-core-infrastructure/05-url-shortener.md",
      "../interview-loops/02-social-and-realtime/01-news-feed-system.md",
      "../interview-loops/07-mobile-and-offline/04-mobile-paging-library-architecture.md",
      "../interview-loops/08-real-world-case-studies/01-netflix-global-streaming-and-zuul-api-architecture.md"
    ],
    "lessons": [
      14,
      15
    ]
  },
  {
    "path": "loop-primitives/07-event-time-watermarks-and-checkpoints.md",
    "sha256": "dba266d47b16094d3ecdee1866010d7bc28eca4222c59a04186cb4286c0949ff",
    "lines": 1543,
    "words": 22667,
    "headings": [
      {
        "line": 1,
        "title": "Event Time, Watermarks & Checkpoints"
      },
      {
        "line": 4,
        "title": "Part 0. Start here"
      },
      {
        "line": 6,
        "title": "A replay that counts one click twice, although no write adds anything"
      },
      {
        "line": 12,
        "title": "The numbers"
      },
      {
        "line": 39,
        "title": "The big picture"
      },
      {
        "line": 61,
        "title": "What you'll be able to do after this page"
      },
      {
        "line": 80,
        "title": "Part 1. The core mental model: two clocks"
      },
      {
        "line": 84,
        "title": "Two clocks"
      },
      {
        "line": 110,
        "title": "Why processing time moves clicks"
      },
      {
        "line": 114,
        "title": "Three questions every streaming design answers"
      },
      {
        "line": 124,
        "title": "Other meanings of the word \"watermark\""
      },
      {
        "line": 135,
        "title": "The running example: one minute of clicks from three shards"
      },
      {
        "line": 192,
        "title": "Part 2. Watermarks"
      },
      {
        "line": 196,
        "title": "What a watermark promises"
      },
      {
        "line": 200,
        "title": "One shard"
      },
      {
        "line": 206,
        "title": "Three shards and the minimum"
      },
      {
        "line": 256,
        "title": "Out of order is not late"
      },
      {
        "line": 260,
        "title": "A quiet shard"
      },
      {
        "line": 287,
        "title": "Clamping the device's time"
      },
      {
        "line": 300,
        "title": "A watermark is an estimate, and it can be wrong"
      },
      {
        "line": 304,
        "title": "Watching the watermark"
      },
      {
        "line": 316,
        "title": "Part 3. Windows"
      },
      {
        "line": 320,
        "title": "Tumbling: the minute closes"
      },
      {
        "line": 346,
        "title": "The hour from closed minutes"
      },
      {
        "line": 364,
        "title": "Sliding windows"
      },
      {
        "line": 377,
        "title": "Session windows"
      },
      {
        "line": 409,
        "title": "Processing-time windows"
      },
      {
        "line": 421,
        "title": "Pre-aggregation without losing clicks"
      },
      {
        "line": 460,
        "title": "Part 4. Late data and early answers"
      },
      {
        "line": 464,
        "title": "Two late clicks"
      },
      {
        "line": 481,
        "title": "Allowed lateness or a side output"
      },
      {
        "line": 502,
        "title": "The acceptance horizon and the recount"
      },
      {
        "line": 519,
        "title": "Early firings"
      },
      {
        "line": 558,
        "title": "Part 5. Keyed state and timers"
      },
      {
        "line": 562,
        "title": "State that belongs to a key"
      },
      {
        "line": 577,
        "title": "Timers on two clocks"
      },
      {
        "line": 583,
        "title": "Why TTL can't be the dedup window"
      },
      {
        "line": 604,
        "title": "Where state lives"
      },
      {
        "line": 624,
        "title": "Part 6. Checkpoints"
      },
      {
        "line": 628,
        "title": "A consistent cut"
      },
      {
        "line": 632,
        "title": "Barriers, step by step"
      },
      {
        "line": 708,
        "title": "Buffers outside state: flush at the barrier"
      },
      {
        "line": 723,
        "title": "When the sink is slow"
      },
      {
        "line": 752,
        "title": "Unaligned barriers"
      },
      {
        "line": 767,
        "title": "Incremental uploads, interval, pause and timeout"
      },
      {
        "line": 774,
        "title": "Savepoints"
      },
      {
        "line": 791,
        "title": "Part 7. Crash, restore and replay"
      },
      {
        "line": 795,
        "title": "What comes back, and what doesn't"
      },
      {
        "line": 818,
        "title": "The replay makes a different lateness decision"
      },
      {
        "line": 861,
        "title": "A closed minute re-opens"
      },
      {
        "line": 878,
        "title": "The watermark floor"
      },
      {
        "line": 900,
        "title": "External state isn't rolled back"
      },
      {
        "line": 912,
        "title": "Part 8. Sinks: making repeated and changed output harmless"
      },
      {
        "line": 916,
        "title": "What a replay does to output written after the checkpoint"
      },
      {
        "line": 927,
        "title": "Absolute values per key"
      },
      {
        "line": 931,
        "title": "Each writer tags its own attributes"
      },
      {
        "line": 970,
        "title": "The full flush, and the index that finds it"
      },
      {
        "line": 1014,
        "title": "The tag must be set at the barrier"
      },
      {
        "line": 1028,
        "title": "Closed rows are final"
      },
      {
        "line": 1051,
        "title": "Transactional sinks and their timeouts"
      },
      {
        "line": 1158,
        "title": "Part 9. Rescaling and upgrades"
      },
      {
        "line": 1162,
        "title": "Key groups"
      },
      {
        "line": 1198,
        "title": "Rescaling is a restore"
      },
      {
        "line": 1223,
        "title": "Blue/green and `job_gen`"
      },
      {
        "line": 1248,
        "title": "Part 10. Compared on equal terms: stream, micro-batch, batch recount"
      },
      {
        "line": 1252,
        "title": "Four ways to count minute 10:14"
      },
      {
        "line": 1266,
        "title": "The table"
      },
      {
        "line": 1279,
        "title": "Lambda and kappa architectures, and what each costs"
      },
      {
        "line": 1295,
        "title": "Reprocessing a day"
      },
      {
        "line": 1312,
        "title": "Part 11. End to end through the layers"
      },
      {
        "line": 1316,
        "title": "One click through six layers"
      },
      {
        "line": 1339,
        "title": "The four numbers to watch"
      },
      {
        "line": 1356,
        "title": "Part 12. On AWS"
      },
      {
        "line": 1360,
        "title": "Managed services that use it"
      },
      {
        "line": 1373,
        "title": "Running it yourself"
      },
      {
        "line": 1379,
        "title": "Shared limits"
      },
      {
        "line": 1390,
        "title": "Look-alikes that are not this mechanism"
      },
      {
        "line": 1411,
        "title": "Part 13. What you've learned"
      },
      {
        "line": 1413,
        "title": "Back to the double count"
      },
      {
        "line": 1425,
        "title": "The whole story, event by event"
      },
      {
        "line": 1454,
        "title": "The cheat card"
      },
      {
        "line": 1474,
        "title": "Failure checklist"
      },
      {
        "line": 1488,
        "title": "Think-first drills"
      },
      {
        "line": 1524,
        "title": "Interview questions"
      },
      {
        "line": 1535,
        "title": "Where to go next"
      }
    ],
    "fences": [
      {
        "start": 41,
        "end": 57,
        "language": "mermaid"
      },
      {
        "start": 227,
        "end": 252,
        "language": "mermaid"
      },
      {
        "start": 324,
        "end": 333,
        "language": "pseudocode"
      },
      {
        "start": 389,
        "end": 405,
        "language": "mermaid"
      },
      {
        "start": 425,
        "end": 438,
        "language": "pseudocode"
      },
      {
        "start": 506,
        "end": 515,
        "language": "mermaid"
      },
      {
        "start": 634,
        "end": 650,
        "language": "pseudocode"
      },
      {
        "start": 654,
        "end": 667,
        "language": "mermaid"
      },
      {
        "start": 684,
        "end": 704,
        "language": "mermaid"
      },
      {
        "start": 820,
        "end": 836,
        "language": "mermaid"
      },
      {
        "start": 880,
        "end": 894,
        "language": "pseudocode"
      },
      {
        "start": 942,
        "end": 966,
        "language": "pseudocode"
      },
      {
        "start": 985,
        "end": 1006,
        "language": "mermaid"
      },
      {
        "start": 1032,
        "end": 1040,
        "language": "pseudocode"
      },
      {
        "start": 1055,
        "end": 1068,
        "language": "pseudocode"
      },
      {
        "start": 1074,
        "end": 1088,
        "language": "mermaid"
      },
      {
        "start": 1127,
        "end": 1146,
        "language": "mermaid"
      },
      {
        "start": 1176,
        "end": 1192,
        "language": "mermaid"
      },
      {
        "start": 1318,
        "end": 1326,
        "language": "mermaid"
      }
    ],
    "links": [
      "../interview-loops/06-media-and-streaming/03-ad-click-event-aggregation.md",
      "../interview-loops/06-media-and-streaming/03-ad-click-event-aggregation.md",
      "../interview-loops/03-storage-and-search/04-metrics-monitoring-and-alerting-system.md",
      "../interview-loops/06-media-and-streaming/01-youtube-video-streaming-platform.md",
      "../components/05-message-queues-vs-event-streams.md",
      "04-idempotency-and-effectively-once-processing.md",
      "../interview-loops/03-storage-and-search/01-s3-like-object-storage.md",
      "04-idempotency-and-effectively-once-processing.md",
      "02-lsm-trees-and-compaction.md",
      "01-write-ahead-log-fsync-and-group-commit.md",
      "04-idempotency-and-effectively-once-processing.md",
      "04-idempotency-and-effectively-once-processing.md",
      "../interview-loops/06-media-and-streaming/01-youtube-video-streaming-platform.md",
      "../interview-loops/01-core-infrastructure/05-url-shortener.md",
      "03-leases-fencing-tokens-and-distributed-locks.md",
      "04-idempotency-and-effectively-once-processing.md",
      "05-change-streams-and-transactional-outbox.md",
      "03-leases-fencing-tokens-and-distributed-locks.md",
      "02-lsm-trees-and-compaction.md",
      "06-replication-quorums-and-read-your-writes.md",
      "../components/05-message-queues-vs-event-streams.md",
      "../components/10-two-phase-commit-and-saga-orchestration.md",
      "../components/23-event-sourcing-and-cqrs-architecture.md",
      "../interview-loops/06-media-and-streaming/03-ad-click-event-aggregation.md",
      "../interview-loops/03-storage-and-search/04-metrics-monitoring-and-alerting-system.md",
      "../interview-loops/06-media-and-streaming/01-youtube-video-streaming-platform.md",
      "../interview-loops/01-core-infrastructure/05-url-shortener.md",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md",
      "../interview-loops/03-storage-and-search/02-search-autocomplete-system.md",
      "../interview-loops/05-location-and-geospatial/02-google-maps-and-routing-engine.md"
    ],
    "lessons": [
      21
    ]
  },
  {
    "path": "loop-primitives/08-sharding-hot-keys-and-rebalancing.md",
    "sha256": "8df2444e014537641df2df386099a09ab4685ea6b3c05b7e954461f6849f9d28",
    "lines": 1451,
    "words": 24176,
    "headings": [
      {
        "line": 1,
        "title": "Sharding, Hot Keys & Rebalancing"
      },
      {
        "line": 4,
        "title": "Part 0. Start here"
      },
      {
        "line": 6,
        "title": "The problem: 66,667 reads and 10,000 likes a second on one post"
      },
      {
        "line": 47,
        "title": "The big picture"
      },
      {
        "line": 62,
        "title": "What you'll be able to do after this page"
      },
      {
        "line": 80,
        "title": "Part 1. What a shard is"
      },
      {
        "line": 84,
        "title": "Key, slot, shard"
      },
      {
        "line": 108,
        "title": "The router, the directory and the slot table"
      },
      {
        "line": 142,
        "title": "Three words for one idea"
      },
      {
        "line": 150,
        "title": "The example we follow"
      },
      {
        "line": 198,
        "title": "Part 2. Choosing a partition key"
      },
      {
        "line": 202,
        "title": "Trace: the same traffic under three layouts"
      },
      {
        "line": 212,
        "title": "Range, hash, directory"
      },
      {
        "line": 237,
        "title": "User, tenant and time keys"
      },
      {
        "line": 255,
        "title": "The key follows the most frequent query and the transaction boundary"
      },
      {
        "line": 265,
        "title": "Key skew vs load skew"
      },
      {
        "line": 272,
        "title": "How many shards, how many buckets"
      },
      {
        "line": 289,
        "title": "Part 3. A hot key appears"
      },
      {
        "line": 293,
        "title": "Trace: `p9` goes viral"
      },
      {
        "line": 337,
        "title": "Why more shards don't help"
      },
      {
        "line": 345,
        "title": "Finding the key"
      },
      {
        "line": 394,
        "title": "Reads or writes?"
      },
      {
        "line": 400,
        "title": "A limit per key"
      },
      {
        "line": 421,
        "title": "Part 4. Hot reads: cache, coalesce, copy"
      },
      {
        "line": 425,
        "title": "Cache it for a second"
      },
      {
        "line": 474,
        "title": "Why not cache it forever?"
      },
      {
        "line": 478,
        "title": "Coalesce identical reads"
      },
      {
        "line": 532,
        "title": "Replica reads"
      },
      {
        "line": 536,
        "title": "Copies of one value"
      },
      {
        "line": 552,
        "title": "Part 5. Hot writes, and every fix on equal terms"
      },
      {
        "line": 556,
        "title": "Salting"
      },
      {
        "line": 597,
        "title": "When salts collide, and the tag trap"
      },
      {
        "line": 623,
        "title": "Pre-aggregation"
      },
      {
        "line": 655,
        "title": "Nothing else may write the hot key"
      },
      {
        "line": 659,
        "title": "When a router crashes"
      },
      {
        "line": 674,
        "title": "Leasing and striping"
      },
      {
        "line": 681,
        "title": "A dedicated shard"
      },
      {
        "line": 685,
        "title": "GSI write sharding"
      },
      {
        "line": 689,
        "title": "Every hot-key fix on equal terms"
      },
      {
        "line": 721,
        "title": "Part 6. Adding capacity: rings, slots and splits"
      },
      {
        "line": 725,
        "title": "Trace: three ways to add S5"
      },
      {
        "line": 735,
        "title": "`hash mod N`"
      },
      {
        "line": 747,
        "title": "The ring, with one point and with many"
      },
      {
        "line": 790,
        "title": "Fixed slots and a directory"
      },
      {
        "line": 805,
        "title": "Split and merge"
      },
      {
        "line": 809,
        "title": "Bounded load, briefly"
      },
      {
        "line": 813,
        "title": "On equal terms"
      },
      {
        "line": 833,
        "title": "Part 7. DynamoDB and Kinesis: managed splitting never splits one key"
      },
      {
        "line": 837,
        "title": "DynamoDB: adaptive capacity and split for consumption"
      },
      {
        "line": 858,
        "title": "Kinesis: shards, splits and lineage"
      },
      {
        "line": 906,
        "title": "Part 8. Moving live data"
      },
      {
        "line": 912,
        "title": "Copy and tail"
      },
      {
        "line": 950,
        "title": "Fence and flip"
      },
      {
        "line": 1020,
        "title": "Without the slot table"
      },
      {
        "line": 1032,
        "title": "When the move fails"
      },
      {
        "line": 1041,
        "title": "Dual writes instead of a log tail"
      },
      {
        "line": 1050,
        "title": "Moving key by key"
      },
      {
        "line": 1070,
        "title": "What the move costs everyone else"
      },
      {
        "line": 1076,
        "title": "Engines that do it for you"
      },
      {
        "line": 1092,
        "title": "Part 9. Across shards: queries and transactions"
      },
      {
        "line": 1096,
        "title": "Trace: \"trending now\""
      },
      {
        "line": 1124,
        "title": "The slowest shard sets the query's latency"
      },
      {
        "line": 1139,
        "title": "A popular query is a hot read"
      },
      {
        "line": 1145,
        "title": "Local and global indexes"
      },
      {
        "line": 1152,
        "title": "Reports run on a copy"
      },
      {
        "line": 1168,
        "title": "Transactions that span shards"
      },
      {
        "line": 1192,
        "title": "Part 10. End to end through the layers"
      },
      {
        "line": 1219,
        "title": "Sizing per AZ"
      },
      {
        "line": 1239,
        "title": "Part 11. On AWS"
      },
      {
        "line": 1243,
        "title": "Managed services that use it"
      },
      {
        "line": 1259,
        "title": "Running it yourself"
      },
      {
        "line": 1266,
        "title": "Look-alikes that are not this mechanism"
      },
      {
        "line": 1285,
        "title": "Part 12. What you've learned"
      },
      {
        "line": 1287,
        "title": "Back to the celebrity post"
      },
      {
        "line": 1299,
        "title": "The whole story, event by event"
      },
      {
        "line": 1356,
        "title": "The cheat card"
      },
      {
        "line": 1393,
        "title": "Think-first drills"
      },
      {
        "line": 1429,
        "title": "Interview questions"
      },
      {
        "line": 1440,
        "title": "Where to go next"
      }
    ],
    "fences": [
      {
        "start": 49,
        "end": 58,
        "language": "mermaid"
      },
      {
        "start": 95,
        "end": 104,
        "language": "mermaid"
      },
      {
        "start": 115,
        "end": 136,
        "language": "pseudocode"
      },
      {
        "start": 222,
        "end": 233,
        "language": "mermaid"
      },
      {
        "start": 295,
        "end": 302,
        "language": "mermaid"
      },
      {
        "start": 306,
        "end": 313,
        "language": "mermaid"
      },
      {
        "start": 354,
        "end": 370,
        "language": "pseudocode"
      },
      {
        "start": 374,
        "end": 390,
        "language": "mermaid"
      },
      {
        "start": 432,
        "end": 448,
        "language": "pseudocode"
      },
      {
        "start": 452,
        "end": 463,
        "language": "mermaid"
      },
      {
        "start": 494,
        "end": 512,
        "language": "mermaid"
      },
      {
        "start": 568,
        "end": 591,
        "language": "mermaid"
      },
      {
        "start": 633,
        "end": 651,
        "language": "pseudocode"
      },
      {
        "start": 751,
        "end": 758,
        "language": "mermaid"
      },
      {
        "start": 767,
        "end": 772,
        "language": "mermaid"
      },
      {
        "start": 881,
        "end": 888,
        "language": "mermaid"
      },
      {
        "start": 923,
        "end": 946,
        "language": "mermaid"
      },
      {
        "start": 952,
        "end": 976,
        "language": "pseudocode"
      },
      {
        "start": 1000,
        "end": 1016,
        "language": "mermaid"
      },
      {
        "start": 1100,
        "end": 1120,
        "language": "mermaid"
      },
      {
        "start": 1196,
        "end": 1204,
        "language": "mermaid"
      }
    ],
    "links": [
      "../interview-loops/02-social-and-realtime/01-news-feed-system.md",
      "../interview-loops/02-social-and-realtime/01-news-feed-system.md",
      "../interview-loops/06-media-and-streaming/03-ad-click-event-aggregation.md",
      "../interview-loops/04-financial-and-transactional/02-digital-wallet-system.md",
      "../interview-loops/08-real-world-case-studies/02-discord-trillions-of-messages-scylladb-storage.md",
      "06-replication-quorums-and-read-your-writes.md",
      "05-change-streams-and-transactional-outbox.md",
      "../interview-loops/04-financial-and-transactional/04-hotel-reservation-system.md",
      "../interview-loops/08-real-world-case-studies/03-uber-realtime-dispatch-and-schemaless-mysql.md",
      "../situations/02-sharding-tenant-hotspot.md",
      "../interview-loops/08-real-world-case-studies/02-discord-trillions-of-messages-scylladb-storage.md",
      "../interview-loops/02-social-and-realtime/04-notification-system.md",
      "../interview-loops/03-storage-and-search/05-distributed-job-scheduler-and-task-queue.md",
      "../interview-loops/04-financial-and-transactional/04-hotel-reservation-system.md",
      "../interview-loops/04-financial-and-transactional/01-payment-processing-system.md",
      "../interview-loops/04-financial-and-transactional/03-stock-exchange-matching-engine.md",
      "../interview-loops/04-financial-and-transactional/02-digital-wallet-system.md",
      "../interview-loops/01-core-infrastructure/01-distributed-key-value-store.md",
      "../interview-loops/01-core-infrastructure/03-distributed-message-queue.md",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md",
      "09-multi-region-failover.md",
      "../situations/01-caching-hot-product-page.md",
      "06-replication-quorums-and-read-your-writes.md",
      "../interview-loops/03-storage-and-search/02-search-autocomplete-system.md",
      "../interview-loops/02-social-and-realtime/01-news-feed-system.md",
      "../interview-loops/07-mobile-and-offline/01-mobile-news-feed-offline-first.md",
      "04-idempotency-and-effectively-once-processing.md",
      "07-event-time-watermarks-and-checkpoints.md",
      "../interview-loops/06-media-and-streaming/03-ad-click-event-aggregation.md",
      "../interview-loops/06-media-and-streaming/01-youtube-video-streaming-platform.md",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md",
      "../interview-loops/04-financial-and-transactional/02-digital-wallet-system.md",
      "../interview-loops/04-financial-and-transactional/01-payment-processing-system.md",
      "../situations/02-sharding-tenant-hotspot.md",
      "../interview-loops/01-core-infrastructure/05-url-shortener.md",
      "../interview-loops/03-storage-and-search/01-s3-like-object-storage.md",
      "../situations/07-consistent-hashing-vnode-skew.md",
      "../interview-loops/03-storage-and-search/03-web-crawler-at-scale.md",
      "05-change-streams-and-transactional-outbox.md",
      "05-change-streams-and-transactional-outbox.md",
      "06-replication-quorums-and-read-your-writes.md",
      "05-change-streams-and-transactional-outbox.md",
      "03-leases-fencing-tokens-and-distributed-locks.md",
      "04-idempotency-and-effectively-once-processing.md",
      "../interview-loops/04-financial-and-transactional/04-hotel-reservation-system.md",
      "../interview-loops/04-financial-and-transactional/02-digital-wallet-system.md",
      "03-leases-fencing-tokens-and-distributed-locks.md",
      "../interview-loops/08-real-world-case-studies/02-discord-trillions-of-messages-scylladb-storage.md",
      "../interview-loops/08-real-world-case-studies/03-uber-realtime-dispatch-and-schemaless-mysql.md",
      "../interview-loops/04-financial-and-transactional/02-digital-wallet-system.md",
      "05-change-streams-and-transactional-outbox.md",
      "04-idempotency-and-effectively-once-processing.md",
      "01-write-ahead-log-fsync-and-group-commit.md",
      "06-replication-quorums-and-read-your-writes.md",
      "05-change-streams-and-transactional-outbox.md",
      "06-replication-quorums-and-read-your-writes.md",
      "06-replication-quorums-and-read-your-writes.md",
      "09-multi-region-failover.md",
      "06-replication-quorums-and-read-your-writes.md",
      "03-leases-fencing-tokens-and-distributed-locks.md",
      "04-idempotency-and-effectively-once-processing.md",
      "05-change-streams-and-transactional-outbox.md",
      "07-event-time-watermarks-and-checkpoints.md",
      "01-write-ahead-log-fsync-and-group-commit.md",
      "09-multi-region-failover.md",
      "../components/08-database-sharding-and-partition-keys.md",
      "../components/01-consistent-hashing.md",
      "../components/04-distributed-cache-patterns-and-eviction.md",
      "../components/10-two-phase-commit-and-saga-orchestration.md",
      "../situations/01-caching-hot-product-page.md",
      "../situations/02-sharding-tenant-hotspot.md",
      "../situations/07-consistent-hashing-vnode-skew.md",
      "../interview-loops/01-core-infrastructure/01-distributed-key-value-store.md",
      "../interview-loops/01-core-infrastructure/03-distributed-message-queue.md",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md",
      "../interview-loops/01-core-infrastructure/05-url-shortener.md",
      "../interview-loops/02-social-and-realtime/01-news-feed-system.md",
      "../interview-loops/02-social-and-realtime/02-chat-and-instant-messaging-system.md",
      "../interview-loops/02-social-and-realtime/03-realtime-gaming-leaderboard.md",
      "../interview-loops/02-social-and-realtime/04-notification-system.md",
      "../interview-loops/03-storage-and-search/01-s3-like-object-storage.md",
      "../interview-loops/03-storage-and-search/02-search-autocomplete-system.md",
      "../interview-loops/03-storage-and-search/03-web-crawler-at-scale.md",
      "../interview-loops/03-storage-and-search/04-metrics-monitoring-and-alerting-system.md",
      "../interview-loops/03-storage-and-search/05-distributed-job-scheduler-and-task-queue.md",
      "../interview-loops/04-financial-and-transactional/01-payment-processing-system.md",
      "../interview-loops/04-financial-and-transactional/02-digital-wallet-system.md",
      "../interview-loops/04-financial-and-transactional/03-stock-exchange-matching-engine.md",
      "../interview-loops/04-financial-and-transactional/04-hotel-reservation-system.md",
      "../interview-loops/04-financial-and-transactional/05-transactional-outbox-and-event-driven-ledger.md",
      "../interview-loops/05-location-and-geospatial/01-proximity-service.md",
      "../interview-loops/05-location-and-geospatial/02-google-maps-and-routing-engine.md",
      "../interview-loops/05-location-and-geospatial/03-nearby-friends-location-service.md",
      "../interview-loops/05-location-and-geospatial/04-ride-sharing-dispatch-service.md",
      "../interview-loops/06-media-and-streaming/01-youtube-video-streaming-platform.md",
      "../interview-loops/06-media-and-streaming/02-google-drive-file-sync-and-storage.md",
      "../interview-loops/06-media-and-streaming/03-ad-click-event-aggregation.md",
      "../interview-loops/06-media-and-streaming/04-distributed-email-service.md",
      "../interview-loops/07-mobile-and-offline/01-mobile-news-feed-offline-first.md",
      "../interview-loops/07-mobile-and-offline/03-mobile-stock-trading-app.md",
      "../interview-loops/08-real-world-case-studies/02-discord-trillions-of-messages-scylladb-storage.md",
      "../interview-loops/08-real-world-case-studies/03-uber-realtime-dispatch-and-schemaless-mysql.md"
    ],
    "lessons": [
      12,
      13,
      17
    ]
  },
  {
    "path": "loop-primitives/09-multi-region-failover.md",
    "sha256": "bdb8a06e5e99771b8fb62f0e05cbc6962545f7c8155e076f472be46398dfa319",
    "lines": 1490,
    "words": 26984,
    "headings": [
      {
        "line": 1,
        "title": "Multi-Region Failover"
      },
      {
        "line": 4,
        "title": "Part 0. Start here"
      },
      {
        "line": 6,
        "title": "The problem: Frankfurt goes dark at 20:00"
      },
      {
        "line": 50,
        "title": "The big picture"
      },
      {
        "line": 78,
        "title": "What you'll be able to do after this page"
      },
      {
        "line": 97,
        "title": "Part 1. Five decisions, one home per key, and two numbers"
      },
      {
        "line": 101,
        "title": "Five decisions, in order"
      },
      {
        "line": 127,
        "title": "One home per key"
      },
      {
        "line": 135,
        "title": "RPO and RTO"
      },
      {
        "line": 150,
        "title": "Words we use"
      },
      {
        "line": 168,
        "title": "The example we follow"
      },
      {
        "line": 196,
        "title": "Snapshot R1: normal Friday, 18:00"
      },
      {
        "line": 215,
        "title": "Part 2. Is it the Region? Detecting from outside"
      },
      {
        "line": 219,
        "title": "Trace: the link breaks at 18:30"
      },
      {
        "line": 229,
        "title": "Snapshot R2: the split"
      },
      {
        "line": 266,
        "title": "Vantage points and a business metric"
      },
      {
        "line": 291,
        "title": "What a split does to the data at risk"
      },
      {
        "line": 297,
        "title": "Route 53 health checks, precisely"
      },
      {
        "line": 313,
        "title": "Trace: detection at 20:00"
      },
      {
        "line": 330,
        "title": "Part 3. Do we move? The decision gate"
      },
      {
        "line": 334,
        "title": "Trace: 20:01, the checklist"
      },
      {
        "line": 338,
        "title": "Snapshot R3: the gate at 20:01"
      },
      {
        "line": 399,
        "title": "Safety rules in the switch itself"
      },
      {
        "line": 412,
        "title": "Why a move to a read-only Region is gated"
      },
      {
        "line": 429,
        "title": "Part 4. Fencing the old writer, promoting the new one"
      },
      {
        "line": 433,
        "title": "Recap: the single-writer handover, at Region scale"
      },
      {
        "line": 437,
        "title": "The lease lives in a table stored in three Regions"
      },
      {
        "line": 447,
        "title": "Which clock each value uses"
      },
      {
        "line": 491,
        "title": "The epoch lives in the database"
      },
      {
        "line": 517,
        "title": "Trace: the fence and the promotion"
      },
      {
        "line": 566,
        "title": "Snapshot R4: after the flip"
      },
      {
        "line": 589,
        "title": "Aurora: switchover or failover"
      },
      {
        "line": 619,
        "title": "Part 5. Moving the users"
      },
      {
        "line": 623,
        "title": "Trace: the flip at 20:05"
      },
      {
        "line": 633,
        "title": "Three ways to move traffic"
      },
      {
        "line": 649,
        "title": "The DNS tail and client caches"
      },
      {
        "line": 677,
        "title": "The timing chain"
      },
      {
        "line": 708,
        "title": "The night `us-east-1` goes dark (side replay U)"
      },
      {
        "line": 724,
        "title": "Read-only mode (side row 19r)"
      },
      {
        "line": 736,
        "title": "Part 6. Where the load can go: capacity, quota, residency, and the ladder"
      },
      {
        "line": 740,
        "title": "Trace: 20:01, capacity starts"
      },
      {
        "line": 748,
        "title": "Capacity options on equal terms"
      },
      {
        "line": 774,
        "title": "Sizing the target (event K1)"
      },
      {
        "line": 787,
        "title": "Residency and prerequisites"
      },
      {
        "line": 803,
        "title": "The ladder"
      },
      {
        "line": 817,
        "title": "Side replay L1: the same EU service on five rungs"
      },
      {
        "line": 843,
        "title": "Active-passive vs active-active"
      },
      {
        "line": 857,
        "title": "Shared limits"
      },
      {
        "line": 880,
        "title": "Part 7. Writes caught in the middle"
      },
      {
        "line": 884,
        "title": "Trace: 160 orders and 60 unknowns"
      },
      {
        "line": 894,
        "title": "Seat 14C"
      },
      {
        "line": 932,
        "title": "The Frankfurt and Virginia booking drill, both questions"
      },
      {
        "line": 939,
        "title": "Stores at failover"
      },
      {
        "line": 951,
        "title": "What a secondary accepts, it must forward"
      },
      {
        "line": 970,
        "title": "Part 8. Failback: bringing the old Region back"
      },
      {
        "line": 974,
        "title": "Trace: Frankfurt returns at 20:40"
      },
      {
        "line": 983,
        "title": "Snapshot R5: Frankfurt back, refusing"
      },
      {
        "line": 994,
        "title": "Late arrivals and old queues"
      },
      {
        "line": 1025,
        "title": "Reconciling the tail"
      },
      {
        "line": 1043,
        "title": "Failing back on purpose"
      },
      {
        "line": 1066,
        "title": "Snapshot R6: failback done, Saturday 06:00"
      },
      {
        "line": 1084,
        "title": "Part 9. Shrinking the blast radius: cells and shuffle sharding"
      },
      {
        "line": 1088,
        "title": "Trace: the 19:00 deploy"
      },
      {
        "line": 1108,
        "title": "Cells"
      },
      {
        "line": 1128,
        "title": "Shuffle sharding"
      },
      {
        "line": 1151,
        "title": "Part 10. Rehearsing failover: game days and the error budget"
      },
      {
        "line": 1155,
        "title": "The budget"
      },
      {
        "line": 1161,
        "title": "This month's spending (event 30)"
      },
      {
        "line": 1186,
        "title": "Rehearsing the split, the move and the return (event G1)"
      },
      {
        "line": 1226,
        "title": "Part 11. End to end through the layers"
      },
      {
        "line": 1230,
        "title": "One checkout after the flip (event 31, 20:07:30)"
      },
      {
        "line": 1270,
        "title": "Part 12. On AWS"
      },
      {
        "line": 1274,
        "title": "Managed services that use it"
      },
      {
        "line": 1286,
        "title": "Running it yourself"
      },
      {
        "line": 1295,
        "title": "Look-alikes that are not this mechanism"
      },
      {
        "line": 1315,
        "title": "Part 13. What you've learned"
      },
      {
        "line": 1317,
        "title": "Back to Frankfurt at 20:00"
      },
      {
        "line": 1328,
        "title": "What it costs"
      },
      {
        "line": 1335,
        "title": "The whole story, event by event"
      },
      {
        "line": 1396,
        "title": "The cheat card"
      },
      {
        "line": 1433,
        "title": "Think-first drills"
      },
      {
        "line": 1469,
        "title": "Interview questions"
      },
      {
        "line": 1480,
        "title": "Where to go next"
      }
    ],
    "fences": [
      {
        "start": 52,
        "end": 74,
        "language": "mermaid"
      },
      {
        "start": 103,
        "end": 115,
        "language": "mermaid"
      },
      {
        "start": 231,
        "end": 248,
        "language": "mermaid"
      },
      {
        "start": 275,
        "end": 287,
        "language": "pseudocode"
      },
      {
        "start": 372,
        "end": 387,
        "language": "pseudocode"
      },
      {
        "start": 458,
        "end": 475,
        "language": "pseudocode"
      },
      {
        "start": 495,
        "end": 513,
        "language": "sql"
      },
      {
        "start": 519,
        "end": 537,
        "language": "mermaid"
      },
      {
        "start": 568,
        "end": 585,
        "language": "mermaid"
      },
      {
        "start": 688,
        "end": 704,
        "language": "mermaid"
      },
      {
        "start": 807,
        "end": 813,
        "language": "mermaid"
      },
      {
        "start": 896,
        "end": 912,
        "language": "mermaid"
      },
      {
        "start": 1000,
        "end": 1012,
        "language": "pseudocode"
      },
      {
        "start": 1055,
        "end": 1062,
        "language": "mermaid"
      },
      {
        "start": 1095,
        "end": 1104,
        "language": "mermaid"
      },
      {
        "start": 1173,
        "end": 1180,
        "language": "mermaid"
      },
      {
        "start": 1232,
        "end": 1245,
        "language": "mermaid"
      }
    ],
    "links": [
      "../interview-loops/08-real-world-case-studies/01-netflix-global-streaming-and-zuul-api-architecture.md",
      "../interview-loops/08-real-world-case-studies/03-uber-realtime-dispatch-and-schemaless-mysql.md",
      "../interview-loops/06-media-and-streaming/02-google-drive-file-sync-and-storage.md",
      "../interview-loops/04-financial-and-transactional/01-payment-processing-system.md",
      "08-sharding-hot-keys-and-rebalancing.md",
      "06-replication-quorums-and-read-your-writes.md",
      "03-leases-fencing-tokens-and-distributed-locks.md",
      "03-leases-fencing-tokens-and-distributed-locks.md",
      "03-leases-fencing-tokens-and-distributed-locks.md",
      "05-change-streams-and-transactional-outbox.md",
      "08-sharding-hot-keys-and-rebalancing.md",
      "04-idempotency-and-effectively-once-processing.md",
      "../situations/05-replication-multi-region-consistency.md",
      "06-replication-quorums-and-read-your-writes.md",
      "03-leases-fencing-tokens-and-distributed-locks.md",
      "../interview-loops/06-media-and-streaming/04-distributed-email-service.md",
      "05-change-streams-and-transactional-outbox.md",
      "../interview-loops/03-storage-and-search/04-metrics-monitoring-and-alerting-system.md",
      "../interview-loops/08-real-world-case-studies/01-netflix-global-streaming-and-zuul-api-architecture.md",
      "07-event-time-watermarks-and-checkpoints.md",
      "01-write-ahead-log-fsync-and-group-commit.md",
      "06-replication-quorums-and-read-your-writes.md",
      "../situations/05-replication-multi-region-consistency.md",
      "03-leases-fencing-tokens-and-distributed-locks.md",
      "04-idempotency-and-effectively-once-processing.md",
      "05-change-streams-and-transactional-outbox.md",
      "06-replication-quorums-and-read-your-writes.md",
      "07-event-time-watermarks-and-checkpoints.md",
      "01-write-ahead-log-fsync-and-group-commit.md",
      "08-sharding-hot-keys-and-rebalancing.md",
      "../situations/05-replication-multi-region-consistency.md",
      "../interview-loops/01-core-infrastructure/01-distributed-key-value-store.md",
      "../interview-loops/01-core-infrastructure/03-distributed-message-queue.md",
      "../interview-loops/02-social-and-realtime/02-chat-and-instant-messaging-system.md",
      "../interview-loops/04-financial-and-transactional/01-payment-processing-system.md",
      "../interview-loops/04-financial-and-transactional/02-digital-wallet-system.md",
      "../interview-loops/04-financial-and-transactional/04-hotel-reservation-system.md",
      "../interview-loops/04-financial-and-transactional/05-transactional-outbox-and-event-driven-ledger.md",
      "../interview-loops/05-location-and-geospatial/04-ride-sharing-dispatch-service.md",
      "../interview-loops/06-media-and-streaming/02-google-drive-file-sync-and-storage.md",
      "../interview-loops/06-media-and-streaming/04-distributed-email-service.md",
      "../interview-loops/07-mobile-and-offline/03-mobile-stock-trading-app.md",
      "../interview-loops/08-real-world-case-studies/01-netflix-global-streaming-and-zuul-api-architecture.md",
      "../interview-loops/08-real-world-case-studies/03-uber-realtime-dispatch-and-schemaless-mysql.md"
    ],
    "lessons": [
      25
    ]
  },
  {
    "path": "loop-primitives/10-retries-timeouts-backpressure-and-load-shedding.md",
    "sha256": "2acd6834df86d94f1694793ae46273bf589f64eb6c73e3903fa2a8595eb7827b",
    "lines": 1488,
    "words": 26463,
    "headings": [
      {
        "line": 1,
        "title": "Retries, Timeouts, Backpressure & Load Shedding"
      },
      {
        "line": 4,
        "title": "Part 0. Start here"
      },
      {
        "line": 6,
        "title": "The problem: an optional widget took the home page down"
      },
      {
        "line": 40,
        "title": "The big picture"
      },
      {
        "line": 57,
        "title": "What you'll be able to do after this page"
      },
      {
        "line": 76,
        "title": "Part 1. Why slow is worse than down"
      },
      {
        "line": 80,
        "title": "Little's law"
      },
      {
        "line": 91,
        "title": "Goodput is not throughput"
      },
      {
        "line": 95,
        "title": "The example we follow"
      },
      {
        "line": 128,
        "title": "The same 30 seconds, as shipped (C0)"
      },
      {
        "line": 162,
        "title": "Why it can't drain"
      },
      {
        "line": 199,
        "title": "Part 2. Timeouts and deadlines"
      },
      {
        "line": 211,
        "title": "A timeout from percentiles"
      },
      {
        "line": 240,
        "title": "Timeout vs deadline"
      },
      {
        "line": 247,
        "title": "Each attempt's own deadline, cancelled when abandoned"
      },
      {
        "line": 300,
        "title": "The same 30 seconds, with bounded waits (C1)"
      },
      {
        "line": 318,
        "title": "Defaults that are too long"
      },
      {
        "line": 328,
        "title": "Why not 50 ms, and why not longer"
      },
      {
        "line": 357,
        "title": "Part 3. Retries: what, where and how many"
      },
      {
        "line": 371,
        "title": "Which errors to retry"
      },
      {
        "line": 382,
        "title": "Side effects: the same key on every retry"
      },
      {
        "line": 388,
        "title": "When the server says when"
      },
      {
        "line": 396,
        "title": "Where to retry: one layer"
      },
      {
        "line": 440,
        "title": "Hidden retry layers"
      },
      {
        "line": 474,
        "title": "Part 4. Backoff and jitter"
      },
      {
        "line": 489,
        "title": "Backoff, then jitter"
      },
      {
        "line": 504,
        "title": "What jitter does and doesn't do"
      },
      {
        "line": 512,
        "title": "Periodic work: a stable offset, not a fresh random wait"
      },
      {
        "line": 563,
        "title": "Part 5. Retry budgets and hedged requests"
      },
      {
        "line": 567,
        "title": "A cap per request vs a budget per client"
      },
      {
        "line": 603,
        "title": "Real budgets"
      },
      {
        "line": 612,
        "title": "Budgets live in each client"
      },
      {
        "line": 618,
        "title": "Hedging the tail"
      },
      {
        "line": 668,
        "title": "Part 6. Backpressure: `stock` protects itself"
      },
      {
        "line": 672,
        "title": "What backpressure is"
      },
      {
        "line": 676,
        "title": "Refuse early and cheaply"
      },
      {
        "line": 680,
        "title": "Find the real queue"
      },
      {
        "line": 725,
        "title": "The same 30 seconds, with `stock` protecting itself (C5)"
      },
      {
        "line": 744,
        "title": "Will it ever drain?"
      },
      {
        "line": 785,
        "title": "Part 7. Circuit breakers"
      },
      {
        "line": 789,
        "title": "The three states"
      },
      {
        "line": 836,
        "title": "Count window or time window"
      },
      {
        "line": 840,
        "title": "The same 30 seconds, with breakers (C6)"
      },
      {
        "line": 848,
        "title": "Probes and their timeout"
      },
      {
        "line": 857,
        "title": "Key it by what fails separately"
      },
      {
        "line": 861,
        "title": "Per host, per dependency, or a cap?"
      },
      {
        "line": 872,
        "title": "Breaker or a retry budget only, on equal terms"
      },
      {
        "line": 908,
        "title": "Part 8. Bulkheads and concurrency limits"
      },
      {
        "line": 932,
        "title": "Size a cap with Little's law"
      },
      {
        "line": 936,
        "title": "A backstop, and when it saves you"
      },
      {
        "line": 947,
        "title": "Pools and semaphores"
      },
      {
        "line": 951,
        "title": "Caps multiply with the fleet"
      },
      {
        "line": 955,
        "title": "Limits that adapt"
      },
      {
        "line": 996,
        "title": "Part 9. Brownouts and fallbacks"
      },
      {
        "line": 1033,
        "title": "Three kinds of fallback"
      },
      {
        "line": 1045,
        "title": "Why checkouts stay below about 50 a second"
      },
      {
        "line": 1049,
        "title": "Shedding and autoscaling"
      },
      {
        "line": 1073,
        "title": "Part 10. Why it stayed down: metastable failures"
      },
      {
        "line": 1077,
        "title": "Stable, vulnerable, metastable"
      },
      {
        "line": 1099,
        "title": "The loops that kept it down"
      },
      {
        "line": 1121,
        "title": "Getting out"
      },
      {
        "line": 1132,
        "title": "Why \"add capacity\" didn't help"
      },
      {
        "line": 1136,
        "title": "Every guard on equal terms"
      },
      {
        "line": 1189,
        "title": "Part 11. End to end through the layers"
      },
      {
        "line": 1233,
        "title": "Sizing for an Availability Zone loss"
      },
      {
        "line": 1245,
        "title": "Part 12. On AWS"
      },
      {
        "line": 1249,
        "title": "Managed services that use it"
      },
      {
        "line": 1275,
        "title": "Running it yourself"
      },
      {
        "line": 1283,
        "title": "Look-alikes that are not this mechanism"
      },
      {
        "line": 1305,
        "title": "Part 13. What you've learned"
      },
      {
        "line": 1307,
        "title": "Back to the home page"
      },
      {
        "line": 1320,
        "title": "The snapshots, side by side"
      },
      {
        "line": 1331,
        "title": "What it costs"
      },
      {
        "line": 1340,
        "title": "The whole story, event by event"
      },
      {
        "line": 1392,
        "title": "The cheat card"
      },
      {
        "line": 1428,
        "title": "Think-first drills"
      },
      {
        "line": 1464,
        "title": "Interview questions"
      },
      {
        "line": 1475,
        "title": "Where to go next"
      }
    ],
    "fences": [
      {
        "start": 42,
        "end": 53,
        "language": "mermaid"
      },
      {
        "start": 180,
        "end": 187,
        "language": "mermaid"
      },
      {
        "start": 215,
        "end": 224,
        "language": "pseudocode"
      },
      {
        "start": 251,
        "end": 267,
        "language": "mermaid"
      },
      {
        "start": 271,
        "end": 294,
        "language": "pseudocode"
      },
      {
        "start": 361,
        "end": 367,
        "language": "mermaid"
      },
      {
        "start": 400,
        "end": 417,
        "language": "pseudocode"
      },
      {
        "start": 421,
        "end": 431,
        "language": "mermaid"
      },
      {
        "start": 478,
        "end": 485,
        "language": "mermaid"
      },
      {
        "start": 516,
        "end": 523,
        "language": "mermaid"
      },
      {
        "start": 572,
        "end": 584,
        "language": "pseudocode"
      },
      {
        "start": 588,
        "end": 595,
        "language": "mermaid"
      },
      {
        "start": 622,
        "end": 632,
        "language": "mermaid"
      },
      {
        "start": 690,
        "end": 698,
        "language": "mermaid"
      },
      {
        "start": 702,
        "end": 721,
        "language": "pseudocode"
      },
      {
        "start": 791,
        "end": 804,
        "language": "mermaid"
      },
      {
        "start": 808,
        "end": 832,
        "language": "pseudocode"
      },
      {
        "start": 912,
        "end": 928,
        "language": "mermaid"
      },
      {
        "start": 1000,
        "end": 1005,
        "language": "mermaid"
      },
      {
        "start": 1081,
        "end": 1089,
        "language": "mermaid"
      },
      {
        "start": 1109,
        "end": 1117,
        "language": "mermaid"
      },
      {
        "start": 1193,
        "end": 1201,
        "language": "mermaid"
      }
    ],
    "links": [
      "../interview-loops/08-real-world-case-studies/01-netflix-global-streaming-and-zuul-api-architecture.md",
      "../interview-loops/08-real-world-case-studies/01-netflix-global-streaming-and-zuul-api-architecture.md",
      "../interview-loops/08-real-world-case-studies/04-shopify-payment-resilience-and-flash-sale-architecture.md",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md",
      "../interview-loops/03-storage-and-search/05-distributed-job-scheduler-and-task-queue.md",
      "../interview-loops/02-social-and-realtime/02-chat-and-instant-messaging-system.md",
      "03-leases-fencing-tokens-and-distributed-locks.md",
      "../situations/23-circuit-breaker-cascading-thread-stall.md",
      "04-idempotency-and-effectively-once-processing.md",
      "../interview-loops/03-storage-and-search/05-distributed-job-scheduler-and-task-queue.md",
      "06-replication-quorums-and-read-your-writes.md",
      "../interview-loops/02-social-and-realtime/01-news-feed-system.md",
      "../interview-loops/04-financial-and-transactional/03-stock-exchange-matching-engine.md",
      "../interview-loops/02-social-and-realtime/02-chat-and-instant-messaging-system.md",
      "05-change-streams-and-transactional-outbox.md",
      "12-queues-and-delivery-semantics.md",
      "../loop-primitives/14-rate-limiting-algorithms.md",
      "08-sharding-hot-keys-and-rebalancing.md",
      "12-queues-and-delivery-semantics.md",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md",
      "../interview-loops/08-real-world-case-studies/04-shopify-payment-resilience-and-flash-sale-architecture.md",
      "../situations/23-circuit-breaker-cascading-thread-stall.md",
      "../interview-loops/03-storage-and-search/05-distributed-job-scheduler-and-task-queue.md",
      "../situations/23-circuit-breaker-cascading-thread-stall.md",
      "../interview-loops/08-real-world-case-studies/01-netflix-global-streaming-and-zuul-api-architecture.md",
      "../interview-loops/08-real-world-case-studies/01-netflix-global-streaming-and-zuul-api-architecture.md",
      "../interview-loops/05-location-and-geospatial/04-ride-sharing-dispatch-service.md",
      "11-caching-and-invalidation.md",
      "08-sharding-hot-keys-and-rebalancing.md",
      "../interview-loops/07-mobile-and-offline/03-mobile-stock-trading-app.md",
      "14-rate-limiting-algorithms.md",
      "06-replication-quorums-and-read-your-writes.md",
      "12-queues-and-delivery-semantics.md",
      "14-rate-limiting-algorithms.md",
      "08-sharding-hot-keys-and-rebalancing.md",
      "12-queues-and-delivery-semantics.md",
      "09-multi-region-failover.md",
      "../interview-loops/04-financial-and-transactional/03-stock-exchange-matching-engine.md",
      "../interview-loops/08-real-world-case-studies/06-bot-defense-and-sybil-resistant-registration-architecture.md",
      "04-idempotency-and-effectively-once-processing.md",
      "08-sharding-hot-keys-and-rebalancing.md",
      "09-multi-region-failover.md",
      "03-leases-fencing-tokens-and-distributed-locks.md",
      "05-change-streams-and-transactional-outbox.md",
      "06-replication-quorums-and-read-your-writes.md",
      "11-caching-and-invalidation.md",
      "12-queues-and-delivery-semantics.md",
      "../loop-primitives/14-rate-limiting-algorithms.md",
      "../situations/23-circuit-breaker-cascading-thread-stall.md",
      "../components/22-circuit-breaker-bulkhead-and-fault-tolerance-patterns.md",
      "../interview-loops/01-core-infrastructure/01-distributed-key-value-store.md",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md",
      "../interview-loops/02-social-and-realtime/01-news-feed-system.md",
      "../interview-loops/02-social-and-realtime/02-chat-and-instant-messaging-system.md",
      "../interview-loops/02-social-and-realtime/04-notification-system.md",
      "../interview-loops/03-storage-and-search/01-s3-like-object-storage.md",
      "../interview-loops/03-storage-and-search/02-search-autocomplete-system.md",
      "../interview-loops/03-storage-and-search/03-web-crawler-at-scale.md",
      "../interview-loops/03-storage-and-search/05-distributed-job-scheduler-and-task-queue.md",
      "../interview-loops/04-financial-and-transactional/01-payment-processing-system.md",
      "../interview-loops/04-financial-and-transactional/03-stock-exchange-matching-engine.md",
      "../interview-loops/04-financial-and-transactional/04-hotel-reservation-system.md",
      "../interview-loops/05-location-and-geospatial/04-ride-sharing-dispatch-service.md",
      "../interview-loops/07-mobile-and-offline/01-mobile-news-feed-offline-first.md",
      "../interview-loops/07-mobile-and-offline/03-mobile-stock-trading-app.md",
      "../interview-loops/07-mobile-and-offline/04-mobile-paging-library-architecture.md",
      "../interview-loops/08-real-world-case-studies/01-netflix-global-streaming-and-zuul-api-architecture.md",
      "../interview-loops/08-real-world-case-studies/03-uber-realtime-dispatch-and-schemaless-mysql.md",
      "../interview-loops/08-real-world-case-studies/04-shopify-payment-resilience-and-flash-sale-architecture.md"
    ],
    "lessons": [
      6,
      7
    ]
  },
  {
    "path": "loop-primitives/11-caching-and-invalidation.md",
    "sha256": "67c47119d49c5db8706db1c0e04d4bcf683e43cda8648451953ad46da5cff6ea",
    "lines": 1522,
    "words": 25919,
    "headings": [
      {
        "line": 1,
        "title": "Caching & Invalidation"
      },
      {
        "line": 4,
        "title": "Part 0. Start here"
      },
      {
        "line": 6,
        "title": "The problem: one price change and 12,000 page views a second"
      },
      {
        "line": 50,
        "title": "The big picture"
      },
      {
        "line": 67,
        "title": "What you'll be able to do after this page"
      },
      {
        "line": 86,
        "title": "Part 1. What a cache is, and why the database now depends on it"
      },
      {
        "line": 90,
        "title": "A copy with no subscription"
      },
      {
        "line": 107,
        "title": "The read path and the write path"
      },
      {
        "line": 150,
        "title": "Hit ratio and the database"
      },
      {
        "line": 170,
        "title": "What a TTL costs in misses"
      },
      {
        "line": 182,
        "title": "Why not more read replicas instead?"
      },
      {
        "line": 186,
        "title": "Why not cache it forever?"
      },
      {
        "line": 190,
        "title": "The example we follow"
      },
      {
        "line": 240,
        "title": "Part 2. Five ways to put a cache in the path, and one alternative"
      },
      {
        "line": 244,
        "title": "Two writers cross"
      },
      {
        "line": 288,
        "title": "Six designs on equal terms"
      },
      {
        "line": 301,
        "title": "Write-behind: the cache is the truth until it flushes"
      },
      {
        "line": 305,
        "title": "A complete copy fed by the stream"
      },
      {
        "line": 317,
        "title": "Part 3. The race that stores the old value again after a delete"
      },
      {
        "line": 321,
        "title": "Trace: the old price comes back"
      },
      {
        "line": 375,
        "title": "Race B: the lagging source"
      },
      {
        "line": 379,
        "title": "Race A: the slow reader"
      },
      {
        "line": 393,
        "title": "What the refill can see"
      },
      {
        "line": 412,
        "title": "Fix 1: versions on both sides"
      },
      {
        "line": 462,
        "title": "Fix 2: leases"
      },
      {
        "line": 473,
        "title": "The TTL is the last bound"
      },
      {
        "line": 477,
        "title": "Every write-path option on equal terms"
      },
      {
        "line": 500,
        "title": "Part 4. Where invalidations come from"
      },
      {
        "line": 504,
        "title": "Trace: the invalidation that timed out"
      },
      {
        "line": 519,
        "title": "Sources on equal terms"
      },
      {
        "line": 564,
        "title": "Invalidate from the copy you fill from"
      },
      {
        "line": 570,
        "title": "A new key per version"
      },
      {
        "line": 583,
        "title": "The invalidator's lag is your staleness"
      },
      {
        "line": 595,
        "title": "Part 5. Stampedes: when a hot key or a batch of keys expires"
      },
      {
        "line": 599,
        "title": "Many keys at once: the avalanche, on a copy"
      },
      {
        "line": 618,
        "title": "Jitter, the main line"
      },
      {
        "line": 622,
        "title": "Trace: `k7` expires under 12,000 reads a second"
      },
      {
        "line": 667,
        "title": "One fill per key"
      },
      {
        "line": 678,
        "title": "Serve stale, refresh once"
      },
      {
        "line": 708,
        "title": "Refresh before it expires"
      },
      {
        "line": 721,
        "title": "The fixes on equal terms"
      },
      {
        "line": 734,
        "title": "Question 1 of the drill The Product Page That Melted Redis: a restarted cache"
      },
      {
        "line": 748,
        "title": "Part 6. Negative caching: the keys you don't have"
      },
      {
        "line": 752,
        "title": "Trace: a deleted product in an email"
      },
      {
        "line": 764,
        "title": "A create must replace a cached \"absent\""
      },
      {
        "line": 768,
        "title": "Random keys never repeat"
      },
      {
        "line": 780,
        "title": "Part 7. Sizing and eviction"
      },
      {
        "line": 784,
        "title": "Working set and hit ratio"
      },
      {
        "line": 819,
        "title": "LRU, LFU and scans"
      },
      {
        "line": 835,
        "title": "Eviction policies"
      },
      {
        "line": 859,
        "title": "Part 8. When the cache is empty: failover, cold starts and warming"
      },
      {
        "line": 863,
        "title": "Trace: shard B dies during the sale"
      },
      {
        "line": 872,
        "title": "Cache error is not a miss, on a copy"
      },
      {
        "line": 899,
        "title": "Cap the fills"
      },
      {
        "line": 934,
        "title": "Trace: back, empty"
      },
      {
        "line": 963,
        "title": "Warming"
      },
      {
        "line": 971,
        "title": "A replica keeps it warm"
      },
      {
        "line": 975,
        "title": "What a failover loses"
      },
      {
        "line": 996,
        "title": "Part 9. Consistency: your own writes, other Regions and DAX"
      },
      {
        "line": 1000,
        "title": "Trace: the merchandiser can't see her price"
      },
      {
        "line": 1010,
        "title": "Read-your-writes through a cache"
      },
      {
        "line": 1043,
        "title": "Trace: the second Region"
      },
      {
        "line": 1086,
        "title": "DAX"
      },
      {
        "line": 1108,
        "title": "Part 10. The edge: CDN, browser and HTTP caching"
      },
      {
        "line": 1112,
        "title": "Back to 20:00, on a copy: every link is a different object"
      },
      {
        "line": 1146,
        "title": "`Cache-Control` directives, and the `Age` header"
      },
      {
        "line": 1159,
        "title": "The cache key"
      },
      {
        "line": 1191,
        "title": "A personal page cached publicly"
      },
      {
        "line": 1199,
        "title": "Purge or version"
      },
      {
        "line": 1212,
        "title": "Serving stale at the edge"
      },
      {
        "line": 1218,
        "title": "Why an edge at all"
      },
      {
        "line": 1230,
        "title": "Part 11. End to end through the layers"
      },
      {
        "line": 1297,
        "title": "Part 12. On AWS"
      },
      {
        "line": 1301,
        "title": "Managed services that use it"
      },
      {
        "line": 1310,
        "title": "Running it yourself"
      },
      {
        "line": 1317,
        "title": "Look-alikes that are not this mechanism"
      },
      {
        "line": 1338,
        "title": "Part 13. What you've learned"
      },
      {
        "line": 1340,
        "title": "Back to the kettle"
      },
      {
        "line": 1355,
        "title": "The whole story, event by event"
      },
      {
        "line": 1421,
        "title": "The cheat card"
      },
      {
        "line": 1459,
        "title": "Think-first drills"
      },
      {
        "line": 1500,
        "title": "Interview questions"
      },
      {
        "line": 1511,
        "title": "Where to go next"
      }
    ],
    "fences": [
      {
        "start": 52,
        "end": 63,
        "language": "mermaid"
      },
      {
        "start": 111,
        "end": 125,
        "language": "pseudocode"
      },
      {
        "start": 129,
        "end": 144,
        "language": "mermaid"
      },
      {
        "start": 257,
        "end": 272,
        "language": "mermaid"
      },
      {
        "start": 345,
        "end": 359,
        "language": "mermaid"
      },
      {
        "start": 416,
        "end": 437,
        "language": "pseudocode"
      },
      {
        "start": 532,
        "end": 542,
        "language": "mermaid"
      },
      {
        "start": 635,
        "end": 641,
        "language": "mermaid"
      },
      {
        "start": 645,
        "end": 651,
        "language": "mermaid"
      },
      {
        "start": 682,
        "end": 698,
        "language": "pseudocode"
      },
      {
        "start": 797,
        "end": 803,
        "language": "mermaid"
      },
      {
        "start": 901,
        "end": 921,
        "language": "pseudocode"
      },
      {
        "start": 952,
        "end": 959,
        "language": "mermaid"
      },
      {
        "start": 1014,
        "end": 1027,
        "language": "pseudocode"
      },
      {
        "start": 1031,
        "end": 1037,
        "language": "mermaid"
      },
      {
        "start": 1047,
        "end": 1062,
        "language": "mermaid"
      },
      {
        "start": 1127,
        "end": 1140,
        "language": "mermaid"
      },
      {
        "start": 1234,
        "end": 1243,
        "language": "mermaid"
      },
      {
        "start": 1247,
        "end": 1258,
        "language": "mermaid"
      }
    ],
    "links": [
      "../situations/01-caching-hot-product-page.md",
      "../interview-loops/01-core-infrastructure/05-url-shortener.md",
      "../interview-loops/02-social-and-realtime/01-news-feed-system.md",
      "../interview-loops/04-financial-and-transactional/04-hotel-reservation-system.md",
      "../interview-loops/08-real-world-case-studies/04-shopify-payment-resilience-and-flash-sale-architecture.md",
      "06-replication-quorums-and-read-your-writes.md",
      "05-change-streams-and-transactional-outbox.md",
      "06-replication-quorums-and-read-your-writes.md",
      "../components/04-distributed-cache-patterns-and-eviction.md",
      "../interview-loops/05-location-and-geospatial/01-proximity-service.md",
      "../interview-loops/04-financial-and-transactional/02-digital-wallet-system.md",
      "../interview-loops/06-media-and-streaming/03-ad-click-event-aggregation.md",
      "05-change-streams-and-transactional-outbox.md",
      "../situations/01-caching-hot-product-page.md",
      "../interview-loops/01-core-infrastructure/05-url-shortener.md",
      "../interview-loops/02-social-and-realtime/01-news-feed-system.md",
      "../interview-loops/05-location-and-geospatial/03-nearby-friends-location-service.md",
      "05-change-streams-and-transactional-outbox.md",
      "03-leases-fencing-tokens-and-distributed-locks.md",
      "04-idempotency-and-effectively-once-processing.md",
      "../interview-loops/08-real-world-case-studies/05-figma-100x-postgres-multi-tenant-scaling.md",
      "../interview-loops/08-real-world-case-studies/04-shopify-payment-resilience-and-flash-sale-architecture.md",
      "../interview-loops/02-social-and-realtime/04-notification-system.md",
      "05-change-streams-and-transactional-outbox.md",
      "10-retries-timeouts-backpressure-and-load-shedding.md",
      "08-sharding-hot-keys-and-rebalancing.md",
      "../situations/01-caching-hot-product-page.md",
      "14-rate-limiting-algorithms.md",
      "10-retries-timeouts-backpressure-and-load-shedding.md",
      "06-replication-quorums-and-read-your-writes.md",
      "06-replication-quorums-and-read-your-writes.md",
      "../interview-loops/02-social-and-realtime/01-news-feed-system.md",
      "08-sharding-hot-keys-and-rebalancing.md",
      "../situations/06-cdn-edge-image-delivery.md",
      "../interview-loops/06-media-and-streaming/01-youtube-video-streaming-platform.md",
      "08-sharding-hot-keys-and-rebalancing.md",
      "06-replication-quorums-and-read-your-writes.md",
      "06-replication-quorums-and-read-your-writes.md",
      "08-sharding-hot-keys-and-rebalancing.md",
      "09-multi-region-failover.md",
      "05-change-streams-and-transactional-outbox.md",
      "06-replication-quorums-and-read-your-writes.md",
      "08-sharding-hot-keys-and-rebalancing.md",
      "04-idempotency-and-effectively-once-processing.md",
      "09-multi-region-failover.md",
      "03-leases-fencing-tokens-and-distributed-locks.md",
      "10-retries-timeouts-backpressure-and-load-shedding.md",
      "14-rate-limiting-algorithms.md",
      "../components/04-distributed-cache-patterns-and-eviction.md",
      "../situations/01-caching-hot-product-page.md",
      "../situations/06-cdn-edge-image-delivery.md",
      "../interview-loops/01-core-infrastructure/01-distributed-key-value-store.md",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md",
      "../interview-loops/01-core-infrastructure/05-url-shortener.md",
      "../interview-loops/02-social-and-realtime/01-news-feed-system.md",
      "../interview-loops/02-social-and-realtime/02-chat-and-instant-messaging-system.md",
      "../interview-loops/02-social-and-realtime/03-realtime-gaming-leaderboard.md",
      "../interview-loops/02-social-and-realtime/04-notification-system.md",
      "../interview-loops/03-storage-and-search/02-search-autocomplete-system.md",
      "../interview-loops/03-storage-and-search/05-distributed-job-scheduler-and-task-queue.md",
      "../interview-loops/04-financial-and-transactional/02-digital-wallet-system.md",
      "../interview-loops/04-financial-and-transactional/04-hotel-reservation-system.md",
      "../interview-loops/05-location-and-geospatial/01-proximity-service.md",
      "../interview-loops/05-location-and-geospatial/02-google-maps-and-routing-engine.md",
      "../interview-loops/05-location-and-geospatial/03-nearby-friends-location-service.md",
      "../interview-loops/05-location-and-geospatial/04-ride-sharing-dispatch-service.md",
      "../interview-loops/06-media-and-streaming/01-youtube-video-streaming-platform.md",
      "../interview-loops/06-media-and-streaming/02-google-drive-file-sync-and-storage.md",
      "../interview-loops/06-media-and-streaming/03-ad-click-event-aggregation.md",
      "../interview-loops/07-mobile-and-offline/01-mobile-news-feed-offline-first.md",
      "../interview-loops/07-mobile-and-offline/03-mobile-stock-trading-app.md",
      "../interview-loops/08-real-world-case-studies/01-netflix-global-streaming-and-zuul-api-architecture.md",
      "../interview-loops/08-real-world-case-studies/02-discord-trillions-of-messages-scylladb-storage.md",
      "../interview-loops/08-real-world-case-studies/04-shopify-payment-resilience-and-flash-sale-architecture.md",
      "../interview-loops/08-real-world-case-studies/05-figma-100x-postgres-multi-tenant-scaling.md"
    ],
    "lessons": [
      10,
      11
    ]
  },
  {
    "path": "loop-primitives/12-queues-and-delivery-semantics.md",
    "sha256": "595dca185264f187929e2c036df5dc7d5a145f557ea411c5b527b0f71fcb68bd",
    "lines": 1607,
    "words": 27384,
    "headings": [
      {
        "line": 1,
        "title": "Queues & Delivery Semantics"
      },
      {
        "line": 4,
        "title": "Part 0. Start here"
      },
      {
        "line": 6,
        "title": "The problem: the order that vanished with every alarm green"
      },
      {
        "line": 50,
        "title": "The big picture"
      },
      {
        "line": 71,
        "title": "What you'll be able to do after this page"
      },
      {
        "line": 90,
        "title": "Part 1. A received message is leased, not removed"
      },
      {
        "line": 94,
        "title": "Queue or log"
      },
      {
        "line": 108,
        "title": "The ack point decides the semantics"
      },
      {
        "line": 144,
        "title": "The example we follow"
      },
      {
        "line": 214,
        "title": "Part 2. One message's round trip"
      },
      {
        "line": 218,
        "title": "The loop every worker runs"
      },
      {
        "line": 240,
        "title": "Long polling"
      },
      {
        "line": 251,
        "title": "A batch starts every timer at once"
      },
      {
        "line": 278,
        "title": "Receive what you'll work"
      },
      {
        "line": 284,
        "title": "In-flight limits"
      },
      {
        "line": 288,
        "title": "What a queue is not"
      },
      {
        "line": 312,
        "title": "Part 3. The attempt that outlives its lease"
      },
      {
        "line": 351,
        "title": "The visibility timeout is a lease"
      },
      {
        "line": 355,
        "title": "Sizing it"
      },
      {
        "line": 359,
        "title": "The heartbeat and its cap"
      },
      {
        "line": 374,
        "title": "A delete with an old receipt handle"
      },
      {
        "line": 386,
        "title": "Slow failures hold resources"
      },
      {
        "line": 390,
        "title": "Releasing a message early"
      },
      {
        "line": 414,
        "title": "Part 4. The message that keeps failing"
      },
      {
        "line": 418,
        "title": "Which errors to retry"
      },
      {
        "line": 429,
        "title": "Count, and back off with the visibility"
      },
      {
        "line": 451,
        "title": "Bound it: `maxReceiveCount` and the dead-letter queue"
      },
      {
        "line": 500,
        "title": "Deferring healthy work"
      },
      {
        "line": 524,
        "title": "Delay queues and timers"
      },
      {
        "line": 548,
        "title": "Part 5. The dead-letter queue that no alarm reports"
      },
      {
        "line": 582,
        "title": "Five ways a DLQ fails silently"
      },
      {
        "line": 592,
        "title": "The age metric's blind spot"
      },
      {
        "line": 596,
        "title": "An alarm that fires (Q3)"
      },
      {
        "line": 633,
        "title": "Bringing it back"
      },
      {
        "line": 684,
        "title": "Part 6. The batch that fails with it"
      },
      {
        "line": 688,
        "title": "How a Lambda reads a queue"
      },
      {
        "line": 738,
        "title": "A failed batch comes back whole"
      },
      {
        "line": 748,
        "title": "Report failures by ID"
      },
      {
        "line": 774,
        "title": "Concurrency at the source, not at the function"
      },
      {
        "line": 800,
        "title": "Part 7. The customer it blocks"
      },
      {
        "line": 804,
        "title": "Message groups"
      },
      {
        "line": 827,
        "title": "Head-of-line blocking, and why the age alarm works here"
      },
      {
        "line": 837,
        "title": "A DLQ breaks the order; parking keeps it"
      },
      {
        "line": 863,
        "title": "Choosing the key"
      },
      {
        "line": 877,
        "title": "Throughput"
      },
      {
        "line": 888,
        "title": "Fair queues: tenants without order"
      },
      {
        "line": 912,
        "title": "Part 8. The partitions it blocks"
      },
      {
        "line": 926,
        "title": "Consumer groups and offsets"
      },
      {
        "line": 950,
        "title": "One record stops a consumer's partitions, then the group"
      },
      {
        "line": 986,
        "title": "Rebalances"
      },
      {
        "line": 995,
        "title": "The fix: bound the retries, then apply the error boundary"
      },
      {
        "line": 1028,
        "title": "The same poison message on four transports"
      },
      {
        "line": 1064,
        "title": "Part 9. The backlog behind it"
      },
      {
        "line": 1068,
        "title": "Two different ages"
      },
      {
        "line": 1104,
        "title": "Alarm on age, not on depth"
      },
      {
        "line": 1122,
        "title": "Adding consumers"
      },
      {
        "line": 1156,
        "title": "Part 10. The copies it fans out to"
      },
      {
        "line": 1160,
        "title": "One publish, one copy per queue"
      },
      {
        "line": 1179,
        "title": "What SNS and EventBridge retry, and what they drop"
      },
      {
        "line": 1220,
        "title": "Size limits"
      },
      {
        "line": 1291,
        "title": "Part 11. One order, end to end"
      },
      {
        "line": 1345,
        "title": "Part 12. On AWS"
      },
      {
        "line": 1349,
        "title": "Managed services that use it"
      },
      {
        "line": 1380,
        "title": "Running it yourself"
      },
      {
        "line": 1390,
        "title": "Look-alikes that are not this mechanism"
      },
      {
        "line": 1411,
        "title": "Part 13. What you've learned"
      },
      {
        "line": 1413,
        "title": "Back to `o-7731`"
      },
      {
        "line": 1426,
        "title": "The snapshots, side by side"
      },
      {
        "line": 1437,
        "title": "What it costs"
      },
      {
        "line": 1446,
        "title": "The whole story, event by event"
      },
      {
        "line": 1509,
        "title": "The cheat card"
      },
      {
        "line": 1548,
        "title": "Think-first drills"
      },
      {
        "line": 1584,
        "title": "Interview questions"
      },
      {
        "line": 1595,
        "title": "Where to go next"
      }
    ],
    "fences": [
      {
        "start": 52,
        "end": 67,
        "language": "mermaid"
      },
      {
        "start": 112,
        "end": 123,
        "language": "mermaid"
      },
      {
        "start": 220,
        "end": 234,
        "language": "pseudocode"
      },
      {
        "start": 255,
        "end": 268,
        "language": "mermaid"
      },
      {
        "start": 316,
        "end": 331,
        "language": "mermaid"
      },
      {
        "start": 361,
        "end": 368,
        "language": "pseudocode"
      },
      {
        "start": 433,
        "end": 445,
        "language": "pseudocode"
      },
      {
        "start": 455,
        "end": 469,
        "language": "mermaid"
      },
      {
        "start": 554,
        "end": 576,
        "language": "mermaid"
      },
      {
        "start": 611,
        "end": 629,
        "language": "mermaid"
      },
      {
        "start": 642,
        "end": 654,
        "language": "pseudocode"
      },
      {
        "start": 708,
        "end": 734,
        "language": "mermaid"
      },
      {
        "start": 750,
        "end": 762,
        "language": "pseudocode"
      },
      {
        "start": 808,
        "end": 823,
        "language": "mermaid"
      },
      {
        "start": 930,
        "end": 946,
        "language": "mermaid"
      },
      {
        "start": 961,
        "end": 978,
        "language": "mermaid"
      },
      {
        "start": 997,
        "end": 1014,
        "language": "pseudocode"
      },
      {
        "start": 1091,
        "end": 1098,
        "language": "mermaid"
      },
      {
        "start": 1164,
        "end": 1175,
        "language": "mermaid"
      },
      {
        "start": 1195,
        "end": 1204,
        "language": "pseudocode"
      },
      {
        "start": 1235,
        "end": 1247,
        "language": "pseudocode"
      },
      {
        "start": 1251,
        "end": 1265,
        "language": "mermaid"
      },
      {
        "start": 1295,
        "end": 1307,
        "language": "mermaid"
      }
    ],
    "links": [
      "../interview-loops/01-core-infrastructure/03-distributed-message-queue.md",
      "04-idempotency-and-effectively-once-processing.md",
      "../interview-loops/01-core-infrastructure/03-distributed-message-queue.md",
      "../interview-loops/02-social-and-realtime/04-notification-system.md",
      "../interview-loops/03-storage-and-search/05-distributed-job-scheduler-and-task-queue.md",
      "../interview-loops/03-storage-and-search/03-web-crawler-at-scale.md",
      "../interview-loops/04-financial-and-transactional/05-transactional-outbox-and-event-driven-ledger.md",
      "../situations/03-message-queue-order-pipeline.md",
      "../interview-loops/01-core-infrastructure/03-distributed-message-queue.md",
      "04-idempotency-and-effectively-once-processing.md",
      "../situations/03-message-queue-order-pipeline.md",
      "05-change-streams-and-transactional-outbox.md",
      "10-retries-timeouts-backpressure-and-load-shedding.md",
      "05-change-streams-and-transactional-outbox.md",
      "../interview-loops/06-media-and-streaming/04-distributed-email-service.md",
      "10-retries-timeouts-backpressure-and-load-shedding.md",
      "../interview-loops/01-core-infrastructure/03-distributed-message-queue.md",
      "../interview-loops/03-storage-and-search/03-web-crawler-at-scale.md",
      "../interview-loops/03-storage-and-search/03-web-crawler-at-scale.md",
      "04-idempotency-and-effectively-once-processing.md",
      "03-leases-fencing-tokens-and-distributed-locks.md",
      "../interview-loops/01-core-infrastructure/03-distributed-message-queue.md",
      "10-retries-timeouts-backpressure-and-load-shedding.md",
      "../interview-loops/03-storage-and-search/05-distributed-job-scheduler-and-task-queue.md",
      "../interview-loops/06-media-and-streaming/01-youtube-video-streaming-platform.md",
      "../interview-loops/03-storage-and-search/03-web-crawler-at-scale.md",
      "10-retries-timeouts-backpressure-and-load-shedding.md",
      "10-retries-timeouts-backpressure-and-load-shedding.md",
      "../interview-loops/02-social-and-realtime/04-notification-system.md",
      "../interview-loops/02-social-and-realtime/04-notification-system.md",
      "../interview-loops/03-storage-and-search/05-distributed-job-scheduler-and-task-queue.md",
      "../interview-loops/05-location-and-geospatial/04-ride-sharing-dispatch-service.md",
      "../interview-loops/01-core-infrastructure/03-distributed-message-queue.md",
      "05-change-streams-and-transactional-outbox.md",
      "05-change-streams-and-transactional-outbox.md",
      "../interview-loops/01-core-infrastructure/03-distributed-message-queue.md",
      "05-change-streams-and-transactional-outbox.md",
      "08-sharding-hot-keys-and-rebalancing.md",
      "04-idempotency-and-effectively-once-processing.md",
      "../interview-loops/02-social-and-realtime/04-notification-system.md",
      "../interview-loops/02-social-and-realtime/04-notification-system.md",
      "../interview-loops/03-storage-and-search/05-distributed-job-scheduler-and-task-queue.md",
      "../interview-loops/01-core-infrastructure/03-distributed-message-queue.md",
      "05-change-streams-and-transactional-outbox.md",
      "05-change-streams-and-transactional-outbox.md",
      "05-change-streams-and-transactional-outbox.md",
      "10-retries-timeouts-backpressure-and-load-shedding.md",
      "../situations/03-message-queue-order-pipeline.md",
      "07-event-time-watermarks-and-checkpoints.md",
      "05-change-streams-and-transactional-outbox.md",
      "../interview-loops/06-media-and-streaming/04-distributed-email-service.md",
      "05-change-streams-and-transactional-outbox.md",
      "04-idempotency-and-effectively-once-processing.md",
      "05-change-streams-and-transactional-outbox.md",
      "01-write-ahead-log-fsync-and-group-commit.md",
      "09-multi-region-failover.md",
      "04-idempotency-and-effectively-once-processing.md",
      "05-change-streams-and-transactional-outbox.md",
      "05-change-streams-and-transactional-outbox.md",
      "04-idempotency-and-effectively-once-processing.md",
      "05-change-streams-and-transactional-outbox.md",
      "03-leases-fencing-tokens-and-distributed-locks.md",
      "10-retries-timeouts-backpressure-and-load-shedding.md",
      "07-event-time-watermarks-and-checkpoints.md",
      "08-sharding-hot-keys-and-rebalancing.md",
      "09-multi-region-failover.md",
      "14-rate-limiting-algorithms.md",
      "../situations/03-message-queue-order-pipeline.md",
      "../components/05-message-queues-vs-event-streams.md",
      "../interview-loops/01-core-infrastructure/03-distributed-message-queue.md",
      "../interview-loops/02-social-and-realtime/04-notification-system.md",
      "../interview-loops/03-storage-and-search/05-distributed-job-scheduler-and-task-queue.md",
      "../interview-loops/03-storage-and-search/03-web-crawler-at-scale.md",
      "../interview-loops/06-media-and-streaming/04-distributed-email-service.md",
      "../interview-loops/06-media-and-streaming/01-youtube-video-streaming-platform.md",
      "../interview-loops/04-financial-and-transactional/05-transactional-outbox-and-event-driven-ledger.md",
      "../interview-loops/05-location-and-geospatial/04-ride-sharing-dispatch-service.md",
      "../interview-loops/02-social-and-realtime/01-news-feed-system.md",
      "../interview-loops/02-social-and-realtime/02-chat-and-instant-messaging-system.md",
      "../interview-loops/03-storage-and-search/04-metrics-monitoring-and-alerting-system.md",
      "../interview-loops/01-core-infrastructure/05-url-shortener.md",
      "../interview-loops/07-mobile-and-offline/02-mobile-chat-architecture.md",
      "../interview-loops/04-financial-and-transactional/02-digital-wallet-system.md",
      "../interview-loops/02-social-and-realtime/03-realtime-gaming-leaderboard.md",
      "../interview-loops/04-financial-and-transactional/04-hotel-reservation-system.md",
      "../interview-loops/06-media-and-streaming/02-google-drive-file-sync-and-storage.md",
      "../interview-loops/07-mobile-and-offline/01-mobile-news-feed-offline-first.md",
      "../interview-loops/07-mobile-and-offline/03-mobile-stock-trading-app.md",
      "../interview-loops/08-real-world-case-studies/03-uber-realtime-dispatch-and-schemaless-mysql.md",
      "../interview-loops/08-real-world-case-studies/01-netflix-global-streaming-and-zuul-api-architecture.md"
    ],
    "lessons": [
      19
    ]
  },
  {
    "path": "loop-primitives/13-distributed-transactions-sagas-vs-two-phase-commit.md",
    "sha256": "24e7c7f88b504c49dbadcfb84b05e8dca6ad8b75d65e8d33c2392c74f2d62299",
    "lines": 1620,
    "words": 25112,
    "headings": [
      {
        "line": 1,
        "title": "Distributed Transactions: Sagas vs Two-Phase Commit"
      },
      {
        "line": 4,
        "title": "Part 0. Start here"
      },
      {
        "line": 6,
        "title": "The problem: a sold-out hotel that cost $36,260"
      },
      {
        "line": 51,
        "title": "The big picture"
      },
      {
        "line": 76,
        "title": "What you'll be able to do after this page"
      },
      {
        "line": 96,
        "title": "Part 1. One trip, three databases, two partners"
      },
      {
        "line": 100,
        "title": "Charge, seat, rooms, refund"
      },
      {
        "line": 121,
        "title": "What one transaction covers"
      },
      {
        "line": 132,
        "title": "Two families"
      },
      {
        "line": 139,
        "title": "The example we follow"
      },
      {
        "line": 208,
        "title": "Part 2. All or nothing, with locks"
      },
      {
        "line": 212,
        "title": "Three databases, one decision"
      },
      {
        "line": 249,
        "title": "Prepare, vote, decide, commit"
      },
      {
        "line": 284,
        "title": "What it looks like in PostgreSQL and MySQL"
      },
      {
        "line": 301,
        "title": "What 2PC can't include"
      },
      {
        "line": 316,
        "title": "Isolation is still the participants'"
      },
      {
        "line": 320,
        "title": "The lock window"
      },
      {
        "line": 336,
        "title": "One store, one call"
      },
      {
        "line": 371,
        "title": "Part 3. The coordinator that died"
      },
      {
        "line": 375,
        "title": "What a crash after the votes blocks: 10 s with a takeover, 90 s without"
      },
      {
        "line": 411,
        "title": "Why a prepared participant waits"
      },
      {
        "line": 420,
        "title": "Presumed abort and the scan"
      },
      {
        "line": 426,
        "title": "What exactly is blocked"
      },
      {
        "line": 470,
        "title": "An operator's manual rollback"
      },
      {
        "line": 484,
        "title": "When a participant crashes"
      },
      {
        "line": 488,
        "title": "Replicated coordinators"
      },
      {
        "line": 514,
        "title": "Part 4. Undo instead of rollback"
      },
      {
        "line": 518,
        "title": "Seat held, rooms gone"
      },
      {
        "line": 541,
        "title": "A saga"
      },
      {
        "line": 563,
        "title": "Semantic undo"
      },
      {
        "line": 575,
        "title": "Backwards, in order"
      },
      {
        "line": 618,
        "title": "Part 5. The point of no return"
      },
      {
        "line": 622,
        "title": "Three zones"
      },
      {
        "line": 652,
        "title": "Ordering rules"
      },
      {
        "line": 666,
        "title": "Fence, then the pivot"
      },
      {
        "line": 680,
        "title": "Telling the customer at the pivot"
      },
      {
        "line": 691,
        "title": "Confirm, then capture"
      },
      {
        "line": 695,
        "title": "Retriable, not \"retried forever\""
      },
      {
        "line": 711,
        "title": "When the pivot fails"
      },
      {
        "line": 745,
        "title": "Part 6. Holds that expire"
      },
      {
        "line": 749,
        "title": "Try, confirm, cancel"
      },
      {
        "line": 763,
        "title": "Why holds expire"
      },
      {
        "line": 789,
        "title": "Empty rollback and hanging"
      },
      {
        "line": 850,
        "title": "The fence, and twenty minutes without an orchestrator"
      },
      {
        "line": 905,
        "title": "Part 7. The reply that never came"
      },
      {
        "line": 909,
        "title": "Timed out at 5 s"
      },
      {
        "line": 940,
        "title": "Unknown, not failed"
      },
      {
        "line": 954,
        "title": "Resolve, then decide"
      },
      {
        "line": 965,
        "title": "When the undo fails"
      },
      {
        "line": 1006,
        "title": "Part 8. The orchestrator that crashed"
      },
      {
        "line": 1010,
        "title": "The saga log"
      },
      {
        "line": 1025,
        "title": "Taking over"
      },
      {
        "line": 1092,
        "title": "Why the key matters"
      },
      {
        "line": 1096,
        "title": "Determinism and versions"
      },
      {
        "line": 1100,
        "title": "Commands through the outbox"
      },
      {
        "line": 1126,
        "title": "Part 9. What other bookings see"
      },
      {
        "line": 1130,
        "title": "The undo that oversold"
      },
      {
        "line": 1155,
        "title": "Three anomalies"
      },
      {
        "line": 1167,
        "title": "Countermeasures"
      },
      {
        "line": 1198,
        "title": "Part 10. Nobody in charge"
      },
      {
        "line": 1202,
        "title": "Two styles"
      },
      {
        "line": 1236,
        "title": "Ordering on the bus"
      },
      {
        "line": 1280,
        "title": "Part 11. The same trip four ways"
      },
      {
        "line": 1284,
        "title": "On equal terms"
      },
      {
        "line": 1303,
        "title": "Choose this when"
      },
      {
        "line": 1321,
        "title": "Part 12. One trip, end to end"
      },
      {
        "line": 1364,
        "title": "Part 13. On AWS"
      },
      {
        "line": 1368,
        "title": "Managed services that use it"
      },
      {
        "line": 1395,
        "title": "Running it yourself"
      },
      {
        "line": 1404,
        "title": "Look-alikes that are not this mechanism"
      },
      {
        "line": 1425,
        "title": "Part 14. What you've learned"
      },
      {
        "line": 1427,
        "title": "Back to the sale"
      },
      {
        "line": 1437,
        "title": "What it costs"
      },
      {
        "line": 1446,
        "title": "The settings"
      },
      {
        "line": 1461,
        "title": "The whole story, event by event"
      },
      {
        "line": 1520,
        "title": "The cheat card"
      },
      {
        "line": 1561,
        "title": "Think-first drills"
      },
      {
        "line": 1597,
        "title": "Interview questions"
      },
      {
        "line": 1608,
        "title": "Where to go next"
      }
    ],
    "fences": [
      {
        "start": 53,
        "end": 72,
        "language": "mermaid"
      },
      {
        "start": 102,
        "end": 117,
        "language": "mermaid"
      },
      {
        "start": 216,
        "end": 235,
        "language": "mermaid"
      },
      {
        "start": 251,
        "end": 278,
        "language": "pseudocode"
      },
      {
        "start": 377,
        "end": 399,
        "language": "mermaid"
      },
      {
        "start": 436,
        "end": 453,
        "language": "mermaid"
      },
      {
        "start": 520,
        "end": 537,
        "language": "mermaid"
      },
      {
        "start": 584,
        "end": 592,
        "language": "pseudocode"
      },
      {
        "start": 630,
        "end": 648,
        "language": "mermaid"
      },
      {
        "start": 772,
        "end": 785,
        "language": "mermaid"
      },
      {
        "start": 791,
        "end": 804,
        "language": "mermaid"
      },
      {
        "start": 817,
        "end": 844,
        "language": "pseudocode"
      },
      {
        "start": 854,
        "end": 871,
        "language": "sql"
      },
      {
        "start": 911,
        "end": 925,
        "language": "mermaid"
      },
      {
        "start": 1027,
        "end": 1039,
        "language": "sql"
      },
      {
        "start": 1041,
        "end": 1052,
        "language": "pseudocode"
      },
      {
        "start": 1058,
        "end": 1074,
        "language": "mermaid"
      },
      {
        "start": 1132,
        "end": 1144,
        "language": "mermaid"
      },
      {
        "start": 1204,
        "end": 1220,
        "language": "mermaid"
      },
      {
        "start": 1325,
        "end": 1337,
        "language": "mermaid"
      }
    ],
    "links": [
      "../interview-loops/04-financial-and-transactional/01-payment-processing-system.md",
      "../interview-loops/04-financial-and-transactional/02-digital-wallet-system.md",
      "../interview-loops/04-financial-and-transactional/04-hotel-reservation-system.md",
      "../interview-loops/03-storage-and-search/05-distributed-job-scheduler-and-task-queue.md",
      "../interview-loops/08-real-world-case-studies/04-shopify-payment-resilience-and-flash-sale-architecture.md",
      "../situations/12-saga-orchestration-compensating-failure.md",
      "05-change-streams-and-transactional-outbox.md",
      "04-idempotency-and-effectively-once-processing.md",
      "../components/10-two-phase-commit-and-saga-orchestration.md",
      "04-idempotency-and-effectively-once-processing.md",
      "../interview-loops/05-location-and-geospatial/04-ride-sharing-dispatch-service.md",
      "06-replication-quorums-and-read-your-writes.md",
      "../components/09-distributed-consensus-raft-and-paxos.md",
      "../interview-loops/04-financial-and-transactional/02-digital-wallet-system.md",
      "../interview-loops/07-mobile-and-offline/01-mobile-news-feed-offline-first.md",
      "../interview-loops/04-financial-and-transactional/04-hotel-reservation-system.md",
      "10-retries-timeouts-backpressure-and-load-shedding.md",
      "../interview-loops/04-financial-and-transactional/01-payment-processing-system.md",
      "../situations/12-saga-orchestration-compensating-failure.md",
      "05-change-streams-and-transactional-outbox.md",
      "../interview-loops/04-financial-and-transactional/02-digital-wallet-system.md",
      "03-leases-fencing-tokens-and-distributed-locks.md",
      "../interview-loops/08-real-world-case-studies/04-shopify-payment-resilience-and-flash-sale-architecture.md",
      "04-idempotency-and-effectively-once-processing.md",
      "../interview-loops/04-financial-and-transactional/01-payment-processing-system.md",
      "../interview-loops/07-mobile-and-offline/03-mobile-stock-trading-app.md",
      "../interview-loops/04-financial-and-transactional/04-hotel-reservation-system.md",
      "../interview-loops/03-storage-and-search/05-distributed-job-scheduler-and-task-queue.md",
      "../interview-loops/04-financial-and-transactional/06-crowdfunding-platform.md",
      "03-leases-fencing-tokens-and-distributed-locks.md",
      "05-change-streams-and-transactional-outbox.md",
      "../interview-loops/03-storage-and-search/05-distributed-job-scheduler-and-task-queue.md",
      "../interview-loops/04-financial-and-transactional/01-payment-processing-system.md",
      "09-multi-region-failover.md",
      "../components/21-database-isolation-levels-acid-and-concurrency-anomalies.md",
      "../interview-loops/04-financial-and-transactional/02-digital-wallet-system.md",
      "../interview-loops/04-financial-and-transactional/01-payment-processing-system.md",
      "../interview-loops/08-real-world-case-studies/03-uber-realtime-dispatch-and-schemaless-mysql.md",
      "04-idempotency-and-effectively-once-processing.md",
      "../interview-loops/03-storage-and-search/05-distributed-job-scheduler-and-task-queue.md",
      "12-queues-and-delivery-semantics.md",
      "../interview-loops/04-financial-and-transactional/04-hotel-reservation-system.md",
      "01-write-ahead-log-fsync-and-group-commit.md",
      "04-idempotency-and-effectively-once-processing.md",
      "../interview-loops/04-financial-and-transactional/05-transactional-outbox-and-event-driven-ledger.md",
      "../interview-loops/08-real-world-case-studies/03-uber-realtime-dispatch-and-schemaless-mysql.md",
      "09-multi-region-failover.md",
      "10-retries-timeouts-backpressure-and-load-shedding.md",
      "04-idempotency-and-effectively-once-processing.md",
      "05-change-streams-and-transactional-outbox.md",
      "03-leases-fencing-tokens-and-distributed-locks.md",
      "06-replication-quorums-and-read-your-writes.md",
      "08-sharding-hot-keys-and-rebalancing.md",
      "10-retries-timeouts-backpressure-and-load-shedding.md",
      "12-queues-and-delivery-semantics.md",
      "09-multi-region-failover.md",
      "../components/10-two-phase-commit-and-saga-orchestration.md",
      "../components/21-database-isolation-levels-acid-and-concurrency-anomalies.md",
      "../components/09-distributed-consensus-raft-and-paxos.md",
      "../situations/12-saga-orchestration-compensating-failure.md",
      "../interview-loops/04-financial-and-transactional/01-payment-processing-system.md",
      "../interview-loops/04-financial-and-transactional/02-digital-wallet-system.md",
      "../interview-loops/04-financial-and-transactional/04-hotel-reservation-system.md",
      "../interview-loops/04-financial-and-transactional/05-transactional-outbox-and-event-driven-ledger.md",
      "../interview-loops/04-financial-and-transactional/06-crowdfunding-platform.md",
      "../interview-loops/03-storage-and-search/05-distributed-job-scheduler-and-task-queue.md",
      "../interview-loops/01-core-infrastructure/01-distributed-key-value-store.md",
      "../interview-loops/02-social-and-realtime/02-chat-and-instant-messaging-system.md",
      "../interview-loops/02-social-and-realtime/01-news-feed-system.md",
      "../interview-loops/02-social-and-realtime/03-realtime-gaming-leaderboard.md",
      "../interview-loops/05-location-and-geospatial/03-nearby-friends-location-service.md",
      "../interview-loops/05-location-and-geospatial/04-ride-sharing-dispatch-service.md",
      "../interview-loops/06-media-and-streaming/01-youtube-video-streaming-platform.md",
      "../interview-loops/06-media-and-streaming/02-google-drive-file-sync-and-storage.md",
      "../interview-loops/06-media-and-streaming/03-ad-click-event-aggregation.md",
      "../interview-loops/06-media-and-streaming/04-distributed-email-service.md",
      "../interview-loops/07-mobile-and-offline/01-mobile-news-feed-offline-first.md",
      "../interview-loops/07-mobile-and-offline/03-mobile-stock-trading-app.md",
      "../interview-loops/08-real-world-case-studies/03-uber-realtime-dispatch-and-schemaless-mysql.md",
      "../interview-loops/08-real-world-case-studies/04-shopify-payment-resilience-and-flash-sale-architecture.md",
      "../interview-loops/08-real-world-case-studies/05-figma-100x-postgres-multi-tenant-scaling.md",
      "../interview-loops/08-real-world-case-studies/06-bot-defense-and-sybil-resistant-registration-architecture.md"
    ],
    "lessons": [
      23,
      24
    ]
  },
  {
    "path": "loop-primitives/14-rate-limiting-algorithms.md",
    "sha256": "300451357eab01fe74f531009ea4f5990d031e3ad410398f6899d1b9c2ba1d61",
    "lines": 3702,
    "words": 49053,
    "headings": [
      {
        "line": 1,
        "title": "Rate Limiting Algorithms"
      },
      {
        "line": 4,
        "title": "Start here"
      },
      {
        "line": 34,
        "title": "What you'll be able to do after this page"
      },
      {
        "line": 70,
        "title": "Mid-level: one limit, enforced correctly"
      },
      {
        "line": 74,
        "title": "The problem: twenty gateways, each with its own count for one key"
      },
      {
        "line": 138,
        "title": "The big picture"
      },
      {
        "line": 168,
        "title": "A limit is two numbers and a key"
      },
      {
        "line": 180,
        "title": "Rate, burst, key, cost"
      },
      {
        "line": 201,
        "title": "Rate limit, quota, shedding"
      },
      {
        "line": 219,
        "title": "The example we follow"
      },
      {
        "line": 344,
        "title": "Counting in windows"
      },
      {
        "line": 368,
        "title": "Fixed window"
      },
      {
        "line": 462,
        "title": "The edge between two fixed windows"
      },
      {
        "line": 515,
        "title": "Sliding log"
      },
      {
        "line": 554,
        "title": "Sliding window counter"
      },
      {
        "line": 588,
        "title": "How wrong the counter's estimate can be"
      },
      {
        "line": 626,
        "title": "The token bucket"
      },
      {
        "line": 632,
        "title": "The token bucket's check"
      },
      {
        "line": 738,
        "title": "The bound on any interval"
      },
      {
        "line": 757,
        "title": "`Retry-After` from the refill"
      },
      {
        "line": 763,
        "title": "GCRA: the same decisions with one number"
      },
      {
        "line": 826,
        "title": "The leaky bucket: a queue or a meter"
      },
      {
        "line": 882,
        "title": "Twenty servers, one limit"
      },
      {
        "line": 920,
        "title": "Why twenty buckets admit twenty times the rate"
      },
      {
        "line": 950,
        "title": "Split the limit into exact shares"
      },
      {
        "line": 974,
        "title": "Send the key to one server"
      },
      {
        "line": 987,
        "title": "Count the key in one place"
      },
      {
        "line": 995,
        "title": "Which way each design errs"
      },
      {
        "line": 1028,
        "title": "The same bucket when you are the caller"
      },
      {
        "line": 1071,
        "title": "One atomic step, one clock"
      },
      {
        "line": 1081,
        "title": "Fifteen calls, five tokens"
      },
      {
        "line": 1128,
        "title": "The atomic script"
      },
      {
        "line": 1169,
        "title": "Whose clock the script reads"
      },
      {
        "line": 1213,
        "title": "DynamoDB as the limiter's store"
      },
      {
        "line": 1251,
        "title": "Keys, memory and expiry"
      },
      {
        "line": 1259,
        "title": "40 accounts, 40 buckets"
      },
      {
        "line": 1281,
        "title": "What to key on"
      },
      {
        "line": 1298,
        "title": "Memory per key"
      },
      {
        "line": 1315,
        "title": "TTL and eviction"
      },
      {
        "line": 1373,
        "title": "When the limiter's store fails"
      },
      {
        "line": 1383,
        "title": "A timeout and a breaker on the limiter's store"
      },
      {
        "line": 1428,
        "title": "Three failure policies"
      },
      {
        "line": 1459,
        "title": "Security limits with more servers than the limit"
      },
      {
        "line": 1478,
        "title": "When the store comes back"
      },
      {
        "line": 1518,
        "title": "What the client is told"
      },
      {
        "line": 1538,
        "title": "`429` or `503`"
      },
      {
        "line": 1546,
        "title": "`Retry-After` in whole seconds"
      },
      {
        "line": 1554,
        "title": "The budget on every response"
      },
      {
        "line": 1579,
        "title": "Limiters that do not send `429` by default"
      },
      {
        "line": 1593,
        "title": "A client that honours `Retry-After`"
      },
      {
        "line": 1677,
        "title": "Mid-level checkpoint"
      },
      {
        "line": 1681,
        "title": "Check yourself: Mid-level"
      },
      {
        "line": 1797,
        "title": "Senior: many limits, many keys, live rules"
      },
      {
        "line": 1801,
        "title": "The problem: one Tuesday, four new failures"
      },
      {
        "line": 1887,
        "title": "Several limits on one request"
      },
      {
        "line": 1898,
        "title": "One script, all buckets"
      },
      {
        "line": 1974,
        "title": "Hash tags"
      },
      {
        "line": 2008,
        "title": "Changing a rule"
      },
      {
        "line": 2014,
        "title": "A rule that reaches 20 gateways one at a time"
      },
      {
        "line": 2065,
        "title": "Rule numbers as script arguments"
      },
      {
        "line": 2112,
        "title": "Versions that only go up"
      },
      {
        "line": 2120,
        "title": "Shadow mode before a rule refuses"
      },
      {
        "line": 2155,
        "title": "A key too hot for one shard"
      },
      {
        "line": 2163,
        "title": "100,000 a second on one shard"
      },
      {
        "line": 2175,
        "title": "Lease a block of tokens"
      },
      {
        "line": 2265,
        "title": "The overshoot of leasing and its bound"
      },
      {
        "line": 2342,
        "title": "One request, end to end"
      },
      {
        "line": 2420,
        "title": "Layer by layer"
      },
      {
        "line": 2475,
        "title": "On AWS"
      },
      {
        "line": 2489,
        "title": "Managed services that use it"
      },
      {
        "line": 2545,
        "title": "Running it yourself"
      },
      {
        "line": 2585,
        "title": "Look-alikes that are not this mechanism"
      },
      {
        "line": 2608,
        "title": "Senior checkpoint"
      },
      {
        "line": 2612,
        "title": "Check yourself: Senior"
      },
      {
        "line": 2639,
        "title": "Architect: what the whole system promises"
      },
      {
        "line": 2643,
        "title": "The problem: one Thursday in three Regions"
      },
      {
        "line": 2730,
        "title": "One limit, three Regions"
      },
      {
        "line": 2736,
        "title": "Shares that follow demand"
      },
      {
        "line": 2767,
        "title": "Moving room by named grants"
      },
      {
        "line": 2816,
        "title": "What run G shows"
      },
      {
        "line": 2875,
        "title": "What the sum of shares bounds"
      },
      {
        "line": 2888,
        "title": "Security limits across Regions"
      },
      {
        "line": 2900,
        "title": "Why one counter that every Region writes fails"
      },
      {
        "line": 2938,
        "title": "When a Region is lost"
      },
      {
        "line": 2946,
        "title": "Three rules for a silent Region"
      },
      {
        "line": 2963,
        "title": "Run G2: Region B is lost"
      },
      {
        "line": 3045,
        "title": "A rate limit is not a billing quota"
      },
      {
        "line": 3051,
        "title": "Per second and per day"
      },
      {
        "line": 3061,
        "title": "Quotas that bill"
      },
      {
        "line": 3081,
        "title": "A bucket is not a count"
      },
      {
        "line": 3091,
        "title": "Counting each admitted request once"
      },
      {
        "line": 3118,
        "title": "How late a quota is enforced"
      },
      {
        "line": 3157,
        "title": "An attack spread over many addresses"
      },
      {
        "line": 3223,
        "title": "A slow backend when every client is under its limit"
      },
      {
        "line": 3277,
        "title": "Build on AWS's limiters or your own"
      },
      {
        "line": 3281,
        "title": "API Gateway, WAF and a limiter you build, on equal terms"
      },
      {
        "line": 3303,
        "title": "What each mechanism costs"
      },
      {
        "line": 3360,
        "title": "Where the loop designs each decision"
      },
      {
        "line": 3381,
        "title": "Architect checkpoint"
      },
      {
        "line": 3385,
        "title": "Check yourself: Architect"
      },
      {
        "line": 3434,
        "title": "Reference"
      },
      {
        "line": 3438,
        "title": "Back to FreshBox"
      },
      {
        "line": 3477,
        "title": "The snapshots, side by side"
      },
      {
        "line": 3490,
        "title": "What it costs"
      },
      {
        "line": 3502,
        "title": "The whole story, event by event"
      },
      {
        "line": 3563,
        "title": "The cheat card"
      },
      {
        "line": 3639,
        "title": "Pages that send readers here"
      },
      {
        "line": 3652,
        "title": "Where to go next"
      },
      {
        "line": 3697,
        "title": "Ready to apply this?"
      }
    ],
    "fences": [
      {
        "start": 20,
        "end": 24,
        "language": "mermaid"
      },
      {
        "start": 142,
        "end": 158,
        "language": "mermaid"
      },
      {
        "start": 374,
        "end": 383,
        "language": "pseudocode"
      },
      {
        "start": 443,
        "end": 450,
        "language": "mermaid"
      },
      {
        "start": 480,
        "end": 503,
        "language": "mermaid"
      },
      {
        "start": 521,
        "end": 528,
        "language": "pseudocode"
      },
      {
        "start": 560,
        "end": 568,
        "language": "pseudocode"
      },
      {
        "start": 642,
        "end": 663,
        "language": "pseudocode"
      },
      {
        "start": 712,
        "end": 718,
        "language": "mermaid"
      },
      {
        "start": 771,
        "end": 782,
        "language": "pseudocode"
      },
      {
        "start": 802,
        "end": 807,
        "language": "mermaid"
      },
      {
        "start": 894,
        "end": 912,
        "language": "mermaid"
      },
      {
        "start": 999,
        "end": 1006,
        "language": "mermaid"
      },
      {
        "start": 1089,
        "end": 1107,
        "language": "mermaid"
      },
      {
        "start": 1134,
        "end": 1146,
        "language": "pseudocode"
      },
      {
        "start": 1175,
        "end": 1189,
        "language": "mermaid"
      },
      {
        "start": 1403,
        "end": 1418,
        "language": "mermaid"
      },
      {
        "start": 1526,
        "end": 1534,
        "language": "http"
      },
      {
        "start": 1608,
        "end": 1623,
        "language": "mermaid"
      },
      {
        "start": 1824,
        "end": 1839,
        "language": "mermaid"
      },
      {
        "start": 1902,
        "end": 1910,
        "language": "mermaid"
      },
      {
        "start": 1938,
        "end": 1953,
        "language": "pseudocode"
      },
      {
        "start": 2047,
        "end": 2055,
        "language": "mermaid"
      },
      {
        "start": 2080,
        "end": 2100,
        "language": "pseudocode"
      },
      {
        "start": 2183,
        "end": 2198,
        "language": "mermaid"
      },
      {
        "start": 2212,
        "end": 2236,
        "language": "pseudocode"
      },
      {
        "start": 2387,
        "end": 2404,
        "language": "mermaid"
      },
      {
        "start": 2424,
        "end": 2434,
        "language": "mermaid"
      },
      {
        "start": 2662,
        "end": 2680,
        "language": "mermaid"
      },
      {
        "start": 2795,
        "end": 2810,
        "language": "mermaid"
      },
      {
        "start": 2850,
        "end": 2858,
        "language": "mermaid"
      },
      {
        "start": 2996,
        "end": 3007,
        "language": "mermaid"
      },
      {
        "start": 3103,
        "end": 3108,
        "language": "mermaid"
      },
      {
        "start": 3187,
        "end": 3194,
        "language": "mermaid"
      },
      {
        "start": 3243,
        "end": 3248,
        "language": "mermaid"
      }
    ],
    "links": [
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md",
      "#mid-level-one-limit-enforced-correctly",
      "#senior-many-limits-many-keys-live-rules",
      "#architect-what-the-whole-system-promises",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md#round-1-mid-level-protect-one-public-api",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md#round-2-senior-a-multi-tenant-gateway-for-the-whole-company",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md#round-3-architect-global-limits-billing-quotas-and-real-attackers",
      "#reference",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md",
      "../situations/04-rate-limiting-partner-api.md",
      "#a-limit-is-two-numbers-and-a-key",
      "#counting-in-windows",
      "#the-token-bucket",
      "#twenty-servers-one-limit",
      "#one-atomic-step-one-clock",
      "#keys-memory-and-expiry",
      "#when-the-limiters-store-fails",
      "#what-the-client-is-told",
      "#several-limits-on-one-request",
      "#changing-a-rule",
      "#a-key-too-hot-for-one-shard",
      "#one-request-end-to-end",
      "#on-aws",
      "#one-limit-three-regions",
      "#when-a-region-is-lost",
      "#a-rate-limit-is-not-a-billing-quota",
      "#an-attack-spread-over-many-addresses",
      "#a-slow-backend-when-every-client-is-under-its-limit",
      "#build-on-awss-limiters-or-your-own",
      "#check-yourself-architect",
      "#pages-that-send-readers-here",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md#round-1-mid-level-protect-one-public-api",
      "#counting-in-windows",
      "#split-the-limit-into-exact-shares",
      "#when-the-limiters-store-fails",
      "#counting-in-windows",
      "#the-token-bucket",
      "#twenty-servers-one-limit",
      "#one-atomic-step-one-clock",
      "#when-the-limiters-store-fails",
      "#a-limit-is-two-numbers-and-a-key",
      "#twenty-servers-one-limit",
      "#what-it-costs",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md#r11",
      "#keys-memory-and-expiry",
      "#several-limits-on-one-request",
      "../interview-loops/01-core-infrastructure/03-distributed-message-queue.md",
      "../interview-loops/03-storage-and-search/04-metrics-monitoring-and-alerting-system.md",
      "#counting-in-windows",
      "#the-token-bucket",
      "#a-rate-limit-is-not-a-billing-quota",
      "10-retries-timeouts-backpressure-and-load-shedding.md",
      "10-retries-timeouts-backpressure-and-load-shedding.md",
      "10-retries-timeouts-backpressure-and-load-shedding.md",
      "../components/02-distributed-rate-limiting.md",
      "10-retries-timeouts-backpressure-and-load-shedding.md",
      "12-queues-and-delivery-semantics.md",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md",
      "../situations/04-rate-limiting-partner-api.md",
      "../interview-loops/07-mobile-and-offline/03-mobile-stock-trading-app.md",
      "10-retries-timeouts-backpressure-and-load-shedding.md",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md#step-14",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md#step-16",
      "#counting-in-windows",
      "#counting-in-windows",
      "#the-token-bucket",
      "#twenty-servers-one-limit",
      "#one-atomic-step-one-clock",
      "#keys-memory-and-expiry",
      "#when-the-limiters-store-fails",
      "#what-the-client-is-told",
      "#the-problem-one-tuesday-four-new-failures",
      "#several-limits-on-one-request",
      "#changing-a-rule",
      "#a-key-too-hot-for-one-shard",
      "#one-request-end-to-end",
      "#the-problem-one-thursday-in-three-regions",
      "#one-limit-three-regions",
      "#when-a-region-is-lost",
      "#a-rate-limit-is-not-a-billing-quota",
      "#an-attack-spread-over-many-addresses",
      "#a-slow-backend-when-every-client-is-under-its-limit",
      "#build-on-awss-limiters-or-your-own",
      "#the-whole-story-event-by-event",
      "#the-bound-on-any-interval",
      "#the-problem-twenty-gateways-each-with-its-own-count-for-one-key",
      "#twenty-servers-one-limit",
      "#the-token-bucket",
      "#one-atomic-step-one-clock",
      "#the-token-bucket",
      "#when-the-limiters-store-fails",
      "#what-the-client-is-told",
      "#the-token-bucket",
      "../situations/04-rate-limiting-partner-api.md",
      "#memory-per-key",
      "#the-atomic-script",
      "#the-edge-between-two-fixed-windows",
      "#the-atomic-script",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md#step-12",
      "#several-limits-on-one-request",
      "#the-budget-on-every-response",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md#step-12",
      "#retry-after-in-whole-seconds",
      "../interview-loops/04-financial-and-transactional/04-hotel-reservation-system.md#step-22",
      "../interview-loops/08-real-world-case-studies/04-shopify-payment-resilience-and-flash-sale-architecture.md#step-23",
      "../situations/04-rate-limiting-partner-api.md",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md#step-10",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md#step-11",
      "../interview-loops/06-media-and-streaming/04-distributed-email-service.md#step-26",
      "../interview-loops/07-mobile-and-offline/03-mobile-stock-trading-app.md#step-26",
      "../interview-loops/07-mobile-and-offline/03-mobile-stock-trading-app.md#step-26",
      "../interview-loops/02-social-and-realtime/04-notification-system.md#step-24",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md#step-11",
      "#one-atomic-step-one-clock",
      "#when-the-limiters-store-fails",
      "#a-key-too-hot-for-one-shard",
      "#a-key-too-hot-for-one-shard",
      "#when-the-limiters-store-fails",
      "#a-key-too-hot-for-one-shard",
      "../interview-loops/04-financial-and-transactional/06-crowdfunding-platform.md#step-23",
      "04-idempotency-and-effectively-once-processing.md",
      "12-queues-and-delivery-semantics.md",
      "../interview-loops/02-social-and-realtime/02-chat-and-instant-messaging-system.md#step-25",
      "#when-the-limiters-store-fails",
      "#a-key-too-hot-for-one-shard",
      "#when-the-limiters-store-fails",
      "03-leases-fencing-tokens-and-distributed-locks.md",
      "#a-key-too-hot-for-one-shard",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md#r111",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md#step-21",
      "#several-limits-on-one-request",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md#step-25",
      "../interview-loops/03-storage-and-search/03-web-crawler-at-scale.md#step-13",
      "../interview-loops/08-real-world-case-studies/02-discord-trillions-of-messages-scylladb-storage.md#step-21",
      "../interview-loops/01-core-infrastructure/03-distributed-message-queue.md#step-27",
      "../interview-loops/03-storage-and-search/04-metrics-monitoring-and-alerting-system.md#step-24",
      "11-caching-and-invalidation.md",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md#r111",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md#r28",
      "#several-limits-on-one-request",
      "10-retries-timeouts-backpressure-and-load-shedding.md",
      "#split-the-limit-into-exact-shares",
      "#send-the-key-to-one-server",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md#step-14",
      "../interview-loops/08-real-world-case-studies/06-bot-defense-and-sybil-resistant-registration-architecture.md#r19",
      "06-replication-quorums-and-read-your-writes.md",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md#step-24",
      "10-retries-timeouts-backpressure-and-load-shedding.md",
      "10-retries-timeouts-backpressure-and-load-shedding.md",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md#r111",
      "#when-the-limiters-store-fails",
      "10-retries-timeouts-backpressure-and-load-shedding.md",
      "#one-request-end-to-end",
      "#mid-level-checkpoint",
      "../situations/04-rate-limiting-partner-api.md",
      "../situations/04-rate-limiting-partner-api.md",
      "#a-key-too-hot-for-one-shard",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md#round-2-senior-a-multi-tenant-gateway-for-the-whole-company",
      "#one-request-end-to-end",
      "#several-limits-on-one-request",
      "#changing-a-rule",
      "#a-key-too-hot-for-one-shard",
      "#one-request-end-to-end",
      "#several-limits-on-one-request",
      "#changing-a-rule",
      "#a-key-too-hot-for-one-shard",
      "#one-request-end-to-end",
      "#the-overshoot-of-leasing-and-its-bound",
      "#mid-level-one-limit-enforced-correctly",
      "#a-key-too-hot-for-one-shard",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md#step-22",
      "#40-accounts-40-buckets",
      "#the-atomic-script",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md#step-22",
      "#the-atomic-script",
      "08-sharding-hot-keys-and-rebalancing.md",
      "08-sharding-hot-keys-and-rebalancing.md",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md#step-23",
      "#the-overshoot-of-leasing-and-its-bound",
      "03-leases-fencing-tokens-and-distributed-locks.md",
      "#twenty-servers-one-limit",
      "#when-the-limiters-store-fails",
      "#when-the-limiters-store-fails",
      "#a-key-too-hot-for-one-shard",
      "#ttl-and-eviction",
      "10-retries-timeouts-backpressure-and-load-shedding.md",
      "../interview-loops/08-real-world-case-studies/06-bot-defense-and-sybil-resistant-registration-architecture.md#r28",
      "#dynamodb-as-the-limiters-store",
      "#twenty-servers-one-limit",
      "12-queues-and-delivery-semantics.md",
      "#the-same-bucket-when-you-are-the-caller",
      "#quotas-that-bill",
      "08-sharding-hot-keys-and-rebalancing.md",
      "10-retries-timeouts-backpressure-and-load-shedding.md",
      "../interview-loops/08-real-world-case-studies/06-bot-defense-and-sybil-resistant-registration-architecture.md",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md#round-3-architect-global-limits-billing-quotas-and-real-attackers",
      "#one-limit-three-regions",
      "#what-run-g-shows",
      "#one-limit-three-regions",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md#step-31",
      "#when-a-region-is-lost",
      "#changing-a-rule",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md#step-31",
      "../interview-loops/06-media-and-streaming/03-ad-click-event-aggregation.md#step-32",
      "#when-a-region-is-lost",
      "#a-key-too-hot-for-one-shard",
      "#a-rate-limit-is-not-a-billing-quota",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md#step-35",
      "09-multi-region-failover.md",
      "09-multi-region-failover.md",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md#step-35",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md#r38",
      "09-multi-region-failover.md",
      "../interview-loops/02-social-and-realtime/04-notification-system.md#step-26",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md#step-32",
      "#on-aws",
      "../interview-loops/04-financial-and-transactional/06-crowdfunding-platform.md#step-23",
      "#several-limits-on-one-request",
      "#three-failure-policies",
      "#the-overshoot-of-leasing-and-its-bound",
      "04-idempotency-and-effectively-once-processing.md",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md#step-32",
      "#security-limits-with-more-servers-than-the-limit",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md#step-33",
      "../interview-loops/08-real-world-case-studies/06-bot-defense-and-sybil-resistant-registration-architecture.md",
      "../interview-loops/08-real-world-case-studies/06-bot-defense-and-sybil-resistant-registration-architecture.md#step-21",
      "../interview-loops/08-real-world-case-studies/06-bot-defense-and-sybil-resistant-registration-architecture.md#step-32",
      "../components/25-bot-defense-sybil-resistance-and-registration-abuse.md",
      "10-retries-timeouts-backpressure-and-load-shedding.md",
      "#rate-limit-quota-shedding",
      "10-retries-timeouts-backpressure-and-load-shedding.md",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md#step-34",
      "#the-token-bucket",
      "#several-limits-on-one-request",
      "#one-limit-three-regions",
      "#when-the-limiters-store-fails",
      "#a-rate-limit-is-not-a-billing-quota",
      "#when-a-region-is-lost",
      "#one-request-end-to-end",
      "../components/14-api-gateway-and-reverse-proxy.md",
      "../components/25-bot-defense-sybil-resistance-and-registration-abuse.md",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md#step-31",
      "#one-limit-three-regions",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md#step-32",
      "#a-rate-limit-is-not-a-billing-quota",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md#step-33",
      "#an-attack-spread-over-many-addresses",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md#step-34",
      "#a-slow-backend-when-every-client-is-under-its-limit",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md#step-35",
      "#when-a-region-is-lost",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md#step-36",
      "#build-on-awss-limiters-or-your-own",
      "#counting-each-admitted-request-once",
      "#how-late-a-quota-is-enforced",
      "#a-slow-backend-when-every-client-is-under-its-limit",
      "#a-limit-is-two-numbers-and-a-key",
      "#counting-in-windows",
      "#the-token-bucket",
      "#twenty-servers-one-limit",
      "#one-atomic-step-one-clock",
      "#keys-memory-and-expiry",
      "#when-the-limiters-store-fails",
      "#what-the-client-is-told",
      "#several-limits-on-one-request",
      "#changing-a-rule",
      "#a-key-too-hot-for-one-shard",
      "#one-request-end-to-end",
      "#one-limit-three-regions",
      "#when-a-region-is-lost",
      "#a-rate-limit-is-not-a-billing-quota",
      "#an-attack-spread-over-many-addresses",
      "#a-slow-backend-when-every-client-is-under-its-limit",
      "#build-on-awss-limiters-or-your-own",
      "#the-token-buckets-check",
      "#twenty-servers-one-limit",
      "#fixed-window",
      "#fifteen-calls-five-tokens",
      "#a-timeout-and-a-breaker-on-the-limiters-store",
      "#the-overshoot-of-leasing-and-its-bound",
      "#the-example-we-follow",
      "#the-example-we-follow",
      "#the-example-we-follow",
      "#fixed-window",
      "#the-edge-between-two-fixed-windows",
      "#sliding-log",
      "#sliding-window-counter",
      "#how-wrong-the-counters-estimate-can-be",
      "#the-token-buckets-check",
      "#the-token-buckets-check",
      "#the-token-buckets-check",
      "#gcra-the-same-decisions-with-one-number",
      "#the-leaky-bucket-a-queue-or-a-meter",
      "#why-twenty-buckets-admit-twenty-times-the-rate",
      "#split-the-limit-into-exact-shares",
      "#send-the-key-to-one-server",
      "#count-the-key-in-one-place",
      "#fifteen-calls-five-tokens",
      "#whose-clock-the-script-reads",
      "#the-atomic-script",
      "#dynamodb-as-the-limiters-store",
      "#40-accounts-40-buckets",
      "#memory-per-key",
      "#ttl-and-eviction",
      "#ttl-and-eviction",
      "#a-timeout-and-a-breaker-on-the-limiters-store",
      "#three-failure-policies",
      "#three-failure-policies",
      "#three-failure-policies",
      "#security-limits-with-more-servers-than-the-limit",
      "#when-the-store-comes-back",
      "#100000-a-second-on-one-shard",
      "#the-overshoot-of-leasing-and-its-bound",
      "#the-overshoot-of-leasing-and-its-bound",
      "#one-limit-three-regions",
      "#what-run-g-shows",
      "#what-run-g-shows",
      "#run-g2-region-b-is-lost",
      "#one-script-all-buckets",
      "#one-script-all-buckets",
      "#a-rule-that-reaches-20-gateways-one-at-a-time",
      "#per-second-and-per-day",
      "#what-the-client-is-told",
      "#a-client-that-honours-retry-after",
      "#limiters-that-do-not-send-429-by-default",
      "#one-request-end-to-end",
      "#one-request-end-to-end",
      "#layer-by-layer",
      "#counting-each-admitted-request-once",
      "#how-late-a-quota-is-enforced",
      "#an-attack-spread-over-many-addresses",
      "#a-slow-backend-when-every-client-is-under-its-limit",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md#step-11",
      "../interview-loops/02-social-and-realtime/04-notification-system.md#step-24",
      "../interview-loops/07-mobile-and-offline/03-mobile-stock-trading-app.md#step-26",
      "../interview-loops/08-real-world-case-studies/04-shopify-payment-resilience-and-flash-sale-architecture.md#step-23",
      "../interview-loops/08-real-world-case-studies/06-bot-defense-and-sybil-resistant-registration-architecture.md#step-11",
      "../situations/04-rate-limiting-partner-api.md",
      "#counting-in-windows",
      "#the-token-bucket",
      "#twenty-servers-one-limit",
      "#check-yourself-mid-level",
      "10-retries-timeouts-backpressure-and-load-shedding.md",
      "08-sharding-hot-keys-and-rebalancing.md",
      "03-leases-fencing-tokens-and-distributed-locks.md",
      "04-idempotency-and-effectively-once-processing.md",
      "12-queues-and-delivery-semantics.md",
      "11-caching-and-invalidation.md",
      "06-replication-quorums-and-read-your-writes.md",
      "09-multi-region-failover.md",
      "../situations/04-rate-limiting-partner-api.md",
      "#counting-in-windows",
      "#the-token-bucket",
      "#twenty-servers-one-limit",
      "#check-yourself-mid-level",
      "../components/02-distributed-rate-limiting.md",
      "../components/14-api-gateway-and-reverse-proxy.md",
      "../components/25-bot-defense-sybil-resistance-and-registration-abuse.md",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md#step-10",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md#step-16",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md#r14",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md#r17",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md#r19",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md#r111",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md#step-21",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md#step-26",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md#r27",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md#r29",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md#r211",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md#step-31",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md#step-32",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md#step-34",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md#step-36",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md#r37",
      "../interview-loops/02-social-and-realtime/04-notification-system.md",
      "../interview-loops/02-social-and-realtime/04-notification-system.md#r17",
      "../interview-loops/02-social-and-realtime/04-notification-system.md#step-24",
      "../interview-loops/02-social-and-realtime/04-notification-system.md#step-26",
      "../interview-loops/06-media-and-streaming/04-distributed-email-service.md",
      "../interview-loops/06-media-and-streaming/04-distributed-email-service.md#step-26",
      "../interview-loops/06-media-and-streaming/04-distributed-email-service.md#r29",
      "../interview-loops/07-mobile-and-offline/03-mobile-stock-trading-app.md",
      "../interview-loops/07-mobile-and-offline/03-mobile-stock-trading-app.md#step-26",
      "../interview-loops/08-real-world-case-studies/04-shopify-payment-resilience-and-flash-sale-architecture.md",
      "../interview-loops/08-real-world-case-studies/04-shopify-payment-resilience-and-flash-sale-architecture.md#step-23",
      "../interview-loops/08-real-world-case-studies/06-bot-defense-and-sybil-resistant-registration-architecture.md",
      "../interview-loops/08-real-world-case-studies/06-bot-defense-and-sybil-resistant-registration-architecture.md#step-11",
      "../interview-loops/08-real-world-case-studies/06-bot-defense-and-sybil-resistant-registration-architecture.md#r17",
      "../interview-loops/08-real-world-case-studies/06-bot-defense-and-sybil-resistant-registration-architecture.md#r19",
      "../interview-loops/08-real-world-case-studies/06-bot-defense-and-sybil-resistant-registration-architecture.md#step-25",
      "../interview-loops/08-real-world-case-studies/06-bot-defense-and-sybil-resistant-registration-architecture.md#r28",
      "../interview-loops/04-financial-and-transactional/06-crowdfunding-platform.md",
      "../interview-loops/04-financial-and-transactional/06-crowdfunding-platform.md#step-22",
      "../interview-loops/04-financial-and-transactional/06-crowdfunding-platform.md#step-23",
      "../interview-loops/04-financial-and-transactional/06-crowdfunding-platform.md#r29",
      "../interview-loops/04-financial-and-transactional/06-crowdfunding-platform.md#step-35",
      "../interview-loops/01-core-infrastructure/01-distributed-key-value-store.md",
      "../interview-loops/01-core-infrastructure/01-distributed-key-value-store.md#step-27",
      "../interview-loops/01-core-infrastructure/02-distributed-unique-id-generator.md",
      "../interview-loops/01-core-infrastructure/02-distributed-unique-id-generator.md#step-24",
      "../interview-loops/01-core-infrastructure/03-distributed-message-queue.md",
      "../interview-loops/01-core-infrastructure/03-distributed-message-queue.md#step-27",
      "../interview-loops/03-storage-and-search/01-s3-like-object-storage.md",
      "../interview-loops/03-storage-and-search/01-s3-like-object-storage.md#step-36",
      "../interview-loops/03-storage-and-search/04-metrics-monitoring-and-alerting-system.md",
      "../interview-loops/03-storage-and-search/04-metrics-monitoring-and-alerting-system.md#step-24",
      "../interview-loops/01-core-infrastructure/05-url-shortener.md",
      "../interview-loops/01-core-infrastructure/05-url-shortener.md#step-24",
      "../interview-loops/01-core-infrastructure/05-url-shortener.md#step-26",
      "../interview-loops/01-core-infrastructure/05-url-shortener.md#r25",
      "../interview-loops/04-financial-and-transactional/04-hotel-reservation-system.md",
      "../interview-loops/04-financial-and-transactional/04-hotel-reservation-system.md#step-22",
      "../interview-loops/06-media-and-streaming/03-ad-click-event-aggregation.md",
      "../interview-loops/06-media-and-streaming/03-ad-click-event-aggregation.md#step-32",
      "../interview-loops/05-location-and-geospatial/01-proximity-service.md",
      "../interview-loops/05-location-and-geospatial/01-proximity-service.md#r19",
      "../interview-loops/05-location-and-geospatial/03-nearby-friends-location-service.md",
      "../interview-loops/05-location-and-geospatial/03-nearby-friends-location-service.md#r19",
      "../interview-loops/03-storage-and-search/03-web-crawler-at-scale.md",
      "../interview-loops/03-storage-and-search/03-web-crawler-at-scale.md#step-13",
      "../interview-loops/03-storage-and-search/03-web-crawler-at-scale.md#step-34",
      "../interview-loops/03-storage-and-search/05-distributed-job-scheduler-and-task-queue.md",
      "../interview-loops/03-storage-and-search/05-distributed-job-scheduler-and-task-queue.md#step-26",
      "../interview-loops/02-social-and-realtime/02-chat-and-instant-messaging-system.md",
      "../interview-loops/02-social-and-realtime/02-chat-and-instant-messaging-system.md#step-25",
      "../interview-loops/08-real-world-case-studies/02-discord-trillions-of-messages-scylladb-storage.md",
      "../interview-loops/08-real-world-case-studies/02-discord-trillions-of-messages-scylladb-storage.md#step-21",
      "../interview-loops/02-social-and-realtime/03-realtime-gaming-leaderboard.md",
      "../interview-loops/02-social-and-realtime/03-realtime-gaming-leaderboard.md#step-14",
      "../interview-loops/06-media-and-streaming/02-google-drive-file-sync-and-storage.md",
      "../interview-loops/06-media-and-streaming/02-google-drive-file-sync-and-storage.md#r28",
      "../interview-loops/04-financial-and-transactional/01-payment-processing-system.md",
      "../interview-loops/04-financial-and-transactional/01-payment-processing-system.md#step-33",
      "../interview-loops/08-real-world-case-studies/01-netflix-global-streaming-and-zuul-api-architecture.md",
      "../interview-loops/08-real-world-case-studies/01-netflix-global-streaming-and-zuul-api-architecture.md#r14",
      "../interview-loops/04-financial-and-transactional/02-digital-wallet-system.md",
      "../interview-loops/04-financial-and-transactional/02-digital-wallet-system.md#r26",
      "../interview-loops/01-core-infrastructure/04-distributed-rate-limiter.md",
      "../situations/04-rate-limiting-partner-api.md"
    ],
    "lessons": [
      8,
      9
    ]
  },
  {
    "path": "loop-primitives/15-time-series-storage-and-roll-ups.md",
    "sha256": "ae24364abd9e2f672d4f4d7a3233cccb34accfb633ffcf6c4c755c4b9d796035",
    "lines": 1539,
    "words": 27771,
    "headings": [
      {
        "line": 1,
        "title": "Time-Series Storage & Roll-ups"
      },
      {
        "line": 4,
        "title": "Part 0. Start here"
      },
      {
        "line": 6,
        "title": "The problem: two questions about one metric"
      },
      {
        "line": 56,
        "title": "The big picture"
      },
      {
        "line": 72,
        "title": "What you'll be able to do after this page"
      },
      {
        "line": 92,
        "title": "Part 1. A series is a name and labels"
      },
      {
        "line": 96,
        "title": "Name plus labels"
      },
      {
        "line": 100,
        "title": "Counter, gauge, histogram"
      },
      {
        "line": 125,
        "title": "The example we follow"
      },
      {
        "line": 195,
        "title": "Part 2. Series, not samples, cost memory"
      },
      {
        "line": 209,
        "title": "Finding series: the label index"
      },
      {
        "line": 221,
        "title": "What a series costs"
      },
      {
        "line": 227,
        "title": "Where to stop a label"
      },
      {
        "line": 257,
        "title": "Part 3. Writing a sample: head, log, chunks, blocks"
      },
      {
        "line": 261,
        "title": "Log first, then the chunk"
      },
      {
        "line": 281,
        "title": "Blocks, written once"
      },
      {
        "line": 295,
        "title": "How old data leaves"
      },
      {
        "line": 301,
        "title": "Why not a table of rows"
      },
      {
        "line": 325,
        "title": "Part 4. About a byte a sample"
      },
      {
        "line": 329,
        "title": "Timestamps: delta of deltas"
      },
      {
        "line": 343,
        "title": "Values: XOR with the previous value"
      },
      {
        "line": 363,
        "title": "What the paper measured, and what our series cost"
      },
      {
        "line": 394,
        "title": "Aligned scrapes keep timestamps at one bit"
      },
      {
        "line": 418,
        "title": "Part 5. Roll-ups keep what can be merged"
      },
      {
        "line": 422,
        "title": "Minute 10:20, rolled up"
      },
      {
        "line": 458,
        "title": "What merges, and what doesn't"
      },
      {
        "line": 476,
        "title": "The type decides"
      },
      {
        "line": 480,
        "title": "Counters and restarts"
      },
      {
        "line": 524,
        "title": "Keeping the last point loses the spike"
      },
      {
        "line": 540,
        "title": "The hour from 60 minutes"
      },
      {
        "line": 544,
        "title": "Set, never add"
      },
      {
        "line": 570,
        "title": "Part 6. Percentiles don't roll up"
      },
      {
        "line": 574,
        "title": "Why a percentile can't be merged"
      },
      {
        "line": 586,
        "title": "Sum the buckets, then estimate"
      },
      {
        "line": 609,
        "title": "The bucket decides the error"
      },
      {
        "line": 620,
        "title": "Sketches with a guarantee"
      },
      {
        "line": 641,
        "title": "Distinct counts"
      },
      {
        "line": 665,
        "title": "Part 7. Tiers: how long each resolution lives"
      },
      {
        "line": 669,
        "title": "What each tier keeps"
      },
      {
        "line": 677,
        "title": "Counting values, not windows"
      },
      {
        "line": 717,
        "title": "The \"60× smaller\" claim"
      },
      {
        "line": 732,
        "title": "Hot, warm and cold"
      },
      {
        "line": 736,
        "title": "Six designs on equal terms"
      },
      {
        "line": 798,
        "title": "Part 8. Reading it back: tiers, steps, rates and gaps"
      },
      {
        "line": 802,
        "title": "Three panels, three tiers"
      },
      {
        "line": 845,
        "title": "Steps and alignment"
      },
      {
        "line": 849,
        "title": "Rates across a restart"
      },
      {
        "line": 864,
        "title": "Staleness and gaps"
      },
      {
        "line": 918,
        "title": "Part 9. Late data after a roll-up"
      },
      {
        "line": 922,
        "title": "Late, or out of order?"
      },
      {
        "line": 928,
        "title": "The out-of-order window"
      },
      {
        "line": 956,
        "title": "Dirty windows and re-rolls"
      },
      {
        "line": 1000,
        "title": "Roll up early and re-roll, or roll up late once"
      },
      {
        "line": 1034,
        "title": "Part 10. Alerting on coarse data"
      },
      {
        "line": 1067,
        "title": "What sets the delay"
      },
      {
        "line": 1088,
        "title": "The hourly tier dilutes a short incident"
      },
      {
        "line": 1116,
        "title": "Part 11. When the pipeline breaks"
      },
      {
        "line": 1120,
        "title": "The job that ran twice"
      },
      {
        "line": 1170,
        "title": "A worker that fell behind"
      },
      {
        "line": 1187,
        "title": "An ingester that crashed"
      },
      {
        "line": 1191,
        "title": "A corrupt block"
      },
      {
        "line": 1215,
        "title": "Part 12. End to end through the layers"
      },
      {
        "line": 1271,
        "title": "Part 13. On AWS"
      },
      {
        "line": 1275,
        "title": "Managed services that use it"
      },
      {
        "line": 1299,
        "title": "Running it yourself"
      },
      {
        "line": 1309,
        "title": "Look-alikes that are not this mechanism"
      },
      {
        "line": 1331,
        "title": "Part 14. What you've learned"
      },
      {
        "line": 1333,
        "title": "Back to the canary"
      },
      {
        "line": 1349,
        "title": "The snapshots, side by side"
      },
      {
        "line": 1360,
        "title": "What it costs"
      },
      {
        "line": 1369,
        "title": "The whole story, event by event"
      },
      {
        "line": 1420,
        "title": "The cheat card"
      },
      {
        "line": 1476,
        "title": "Think-first drills"
      },
      {
        "line": 1516,
        "title": "Interview questions"
      },
      {
        "line": 1527,
        "title": "Where to go next"
      }
    ],
    "fences": [
      {
        "start": 58,
        "end": 68,
        "language": "mermaid"
      },
      {
        "start": 112,
        "end": 121,
        "language": "text"
      },
      {
        "start": 270,
        "end": 277,
        "language": "mermaid"
      },
      {
        "start": 377,
        "end": 383,
        "language": "mermaid"
      },
      {
        "start": 426,
        "end": 446,
        "language": "mermaid"
      },
      {
        "start": 484,
        "end": 490,
        "language": "mermaid"
      },
      {
        "start": 496,
        "end": 509,
        "language": "pseudocode"
      },
      {
        "start": 594,
        "end": 605,
        "language": "mermaid"
      },
      {
        "start": 705,
        "end": 713,
        "language": "mermaid"
      },
      {
        "start": 751,
        "end": 774,
        "language": "mermaid"
      },
      {
        "start": 816,
        "end": 831,
        "language": "pseudocode"
      },
      {
        "start": 868,
        "end": 880,
        "language": "mermaid"
      },
      {
        "start": 940,
        "end": 950,
        "language": "pseudocode"
      },
      {
        "start": 975,
        "end": 994,
        "language": "mermaid"
      },
      {
        "start": 1046,
        "end": 1063,
        "language": "mermaid"
      },
      {
        "start": 1131,
        "end": 1140,
        "language": "mermaid"
      },
      {
        "start": 1146,
        "end": 1166,
        "language": "pseudocode"
      },
      {
        "start": 1219,
        "end": 1230,
        "language": "mermaid"
      }
    ],
    "links": [
      "../interview-loops/03-storage-and-search/04-metrics-monitoring-and-alerting-system.md",
      "../interview-loops/06-media-and-streaming/03-ad-click-event-aggregation.md",
      "../interview-loops/01-core-infrastructure/05-url-shortener.md",
      "../interview-loops/02-social-and-realtime/03-realtime-gaming-leaderboard.md",
      "10-retries-timeouts-backpressure-and-load-shedding.md",
      "12-queues-and-delivery-semantics.md",
      "14-rate-limiting-algorithms.md",
      "../components/13-trie-data-structure-and-inverted-index.md",
      "../interview-loops/03-storage-and-search/04-metrics-monitoring-and-alerting-system.md",
      "14-rate-limiting-algorithms.md",
      "08-sharding-hot-keys-and-rebalancing.md",
      "01-write-ahead-log-fsync-and-group-commit.md",
      "02-lsm-trees-and-compaction.md",
      "../situations/10-wal-lsm-compaction-storm.md",
      "../interview-loops/03-storage-and-search/04-metrics-monitoring-and-alerting-system.md",
      "../interview-loops/07-mobile-and-offline/03-mobile-stock-trading-app.md",
      "../interview-loops/04-financial-and-transactional/01-payment-processing-system.md",
      "../interview-loops/06-media-and-streaming/03-ad-click-event-aggregation.md",
      "../interview-loops/03-storage-and-search/05-distributed-job-scheduler-and-task-queue.md",
      "07-event-time-watermarks-and-checkpoints.md",
      "../interview-loops/01-core-infrastructure/05-url-shortener.md",
      "04-idempotency-and-effectively-once-processing.md",
      "../interview-loops/02-social-and-realtime/03-realtime-gaming-leaderboard.md",
      "../interview-loops/07-mobile-and-offline/04-mobile-paging-library-architecture.md",
      "../interview-loops/01-core-infrastructure/05-url-shortener.md",
      "../components/03-bloom-filters-and-counting-filters.md",
      "../interview-loops/03-storage-and-search/04-metrics-monitoring-and-alerting-system.md",
      "11-caching-and-invalidation.md",
      "../interview-loops/03-storage-and-search/04-metrics-monitoring-and-alerting-system.md",
      "../interview-loops/03-storage-and-search/04-metrics-monitoring-and-alerting-system.md",
      "../interview-loops/03-storage-and-search/01-s3-like-object-storage.md",
      "../interview-loops/06-media-and-streaming/03-ad-click-event-aggregation.md",
      "07-event-time-watermarks-and-checkpoints.md",
      "01-write-ahead-log-fsync-and-group-commit.md",
      "06-replication-quorums-and-read-your-writes.md",
      "02-lsm-trees-and-compaction.md",
      "12-queues-and-delivery-semantics.md",
      "12-queues-and-delivery-semantics.md",
      "07-event-time-watermarks-and-checkpoints.md",
      "01-write-ahead-log-fsync-and-group-commit.md",
      "02-lsm-trees-and-compaction.md",
      "../situations/10-wal-lsm-compaction-storm.md",
      "07-event-time-watermarks-and-checkpoints.md",
      "08-sharding-hot-keys-and-rebalancing.md",
      "04-idempotency-and-effectively-once-processing.md",
      "11-caching-and-invalidation.md",
      "12-queues-and-delivery-semantics.md",
      "14-rate-limiting-algorithms.md",
      "06-replication-quorums-and-read-your-writes.md",
      "09-multi-region-failover.md",
      "../components/07-write-ahead-log-and-lsm-trees.md",
      "../components/13-trie-data-structure-and-inverted-index.md",
      "../components/03-bloom-filters-and-counting-filters.md",
      "../interview-loops/03-storage-and-search/04-metrics-monitoring-and-alerting-system.md",
      "../interview-loops/06-media-and-streaming/03-ad-click-event-aggregation.md",
      "../interview-loops/01-core-infrastructure/05-url-shortener.md",
      "../interview-loops/02-social-and-realtime/03-realtime-gaming-leaderboard.md",
      "../interview-loops/03-storage-and-search/01-s3-like-object-storage.md",
      "../interview-loops/03-storage-and-search/05-distributed-job-scheduler-and-task-queue.md",
      "../interview-loops/04-financial-and-transactional/01-payment-processing-system.md",
      "../interview-loops/07-mobile-and-offline/03-mobile-stock-trading-app.md",
      "../interview-loops/07-mobile-and-offline/04-mobile-paging-library-architecture.md"
    ],
    "lessons": [
      22
    ]
  }
]