{"id":32474901,"url":"https://github.com/forattini-dev/s3db.js","last_synced_at":"2026-04-08T06:02:17.243Z","repository":{"id":63544518,"uuid":"548959656","full_name":"forattini-dev/s3db.js","owner":"forattini-dev","description":"Use AWS S3 as a cheap document database.","archived":false,"fork":false,"pushed_at":"2026-04-02T02:35:30.000Z","size":164460,"stargazers_count":11,"open_issues_count":0,"forks_count":0,"subscribers_count":1,"default_branch":"main","last_synced_at":"2026-04-02T10:32:08.869Z","etag":null,"topics":["aws","database","plugin","recker","s3","vector"],"latest_commit_sha":null,"homepage":"https://forattini-dev.github.io/s3db.js/","language":"TypeScript","has_issues":true,"has_wiki":null,"has_pages":null,"mirror_url":null,"source_name":null,"license":"unlicense","status":null,"scm":"git","pull_requests_enabled":true,"icon_url":"https://github.com/forattini-dev.png","metadata":{"files":{"readme":"README.md","changelog":null,"contributing":null,"funding":null,"license":null,"code_of_conduct":null,"threat_model":null,"audit":null,"citation":null,"codeowners":null,"security":null,"support":null,"governance":null,"roadmap":null,"authors":null,"dei":null,"publiccode":null,"codemeta":null,"zenodo":null,"notice":null,"maintainers":null,"copyright":null,"agents":null,"dco":null,"cla":null}},"created_at":"2022-10-10T13:04:06.000Z","updated_at":"2026-04-02T02:35:34.000Z","dependencies_parsed_at":"2026-01-22T17:04:56.006Z","dependency_job_id":null,"html_url":"https://github.com/forattini-dev/s3db.js","commit_stats":{"total_commits":73,"total_committers":1,"mean_commits":73.0,"dds":0.0,"last_synced_commit":"e43bf02d152696014b4c383c4e7a9ff0f8df5f37"},"previous_names":[],"tags_count":368,"template":false,"template_full_name":null,"purl":"pkg:github/forattini-dev/s3db.js","repository_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/forattini-dev%2Fs3db.js","tags_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/forattini-dev%2Fs3db.js/tags","releases_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/forattini-dev%2Fs3db.js/releases","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/forattini-dev%2Fs3db.js/manifests","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/forattini-dev","download_url":"https://codeload.github.com/forattini-dev/s3db.js/tar.gz/refs/heads/main","sbom_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/forattini-dev%2Fs3db.js/sbom","scorecard":null,"host":{"name":"GitHub","url":"https://github.com","kind":"github","repositories_count":286080680,"owners_count":31542384,"icon_url":"https://github.com/github.png","version":null,"created_at":"2022-05-30T11:31:42.601Z","updated_at":"2026-04-07T16:28:08.000Z","status":"online","status_checked_at":"2026-04-08T02:00:06.127Z","response_time":54,"last_error":null,"robots_txt_status":"success","robots_txt_updated_at":"2025-07-24T06:49:26.215Z","robots_txt_url":"https://github.com/robots.txt","online":true,"can_crawl_api":true,"host_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories","repository_names_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repository_names","owners_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners"}},"keywords":["aws","database","plugin","recker","s3","vector"],"created_at":"2025-10-26T20:15:16.937Z","updated_at":"2026-04-08T06:02:17.235Z","avatar_url":"https://github.com/forattini-dev.png","language":"TypeScript","funding_links":[],"categories":[],"sub_categories":[],"readme":"# 🗃️ s3db.js\n\n\u003cp align=\"center\"\u003e\n  \u003cimg width=\"200\" src=\"https://img.icons8.com/fluency/200/database.png\" alt=\"s3db.js\"\u003e\n\u003c/p\u003e\n\n\u003cp align=\"center\"\u003e\n  \u003cstrong\u003eTransform AWS S3 into a powerful document database\u003c/strong\u003e\u003cbr\u003e\n  \u003cem\u003eCost-effective storage • Automatic encryption • ORM-like interface • Streaming API\u003c/em\u003e\n\u003c/p\u003e\n\n\u003cp align=\"center\"\u003e\n  \u003ca href=\"https://www.npmjs.com/package/s3db.js\"\u003e\u003cimg src=\"https://img.shields.io/npm/v/s3db.js.svg?style=flat\u0026color=brightgreen\" alt=\"npm version\"\u003e\u003c/a\u003e\n  \u0026nbsp;\n  \u003ca href=\"https://github.com/forattini-dev/s3db.js\"\u003e\u003cimg src=\"https://img.shields.io/github/stars/forattini-dev/s3db.js?style=flat\u0026color=yellow\" alt=\"GitHub stars\"\u003e\u003c/a\u003e\n  \u0026nbsp;\n  \u003ca href=\"https://github.com/forattini-dev/s3db.js/blob/main/LICENSE\"\u003e\u003cimg src=\"https://img.shields.io/badge/license-Unlicense-blue.svg?style=flat\" alt=\"License\"\u003e\u003c/a\u003e\n  \u0026nbsp;\n  \u003ca href=\"https://api.codeclimate.com/v1/badges/26e3dc46c42367d44f18/maintainability\"\u003e\u003cimg src=\"https://api.codeclimate.com/v1/badges/26e3dc46c42367d44f18/maintainability\" alt=\"Maintainability\"\u003e\u003c/a\u003e\n  \u0026nbsp;\n  \u003ca href=\"https://coveralls.io/github/forattini-dev/s3db.js?branch=main\"\u003e\u003cimg src=\"https://coveralls.io/repos/github/forattini-dev/s3db.js/badge.svg?branch=main\u0026style=flat\" alt=\"Coverage Status\"\u003e\u003c/a\u003e\n\u003c/p\u003e\n\n\u003cp align=\"center\"\u003e\n  \u003ca href=\"https://github.com/forattini-dev/s3db.js\"\u003e\u003cimg src=\"https://img.shields.io/badge/Built_with-Node.js-339933.svg?style=flat\u0026logo=node.js\" alt=\"Built with Node.js\"\u003e\u003c/a\u003e\n  \u0026nbsp;\n  \u003ca href=\"https://aws.amazon.com/s3/\"\u003e\u003cimg src=\"https://img.shields.io/badge/Powered_by-AWS_S3-FF9900.svg?style=flat\u0026logo=amazon-aws\" alt=\"Powered by AWS S3\"\u003e\u003c/a\u003e\n  \u0026nbsp;\n  \u003ca href=\"https://nodejs.org/\"\u003e\u003cimg src=\"https://img.shields.io/badge/Runtime-Node.js-339933.svg?style=flat\u0026logo=node.js\" alt=\"Node.js Runtime\"\u003e\u003c/a\u003e\n\u003c/p\u003e\n\n\u003cp align=\"center\"\u003e\n  \u003ca href=\"https://forattini-dev.github.io/s3db.js/\"\u003e\u003cstrong\u003e📖 Full Documentation\u003c/strong\u003e\u003c/a\u003e\n\u003c/p\u003e\n\n\u003cbr\u003e\n\n## 🚀 What is s3db.js?\n\n**s3db.js** is a document database that transforms AWS S3 into a fully functional database using S3's metadata capabilities. Instead of traditional storage methods, it stores document data in S3's metadata fields (up to 2KB), making it highly cost-effective while providing a familiar ORM-like interface.\n\n**Perfect for:**\n- 🌐 **Serverless applications** - No database servers to manage\n- 💰 **Cost-conscious projects** - Pay only for what you use\n- 🔒 **Secure applications** - Built-in encryption and validation\n- 📊 **Analytics platforms** - Efficient data streaming and processing\n- 🚀 **Rapid prototyping** - Get started in minutes, not hours\n\n---\n\n## ✨ Key Features\n\n\u003ctable\u003e\n\u003ctr\u003e\n\u003ctd width=\"50%\"\u003e\n\n### 🎯 **Database Operations**\n- **ORM-like Interface** - Familiar CRUD operations\n- **Schema Validation** - Automatic data validation\n- **Schema Registry** - Stable attribute mapping across schema changes\n- **Streaming API** - Handle large datasets efficiently\n- **Event System** - Real-time notifications\n\n\u003c/td\u003e\n\u003ctd width=\"50%\"\u003e\n\n### 🔐 **Security \u0026 Performance**\n- **Field-level Encryption** - Secure sensitive data\n- **Intelligent Caching** - Reduce API calls\n- **Concurrency Control** - Optimized bulk operations\n- **Auto-generated Passwords** - Secure by default\n- **Cost Tracking** - Monitor AWS expenses\n\n\u003c/td\u003e\n\u003c/tr\u003e\n\u003ctr\u003e\n\u003ctd width=\"50%\"\u003e\n\n### 📦 **Data Management**\n- **Partitions** - Organize data efficiently\n- **Bulk Operations** - Handle multiple records\n- **Nested Objects** - Complex data structures\n- **Automatic Timestamps** - Track changes\n\n\u003c/td\u003e\n\u003ctd width=\"50%\"\u003e\n\n### 🔧 **Extensibility**\n- **Custom Behaviors** - Handle large documents\n- **Hooks System** - Custom business logic\n- **Plugin Architecture** - Extend functionality\n- **Event System** - Real-time notifications\n\n\u003c/td\u003e\n\u003c/tr\u003e\n\u003c/table\u003e\n\n---\n\n## 📋 Table of Contents\n\n- [🚀 What is s3db.js?](#-what-is-s3dbjs)\n- [✨ Key Features](#-key-features)\n- [🚀 Quick Start](#-quick-start)\n- [💾 Installation](#-installation)\n- [🔌 Storage Backends](#-storage-backends)\n- [🗄️ Database](#️-database)\n- [🔀 DatabaseManager](#-databasemanager--multiple-backends)\n- [🪵 Logging](#-logging)\n- [📋 Resources](#-resources)\n- [⚡ Performance \u0026 Concurrency](#-performance--concurrency)\n- [🔌 Plugins](#-plugins)\n- [🤖 MCP \u0026 Integrations](#-mcp--integrations)\n- [🔧 CLI](#-cli)\n- [📖 Documentation](#-documentation)\n\n---\n\n## 📚 Documentation Quick Links\n\n\u003e **Core Concepts:** [Schema \u0026 Validation](./docs/core/schema.md) • [Schema Registry](./docs/core/schema.md#schema-registry-stable-attribute-mapping) • [Clients](./docs/clients/README.md) • [Fastest Validator](./docs/dependencies/fastest-validator.md)\n\n\u003e **Plugins:** [API Plugin](./docs/plugins/api/README.md) • [Identity Plugin](./docs/plugins/identity/README.md) • [All Plugins](#-plugins)\n\n\u003e **Guides:** [Path-based Basic + OIDC Example](./docs/examples/e101-path-based-basic-oidc.js)\n\n\n\u003e **Integrations:** [MCP Server](./docs/mcp.md) • [Model Context Protocol](./docs/mcp.md)\n\n\u003e **Advanced:** [Executor Pool Benchmark](./docs/benchmarks/executor-pool.md) • [Performance Tuning](./docs/benchmarks/README.md) • [Examples](./docs/examples/README.md) • [TypeScript Support](./docs/guides/typescript.md)\n\n---\n\n## 🚀 Quick Start\n\nGet up and running in less than 5 minutes!\n\n### 1. Install s3db.js\n\n```bash\nnpm install s3db.js\n```\n\n\u003e Need deeper telemetry? Pass `taskExecutorMonitoring` alongside `executorPool`. It merges into the pool's monitoring block, making it easy to enable verbose stats/heap tracking for any database instance without touching individual resources.\n\n### 2. Connect to your S3 database\n\n```javascript\nimport { S3db } from \"s3db.js\";\n\nconst s3db = new S3db({\n  connectionString: \"s3://ACCESS_KEY:SECRET_KEY@BUCKET_NAME/databases/myapp\"\n});\n\nawait s3db.connect();\nconsole.log(\"🎉 Connected to S3 database!\");\n```\n\n\u003e **⚡ Performance Tip:** s3db.js comes with optimized HTTP client settings by default for excellent S3 performance. The default configuration includes keep-alive enabled, balanced connection pooling, and appropriate timeouts for most applications.\n\n\u003e **ℹ️ Note:** You do **not** need to provide `ACCESS_KEY` and `SECRET_KEY` in the connection string if your environment already has S3 permissions (e.g., via IAM Role on EKS, EC2, Lambda, or other compatible clouds). s3db.js will use the default AWS credential provider chain, so credentials can be omitted for role-based or environment-based authentication. This also applies to S3-compatible clouds (MinIO, DigitalOcean Spaces, etc.) if they support such mechanisms.\n\n---\n\n### 3. Create your first resource\n\nSchema validation powered by **[fastest-validator](https://github.com/icebob/fastest-validator)** ⚡\n\n```javascript\nconst users = await s3db.createResource({\n  name: \"users\",\n  attributes: {\n    name: \"string|min:2|max:100\",\n    email: \"email|unique\",\n    age: \"number|integer|positive\",\n    isActive: \"boolean\"\n  },\n  timestamps: true\n});\n```\n\n### 4. Start storing data\n\n```javascript\n// Insert a user\nconst user = await users.insert({\n  name: \"John Doe\",\n  email: \"john@example.com\",\n  age: 30,\n  isActive: true,\n  createdAt: new Date()\n});\n\n// Query the user\nconst foundUser = await users.get(user.id);\nconsole.log(`Hello, ${foundUser.name}! 👋`);\n\n// Update the user\nawait users.update(user.id, { age: 31 });\n\n// List all users\nconst allUsers = await users.list();\nconsole.log(`Total users: ${allUsers.length}`);\n```\n\n**That's it!** You now have a fully functional document database running on AWS S3. 🎉\n\n### 5. Add plugins for a better experience (Optional)\n\nEnhance your database with powerful plugins for production-ready features. All plugins are available from the main `s3db.js` package:\n\n```javascript\nimport { S3db } from \"s3db.js\";\nimport { RelationPlugin, TTLPlugin, ReplicatorPlugin, CachePlugin } from \"s3db.js\";\n\nconst s3db = new S3db({\n  connectionString: \"s3://ACCESS_KEY:SECRET_KEY@BUCKET_NAME/databases/myapp\",\n  plugins: [\n    // Auto-cleanup expired records (no cron jobs needed!)\n    new TTLPlugin({\n      resources: {\n        sessions: { ttl: 86400, onExpire: 'soft-delete' }  // 24h\n      }\n    }),\n\n    // ORM-like relationships with 10-100x faster queries\n    new RelationPlugin({\n      relations: {\n        users: {\n          posts: { type: 'hasMany', resource: 'posts', foreignKey: 'userId' }\n        }\n      }\n    }),\n\n    // Real-time replication to BigQuery, PostgreSQL, etc.\n    new ReplicatorPlugin({\n      replicators: [{\n        driver: 'bigquery',\n        config: { projectId: 'my-project', datasetId: 'analytics' },\n        resources: { users: 'users_table', posts: 'posts_table' }\n      }]\n    }),\n\n    // Cache frequently accessed data (memory, S3, or filesystem)\n    new CachePlugin({\n      driver: 'memory',\n      ttl: 300000  // 5 minutes\n    })\n  ]\n});\n```\n\n**Learn more** about available plugins and their features in the [Plugin Documentation](docs/plugins/README.md).\n\n---\n\n## 💾 Installation\n\n### Package Manager\n\n```bash\n# npm\nnpm install s3db.js\n# pnpm\npnpm add s3db.js\n# yarn\nyarn add s3db.js\n```\n\n### 📦 Optional Dependencies\n\nSome features require additional dependencies to be installed manually:\n\n#### API Plugin Dependencies\n\nIf you plan to use the API plugin, install these dependencies:\n\n```bash\n# Core API runtime\nnpm install s3db.js\n\n# HTTP logging (optional, recommended)\nnpm install pino-http\n\n# Authentication (optional)\nnpm install jose  # For JWT auth\n\n# Standalone Raffel integrations (optional)\nnpm install raffel\n```\n\n#### Replicator Dependencies\n\nIf you plan to use the replicator system with external services, install the corresponding dependencies:\n\n```bash\n# For SQS replicator (AWS SQS queues)\nnpm install @aws-sdk/client-sqs\n\n# For BigQuery replicator (Google BigQuery)\nnpm install @google-cloud/bigquery\n\n# For PostgreSQL replicator (PostgreSQL databases)\nnpm install pg\n```\n\n**Why manual installation?** These are marked as `peerDependencies` to keep the main package lightweight (~500KB). Only install what you need!\n\n#### 🛠️ Development Setup\n\nContributing to s3db.js? Use our **modular installation system** to install only what you need:\n\n```bash\n# Clone the repo\ngit clone https://github.com/forattini-dev/s3db.js.git\ncd s3db.js\n\n# Install base dependencies (required)\npnpm install\n\n# Choose your dev setup:\n./scripts/install-deps.sh minimal     # Core only (~50MB)\n./scripts/install-deps.sh common      # + Replicators + Plugins (~500MB)\n./scripts/install-deps.sh full        # Everything (~2GB)\n\n# Or install specific groups:\npnpm run install:dev:replicators  # PostgreSQL, BigQuery, etc.\npnpm run install:dev:plugins      # API, Identity, ML, etc.\npnpm run install:dev:puppeteer    # Web scraping suite\npnpm run install:dev:cloud        # AWS SDK clients\n```\n\nSee [docs/README.md](./docs/README.md) for the main documentation entrypoint and [package.json](./package.json) for the current dependency groups.\n\n### 📘 TypeScript Support\n\ns3db.js includes comprehensive TypeScript definitions out of the box. Get full type safety, autocomplete, and IntelliSense support in your IDE!\n\n#### Basic Usage (Automatic Types)\n\n```typescript\nimport { Database, DatabaseConfig, Resource } from 's3db.js';\n\n// Type-safe configuration\nconst config: DatabaseConfig = {\n  connectionString: 's3://ACCESS_KEY:SECRET@bucket/path',\n  logLevel: 'debug',\n  executorPool: { concurrency: 100 }  // Default - nested under executorPool\n};\n\nconst db = new Database(config);\n\n// TypeScript knows all methods and options!\nawait db.createResource({\n  name: 'users',\n  attributes: {\n    name: 'string|required',\n    email: 'string|required|email',\n    age: 'number|min:0'\n  }\n});\n\n// Full autocomplete for all operations\nconst users: Resource\u003cany\u003e = db.resources.users;\nconst user = await users.insert({ name: 'Alice', email: 'alice@example.com', age: 28 });\n```\n\n#### Advanced: Generate Resource Types\n\nFor even better type safety, auto-generate TypeScript interfaces from your resources:\n\n```typescript\nimport { generateTypes } from 's3db.js/typescript-generator';\n\n// Generate types after creating resources\nawait generateTypes(db, { outputPath: './types/database.d.ts' });\n```\n\nSee the complete example in [`docs/examples/typescript-usage-example.ts`](docs/examples/typescript-usage-example.ts).\n\n---\n\n## 🔌 Storage Backends\n\ns3db.js is backend-portable. Same code, same resources, same plugins — just change the connection string.\n\n| Backend | Connection String | Best For |\n|---------|------------------|----------|\n| **AWS S3** | `s3://KEY:SECRET@bucket?region=us-east-1` | Production, large datasets (100 GB+) |\n| **Cloudflare R2** | `https://KEY:SECRET@ACCOUNT.r2.cloudflarestorage.com/bucket` | Production, zero egress, auto 8 KB metadata |\n| **Cloudflare D1** | `sqlite+d1://ACCOUNT/DB_ID?apiToken=TOKEN` | Read-heavy, serverless, \u003c 10 GB |\n| **Cloudflare D1** (Worker) | `sqlite+d1://binding/DB` + `clientOptions: { d1Binding: env.DB }` | Inside Workers (~1-5ms latency) |\n| **Turso** | `sqlite+libsql://db-org.turso.io?authToken=TOKEN` | Edge reads, any runtime |\n| **Turso** (embedded) | `sqlite+libsql:///tmp/local.db?syncUrl=libsql://db.turso.io\u0026authToken=TOKEN` | 0ms local reads + remote sync |\n| **MinIO** | `http://user:pass@localhost:9000/bucket` | Self-hosted, local dev |\n| **SQLite** (file) | `sqlite:///path/to/db.sqlite` | Local dev, CI, single-process |\n| **SQLite** (memory) | `sqlite:///:memory:` | Fast tests with SQLite behavior |\n| **Memory** | `memory://bucket/prefix` | Tests (100-1000x faster) |\n| **Filesystem** | `file:///path/to/data` | Local dev, debugging |\n| **RedDB** | `reddb://TOKEN@host:8080/prefix` | Multi-structure (tables, graphs, vectors) |\n\n```javascript\n// Just change the connection string — everything else stays the same\nconst db = new Database({\n  connectionString: process.env.S3DB_CONNECTION_STRING\n})\n```\n\n\u003e For detailed pricing comparison and decision guidance, see [Choosing a Backend](./docs/guides/choosing-a-backend.md).\n\n---\n\n## 🗄️ Database\n\nA Database is a logical container for your resources, stored in a specific S3 bucket path. The database manages resource metadata, connections, and provides the core interface for all operations.\n\n### Configuration Parameters\n\n| Parameter | Type | Default | Description |\n|-----------|------|---------|-------------|\n| `connectionString` | `string` | **required** | S3 connection string (see formats below) |\n| `httpClientOptions` | `object` | optimized | HTTP client configuration for S3 requests |\n| `logLevel` | `boolean` | `false` | Enable debug logging for debugging |\n| `parallelism` | `number` | `100` | Concurrent operations for bulk operations (Separate Executor Pools per Database) |\n| `versioningEnabled` | `boolean` | `false` | Enable automatic resource versioning |\n| `passphrase` | `string` | `'secret'` | Default passphrase for field encryption |\n| `plugins` | `array` | `[]` | Array of plugin instances to extend functionality |\n\n### Connection Strings\n\ns3db.js supports multiple connection string formats for different S3 providers:\n\n```javascript\n// AWS S3 (with credentials)\n\"s3://ACCESS_KEY:SECRET_KEY@BUCKET_NAME/databases/myapp?region=us-east-1\"\n\n// AWS S3 (IAM role - recommended for production)\n\"s3://BUCKET_NAME/databases/myapp?region=us-east-1\"\n\n// MinIO (self-hosted)\n\"http://minioadmin:minioadmin@localhost:9000/bucket/databases/myapp\"\n\n// Digital Ocean Spaces\n\"https://SPACES_KEY:SPACES_SECRET@nyc3.digitaloceanspaces.com/SPACE_NAME/databases/myapp\"\n\n// LocalStack (local testing)\n\"http://test:test@localhost:4566/mybucket/databases/myapp\"\n\n// MemoryClient (ultra-fast in-memory testing - no S3 required!)\n\"memory://mybucket/databases/myapp\"\n\n// Backblaze B2\n\"https://KEY_ID:APPLICATION_KEY@s3.us-west-002.backblazeb2.com/BUCKET/databases/myapp\"\n\n// Cloudflare R2\n\"https://ACCESS_KEY:SECRET_KEY@ACCOUNT_ID.r2.cloudflarestorage.com/BUCKET/databases/myapp\"\n```\n\n\u003cdetails\u003e\n\u003csummary\u003e\u003cstrong\u003e🔑 Complete authentication examples\u003c/strong\u003e\u003c/summary\u003e\n\n#### 1. Access Keys (Development)\n```javascript\nconst s3db = new S3db({\n  connectionString: \"s3://ACCESS_KEY:SECRET_KEY@BUCKET_NAME/databases/myapp\"\n});\n```\n\n#### 2. IAM Roles (Production - Recommended)\n```javascript\n// No credentials needed - uses IAM role permissions\nconst s3db = new S3db({\n  connectionString: \"s3://BUCKET_NAME/databases/myapp\"\n});\n```\n\n#### 3. MinIO (Self-hosted S3)\n```javascript\n// MinIO running locally (note: http:// protocol and port)\nconst s3db = new S3db({\n  connectionString: \"http://minioadmin:minioadmin@localhost:9000/mybucket/databases/myapp\"\n});\n```\n\n#### 4. Digital Ocean Spaces (SaaS)\n```javascript\n// Digital Ocean Spaces (NYC3 datacenter)\nconst s3db = new S3db({\n  connectionString: \"https://SPACES_KEY:SPACES_SECRET@nyc3.digitaloceanspaces.com/SPACE_NAME/databases/myapp\"\n});\n```\n\n\u003c/details\u003e\n\n---\n\n### 🚀 MemoryClient - Ultra-Fast Testing (100-1000x faster!)\n\nFor testing, s3db.js provides **MemoryClient** - a pure in-memory implementation that's **100-1000x faster** than LocalStack and requires **zero dependencies**.\n\n**Why MemoryClient?**\n- ⚡ **100-1000x faster** than LocalStack/MinIO\n- 🎯 **Zero dependencies** - no Docker, LocalStack, or S3 needed\n- 💯 **100% compatible** - same API as S3Client\n- 🧪 **Perfect for tests** - instant setup and teardown\n- 💾 **Optional persistence** - save/load snapshots to disk\n\n**Quick Start with Connection String:**\n\n```javascript\nimport { S3db } from 's3db.js';\n\n// Simple - just use memory:// protocol!\nconst db = new S3db({\n  connectionString: 'memory://mybucket'\n});\n\nawait db.connect();\n```\n\n**Alternative - Manual Instantiation:**\n\n```javascript\nimport { S3db, MemoryClient } from 's3db.js';\n\n// Create database with MemoryClient\nconst db = new S3db({\n  client: new MemoryClient({ bucket: 'test-bucket' })\n});\n\nawait db.connect();\n\n// Use exactly like S3 - same API!\nconst users = await db.createResource({\n  name: 'users',\n  attributes: {\n    name: 'string|required',\n    email: 'email|required'\n  }\n});\n\nawait users.insert({ id: 'u1', name: 'John', email: 'john@test.com' });\nconst user = await users.get('u1');\n```\n\n**Connection String Options:**\n\n```javascript\n// Basic usage\n\"memory://mybucket\"\n\n// With key prefix (path)\n\"memory://mybucket/databases/myapp\"\n\n// With multiple path segments\n\"memory://testdb/level1/level2/level3\"\n\n// With query parameters\n\"memory://mybucket?region=us-west-2\"\n```\n\n**Advanced Features (Manual Client):**\n\n```javascript\nimport { S3db, MemoryClient } from 's3db.js';\n\n// Option 1: Connection string (recommended)\nconst db1 = new S3db({\n  connectionString: 'memory://test-bucket/tests/'\n});\n\n// Option 2: Manual client configuration\nconst db2 = new S3db({\n  client: new MemoryClient({\n    bucket: 'test-bucket',\n    keyPrefix: 'tests/',              // Optional prefix for all keys\n    enforceLimits: true,               // Enforce S3 2KB metadata limit\n    persistPath: './test-data.json',  // Optional: persist to disk\n    logLevel: 'silent'                     // Disable logging\n  })\n});\n\n// Snapshot/Restore (perfect for tests)\nconst snapshot = client.snapshot();        // Capture current state\n// ... run tests that modify data ...\nclient.restore(snapshot);                  // Restore to original state\n\n// Persistence\nawait client.saveToDisk();                 // Save to persistPath\nawait client.loadFromDisk();               // Load from persistPath\n\n// Statistics\nconst stats = client.getStats();\nconsole.log(`Objects: ${stats.objectCount}, Size: ${stats.totalSizeFormatted}`);\n\n// Clear all data\nclient.clear();\n```\n\n**Testing Example:**\n\n```javascript\nimport {\n  describe,\n  test,\n  beforeEach,\n  afterEach } from 'vitest';\nimport { S3db\n} from 's3db.js';\n\ndescribe('User Tests', () =\u003e {\n  let db, users, snapshot;\n\n  beforeEach(async () =\u003e {\n    // Simple connection string setup!\n    db = new S3db({\n      connectionString: 'memory://test-db/my-tests'\n    });\n    await db.connect();\n\n    users = await db.createResource({\n      name: 'users',\n      attributes: { name: 'string', email: 'email' }\n    });\n\n    // Save snapshot for each test\n    snapshot = db.client.snapshot();\n  });\n\n  afterEach(() =\u003e {\n    // Restore to clean state (faster than recreating)\n    db.client.restore(snapshot);\n  });\n\n  test('should insert user', async () =\u003e {\n    await users.insert({ id: 'u1', name: 'John', email: 'john@test.com' });\n    const user = await users.get('u1');\n    expect(user.name).toBe('John');\n  });\n});\n```\n\n**Illustrative Performance Comparison:**\n\n| Operation | LocalStack | MemoryClient | Speedup |\n|-----------|------------|--------------|---------|\n| Insert 100 records | ~2000ms | ~50ms | **40x faster** |\n| Query 1000 records | ~5000ms | ~100ms | **50x faster** |\n| Full test suite | ~120s | ~2s | **60x faster** |\n\nThese numbers are workload-dependent and should be treated as directional, not\nas a current benchmark contract.\n\n📚 [**Full MemoryClient Documentation**](./docs/clients/memory-client.md) • [**Core Memory Benchmark**](./docs/benchmarks/core-memory.md)\n\n---\n\n### 🗄️ SqliteClient - Local Persistent Storage\n\nFor workloads that need persistence without S3 infrastructure (single-process local dev,\nCI, migration drills), use `SqliteClient`.\n\n**Why SqliteClient?**\n- 🧰 **Simple local deployment** - file-based storage, no object storage setup\n- 📈 **Predictable performance** - consistent local latency\n- 🧪 **Reliable integration tests** - data persists between test steps\n- 🔐 **Memory budget controls** - `maxMemoryMB` to avoid OOM in heavy writes\n\n**Quick Start (Connection String):**\n\n```javascript\nimport { S3db } from 's3db.js';\n\nconst db = new S3db({\n  connectionString: 'sqlite:///tmp/s3db.sqlite'\n});\n\nawait db.connect();\n```\n\n**Example with options:**\n\n```javascript\nimport { S3db, SqliteClient } from 's3db.js';\n\nconst client = new SqliteClient({\n  basePath: '/tmp/s3db.sqlite',\n  bucket: 'myapp',\n  maxObjectSize: 5 * 1024 * 1024,\n  maxMemoryMB: 256,\n});\n\nconst db = new S3db({ client });\nawait db.connect();\n```\n\n### 🌐 Remote SQLite - Turso and Cloudflare D1\n\nFor remote SQLite-compatible backends, use the explicit remote connection string schemes:\n\n```javascript\nimport { Database } from 's3db.js';\n\nconst tursoDb = new Database({\n  connectionString: 'sqlite+libsql://my-db-my-org.turso.io?authToken=YOUR_TOKEN'\n});\n\nconst d1Db = new Database({\n  connectionString: 'sqlite+d1://ACCOUNT_ID/DATABASE_ID?apiToken=YOUR_TOKEN'\n});\n```\n\nNotes:\n- `sqlite://` remains local-only.\n- `sqlite+libsql://` uses the remote SQLite client over libsql/Turso.\n- `sqlite+d1://` uses the remote SQLite client over the Cloudflare D1 HTTP API.\n- `@libsql/client` is optional and only required when you use `sqlite+libsql://`.\n\nInstall the optional Turso dependency in your app:\n\n```bash\npnpm add @libsql/client\n```\n\n📚 [**Full SqliteClient Documentation**](./docs/clients/sqlite-client.md)\n\n---\n\n### S3 Bucket Structure\n\nWhen you create a database, s3db.js organizes your data in a structured way within your S3 bucket:\n\n```\nbucket-name/\n└── databases/\n    └── myapp/                                  # Database root (from connection string)\n        ├── s3db.json                           # Database metadata \u0026 resource definitions\n        │\n        ├── resource=users/                     # Resource: users\n        │   ├── data/\n        │   │   ├── id=user-123                 # Document (metadata in S3 metadata, optional body)\n        │   │   └── id=user-456\n        │   └── partition=byRegion/             # Partition: byRegion\n        │       ├── region=US/\n        │       │   ├── id=user-123             # Partition reference\n        │       │   └── id=user-789\n        │       └── region=EU/\n        │           └── id=user-456\n        │\n        ├── resource=posts/                     # Resource: posts\n        │   └── data/\n        │       ├── id=post-abc\n        │       └── id=post-def\n        │\n        ├── resource=sessions/                  # Resource: sessions (with TTL)\n        │   └── data/\n        │       ├── id=session-xyz\n        │       └── id=session-qwe\n        │\n        ├── plugin=cache/                       # Plugin: CachePlugin (global data)\n        │   ├── config                          # Plugin configuration\n        │   └── locks/\n        │       └── cache-cleanup               # Distributed lock\n        │\n        └── resource=wallets/                   # Resource: wallets\n            ├── data/\n            │   └── id=wallet-123\n            └── plugin=eventual-consistency/    # Plugin: scoped to resource\n                ├── balance/\n                │   └── transactions/\n                │       └── id=txn-123          # Plugin-specific data\n                └── locks/\n                    └── balance-sync            # Resource-scoped lock\n```\n\n**Key Path Patterns:**\n\n| Type | Pattern | Example |\n|------|---------|---------|\n| **Metadata** | `s3db.json` | Database schema, resources, versions |\n| **Document** | `resource={name}/data/id={id}` | `resource=users/data/id=user-123` |\n| **Partition** | `resource={name}/partition={partition}/{field}={value}/id={id}` | `resource=users/partition=byRegion/region=US/id=user-123` |\n| **Plugin (global)** | `plugin={slug}/{path}` | `plugin=cache/config` |\n| **Plugin (resource)** | `resource={name}/plugin={slug}/{path}` | `resource=wallets/plugin=eventual-consistency/balance/transactions/id=txn-123` |\n| **Lock (global)** | `plugin={slug}/locks/{lockName}` | `plugin=ttl/locks/cleanup` |\n| **Lock (resource)** | `resource={name}/plugin={slug}/locks/{lockName}` | `resource=wallets/plugin=eventual-consistency/locks/balance-sync` |\n\n**Storage Layers:**\n\n1. **Documents** - User data stored in resources\n   - Metadata: Stored in S3 object metadata (up to 2KB)\n   - Body: Large content stored in S3 object body (unlimited)\n\n2. **Partitions** - Organized references for O(1) queries\n   - Hierarchical paths with field values\n   - References point to main document\n\n3. **Plugin Storage** - Plugin-specific data\n   - **Global**: `plugin={slug}/...` - Shared config, caches, locks\n   - **Resource-scoped**: `resource={name}/plugin={slug}/...` - Per-resource data\n   - Supports same behaviors as resources (body-overflow, body-only, etc.)\n   - 3-5x faster than creating full resources\n   - Examples: EventualConsistency transactions, TTL expiration queues, Cache entries, Audit logs\n\n**Why This Structure?**\n\n- ✅ **Flat hierarchy** - No deep nesting, better S3 performance\n- ✅ **Self-documenting** - Path tells you what data it contains\n- ✅ **Partition-friendly** - O(1) lookups via S3 prefix queries\n- ✅ **Plugin isolation** - Each plugin has its own namespace\n- ✅ **Consistent naming** - `resource=`, `partition=`, `plugin=`, `id=` prefixes\n\n### Creating a Database\n\n```javascript\nimport { S3db } from 's3db.js';\n\n// Simple connection\nconst db = new S3db({\n  connectionString: 's3://ACCESS_KEY:SECRET@bucket/databases/myapp'\n});\n\nawait db.connect();\n\n// With plugins and options\nconst db = new S3db({\n  connectionString: 's3://bucket/databases/myapp',\n  logLevel: 'debug',\n  versioningEnabled: true,\n  executorPool: {\n    concurrency: 100,  // Default concurrency (can increase for high-throughput)\n    retries: 3,\n    retryDelay: 1000\n  },\n  taskExecutorMonitoring: {\n    enabled: true,\n    collectMetrics: true,\n    sampleRate: 0.2\n  },\n  plugins: [\n    new CachePlugin({ ttl: 300000 }),\n    new MetricsPlugin()\n  ],\n  httpClientOptions: {\n    keepAlive: true,\n    maxSockets: 100,\n    timeout: 60000\n  }\n});\n\nawait db.connect();\n```\n\n### Database Methods\n\n| Method | Description |\n|--------|-------------|\n| `connect()` | Initialize database connection and load metadata |\n| `createResource(config)` | Create or update a resource |\n| `getResource(name, options?)` | Get existing resource instance |\n| `resourceExists(name)` | Check if resource exists |\n| `resources.{name}` | Access resource by property |\n| `uploadMetadataFile()` | Save metadata changes to S3 |\n\n### HTTP Client Configuration\n\nCustomize HTTP performance for your workload:\n\n```javascript\nconst db = new S3db({\n  connectionString: '...',\n  httpClientOptions: {\n    keepAlive: true,          // Enable connection reuse\n    keepAliveMsecs: 1000,     // Keep connections alive for 1s\n    maxSockets: 50,           // Max 50 concurrent connections\n    maxFreeSockets: 10,       // Keep 10 free connections in pool\n    timeout: 60000            // 60 second timeout\n  }\n});\n```\n\n**Presets:**\n\n\u003cdetails\u003e\n\u003csummary\u003e\u003cstrong\u003eHigh Concurrency (APIs)\u003c/strong\u003e\u003c/summary\u003e\n\n```javascript\nhttpClientOptions: {\n  keepAlive: true,\n  keepAliveMsecs: 1000,\n  maxSockets: 100,         // Higher concurrency\n  maxFreeSockets: 20,      // More free connections\n  timeout: 60000\n}\n```\n\u003c/details\u003e\n\n\u003cdetails\u003e\n\u003csummary\u003e\u003cstrong\u003eAggressive Performance (High-throughput)\u003c/strong\u003e\u003c/summary\u003e\n\n```javascript\nhttpClientOptions: {\n  keepAlive: true,\n  keepAliveMsecs: 5000,    // Longer keep-alive\n  maxSockets: 200,         // High concurrency\n  maxFreeSockets: 50,      // Large connection pool\n  timeout: 120000          // 2 minute timeout\n}\n```\n\u003c/details\u003e\n\n**Complete documentation**: See above for all Database configuration options\n\n---\n\n## 🔀 DatabaseManager — Multiple Backends\n\nWhen you need resources across different storage backends (S3 + RedDB + SQLite, etc.), `DatabaseManager` orchestrates multiple `Database` instances with a unified API.\n\n```javascript\nimport { DatabaseManager } from 's3db.js'\n\nconst manager = new DatabaseManager({\n  // Shared config — applied to all connections\n  defaults: { logLevel: 'info', security: { passphrase: 'my-secret' } },\n  // Per-connection config (plugins go here, not in defaults)\n  connections: {\n    primary: {\n      connectionString: 's3://KEY:SECRET@bucket?region=us-east-1',\n      plugins: [new CachePlugin(), new MetricsPlugin()],\n    },\n    analytics: {\n      connectionString: 'reddb://localhost:8080',\n      plugins: [new CostsPlugin()],\n    },\n    cache: { connectionString: 'sqlite:///tmp/cache.db' },\n  },\n  default: 'primary',\n})\n\nawait manager.connect()  // connects all in parallel\n\n// Create resources — `connection` field specifies which backend\nconst users = await manager.createResource({\n  name: 'users',\n  connection: 'primary',\n  attributes: { email: 'email|required', name: 'string' },\n})\n\nconst events = await manager.createResource({\n  name: 'events',\n  connection: 'analytics',\n  attributes: { type: 'string', data: 'json' },\n})\n\n// Without `connection` — uses default\nconst settings = await manager.createResource({\n  name: 'settings',\n  attributes: { key: 'string', value: 'json' },\n})\n\n// Unified resource lookup across all connections\nmanager.resource('users')                    // from primary\nmanager.resource('events')                   // from analytics\nmanager.resources                            // merged view of all\n\n// Direct database access\nmanager.connection('analytics')              // Database instance\nmanager.getConnectionForResource('events')   // 'analytics'\n\nawait manager.disconnect()\n```\n\n**Key points:**\n- `defaults` sets shared config (logLevel, security, etc.) applied to all connections — connection values override\n- `plugins` are always per-connection (excluded from defaults) — each backend has different needs\n- Resource names must be globally unique across connections\n- Each `Database` has its own metadata, plugins, and client — full isolation\n- Events are forwarded with connection prefix: `manager.on('analytics:db:resource-created', ...)`\n\n\u003e **When to use:** Use `Database` when all resources live on the same backend. Use `DatabaseManager` when you need resources on different backends.\n\n\u003e **Full documentation:** See [DatabaseManager docs](./docs/core/database-manager.md)\n\n---\n\n## 🪵 Logging\n\ns3db.js uses **[Pino](https://getpino.io)** - a blazing-fast, low-overhead JSON logger (5-10x faster than console.*). The logging system is hierarchical: Database → Plugins → Resources automatically inherit log levels, with per-component override capabilities.\n\n### Quick Start - Automatic Inheritance\n\nAll components (Database, Plugins, Resources) automatically inherit the global log level:\n\n```javascript\nconst db = new Database({\n  connectionString: 's3://bucket/db',\n  loggerOptions: {\n    level: 'warn'  // ← Database, Resources, and Plugins all inherit 'warn'\n  }\n});\n\nawait db.usePlugin(new CachePlugin(), 'cache');  // Inherits 'warn'\nawait db.usePlugin(new TTLPlugin(), 'ttl');      // Inherits 'warn'\n```\n\n### Format Presets\n\ns3db.js provides two built-in format presets for different environments:\n\n**JSON Format** (Production - Structured Logs):\n```javascript\nconst db = new Database({\n  connectionString: 's3://bucket/db',\n  loggerOptions: {\n    level: 'info',\n    format: 'json'  // ← Compact JSON for log aggregation\n  }\n});\n\n// Output: {\"level\":30,\"time\":1234567890,\"msg\":\"User created\",\"userId\":\"123\"}\n```\n\n**Pretty Format** (Development - Human Readable):\n```javascript\nconst db = new Database({\n  connectionString: 's3://bucket/db',\n  loggerOptions: {\n    level: 'debug',\n    format: 'pretty'  // ← Colorized, readable output\n  }\n});\n\n// Output: [14:23:45.123] INFO: User created\n//           userId: \"123\"\n```\n\n**Auto-Detection** (Default):\n```javascript\n// Automatically chooses format based on:\n// - TTY detection (terminal vs piped)\n// - NODE_ENV (development vs production)\nconst db = new Database({\n  connectionString: 's3://bucket/db',\n  loggerOptions: {\n    level: 'info'\n    // format is auto-detected\n  }\n});\n```\n\n### Custom Error Serialization\n\ns3db.js errors automatically use `toJSON()` for structured logging:\n\n```javascript\nimport { ValidationError } from 's3db.js';\n\nconst error = new ValidationError('Invalid email', {\n  field: 'email',\n  value: 'invalid@',\n  statusCode: 422\n});\n\n// Logs include full error context automatically\nlogger.error({ err: error }, 'Validation failed');\n\n// Output includes: name, message, code, statusCode, suggestion, stack, etc.\n```\n\n### Per-Component Override\n\nFine-tune log levels for specific plugins or resources using `childLevels`:\n\n```javascript\nconst db = new Database({\n  connectionString: 's3://bucket/db',\n  loggerOptions: {\n    level: 'warn',  // ← Global default\n\n    childLevels: {\n      // Override specific plugins\n      'Plugin:cache': 'debug',      // Cache plugin in debug mode\n      'Plugin:ttl': 'trace',         // TTL plugin in trace mode\n      'Plugin:metrics': 'error',     // Metrics plugin only shows errors\n      'Plugin:s3-queue': 'info',     // S3Queue plugin in info mode\n\n      // Override specific resources\n      'Resource:users': 'debug',     // Users resource in debug\n      'Resource:logs': 'silent'      // Logs resource silenced\n    }\n  }\n});\n```\n\n**Result:**\n- Database → `warn`\n- CachePlugin → `debug` (override)\n- TTLPlugin → `trace` (override)\n- MetricsPlugin → `error` (override)\n- All other plugins → `warn` (inherited)\n\n### Custom Logger (No Inheritance)\n\nPlugins can use completely custom loggers that don't inherit from Database:\n\n```javascript\nimport { createLogger } from 's3db.js/logger';\n\n// Create custom logger\nconst customLogger = createLogger({\n  name: 'MyApp',\n  level: 'trace',\n  // Pino options\n  transport: {\n    target: 'pino-pretty',\n    options: { colorize: true }\n  }\n});\n\n// Plugin uses custom logger instead of inheriting\nconst plugin = new CachePlugin({\n  logger: customLogger  // ← Ignores inheritance\n});\n\nawait db.usePlugin(plugin, 'cache');\n```\n\n### Runtime Log Level Changes\n\nChange log levels on the fly for specific components:\n\n```javascript\n// Increase verbosity for debugging\ndb.setChildLevel('Plugin:cache', 'debug');\n\n// Silence a noisy plugin\ndb.setChildLevel('Plugin:ttl', 'silent');\n\n// Debug specific resource\ndb.setChildLevel('Resource:clicks', 'trace');\n```\n\n**⚠️ Limitation:** `setChildLevel()` only affects **new child loggers**. Loggers already created maintain their previous level.\n\n### Environment Variables\n\nOverride logging globally using environment variables:\n\n```bash\n# Set log level\nS3DB_LOG_LEVEL=debug node app.js\n\n# Set output format (using presets)\nS3DB_LOG_FORMAT=pretty node app.js  # Pretty format (colorized, human-readable)\nS3DB_LOG_FORMAT=json node app.js    # JSON format (structured logs for production)\n\n# Combined example\nS3DB_LOG_LEVEL=debug S3DB_LOG_FORMAT=pretty node app.js\n```\n\n**Legacy Support:** The old `S3DB_LOG_PRETTY` environment variable is still supported for backward compatibility:\n```bash\nS3DB_LOG_PRETTY=true node app.js   # Same as S3DB_LOG_FORMAT=pretty\nS3DB_LOG_PRETTY=false node app.js  # Same as S3DB_LOG_FORMAT=json\n```\n\n### Available Log Levels\n\n| Level | Use Case | When to Use |\n|-------|----------|-------------|\n| `silent` | No logs | Tests, silent components |\n| `fatal` | Critical errors | System unusable |\n| `error` | Errors | Failed operations |\n| `warn` | Warnings | Deprecations, fallbacks |\n| `info` | Information | **Default for production** |\n| `debug` | Debug | Development |\n| `trace` | Full trace | Deep debugging |\n\n### Practical Examples\n\n#### Production (Minimal Logs)\n\n```javascript\nconst db = new Database({\n  connectionString: process.env.S3DB_CONNECTION,\n  loggerOptions: {\n    level: 'warn',\n    format: 'json',  // ← Structured logs for aggregation\n\n    childLevels: {\n      // Info-level logging only for critical plugins\n      'Plugin:metrics': 'info',\n      'Plugin:audit': 'info'\n    }\n  }\n});\n```\n\n#### Development (Verbose)\n\n```javascript\nconst db = new Database({\n  connectionString: 'http://localhost:9000/bucket',\n  loggerOptions: {\n    level: 'debug',\n    format: 'pretty',  // ← Human-readable, colorized\n\n    childLevels: {\n      // Trace the specific plugin you're debugging\n      'Plugin:cache': 'trace',\n\n      // Silence noisy plugins\n      'Plugin:metrics': 'silent'\n    }\n  }\n});\n```\n\n#### Debug Specific Plugin\n\n```javascript\nconst db = new Database({\n  connectionString: 's3://bucket/db',\n  loggerOptions: {\n    level: 'warn',\n    format: 'json',  // ← Production format\n\n    childLevels: {\n      // Debug ONLY the TTL plugin\n      'Plugin:ttl': 'trace'\n    }\n  }\n});\n```\n\n### Discovering Child Logger Names\n\n**Plugins:** Format is `Plugin:{name}`\n\n```javascript\nawait db.usePlugin(new CachePlugin(), 'cache');\n// Child logger: 'Plugin:cache'\n\nawait db.usePlugin(new TTLPlugin(), 'my-ttl');\n// Child logger: 'Plugin:my-ttl'\n```\n\n**Resources:** Format is `Resource:{name}`\n\n```javascript\nawait db.createResource({ name: 'users', ... });\n// Child logger: 'Resource:users'\n```\n\n### HTTP Request Logging (API Plugin)\n\nThe API Plugin includes automatic HTTP request/response logging with smart detection:\n\n**Smart Detection:**\n- **If `pino-http` is installed:** Uses full-featured pino-http with all bells and whistles\n- **If `pino-http` is NOT installed:** Falls back to simple built-in HTTP logging\n\n**Installation (optional, recommended):**\n```bash\nnpm install pino-http\n```\n\n**Usage:**\n```javascript\nimport { APIPlugin } from 's3db.js';\n\nconst api = new APIPlugin({\n  port: 3000,\n\n  // Enable HTTP logging (works with or without pino-http!)\n  httpLogger: {\n    enabled: true,\n    autoLogging: true,              // Log all requests/responses\n    ignorePaths: ['/health'],       // Skip logging for these paths\n\n    // Custom log level based on status code\n    customLogLevel: (req, res, err) =\u003e {\n      if (err || res.statusCode \u003e= 500) return 'error';\n      if (res.statusCode \u003e= 400) return 'warn';\n      return 'info';\n    }\n  },\n\n  // Enable request ID tracking (recommended)\n  requestId: {\n    enabled: true,\n    headerName: 'X-Request-ID'\n  }\n});\n```\n\n**What you get:**\n\n| Feature | With pino-http | Without pino-http |\n|---------|---------------|-------------------|\n| Request logging | ✅ Full | ✅ Basic |\n| Response logging | ✅ Full | ✅ Basic |\n| Error logging | ✅ Full | ✅ Basic |\n| Request ID | ✅ Auto | ✅ Manual |\n| Custom serializers | ✅ Yes | ✅ Basic |\n| Performance overhead | ⚡ Minimal | ⚡ Minimal |\n\n**No installation required!** HTTP logging works out-of-the-box with basic features. Install `pino-http` for enhanced capabilities.\n\n**Automatic Logging Output:**\n```json\n{\n  \"level\": 30,\n  \"time\": 1234567890,\n  \"req\": {\n    \"id\": \"abc123\",\n    \"method\": \"POST\",\n    \"url\": \"/users\",\n    \"headers\": { \"user-agent\": \"...\", \"content-type\": \"application/json\" }\n  },\n  \"res\": {\n    \"statusCode\": 201,\n    \"headers\": { \"content-type\": \"application/json\" }\n  },\n  \"responseTime\": 45,\n  \"msg\": \"request completed\"\n}\n```\n\n**Features:**\n- Request/response correlation with request IDs\n- Automatic status code-based log levels\n- Error serialization with `toJSON()`\n- Path filtering (e.g., skip `/health`, `/metrics`)\n- Zero configuration required\n\n### Tips \u0026 Best Practices\n\n1. **Production**: Use `format: 'json'` with `level: 'warn'` for structured logging\n2. **Development**: Use `format: 'pretty'` with `level: 'debug'` for readability\n3. **Debugging**: Use `childLevels` to isolate specific components\n4. **Performance**: Lower levels (`trace`, `debug`) have performance impact\n5. **Inheritance**: Components automatically inherit global level if not overridden\n6. **Error Logging**: Custom errors automatically use `toJSON()` for rich context\n7. **CI/CD**: Use `format: 'json'` in automated environments for parsing\n8. **HTTP Logging**: Enable `httpLogger` in API Plugin for automatic request tracking\n\n---\n\n## 📋 Resources\n\nResources are the core abstraction in s3db.js - they define your data structure, validation rules, and behavior. Think of them as tables in traditional databases, but with much more flexibility and features.\n\n### TL;DR\n\nResources provide:\n- ✅ **Schema validation** with 30+ field types\n- ✅ **5 behavior strategies** for handling 2KB S3 metadata limit\n- ✅ **Partitioning** for O(1) queries vs O(n) scans\n- ✅ **Hooks \u0026 middlewares** for custom logic\n- ✅ **Events** for real-time notifications\n- ✅ **Versioning** for schema evolution\n- ✅ **Encryption** for sensitive fields\n- ✅ **Streaming** for large datasets\n\n**Quick example:**\n```javascript\nconst users = await db.createResource({\n  name: 'users',\n  attributes: {\n    email: 'email|required|unique',\n    password: 'secret|required',\n    age: 'number|min:18|max:120'\n  },\n  behavior: 'enforce-limits',\n  timestamps: true,\n  partitions: {\n    byAge: { fields: { age: 'number' } }\n  }\n});\n\nawait users.insert({ email: 'john@example.com', password: 'secret123', age: 25 });\n```\n\n### Schema \u0026 Field Types\n\nDefine your data structure with powerful validation using **[fastest-validator](https://github.com/icebob/fastest-validator)** - a blazing-fast validation library with comprehensive type support:\n\n#### Basic Types\n\n| Type | Example | Validation Rules |\n|------|---------|------------------|\n| `string` | `\"name: 'string\\|required'\"` | `min`, `max`, `length`, `pattern`, `enum` |\n| `number` | `\"age: 'number\\|min:0'\"` | `min`, `max`, `integer`, `positive`, `negative` |\n| `boolean` | `\"isActive: 'boolean'\"` | `true`, `false` |\n| `email` | `\"email: 'email\\|required'\"` | RFC 5322 validation |\n| `url` | `\"website: 'url'\"` | Valid URL format |\n| `date` | `\"createdAt: 'date'\"` | ISO 8601 dates |\n| `array` | `\"tags: 'array\\|items:string'\"` | `items`, `min`, `max`, `unique` |\n| `object` | `\"profile: { type: 'object', props: {...} }\"` | Nested validation |\n\n#### Advanced Types (with encoding)\n\n| Type | Savings | Example |\n|------|---------|---------|\n| `secret` | Encrypted | `\"password: 'secret\\|required'\"` - AES-256-GCM |\n| `embedding:N` | 77% | `\"vector: 'embedding:1536'\"` - Fixed-point Base62 |\n| `ip4` | 47% | `\"ipAddress: 'ip4'\"` - Binary Base64 |\n| `ip6` | 44% | `\"ipv6: 'ip6'\"` - Binary Base64 |\n\n**Encoding optimizations:**\n- ISO timestamps → Unix Base62 (67% savings)\n- UUIDs → Binary Base64 (33% savings)\n- Dictionary values → Single bytes (95% savings)\n\n#### Schema Examples\n\n\u003e **📖 Validation powered by [fastest-validator](https://github.com/icebob/fastest-validator)**\n\u003e All schemas use fastest-validator's syntax with full support for shorthand notation.\n\n```javascript\n// Simple schema\n{\n  name: 'string|required|min:2|max:100',\n  email: 'email|required|unique',\n  age: 'number|integer|min:0|max:150'\n}\n\n// Nested objects - MAGIC AUTO-DETECT! ✨ (recommended)\n// Just write your object structure - s3db detects it automatically!\n{\n  name: 'string|required',\n  profile: {               // ← No $$type needed! Auto-detected as optional object\n    bio: 'string|max:500',\n    avatar: 'url|optional',\n    social: {              // ← Deeply nested also works!\n      twitter: 'string|optional',\n      github: 'string|optional'\n    }\n  }\n}\n\n// Need validation control? Use $$type (when you need required/optional)\n{\n  name: 'string|required',\n  profile: {\n    $$type: 'object|required',  // ← Add required validation\n    bio: 'string|max:500',\n    avatar: 'url|optional'\n  }\n}\n\n// Advanced: Full control (rare cases - strict mode, etc)\n{\n  name: 'string|required',\n  profile: {\n    type: 'object',\n    optional: false,\n    strict: true,        // ← Enable strict validation\n    props: {\n      bio: 'string|max:500',\n      avatar: 'url|optional'\n    }\n  }\n}\n\n// Arrays with validation\n{\n  name: 'string|required',\n  tags: 'array|items:string|min:1|max:10|unique',\n  scores: 'array|items:number|min:0|max:100'\n}\n\n// Encrypted fields\n{\n  email: 'email|required',\n  password: 'secret|required',\n  apiKey: 'secret|required'\n}\n```\n\n### Behaviors (Handling 2KB Metadata Limit)\n\nS3 metadata has a 2KB limit. Behaviors define how to handle data that exceeds this:\n\n| Behavior | Enforcement | Data Loss | Use Case |\n|----------|-------------|-----------|----------|\n| `user-managed` | None | Possible | Dev/Test - warnings only |\n| `enforce-limits` | Strict | No | Production - throws errors |\n| `truncate-data` | Truncates | Yes | Content management - smart truncation |\n| `body-overflow` | Splits | No | Mixed data - metadata + body |\n| `body-only` | Unlimited | No | Large docs - everything in body |\n\n```javascript\n// Enforce limits (recommended for production)\nconst users = await db.createResource({\n  name: 'users',\n  behavior: 'enforce-limits',\n  attributes: { name: 'string', bio: 'string' }\n});\n\n// Body overflow for large content\nconst blogs = await db.createResource({\n  name: 'blogs',\n  behavior: 'body-overflow',\n  attributes: { title: 'string', content: 'string' }\n});\n\n// Body-only for documents\nconst documents = await db.createResource({\n  name: 'documents',\n  behavior: 'body-only',\n  attributes: { title: 'string', content: 'string', metadata: 'object' }\n});\n```\n\n### Resource Methods\n\n#### CRUD Operations\n\n```javascript\n// Create\nconst user = await users.insert({ name: 'John', email: 'john@example.com' });\n\n// Read\nconst user = await users.get('user-123');\nconst all = await users.list({ limit: 10, offset: 0 });\nconst filtered = await users.query({ isActive: true });\n\n// Update (3 methods with different performance)\nawait users.update(id, { name: 'Jane' });      // GET+PUT merge (baseline)\nawait users.patch(id, { name: 'Jane' });       // HEAD+COPY (40-60% faster*)\nawait users.replace(id, fullObject);           // PUT only (30-40% faster)\n// *patch() uses HEAD+COPY for metadata-only behaviors\n\n// Delete\nawait users.delete('user-123');\n```\n\n#### Bulk Operations\n\n```javascript\n// Bulk insert\nawait users.insertMany([\n  { name: 'User 1', email: 'user1@example.com' },\n  { name: 'User 2', email: 'user2@example.com' }\n]);\n\n// Bulk get\nconst data = await users.getMany(['user-1', 'user-2', 'user-3']);\n\n// Bulk delete\nawait users.deleteMany(['user-1', 'user-2']);\n```\n\n### Partitions (O(1) Queries)\n\nOrganize data for fast queries without scanning:\n\n```javascript\nconst analytics = await db.createResource({\n  name: 'analytics',\n  attributes: {\n    userId: 'string',\n    event: 'string',\n    timestamp: 'date',\n    region: 'string'\n  },\n  partitions: {\n    // Single field\n    byEvent: { fields: { event: 'string' } },\n\n    // Multiple fields (composite)\n    byEventAndRegion: {\n      fields: {\n        event: 'string',\n        region: 'string'\n      }\n    },\n\n    // Nested field\n    byUserCountry: {\n      fields: {\n        'profile.country': 'string'\n      }\n    }\n  },\n\n  // Async partitions for 70-100% faster writes\n  asyncPartitions: true\n});\n\n// Query by partition (O(1))\nconst usEvents = await analytics.list({\n  partition: 'byEventAndRegion',\n  partitionValues: { event: 'click', region: 'US' }\n});\n```\n\n**Automatic timestamp partitions:**\n\n```javascript\nconst events = await db.createResource({\n  name: 'events',\n  attributes: { name: 'string', data: 'object' },\n  timestamps: true  // Auto-creates byCreatedDate and byUpdatedDate partitions\n});\n\nconst todayEvents = await events.list({\n  partition: 'byCreatedDate',\n  partitionValues: { createdAt: '2024-01-15' }\n});\n```\n\n### Hooks (Lifecycle Functions)\n\nAdd custom logic before/after operations:\n\n```javascript\nconst products = await db.createResource({\n  name: 'products',\n  attributes: { name: 'string', price: 'number', sku: 'string' },\n  hooks: {\n    // Before operations\n    beforeInsert: [\n      async (data) =\u003e {\n        data.sku = `PROD-${Date.now()}`;\n        return data;\n      }\n    ],\n    beforeUpdate: [\n      async (data) =\u003e {\n        data.updatedAt = new Date().toISOString();\n        return data;\n      }\n    ],\n\n    // After operations\n    afterInsert: [\n      async (data) =\u003e {\n        console.log(`Product ${data.name} created with SKU ${data.sku}`);\n      }\n    ],\n    afterDelete: [\n      async (data) =\u003e {\n        await notifyWarehouse(data.sku);\n      }\n    ]\n  }\n});\n```\n\n**Available hooks:**\n- `beforeInsert`, `afterInsert`\n- `beforeUpdate`, `afterUpdate`\n- `beforeDelete`, `afterDelete`\n- `beforeGet`, `afterGet`\n- `beforeList`, `afterList`\n\n### Middlewares (Method Wrappers)\n\nIntercept and transform method calls:\n\n```javascript\n// Authentication middleware\nusers.useMiddleware('inserted', async (ctx, next) =\u003e {\n  if (!ctx.args[0].userId) {\n    throw new Error('Authentication required');\n  }\n  return await next();\n});\n\n// Logging middleware\nusers.useMiddleware('updated', async (ctx, next) =\u003e {\n  const start = Date.now();\n  const result = await next();\n  console.log(`Update took ${Date.now() - start}ms`);\n  return result;\n});\n\n// Validation middleware\nusers.useMiddleware('inserted', async (ctx, next) =\u003e {\n  ctx.args[0].name = ctx.args[0].name.toUpperCase();\n  return await next();\n});\n```\n\n**Supported methods:**\n`fetched`, `list`, `inserted`, `updated`, `deleted`, `deleteMany`, `exists`, `getMany`, `count`, `page`, `listIds`, `getAll`\n\n### Events (Real-time Notifications)\n\nListen to resource operations:\n\n```javascript\nconst users = await db.createResource({\n  name: 'users',\n  attributes: { name: 'string', email: 'string' },\n\n  // Declarative event listeners\n  events: {\n    insert: (event) =\u003e {\n      console.log('User created:', event.id, event.name);\n    },\n\n    update: [\n      (event) =\u003e console.log('Update detected:', event.id),\n      (event) =\u003e {\n        if (event.$before.email !== event.$after.email) {\n          console.log('Email changed!');\n        }\n      }\n    ],\n\n    delete: (event) =\u003e {\n      console.log('User deleted:', event.id);\n    }\n  }\n});\n\n// Programmatic listeners\nusers.on('inserted', (event) =\u003e {\n  sendWelcomeEmail(event.email);\n});\n```\n\n**Available events:**\n`inserted`, `updated`, `deleted`, `insertMany`, `deleteMany`, `list`, `count`, `fetched`, `getMany`\n\n### Streaming API\n\nProcess large datasets efficiently:\n\n```javascript\n// Readable stream\nconst readableStream = await users.readable({\n  batchSize: 50,\n  concurrency: 10\n});\n\nreadableStream.on('data', (user) =\u003e {\n  console.log('Processing:', user.name);\n});\n\nreadableStream.on('end', () =\u003e {\n  console.log('Stream completed');\n});\n\n// Writable stream\nconst writableStream = await users.writable({\n  batchSize: 25,\n  concurrency: 5\n});\n\nuserData.forEach(user =\u003e writableStream.write(user));\nwritableStream.end();\n```\n\n### Complete Resource Example\n\nA complex, production-ready resource showing all capabilities:\n\n```javascript\nconst orders = await db.createResource({\n  name: 'orders',\n\n  // Schema with all features\n  attributes: {\n    // Basic fields\n    orderId: 'string|required|unique',\n    userId: 'string|required',\n    status: 'string|required|enum:pending,processing,completed,cancelled',\n    total: 'number|required|min:0',\n\n    // Encrypted sensitive data\n    paymentToken: 'secret|required',\n\n    // Nested objects\n    customer: {\n      type: 'object',\n      props: {\n        name: 'string|required',\n        email: 'email|required',\n        phone: 'string|optional',\n        address: {\n          type: 'object',\n          props: {\n            street: 'string|required',\n            city: 'string|required',\n            country: 'string|required|length:2',\n            zipCode: 'string|required'\n          }\n        }\n      }\n    },\n\n    // Arrays\n    items: 'array|items:object|min:1',\n    tags: 'array|items:string|unique|optional',\n\n    // Special types\n    ipAddress: 'ip4',\n    userAgent: 'string|max:500',\n\n    // Embeddings for AI/ML\n    orderEmbedding: 'embedding:384'\n  },\n\n  // Behavior for large orders\n  behavior: 'body-overflow',\n\n  // Automatic timestamps\n  timestamps: true,\n\n  // Versioning for schema evolution\n  versioningEnabled: true,\n\n  // Custom ID generation\n  idGenerator: () =\u003e `ORD-${Date.now()}-${Math.random().toString(36).substr(2, 5)}`,\n\n  // Partitions for efficient queries\n  partitions: {\n    byStatus: { fields: { status: 'string' } },\n    byUser: { fields: { userId: 'string' } },\n    byCountry: { fields: { 'customer.address.country': 'string' } },\n    byUserAndStatus: {\n      fields: {\n        userId: 'string',\n        status: 'string'\n      }\n    }\n  },\n\n  // Async partitions for faster writes\n  asyncPartitions: true,\n\n  // Hooks for business logic\n  hooks: {\n    beforeInsert: [\n      async function(data) {\n        // Validate stock availability\n        const available = await this.validateStock(data.items);\n        if (!available) throw new Error('Insufficient stock');\n\n        // Calculate total\n        data.total = data.items.reduce((sum, item) =\u003e sum + item.price * item.quantity, 0);\n\n        return data;\n      },\n      async (data) =\u003e {\n        // Add metadata\n        data.processedAt = new Date().toISOString();\n        return data;\n      }\n    ],\n\n    afterInsert: [\n      async (data) =\u003e {\n        // Send confirmation email\n        await sendOrderConfirmation(data.customer.email, data.orderId);\n      },\n      async (data) =\u003e {\n        // Update inventory\n        await updateInventory(data.items);\n      }\n    ],\n\n    beforeUpdate: [\n      async function(data) {\n        // Prevent status rollback\n        if (data.status === 'cancelled' \u0026\u0026 this.previousStatus === 'completed') {\n          throw new Error('Cannot cancel completed order');\n        }\n        return data;\n      }\n    ],\n\n    afterUpdate: [\n      async (data) =\u003e {\n        // Notify customer of status change\n        if (data.$before.status !== data.$after.status) {\n          await notifyStatusChange(data.customer.email, data.status);\n        }\n      }\n    ]\n  },\n\n  // Events for monitoring\n  events: {\n    insert: (event) =\u003e {\n      console.log(`Order ${event.orderId} created - Total: $${event.total}`);\n      metrics.increment('orders.created');\n    },\n\n    update: [\n      (event) =\u003e {\n        if (event.$before.status !== event.$after.status) {\n          console.log(`Order ${event.orderId}: ${event.$before.status} → ${event.$after.status}`);\n          metrics.increment(`orders.status.${event.$after.status}`);\n        }\n      }\n    ],\n\n    delete: (event) =\u003e {\n      console.warn(`Order ${event.orderId} deleted`);\n      metrics.increment('orders.deleted');\n    }\n  }\n});\n\n// Add middlewares for cross-cutting concerns\norders.useMiddleware('inserted', async (ctx, next) =\u003e {\n  // Rate limiting\n  await checkRateLimit(ctx.args[0].userId);\n  return await next();\n});\n\norders.useMiddleware('updated', async (ctx, next) =\u003e {\n  // Audit logging\n  const start = Date.now();\n  const result = await next();\n  await auditLog.write({\n    action: 'order.update',\n    orderId: ctx.args[0],\n    duration: Date.now() - start,\n    timestamp: new Date()\n  });\n  return result;\n});\n```\n\n**Complete documentation**: [**docs/core/resource.md**](./docs/core/resource.md)\n\n---\n\n## ⚡ Performance \u0026 Concurrency\n\ns3db.js features **Separate Executor Pools** - a revolutionary architecture where each Database instance gets its own independent executor pool for maximum efficiency and zero contention.\n\n### Separate Pools Architecture (NEW!)\n\nEach database instance gets **its own executor pool**, enabling:\n\n- **🚀 40-50% faster** at medium scale (5,000+ operations)\n- **📈 13x less memory** at large scale (10,000+ operations)\n- **⏱️ Zero contention** between concurrent databases\n- **🛡️ Auto-retry** with exponential backoff\n- **🧠 Adaptive tuning** - automatically adjusts concurrency based on performance\n- **Default parallelism: 100** (up from 10, optimized for S3 throughput)\n\n### Quick Start\n\nExecutor pool is **enabled by default** with optimized settings:\n\n```javascript\nimport { Database } from 's3db.js';const db = new Database({\n  connectionString: 's3://bucket/database'\n  // That's it! Executor pool is automatically configured with:\n  // - Separate pool per database (zero contention)\n  // - Concurrency: 100 (default)\n  // - Auto-retry with exponential backoff\n  // - Priority queue for important operations\n  // - Real-time metrics\n})\n\nawait db.connect()\n```\n\n### Profiles, Monitoring \u0026 Auto-Tuning\n\nExecutor pools (and the standalone `TasksRunner`/`TasksPool`) support lightweight vs full-featured schedulers, observability exports, and adaptive concurrency:\n\n```javascript\nconst db = new Database({\n  connectionString: 's3://bucket/database',\n  executorPool: {\n    features: { profile: 'light', emitEvents: false }, // or 'balanced'\n    monitoring: {\n      enabled: true,\n      reportInterval: 1000,\n      exporter: (snapshot) =\u003e console.log('[executor]', snapshot)\n    },\n    autoTuning: {\n      enabled: true,\n      minConcurrency: 10,\n      maxConcurrency: 200,\n      targetLatency: 250,\n      adjustmentInterval: 5000\n    }\n  }\n})\n```\n\nUse the light profile for PromisePool-style throughput when you just need FIFO fan-out. Switch to balanced when you need retries, priority aging, rich metrics, or adaptive scaling. The same options apply to filesystem/memory clients via `taskExecutorMonitoring`, `autoTuning`, and `features.profile`.\n\n### Adaptive Tuning (Optional)\n\nCustomize concurrency for your specific workload:\n\n```javascript\nimport { Database } from 's3db.js';const db = new Database({\n  connectionString: 's3://bucket/database',\n  executorPool: {\n    concurrency: 200,         // Increase for high-throughput scenarios\n    // Or use auto-tuning:\n    // concurrency: 'auto',   // Auto-tune based on system load\n    autotune: {\n      targetLatency: 100,     // Target 100ms per operation\n      minConcurrency: 50,     // Never go below 50\n      maxConcurrency: 500     // Never exceed 500\n    }\n  }\n})\n```\n\n### Monitoring \u0026 Control\n\n```javascript\n// Get queue statistics\nconst stats = db.client.getQueueStats()\nconsole.log(stats)\n// {\n//   queueSize: 0,\n//   activeCount: 50,\n//   processedCount: 15420,\n//   errorCount: 3,\n//   retryCount: 8\n// }\n\n// Get performance metrics\nconst metrics = db.client.getAggregateMetrics()\nconsole.log(metrics)\n// {\n//   count: 15420,\n//   avgExecution: 45,\n//   p50: 42,\n//   p95: 78,\n//   p99: 125\n// }\n\n// Lifecycle control\nawait db.client.pausePool()    // Pause processing\ndb.client.resumePool()         // Resume processing\nawait db.client.drainPool()    // Wait for queue to empty\ndb.client.stopPool()           // Stop and cleanup\n```\n\n### Event Monitoring\n\nOperationPool emits events for monitoring and observability:\n\n| Event | Parameters | Description |\n|-------|------------|-------------|\n| `pool:taskStarted` | `(task)` | Task execution started |\n| `pool:taskCompleted` | `(task, result)` | Task completed successfully |\n| `pool:taskError` | `(task, error)` | Task failed with error |\n| `pool:taskRetry` | `(task, attempt)` | Task retry attempt (1-based) |\n| `pool:taskMetrics` | `(metrics)` | Task performance metrics |\n| `pool:paused` | `()` | Pool paused (waiting for active tasks) |\n| `pool:resumed` | `()` | Pool resumed processing |\n| `pool:drained` | `()` | All tasks completed (queue empty) |\n| `pool:stopped` | `()` | Pool stopped (pending tasks cancelled) |\n\n**Example:**\n```javascript\ndb.client.on('pool:taskCompleted', (task, result) =\u003e {\n  console.log(`✓ ${task.id}: ${task.timings.execution}ms`)\n})\n\ndb.client.on('pool:taskError', (task, error) =\u003e {\n  console.error(`✗ ${task.id}:`, error.message)\n})\n```\n\nSee [docs/benchmarks/operation-pool.md](./docs/benchmarks/operation-pool.md) for the operation-pool benchmark discussion and implementation notes.\n\n### Performance Comparison\n\nBenchmark results from comprehensive testing of 108 scenarios (see [docs/benchmarks/operation-pool.md](./docs/benchmarks/operation-pool.md) and the [benchmark index](./docs/benchmarks/README.md)):\n\n| Scale | Separate Pools | Promise.all | Shared Pool | Winner |\n|-------|----------------|------------|------------|--------|\n| **1,000 ops** | 2.1ms | 1.8ms | 2.5ms | Promise.all (marginal) |\n| **5,000 ops** | 18ms | 28ms | 32ms | **Separate Pools (+40%)** |\n| **10,000 ops** | 35ms | 45ms | 52ms | **Separate Pools (+37%)** |\n| **Memory (10K)** | 88 MB | 1,142 MB | 278 MB | **Separate Pools (13x better)** |\n\n### When to Use\n\n**✅ Automatic (no configuration needed):**\n- All operations benefit from Separate Pools\n- Default concurrency: 100 (optimized for S3)\n- Zero contention between databases\n- Auto-retry with exponential backoff\n- Adaptive tuning available for custom scenarios\n\n**Customize concurrency for:**\n- **High-throughput APIs**: `executorPool: { concurrency: 200 }`\n- **Data pipelines**: `executorPool: { concurrency: 300-500 }`\n- **Single/low-frequency ops**: `executorPool: { concurrency: 10 }`\n- **Memory-constrained**: `executorPool: { concurrency: 25-50 }`\n\n### Configuration Reference\n\nSeparate Pools comes pre-configured with production-ready defaults. Override only what you need:\n\n```javascript\n// Minimal - uses all defaults (recommended)\nconst db = new Database({\n  connectionString: 's3://bucket/database'\n  // executorPool uses defaults: { concurrency: 100 }\n})\n\n// Custom - override specific settings\nconst db = new Database({\n  connectionString: 's3://bucket/database',\n  executorPool: {\n    concurrency: 200,               // Concurrency per database pool (default: 100)\n    retries: 3,                     // Max retry attempts\n    retryDelay: 1000,               // Initial retry delay (ms)\n    timeout: 30000,                 // Operation timeout (ms)\n    retryableErrors: [              // Errors to retry (empty = all)\n      'NetworkingError',\n      'TimeoutError',\n      'RequestTimeout',\n      'ServiceUnavailable',\n      'SlowDown',\n      'RequestLimitExceeded'\n    ],\n    autotune: {                     // Auto-tuning (optional)\n      enabled: true,\n      targetLatency: 100,           // Target latency (ms)\n      minConcurrency: 50,           // Min per database\n      maxConcurrency: 500,          // Max per database\n      targetMemoryPercent: 0.7,     // Target memory usage (70%)\n      adjustmentInterval: 5000      // Check interval (ms)\n    }\n  },\n  taskExecutorMonitoring: {\n    enabled: true,\n    collectMetrics: true,\n    sampleRate: 1,\n    mode: 'balanced'\n  }\n})\n```\n\n**Complete documentation**: [**docs/benchmarks/executor-pool.md**](./docs/benchmarks/executor-pool.md)\n\n---\n\n## 🔌 Plugins\n\n\u003e **Quick Jump:** [🌐 API](#-api) | [🔐 Identity](#-identity) | [⚡ Performance](#-performance) | [📊 Data](#-data) | [🎮 Gaming](#-gaming--esports) | [🔧 DevOps](#-devops) | [🤖 ML/AI](#-mlai) | [🕷️ Web Scraping](#️-web-scraping--automation)\n\nExtend s3db.js with powerful plugins. All plugins are optional and can be installed independently.\n\n### 🌐 API \u0026 Auth\n\n[**APIPlugin**](./docs/plugins/api/README.md) • [**IdentityPlugin**](./docs/plugins/identity/README.md)\n\n**APIPlugin** - Transform s3db.js into production-ready REST API with OpenAPI, multi-auth (JWT/OIDC/Basic/API Key), rate limiting, and template engines.\n\n**IdentityPlugin** - Complete OAuth2/OIDC server with MFA, whitelabel UI, and enterprise SSO.\n\n### ⚡ Performance\n\n[**CachePlugin**](./docs/plugins/cache/README.md) • [**TTLPlugin**](./docs/plugins/ttl/README.md) • [**EventualConsistencyPlugin**](./docs/plugins/eventual-consistency/README.md) • [**MetricsPlugin**](./docs/plugins/metrics/README.md)\n\n**CachePlugin** - Memory/S3/filesystem caching with compression and automatic invalidation.\n\n**TTLPlugin** - Auto-cleanup expired records with O(1) partition-based deletion.\n\n**EventualConsistencyPlugin** - Eventually consistent counters and high-performance analytics.\n\n**MetricsPlugin** - Performance monitoring with Prometheus export.\n\n### 📊 Data \u0026 Replication\n\n[**ReplicatorPlugin**](./docs/plugins/replicator/README.md) • [**ImporterPlugin**](./docs/plugins/importer/README.md) • [**BackupPlugin**](./docs/plugins/backup/README.md) • [**AuditPlugin**](./docs/plugins/audit/README.md)\n\n**ReplicatorPlugin** - Real-time replication to BigQuery, PostgreSQL, MySQL, Turso, PlanetScale, and SQS.\n\n**ImporterPlugin** - Stream processing for large JSON/CSV imports.\n\n**BackupPlugin** - Automated backups to S3, filesystem, or cross-cloud.\n\n**AuditPlugin** - Compliance logging for all database operations.\n\n### 🎮 Gaming \u0026 Esports\n\n[**TournamentPlugin**](./docs/plugins/tournament/README.md)\n\n**TournamentPlugin** - Complete tournament engine supporting Single/Double Elimination, Round Robin, Swiss, and League formats with automated bracket generation.\n\n### 🔧 DevOps\n\n[**QueueConsumerPlugin**](./docs/plugins/queue-consumer/README.md) • [**SchedulerPlugin**](./docs/plugins/scheduler/README.md) • [**TfstatePlugin**](./docs/plugins/tfstate/README.md) • [**CloudInventoryPlugin**](./docs/plugins/cloud-inventory/README.md) • [**CostsPlugin**](./docs/plugins/costs/README.md)\n\n**QueueConsumerPlugin** - Process RabbitMQ/SQS messages for event-driven architectures.\n\n**SchedulerPlugin** - Cron-based job scheduling for maintenance tasks.\n\n**TfstatePlugin** - Track Terraform infrastructure changes and drift detection.\n\n**CloudInventoryPlugin** - Multi-cloud inventory with versioning and diff tracking.\n\n**CostsPlugin** - AWS cost tracking and optimization insights.\n\n### 🤖 ML/AI \u0026 Advanced Features\n\n[**MLPlugin**](./docs/plugins/ml-plugin/README.md) • [**VectorPlugin**](./docs/plugins/vector/README.md) • [**FullTextPlugin**](./docs/plugins/fulltext/README.md) • [**GeoPlugin**](./docs/plugins/geo/README.md)\n\n**MLPlugin** - Machine learning model management and inference pipelines.\n\n**VectorPlugin** - Vector similarity search (cosine, euclidean) for RAG and ML applications.\n\n**FullTextPlugin** - Full-text search with tokenization and indexing.\n\n**GeoPlugin** - Geospatial queries and distance calculations.\n\n### 🕷️ Web Scraping \u0026 Automation\n\n[**PuppeteerPlugin**](./docs/plugins/puppeteer/README.md)\n\n**PuppeteerPlugin** - Enterprise-grade browser automation with anti-bot detection, cookie farming, proxy rotation, and intelligent pooling for web scraping at scale.\n\n### 🔗 Other Plugins\n\n[**RelationPlugin**](./docs/plugins/relation/README.md) • [**StateMachinePlugin**](./docs/plugins/state-machine/README.md) • [**S3QueuePlugin**](./docs/plugins/s3-queue/README.md)\n\n**RelationPlugin** - ORM-like relationships with join optimization (10-100x faster queries).\n\n**StateMachinePlugin** - Finite state machine workflows for business processes.\n\n**S3QueuePlugin** - Distributed queue with zero race conditions using S3.\n\n### Plugin Installation\n\n```bash\n# Core plugins (no dependencies)\n# Included in s3db.js package\n\n# External dependencies (install only what you need)\npnpm add pg                      # PostgreSQL replication (ReplicatorPlugin)\npnpm add @google-cloud/bigquery  # BigQuery replication (ReplicatorPlugin)\npnpm add @aws-sdk/client-sqs     # SQS replication/consumption (ReplicatorPlugin, QueueConsumerPlugin)\npnpm add amqplib                 # RabbitMQ consumption (QueueConsumerPlugin)\npnpm add ejs                     # Template engine (APIPlugin - optional)\n```\n\n### Quick Example\n\n```javascript\nimport { S3db } from 's3db.js';\nimport { CachePlugin, MetricsPlugin, TTLPlugin } from 's3db.js';\n\nconst db = new S3db({\n  connectionString: 's3://bucket/databases/myapp',\n  plugins: [\n    // Cache frequently accessed data\n    new CachePlugin({\n      driver: 'memory',\n      ttl: 300000,  // 5 minutes\n      config: {\n        maxMemoryPercent: 0.1,  // 10% of system memory\n        enableCompression: true\n      }\n    }),\n\n    // Track performance metrics\n    new MetricsPlugin({\n      enablePrometheus: true,\n      port: 9090\n    }),\n\n    // Auto-cleanup expired sessions\n    new TTLPlugin({\n      resources: {\n        sessions: { ttl: 86400, onExpire: 'soft-delete' }  // 24h\n      }\n    })\n  ]\n});\n```\n\n### Creating Custom Plugins\n\nSimple plugin example:\n\n```javascript\nimport { Plugin } from 's3db.js';\n\nexport class MyPlugin extends Plugin {\n  constructor(options = {}) {\n    super(options);\n    this.name = 'MyPlugin';\n  }\n\n  async initialize(database) {\n    console.log('Plugin initialized!');\n\n    // Wrap methods\n    this.wrapMethod('Resource', 'inserted', async (original, resource, args) =\u003e {\n      console.log(`Inserting into ${resource.name}`);\n      const result = await original(...args);\n      console.log(`Inserted: ${result.id}`);\n      return result;\n    });\n  }\n}\n```\n\n**Complete documentation**: [**docs/plugins/README.md**](./docs/plugins/README.md)\n\n---\n\n## 🤖 MCP \u0026 Integrations\n\n### Model Context Protocol (MCP) Server\n\nS3DB includes a built-in MCP server that works in two modes depending on whether a connection string is provided.\n\n#### Library Mode (no credentials needed)\n\n```bash\n# Claude Code\nclaude mcp add s3db -- npx -y s3db.js mcp\n```\n\nExposes documentation tools, resources, and prompts. Helps you design schemas, choose field types, configure plugins, and learn s3db — no AWS credentials required.\n\n#### Full Mode (with database connection)\n\n```bash\n# Claude Code\nclaude mcp add s3db \\\n  -e S3DB_CONNECTION_STRING=s3://KEY:SECRET@my-bucket \\\n  -- npx -y s3db.js mcp\n\n# HTTP transport\nS3DB_CONNECTION_STRING=s3://KEY:SECRET@my-bucket npx s3db.js mcp --transport=http\n```\n\nAdds CRUD tools, live resource introspection (`s3db://resource/{name}`), and data-aware prompts on top of everything in Library Mode.\n\n#### What's exposed per mode\n\n| | Library Mode | Full Mode |\n|---|:---:|:---:|\n| `s3dbSearchDocs` | ✅ | ✅ |\n| `s3db://core/`, `s3db://plugin/`, `s3db://guide/` resources | ✅ | ✅ |\n| Schema design \u0026 migration prompts | ✅ | ✅ |\n| CRUD tools (resourceGet, resourceList, resourceInsert…) | — | ✅ |\n| `s3db://resource/{name}` (live schema) | — | ✅ |\n| Debug \u0026 optimization prompts | — | ✅ |\n\nLibrary Mode can be promoted to Full Mode at runtime by calling the `dbConnect` tool — no restart needed.\n\n**Complete documentation**: [**docs/mcp.md**](./docs/mcp.md)\n\n### Integrations\n\ns3db.js integrates seamlessly with:\n\n- **BigQuery** - Real-time data replication via ReplicatorPlugin\n- **PostgreSQL** - Sync to traditional databases via ReplicatorPlugin\n- **AWS SQS** - Event streaming and message queues\n- **RabbitMQ** - Message queue integration\n- **Prometheus** - Metrics export via MetricsPlugin\n- **Vector Databases** - Embedding field type with 77% compression\n\n---\n\n## 🔧 CLI\n\ns3db.js includes a powerful CLI for database management and operations.\n\n### Installation\n\n```bash\n# Global\nnpm install -g s3db.js\n\n# Project\nnpm install s3db.js\nnpx s3db [command]\n```\n\n### Commands\n\n```bash\n# List resources\ns3db list\n\n# Query resources\ns3db query users\ns3db query users --filter '{\"status\":\"active\"}'\n\n# Insert records\ns3db insert users --data '{\"name\":\"John\",\"email\":\"john@example.com\"}'\n\n# Update records\ns3db update users user-123 --data '{\"age\":31}'\n\n# Delete records\ns3db delete users user-123\n\n# Export data\ns3db export users --format json \u003e users.json\ns3db export users --format csv \u003e users.csv\n\n# Import data\ns3db import users \u003c users.json\n\n# Stats\ns3db stats\ns3db stats users\n\n# MCP Server\ns3db mcp --transport=stdio\ns3db mcp --transport=sse --port=17500\n```\n\n### Environment Variables\n\n```bash\nS3DB_CONNECTION_STRING=s3://bucket/databases/myapp\nS3DB_CACHE_ENABLED=true\nS3DB_COSTS_ENABLED=true\nS3DB_VERBOSE=false\n```\n\n---\n\n## 📖 Documentation\n\n### Core Documentation\n\n- [**Resources (Complete Guide)**](./docs/core/resource.md) - Everything about resources, schemas, behaviors, partitions, hooks, middlewares, events\n- [**Clients**](./docs/clients/README.md) - Low-level S3 operations, HTTP configuration, and client selection\n- [**Schema Validation**](./docs/core/schema.md) - Comprehensive schema validation and field types\n- [**Plugins Overview**](./docs/plugins/README.md) - All available plugins and how to create custom ones\n\n### Plugin Documentation\n\n- [Cache Plugin](./docs/plugins/cache/README.md)\n- [Costs Plugin](./docs/plugins/costs/README.md)\n- [Metrics Plugin](./docs/plugins/metrics/README.md)\n- [Audit Plugin](./docs/plugins/audit/README.md)\n- [TTL Plugin](./docs/plugins/ttl/README.md)\n- [Relation Plugin](./docs/plugins/relation/README.md)\n- [Replicator Plugin](./docs/plugins/replicator/README.md)\n\n### MCP \u0026 Integrations\n\n- [MCP Server Guide](./docs/mcp.md) - Complete MCP documentation with all 28 tools\n- [MCP Server Guide](./docs/mcp.md) - Use MCP with stdio or SSE transports\n- [Claude CLI Example](./docs/mcp.md) - Claude CLI configuration and examples\n\n### Benchmarks \u0026 Performance\n\n- [Benchmark Index](./docs/benchmarks/README.md)\n- [Base62 Encoding](./docs/benchmarks/base62.md)\n- [All Types Encoding](./docs/benchmarks/all-types-encoding.md)\n- [String Encoding Optimizations](./docs/benchmarks/STRING-ENCODING-OPTIMIZATIONS.md)\n- [EventualConsistency Plugin](./docs/benchmarks/eventual-consistency.md)\n- [Partitions Matrix](./docs/benchmarks/partitions.md)\n- [Vector Clustering](./docs/benchmarks/vector-clustering.md)\n\n### Examples\n\nBrowse [**60+ examples**](./docs/examples/) covering:\n- Basic CRUD (e01-e07)\n- Advanced features (e08-e17)\n- Plugins (e18-e33)\n- Vectors \u0026 RAG (e41-e43)\n- Testing patterns (e38-e40, e64-e65)\n\n### API Reference\n\n| Resource | Link |\n|----------|------|\n| Resource API | [docs/core/resource.md](./docs/core/resource.md) |\n| Client API | [docs/clients/README.md](./docs/clients/README.md) |\n| Schema Validation | [docs/core/schema.md](./docs/core/schema.md) |\n| Plugin API | [docs/plugins/README.md](./docs/plugins/README.md) |\n\n---\n\n## 🔧 Troubleshooting\n\nCommon issues and solutions:\n\n\u003cdetails\u003e\n\u003csummary\u003e\u003cstrong\u003eConnection Issues\u003c/strong\u003e\u003c/summary\u003e\n\n**Problem:** Cannot connect to S3 bucket\n\n**Solutions:**\n1. Verify credentials in connection string\n2. Check IAM permissions (s3:ListBucket, s3:GetObject, s3:PutObject)\n3. Ensure bucket exists\n4. Check network connectivity\n\n```javascript\n// Enable debug logging\nconst db = new S3db({\n  connectionString: '...',\n  logLevel: 'debug'\n});\n```\n\u003c/details\u003e\n\n\u003cdetails\u003e\n\u003csummary\u003e\u003cstrong\u003eMetadata Size Exceeded\u003c/strong\u003e\u003c/summary\u003e\n\n**Problem:** Error: \"S3 metadata size exceeds 2KB limit\"\n\n**Solutions:**\n1. Change behavior to `body-overflow` or `body-only`\n2. Reduce field sizes or use truncation\n3. Move large content to separate fields\n\n```javascript\nconst resource = await db.createResource({\n  name: 'blogs',\n  behavior: 'body-overflow',  // Automatically handle overflow\n  attributes: { title: 'string', content: 'string' }\n});\n```\n\u003c/details\u003e\n\n\u003cdetails\u003e\n\u003csummary\u003e\u003cstrong\u003ePerformance Issues\u003c/strong\u003e\u003c/summary\u003e\n\n**Problem:** Slow queries or operations\n\n**Solutions:**\n1. Use partitions for frequently queried fields\n2. Enable caching with CachePlugin\n3. Increase HTTP client concurrency\n4. Use bulk operations instead of loops\n\n```javascript\n// Add partitions\nconst resource = await db.createResource({\n  name: 'analytics',\n  attributes: { event: 'string', region: 'string' },\n  partitions: {\n    byEvent: { fields: { event: 'string' } }\n  },\n  asyncPartitions: true  // 70-100% faster writes\n});\n\n// Enable caching\nconst db = new S3db({\n  connectionString: '...',\n  plugins: [new CachePlugin({ ttl: 300000 })]\n});\n```\n\u003c/details\u003e\n\n\u003cdetails\u003e\n\u003csummary\u003e\u003cstrong\u003eOrphaned Partitions\u003c/strong\u003e\u003c/summary\u003e\n\n**Problem:** Partition references deleted field\n\n**Solutions:**\n\n```javascript\nconst resource = await db.getResource('users', { strictValidation: false });\nconst orphaned = resource.findOrphanedPartitions();\nconsole.log('Orphaned:', orphaned);\n\n// Remove them\nresource.removeOrphanedPartitions();\nawait db.uploadMetadataFile();\n```\n\u003c/details\u003e\n\n---\n\n## 📊 Performance Benchmarks\n\n\u003e **⚠️ Important**: All benchmark results documented were generated using **Node.js v22.6.0**. Performance may vary with different Node.js versions.\n\ns3db.js includes comprehensive benchmarks demonstrating real-world performance optimizations:\n\n- [**Base62 Encoding**](./docs/benchmarks/base62.md) - 40-46% space savings, 5x faster than Base36\n- [**All Types Encoding**](./docs/benchmarks/all-types-encoding.md) - Comprehensive encoding across all field types\n- [**String Encoding Optimizations**](./docs/benchmarks/STRING-ENCODING-OPTIMIZATIONS.md) - 2-3x faster UTF-8 calculations\n- [**EventualConsistency Plugin**](./docs/benchmarks/eventual-consistency.md) - 70-100% faster writes\n- [**Partitions Matrix**](./docs/benchmarks/partitions.md) - Test 110 combinations to find optimal config\n- [**Vector Clustering**](./docs/benchmarks/vector-clustering.md) - Vector similarity and clustering performance\n\n**[📋 Complete Benchmark Index](./docs/benchmarks/README.md)**\n\n---\n\n## 🤝 Contributing\n\nContributions are welcome. Use the [repository on GitHub](https://github.com/forattini-dev/s3db.js) to open issues or pull requests.\n\n---\n\n## 📄 License\n\nThis project is licensed under the [Unlicense](https://github.com/forattini-dev/s3db.js/blob/main/LICENSE).\n\n---\n\n## 🙏 Acknowledgments\n\n- Built with [AWS SDK for JavaScript](https://aws.amazon.com/sdk-for-javascript/)\n- Validation powered by [@icebob/fastest-validator](https://github.com/icebob/fastest-validator)\n- ID generation using the built-in generator\n\n---\n\n\u003cp align=\"center\"\u003e\n  Made with ❤️ by the s3db.js community\n\u003c/p\u003e\n","project_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fforattini-dev%2Fs3db.js","html_url":"https://awesome.ecosyste.ms/projects/github.com%2Fforattini-dev%2Fs3db.js","lists_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fforattini-dev%2Fs3db.js/lists"}