mirror of
https://github.com/supermemoryai/supermemory.git
synced 2026-05-18 23:36:00 +00:00
docs: fix s3 connector config shape
This commit is contained in:
Prasanna A P
parent
62ac31ecaa
commit
54d32e3676
1 changed files with 84 additions and 44 deletions
|
|
@ -22,10 +22,12 @@ The S3 connector requires a **Scale Plan** or higher. You can also create S3 con
|
|||
});
|
||||
|
||||
const connection = await client.connections.create('s3', {
|
||||
accessKeyId: process.env.AWS_ACCESS_KEY_ID!,
|
||||
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!,
|
||||
bucket: 'my-documents-bucket',
|
||||
region: 'us-east-1',
|
||||
metadata: {
|
||||
accessKeyId: process.env.AWS_ACCESS_KEY_ID!,
|
||||
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!,
|
||||
bucket: 'my-documents-bucket',
|
||||
region: 'us-east-1'
|
||||
},
|
||||
containerTags: ['org-123']
|
||||
});
|
||||
```
|
||||
|
|
@ -35,14 +37,16 @@ The S3 connector requires a **Scale Plan** or higher. You can also create S3 con
|
|||
from supermemory import Supermemory
|
||||
import os
|
||||
|
||||
client = Supermemory(api_key=os.environ.get("SUPERMEMORY_API_KEY"))
|
||||
client = Supermemory(api_key=os.environ["SUPERMEMORY_API_KEY"])
|
||||
|
||||
connection = client.connections.create(
|
||||
's3',
|
||||
access_key_id=os.environ.get("AWS_ACCESS_KEY_ID"),
|
||||
secret_access_key=os.environ.get("AWS_SECRET_ACCESS_KEY"),
|
||||
bucket='my-documents-bucket',
|
||||
region='us-east-1',
|
||||
metadata={
|
||||
'accessKeyId': os.environ["AWS_ACCESS_KEY_ID"],
|
||||
'secretAccessKey': os.environ["AWS_SECRET_ACCESS_KEY"],
|
||||
'bucket': 'my-documents-bucket',
|
||||
'region': 'us-east-1'
|
||||
},
|
||||
container_tags=['org-123', 's3-sync']
|
||||
)
|
||||
```
|
||||
|
|
@ -53,10 +57,12 @@ The S3 connector requires a **Scale Plan** or higher. You can also create S3 con
|
|||
-H "Authorization: Bearer $SUPERMEMORY_API_KEY" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"accessKeyId": "AKIAIOSFODNN7EXAMPLE",
|
||||
"secretAccessKey": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
|
||||
"bucket": "my-documents-bucket",
|
||||
"region": "us-east-1",
|
||||
"metadata": {
|
||||
"accessKeyId": "AKIAIOSFODNN7EXAMPLE",
|
||||
"secretAccessKey": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
|
||||
"bucket": "my-documents-bucket",
|
||||
"region": "us-east-1"
|
||||
},
|
||||
"containerTags": ["org-123"]
|
||||
}'
|
||||
```
|
||||
|
|
@ -65,50 +71,81 @@ The S3 connector requires a **Scale Plan** or higher. You can also create S3 con
|
|||
|
||||
## Configuration Options
|
||||
|
||||
| Parameter | Required | Description |
|
||||
|-----------|----------|-------------|
|
||||
| `accessKeyId` | Yes | AWS access key ID or S3-compatible service key |
|
||||
| `secretAccessKey` | Yes | AWS secret access key |
|
||||
| `bucket` | Yes | S3 bucket name |
|
||||
| `region` | Yes | AWS region (e.g., `us-east-1`) |
|
||||
| `endpoint` | No | Custom endpoint for S3-compatible services |
|
||||
| `prefix` | No | Key prefix filter (e.g., `documents/`) |
|
||||
| `containerTagRegex` | No | Regex to extract container tags from file paths |
|
||||
| `containerTags` | No | Tags for organizing connections |
|
||||
| `documentLimit` | No | Maximum documents to sync (default: 10,000) |
|
||||
For S3, provider-specific connection fields are passed inside the top-level `metadata` object. General connection options stay top-level.
|
||||
|
||||
| Parameter | Location | Required | Description |
|
||||
|-----------|----------|----------|-------------|
|
||||
| `accessKeyId` | `metadata.accessKeyId` | Yes | AWS access key ID or S3-compatible service key |
|
||||
| `secretAccessKey` | `metadata.secretAccessKey` | Yes | AWS secret access key |
|
||||
| `bucket` | `metadata.bucket` | Yes | S3 bucket name |
|
||||
| `region` | `metadata.region` | Yes | AWS region (e.g., `us-east-1`). Use `auto` for Cloudflare R2. |
|
||||
| `endpoint` | `metadata.endpoint` | No | Custom endpoint for S3-compatible services |
|
||||
| `prefix` | `metadata.prefix` | No | Key prefix filter (e.g., `documents/`) |
|
||||
| `containerTagRegex` | `metadata.containerTagRegex` | No | Regex to extract container tags from file paths |
|
||||
| `containerTags` | top-level | No | Tags for organizing connections |
|
||||
| `documentLimit` | top-level | No | Maximum documents to sync (default: 10,000) |
|
||||
|
||||
<Note>
|
||||
In the Python SDK, use `container_tags` for the top-level option, but keep S3 metadata keys in camelCase: `accessKeyId`, `secretAccessKey`, and `containerTagRegex`.
|
||||
</Note>
|
||||
|
||||
## S3-Compatible Services
|
||||
|
||||
Use a custom `endpoint` to connect to S3-compatible storage:
|
||||
Use `metadata.endpoint` to connect to S3-compatible storage:
|
||||
|
||||
```typescript
|
||||
// MinIO
|
||||
const connection = await client.connections.create('s3', {
|
||||
accessKeyId: 'minio-key',
|
||||
secretAccessKey: 'minio-secret',
|
||||
bucket: 'my-bucket',
|
||||
region: 'us-east-1',
|
||||
endpoint: 'https://minio.example.com',
|
||||
metadata: {
|
||||
accessKeyId: 'minio-key',
|
||||
secretAccessKey: 'minio-secret',
|
||||
bucket: 'my-bucket',
|
||||
region: 'us-east-1',
|
||||
endpoint: 'https://minio.example.com'
|
||||
},
|
||||
containerTags: ['minio-sync']
|
||||
});
|
||||
|
||||
// DigitalOcean Spaces
|
||||
endpoint: 'https://nyc3.digitaloceanspaces.com'
|
||||
|
||||
// Cloudflare R2
|
||||
endpoint: 'https://ACCOUNT_ID.r2.cloudflarestorage.com'
|
||||
```
|
||||
|
||||
Common S3-compatible endpoint values:
|
||||
|
||||
| Service | `metadata.endpoint` | `metadata.region` |
|
||||
|---------|----------------------|-------------------|
|
||||
| DigitalOcean Spaces | `https://nyc3.digitaloceanspaces.com` | `nyc3` |
|
||||
| Cloudflare R2 | `https://<account-id>.r2.cloudflarestorage.com` | `auto` |
|
||||
|
||||
Cloudflare R2 example:
|
||||
|
||||
```typescript
|
||||
const connection = await client.connections.create('s3', {
|
||||
metadata: {
|
||||
accessKeyId: process.env.R2_ACCESS_KEY_ID!,
|
||||
secretAccessKey: process.env.R2_SECRET_ACCESS_KEY!,
|
||||
bucket: 'my-bucket',
|
||||
region: 'auto',
|
||||
endpoint: 'https://<account-id>.r2.cloudflarestorage.com'
|
||||
},
|
||||
containerTags: ['r2-sync']
|
||||
});
|
||||
```
|
||||
|
||||
<Note>
|
||||
For S3-compatible services, `metadata.endpoint` is the base S3 endpoint. Do not include the bucket name in the endpoint URL; pass the bucket separately as `metadata.bucket`.
|
||||
</Note>
|
||||
|
||||
## Prefix Filtering
|
||||
|
||||
Sync only files within a specific path:
|
||||
|
||||
```typescript
|
||||
const connection = await client.connections.create('s3', {
|
||||
// ... credentials
|
||||
bucket: 'company-data',
|
||||
region: 'us-east-1',
|
||||
prefix: 'documents/engineering/', // Only syncs files under this path
|
||||
metadata: {
|
||||
accessKeyId: process.env.AWS_ACCESS_KEY_ID!,
|
||||
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!,
|
||||
bucket: 'company-data',
|
||||
region: 'us-east-1',
|
||||
prefix: 'documents/engineering/' // Only syncs files under this path
|
||||
},
|
||||
containerTags: ['engineering-docs']
|
||||
});
|
||||
```
|
||||
|
|
@ -119,10 +156,13 @@ Extract container tags from S3 key paths for multi-tenant setups:
|
|||
|
||||
```typescript
|
||||
const connection = await client.connections.create('s3', {
|
||||
// ... credentials
|
||||
bucket: 'user-files',
|
||||
region: 'us-east-1',
|
||||
containerTagRegex: 'users/(?<userId>[^/]+)/',
|
||||
metadata: {
|
||||
accessKeyId: process.env.AWS_ACCESS_KEY_ID!,
|
||||
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!,
|
||||
bucket: 'user-files',
|
||||
region: 'us-east-1',
|
||||
containerTagRegex: 'users/(?<userId>[^/]+)/'
|
||||
},
|
||||
containerTags: ['user-files']
|
||||
});
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue