diff --git a/.env.example b/.env.example index 4f805a7..2256ba9 100644 --- a/.env.example +++ b/.env.example @@ -65,14 +65,7 @@ SURREAL_PASSWORD="root" SURREAL_NAMESPACE="open_notebook" SURREAL_DATABASE="staging" -# Old format (backward compatible) - will be converted automatically -# SURREAL_ADDRESS="localhost" -# SURREAL_PORT=8000 -# SURREAL_USER="root" -# SURREAL_PASS="root" -# SURREAL_NAMESPACE="open_notebook" -# SURREAL_DATABASE="staging" - +# OPEN_NOTEBOOK_PASSWORD= # FIRECRAWL - Get a key at https://firecrawl.dev/ FIRECRAWL_API_KEY= diff --git a/.github/workflows/build-and-release.yml b/.github/workflows/build-and-release.yml new file mode 100644 index 0000000..b62bf8c --- /dev/null +++ b/.github/workflows/build-and-release.yml @@ -0,0 +1,168 @@ +name: Build and Release + +on: + workflow_dispatch: + inputs: + build_type: + description: 'Build type to create' + required: true + default: 'both' + type: choice + options: + - both + - regular + - single + push_latest: + description: 'Also push latest tags' + required: false + default: true + type: boolean + release: + types: [published] + +env: + REGISTRY: docker.io + IMAGE_NAME: lfnovo/open_notebook + +jobs: + extract-version: + runs-on: ubuntu-latest + outputs: + version: ${{ steps.version.outputs.version }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Extract version from pyproject.toml + id: version + run: | + VERSION=$(grep -m1 '^version = ' pyproject.toml | cut -d'"' -f2) + echo "version=$VERSION" >> $GITHUB_OUTPUT + echo "Extracted version: $VERSION" + + build-regular: + needs: extract-version + runs-on: ubuntu-latest + if: github.event.inputs.build_type == 'regular' || github.event.inputs.build_type == 'both' || github.event_name == 'release' + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Cache Docker layers + uses: actions/cache@v3 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-regular-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx-regular- + + - name: Build and push regular image + uses: docker/build-push-action@v5 + with: + context: . + file: ./Dockerfile + platforms: linux/amd64,linux/arm64 + push: true + tags: | + ${{ env.IMAGE_NAME }}:${{ needs.extract-version.outputs.version }} + ${{ github.event.inputs.push_latest == 'true' && format('{0}:latest', env.IMAGE_NAME) || '' }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max + + - name: Move cache + run: | + rm -rf /tmp/.buildx-cache + mv /tmp/.buildx-cache-new /tmp/.buildx-cache + + build-single: + needs: extract-version + runs-on: ubuntu-latest + if: github.event.inputs.build_type == 'single' || github.event.inputs.build_type == 'both' || github.event_name == 'release' + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Cache Docker layers + uses: actions/cache@v3 + with: + path: /tmp/.buildx-cache-single + key: ${{ runner.os }}-buildx-single-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx-single- + + - name: Build and push single-container image + uses: docker/build-push-action@v5 + with: + context: . + file: ./Dockerfile.single + platforms: linux/amd64,linux/arm64 + push: true + tags: | + ${{ env.IMAGE_NAME }}:${{ needs.extract-version.outputs.version }}-single + ${{ github.event.inputs.push_latest == 'true' && format('{0}:latest-single', env.IMAGE_NAME) || '' }} + cache-from: type=local,src=/tmp/.buildx-cache-single + cache-to: type=local,dest=/tmp/.buildx-cache-single-new,mode=max + + - name: Move cache + run: | + rm -rf /tmp/.buildx-cache-single + mv /tmp/.buildx-cache-single-new /tmp/.buildx-cache-single + + summary: + needs: [extract-version, build-regular, build-single] + runs-on: ubuntu-latest + if: always() + steps: + - name: Build Summary + run: | + echo "## Build Summary" >> $GITHUB_STEP_SUMMARY + echo "**Version:** ${{ needs.extract-version.outputs.version }}" >> $GITHUB_STEP_SUMMARY + echo "**Build Type:** ${{ github.event.inputs.build_type || 'both' }}" >> $GITHUB_STEP_SUMMARY + echo "**Push Latest:** ${{ github.event.inputs.push_latest || 'true' }}" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### Images Built:" >> $GITHUB_STEP_SUMMARY + + if [[ "${{ needs.build-regular.result }}" == "success" ]]; then + echo "✅ **Regular:** \`${{ env.IMAGE_NAME }}:${{ needs.extract-version.outputs.version }}\`" >> $GITHUB_STEP_SUMMARY + if [[ "${{ github.event.inputs.push_latest }}" == "true" ]]; then + echo "✅ **Regular Latest:** \`${{ env.IMAGE_NAME }}:latest\`" >> $GITHUB_STEP_SUMMARY + fi + elif [[ "${{ needs.build-regular.result }}" == "skipped" ]]; then + echo "⏭️ **Regular:** Skipped" >> $GITHUB_STEP_SUMMARY + else + echo "❌ **Regular:** Failed" >> $GITHUB_STEP_SUMMARY + fi + + if [[ "${{ needs.build-single.result }}" == "success" ]]; then + echo "✅ **Single:** \`${{ env.IMAGE_NAME }}:${{ needs.extract-version.outputs.version }}-single\`" >> $GITHUB_STEP_SUMMARY + if [[ "${{ github.event.inputs.push_latest }}" == "true" ]]; then + echo "✅ **Single Latest:** \`${{ env.IMAGE_NAME }}:latest-single\`" >> $GITHUB_STEP_SUMMARY + fi + elif [[ "${{ needs.build-single.result }}" == "skipped" ]]; then + echo "⏭️ **Single:** Skipped" >> $GITHUB_STEP_SUMMARY + else + echo "❌ **Single:** Failed" >> $GITHUB_STEP_SUMMARY + fi + + echo "" >> $GITHUB_STEP_SUMMARY + echo "### Platforms:" >> $GITHUB_STEP_SUMMARY + echo "- linux/amd64" >> $GITHUB_STEP_SUMMARY + echo "- linux/arm64" >> $GITHUB_STEP_SUMMARY \ No newline at end of file diff --git a/.github/workflows/build-dev.yml b/.github/workflows/build-dev.yml new file mode 100644 index 0000000..6138be7 --- /dev/null +++ b/.github/workflows/build-dev.yml @@ -0,0 +1,186 @@ +name: Development Build + +on: + pull_request: + branches: [ main ] + push: + branches: [ main ] + paths-ignore: + - '**.md' + - 'docs/**' + - 'notebooks/**' + - '.github/workflows/claude*.yml' + workflow_dispatch: + inputs: + dockerfile: + description: 'Dockerfile to test' + required: true + default: 'both' + type: choice + options: + - both + - regular + - single + platform: + description: 'Platform to build' + required: true + default: 'linux/amd64' + type: choice + options: + - linux/amd64 + - linux/arm64 + - linux/amd64,linux/arm64 + +env: + REGISTRY: docker.io + IMAGE_NAME: lfnovo/open_notebook + +jobs: + extract-version: + runs-on: ubuntu-latest + outputs: + version: ${{ steps.version.outputs.version }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Extract version from pyproject.toml + id: version + run: | + VERSION=$(grep -m1 '^version = ' pyproject.toml | cut -d'"' -f2) + echo "version=$VERSION" >> $GITHUB_OUTPUT + echo "Extracted version: $VERSION" + + lint-and-check: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.12' + + - name: Install uv + uses: astral-sh/setup-uv@v3 + + - name: Install dependencies + run: uv sync --dev + + - name: Run ruff + run: uv run ruff check . --output-format=github + + - name: Run mypy + run: uv run python -m mypy . + + test-build-regular: + needs: extract-version + runs-on: ubuntu-latest + if: github.event.inputs.dockerfile == 'regular' || github.event.inputs.dockerfile == 'both' || github.event_name != 'workflow_dispatch' + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Cache Docker layers + uses: actions/cache@v3 + with: + path: /tmp/.buildx-cache-dev + key: ${{ runner.os }}-buildx-dev-regular-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx-dev-regular- + + - name: Build regular image (test only) + uses: docker/build-push-action@v5 + with: + context: . + file: ./Dockerfile + platforms: ${{ github.event.inputs.platform || 'linux/amd64' }} + push: false + tags: ${{ env.IMAGE_NAME }}:${{ needs.extract-version.outputs.version }}-dev-regular + cache-from: type=local,src=/tmp/.buildx-cache-dev + cache-to: type=local,dest=/tmp/.buildx-cache-dev-new,mode=max + + - name: Move cache + run: | + rm -rf /tmp/.buildx-cache-dev + mv /tmp/.buildx-cache-dev-new /tmp/.buildx-cache-dev + + test-build-single: + needs: extract-version + runs-on: ubuntu-latest + if: github.event.inputs.dockerfile == 'single' || github.event.inputs.dockerfile == 'both' || github.event_name != 'workflow_dispatch' + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Cache Docker layers + uses: actions/cache@v3 + with: + path: /tmp/.buildx-cache-dev-single + key: ${{ runner.os }}-buildx-dev-single-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx-dev-single- + + - name: Build single-container image (test only) + uses: docker/build-push-action@v5 + with: + context: . + file: ./Dockerfile.single + platforms: ${{ github.event.inputs.platform || 'linux/amd64' }} + push: false + tags: ${{ env.IMAGE_NAME }}:${{ needs.extract-version.outputs.version }}-dev-single + cache-from: type=local,src=/tmp/.buildx-cache-dev-single + cache-to: type=local,dest=/tmp/.buildx-cache-dev-single-new,mode=max + + - name: Move cache + run: | + rm -rf /tmp/.buildx-cache-dev-single + mv /tmp/.buildx-cache-dev-single-new /tmp/.buildx-cache-dev-single + + summary: + needs: [extract-version, lint-and-check, test-build-regular, test-build-single] + runs-on: ubuntu-latest + if: always() + steps: + - name: Development Build Summary + run: | + echo "## Development Build Summary" >> $GITHUB_STEP_SUMMARY + echo "**Version:** ${{ needs.extract-version.outputs.version }}" >> $GITHUB_STEP_SUMMARY + echo "**Platform:** ${{ github.event.inputs.platform || 'linux/amd64' }}" >> $GITHUB_STEP_SUMMARY + echo "**Dockerfile:** ${{ github.event.inputs.dockerfile || 'both' }}" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### Results:" >> $GITHUB_STEP_SUMMARY + + if [[ "${{ needs.lint-and-check.result }}" == "success" ]]; then + echo "✅ **Lint & Type Check:** Passed" >> $GITHUB_STEP_SUMMARY + else + echo "❌ **Lint & Type Check:** Failed" >> $GITHUB_STEP_SUMMARY + fi + + if [[ "${{ needs.test-build-regular.result }}" == "success" ]]; then + echo "✅ **Regular Dockerfile:** Build successful" >> $GITHUB_STEP_SUMMARY + elif [[ "${{ needs.test-build-regular.result }}" == "skipped" ]]; then + echo "⏭️ **Regular Dockerfile:** Skipped" >> $GITHUB_STEP_SUMMARY + else + echo "❌ **Regular Dockerfile:** Build failed" >> $GITHUB_STEP_SUMMARY + fi + + if [[ "${{ needs.test-build-single.result }}" == "success" ]]; then + echo "✅ **Single Dockerfile:** Build successful" >> $GITHUB_STEP_SUMMARY + elif [[ "${{ needs.test-build-single.result }}" == "skipped" ]]; then + echo "⏭️ **Single Dockerfile:** Skipped" >> $GITHUB_STEP_SUMMARY + else + echo "❌ **Single Dockerfile:** Build failed" >> $GITHUB_STEP_SUMMARY + fi + + echo "" >> $GITHUB_STEP_SUMMARY + echo "### Notes:" >> $GITHUB_STEP_SUMMARY + echo "- This is a development build (no images pushed to registry)" >> $GITHUB_STEP_SUMMARY + echo "- For production releases, use the 'Build and Release' workflow" >> $GITHUB_STEP_SUMMARY \ No newline at end of file diff --git a/open_notebook/database/new.py b/open_notebook/database/new.py deleted file mode 100644 index bf5843f..0000000 --- a/open_notebook/database/new.py +++ /dev/null @@ -1,178 +0,0 @@ -import os -from contextlib import asynccontextmanager -from datetime import datetime, timezone -from typing import Any, Dict, List, Optional, TypeVar, Union - -from loguru import logger -from surrealdb import AsyncSurreal, RecordID # type: ignore - -T = TypeVar("T", Dict[str, Any], List[Dict[str, Any]]) - - -def get_database_url(): - """Get database URL with backward compatibility""" - surreal_url = os.getenv("SURREAL_URL") - if surreal_url: - return surreal_url - - # Fallback to old format - WebSocket URL format - address = os.getenv("SURREAL_ADDRESS", "localhost") - port = os.getenv("SURREAL_PORT", "8000") - return f"ws://{address}/rpc:{port}" - - -def get_database_password(): - """Get password with backward compatibility""" - return os.getenv("SURREAL_PASSWORD") or os.getenv("SURREAL_PASS") - - -def parse_record_ids(obj: Any) -> Any: - """Recursively parse and convert RecordIDs into strings.""" - if isinstance(obj, dict): - return {k: parse_record_ids(v) for k, v in obj.items()} - elif isinstance(obj, list): - return [parse_record_ids(item) for item in obj] - elif isinstance(obj, RecordID): - return str(obj) - return obj - - -def ensure_record_id(value: Union[str, RecordID]) -> RecordID: - """Ensure a value is a RecordID.""" - if isinstance(value, RecordID): - return value - return RecordID.parse(value) - - -@asynccontextmanager -async def db_connection(): - db = AsyncSurreal(get_database_url()) - await db.signin( - { - "username": os.environ["SURREAL_USER"], - "password": get_database_password(), - } - ) - await db.use(os.environ["SURREAL_NAMESPACE"], os.environ["SURREAL_DATABASE"]) - try: - yield db - finally: - await db.close() - - -async def repo_query( - query_str: str, vars: Optional[Dict[str, Any]] = None -) -> List[Dict[str, Any]]: - """Execute a SurrealQL query and return the results""" - - async with db_connection() as connection: - try: - result = parse_record_ids(await connection.query(query_str, vars)) - if isinstance(result, str): - raise RuntimeError(result) - return result - except Exception as e: - logger.error(f"Query: {query_str[:200]} vars: {vars}") - logger.exception(e) - raise - - -async def repo_create(table: str, data: Dict[str, Any]) -> Dict[str, Any]: - """Create a new record in the specified table""" - # Remove 'id' attribute if it exists in data - data.pop("id", None) - data["created"] = datetime.now(timezone.utc) - data["updated"] = datetime.now(timezone.utc) - try: - async with db_connection() as connection: - return parse_record_ids(await connection.insert(table, data)) - except Exception as e: - logger.exception(e) - raise RuntimeError("Failed to create record") - - -async def repo_relate( - source: str, relationship: str, target: str, data: Optional[Dict[str, Any]] = None -) -> List[Dict[str, Any]]: - """Create a relationship between two records with optional data""" - if data is None: - data = {} - query = f"RELATE {source}->{relationship}->{target} CONTENT $data;" - # logger.debug(f"Relate query: {query}") - - return await repo_query( - query, - { - "data": data, - }, - ) - - -async def repo_upsert( - table: str, id: Optional[str], data: Dict[str, Any], add_timestamp: bool = False -) -> List[Dict[str, Any]]: - """Create or update a record in the specified table""" - data.pop("id", None) - if add_timestamp: - data["updated"] = datetime.now(timezone.utc) - query = f"UPSERT {id if id else table} MERGE $data;" - return await repo_query(query, {"data": data}) - - -async def repo_update( - table: str, id: str, data: Dict[str, Any] -) -> List[Dict[str, Any]]: - """Update an existing record by table and id""" - # If id already contains the table name, use it as is - try: - if isinstance(id, RecordID) or (":" in id and id.startswith(f"{table}:")): - record_id = id - else: - record_id = f"{table}:{id}" - - data["updated"] = datetime.now(timezone.utc) - query = f"UPDATE {record_id} MERGE $data;" - # logger.debug(f"Update query: {query}") - result = await repo_query(query, {"data": data}) - # if isinstance(result, list): - # return [_return_data(item) for item in result] - return [parse_record_ids(result)] - except Exception as e: - raise RuntimeError(f"Failed to update record: {str(e)}") - - -async def repo_get_news_by_jota_id(jota_id: str) -> Dict[str, Any]: - try: - results = await repo_query( - "SELECT * omit embedding FROM news where jota_id=$jota_id", - {"jota_id": jota_id}, - ) - return parse_record_ids(results) - except Exception as e: - logger.exception(e) - raise RuntimeError(f"Failed to fetch record: {str(e)}") - - -async def repo_delete(record_id: Union[str, RecordID]): - """Delete a record by record id""" - - try: - async with db_connection() as connection: - return await connection.delete(record_id) - except Exception as e: - logger.exception(e) - raise RuntimeError(f"Failed to delete record: {str(e)}") - - -async def repo_insert( - table: str, data: List[Dict[str, Any]], ignore_duplicates: bool = False -) -> List[Dict[str, Any]]: - """Create a new record in the specified table""" - try: - async with db_connection() as connection: - return parse_record_ids(await connection.insert(table, data)) - except Exception as e: - if ignore_duplicates and "already contains" in str(e): - return [] - logger.exception(e) - raise RuntimeError("Failed to create record") diff --git a/open_notebook/database/repository.py b/open_notebook/database/repository.py index c29570d..1b241f5 100644 --- a/open_notebook/database/repository.py +++ b/open_notebook/database/repository.py @@ -49,11 +49,13 @@ async def db_connection(): db = AsyncSurreal(get_database_url()) await db.signin( { - "username": os.environ["SURREAL_USER"], + "username": os.environ.get("SURREAL_USER"), "password": get_database_password(), } ) - await db.use(os.environ["SURREAL_NAMESPACE"], os.environ["SURREAL_DATABASE"]) + await db.use( + os.environ.get("SURREAL_NAMESPACE"), os.environ.get("SURREAL_DATABASE") + ) try: yield db finally: diff --git a/open_notebook/database/repository_old.py b/open_notebook/database/repository_old.py deleted file mode 100644 index d90ac9d..0000000 --- a/open_notebook/database/repository_old.py +++ /dev/null @@ -1,63 +0,0 @@ -import os -from contextlib import contextmanager -from typing import Any, Dict, Optional - -from loguru import logger -from sblpy.connection import SurrealSyncConnection - - -@contextmanager -def db_connection(): - connection = SurrealSyncConnection( - host=os.environ["SURREAL_ADDRESS"], - port=int(os.environ["SURREAL_PORT"]), - user=os.environ["SURREAL_USER"], - password=os.environ["SURREAL_PASS"], - namespace=os.environ["SURREAL_NAMESPACE"], - database=os.environ["SURREAL_DATABASE"], - max_size=2.2**20, - encrypted=False, # Set to True if using SSL - ) - try: - yield connection - finally: - connection.socket.close() - - -def repo_query(query_str: str, vars: Optional[Dict[str, Any]] = None): - with db_connection() as connection: - try: - result = connection.query(query_str, vars) - return result - except Exception as e: - logger.critical(f"Query: {query_str}") - logger.exception(e) - raise - - -def repo_create(table: str, data: Dict[str, Any]): - query = f"CREATE {table} CONTENT {data};" - return repo_query(query) - - -def repo_upsert(table: str, data: Dict[str, Any]): - query = f"UPSERT {table} CONTENT {data};" - return repo_query(query) - - -def repo_update(id: str, data: Dict[str, Any]): - query = "UPDATE $id CONTENT $data;" - vars = {"id": id, "data": data} - return repo_query(query, vars) - - -def repo_delete(id: str): - query = "DELETE $id;" - vars = {"id": id} - return repo_query(query, vars) - - -def repo_relate(source: str, relationship: str, target: str, data: Optional[Dict] = {}): - query = f"RELATE {source}->{relationship}->{target} CONTENT $content;" - result = repo_query(query, {"content": data}) - return result diff --git a/test_commands.sh b/test_commands.sh deleted file mode 100755 index 828270b..0000000 --- a/test_commands.sh +++ /dev/null @@ -1,90 +0,0 @@ -#!/bin/bash - -echo "=== Testing Surreal Commands Integration ===" -echo "" - -# Base URL -BASE_URL="http://localhost:5055/api" - -# 1. Test text processing command -echo "1. Testing text processing command (uppercase)..." -curl -X POST "$BASE_URL/commands/jobs" \ - -H "Content-Type: application/json" \ - -d '{ - "command": "process_text", - "app": "open_notebook", - "input": { - "text": "Hello, this is a test message!", - "operation": "uppercase" - } - }' | jq . - -echo "" -echo "2. Testing text processing with delay (3 seconds)..." -curl -X POST "$BASE_URL/commands/jobs" \ - -H "Content-Type: application/json" \ - -d '{ - "command": "process_text", - "app": "open_notebook", - "input": { - "text": "Testing async behavior with delay", - "operation": "reverse", - "delay_seconds": 3 - } - }' | jq . - -echo "" -echo "3. Testing data analysis command..." -curl -X POST "$BASE_URL/commands/jobs" \ - -H "Content-Type: application/json" \ - -d '{ - "command": "analyze_data", - "app": "open_notebook", - "input": { - "numbers": [10, 20, 30, 40, 50], - "analysis_type": "basic" - } - }' | jq . - -echo "" -echo "4. Testing error scenario (empty numbers array)..." -curl -X POST "$BASE_URL/commands/jobs" \ - -H "Content-Type: application/json" \ - -d '{ - "command": "analyze_data", - "app": "open_notebook", - "input": { - "numbers": [], - "analysis_type": "basic" - } - }' | jq . - -echo "" -echo "5. Testing word count operation..." -curl -X POST "$BASE_URL/commands/jobs" \ - -H "Content-Type: application/json" \ - -d '{ - "command": "process_text", - "app": "open_notebook", - "input": { - "text": "This is a sample text with multiple words to count", - "operation": "word_count" - } - }' | jq . - -echo "" -echo "Please save the job_ids from above to check status!" -echo "" -echo "6. To check job status (replace JOB_ID with actual ID):" -echo "curl \"$BASE_URL/commands/jobs/{JOB_ID}\" | jq ." - -echo "" -echo "7. To list all jobs:" -echo "curl \"$BASE_URL/commands/jobs\" | jq ." - -echo "" -echo "=== Test Commands Complete ===" -echo "" -echo "Manual status check example:" -echo "Replace JOB_ID with one of the job IDs returned above:" -echo "curl \"$BASE_URL/commands/jobs/JOB_ID\" | jq ." \ No newline at end of file