diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..ef464e25 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,21 @@ +**/videos +**/artifacts +**/traces +**/inputs +**/har +**/.git +**/.github +**/.mypy_cache +**/.venv +**/.vscode +*.env* +# Streamlit ignores +**/secrets*.toml + +# Skyvern +docs +images + +.dockerignore +.gitignore +Dockerfile \ No newline at end of file diff --git a/.github/workflows/build-docker-image.yml b/.github/workflows/build-docker-image.yml new file mode 100644 index 00000000..f8bbb83a --- /dev/null +++ b/.github/workflows/build-docker-image.yml @@ -0,0 +1,53 @@ +name: Build Docker Image and Push to ECR + +on: + release: + types: [ published ] + +env: + AWS_REGION: us-east-1 + ECR_REPOSITORY: skyvern + REGISTRY_ALIAS: t6d4b5t4 # skyvern + +jobs: + run-ci: + uses: ./.github/workflows/ci.yml + + build-docker-image: + runs-on: ubuntu-latest + needs: [ run-ci ] + steps: + - name: Check out Git repository + uses: actions/checkout@v4 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ env.AWS_REGION }} + + - name: Login to Amazon ECR Public + id: login-ecr-public + uses: aws-actions/amazon-ecr-login@v2 + with: + registry-type: public + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Build, tag, and push image to Amazon Public ECR + id: build-image + uses: docker/build-push-action@v2 + env: + ECR_REGISTRY: ${{ steps.login-ecr-public.outputs.registry }} + with: + context: . + platforms: | + linux/amd64 + linux/arm64 + push: true + tags: | + ${{ env.ECR_REGISTRY}}/${{ env.REGISTRY_ALIAS }}/${{ env.ECR_REPOSITORY }}:${{ github.sha }} + ${{ env.ECR_REGISTRY}}/${{ env.REGISTRY_ALIAS }}/${{ env.ECR_REPOSITORY }}:${{ github.event.release.tag_name }} + ${{ env.ECR_REGISTRY}}/${{ env.REGISTRY_ALIAS }}/${{ env.ECR_REPOSITORY }}:latest diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fa894362..80f310ca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,6 +4,7 @@ name: Run tests and pre-commit # `branches: [main], then this actions runs _twice_ on pull requests, which is # annoying. on: + workflow_call: pull_request: push: branches: [main] diff --git a/.gitignore b/.gitignore index 4f9f733e..f237442a 100644 --- a/.gitignore +++ b/.gitignore @@ -164,6 +164,7 @@ artifacts/ traces/ *.pkl har/ +postgres-data # Streamlit ignores **/secrets*.toml diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..87123311 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,34 @@ +FROM python:3.11 as requirements-stage + +WORKDIR /tmp +RUN pip install poetry +COPY ./pyproject.toml /tmp/pyproject.toml +COPY ./poetry.lock /tmp/poetry.lock +RUN poetry export -f requirements.txt --output requirements.txt --without-hashes + +FROM python:3.11-slim-bookworm +WORKDIR /app +COPY --from=requirements-stage /tmp/requirements.txt /app/requirements.txt +RUN pip install --no-cache-dir --upgrade -r requirements.txt +RUN pip install --no-cache-dir streamlit +RUN playwright install-deps +RUN playwright install +RUN apt-get install -y xauth && apt-get clean + +COPY . /app + +ENV PYTHONPATH="/app:$PYTHONPATH" +ENV VIDEO_PATH=/data/videos +ENV HAR_PATH=/data/har +ENV ARTIFACT_STORAGE_PATH=/data/artifacts + +COPY ./entrypoint-skyvern.sh /app/entrypoint-skyvern.sh +RUN chmod +x /app/entrypoint-skyvern.sh + +COPY ./entrypoint-streamlit.sh /app/entrypoint-streamlit.sh +RUN chmod +x /app/entrypoint-streamlit.sh + +CMD [ "/bin/bash", "/app/entrypoint-skyvern.sh" ] + + + diff --git a/README.md b/README.md index d5e15a13..d8206786 100644 --- a/README.md +++ b/README.md @@ -105,6 +105,16 @@ Note: Our setup script does these two for you, but they are here for reference. ``` 1. Navigate to `http://localhost:8501` in your browser to start using the UI +## Docker Compose setup + +1. Fill in the LLM provider key on the [docker-compose.yml](./docker-compose.yml) +2. Run the following command: + ```bash + docker compose up -d + ``` +3. Navigate to `http://localhost:8501` in your browser to start using the UI + + ## Additional Setup for Contributors If you're looking to contribute to Skyvern, you'll need to install the pre-commit hooks to ensure code quality and consistency. You can do this by running the following command: ```bash diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..45877aad --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,64 @@ +version: '3.8' + +services: + postgres: + image: postgres:14-alpine + # comment out if you want to externally connect DB + # ports: + # - 5432:5432 + volumes: + - ./postgres-data:/var/lib/postgresql/data + environment: + - PGDATA=/var/lib/postgresql/data/pgdata + - POSTGRES_USER=skyvern + - POSTGRES_PASSWORD=skyvern + - POSTGRES_POSTGRES_DB=skyvern + healthcheck: + test: ["CMD-SHELL", "pg_isready -U skyvern"] + interval: 5s + timeout: 5s + retries: 5 + + skyvern: + image: public.ecr.aws/t6d4b5t4/skyvern:latest + restart: on-failure + # comment out if you want to externally call skyvern API + # ports: + # - 8000:8000 + volumes: + - ./artifacts:/data/artifacts + - ./videos:/data/videos + - ./har:/data/har + - ./.streamlit:/app/.streamlit + environment: + - DATABASE_STRING=postgresql+psycopg://skyvern:skyvern@postgres:5432/skyvern + - BROWSER_TYPE=chromium-headful + - ENABLE_OPENAI=true + - OPENAI_API_KEY= + # If you want to use other LLM provider, like azure and anthropic: + # - ENABLE_ANTHROPIC=true + # - ANTHROPIC_API_KEY= + # - ENABLE_AZURE=true + # - LLM_KEY=AZURE_OPENAI_GPT4V + # - AZURE_DEPLOYMENT= + # - AZURE_API_KEY= + # - AZURE_API_BASE= + # - AZURE_API_VERSION= + depends_on: + postgres: + condition: service_healthy + + streamlit: + image: public.ecr.aws/t6d4b5t4/skyvern:latest + restart: on-failure + ports: + - 8501:8501 + volumes: + - ./artifacts:/data/artifacts + - ./videos:/data/videos + - ./har:/data/har + - ./.streamlit:/app/.streamlit + command: ["/bin/bash", "entrypoint-streamlit.sh"] + depends_on: + - skyvern + diff --git a/entrypoint-skyvern.sh b/entrypoint-skyvern.sh new file mode 100644 index 00000000..50385118 --- /dev/null +++ b/entrypoint-skyvern.sh @@ -0,0 +1,20 @@ + +#!/bin/bash + +set -e + +# check alembic +alembic upgrade head +alembic check + +if [ ! -f ".streamlit/secrets.toml" ]; then + echo "Creating organization and API token..." + org_output=$(python scripts/create_organization.py Skyvern-Open-Source) + api_token=$(echo "$org_output" | awk '/token=/{gsub(/.*token='\''|'\''.*/, ""); print}') + # Update the secrets-open-source.toml file + echo -e "[skyvern]\nconfigs = [\n {\"env\" = \"local\", \"host\" = \"http://skyvern:8000/api/v1\", \"orgs\" = [{name=\"Skyvern\", cred=\"$api_token\"}]}\n]" > .streamlit/secrets.toml + echo ".streamlit/secrets.toml file updated with organization details." +fi + +# Run the command and pass in all three arguments +xvfb-run python -m skyvern.forge diff --git a/entrypoint-streamlit.sh b/entrypoint-streamlit.sh new file mode 100644 index 00000000..b0f62435 --- /dev/null +++ b/entrypoint-streamlit.sh @@ -0,0 +1,7 @@ + +#!/bin/bash + +set -e + +# Run the command and pass in all three arguments +streamlit run streamlit_app/visualizer/streamlit.py --server.port 8501 \ No newline at end of file