mirror of
https://github.com/cyclotruc/gitingest.git
synced 2026-04-26 15:40:40 +00:00
chore: add pre-commit config, type hints, badges, and lint codebase (#57)
* chore: add pre-commit config, type hints, badges, and lint codebase - Add .pre-commit-config.yaml and pyproject.toml for Black and isort - Add missing type hints throughout the code (Dict[...] for Python 3.8 compatibility) - Added badges and convert existing badges to use <a><img></a> format - Lint Markdown files - Lint Jinja templates with djlint * Resolve error and fix remaining type hint violations * Fix absolute imports and mock paths in test_clone.py to resolve test failures. * Replace deprecated 'dotenv' with 'python-dotenv' in requirements.txt to resolve installation errors.
This commit is contained in:
Filip Christiansen
GitHub
parent
96fb7fe4fb
commit
eb73a0cc1f
41 changed files with 1050 additions and 725 deletions
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -131,6 +131,7 @@ venv/
|
|||
ENV/
|
||||
env.bak/
|
||||
venv.bak/
|
||||
.python-version
|
||||
|
||||
# Spyder project settings
|
||||
.spyderproject
|
||||
|
|
|
|||
78
.pre-commit-config.yaml
Normal file
78
.pre-commit-config.yaml
Normal file
|
|
@ -0,0 +1,78 @@
|
|||
repos:
|
||||
- repo: https://github.com/pre-commit/pre-commit-hooks
|
||||
rev: v5.0.0
|
||||
hooks:
|
||||
# Files
|
||||
- id: check-added-large-files
|
||||
description: 'Prevent large files from being committed.'
|
||||
args: ['--maxkb=10000']
|
||||
- id: check-case-conflict
|
||||
description: 'Check for files that would conflict in case-insensitive filesystems.'
|
||||
- id: fix-byte-order-marker
|
||||
description: 'Remove utf-8 byte order marker.'
|
||||
- id: mixed-line-ending
|
||||
description: 'Replace mixed line ending.'
|
||||
|
||||
# Links
|
||||
- id: destroyed-symlinks
|
||||
description: 'Detect symlinks which are changed to regular files with a content of a path which that symlink was pointing to.'
|
||||
|
||||
# File files for parseable syntax: python
|
||||
- id: check-ast
|
||||
|
||||
# File and line endings
|
||||
- id: end-of-file-fixer
|
||||
description: 'Ensure that a file is either empty, or ends with one newline.'
|
||||
- id: trailing-whitespace
|
||||
description: 'Trim trailing whitespace.'
|
||||
|
||||
# Python
|
||||
- id: check-docstring-first
|
||||
description: 'Check a common error of defining a docstring after code.'
|
||||
- id: requirements-txt-fixer
|
||||
description: 'Sort entries in requirements.txt.'
|
||||
|
||||
- repo: https://github.com/MarcoGorelli/absolufy-imports
|
||||
rev: v0.3.1
|
||||
hooks:
|
||||
- id: absolufy-imports
|
||||
description: 'Automatically convert relative imports to absolute. (Use `args: [--never]` to revert.)'
|
||||
|
||||
- repo: https://github.com/psf/black
|
||||
rev: 24.10.0
|
||||
hooks:
|
||||
- id: black
|
||||
|
||||
- repo: https://github.com/asottile/pyupgrade
|
||||
rev: v3.19.1
|
||||
hooks:
|
||||
- id: pyupgrade
|
||||
description: 'Automatically upgrade syntax for newer versions.'
|
||||
args: [--py3-plus, --py36-plus, --py38-plus]
|
||||
|
||||
- repo: https://github.com/pre-commit/pygrep-hooks
|
||||
rev: v1.10.0
|
||||
hooks:
|
||||
- id: python-check-blanket-noqa
|
||||
description: 'Enforce that `noqa` annotations always occur with specific codes. Sample annotations: `# noqa: F401`, `# noqa: F401,W203`.'
|
||||
- id: python-check-blanket-type-ignore
|
||||
description: 'Enforce that `# type: ignore` annotations always occur with specific codes. Sample annotations: `# type: ignore[attr-defined]`, `# type: ignore[attr-defined, name-defined]`.'
|
||||
- id: python-use-type-annotations
|
||||
description: 'Enforce that python3.6+ type annotations are used instead of type comments.'
|
||||
|
||||
- repo: https://github.com/PyCQA/isort
|
||||
rev: 5.13.2
|
||||
hooks:
|
||||
- id: isort
|
||||
description: 'Sort imports alphabetically, and automatically separated into sections and by type.'
|
||||
|
||||
- repo: https://github.com/hadialqattan/pycln
|
||||
rev: v2.4.0
|
||||
hooks:
|
||||
- id: pycln
|
||||
description: 'Remove unused import statements.'
|
||||
|
||||
- repo: https://github.com/djlint/djLint
|
||||
rev: v1.36.4
|
||||
hooks:
|
||||
- id: djlint-reformat-jinja
|
||||
|
|
@ -60,7 +60,7 @@ representative at an online or offline event.
|
|||
|
||||
Instances of abusive, harassing, or otherwise unacceptable behavior may be
|
||||
reported to the community leaders responsible for enforcement at
|
||||
romain@coderamp.io.
|
||||
<romain@coderamp.io>.
|
||||
All complaints will be reviewed and investigated promptly and fairly.
|
||||
|
||||
All community leaders are obligated to respect the privacy and security of the
|
||||
|
|
@ -114,15 +114,13 @@ the community.
|
|||
|
||||
## Attribution
|
||||
|
||||
This Code of Conduct is adapted from the [Contributor Covenant][homepage],
|
||||
This Code of Conduct is adapted from the [Contributor Covenant](https://www.contributor-covenant.org),
|
||||
version 2.0, available at
|
||||
https://www.contributor-covenant.org/version/2/0/code_of_conduct.html.
|
||||
<https://www.contributor-covenant.org/version/2/0/code_of_conduct.html>.
|
||||
|
||||
Community Impact Guidelines were inspired by [Mozilla's code of conduct
|
||||
enforcement ladder](https://github.com/mozilla/diversity).
|
||||
|
||||
[homepage]: https://www.contributor-covenant.org
|
||||
|
||||
For answers to common questions about this code of conduct, see the FAQ at
|
||||
https://www.contributor-covenant.org/faq. Translations are available at
|
||||
https://www.contributor-covenant.org/translations.
|
||||
<https://www.contributor-covenant.org/faq>. Translations are available at
|
||||
<https://www.contributor-covenant.org/translations>.
|
||||
|
|
|
|||
62
README.md
62
README.md
|
|
@ -1,34 +1,56 @@
|
|||
[](https://gitingest.com/)
|
||||
[](https://gitingest.com)
|
||||
|
||||
|
||||
<!-- License -->
|
||||
<a href="https://github.com/cyclotruc/gitingest/blob/main/LICENSE">
|
||||
<img alt="License" src="https://img.shields.io/badge/license-MIT-blue.svg" />
|
||||
</a>
|
||||
<!-- PyPI version -->
|
||||
<a href="https://badge.fury.io/py/gitingest">
|
||||
<img src="https://badge.fury.io/py/gitingest.svg" alt="PyPI version" />
|
||||
</a>
|
||||
<!-- Downloads -->
|
||||
<a href="https://pepy.tech/project/gitingest">
|
||||
<img src="https://pepy.tech/badge/gitingest" alt="Downloads" />
|
||||
</a>
|
||||
<!-- GitHub issues -->
|
||||
<a href="https://github.com/cyclotruc/gitingest/issues">
|
||||
<img src="https://img.shields.io/github/issues/cyclotruc/gitingest" alt="GitHub issues" />
|
||||
</a>
|
||||
<!-- Black code style -->
|
||||
<a href="https://github.com/psf/black">
|
||||
<img alt="Code style: black" src="https://img.shields.io/badge/code%20style-black-000000.svg" />
|
||||
</a>
|
||||
|
||||
<!-- Discord -->
|
||||
<a href="https://discord.com/invite/zerRaGK9EC">
|
||||
<img src="https://dcbadge.limes.pink/api/server/https://discord.com/invite/zerRaGK9EC" alt="Discord" />
|
||||
</a>
|
||||
|
||||
# GitIngest
|
||||
|
||||
# GitIngest 🔍
|
||||
Turn any Git repository into a prompt-friendly text ingest for LLMs.
|
||||
|
||||
You can also replace `hub` with `ingest` in any github url to access the coresponding digest
|
||||
|
||||
[gitingest.com](https://gitingest.com/)
|
||||
|
||||
[gitingest.com](https://gitingest.com)
|
||||
|
||||
## 🚀 Features
|
||||
|
||||
- **Easy code context**: Get a text digest from a git repository URL or a directory
|
||||
- **Smart Formatting**: Optimized output format for LLM prompts
|
||||
- **Statistics about**: :
|
||||
- **Statistics about**:
|
||||
- File and directory structure
|
||||
- Size of the extract
|
||||
- Token count
|
||||
- **CLI tool**: Run it as a command (Currently on Linux only)
|
||||
- **Python package**: Import it in your code
|
||||
|
||||
|
||||
## 📦 Installation
|
||||
|
||||
```
|
||||
``` bash
|
||||
pip install gitingest
|
||||
```
|
||||
|
||||
|
||||
## 💡 Command Line usage
|
||||
|
||||
The `gitingest` command line tool allows you to analyze codebases and create a text dump of their contents.
|
||||
|
|
@ -46,10 +68,8 @@ gitingest --help
|
|||
|
||||
This will write the digest in a text file (default `digest.txt`) in your current working directory.
|
||||
|
||||
|
||||
## 🐛 Python package usage
|
||||
|
||||
|
||||
```python
|
||||
from gitingest import ingest
|
||||
|
||||
|
|
@ -61,24 +81,27 @@ summary, tree, content = ingest("https://github.com/cyclotruc/gitingest")
|
|||
|
||||
By default, this won't write a file but can be enabled with the `output` argument
|
||||
|
||||
|
||||
## 🛠️ Using
|
||||
|
||||
- Tailwind CSS - Frontend
|
||||
- [FastAPI](https://github.com/fastapi/fastapi) - Backend framework
|
||||
- [tiktoken](https://github.com/openai/tiktoken) - Token estimation
|
||||
- [apianalytics.dev](https://www.apianalytics.dev/) - Simple Analytics
|
||||
|
||||
|
||||
## 🌐 Self-host
|
||||
|
||||
1. Build the image:
|
||||
```
|
||||
|
||||
``` bash
|
||||
docker build -t gitingest .
|
||||
```
|
||||
|
||||
2. Run the container:
|
||||
```
|
||||
|
||||
``` bash
|
||||
docker run -d --name gitingest -p 8000:8000 gitingest
|
||||
```
|
||||
|
||||
The application will be available at `http://localhost:8000`
|
||||
Ensure environment variables are set before running the application or deploying it via Docker.
|
||||
|
||||
|
|
@ -92,7 +115,7 @@ Gitingest aims to be friendly for first time contributors, with a simple python
|
|||
|
||||
1. Provide your feedback and ideas on discord
|
||||
2. Open an Issue on github to report a bug
|
||||
2. Create a Pull request
|
||||
3. Create a Pull request
|
||||
- Fork the repository
|
||||
- Make your changes and test them locally
|
||||
- Open a pull request for review and feedback
|
||||
|
|
@ -100,6 +123,7 @@ Gitingest aims to be friendly for first time contributors, with a simple python
|
|||
### 🔧 Local dev
|
||||
|
||||
#### Environment Configuration
|
||||
|
||||
- **`ALLOWED_HOSTS`**: Specify allowed hostnames for the application. Default: `"gitingest.com,*.gitingest.com,gitdigest.dev,localhost"`.
|
||||
You can configure the application using the following environment variables:
|
||||
|
||||
|
|
@ -108,23 +132,25 @@ ALLOWED_HOSTS="gitingest.local,localhost"
|
|||
```
|
||||
|
||||
#### Run locally
|
||||
|
||||
1. Clone the repository
|
||||
|
||||
```bash
|
||||
git clone https://github.com/cyclotruc/gitingest.git
|
||||
cd gitingest
|
||||
```
|
||||
|
||||
2. Install dependencies
|
||||
|
||||
```bash
|
||||
pip install -r requirements.txt
|
||||
```
|
||||
|
||||
3. Run the application:
|
||||
|
||||
```bash
|
||||
cd src
|
||||
uvicorn main:app --reload
|
||||
```
|
||||
|
||||
The frontend will be available at `localhost:8000`
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -2,4 +2,4 @@
|
|||
|
||||
## Reporting a Vulnerability
|
||||
|
||||
If you have discovered a vulnerability inside the project, report it privately at romain@coderamp.io. This way the maintainer can work on a proper fix without disclosing the problem to the public before it has been solved.
|
||||
If you have discovered a vulnerability inside the project, report it privately at <romain@coderamp.io>. This way the maintainer can work on a proper fix without disclosing the problem to the public before it has been solved.
|
||||
|
|
|
|||
17
pyproject.toml
Normal file
17
pyproject.toml
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
[tool.pylint.format]
|
||||
max-line-length = 119
|
||||
|
||||
[tool.pycln]
|
||||
all = true
|
||||
|
||||
[tool.isort]
|
||||
profile = "black"
|
||||
line_length = 119
|
||||
remove_redundant_aliases = true
|
||||
float_to_top = true
|
||||
order_by_type = true
|
||||
filter_files = true
|
||||
|
||||
[tool.black]
|
||||
line-length = 119
|
||||
skip-string-normalization = true
|
||||
|
|
@ -3,7 +3,6 @@ pythonpath = src
|
|||
testpaths = src/gitingest/tests
|
||||
asyncio_mode = auto
|
||||
|
||||
|
||||
python_files = test_*.py
|
||||
python_classes = Test*
|
||||
python_functions = test_*
|
||||
|
|
@ -1,8 +1,13 @@
|
|||
fastapi[standard]
|
||||
uvicorn
|
||||
black
|
||||
click>=8.0.0
|
||||
djlint
|
||||
fastapi-analytics
|
||||
slowapi
|
||||
tiktoken
|
||||
fastapi[standard]
|
||||
pre-commit
|
||||
pytest
|
||||
pytest-asyncio
|
||||
click>=8.0.0
|
||||
python-dotenv
|
||||
slowapi
|
||||
starlette
|
||||
tiktoken
|
||||
uvicorn
|
||||
|
|
|
|||
2
setup.py
2
setup.py
|
|
@ -1,4 +1,4 @@
|
|||
from setuptools import setup, find_packages
|
||||
from setuptools import find_packages, setup
|
||||
|
||||
setup(
|
||||
name="gitingest",
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
MAX_DISPLAY_SIZE = 300000
|
||||
MAX_DISPLAY_SIZE = 300_000
|
||||
TMP_BASE_PATH = "../tmp"
|
||||
|
||||
EXAMPLE_REPOS = [
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
from .ingest_from_query import ingest_from_query
|
||||
from .clone import clone_repo
|
||||
from .parse_query import parse_query
|
||||
from .ingest import ingest
|
||||
from gitingest.clone import clone_repo
|
||||
from gitingest.ingest import ingest
|
||||
from gitingest.ingest_from_query import ingest_from_query
|
||||
from gitingest.parse_query import parse_query
|
||||
|
||||
__all__ = ["ingest_from_query", "clone_repo", "parse_query", "ingest"]
|
||||
|
|
@ -1,10 +1,12 @@
|
|||
import os
|
||||
import pathlib
|
||||
from typing import Optional, Tuple
|
||||
|
||||
import click
|
||||
|
||||
from gitingest.ignore_patterns import DEFAULT_IGNORE_PATTERNS
|
||||
from gitingest.ingest import ingest
|
||||
from gitingest.ingest_from_query import MAX_FILE_SIZE
|
||||
from gitingest.parse_query import DEFAULT_IGNORE_PATTERNS
|
||||
|
||||
|
||||
def normalize_pattern(pattern: str) -> str:
|
||||
pattern = pattern.strip()
|
||||
|
|
@ -13,13 +15,20 @@ def normalize_pattern(pattern: str) -> str:
|
|||
pattern += "*"
|
||||
return pattern
|
||||
|
||||
|
||||
@click.command()
|
||||
@click.argument('source', type=str, required=True)
|
||||
@click.option('--output', '-o', default=None, help='Output file path (default: <repo_name>.txt in current directory)')
|
||||
@click.option('--max-size', '-s', default=MAX_FILE_SIZE, help='Maximum file size to process in bytes')
|
||||
@click.option('--exclude-pattern', '-e', multiple=True, help='Patterns to exclude')
|
||||
@click.option('--include-pattern', '-i', multiple=True, help='Patterns to include')
|
||||
def main(source, output, max_size, exclude_pattern, include_pattern):
|
||||
def main(
|
||||
source: str,
|
||||
output: Optional[str],
|
||||
max_size: int,
|
||||
exclude_pattern: Tuple[str, ...],
|
||||
include_pattern: Tuple[str, ...],
|
||||
) -> None:
|
||||
"""Analyze a directory and create a text dump of its contents."""
|
||||
try:
|
||||
# Combine default and custom ignore patterns
|
||||
|
|
@ -38,5 +47,6 @@ def main(source, output, max_size, exclude_pattern, include_pattern):
|
|||
click.echo(f"Error: {str(e)}", err=True)
|
||||
raise click.Abort()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
|
@ -1,10 +1,11 @@
|
|||
import asyncio
|
||||
from typing import Tuple
|
||||
from typing import Any, Dict, Tuple
|
||||
|
||||
from gitingest.utils import async_timeout
|
||||
|
||||
CLONE_TIMEOUT = 20
|
||||
|
||||
|
||||
async def check_repo_exists(url: str) -> bool:
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
"curl",
|
||||
|
|
@ -20,8 +21,9 @@ async def check_repo_exists(url: str) -> bool:
|
|||
stdout_str = stdout.decode()
|
||||
return "HTTP/1.1 404" not in stdout_str and "HTTP/2 404" not in stdout_str
|
||||
|
||||
|
||||
@async_timeout(CLONE_TIMEOUT)
|
||||
async def clone_repo(query: dict) -> str:
|
||||
async def clone_repo(query: Dict[str, Any]) -> Tuple[bytes, bytes]:
|
||||
if not await check_repo_exists(query['url']):
|
||||
raise ValueError("Repository not found, make sure it is public")
|
||||
|
||||
|
|
|
|||
102
src/gitingest/ignore_patterns.py
Normal file
102
src/gitingest/ignore_patterns.py
Normal file
|
|
@ -0,0 +1,102 @@
|
|||
from typing import List
|
||||
|
||||
DEFAULT_IGNORE_PATTERNS: List[str] = [
|
||||
# Python
|
||||
'*.pyc',
|
||||
'*.pyo',
|
||||
'*.pyd',
|
||||
'__pycache__',
|
||||
'.pytest_cache',
|
||||
'.coverage',
|
||||
'.tox',
|
||||
'.nox',
|
||||
'.mypy_cache',
|
||||
'.ruff_cache',
|
||||
'.hypothesis',
|
||||
'poetry.lock',
|
||||
'Pipfile.lock',
|
||||
# JavaScript/Node
|
||||
'node_modules',
|
||||
'bower_components',
|
||||
'package-lock.json',
|
||||
'yarn.lock',
|
||||
'.npm',
|
||||
'.yarn',
|
||||
'.pnpm-store',
|
||||
# Version control
|
||||
'.git',
|
||||
'.svn',
|
||||
'.hg',
|
||||
'.gitignore',
|
||||
'.gitattributes',
|
||||
'.gitmodules',
|
||||
# Images and media
|
||||
'*.svg',
|
||||
'*.png',
|
||||
'*.jpg',
|
||||
'*.jpeg',
|
||||
'*.gif',
|
||||
'*.ico',
|
||||
'*.pdf',
|
||||
'*.mov',
|
||||
'*.mp4',
|
||||
'*.mp3',
|
||||
'*.wav',
|
||||
# Virtual environments
|
||||
'venv',
|
||||
'.venv',
|
||||
'env',
|
||||
'.env',
|
||||
'virtualenv',
|
||||
# IDEs and editors
|
||||
'.idea',
|
||||
'.vscode',
|
||||
'.vs',
|
||||
'*.swp',
|
||||
'*.swo',
|
||||
'*.swn',
|
||||
'.settings',
|
||||
'.project',
|
||||
'.classpath',
|
||||
'*.sublime-*',
|
||||
# Temporary and cache files
|
||||
'*.log',
|
||||
'*.bak',
|
||||
'*.swp',
|
||||
'*.tmp',
|
||||
'*.temp',
|
||||
'.cache',
|
||||
'.sass-cache',
|
||||
'.eslintcache',
|
||||
'.DS_Store',
|
||||
'Thumbs.db',
|
||||
'desktop.ini',
|
||||
# Build directories and artifacts
|
||||
'build',
|
||||
'dist',
|
||||
'target',
|
||||
'out',
|
||||
'*.egg-info',
|
||||
'*.egg',
|
||||
'*.whl',
|
||||
'*.so',
|
||||
'*.dylib',
|
||||
'*.dll',
|
||||
'*.class',
|
||||
# Documentation
|
||||
'site-packages',
|
||||
'.docusaurus',
|
||||
'.next',
|
||||
'.nuxt',
|
||||
# Other common patterns
|
||||
## Minified files
|
||||
'*.min.js',
|
||||
'*.min.css',
|
||||
## Source maps
|
||||
'*.map',
|
||||
## Terraform
|
||||
'.terraform',
|
||||
'*.tfstate*',
|
||||
## Dependencies in various languages
|
||||
'vendor/',
|
||||
]
|
||||
|
|
@ -1,17 +1,35 @@
|
|||
import asyncio
|
||||
import inspect
|
||||
import shutil
|
||||
from typing import Union, List
|
||||
from pathlib import Path
|
||||
from typing import List, Optional, Tuple, Union
|
||||
|
||||
from .ingest_from_query import ingest_from_query
|
||||
from .clone import clone_repo
|
||||
from .parse_query import parse_query
|
||||
from gitingest.clone import clone_repo
|
||||
from gitingest.ingest_from_query import ingest_from_query
|
||||
from gitingest.parse_query import parse_query
|
||||
|
||||
def ingest(source: str, max_file_size: int = 10 * 1024 * 1024, include_patterns: Union[List[str], str] = None, exclude_patterns: Union[List[str], str] = None, output: str = None) -> str:
|
||||
|
||||
def ingest(
|
||||
source: str,
|
||||
max_file_size: int = 10 * 1024 * 1024,
|
||||
include_patterns: Union[List[str], str, None] = None,
|
||||
exclude_patterns: Union[List[str], str, None] = None,
|
||||
output: Optional[str] = None,
|
||||
) -> Tuple[str, str, str]:
|
||||
try:
|
||||
query = parse_query(source, max_file_size, False, include_patterns, exclude_patterns)
|
||||
query = parse_query(
|
||||
source=source,
|
||||
max_file_size=max_file_size,
|
||||
from_web=False,
|
||||
include_patterns=include_patterns,
|
||||
ignore_patterns=exclude_patterns,
|
||||
)
|
||||
if query['url']:
|
||||
asyncio.run(clone_repo(query))
|
||||
clone_result = clone_repo(query)
|
||||
if inspect.iscoroutine(clone_result):
|
||||
asyncio.run(clone_result)
|
||||
else:
|
||||
raise TypeError("clone_repo did not return a coroutine as expected.")
|
||||
|
||||
summary, tree, content = ingest_from_query(query)
|
||||
|
||||
|
|
@ -20,6 +38,7 @@ def ingest(source: str, max_file_size: int = 10 * 1024 * 1024, include_patterns:
|
|||
f.write(tree + "\n" + content)
|
||||
|
||||
return summary, tree, content
|
||||
|
||||
finally:
|
||||
# Clean up the temporary directory if it was created
|
||||
if query['url']:
|
||||
|
|
|
|||
|
|
@ -1,12 +1,12 @@
|
|||
import os
|
||||
from fnmatch import fnmatch
|
||||
from typing import Dict, List, Union
|
||||
import tiktoken
|
||||
from typing import Any, Dict, List, Optional, Set, Tuple
|
||||
|
||||
import tiktoken
|
||||
|
||||
MAX_FILE_SIZE = 10 * 1024 * 1024 # 10 MB
|
||||
MAX_DIRECTORY_DEPTH = 20 # Maximum depth of directory traversal
|
||||
MAX_FILES = 10000 # Maximum number of files to process
|
||||
MAX_FILES = 10_000 # Maximum number of files to process
|
||||
MAX_TOTAL_SIZE_BYTES = 500 * 1024 * 1024 # 500 MB
|
||||
|
||||
|
||||
|
|
@ -18,6 +18,7 @@ def should_include(path: str, base_path: str, include_patterns: List[str]) -> bo
|
|||
include = True
|
||||
return include
|
||||
|
||||
|
||||
def should_exclude(path: str, base_path: str, ignore_patterns: List[str]) -> bool:
|
||||
rel_path = path.replace(base_path, "").lstrip(os.sep)
|
||||
for pattern in ignore_patterns:
|
||||
|
|
@ -27,6 +28,7 @@ def should_exclude(path: str, base_path: str, ignore_patterns: List[str]) -> boo
|
|||
return True
|
||||
return False
|
||||
|
||||
|
||||
def is_safe_symlink(symlink_path: str, base_path: str) -> bool:
|
||||
"""Check if a symlink points to a location within the base directory."""
|
||||
try:
|
||||
|
|
@ -37,23 +39,32 @@ def is_safe_symlink(symlink_path: str, base_path: str) -> bool:
|
|||
# If there's any error resolving the paths, consider it unsafe
|
||||
return False
|
||||
|
||||
|
||||
def is_text_file(file_path: str) -> bool:
|
||||
"""Determines if a file is likely a text file based on its content."""
|
||||
try:
|
||||
with open(file_path, 'rb') as file:
|
||||
chunk = file.read(1024)
|
||||
return not bool(chunk.translate(None, bytes([7, 8, 9, 10, 12, 13, 27] + list(range(0x20, 0x100)))))
|
||||
except IOError:
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def read_file_content(file_path: str) -> str:
|
||||
try:
|
||||
with open(file_path, 'r', encoding='utf-8', errors='ignore') as f:
|
||||
with open(file_path, encoding='utf-8', errors='ignore') as f:
|
||||
return f.read()
|
||||
except Exception as e:
|
||||
return f"Error reading file: {str(e)}"
|
||||
|
||||
def scan_directory(path: str, query: dict, seen_paths: set = None, depth: int = 0, stats: Dict = None) -> Dict:
|
||||
|
||||
def scan_directory(
|
||||
path: str,
|
||||
query: Dict[str, Any],
|
||||
seen_paths: Optional[Set[str]] = None,
|
||||
depth: int = 0,
|
||||
stats: Optional[Dict[str, int]] = None,
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Recursively analyzes a directory and its contents with safety limits."""
|
||||
if seen_paths is None:
|
||||
seen_paths = set()
|
||||
|
|
@ -76,6 +87,7 @@ def scan_directory(path: str, query: dict, seen_paths: set = None, depth: int =
|
|||
if real_path in seen_paths:
|
||||
print(f"Skipping already visited path: {path}")
|
||||
return None
|
||||
|
||||
seen_paths.add(real_path)
|
||||
|
||||
result = {
|
||||
|
|
@ -86,7 +98,7 @@ def scan_directory(path: str, query: dict, seen_paths: set = None, depth: int =
|
|||
"file_count": 0,
|
||||
"dir_count": 0,
|
||||
"path": path,
|
||||
"ignore_content": False
|
||||
"ignore_content": False,
|
||||
}
|
||||
|
||||
ignore_patterns = query['ignore_patterns']
|
||||
|
|
@ -137,14 +149,20 @@ def scan_directory(path: str, query: dict, seen_paths: set = None, depth: int =
|
|||
"type": "file",
|
||||
"size": file_size,
|
||||
"content": content,
|
||||
"path": item_path
|
||||
"path": item_path,
|
||||
}
|
||||
result["children"].append(child)
|
||||
result["size"] += file_size
|
||||
result["file_count"] += 1
|
||||
|
||||
elif os.path.isdir(real_path):
|
||||
subdir = scan_directory(real_path, query, seen_paths, depth + 1, stats)
|
||||
subdir = scan_directory(
|
||||
path=real_path,
|
||||
query=query,
|
||||
seen_paths=seen_paths,
|
||||
depth=depth + 1,
|
||||
stats=stats,
|
||||
)
|
||||
if subdir and (not include_patterns or subdir["file_count"] > 0):
|
||||
subdir["name"] = item
|
||||
subdir["path"] = item_path
|
||||
|
|
@ -175,14 +193,20 @@ def scan_directory(path: str, query: dict, seen_paths: set = None, depth: int =
|
|||
"type": "file",
|
||||
"size": file_size,
|
||||
"content": content,
|
||||
"path": item_path
|
||||
"path": item_path,
|
||||
}
|
||||
result["children"].append(child)
|
||||
result["size"] += file_size
|
||||
result["file_count"] += 1
|
||||
|
||||
elif os.path.isdir(item_path):
|
||||
subdir = scan_directory(item_path, query, seen_paths, depth + 1, stats)
|
||||
subdir = scan_directory(
|
||||
path=item_path,
|
||||
query=query,
|
||||
seen_paths=seen_paths,
|
||||
depth=depth + 1,
|
||||
stats=stats,
|
||||
)
|
||||
if subdir and (not include_patterns or subdir["file_count"] > 0):
|
||||
result["children"].append(subdir)
|
||||
result["size"] += subdir["size"]
|
||||
|
|
@ -194,7 +218,13 @@ def scan_directory(path: str, query: dict, seen_paths: set = None, depth: int =
|
|||
|
||||
return result
|
||||
|
||||
def extract_files_content(query: dict, node: Dict, max_file_size: int, files: List = None) -> List[Dict]:
|
||||
|
||||
def extract_files_content(
|
||||
query: Dict[str, Any],
|
||||
node: Dict[str, Any],
|
||||
max_file_size: int,
|
||||
files: Optional[List[Dict[str, Any]]] = None,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Recursively collects all text files with their contents."""
|
||||
if files is None:
|
||||
files = []
|
||||
|
|
@ -204,17 +234,21 @@ def extract_files_content(query: dict, node: Dict, max_file_size: int, files: Li
|
|||
if node["size"] > max_file_size:
|
||||
content = None
|
||||
|
||||
files.append({
|
||||
files.append(
|
||||
{
|
||||
"path": node["path"].replace(query['local_path'], ""),
|
||||
"content": content,
|
||||
"size": node["size"]
|
||||
})
|
||||
"size": node["size"],
|
||||
},
|
||||
)
|
||||
elif node["type"] == "directory":
|
||||
for child in node["children"]:
|
||||
extract_files_content(query, child, max_file_size, files)
|
||||
extract_files_content(query=query, node=child, max_file_size=max_file_size, files=files)
|
||||
|
||||
return files
|
||||
|
||||
def create_file_content_string(files: List[Dict]) -> str:
|
||||
|
||||
def create_file_content_string(files: List[Dict[str, Any]]) -> str:
|
||||
"""Creates a formatted string of file contents with separators."""
|
||||
output = ""
|
||||
separator = "=" * 48 + "\n"
|
||||
|
|
@ -223,6 +257,7 @@ def create_file_content_string(files: List[Dict]) -> str:
|
|||
for file in files:
|
||||
if not file['content']:
|
||||
continue
|
||||
|
||||
if file['path'].lower() == '/readme.md':
|
||||
output += separator
|
||||
output += f"File: {file['path']}\n"
|
||||
|
|
@ -234,6 +269,7 @@ def create_file_content_string(files: List[Dict]) -> str:
|
|||
for file in files:
|
||||
if not file['content'] or file['path'].lower() == '/readme.md':
|
||||
continue
|
||||
|
||||
output += separator
|
||||
output += f"File: {file['path']}\n"
|
||||
output += separator
|
||||
|
|
@ -241,12 +277,14 @@ def create_file_content_string(files: List[Dict]) -> str:
|
|||
|
||||
return output
|
||||
|
||||
def create_summary_string(query: dict, nodes: Dict, files: List[Dict]) -> str:
|
||||
|
||||
def create_summary_string(query: Dict[str, Any], nodes: Dict[str, Any], files: List[Dict[str, Any]]) -> str:
|
||||
"""Creates a summary string with file counts and content size."""
|
||||
if "user_name" in query:
|
||||
summary = f"Repository: {query['user_name']}/{query['repo_name']}\n"
|
||||
else:
|
||||
summary = f"Repository: {query['slug']}\n"
|
||||
|
||||
summary += f"Files analyzed: {nodes['file_count']}\n"
|
||||
|
||||
if 'subpath' in query and query['subpath'] != '/':
|
||||
|
|
@ -255,11 +293,19 @@ def create_summary_string(query: dict, nodes: Dict, files: List[Dict]) -> str:
|
|||
summary += f"Commit: {query['commit']}\n"
|
||||
elif 'branch' in query and query['branch'] != 'main' and query['branch'] != 'master' and query['branch']:
|
||||
summary += f"Branch: {query['branch']}\n"
|
||||
|
||||
return summary
|
||||
|
||||
def create_tree_structure(query: dict, node: Dict, prefix: str = "", is_last: bool = True) -> str:
|
||||
|
||||
def create_tree_structure(
|
||||
query: Dict[str, Any],
|
||||
node: Dict[str, Any],
|
||||
prefix: str = "",
|
||||
is_last: bool = True,
|
||||
) -> str:
|
||||
"""Creates a tree-like string representation of the file structure."""
|
||||
tree = ""
|
||||
|
||||
if not node["name"]:
|
||||
node["name"] = query['slug']
|
||||
|
||||
|
|
@ -267,6 +313,7 @@ def create_tree_structure(query: dict, node: Dict, prefix: str = "", is_last: bo
|
|||
current_prefix = "└── " if is_last else "├── "
|
||||
name = node["name"] + "/" if node["type"] == "directory" else node["name"]
|
||||
tree += prefix + current_prefix + name + "\n"
|
||||
|
||||
if node["type"] == "directory":
|
||||
# Adjust prefix only if we added a node name
|
||||
new_prefix = prefix + (" " if is_last else "│ ") if node["name"] else prefix
|
||||
|
|
@ -276,25 +323,29 @@ def create_tree_structure(query: dict, node: Dict, prefix: str = "", is_last: bo
|
|||
|
||||
return tree
|
||||
|
||||
def generate_token_string(context_string: str) -> str:
|
||||
|
||||
def generate_token_string(context_string: str) -> Optional[str]:
|
||||
"""Returns the number of tokens in a text string."""
|
||||
formatted_tokens = ""
|
||||
try:
|
||||
encoding = tiktoken.get_encoding("cl100k_base", )
|
||||
encoding = tiktoken.get_encoding("cl100k_base")
|
||||
total_tokens = len(encoding.encode(context_string, disallowed_special=()))
|
||||
|
||||
except Exception as e:
|
||||
print(e)
|
||||
return None
|
||||
if total_tokens > 1000000:
|
||||
formatted_tokens = f"{total_tokens/1000000:.1f}M"
|
||||
elif total_tokens > 1000:
|
||||
formatted_tokens = f"{total_tokens/1000:.1f}k"
|
||||
|
||||
if total_tokens > 1_000_000:
|
||||
formatted_tokens = f"{total_tokens / 1_000_000:.1f}M"
|
||||
elif total_tokens > 1_000:
|
||||
formatted_tokens = f"{total_tokens / 1_000:.1f}k"
|
||||
else:
|
||||
formatted_tokens = f"{total_tokens}"
|
||||
|
||||
return formatted_tokens
|
||||
|
||||
def ingest_single_file(path: str, query: dict) -> Dict:
|
||||
|
||||
def ingest_single_file(path: str, query: Dict[str, Any]) -> Tuple[str, str, str]:
|
||||
if not os.path.isfile(path):
|
||||
raise ValueError(f"Path {path} is not a file")
|
||||
|
||||
|
|
@ -310,7 +361,7 @@ def ingest_single_file(path: str, query: dict) -> Dict:
|
|||
file_info = {
|
||||
"path": path.replace(query['local_path'], ""),
|
||||
"content": content,
|
||||
"size": file_size
|
||||
"size": file_size,
|
||||
}
|
||||
|
||||
summary = (
|
||||
|
|
@ -326,11 +377,15 @@ def ingest_single_file(path: str, query: dict) -> Dict:
|
|||
formatted_tokens = generate_token_string(files_content)
|
||||
if formatted_tokens:
|
||||
summary += f"\nEstimated tokens: {formatted_tokens}"
|
||||
return (summary, tree, files_content)
|
||||
|
||||
def ingest_directory(path: str, query: dict) -> Dict:
|
||||
nodes = scan_directory(path, query)
|
||||
files = extract_files_content(query, nodes, query['max_file_size'])
|
||||
return summary, tree, files_content
|
||||
|
||||
|
||||
def ingest_directory(path: str, query: Dict[str, Any]) -> Tuple[str, str, str]:
|
||||
nodes = scan_directory(path=path, query=query)
|
||||
if not nodes:
|
||||
raise ValueError(f"No files found in {path}")
|
||||
files = extract_files_content(query=query, node=nodes, max_file_size=query['max_file_size'])
|
||||
summary = create_summary_string(query, nodes, files)
|
||||
tree = "Directory structure:\n" + create_tree_structure(query, nodes)
|
||||
files_content = create_file_content_string(files)
|
||||
|
|
@ -338,9 +393,11 @@ def ingest_directory(path: str, query: dict) -> Dict:
|
|||
formatted_tokens = generate_token_string(tree + files_content)
|
||||
if formatted_tokens:
|
||||
summary += f"\nEstimated tokens: {formatted_tokens}"
|
||||
return (summary, tree, files_content)
|
||||
|
||||
def ingest_from_query(query: dict) -> Dict:
|
||||
return summary, tree, files_content
|
||||
|
||||
|
||||
def ingest_from_query(query: Dict[str, Any]) -> Tuple[str, str, str]:
|
||||
"""Main entry point for analyzing a codebase directory or single file."""
|
||||
path = f"{query['local_path']}{query['subpath']}"
|
||||
if not os.path.exists(path):
|
||||
|
|
@ -348,6 +405,5 @@ def ingest_from_query(query: dict) -> Dict:
|
|||
|
||||
if query.get('type') == 'blob':
|
||||
return ingest_single_file(path, query)
|
||||
else:
|
||||
return ingest_directory(path, query)
|
||||
|
||||
return ingest_directory(path, query)
|
||||
|
|
|
|||
|
|
@ -1,55 +1,13 @@
|
|||
from typing import List, Union
|
||||
import uuid
|
||||
import os
|
||||
import uuid
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
|
||||
|
||||
DEFAULT_IGNORE_PATTERNS = [
|
||||
# Python
|
||||
'*.pyc', '*.pyo', '*.pyd', '__pycache__', '.pytest_cache', '.coverage',
|
||||
'.tox', '.nox', '.mypy_cache', '.ruff_cache', '.hypothesis',
|
||||
'poetry.lock', 'Pipfile.lock',
|
||||
|
||||
# JavaScript/Node
|
||||
'node_modules', 'bower_components', 'package-lock.json', 'yarn.lock',
|
||||
'.npm', '.yarn', '.pnpm-store',
|
||||
|
||||
# Version control
|
||||
'.git', '.svn', '.hg', '.gitignore', '.gitattributes', '.gitmodules',
|
||||
|
||||
# Images and media
|
||||
'*.svg', '*.png', '*.jpg', '*.jpeg', '*.gif', '*.ico', '*.pdf',
|
||||
'*.mov', '*.mp4', '*.mp3', '*.wav',
|
||||
|
||||
# Virtual environments
|
||||
'venv', '.venv', 'env', '.env', 'virtualenv',
|
||||
|
||||
# IDEs and editors
|
||||
'.idea', '.vscode', '.vs', '*.swp', '*.swo', '*.swn',
|
||||
'.settings', '.project', '.classpath', '*.sublime-*',
|
||||
|
||||
# Temporary and cache files
|
||||
'*.log', '*.bak', '*.swp', '*.tmp', '*.temp',
|
||||
'.cache', '.sass-cache', '.eslintcache',
|
||||
'.DS_Store', 'Thumbs.db', 'desktop.ini',
|
||||
|
||||
# Build directories and artifacts
|
||||
'build', 'dist', 'target', 'out',
|
||||
'*.egg-info', '*.egg', '*.whl',
|
||||
'*.so', '*.dylib', '*.dll', '*.class',
|
||||
|
||||
# Documentation
|
||||
'site-packages', '.docusaurus', '.next', '.nuxt',
|
||||
|
||||
# Other common patterns
|
||||
'*.min.js', '*.min.css', # Minified files
|
||||
'*.map', # Source maps
|
||||
'.terraform', '*.tfstate*', # Terraform
|
||||
'vendor/', # Dependencies in various languages
|
||||
]
|
||||
from gitingest.ignore_patterns import DEFAULT_IGNORE_PATTERNS
|
||||
|
||||
TMP_BASE_PATH = "../tmp"
|
||||
|
||||
def parse_url(url: str) -> dict:
|
||||
|
||||
def parse_url(url: str) -> Dict[str, Any]:
|
||||
parsed = {
|
||||
"user_name": None,
|
||||
"repo_name": None,
|
||||
|
|
@ -91,8 +49,10 @@ def parse_url(url: str) -> dict:
|
|||
parsed["commit"] = parsed['branch']
|
||||
|
||||
parsed["subpath"] = "/" + "/".join(path_parts[4:])
|
||||
|
||||
return parsed
|
||||
|
||||
|
||||
def normalize_pattern(pattern: str) -> str:
|
||||
pattern = pattern.strip()
|
||||
pattern = pattern.lstrip(os.sep)
|
||||
|
|
@ -100,16 +60,21 @@ def normalize_pattern(pattern: str) -> str:
|
|||
pattern += "*"
|
||||
return pattern
|
||||
|
||||
|
||||
def parse_patterns(pattern: Union[List[str], str]) -> List[str]:
|
||||
if isinstance(pattern, list):
|
||||
pattern = ",".join(pattern)
|
||||
|
||||
for p in pattern.split(","):
|
||||
if not all(c.isalnum() or c in "-_./+*" for c in p.strip()):
|
||||
raise ValueError(f"Pattern '{p}' contains invalid characters. Only alphanumeric characters, dash (-), underscore (_), dot (.), forward slash (/), plus (+), and asterisk (*) are allowed.")
|
||||
raise ValueError(
|
||||
f"Pattern '{p}' contains invalid characters. Only alphanumeric characters, dash (-), "
|
||||
"underscore (_), dot (.), forward slash (/), plus (+), and asterisk (*) are allowed."
|
||||
)
|
||||
patterns = [normalize_pattern(p) for p in pattern.split(",")]
|
||||
return patterns
|
||||
|
||||
|
||||
def override_ignore_patterns(ignore_patterns: List[str], include_patterns: List[str]) -> List[str]:
|
||||
for pattern in include_patterns:
|
||||
if pattern in ignore_patterns:
|
||||
|
|
@ -117,8 +82,7 @@ def override_ignore_patterns(ignore_patterns: List[str], include_patterns: List[
|
|||
return ignore_patterns
|
||||
|
||||
|
||||
def parse_path(path: str) -> dict:
|
||||
|
||||
def parse_path(path: str) -> Dict[str, Any]:
|
||||
query = {
|
||||
"local_path": os.path.abspath(path),
|
||||
"slug": os.path.basename(os.path.dirname(path)) + "/" + os.path.basename(path),
|
||||
|
|
@ -128,7 +92,14 @@ def parse_path(path: str) -> dict:
|
|||
}
|
||||
return query
|
||||
|
||||
def parse_query(source: str, max_file_size: int, from_web: bool, include_patterns: Union[List[str], str] = None, ignore_patterns: Union[List[str], str] = None) -> dict:
|
||||
|
||||
def parse_query(
|
||||
source: str,
|
||||
max_file_size: int,
|
||||
from_web: bool,
|
||||
include_patterns: Optional[Union[List[str], str]] = None,
|
||||
ignore_patterns: Optional[Union[List[str], str]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
if from_web:
|
||||
query = parse_url(source)
|
||||
else:
|
||||
|
|
@ -136,6 +107,7 @@ def parse_query(source: str, max_file_size: int, from_web: bool, include_pattern
|
|||
query = parse_url(source)
|
||||
else:
|
||||
query = parse_path(source)
|
||||
|
||||
query['max_file_size'] = max_file_size
|
||||
|
||||
if ignore_patterns and ignore_patterns != "":
|
||||
|
|
@ -153,4 +125,3 @@ def parse_query(source: str, max_file_size: int, from_web: bool, include_pattern
|
|||
query['include_patterns'] = include_patterns
|
||||
|
||||
return query
|
||||
|
||||
|
|
|
|||
|
|
@ -1,17 +1,20 @@
|
|||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
import pytest
|
||||
from clone import clone_repo, check_repo_exists
|
||||
from unittest.mock import patch, AsyncMock
|
||||
|
||||
from gitingest.clone import check_repo_exists, clone_repo
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_clone_repo_with_commit():
|
||||
async def test_clone_repo_with_commit() -> None:
|
||||
query = {
|
||||
'commit': 'a' * 40, # Simulating a valid commit hash
|
||||
'branch': 'main',
|
||||
'url': 'https://github.com/user/repo',
|
||||
'local_path': '/tmp/repo'
|
||||
'local_path': '/tmp/repo',
|
||||
}
|
||||
|
||||
with patch('clone.check_repo_exists', return_value=True) as mock_check:
|
||||
with patch('gitingest.clone.check_repo_exists', return_value=True) as mock_check:
|
||||
with patch('asyncio.create_subprocess_exec', new_callable=AsyncMock) as mock_exec:
|
||||
mock_process = AsyncMock()
|
||||
mock_process.communicate.return_value = (b'output', b'error')
|
||||
|
|
@ -21,16 +24,17 @@ async def test_clone_repo_with_commit():
|
|||
mock_check.assert_called_once_with(query['url'])
|
||||
assert mock_exec.call_count == 2 # Clone and checkout calls
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_clone_repo_without_commit():
|
||||
async def test_clone_repo_without_commit() -> None:
|
||||
query = {
|
||||
'commit': None,
|
||||
'branch': 'main',
|
||||
'url': 'https://github.com/user/repo',
|
||||
'local_path': '/tmp/repo'
|
||||
'local_path': '/tmp/repo',
|
||||
}
|
||||
|
||||
with patch('clone.check_repo_exists', return_value=True) as mock_check:
|
||||
with patch('gitingest.clone.check_repo_exists', return_value=True) as mock_check:
|
||||
with patch('asyncio.create_subprocess_exec', new_callable=AsyncMock) as mock_exec:
|
||||
mock_process = AsyncMock()
|
||||
mock_process.communicate.return_value = (b'output', b'error')
|
||||
|
|
@ -40,13 +44,14 @@ async def test_clone_repo_without_commit():
|
|||
mock_check.assert_called_once_with(query['url'])
|
||||
assert mock_exec.call_count == 1 # Only clone call
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_clone_repo_nonexistent_repository():
|
||||
async def test_clone_repo_nonexistent_repository() -> None:
|
||||
query = {
|
||||
'commit': None,
|
||||
'branch': 'main',
|
||||
'url': 'https://github.com/user/nonexistent-repo',
|
||||
'local_path': '/tmp/repo'
|
||||
'local_path': '/tmp/repo',
|
||||
}
|
||||
|
||||
with patch('gitingest.clone.check_repo_exists', return_value=False) as mock_check:
|
||||
|
|
@ -54,8 +59,9 @@ async def test_clone_repo_nonexistent_repository():
|
|||
await clone_repo(query)
|
||||
mock_check.assert_called_once_with(query['url'])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_check_repo_exists():
|
||||
async def test_check_repo_exists() -> None:
|
||||
url = "https://github.com/user/repo"
|
||||
|
||||
with patch('asyncio.create_subprocess_exec', new_callable=AsyncMock) as mock_exec:
|
||||
|
|
|
|||
|
|
@ -1,12 +1,14 @@
|
|||
from pathlib import Path
|
||||
from typing import Any, Dict
|
||||
|
||||
import pytest
|
||||
from src.gitingest.ingest_from_query import (
|
||||
scan_directory,
|
||||
extract_files_content,
|
||||
)
|
||||
|
||||
from gitingest.ingest_from_query import extract_files_content, scan_directory
|
||||
|
||||
|
||||
# Test fixtures
|
||||
@pytest.fixture
|
||||
def sample_query():
|
||||
def sample_query() -> Dict[str, Any]:
|
||||
return {
|
||||
'user_name': 'test_user',
|
||||
'repo_name': 'test_repo',
|
||||
|
|
@ -14,16 +16,16 @@ def sample_query():
|
|||
'subpath': '/',
|
||||
'branch': 'main',
|
||||
'commit': None,
|
||||
'max_file_size': 1000000,
|
||||
'max_file_size': 1_000_000,
|
||||
'slug': 'test_user/test_repo',
|
||||
'ignore_patterns': ['*.pyc', '__pycache__', '.git'],
|
||||
'include_patterns': None,
|
||||
'pattern_type': 'exclude'
|
||||
|
||||
'pattern_type': 'exclude',
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def temp_directory(tmp_path):
|
||||
def temp_directory(tmp_path: Path) -> Path:
|
||||
# Creates the following structure:
|
||||
# test_repo/
|
||||
# ├── file1.txt
|
||||
|
|
@ -70,24 +72,23 @@ def temp_directory(tmp_path):
|
|||
|
||||
return test_dir
|
||||
|
||||
def test_scan_directory(temp_directory, sample_query):
|
||||
result = scan_directory(
|
||||
str(temp_directory),
|
||||
query=sample_query
|
||||
)
|
||||
|
||||
def test_scan_directory(temp_directory: Path, sample_query: Dict[str, Any]) -> None:
|
||||
result = scan_directory(str(temp_directory), query=sample_query)
|
||||
if result is None:
|
||||
assert False, "Result is None"
|
||||
|
||||
assert result['type'] == 'directory'
|
||||
assert result['file_count'] == 8 # All .txt and .py files
|
||||
assert result['dir_count'] == 4 # src, src/subdir, dir1, dir2
|
||||
assert len(result['children']) == 5 # file1.txt, file2.py, src, dir1, dir2
|
||||
|
||||
def test_extract_files_content(temp_directory, sample_query):
|
||||
nodes = scan_directory(
|
||||
str(temp_directory),
|
||||
query=sample_query
|
||||
)
|
||||
|
||||
files = extract_files_content(sample_query, nodes, max_file_size=1000000)
|
||||
def test_extract_files_content(temp_directory: Path, sample_query: Dict[str, Any]) -> None:
|
||||
nodes = scan_directory(str(temp_directory), query=sample_query)
|
||||
if nodes is None:
|
||||
assert False, "Nodes is None"
|
||||
files = extract_files_content(query=sample_query, node=nodes, max_file_size=1_000_000)
|
||||
assert len(files) == 8 # All .txt and .py files
|
||||
|
||||
# Check for presence of key files
|
||||
|
|
@ -101,7 +102,6 @@ def test_extract_files_content(temp_directory, sample_query):
|
|||
assert any('file_dir2.txt' in p for p in paths)
|
||||
|
||||
|
||||
|
||||
# TODO: test with include patterns: ['*.txt']
|
||||
# TODO: test with wrong include patterns: ['*.qwerty']
|
||||
|
||||
|
|
@ -116,7 +116,3 @@ def test_extract_files_content(temp_directory, sample_query):
|
|||
# TODO: test with multiple include patterns: ['*.txt', '*.py']
|
||||
# TODO: test with multiple include patterns: ['/src/*', '*.txt']
|
||||
# TODO: test with multiple include patterns: ['/src*', '*.txt']
|
||||
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,12 +1,14 @@
|
|||
import pytest
|
||||
from gitingest.parse_query import parse_query, parse_url, DEFAULT_IGNORE_PATTERNS
|
||||
|
||||
from gitingest.ignore_patterns import DEFAULT_IGNORE_PATTERNS
|
||||
from gitingest.parse_query import parse_query, parse_url
|
||||
|
||||
|
||||
def test_parse_url_valid():
|
||||
def test_parse_url_valid() -> None:
|
||||
test_cases = [
|
||||
"https://github.com/user/repo",
|
||||
"https://gitlab.com/user/repo",
|
||||
"https://bitbucket.org/user/repo"
|
||||
"https://bitbucket.org/user/repo",
|
||||
]
|
||||
for url in test_cases:
|
||||
result = parse_url(url)
|
||||
|
|
@ -14,16 +16,15 @@ def test_parse_url_valid():
|
|||
assert result["repo_name"] == "repo"
|
||||
assert result["url"] == url
|
||||
|
||||
def test_parse_url_invalid():
|
||||
|
||||
def test_parse_url_invalid() -> None:
|
||||
url = "https://only-domain.com"
|
||||
with pytest.raises(ValueError, match="Invalid repository URL"):
|
||||
parse_url(url)
|
||||
|
||||
def test_parse_query_basic():
|
||||
test_cases = [
|
||||
"https://github.com/user/repo",
|
||||
"https://gitlab.com/user/repo"
|
||||
]
|
||||
|
||||
def test_parse_query_basic() -> None:
|
||||
test_cases = ["https://github.com/user/repo", "https://gitlab.com/user/repo"]
|
||||
for url in test_cases:
|
||||
result = parse_query(url, max_file_size=50, from_web=True, ignore_patterns='*.txt')
|
||||
assert result["user_name"] == "user"
|
||||
|
|
@ -31,13 +32,15 @@ def test_parse_query_basic():
|
|||
assert result["url"] == url
|
||||
assert "*.txt" in result["ignore_patterns"]
|
||||
|
||||
def test_parse_query_include_pattern():
|
||||
|
||||
def test_parse_query_include_pattern() -> None:
|
||||
url = "https://github.com/user/repo"
|
||||
result = parse_query(url, max_file_size=50, from_web=True, include_patterns='*.py')
|
||||
assert result["include_patterns"] == ["*.py"]
|
||||
assert result["ignore_patterns"] == DEFAULT_IGNORE_PATTERNS
|
||||
|
||||
def test_parse_query_invalid_pattern():
|
||||
|
||||
def test_parse_query_invalid_pattern() -> None:
|
||||
url = "https://github.com/user/repo"
|
||||
with pytest.raises(ValueError, match="Pattern.*contains invalid characters"):
|
||||
parse_query(url, max_file_size=50, from_web=True, include_patterns='*.py;rm -rf')
|
||||
|
|
@ -1,22 +1,27 @@
|
|||
|
||||
## Async Timeout decorator
|
||||
import asyncio
|
||||
import functools
|
||||
from typing import TypeVar, Callable
|
||||
from typing import Awaitable, Callable, ParamSpec, TypeVar
|
||||
|
||||
T = TypeVar("T")
|
||||
P = ParamSpec("P")
|
||||
|
||||
|
||||
class AsyncTimeoutError(Exception):
|
||||
"""Raised when an async operation exceeds its timeout limit."""
|
||||
|
||||
pass
|
||||
|
||||
def async_timeout(seconds: int = 10):
|
||||
def decorator(func: Callable[..., T]) -> Callable[..., T]:
|
||||
|
||||
def async_timeout(seconds: int = 10) -> Callable[[Callable[P, Awaitable[T]]], Callable[P, Awaitable[T]]]:
|
||||
def decorator(func: Callable[P, Awaitable[T]]) -> Callable[P, Awaitable[T]]:
|
||||
@functools.wraps(func)
|
||||
async def wrapper(*args, **kwargs) -> T:
|
||||
async def wrapper(*args: P.args, **kwargs: P.kwargs) -> T:
|
||||
try:
|
||||
return await asyncio.wait_for(func(*args, **kwargs), timeout=seconds)
|
||||
except asyncio.TimeoutError:
|
||||
raise AsyncTimeoutError(f"Clone timed out after {seconds} seconds")
|
||||
raise AsyncTimeoutError(f"Operation timed out after {seconds} seconds")
|
||||
|
||||
return wrapper
|
||||
|
||||
return decorator
|
||||
58
src/main.py
58
src/main.py
|
|
@ -1,27 +1,41 @@
|
|||
import os
|
||||
from dotenv import load_dotenv
|
||||
from typing import Dict
|
||||
|
||||
from fastapi import FastAPI, Request
|
||||
from fastapi.templating import Jinja2Templates
|
||||
from fastapi.responses import HTMLResponse, FileResponse, Response
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from starlette.middleware.trustedhost import TrustedHostMiddleware
|
||||
from api_analytics.fastapi import Analytics
|
||||
from dotenv import load_dotenv
|
||||
from fastapi import FastAPI, Request
|
||||
from fastapi.responses import FileResponse, HTMLResponse, Response
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from fastapi.templating import Jinja2Templates
|
||||
from slowapi import _rate_limit_exceeded_handler
|
||||
from slowapi.errors import RateLimitExceeded
|
||||
from starlette.middleware.trustedhost import TrustedHostMiddleware
|
||||
|
||||
from server_utils import limiter
|
||||
from routers import download, dynamic, index
|
||||
|
||||
from server_utils import limiter
|
||||
|
||||
load_dotenv()
|
||||
|
||||
app = FastAPI()
|
||||
app.state.limiter = limiter
|
||||
app.add_exception_handler(RateLimitExceeded, _rate_limit_exceeded_handler)
|
||||
|
||||
|
||||
# Define a wrapper handler with the correct signature
|
||||
async def rate_limit_exception_handler(request: Request, exc: Exception) -> Response:
|
||||
if isinstance(exc, RateLimitExceeded):
|
||||
# Delegate to the actual handler
|
||||
return _rate_limit_exceeded_handler(request, exc)
|
||||
# Optionally, handle other exceptions or re-raise
|
||||
raise exc
|
||||
|
||||
|
||||
# Register the wrapper handler
|
||||
app.add_exception_handler(RateLimitExceeded, rate_limit_exception_handler)
|
||||
|
||||
app.mount("/static", StaticFiles(directory="static"), name="static")
|
||||
app.add_middleware(Analytics, api_key=os.getenv('API_ANALYTICS_KEY'))
|
||||
app_analytics_key = os.getenv('API_ANALYTICS_KEY')
|
||||
if app_analytics_key:
|
||||
app.add_middleware(Analytics, api_key=app_analytics_key)
|
||||
|
||||
# Define the default allowed hosts
|
||||
default_allowed_hosts = ["gitingest.com", "*.gitingest.com", "localhost", "127.0.0.1"]
|
||||
|
|
@ -36,31 +50,29 @@ else:
|
|||
app.add_middleware(TrustedHostMiddleware, allowed_hosts=allowed_hosts)
|
||||
templates = Jinja2Templates(directory="templates")
|
||||
|
||||
|
||||
@app.get("/health")
|
||||
async def health_check():
|
||||
async def health_check() -> Dict[str, str]:
|
||||
return {"status": "healthy"}
|
||||
|
||||
|
||||
@app.head("/")
|
||||
async def head_root():
|
||||
async def head_root() -> HTMLResponse:
|
||||
"""Mirror the headers and status code of the index page"""
|
||||
return HTMLResponse(
|
||||
content=None,
|
||||
headers={
|
||||
"content-type": "text/html; charset=utf-8"
|
||||
}
|
||||
)
|
||||
return HTMLResponse(content=None, headers={"content-type": "text/html; charset=utf-8"})
|
||||
|
||||
|
||||
@app.get("/api/", response_class=HTMLResponse)
|
||||
@app.get("/api", response_class=HTMLResponse)
|
||||
async def api_docs(request: Request):
|
||||
return templates.TemplateResponse(
|
||||
"api.jinja", {"request": request}
|
||||
)
|
||||
async def api_docs(request: Request) -> HTMLResponse:
|
||||
return templates.TemplateResponse("api.jinja", {"request": request})
|
||||
|
||||
|
||||
@app.get("/robots.txt")
|
||||
async def robots():
|
||||
async def robots() -> FileResponse:
|
||||
return FileResponse('static/robots.txt')
|
||||
|
||||
|
||||
app.include_router(index)
|
||||
app.include_router(download)
|
||||
app.include_router(dynamic)
|
||||
|
|
@ -1,14 +1,25 @@
|
|||
from typing import List
|
||||
from fastapi.templating import Jinja2Templates
|
||||
from fastapi import Request
|
||||
from typing import Any, Dict
|
||||
|
||||
from config import MAX_DISPLAY_SIZE, EXAMPLE_REPOS
|
||||
from gitingest import ingest_from_query, clone_repo, parse_query
|
||||
from server_utils import logSliderToSize, Colors
|
||||
from fastapi import Request
|
||||
from fastapi.templating import Jinja2Templates
|
||||
from starlette.templating import _TemplateResponse
|
||||
|
||||
from config import EXAMPLE_REPOS, MAX_DISPLAY_SIZE
|
||||
from gitingest.clone import clone_repo
|
||||
from gitingest.ingest_from_query import ingest_from_query
|
||||
from gitingest.parse_query import parse_query
|
||||
from server_utils import Colors, logSliderToSize
|
||||
|
||||
templates = Jinja2Templates(directory="templates")
|
||||
|
||||
def print_query(query, request, max_file_size, pattern_type, pattern):
|
||||
|
||||
def print_query(
|
||||
query: Dict[str, Any],
|
||||
request: Request,
|
||||
max_file_size: int,
|
||||
pattern_type: str,
|
||||
pattern: str,
|
||||
) -> None:
|
||||
print(f"{Colors.WHITE}{query['url']:<20}{Colors.END}", end="")
|
||||
if int(max_file_size / 1024) != 50:
|
||||
print(f" | {Colors.YELLOW}Size: {int(max_file_size/1024)}kb{Colors.END}", end="")
|
||||
|
|
@ -18,37 +29,64 @@ def print_query(query, request, max_file_size, pattern_type, pattern):
|
|||
print(f" | {Colors.YELLOW}Exclude {pattern}{Colors.END}", end="")
|
||||
|
||||
|
||||
def print_error(query, request, e, max_file_size, pattern_type, pattern):
|
||||
def print_error(
|
||||
query: Dict[str, Any],
|
||||
request: Request,
|
||||
e: Exception,
|
||||
max_file_size: int,
|
||||
pattern_type: str,
|
||||
pattern: str,
|
||||
) -> None:
|
||||
print(f"{Colors.BROWN}WARN{Colors.END}: {Colors.RED}<- {Colors.END}", end="")
|
||||
print_query(query, request, max_file_size, pattern_type, pattern)
|
||||
print(f" | {Colors.RED}{e}{Colors.END}")
|
||||
|
||||
def print_success(query, request, max_file_size, pattern_type, pattern, summary):
|
||||
|
||||
def print_success(
|
||||
query: Dict[str, Any],
|
||||
request: Request,
|
||||
max_file_size: int,
|
||||
pattern_type: str,
|
||||
pattern: str,
|
||||
summary: str,
|
||||
) -> None:
|
||||
estimated_tokens = summary[summary.index("Estimated tokens:") + len("Estimated ") :]
|
||||
print(f"{Colors.GREEN}INFO{Colors.END}: {Colors.GREEN}<- {Colors.END}", end="")
|
||||
print_query(query, request, max_file_size, pattern_type, pattern)
|
||||
print(f" | {Colors.PURPLE}{estimated_tokens}{Colors.END}")
|
||||
|
||||
|
||||
|
||||
async def process_query(request: Request, input_text: str, slider_position: int, pattern_type: str = "exclude", pattern: str = "", is_index: bool = False) -> str:
|
||||
async def process_query(
|
||||
request: Request,
|
||||
input_text: str,
|
||||
slider_position: int,
|
||||
pattern_type: str = "exclude",
|
||||
pattern: str = "",
|
||||
is_index: bool = False,
|
||||
) -> _TemplateResponse:
|
||||
template = "index.jinja" if is_index else "github.jinja"
|
||||
max_file_size = logSliderToSize(slider_position)
|
||||
|
||||
if pattern_type == "include":
|
||||
include_patterns = pattern
|
||||
exclude_patterns = None
|
||||
elif pattern_type == "exclude":
|
||||
exclude_patterns = pattern
|
||||
include_patterns = None
|
||||
|
||||
try:
|
||||
query = parse_query(input_text, max_file_size, True, include_patterns, exclude_patterns)
|
||||
query = parse_query(
|
||||
source=input_text,
|
||||
max_file_size=max_file_size,
|
||||
from_web=True,
|
||||
include_patterns=include_patterns,
|
||||
ignore_patterns=exclude_patterns,
|
||||
)
|
||||
await clone_repo(query)
|
||||
summary, tree, content = ingest_from_query(query)
|
||||
with open(f"{query['local_path']}.txt", "w") as f:
|
||||
f.write(tree + "\n" + content)
|
||||
|
||||
|
||||
|
||||
except Exception as e:
|
||||
# hack to print error message when query is not defined
|
||||
if 'query' in locals() and query is not None and isinstance(query, dict):
|
||||
|
|
@ -56,6 +94,7 @@ async def process_query(request: Request, input_text: str, slider_position: int,
|
|||
else:
|
||||
print(f"{Colors.BROWN}WARN{Colors.END}: {Colors.RED}<- {Colors.END}", end="")
|
||||
print(f"{Colors.RED}{e}{Colors.END}")
|
||||
|
||||
return templates.TemplateResponse(
|
||||
template,
|
||||
{
|
||||
|
|
@ -66,12 +105,24 @@ async def process_query(request: Request, input_text: str, slider_position: int,
|
|||
"default_file_size": slider_position,
|
||||
"pattern_type": pattern_type,
|
||||
"pattern": pattern,
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
if len(content) > MAX_DISPLAY_SIZE:
|
||||
content = f"(Files content cropped to {int(MAX_DISPLAY_SIZE/1000)}k characters, download full ingest to see more)\n" + content[:MAX_DISPLAY_SIZE]
|
||||
print_success(query, request, max_file_size, pattern_type, pattern, summary)
|
||||
content = (
|
||||
f"(Files content cropped to {int(MAX_DISPLAY_SIZE / 1_000)}k characters, "
|
||||
"download full ingest to see more)\n" + content[:MAX_DISPLAY_SIZE]
|
||||
)
|
||||
|
||||
print_success(
|
||||
query=query,
|
||||
request=request,
|
||||
max_file_size=max_file_size,
|
||||
pattern_type=pattern_type,
|
||||
pattern=pattern,
|
||||
summary=summary,
|
||||
)
|
||||
|
||||
return templates.TemplateResponse(
|
||||
template,
|
||||
{
|
||||
|
|
@ -86,5 +137,5 @@ async def process_query(request: Request, input_text: str, slider_position: int,
|
|||
"default_file_size": slider_position,
|
||||
"pattern_type": pattern_type,
|
||||
"pattern": pattern,
|
||||
}
|
||||
},
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
from .download import router as download
|
||||
from .dynamic import router as dynamic
|
||||
from .index import router as index
|
||||
from routers.download import router as download
|
||||
from routers.dynamic import router as dynamic
|
||||
from routers.index import router as index
|
||||
|
||||
__all__ = ["download", "dynamic", "index"]
|
||||
|
|
@ -1,12 +1,15 @@
|
|||
from fastapi import HTTPException, APIRouter
|
||||
from fastapi.responses import Response
|
||||
from config import TMP_BASE_PATH
|
||||
import os
|
||||
|
||||
from fastapi import APIRouter, HTTPException
|
||||
from fastapi.responses import Response
|
||||
|
||||
from config import TMP_BASE_PATH
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.get("/download/{digest_id}")
|
||||
async def download_ingest(digest_id: str):
|
||||
async def download_ingest(digest_id: str) -> Response:
|
||||
try:
|
||||
# Find the first .txt file in the directory
|
||||
directory = f"{TMP_BASE_PATH}/{digest_id}"
|
||||
|
|
@ -15,15 +18,13 @@ async def download_ingest(digest_id: str):
|
|||
if not txt_files:
|
||||
raise FileNotFoundError("No .txt file found")
|
||||
|
||||
with open(f"{directory}/{txt_files[0]}", "r") as f:
|
||||
with open(f"{directory}/{txt_files[0]}") as f:
|
||||
content = f.read()
|
||||
|
||||
return Response(
|
||||
content=content,
|
||||
media_type="text/plain",
|
||||
headers={
|
||||
"Content-Disposition": f"attachment; filename={txt_files[0]}"
|
||||
}
|
||||
headers={"Content-Disposition": f"attachment; filename={txt_files[0]}"},
|
||||
)
|
||||
except FileNotFoundError:
|
||||
raise HTTPException(status_code=404, detail="Digest not found")
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
from fastapi import APIRouter, Request, Form
|
||||
from fastapi import APIRouter, Form, Request
|
||||
from fastapi.responses import HTMLResponse
|
||||
from fastapi.templating import Jinja2Templates
|
||||
|
||||
|
|
@ -8,18 +8,20 @@ from server_utils import limiter
|
|||
router = APIRouter()
|
||||
templates = Jinja2Templates(directory="templates")
|
||||
|
||||
|
||||
@router.get("/{full_path:path}")
|
||||
async def catch_all(request: Request, full_path: str):
|
||||
async def catch_all(request: Request, full_path: str) -> HTMLResponse:
|
||||
return templates.TemplateResponse(
|
||||
"github.jinja",
|
||||
{
|
||||
"request": request,
|
||||
"github_url": f"https://github.com/{full_path}",
|
||||
"loading": True,
|
||||
"default_file_size": 243
|
||||
}
|
||||
"default_file_size": 243,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@router.post("/{full_path:path}", response_class=HTMLResponse)
|
||||
@limiter.limit("10/minute")
|
||||
async def process_catch_all(
|
||||
|
|
@ -27,7 +29,13 @@ async def process_catch_all(
|
|||
input_text: str = Form(...),
|
||||
max_file_size: int = Form(...),
|
||||
pattern_type: str = Form(...),
|
||||
pattern: str = Form(...)
|
||||
):
|
||||
return await process_query(request, input_text, max_file_size, pattern_type, pattern, is_index=False)
|
||||
|
||||
pattern: str = Form(...),
|
||||
) -> HTMLResponse:
|
||||
return await process_query(
|
||||
request,
|
||||
input_text,
|
||||
max_file_size,
|
||||
pattern_type,
|
||||
pattern,
|
||||
is_index=False,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,25 +1,24 @@
|
|||
from fastapi import APIRouter, Request, Form
|
||||
from fastapi import APIRouter, Form, Request
|
||||
from fastapi.responses import HTMLResponse
|
||||
from fastapi.templating import Jinja2Templates
|
||||
|
||||
from server_utils import limiter
|
||||
from process_query import process_query
|
||||
from config import EXAMPLE_REPOS
|
||||
|
||||
from process_query import process_query
|
||||
from server_utils import limiter
|
||||
|
||||
router = APIRouter()
|
||||
templates = Jinja2Templates(directory="templates")
|
||||
|
||||
|
||||
@router.get("/", response_class=HTMLResponse)
|
||||
async def home(request: Request):
|
||||
async def home(request: Request) -> HTMLResponse:
|
||||
return templates.TemplateResponse(
|
||||
"index.jinja",
|
||||
{
|
||||
"request": request,
|
||||
"examples": EXAMPLE_REPOS,
|
||||
"default_file_size": 243
|
||||
}
|
||||
"default_file_size": 243,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -30,11 +29,13 @@ async def index_post(
|
|||
input_text: str = Form(...),
|
||||
max_file_size: int = Form(...),
|
||||
pattern_type: str = Form(...),
|
||||
pattern: str = Form(...)
|
||||
):
|
||||
return await process_query(request, input_text, max_file_size, pattern_type, pattern, is_index=True)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
pattern: str = Form(...),
|
||||
) -> HTMLResponse:
|
||||
return await process_query(
|
||||
request,
|
||||
input_text,
|
||||
max_file_size,
|
||||
pattern_type,
|
||||
pattern,
|
||||
is_index=True,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,11 +1,14 @@
|
|||
import math
|
||||
|
||||
## Rate Limiter
|
||||
from slowapi import Limiter
|
||||
from slowapi.util import get_remote_address
|
||||
|
||||
limiter = Limiter(key_func=get_remote_address)
|
||||
|
||||
## Logarithmic slider to file size
|
||||
import math
|
||||
def logSliderToSize(position):
|
||||
|
||||
## Logarithmic slider to file size conversion
|
||||
def logSliderToSize(position: int) -> int:
|
||||
"""Convert slider position to file size in KB"""
|
||||
maxp = 500
|
||||
minv = math.log(1)
|
||||
|
|
@ -13,9 +16,11 @@ def logSliderToSize(position):
|
|||
|
||||
return round(math.exp(minv + (maxv - minv) * pow(position / maxp, 1.5))) * 1024
|
||||
|
||||
|
||||
## Color printing utility
|
||||
class Colors:
|
||||
"""ANSI color codes"""
|
||||
|
||||
BLACK = "\033[0;30m"
|
||||
RED = "\033[0;31m"
|
||||
GREEN = "\033[0;32m"
|
||||
|
|
|
|||
|
|
@ -2,4 +2,3 @@ User-agent: *
|
|||
Allow: /
|
||||
Allow: /api/
|
||||
Allow: /cyclotruc/gitingest/
|
||||
|
||||
|
|
|
|||
|
|
@ -1,38 +1,32 @@
|
|||
{% extends "base.jinja" %}
|
||||
|
||||
{% block title %}Git ingest API{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<div class="relative">
|
||||
<div class="w-full h-full absolute inset-0 bg-black rounded-xl translate-y-2 translate-x-2"></div>
|
||||
<div class="bg-[#fff4da] rounded-xl border-[3px] border-gray-900 p-8 relative z-20">
|
||||
<h1 class="text-3xl font-bold text-gray-900 mb-4">API Documentation</h1>
|
||||
|
||||
|
||||
<div class="prose prose-blue max-w-none">
|
||||
<div class="bg-yellow-50 border-[3px] border-gray-900 p-4 mb-6 rounded-lg">
|
||||
<div class="flex">
|
||||
<div class="flex-shrink-0">
|
||||
<svg class="h-5 w-5 text-yellow-400" viewBox="0 0 20 20" fill="currentColor">
|
||||
<path fill-rule="evenodd"
|
||||
d="M8.257 3.099c.765-1.36 2.722-1.36 3.486 0l5.58 9.92c.75 1.334-.213 2.98-1.742 2.98H4.42c-1.53 0-2.493-1.646-1.743-2.98l5.58-9.92zM11 13a1 1 0 11-2 0 1 1 0 012 0zm-1-8a1 1 0 00-1 1v3a1 1 0 002 0V6a1 1 0 00-1-1z"
|
||||
clip-rule="evenodd" />
|
||||
<svg class="h-5 w-5 text-yellow-400"
|
||||
viewBox="0 0 20 20"
|
||||
fill="currentColor">
|
||||
<path fill-rule="evenodd" d="M8.257 3.099c.765-1.36 2.722-1.36 3.486 0l5.58 9.92c.75 1.334-.213 2.98-1.742 2.98H4.42c-1.53 0-2.493-1.646-1.743-2.98l5.58-9.92zM11 13a1 1 0 11-2 0 1 1 0 012 0zm-1-8a1 1 0 00-1 1v3a1 1 0 002 0V6a1 1 0 00-1-1z" clip-rule="evenodd" />
|
||||
</svg>
|
||||
</div>
|
||||
<div class="ml-3">
|
||||
<p class="text-sm text-gray-900">
|
||||
The API is currently under development..
|
||||
</p>
|
||||
<p class="text-sm text-gray-900">The API is currently under development..</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<p class="text-gray-900">
|
||||
We're working on making our API available to the public.
|
||||
In the meantime, you can
|
||||
<a href="https://github.com/cyclotruc/gitingest/issues/new" target="_blank"
|
||||
rel="noopener noreferrer" class="text-[#6e5000] hover:underline">
|
||||
open an issue on github
|
||||
</a>
|
||||
<a href="https://github.com/cyclotruc/gitingest/issues/new"
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
class="text-[#6e5000] hover:underline">open an issue on github</a>
|
||||
to suggest features.
|
||||
</p>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -4,34 +4,37 @@
|
|||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<link rel="icon" type="image/x-icon" href="/static/favicon.ico">
|
||||
|
||||
<!-- Search Engine Meta Tags -->
|
||||
<meta name="description" content="Replace 'hub' with 'ingest' in any Github Url for a prompt-friendly text">
|
||||
<meta name="keywords" content="GitIngest, AI tools, LLM integration, Ingest, Digest, Context, Prompt, Git workflow, codebase extraction, Git repository, Git automation, Summarize, prompt-friendly">
|
||||
<meta name="description"
|
||||
content="Replace 'hub' with 'ingest' in any Github Url for a prompt-friendly text">
|
||||
<meta name="keywords"
|
||||
content="GitIngest, AI tools, LLM integration, Ingest, Digest, Context, Prompt, Git workflow, codebase extraction, Git repository, Git automation, Summarize, prompt-friendly">
|
||||
<meta name="robots" content="index, follow">
|
||||
|
||||
<!-- Favicons -->
|
||||
<link rel="icon" type="image/svg+xml" href="/static/favicon.svg">
|
||||
<link rel="icon" type="image/png" sizes="64x64" href="/static/favicon-64.png">
|
||||
<link rel="apple-touch-icon" sizes="180x180" href="/static/apple-touch-icon.png">
|
||||
|
||||
<link rel="icon"
|
||||
type="image/png"
|
||||
sizes="64x64"
|
||||
href="/static/favicon-64.png">
|
||||
<link rel="apple-touch-icon"
|
||||
sizes="180x180"
|
||||
href="/static/apple-touch-icon.png">
|
||||
<!-- Web App Meta -->
|
||||
<meta name="apple-mobile-web-app-title" content="GitIngest">
|
||||
<meta name="application-name" content="GitIngest">
|
||||
<meta name="theme-color" content="#FCA847">
|
||||
<meta name="apple-mobile-web-app-capable" content="yes">
|
||||
<meta name="apple-mobile-web-app-status-bar-style" content="default">
|
||||
|
||||
|
||||
<!-- OpenGraph Meta Tags -->
|
||||
<meta property="og:title" content="Git ingest">
|
||||
<meta property="og:description" content="Replace 'hub' with 'ingest' in any Github Url for a prompt-friendly text">
|
||||
<meta property="og:description"
|
||||
content="Replace 'hub' with 'ingest' in any Github Url for a prompt-friendly text">
|
||||
<meta property="og:type" content="website">
|
||||
<meta property="og:url" content="{{ request.url }}">
|
||||
<meta property="og:image" content="/static/og-image.png">
|
||||
|
||||
<title>{% block title %}Git ingest{% endblock %}</title>
|
||||
|
||||
<title>
|
||||
{% block title %}Git ingest{% endblock %}
|
||||
</title>
|
||||
<script src="https://cdn.tailwindcss.com"></script>
|
||||
<script src="/static/js/utils.js"></script>
|
||||
<script src="/static/js/snow.js"></script>
|
||||
|
|
@ -48,20 +51,16 @@
|
|||
</style>
|
||||
{% block extra_head %}{% endblock %}
|
||||
</head>
|
||||
|
||||
<body class="bg-[#FFFDF8] min-h-screen flex flex-col">
|
||||
<canvas id="snow-canvas"></canvas>
|
||||
{% include 'components/navbar.jinja' %}
|
||||
|
||||
<!-- Main content wrapper -->
|
||||
<main class="flex-1 w-full">
|
||||
<div class="max-w-4xl mx-auto px-4 py-8">
|
||||
{% block content %}{% endblock %}
|
||||
</div>
|
||||
</main>
|
||||
|
||||
{% include 'components/footer.jinja' %}
|
||||
|
||||
{% block extra_scripts %}{% endblock %}
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -4,14 +4,18 @@
|
|||
<div class="flex flex-col items-center">
|
||||
<div class="flex items-center">
|
||||
made with ❤️ by
|
||||
<a href="https://bsky.app/profile/yasbaltrine.bsky.social" target="_blank" rel="noopener noreferrer"
|
||||
class="ml-1 hover:underline">
|
||||
@rom2
|
||||
</a>
|
||||
<a href="https://bsky.app/profile/yasbaltrine.bsky.social"
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
class="ml-1 hover:underline">@rom2</a>
|
||||
</div>
|
||||
<a href="https://discord.gg/zerRaGK9EC" target="_blank" rel="noopener noreferrer"
|
||||
<a href="https://discord.gg/zerRaGK9EC"
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
class="mt-1 hover:underline flex items-center">
|
||||
<svg class="w-4 h-4 mr-1" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 640 512">
|
||||
<svg class="w-4 h-4 mr-1"
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
viewBox="0 0 640 512">
|
||||
<path fill="currentColor" d="M524.531,69.836a1.5,1.5,0,0,0-.764-.7A485.065,485.065,0,0,0,404.081,32.03a1.816,1.816,0,0,0-1.923.91,337.461,337.461,0,0,0-14.9,30.6,447.848,447.848,0,0,0-134.426,0,309.541,309.541,0,0,0-15.135-30.6,1.89,1.89,0,0,0-1.924-.91A483.689,483.689,0,0,0,116.085,69.137a1.712,1.712,0,0,0-.788.676C39.068,183.651,18.186,294.69,28.43,404.354a2.016,2.016,0,0,0,.765,1.375A487.666,487.666,0,0,0,176.02,479.918a1.9,1.9,0,0,0,2.063-.676A348.2,348.2,0,0,0,208.12,430.4a1.86,1.86,0,0,0-1.019-2.588,321.173,321.173,0,0,1-45.868-21.853,1.885,1.885,0,0,1-.185-3.126c3.082-2.309,6.166-4.711,9.109-7.137a1.819,1.819,0,0,1,1.9-.256c96.229,43.917,200.41,43.917,295.5,0a1.812,1.812,0,0,1,1.924.233c2.944,2.426,6.027,4.851,9.132,7.16a1.884,1.884,0,0,1-.162,3.126,301.407,301.407,0,0,1-45.89,21.83,1.875,1.875,0,0,0-1,2.611,391.055,391.055,0,0,0,30.014,48.815,1.864,1.864,0,0,0,2.063.7A486.048,486.048,0,0,0,610.7,405.729a1.882,1.882,0,0,0,.765-1.352C623.729,277.594,590.933,167.465,524.531,69.836ZM222.491,337.58c-28.972,0-52.844-26.587-52.844-59.239S193.056,219.1,222.491,219.1c29.665,0,53.306,26.82,52.843,59.239C275.334,310.993,251.924,337.58,222.491,337.58Zm195.38,0c-28.971,0-52.843-26.587-52.843-59.239S388.437,219.1,417.871,219.1c29.667,0,53.307,26.82,52.844,59.239C470.715,310.993,447.538,337.58,417.871,337.58Z" />
|
||||
</svg>
|
||||
Discord
|
||||
|
|
|
|||
|
|
@ -3,12 +3,15 @@
|
|||
<div class="rounded-xl relative z-20 pl-8 sm:pl-10 pr-8 sm:pr-16 py-8 border-[3px] border-gray-900 bg-[#fff4da]">
|
||||
<img src="https://cdn.devdojo.com/images/january2023/shape-1.png"
|
||||
class="absolute md:block hidden left-0 h-[4.5rem] w-[4.5rem] bottom-0 -translate-x-full ml-3">
|
||||
|
||||
<form class="flex md:flex-row flex-col w-full h-full justify-center items-stretch space-y-5 md:space-y-0 md:space-x-5"
|
||||
id="ingestForm" onsubmit="handleSubmit(event{% if is_index %}, true{% endif %})">
|
||||
id="ingestForm"
|
||||
onsubmit="handleSubmit(event{% if is_index %}, true{% endif %})">
|
||||
<div class="relative w-full h-full">
|
||||
<div class="w-full h-full rounded bg-gray-900 translate-y-1 translate-x-1 absolute inset-0 z-10"></div>
|
||||
<input type="text" name="input_text" id="input_text" placeholder="https://github.com/..."
|
||||
<input type="text"
|
||||
name="input_text"
|
||||
id="input_text"
|
||||
placeholder="https://github.com/..."
|
||||
value="{{ github_url if github_url else '' }}"
|
||||
required
|
||||
class="border-[3px] w-full relative z-20 border-gray-900 placeholder-gray-600 text-lg font-medium focus:outline-none py-3.5 px-6 rounded">
|
||||
|
|
@ -23,7 +26,6 @@
|
|||
<input type="hidden" name="pattern_type" value="exclude">
|
||||
<input type="hidden" name="pattern" value="">
|
||||
</form>
|
||||
|
||||
<div class="mt-4 relative z-20 flex flex-wrap gap-4 items-start">
|
||||
<!-- Pattern selector -->
|
||||
<div class="w-[200px] sm:w-[250px] mr-9 mt-4">
|
||||
|
|
@ -34,10 +36,20 @@
|
|||
<select id="pattern_type"
|
||||
name="pattern_type"
|
||||
class="w-21 py-2 pl-2 pr-6 appearance-none bg-[#e6e8eb] focus:outline-none border-r-[3px] border-gray-900">
|
||||
<option value="exclude" {% if pattern_type == 'exclude' or not pattern_type %}selected{% endif %}>Exclude</option>
|
||||
<option value="exclude"
|
||||
{% if pattern_type == 'exclude' or not pattern_type %}selected{% endif %}>
|
||||
Exclude
|
||||
</option>
|
||||
<option value="include" {% if pattern_type == 'include' %}selected{% endif %}>Include</option>
|
||||
</select>
|
||||
<svg class="absolute right-2 w-4 h-4 pointer-events-none" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
|
||||
<svg class="absolute right-2 w-4 h-4 pointer-events-none"
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
viewBox="0 0 24 24"
|
||||
fill="none"
|
||||
stroke="currentColor"
|
||||
stroke-width="2"
|
||||
stroke-linecap="round"
|
||||
stroke-linejoin="round">
|
||||
<polyline points="6 9 12 15 18 9" />
|
||||
</svg>
|
||||
</div>
|
||||
|
|
@ -50,9 +62,10 @@
|
|||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="w-[200px] sm:w-[200px] mt-3">
|
||||
<label for="file_size" class="block text-gray-700 mb-1">Include files under: <span id="size_value" class="font-bold">50kb</span></label>
|
||||
<label for="file_size" class="block text-gray-700 mb-1">
|
||||
Include files under: <span id="size_value" class="font-bold">50kb</span>
|
||||
</label>
|
||||
<input type="range"
|
||||
id="file_size"
|
||||
name="max_file_size"
|
||||
|
|
@ -60,32 +73,9 @@
|
|||
max="500"
|
||||
required
|
||||
value="{{ default_file_size }}"
|
||||
class="w-full h-3
|
||||
bg-[#FAFAFA]
|
||||
bg-no-repeat
|
||||
bg-[length:50%_100%]
|
||||
bg-[#ebdbb7]
|
||||
appearance-none
|
||||
border-[3px]
|
||||
border-gray-900
|
||||
rounded-sm
|
||||
focus:outline-none
|
||||
bg-gradient-to-r from-[#FE4A60] to-[#FE4A60]
|
||||
[&::-webkit-slider-thumb]:w-5
|
||||
[&::-webkit-slider-thumb]:h-7
|
||||
[&::-webkit-slider-thumb]:appearance-none
|
||||
[&::-webkit-slider-thumb]:bg-white
|
||||
[&::-webkit-slider-thumb]:rounded-sm
|
||||
[&::-webkit-slider-thumb]:cursor-pointer
|
||||
[&::-webkit-slider-thumb]:border-solid
|
||||
[&::-webkit-slider-thumb]:border-[3px]
|
||||
[&::-webkit-slider-thumb]:border-gray-900
|
||||
[&::-webkit-slider-thumb]:shadow-[3px_3px_0_#000]
|
||||
|
||||
">
|
||||
class="w-full h-3 bg-[#FAFAFA] bg-no-repeat bg-[length:50%_100%] bg-[#ebdbb7] appearance-none border-[3px] border-gray-900 rounded-sm focus:outline-none bg-gradient-to-r from-[#FE4A60] to-[#FE4A60] [&::-webkit-slider-thumb]:w-5 [&::-webkit-slider-thumb]:h-7 [&::-webkit-slider-thumb]:appearance-none [&::-webkit-slider-thumb]:bg-white [&::-webkit-slider-thumb]:rounded-sm [&::-webkit-slider-thumb]:cursor-pointer [&::-webkit-slider-thumb]:border-solid [&::-webkit-slider-thumb]:border-[3px] [&::-webkit-slider-thumb]:border-gray-900 [&::-webkit-slider-thumb]:shadow-[3px_3px_0_#000] ">
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{% if show_examples %}
|
||||
<!-- Example repositories section -->
|
||||
<div class="mt-4">
|
||||
|
|
|
|||
|
|
@ -21,7 +21,6 @@
|
|||
|
||||
fetchGitHubStars();
|
||||
</script>
|
||||
|
||||
<header class="sticky top-0 bg-[#FFFDF8] border-b-[3px] border-gray-900 z-50">
|
||||
<div class="max-w-4xl mx-auto px-4">
|
||||
<div class="flex justify-between items-center h-16">
|
||||
|
|
@ -33,24 +32,29 @@
|
|||
</a>
|
||||
</h1>
|
||||
</div>
|
||||
|
||||
<!-- Navigation with updated styling -->
|
||||
<nav class="flex items-center space-x-6">
|
||||
<a href="/api" class="text-gray-900 hover:-translate-y-0.5 transition-transform">API</a>
|
||||
<a href="/api"
|
||||
class="text-gray-900 hover:-translate-y-0.5 transition-transform">API</a>
|
||||
<div class="flex items-center gap-2">
|
||||
<a href="https://github.com/cyclotruc/gitingest" target="_blank" rel="noopener noreferrer"
|
||||
<a href="https://github.com/cyclotruc/gitingest"
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
class="text-gray-900 hover:-translate-y-0.5 transition-transform flex items-center gap-1.5">
|
||||
<svg class="w-4 h-4" fill="currentColor" viewBox="0 0 24 24" aria-hidden="true">
|
||||
<path fill-rule="evenodd"
|
||||
d="M12 2C6.477 2 2 6.484 2 12.017c0 4.425 2.865 8.18 6.839 9.504.5.092.682-.217.682-.483 0-.237-.008-.868-.013-1.703-2.782.605-3.369-1.343-3.369-1.343-.454-1.158-1.11-1.466-1.11-1.466-.908-.62.069-.608.069-.608 1.003.07 1.531 1.032 1.531 1.032.892 1.53 2.341 1.088 2.91.832.092-.647.35-1.088.636-1.338-2.22-.253-4.555-1.113-4.555-4.951 0-1.093.39-1.988 1.029-2.688-.103-.253-.446-1.272.098-2.65 0 0 .84-.27 2.75 1.026A9.564 9.564 0 0112 6.844c.85.004 1.705.115 2.504.337 1.909-1.296 2.747-1.027 2.747-1.027.546 1.379.202 2.398.1 2.651.64.7 1.028 1.595 1.028 2.688 0 3.848-2.339 4.695-4.566 4.943.359.309.678.92.678 1.855 0 1.338-.012 2.419-.012 2.747 0 .268.18.58.688.482A10.019 10.019 0 0022 12.017C22 6.484 17.522 2 12 2z"
|
||||
clip-rule="evenodd"></path>
|
||||
<svg class="w-4 h-4"
|
||||
fill="currentColor"
|
||||
viewBox="0 0 24 24"
|
||||
aria-hidden="true">
|
||||
<path fill-rule="evenodd" d="M12 2C6.477 2 2 6.484 2 12.017c0 4.425 2.865 8.18 6.839 9.504.5.092.682-.217.682-.483 0-.237-.008-.868-.013-1.703-2.782.605-3.369-1.343-3.369-1.343-.454-1.158-1.11-1.466-1.11-1.466-.908-.62.069-.608.069-.608 1.003.07 1.531 1.032 1.531 1.032.892 1.53 2.341 1.088 2.91.832.092-.647.35-1.088.636-1.338-2.22-.253-4.555-1.113-4.555-4.951 0-1.093.39-1.988 1.029-2.688-.103-.253-.446-1.272.098-2.65 0 0 .84-.27 2.75 1.026A9.564 9.564 0 0112 6.844c.85.004 1.705.115 2.504.337 1.909-1.296 2.747-1.027 2.747-1.027.546 1.379.202 2.398.1 2.651.64.7 1.028 1.595 1.028 2.688 0 3.848-2.339 4.695-4.566 4.943.359.309.678.92.678 1.855 0 1.338-.012 2.419-.012 2.747 0 .268.18.58.688.482A10.019 10.019 0 0022 12.017C22 6.484 17.522 2 12 2z" clip-rule="evenodd">
|
||||
</path>
|
||||
</svg>
|
||||
GitHub
|
||||
</a>
|
||||
<div class="flex items-center text-sm text-gray-600">
|
||||
<svg class="w-4 h-4 text-[#ffc480] mr-1" fill="currentColor" viewBox="0 0 20 20">
|
||||
<path
|
||||
d="M9.049 2.927c.3-.921 1.603-.921 1.902 0l1.07 3.292a1 1 0 00.95.69h3.462c.969 0 1.371 1.24.588 1.81l-2.8 2.034a1 1 0 00-.364 1.118l1.07 3.292c.3.921-.755 1.688-1.54 1.118l-2.8-2.034a1 1 0 00-1.175 0l-2.8 2.034c-.784.57-1.838-.197-1.539-1.118l1.07-3.292a1 1 0 00-.364-1.118L2.98 8.72c-.783-.57-.38-1.81.588-1.81h3.461a1 1 0 00.951-.69l1.07-3.292z" />
|
||||
<svg class="w-4 h-4 text-[#ffc480] mr-1"
|
||||
fill="currentColor"
|
||||
viewBox="0 0 20 20">
|
||||
<path d="M9.049 2.927c.3-.921 1.603-.921 1.902 0l1.07 3.292a1 1 0 00.95.69h3.462c.969 0 1.371 1.24.588 1.81l-2.8 2.034a1 1 0 00-.364 1.118l1.07 3.292c.3.921-.755 1.688-1.54 1.118l-2.8-2.034a1 1 0 00-1.175 0l-2.8 2.034c-.784.57-1.838-.197-1.539-1.118l1.07-3.292a1 1 0 00-.364-1.118L2.98 8.72c-.783-.57-.38-1.81.588-1.81h3.461a1 1 0 00.951-.69l1.07-3.292z" />
|
||||
</svg>
|
||||
<span id="github-stars">0</span>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -10,101 +10,80 @@
|
|||
<div class="flex justify-between items-center mb-4 py-2">
|
||||
<h3 class="text-lg font-bold text-gray-900">Summary</h3>
|
||||
</div>
|
||||
|
||||
|
||||
<div class="relative">
|
||||
<div
|
||||
class="w-full h-full rounded bg-gray-900 translate-y-1 translate-x-1 absolute inset-0">
|
||||
</div>
|
||||
<textarea
|
||||
class="w-full h-[160px] p-4 bg-[#fff4da] border-[3px] border-gray-900 rounded font-mono text-sm resize-none focus:outline-none relative z-10"
|
||||
<div class="w-full h-full rounded bg-gray-900 translate-y-1 translate-x-1 absolute inset-0"></div>
|
||||
<textarea class="w-full h-[160px] p-4 bg-[#fff4da] border-[3px] border-gray-900 rounded font-mono text-sm resize-none focus:outline-none relative z-10"
|
||||
readonly>{{ summary }}</textarea>
|
||||
</div>
|
||||
{% if ingest_id %}
|
||||
<div class="relative mt-4 inline-block group">
|
||||
<div
|
||||
class="w-full h-full rounded bg-gray-900 translate-y-1 translate-x-1 absolute inset-0">
|
||||
</div>
|
||||
<div class="w-full h-full rounded bg-gray-900 translate-y-1 translate-x-1 absolute inset-0"></div>
|
||||
<a href="/download/{{ ingest_id }}"
|
||||
class="inline-flex items-center px-4 py-2 bg-[#ffc480] border-[3px] border-gray-900 text-gray-900 rounded group-hover:-translate-y-px group-hover:-translate-x-px transition-transform relative z-10">
|
||||
<svg class="w-4 h-4 mr-2" fill="none" stroke="currentColor" viewBox="0 0 24 24">
|
||||
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2"
|
||||
d="M4 16v1a3 3 0 003 3h10a3 3 0 003-3v-1m-4-4l-4 4m0 0l-4-4m4 4V4" />
|
||||
<svg class="w-4 h-4 mr-2"
|
||||
fill="none"
|
||||
stroke="currentColor"
|
||||
viewBox="0 0 24 24">
|
||||
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M4 16v1a3 3 0 003 3h10a3 3 0 003-3v-1m-4-4l-4 4m0 0l-4-4m4 4V4" />
|
||||
</svg>
|
||||
Download
|
||||
</a>
|
||||
</div>
|
||||
<div class="relative mt-4 inline-block group ml-4">
|
||||
<div
|
||||
class="w-full h-full rounded bg-gray-900 translate-y-1 translate-x-1 absolute inset-0">
|
||||
</div>
|
||||
<div class="w-full h-full rounded bg-gray-900 translate-y-1 translate-x-1 absolute inset-0"></div>
|
||||
<button onclick="copyFullDigest()"
|
||||
class="inline-flex items-center px-4 py-2 bg-[#ffc480] border-[3px] border-gray-900 text-gray-900 rounded group-hover:-translate-y-px group-hover:-translate-x-px transition-transform relative z-10">
|
||||
<svg class="w-4 h-4 mr-2" fill="none" stroke="currentColor" viewBox="0 0 24 24">
|
||||
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2"
|
||||
d="M8 5H6a2 2 0 00-2 2v12a2 2 0 002 2h10a2 2 0 002-2v-1M8 5a2 2 0 002 2h2a2 2 0 002-2M8 5a2 2 0 012-2h2a2 2 0 012 2m0 0h2a2 2 0 012 2v3m2 4H10m0 0l3-3m-3 3l3 3" />
|
||||
<svg class="w-4 h-4 mr-2"
|
||||
fill="none"
|
||||
stroke="currentColor"
|
||||
viewBox="0 0 24 24">
|
||||
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M8 5H6a2 2 0 00-2 2v12a2 2 0 002 2h10a2 2 0 002-2v-1M8 5a2 2 0 002 2h2a2 2 0 002-2M8 5a2 2 0 012-2h2a2 2 0 012 2m0 0h2a2 2 0 012 2v3m2 4H10m0 0l3-3m-3 3l3 3" />
|
||||
</svg>
|
||||
Copy all
|
||||
</button>
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<!-- Directory Structure Column -->
|
||||
<div class="md:col-span-7">
|
||||
<div class="flex justify-between items-center mb-4">
|
||||
<h3 class="text-lg font-bold text-gray-900">Directory Structure</h3>
|
||||
<div class="relative group">
|
||||
<div
|
||||
class="w-full h-full rounded bg-gray-900 translate-y-1 translate-x-1 absolute inset-0">
|
||||
</div>
|
||||
<div class="w-full h-full rounded bg-gray-900 translate-y-1 translate-x-1 absolute inset-0"></div>
|
||||
<button onclick="copyText('directory-structure')"
|
||||
class="px-4 py-2 bg-[#ffc480] border-[3px] border-gray-900 text-gray-900 rounded group-hover:-translate-y-px group-hover:-translate-x-px transition-transform relative z-10 flex items-center gap-2">
|
||||
<svg class="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24">
|
||||
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2"
|
||||
d="M8 5H6a2 2 0 00-2 2v12a2 2 0 002 2h10a2 2 0 002-2v-1M8 5a2 2 0 002 2h2a2 2 0 002-2M8 5a2 2 0 012-2h2a2 2 0 012 2m0 0h2a2 2 0 012 2v3m2 4H10m0 0l3-3m-3 3l3 3" />
|
||||
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M8 5H6a2 2 0 00-2 2v12a2 2 0 002 2h10a2 2 0 002-2v-1M8 5a2 2 0 002 2h2a2 2 0 002-2M8 5a2 2 0 012-2h2a2 2 0 012 2m0 0h2a2 2 0 012 2v3m2 4H10m0 0l3-3m-3 3l3 3" />
|
||||
</svg>
|
||||
Copy
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="relative">
|
||||
<div
|
||||
class="w-full h-full rounded bg-gray-900 translate-y-1 translate-x-1 absolute inset-0">
|
||||
</div>
|
||||
<textarea
|
||||
class="directory-structure w-full p-4 bg-[#fff4da] border-[3px] border-gray-900 rounded font-mono text-sm resize-y focus:outline-none relative z-10 h-[215px]"
|
||||
<div class="w-full h-full rounded bg-gray-900 translate-y-1 translate-x-1 absolute inset-0"></div>
|
||||
<textarea class="directory-structure w-full p-4 bg-[#fff4da] border-[3px] border-gray-900 rounded font-mono text-sm resize-y focus:outline-none relative z-10 h-[215px]"
|
||||
readonly>{{ tree }}</textarea>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Full Digest -->
|
||||
<div>
|
||||
<div class="flex justify-between items-center mb-4">
|
||||
<h3 class="text-lg font-bold text-gray-900">Files Content</h3>
|
||||
<div class="relative group">
|
||||
<div
|
||||
class="w-full h-full rounded bg-gray-900 translate-y-1 translate-x-1 absolute inset-0">
|
||||
</div>
|
||||
<div class="w-full h-full rounded bg-gray-900 translate-y-1 translate-x-1 absolute inset-0"></div>
|
||||
<button onclick="copyText('result-text')"
|
||||
class="px-4 py-2 bg-[#ffc480] border-[3px] border-gray-900 text-gray-900 rounded group-hover:-translate-y-px group-hover:-translate-x-px transition-transform relative z-10 flex items-center gap-2">
|
||||
<svg class="w-4 h-4" fill="none" stroke="currentColor" viewBox="0 0 24 24">
|
||||
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2"
|
||||
d="M8 5H6a2 2 0 00-2 2v12a2 2 0 002 2h10a2 2 0 002-2v-1M8 5a2 2 0 002 2h2a2 2 0 002-2M8 5a2 2 0 012-2h2a2 2 0 012 2m0 0h2a2 2 0 012 2v3m2 4H10m0 0l3-3m-3 3l3 3" />
|
||||
<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M8 5H6a2 2 0 00-2 2v12a2 2 0 002 2h10a2 2 0 002-2v-1M8 5a2 2 0 002 2h2a2 2 0 002-2M8 5a2 2 0 012-2h2a2 2 0 012 2m0 0h2a2 2 0 012 2v3m2 4H10m0 0l3-3m-3 3l3 3" />
|
||||
</svg>
|
||||
Copy
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="relative">
|
||||
<div
|
||||
class="w-full h-full rounded bg-gray-900 translate-y-1 translate-x-1 absolute inset-0">
|
||||
</div>
|
||||
<textarea
|
||||
class="result-text w-full p-4 bg-[#fff4da] border-[3px] border-gray-900 rounded font-mono text-sm resize-y focus:outline-none relative z-10"
|
||||
<div class="w-full h-full rounded bg-gray-900 translate-y-1 translate-x-1 absolute inset-0"></div>
|
||||
<textarea class="result-text w-full p-4 bg-[#fff4da] border-[3px] border-gray-900 rounded font-mono text-sm resize-y focus:outline-none relative z-10"
|
||||
style="min-height: {{ '600px' if content else 'calc(100vh-800px)' }}"
|
||||
readonly>{{ content }}</textarea>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -1,17 +1,13 @@
|
|||
{% extends "base.jinja" %}
|
||||
|
||||
{% block content %}
|
||||
{% if error_message %}
|
||||
<div class="mb-6 p-4 bg-red-50 border border-red-200 rounded-lg text-red-700" id="error-message"
|
||||
data-message="{{ error_message }}">
|
||||
{{ error_message }}
|
||||
</div>
|
||||
<div class="mb-6 p-4 bg-red-50 border border-red-200 rounded-lg text-red-700"
|
||||
id="error-message"
|
||||
data-message="{{ error_message }}">{{ error_message }}</div>
|
||||
{% endif %}
|
||||
|
||||
{% with is_index=true, show_examples=false %}
|
||||
{% include 'components/github_form.jinja' %}
|
||||
{% endwith %}
|
||||
|
||||
{% if loading %}
|
||||
<div class="relative mt-10">
|
||||
<div class="w-full h-full absolute inset-0 bg-black rounded-xl translate-y-2 translate-x-2"></div>
|
||||
|
|
@ -21,10 +17,8 @@
|
|||
</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
{% include 'components/result.jinja' %}
|
||||
{% endblock content %}
|
||||
|
||||
{% block extra_scripts %}
|
||||
<script>
|
||||
document.addEventListener('DOMContentLoaded', function() {
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
{% extends "base.jinja" %}
|
||||
|
||||
{% block extra_head %}
|
||||
<script>
|
||||
function submitExample(repoName) {
|
||||
|
|
@ -9,32 +8,31 @@
|
|||
}
|
||||
</script>
|
||||
{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<div class="mb-12">
|
||||
<div
|
||||
class="relative w-full mx-auto flex sm:flex-row flex-col justify-center items-start sm:items-center">
|
||||
<div class="relative w-full mx-auto flex sm:flex-row flex-col justify-center items-start sm:items-center">
|
||||
<svg class="h-auto w-16 sm:w-20 md:w-24 flex-shrink-0 p-2 md:relative sm:absolute lg:absolute left-0 lg:-translate-x-full lg:ml-32 md:translate-x-10 sm:-translate-y-16 md:-translate-y-0 -translate-x-2 lg:-translate-y-10"
|
||||
viewBox="0 0 91 98" fill="none" xmlns="http://www.w3.org/2000/svg">
|
||||
<path
|
||||
d="m35.878 14.162 1.333-5.369 1.933 5.183c4.47 11.982 14.036 21.085 25.828 24.467l5.42 1.555-5.209 2.16c-11.332 4.697-19.806 14.826-22.888 27.237l-1.333 5.369-1.933-5.183C34.56 57.599 24.993 48.496 13.201 45.114l-5.42-1.555 5.21-2.16c11.331-4.697 19.805-14.826 22.887-27.237Z"
|
||||
fill="#FE4A60" stroke="#000" stroke-width="3.445"></path>
|
||||
<path
|
||||
d="M79.653 5.729c-2.436 5.323-9.515 15.25-18.341 12.374m9.197 16.336c2.6-5.851 10.008-16.834 18.842-13.956m-9.738-15.07c-.374 3.787 1.076 12.078 9.869 14.943M70.61 34.6c.503-4.21-.69-13.346-9.49-16.214M14.922 65.967c1.338 5.677 6.372 16.756 15.808 15.659M18.21 95.832c-1.392-6.226-6.54-18.404-15.984-17.305m12.85-12.892c-.41 3.771-3.576 11.588-12.968 12.681M18.025 96c.367-4.21 3.453-12.905 12.854-14"
|
||||
stroke="#000" stroke-width="2.548" stroke-linecap="round"></path>
|
||||
viewBox="0 0 91 98"
|
||||
fill="none"
|
||||
xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="m35.878 14.162 1.333-5.369 1.933 5.183c4.47 11.982 14.036 21.085 25.828 24.467l5.42 1.555-5.209 2.16c-11.332 4.697-19.806 14.826-22.888 27.237l-1.333 5.369-1.933-5.183C34.56 57.599 24.993 48.496 13.201 45.114l-5.42-1.555 5.21-2.16c11.331-4.697 19.805-14.826 22.887-27.237Z" fill="#FE4A60" stroke="#000" stroke-width="3.445">
|
||||
</path>
|
||||
<path d="M79.653 5.729c-2.436 5.323-9.515 15.25-18.341 12.374m9.197 16.336c2.6-5.851 10.008-16.834 18.842-13.956m-9.738-15.07c-.374 3.787 1.076 12.078 9.869 14.943M70.61 34.6c.503-4.21-.69-13.346-9.49-16.214M14.922 65.967c1.338 5.677 6.372 16.756 15.808 15.659M18.21 95.832c-1.392-6.226-6.54-18.404-15.984-17.305m12.85-12.892c-.41 3.771-3.576 11.588-12.968 12.681M18.025 96c.367-4.21 3.453-12.905 12.854-14" stroke="#000" stroke-width="2.548" stroke-linecap="round">
|
||||
</path>
|
||||
</svg>
|
||||
<h1
|
||||
class="text-4xl sm:text-5xl sm:pt-20 lg:pt-5 md:text-6xl lg:text-7xl font-bold tracking-tighter w-full inline-block text-left md:text-center relative">
|
||||
Prompt-friendly <br>codebase
|
||||
<h1 class="text-4xl sm:text-5xl sm:pt-20 lg:pt-5 md:text-6xl lg:text-7xl font-bold tracking-tighter w-full inline-block text-left md:text-center relative">
|
||||
Prompt-friendly
|
||||
<br>
|
||||
codebase
|
||||
</h1>
|
||||
<svg class="w-16 lg:w-20 h-auto lg:absolute flex-shrink-0 right-0 bottom-0 md:block hidden translate-y-10 md:translate-y-20 lg:translate-y-4 lg:-translate-x-12 -translate-x-10"
|
||||
viewBox="0 0 92 80" fill="none" xmlns="http://www.w3.org/2000/svg">
|
||||
<path
|
||||
d="m35.213 16.953.595-5.261 2.644 4.587a35.056 35.056 0 0 0 26.432 17.33l5.261.594-4.587 2.644A35.056 35.056 0 0 0 48.23 63.28l-.595 5.26-2.644-4.587a35.056 35.056 0 0 0-26.432-17.328l-5.261-.595 4.587-2.644a35.056 35.056 0 0 0 17.329-26.433Z"
|
||||
fill="#5CF1A4" stroke="#000" stroke-width="2.868" class=""></path>
|
||||
<path
|
||||
d="M75.062 40.108c1.07 5.255 1.072 16.52-7.472 19.54m7.422-19.682c1.836 2.965 7.643 8.14 16.187 5.121-8.544 3.02-8.207 15.23-6.971 20.957-1.97-3.343-8.044-9.274-16.588-6.254M12.054 28.012c1.34-5.22 6.126-15.4 14.554-14.369M12.035 28.162c-.274-3.487-2.93-10.719-11.358-11.75C9.104 17.443 14.013 6.262 15.414.542c.226 3.888 2.784 11.92 11.212 12.95"
|
||||
stroke="#000" stroke-width="2.319" stroke-linecap="round"></path>
|
||||
viewBox="0 0 92 80"
|
||||
fill="none"
|
||||
xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="m35.213 16.953.595-5.261 2.644 4.587a35.056 35.056 0 0 0 26.432 17.33l5.261.594-4.587 2.644A35.056 35.056 0 0 0 48.23 63.28l-.595 5.26-2.644-4.587a35.056 35.056 0 0 0-26.432-17.328l-5.261-.595 4.587-2.644a35.056 35.056 0 0 0 17.329-26.433Z" fill="#5CF1A4" stroke="#000" stroke-width="2.868" class="">
|
||||
</path>
|
||||
<path d="M75.062 40.108c1.07 5.255 1.072 16.52-7.472 19.54m7.422-19.682c1.836 2.965 7.643 8.14 16.187 5.121-8.544 3.02-8.207 15.23-6.971 20.957-1.97-3.343-8.044-9.274-16.588-6.254M12.054 28.012c1.34-5.22 6.126-15.4 14.554-14.369M12.035 28.162c-.274-3.487-2.93-10.719-11.358-11.75C9.104 17.443 14.013 6.262 15.414.542c.226 3.888 2.784 11.92 11.212 12.95" stroke="#000" stroke-width="2.319" stroke-linecap="round">
|
||||
</path>
|
||||
</svg>
|
||||
</div>
|
||||
<p class="text-gray-600 text-lg max-w-2xl mx-auto text-center mt-8">
|
||||
|
|
@ -47,21 +45,13 @@
|
|||
You can also replace 'hub' with 'ingest' in any Github URL
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{% if error_message %}
|
||||
<div class="mb-6 p-4 bg-red-50 border border-red-200 rounded-lg text-red-700" id="error-message"
|
||||
data-message="{{ error_message }}">
|
||||
{{ error_message }}
|
||||
</div>
|
||||
<div class="mb-6 p-4 bg-red-50 border border-red-200 rounded-lg text-red-700"
|
||||
id="error-message"
|
||||
data-message="{{ error_message }}">{{ error_message }}</div>
|
||||
{% endif %}
|
||||
|
||||
{% with is_index=true, show_examples=true %}
|
||||
{% include 'components/github_form.jinja' %}
|
||||
{% endwith %}
|
||||
|
||||
{% include 'components/result.jinja' %}
|
||||
|
||||
|
||||
|
||||
|
||||
{% endblock %}
|
||||
Loading…
Add table
Add a link
Reference in a new issue