Skip to content

PLT-0000 fix(deploy): serialize repocache access to prevent stale git locks #151

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
uptickmetachu wants to merge 2 commits into from
Closed

PLT-0000 fix(deploy): serialize repocache access to prevent stale git locks #151

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e21bb5f
fix(deploy): serialize repocache access to prevent stale git locks
uptickmetachu Apr 24, 2026
b21025a
ci: disable mise python github attestation verification
uptickmetachu Apr 24, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/workflows/test.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ permissions:

env:
PYTHONUNBUFFERED: 1
MISE_PYTHON_GITHUB_ATTESTATIONS: false

jobs:
ci:
Expand Down
41 changes: 27 additions & 14 deletions gitops_server/utils/git.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,17 @@

REPO_CACHE_DIR = Path("/tmp/gitops/repocache")
REPO_CACHE: dict[str, Path] = {}
repo_lock = asyncio.Semaphore(1)
_cache_locks: dict[str, asyncio.Lock] = {}
_cache_locks_guard = asyncio.Lock()


async def _get_cache_lock(git_repo_url: str) -> asyncio.Lock:
async with _cache_locks_guard:
lock = _cache_locks.get(git_repo_url)
if lock is None:
lock = asyncio.Lock()
_cache_locks[git_repo_url] = lock
return lock


async def clone_repo(git_repo_url: str, path: str, sha: str | None = None, branch: str | None = None) -> None:
Expand Down Expand Up @@ -57,22 +67,25 @@ async def temp_repo(git_repo_url: str, ref: str | None) -> AsyncGenerator[str, N

with tracer.start_as_current_span("checkout_temp_repo"):
cache_path = REPO_CACHE_DIR / git_repo_url.split("/")[-1].split(".")[0]
cache_lock = await _get_cache_lock(git_repo_url)

# Prep the repo cache
if not (cache_path / ".git").exists():
logger.info("Repo %s not in cache, cloning", git_repo_url)
async with repo_lock:
if cache_path.exists():
await run(f"rm -rf {cache_path}", suppress_errors=True)
if not cache_path.exists():
with tempfile.TemporaryDirectory() as temporary_folder_path:
# Serialize cache init and reads against the shared cache so a
# concurrent clone/fetch can't leak a half-written .git/index.lock
# into the copy.
async with cache_lock:
if not (cache_path / ".git").exists():
logger.info("Repo %s not in cache, cloning", git_repo_url)
if cache_path.exists():
await run(f"rm -rf {cache_path}", suppress_errors=True)
cache_path.mkdir(parents=True)
REPO_CACHE[git_repo_url] = cache_path
await clone_repo(git_repo_url, path=str(cache_path))
REPO_CACHE[git_repo_url] = cache_path
await clone_repo(git_repo_url, path=str(cache_path))

# Copy the repo cache to a temporary folder
with tempfile.TemporaryDirectory() as temporary_folder_path:
await run(f"rm -rf {temporary_folder_path}", suppress_errors=True)
await run(f"cp -r {cache_path} {temporary_folder_path}")
await run(f"rm -rf {temporary_folder_path}", suppress_errors=True)
await run(f"cp -r {cache_path} {temporary_folder_path}")

await run(f"rm -f {temporary_folder_path}/.git/index.lock", suppress_errors=True)
await run(f"cd {temporary_folder_path};git fetch; git checkout {ref}")

yield temporary_folder_path
Expand Down
Loading