mirror of
https://github.com/hanzo-ml/trainer.git
synced 2026-09-24 00:31:34 +00:00
feat(ci): add Python dependency scanning to OSV-Scanner workflow (#3530)
* feat(ci): add Python dependency scanning to OSV-Scanner workflow Add lockfiles and scanning for Python dependencies in the initializers and python_api. This extends the nightly OSV-Scanner workflow to cover Python in addition to Go. - Generate requirements-lock.txt files via pip-compile for: - cmd/initializers/dataset/ - cmd/initializers/model/ - api/python_api/ - Add osv-scan-python job with SARIF scanning and auto-fix - Add validate-lockfile.yaml to ensure lockfiles stay in sync on PRs - Handle both direct and transitive dependency upgrades Closes #3528 Signed-off-by: Fiona-Waters <[email protected]> * fix(ci): address review feedback on Python OSV-Scanner workflow - Fix validate-lockfile false positives by stripping header comments before diffing (pip-compile embeds output filename in header) - Regenerate lockfiles in clean Python 3.12 venv with --strip-extras to include all transitive deps and silence pip-tools deprecation warning - Fix PR body table formatting: use Markdown table rows instead of bullet items in both Go and Python auto-fix jobs - Escape dots in package names for grep/sed regex safety Signed-off-by: Fiona-Waters <[email protected]> * fix(ci): address review feedback on OSV-Scanner Python workflow - Replace shell grep/sed with Python helper script using packaging.requirements.Requirement for exact name matching, fixing prefix collision (requests vs requests-oauthlib), constraint dropping, and extras stripping - Skip entry when package not found in lockfile after pip-compile instead of recording a false version - Bump actions/setup-python from v5 to v6 for consistency Signed-off-by: Fiona-Waters <[email protected]> * fix(ci): pin pip-tools version in CI workflows Pin pip-tools to 7.4.1 in osv-scanner and validate-lockfile workflows to mitigate supply-chain risk in workflows with elevated permissions. Signed-off-by: Fiona-Waters <[email protected]> * fix(deps): update lockfiles for certifi 2026.5.20 Signed-off-by: Fiona-Waters <[email protected]> --------- Signed-off-by: Fiona-Waters <[email protected]> Co-authored-by: Hanzo Dev <[email protected]>
This commit is contained in:
Fiona Waters
committed by
GitHub
GitHub
co-authored by
Hanzo Dev
Hanzo Dev
parent
8704ba0bd4
commit
4af1d0a81b
@@ -161,7 +161,7 @@ jobs:
|
||||
fi
|
||||
|
||||
echo "Installed version: v${INSTALLED_VER}"
|
||||
printf '%s\n' "- ${pkg}@v${INSTALLED_VER} | [$advisory_id]($advisory_url)" >> applied_fixes.txt
|
||||
printf '%s\n' "| ${pkg}@v${INSTALLED_VER} | [$advisory_id]($advisory_url) |" >> applied_fixes.txt
|
||||
done <<< "$FIX_DATA"
|
||||
|
||||
if [ ! -s applied_fixes.txt ]; then
|
||||
@@ -209,3 +209,240 @@ jobs:
|
||||
delete-branch: true
|
||||
labels: |
|
||||
"area/security"
|
||||
|
||||
osv-scan-python:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: write
|
||||
security-events: write
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Setup Python
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: '3.12'
|
||||
|
||||
- name: Install OSV-Scanner
|
||||
run: |
|
||||
OSV_VERSION="2.3.8"
|
||||
EXPECTED_SHA="bc98e15319ed0d515e3f9235287ba53cdc5535d576d24fd573978ecfe9ab92dc"
|
||||
curl -fsSL "https://github.com/google/osv-scanner/releases/download/v${OSV_VERSION}/osv-scanner_linux_amd64" -o osv-scanner
|
||||
ACTUAL_SHA=$(sha256sum osv-scanner | awk '{print $1}')
|
||||
if [ "$ACTUAL_SHA" != "$EXPECTED_SHA" ]; then
|
||||
echo "::error::OSV-Scanner checksum mismatch! Expected ${EXPECTED_SHA}, got ${ACTUAL_SHA}"
|
||||
exit 1
|
||||
fi
|
||||
chmod +x osv-scanner
|
||||
sudo mv osv-scanner /usr/local/bin/
|
||||
|
||||
- name: Install pip-tools
|
||||
run: pip install 'pip-tools==7.5.3'
|
||||
|
||||
- name: Run OSV-Scanner (SARIF for Security Tab)
|
||||
continue-on-error: true
|
||||
run: |
|
||||
rc=0
|
||||
osv-scanner scan \
|
||||
--format sarif \
|
||||
--output osv-results-python.sarif \
|
||||
-L cmd/initializers/dataset/requirements-lock.txt \
|
||||
-L cmd/initializers/model/requirements-lock.txt \
|
||||
-L api/python_api/requirements-lock.txt || rc=$?
|
||||
if [ "$rc" -ne 0 ] && [ "$rc" -ne 1 ]; then
|
||||
echo "::error::OSV-Scanner failed with unexpected exit code $rc"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Upload SARIF to GitHub Security tab
|
||||
if: always() && hashFiles('osv-results-python.sarif') != ''
|
||||
uses: github/codeql-action/upload-sarif@v4
|
||||
with:
|
||||
sarif_file: osv-results-python.sarif
|
||||
category: kubeflow-trainer-osv-scanner-python
|
||||
|
||||
- name: Run OSV-Scanner (JSON for auto-fix)
|
||||
run: |
|
||||
rc=0
|
||||
osv-scanner scan \
|
||||
--format json \
|
||||
--output osv-results-python.json \
|
||||
-L cmd/initializers/dataset/requirements-lock.txt \
|
||||
-L cmd/initializers/model/requirements-lock.txt \
|
||||
-L api/python_api/requirements-lock.txt || rc=$?
|
||||
if [ "$rc" -ne 0 ] && [ "$rc" -ne 1 ]; then
|
||||
echo "::error::OSV-Scanner failed with unexpected exit code $rc"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Process vulnerabilities and apply fixes
|
||||
id: fixer
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
if [ ! -f osv-results-python.json ] || [ ! -s osv-results-python.json ]; then
|
||||
echo "No scan results found."
|
||||
echo "updates_found=false" >> $GITHUB_OUTPUT
|
||||
exit 0
|
||||
fi
|
||||
|
||||
FIX_DATA=$(jq -r '
|
||||
[
|
||||
.results[]?.packages[]? |
|
||||
.package as $pkg |
|
||||
.vulnerabilities[]? |
|
||||
. as $vuln |
|
||||
.affected[]? |
|
||||
select(.package.name == $pkg.name) |
|
||||
.ranges[]? |
|
||||
.events[] |
|
||||
select(.fixed != null) |
|
||||
{
|
||||
pkg: $pkg.name,
|
||||
ver: $pkg.version,
|
||||
fix: (.fixed | sub("^v"; "")),
|
||||
id: $vuln.id,
|
||||
url: ($vuln.references[0].url // "https://osv.dev/vulnerability/\($vuln.id)")
|
||||
}
|
||||
] |
|
||||
.[] |
|
||||
"\(.pkg)|\(.ver)|\(.fix)|\(.id)|\(.url)"
|
||||
' osv-results-python.json \
|
||||
| sort -t'|' -k1,1 -k4,4 -k3,3Vr \
|
||||
| awk -F'|' '!seen[$1,$4]++' \
|
||||
)
|
||||
|
||||
if [ -z "$FIX_DATA" ]; then
|
||||
echo "No fixable vulnerabilities found."
|
||||
echo "updates_found=false" >> $GITHUB_OUTPUT
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "Fixable vulnerabilities found:"
|
||||
echo "$FIX_DATA"
|
||||
echo ""
|
||||
|
||||
# Map lockfiles back to their source requirements files
|
||||
LOCKFILES=(
|
||||
"cmd/initializers/dataset/requirements-lock.txt|cmd/initializers/dataset/requirements.txt"
|
||||
"cmd/initializers/model/requirements-lock.txt|cmd/initializers/model/requirements.txt"
|
||||
"api/python_api/requirements-lock.txt|api/python_api/pyproject.toml"
|
||||
)
|
||||
|
||||
rm -f applied_fixes.txt
|
||||
|
||||
while IFS='|' read -r pkg current_ver fix_ver advisory_id advisory_url; do
|
||||
[ -z "$pkg" ] && continue
|
||||
|
||||
echo ""
|
||||
echo "=== Processing: $pkg $current_ver -> $fix_ver ($advisory_id) ==="
|
||||
|
||||
# Escape dots in package name for safe use in grep/sed regex
|
||||
pkg_re=$(echo "$pkg" | sed 's/\./\\./g')
|
||||
|
||||
FIXED=false
|
||||
for mapping in "${LOCKFILES[@]}"; do
|
||||
lockfile="${mapping%%|*}"
|
||||
source_file="${mapping##*|}"
|
||||
|
||||
# Check if this lockfile contains the vulnerable package (case-insensitive)
|
||||
if ! grep -qi "^${pkg_re}==" "$lockfile"; then
|
||||
continue
|
||||
fi
|
||||
|
||||
echo "Found $pkg in $lockfile (source: $source_file)"
|
||||
|
||||
cp "$lockfile" "${lockfile}.bak"
|
||||
cp "$source_file" "${source_file}.bak"
|
||||
|
||||
# Detect if pkg is a direct dependency and update the constraint in-place.
|
||||
# Uses packaging.requirements.Requirement for exact name matching (no prefix
|
||||
# collision) and specifier intersection (preserves extras and upper bounds).
|
||||
UPDATE_RESULT=$(python3 hack/scripts/update-source-dep.py "$pkg" "$fix_ver" "$source_file")
|
||||
|
||||
if [ "$UPDATE_RESULT" = "direct" ]; then
|
||||
echo "Direct dependency — updated version constraint in $source_file"
|
||||
else
|
||||
echo "Transitive dependency — upgrading via pip-compile"
|
||||
fi
|
||||
|
||||
# Regenerate the lockfile, upgrading the specific package
|
||||
COMPILE_RC=0
|
||||
pip-compile "$source_file" -o "$lockfile" --upgrade-package "${pkg}>=${fix_ver}" --strip-extras --quiet 2>&1 || COMPILE_RC=$?
|
||||
if [ "$COMPILE_RC" -ne 0 ]; then
|
||||
echo "::warning::pip-compile failed for $source_file (exit code $COMPILE_RC), skipping"
|
||||
mv "${lockfile}.bak" "$lockfile"
|
||||
mv "${source_file}.bak" "$source_file"
|
||||
continue
|
||||
fi
|
||||
|
||||
# Verify the fix was applied (case-insensitive)
|
||||
INSTALLED_VER=$(grep -i "^${pkg_re}==" "$lockfile" | sed 's/.*==//' || true)
|
||||
if [ -z "$INSTALLED_VER" ]; then
|
||||
echo "::warning::$pkg not found in $lockfile after pip-compile (may have been removed), skipping"
|
||||
rm -f "${lockfile}.bak" "${source_file}.bak"
|
||||
continue
|
||||
fi
|
||||
|
||||
rm -f "${lockfile}.bak" "${source_file}.bak"
|
||||
|
||||
echo "Installed version: ${INSTALLED_VER}"
|
||||
printf '%s\n' "| ${pkg}==${INSTALLED_VER} | [$advisory_id]($advisory_url) |" >> applied_fixes.txt
|
||||
FIXED=true
|
||||
done
|
||||
|
||||
if [ "$FIXED" = false ]; then
|
||||
echo "::warning::Could not find $pkg in any lockfile, skipping"
|
||||
fi
|
||||
done <<< "$FIX_DATA"
|
||||
|
||||
if [ ! -s applied_fixes.txt ]; then
|
||||
echo "No fixes were applied."
|
||||
echo "updates_found=false" >> $GITHUB_OUTPUT
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "updates_found=true" >> $GITHUB_OUTPUT
|
||||
{
|
||||
echo "fix_details<<EOF"
|
||||
cat applied_fixes.txt
|
||||
echo "EOF"
|
||||
} >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Create Pull Request
|
||||
if: steps.fixer.outputs.updates_found == 'true'
|
||||
uses: peter-evans/create-pull-request@v8
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
commit-message: "fix: nightly automated Python dependency update for security vulnerabilities"
|
||||
title: "fix: nightly security Python dependency updates"
|
||||
add-paths: |
|
||||
cmd/initializers/dataset/requirements.txt
|
||||
cmd/initializers/dataset/requirements-lock.txt
|
||||
cmd/initializers/model/requirements.txt
|
||||
cmd/initializers/model/requirements-lock.txt
|
||||
api/python_api/pyproject.toml
|
||||
api/python_api/requirements-lock.txt
|
||||
body: |
|
||||
## Security Update (Python)
|
||||
|
||||
This is an automated PR triggered by the nightly OSV-Scanner security scan.
|
||||
|
||||
The following Python dependencies were updated to resolve known vulnerabilities:
|
||||
|
||||
| Package & Version | Advisory Link |
|
||||
| :--- | :--- |
|
||||
${{ steps.fixer.outputs.fix_details }}
|
||||
|
||||
**Verification:** Updated via `pip-compile` lockfile regeneration.
|
||||
|
||||
> **Note:** This PR was created with `GITHUB_TOKEN`, which does not trigger `pull_request` workflows.
|
||||
> CI checks will not run automatically. Push an empty commit (`git commit --allow-empty -m "trigger CI"`)
|
||||
> to kick off CI before merging.
|
||||
branch: security-nightly-python-updates
|
||||
delete-branch: true
|
||||
labels: |
|
||||
"area/security"
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
name: Validate Python Lockfiles
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'cmd/initializers/dataset/requirements.txt'
|
||||
- 'cmd/initializers/dataset/requirements-lock.txt'
|
||||
- 'cmd/initializers/model/requirements.txt'
|
||||
- 'cmd/initializers/model/requirements-lock.txt'
|
||||
- 'api/python_api/pyproject.toml'
|
||||
- 'api/python_api/requirements-lock.txt'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
validate-lockfiles:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Setup Python
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: '3.12'
|
||||
|
||||
- name: Install pip-tools
|
||||
run: pip install 'pip-tools==7.5.3'
|
||||
|
||||
- name: Verify lockfiles are in sync
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
LOCKFILE_PAIRS=(
|
||||
"cmd/initializers/dataset/requirements.txt|cmd/initializers/dataset/requirements-lock.txt"
|
||||
"cmd/initializers/model/requirements.txt|cmd/initializers/model/requirements-lock.txt"
|
||||
"api/python_api/pyproject.toml|api/python_api/requirements-lock.txt"
|
||||
)
|
||||
|
||||
FAILED=false
|
||||
|
||||
for pair in "${LOCKFILE_PAIRS[@]}"; do
|
||||
source_file="${pair%%|*}"
|
||||
lockfile="${pair##*|}"
|
||||
|
||||
echo "=== Checking $source_file -> $lockfile ==="
|
||||
|
||||
pip-compile "$source_file" -o "${lockfile}.check" --strip-extras --quiet 2>&1
|
||||
|
||||
# Strip auto-generated header comments before comparing — pip-compile
|
||||
# embeds the output filename in the header, which differs between the
|
||||
# committed lockfile and the .check file.
|
||||
grep -v '^#' "$lockfile" > "${lockfile}.stripped"
|
||||
grep -v '^#' "${lockfile}.check" > "${lockfile}.check.stripped"
|
||||
|
||||
if ! diff -q "${lockfile}.stripped" "${lockfile}.check.stripped" > /dev/null 2>&1; then
|
||||
echo "::error::$lockfile is out of sync with $source_file. Please run 'pip-compile $source_file -o $lockfile --strip-extras' and commit the changes."
|
||||
diff "${lockfile}.stripped" "${lockfile}.check.stripped" || true
|
||||
FAILED=true
|
||||
else
|
||||
echo "In sync"
|
||||
fi
|
||||
|
||||
rm -f "${lockfile}.check" "${lockfile}.stripped" "${lockfile}.check.stripped"
|
||||
done
|
||||
|
||||
if [ "$FAILED" = true ]; then
|
||||
echo ""
|
||||
echo "One or more lockfiles are out of sync."
|
||||
echo "To fix, run the following and commit the updated lockfiles:"
|
||||
echo ""
|
||||
echo " pip install pip-tools"
|
||||
echo " pip-compile cmd/initializers/dataset/requirements.txt -o cmd/initializers/dataset/requirements-lock.txt --strip-extras"
|
||||
echo " pip-compile cmd/initializers/model/requirements.txt -o cmd/initializers/model/requirements-lock.txt --strip-extras"
|
||||
echo " pip-compile api/python_api/pyproject.toml -o api/python_api/requirements-lock.txt --strip-extras"
|
||||
exit 1
|
||||
fi
|
||||
Generated
+19
@@ -0,0 +1,19 @@
|
||||
#
|
||||
# This file is autogenerated by pip-compile with Python 3.12
|
||||
# by the following command:
|
||||
#
|
||||
# pip-compile --output-file=api/python_api/requirements-lock.txt --strip-extras api/python_api/pyproject.toml
|
||||
#
|
||||
annotated-types==0.7.0
|
||||
# via pydantic
|
||||
pydantic==2.13.4
|
||||
# via kubeflow_trainer_api (api/python_api/pyproject.toml)
|
||||
pydantic-core==2.46.4
|
||||
# via pydantic
|
||||
typing-extensions==4.15.0
|
||||
# via
|
||||
# pydantic
|
||||
# pydantic-core
|
||||
# typing-inspection
|
||||
typing-inspection==0.4.2
|
||||
# via pydantic
|
||||
@@ -0,0 +1,89 @@
|
||||
#
|
||||
# This file is autogenerated by pip-compile with Python 3.12
|
||||
# by the following command:
|
||||
#
|
||||
# pip-compile --output-file=cmd/initializers/dataset/requirements-lock.txt --strip-extras cmd/initializers/dataset/requirements.txt
|
||||
#
|
||||
annotated-doc==0.0.4
|
||||
# via typer
|
||||
anyio==4.13.0
|
||||
# via httpx
|
||||
certifi==2026.5.20
|
||||
# via
|
||||
# httpcore
|
||||
# httpx
|
||||
# kubernetes
|
||||
# requests
|
||||
charset-normalizer==3.4.7
|
||||
# via requests
|
||||
click==8.4.0
|
||||
# via typer
|
||||
durationpy==0.10
|
||||
# via kubernetes
|
||||
filelock==3.29.0
|
||||
# via huggingface-hub
|
||||
fsspec==2026.4.0
|
||||
# via huggingface-hub
|
||||
h11==0.16.0
|
||||
# via httpcore
|
||||
hf-xet==1.5.0
|
||||
# via huggingface-hub
|
||||
httpcore==1.0.9
|
||||
# via httpx
|
||||
httpx==0.28.1
|
||||
# via huggingface-hub
|
||||
huggingface-hub==1.14.0
|
||||
# via -r cmd/initializers/dataset/requirements.txt
|
||||
idna==3.15
|
||||
# via
|
||||
# anyio
|
||||
# httpx
|
||||
# requests
|
||||
kubernetes==35.0.0
|
||||
# via -r cmd/initializers/dataset/requirements.txt
|
||||
markdown-it-py==4.2.0
|
||||
# via rich
|
||||
mdurl==0.1.2
|
||||
# via markdown-it-py
|
||||
oauthlib==3.3.1
|
||||
# via requests-oauthlib
|
||||
opendal==0.47.1
|
||||
# via -r cmd/initializers/dataset/requirements.txt
|
||||
packaging==26.2
|
||||
# via huggingface-hub
|
||||
pygments==2.20.0
|
||||
# via rich
|
||||
python-dateutil==2.9.0.post0
|
||||
# via kubernetes
|
||||
pyyaml==6.0.3
|
||||
# via
|
||||
# huggingface-hub
|
||||
# kubernetes
|
||||
requests==2.34.2
|
||||
# via
|
||||
# kubernetes
|
||||
# requests-oauthlib
|
||||
requests-oauthlib==2.0.0
|
||||
# via kubernetes
|
||||
rich==15.0.0
|
||||
# via typer
|
||||
shellingham==1.5.4
|
||||
# via typer
|
||||
six==1.17.0
|
||||
# via
|
||||
# kubernetes
|
||||
# python-dateutil
|
||||
tqdm==4.67.3
|
||||
# via huggingface-hub
|
||||
typer==0.25.1
|
||||
# via huggingface-hub
|
||||
typing-extensions==4.15.0
|
||||
# via
|
||||
# anyio
|
||||
# huggingface-hub
|
||||
urllib3==2.7.0
|
||||
# via
|
||||
# kubernetes
|
||||
# requests
|
||||
websocket-client==1.9.0
|
||||
# via kubernetes
|
||||
@@ -0,0 +1,58 @@
|
||||
#
|
||||
# This file is autogenerated by pip-compile with Python 3.12
|
||||
# by the following command:
|
||||
#
|
||||
# pip-compile --output-file=cmd/initializers/model/requirements-lock.txt --strip-extras cmd/initializers/model/requirements.txt
|
||||
#
|
||||
annotated-doc==0.0.4
|
||||
# via typer
|
||||
anyio==4.13.0
|
||||
# via httpx
|
||||
certifi==2026.5.20
|
||||
# via
|
||||
# httpcore
|
||||
# httpx
|
||||
click==8.4.0
|
||||
# via typer
|
||||
filelock==3.29.0
|
||||
# via huggingface-hub
|
||||
fsspec==2026.4.0
|
||||
# via huggingface-hub
|
||||
h11==0.16.0
|
||||
# via httpcore
|
||||
hf-xet==1.5.0
|
||||
# via huggingface-hub
|
||||
httpcore==1.0.9
|
||||
# via httpx
|
||||
httpx==0.28.1
|
||||
# via huggingface-hub
|
||||
huggingface-hub==1.14.0
|
||||
# via -r cmd/initializers/model/requirements.txt
|
||||
idna==3.15
|
||||
# via
|
||||
# anyio
|
||||
# httpx
|
||||
markdown-it-py==4.2.0
|
||||
# via rich
|
||||
mdurl==0.1.2
|
||||
# via markdown-it-py
|
||||
opendal==0.47.1
|
||||
# via -r cmd/initializers/model/requirements.txt
|
||||
packaging==26.2
|
||||
# via huggingface-hub
|
||||
pygments==2.20.0
|
||||
# via rich
|
||||
pyyaml==6.0.3
|
||||
# via huggingface-hub
|
||||
rich==15.0.0
|
||||
# via typer
|
||||
shellingham==1.5.4
|
||||
# via typer
|
||||
tqdm==4.67.3
|
||||
# via huggingface-hub
|
||||
typer==0.25.1
|
||||
# via huggingface-hub
|
||||
typing-extensions==4.15.0
|
||||
# via
|
||||
# anyio
|
||||
# huggingface-hub
|
||||
@@ -0,0 +1,81 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Update a direct dependency's version constraint in a requirements.txt or pyproject.toml.
|
||||
|
||||
Uses packaging.requirements.Requirement for exact name matching (no prefix collision)
|
||||
and specifier intersection (preserves extras and upper bounds).
|
||||
|
||||
Usage:
|
||||
python3 update-source-dep.py <package> <fix_version> <source_file>
|
||||
|
||||
Prints "direct" if the package was found and updated, "transitive" otherwise.
|
||||
"""
|
||||
|
||||
import re
|
||||
import sys
|
||||
import tomllib
|
||||
|
||||
from packaging.requirements import Requirement
|
||||
from packaging.specifiers import SpecifierSet
|
||||
|
||||
|
||||
def normalize(name):
|
||||
return re.sub(r"[-_.]+", "-", name).lower()
|
||||
|
||||
|
||||
def update_requirements_txt(target, fix_ver, source_file):
|
||||
with open(source_file, "r") as f:
|
||||
lines = f.readlines()
|
||||
found = False
|
||||
new_lines = []
|
||||
for line in lines:
|
||||
stripped = line.strip()
|
||||
if stripped and not stripped.startswith("#"):
|
||||
try:
|
||||
req = Requirement(stripped)
|
||||
if normalize(req.name) == target:
|
||||
found = True
|
||||
req.specifier = req.specifier & SpecifierSet(f">={fix_ver}")
|
||||
new_lines.append(str(req) + "\n")
|
||||
continue
|
||||
except Exception:
|
||||
pass
|
||||
new_lines.append(line)
|
||||
if found:
|
||||
with open(source_file, "w") as f:
|
||||
f.writelines(new_lines)
|
||||
return found
|
||||
|
||||
|
||||
def update_pyproject_toml(target, fix_ver, source_file):
|
||||
with open(source_file, "rb") as f:
|
||||
deps = tomllib.load(f).get("project", {}).get("dependencies", [])
|
||||
found = False
|
||||
with open(source_file, "r") as f:
|
||||
content = f.read()
|
||||
for dep in deps:
|
||||
req = Requirement(dep)
|
||||
if normalize(req.name) == target:
|
||||
found = True
|
||||
new_req = Requirement(dep)
|
||||
new_req.specifier = req.specifier & SpecifierSet(f">={fix_ver}")
|
||||
content = content.replace(f'"{dep}"', f'"{new_req}"')
|
||||
if found:
|
||||
with open(source_file, "w") as f:
|
||||
f.write(content)
|
||||
return found
|
||||
|
||||
|
||||
def main():
|
||||
pkg_name, fix_ver, source_file = sys.argv[1], sys.argv[2], sys.argv[3]
|
||||
target = normalize(pkg_name)
|
||||
|
||||
if source_file.endswith(".toml"):
|
||||
found = update_pyproject_toml(target, fix_ver, source_file)
|
||||
else:
|
||||
found = update_requirements_txt(target, fix_ver, source_file)
|
||||
|
||||
print("direct" if found else "transitive")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user