From 26d3dd4760063aa5e3e83a8155cac73fbe0d6c47 Mon Sep 17 00:00:00 2001 From: David Soria Parra Date: Fri, 10 Jan 2025 21:05:09 +0000 Subject: [PATCH 01/11] feat: add weekly release automation --- .github/workflows/weekly-release.yml | 102 +++++++++++++++ scripts/release.py | 179 +++++++++++++++++++++++++++ 2 files changed, 281 insertions(+) create mode 100644 .github/workflows/weekly-release.yml create mode 100755 scripts/release.py diff --git a/.github/workflows/weekly-release.yml b/.github/workflows/weekly-release.yml new file mode 100644 index 00000000..e4952bd4 --- /dev/null +++ b/.github/workflows/weekly-release.yml @@ -0,0 +1,102 @@ +name: Weekly Release + +on: + schedule: + # Run every Monday at 9:00 UTC + - cron: '0 9 * * 1' + # Allow manual trigger for testing + workflow_dispatch: + +jobs: + prepare: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + last_release: ${{ steps.last-release.outputs.hash }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Find package directories + id: set-matrix + run: | + DIRS=$(git ls-tree -r HEAD --name-only | grep -E "package.json|pyproject.toml" | xargs dirname | grep -v "^.$" | jq -R -s -c 'split("\n")[:-1]') + echo "matrix=${DIRS}" >> $GITHUB_OUTPUT + + - name: Get last release hash + id: last-release + run: | + HASH=$(git rev-list --tags --max-count=1 || echo "HEAD~1") + echo "hash=${HASH}" >> $GITHUB_OUTPUT + + release: + needs: prepare + runs-on: ubuntu-latest + strategy: + matrix: + directory: ${{ fromJson(needs.prepare.outputs.matrix) }} + fail-fast: false + permissions: + contents: write + packages: write + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - uses: astral-sh/setup-uv@v5 + + - name: Setup Node.js + if: endsWith(matrix.directory, 'package.json') + uses: actions/setup-node@v4 + with: + node-version: '18' + registry-url: 'https://registry.npmjs.org' + + - name: Setup Python + if: endsWith(matrix.directory, 'pyproject.toml') + run: uv python install + + - name: Release package + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + UV_PUBLISH_TOKEN: ${{ secrets.PYPI_TOKEN }} + run: uv run --script scripts/release.py "${{ matrix.directory }}" "${{ needs.prepare.outputs.last_release }}" >> "$GITHUB_OUTPUT" + + create-release: + needs: [prepare, release] + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v4 + + - name: Create Release + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + # Check if there's output from release step + if [ -s "$GITHUB_OUTPUT" ]; then + DATE=$(date +%Y.%m.%d) + + # Create git tag + git tag -s -a -m"automated release v${DATE}" "v${DATE}" + git push origin "v${DATE}" + + # Create release notes + echo "# Release ${DATE}" > notes.md + echo "" >> notes.md + echo "## Updated Packages" >> notes.md + + # Read updated packages from github output + while IFS= read -r line; do + echo "- ${line}" >> notes.md + done < "$GITHUB_OUTPUT" + + # Create GitHub release + gh release create "v${DATE}" \ + --title "Release ${DATE}" \ + --notes-file notes.md + fi diff --git a/scripts/release.py b/scripts/release.py new file mode 100755 index 00000000..0c1eb4da --- /dev/null +++ b/scripts/release.py @@ -0,0 +1,179 @@ +#!/usr/bin/env uv run --script +# /// script +# requires-python = ">=3.11" +# dependencies = [ +# "click>=8.1.8", +# "tomlkit>=0.13.2" +# ] +# /// +import sys +import re +import click +from pathlib import Path +import json +import tomlkit +import datetime +import subprocess +from enum import Enum +from typing import Any, NewType + + +Version = NewType("Version", str) +GitHash = NewType("GitHash", str) + + +class GitHashParamType(click.ParamType): + name = "git_hash" + + def convert( + self, value: Any, param: click.Parameter | None, ctx: click.Context | None + ) -> GitHash | None: + if value is None: + return None + + if not (8 <= len(value) <= 40): + self.fail(f"Git hash must be between 8 and 40 characters, got {len(value)}") + + if not re.match(r"^[0-9a-fA-F]+$", value): + self.fail("Git hash must contain only hex digits (0-9, a-f)") + + try: + # Verify hash exists in repo + subprocess.run( + ["git", "rev-parse", "--verify", value], check=True, capture_output=True + ) + except subprocess.CalledProcessError: + self.fail(f"Git hash {value} not found in repository") + + return GitHash(value.lower()) + + +GIT_HASH = GitHashParamType() + + +class PackageType(Enum): + NPM = 1 + PYPI = 2 + + @classmethod + def from_path(cls, directory: Path) -> "PackageType": + if (directory / "package.json").exists(): + return cls.NPM + elif (directory / "pyproject.toml").exists(): + return cls.PYPI + else: + raise Exception("No package.json or pyproject.toml found") + + +def get_changes(path: Path, git_hash: str) -> bool: + """Check if any files changed between current state and git hash""" + try: + output = subprocess.run( + ["git", "diff", "--name-only", git_hash, "--", path], + cwd=path, + check=True, + capture_output=True, + text=True, + ) + + changed_files = [Path(f) for f in output.stdout.splitlines()] + relevant_files = [f for f in changed_files if f.suffix in ['.py', '.ts']] + return len(relevant_files) >= 1 + except subprocess.CalledProcessError: + return False + + +def get_package_name(path: Path, pkg_type: PackageType) -> str: + """Get package name from package.json or pyproject.toml""" + match pkg_type: + case PackageType.NPM: + with open(path / "package.json", "rb") as f: + return json.load(f)["name"] + case PackageType.PYPI: + with open(path / "pyproject.toml") as f: + toml_data = tomlkit.parse(f.read()) + name = toml_data.get("project", {}).get("name") + if not name: + raise Exception("No name in pyproject.toml project section") + return str(name) + + +def generate_version() -> Version: + """Generate version based on current date""" + now = datetime.datetime.now() + return Version(f"{now.year}.{now.month}.{now.day}") + + +def publish_package( + path: Path, pkg_type: PackageType, version: Version, dry_run: bool = False +): + """Publish package based on type""" + try: + match pkg_type: + case PackageType.NPM: + # Update version in package.json + with open(path / "package.json", "rb+") as f: + data = json.load(f) + data["version"] = version + f.seek(0) + json.dump(data, f, indent=2) + f.truncate() + + if not dry_run: + # Publish to npm + subprocess.run(["npm", "publish"], cwd=path, check=True) + case PackageType.PYPI: + # Update version in pyproject.toml + with open(path / "pyproject.toml") as f: + data = tomlkit.parse(f.read()) + data["project"]["version"] = version + + with open(path / "pyproject.toml", "w") as f: + f.write(tomlkit.dumps(data)) + + if not dry_run: + # Build and publish to PyPI + subprocess.run(["uv", "build"], cwd=path, check=True) + subprocess.run( + ["uv", "publish", "--username", "__token__"], + cwd=path, + check=True, + ) + except Exception as e: + raise Exception(f"Failed to publish: {e}") from e + + +@click.command() +@click.argument("directory", type=click.Path(exists=True, path_type=Path)) +@click.argument("git_hash", type=GIT_HASH) +@click.option( + "--dry-run", is_flag=True, help="Update version numbers but don't publish" +) +def main(directory: Path, git_hash: GitHash, dry_run: bool) -> int: + """Release package if changes detected""" + # Detect package type + try: + path = directory.resolve(strict=True) + pkg_type = PackageType.from_path(path) + except Exception as e: + return 1 + + # Check for changes + if not get_changes(path, git_hash): + return 0 + + try: + # Generate version and publish + version = generate_version() + name = get_package_name(path, pkg_type) + + publish_package(path, pkg_type, version, dry_run) + if not dry_run: + click.echo(f"{name}@{version}") + return 0 + except Exception as e: + return 1 + + +if __name__ == "__main__": + sys.exit(main()) From 6d36b5a1ff49b091845ec141048b6f5e45cfdc84 Mon Sep 17 00:00:00 2001 From: David Soria Parra Date: Mon, 13 Jan 2025 11:23:01 +0000 Subject: [PATCH 02/11] feat: add daily release check workflow --- .github/workflows/daily-release-check.yml | 28 +++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 .github/workflows/daily-release-check.yml diff --git a/.github/workflows/daily-release-check.yml b/.github/workflows/daily-release-check.yml new file mode 100644 index 00000000..135e76d7 --- /dev/null +++ b/.github/workflows/daily-release-check.yml @@ -0,0 +1,28 @@ +name: Daily Release Check + +on: + schedule: + - cron: '0 0 * * *' # Run at midnight UTC daily + workflow_dispatch: # Allow manual triggers + +jobs: + check-releases: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # Need full history for git log + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install dependencies + run: | + python -m pip install semver PyGithub rich toml click + + - name: Run release check + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: python scripts/weekly-release.py --dry-run \ No newline at end of file From 0989068ef1d62f1eb44eb00089001fb3511337ad Mon Sep 17 00:00:00 2001 From: David Soria Parra Date: Mon, 13 Jan 2025 11:34:12 +0000 Subject: [PATCH 03/11] feat: enhance release automation with daily checks --- .github/workflows/daily-release-check.yml | 66 +++++++++++++++++------ scripts/release.py | 2 + 2 files changed, 51 insertions(+), 17 deletions(-) diff --git a/.github/workflows/daily-release-check.yml b/.github/workflows/daily-release-check.yml index 135e76d7..128fd557 100644 --- a/.github/workflows/daily-release-check.yml +++ b/.github/workflows/daily-release-check.yml @@ -2,27 +2,59 @@ name: Daily Release Check on: schedule: - - cron: '0 0 * * *' # Run at midnight UTC daily - workflow_dispatch: # Allow manual triggers + # Run every day at 9:00 UTC + - cron: '0 9 * * *' + # Allow manual trigger for testing + workflow_dispatch: jobs: - check-releases: + prepare: runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + last_release: ${{ steps.last-release.outputs.hash }} steps: - uses: actions/checkout@v4 with: - fetch-depth: 0 # Need full history for git log - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.11' - - - name: Install dependencies + fetch-depth: 0 + + - name: Find package directories + id: set-matrix run: | - python -m pip install semver PyGithub rich toml click - - - name: Run release check - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: python scripts/weekly-release.py --dry-run \ No newline at end of file + DIRS=$(git ls-tree -r HEAD --name-only | grep -E "package.json|pyproject.toml" | xargs dirname | grep -v "^.$" | jq -R -s -c 'split("\n")[:-1]') + echo "matrix=${DIRS}" >> $GITHUB_OUTPUT + + - name: Get last release hash + id: last-release + run: | + HASH=$(git rev-list --tags --max-count=1 || echo "HEAD~1") + echo "hash=${HASH}" >> $GITHUB_OUTPUT + + check-release: + needs: prepare + runs-on: ubuntu-latest + strategy: + matrix: + directory: ${{ fromJson(needs.prepare.outputs.matrix) }} + fail-fast: false + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - uses: astral-sh/setup-uv@v5 + + - name: Setup Node.js + if: endsWith(matrix.directory, 'package.json') + uses: actions/setup-node@v4 + with: + node-version: '18' + + - name: Setup Python + if: endsWith(matrix.directory, 'pyproject.toml') + run: uv python install + + - name: Check release + run: | + uv run --script scripts/release.py --dry-run "${{ matrix.directory }}" "${{ needs.prepare.outputs.last_release }}" diff --git a/scripts/release.py b/scripts/release.py index 0c1eb4da..0853d36e 100755 --- a/scripts/release.py +++ b/scripts/release.py @@ -170,6 +170,8 @@ def main(directory: Path, git_hash: GitHash, dry_run: bool) -> int: publish_package(path, pkg_type, version, dry_run) if not dry_run: click.echo(f"{name}@{version}") + else: + click.echo(f"🔍 Dry run: Would have published {name}@{version} if this was a real release") return 0 except Exception as e: return 1 From 9db47b20e7a7f39be4d30f27c756a8a0cc433a42 Mon Sep 17 00:00:00 2001 From: David Soria Parra Date: Mon, 13 Jan 2025 11:35:46 +0000 Subject: [PATCH 04/11] feat: rename --- .../{daily-release-check.yml => release-check.yml} | 3 --- .github/workflows/{weekly-release.yml => release.yml} | 6 +++--- 2 files changed, 3 insertions(+), 6 deletions(-) rename .github/workflows/{daily-release-check.yml => release-check.yml} (95%) rename .github/workflows/{weekly-release.yml => release.yml} (97%) diff --git a/.github/workflows/daily-release-check.yml b/.github/workflows/release-check.yml similarity index 95% rename from .github/workflows/daily-release-check.yml rename to .github/workflows/release-check.yml index 128fd557..5350157b 100644 --- a/.github/workflows/daily-release-check.yml +++ b/.github/workflows/release-check.yml @@ -1,9 +1,6 @@ name: Daily Release Check on: - schedule: - # Run every day at 9:00 UTC - - cron: '0 9 * * *' # Allow manual trigger for testing workflow_dispatch: diff --git a/.github/workflows/weekly-release.yml b/.github/workflows/release.yml similarity index 97% rename from .github/workflows/weekly-release.yml rename to .github/workflows/release.yml index e4952bd4..252c89fd 100644 --- a/.github/workflows/weekly-release.yml +++ b/.github/workflows/release.yml @@ -1,9 +1,9 @@ -name: Weekly Release +name: Daily Release on: schedule: - # Run every Monday at 9:00 UTC - - cron: '0 9 * * 1' + # Run every day at 9:00 UTC + - cron: '0 9 * * *' # Allow manual trigger for testing workflow_dispatch: From b3f20decdc453f34c2f0733d60e2fcbed641763e Mon Sep 17 00:00:00 2001 From: David Soria Parra Date: Mon, 13 Jan 2025 11:38:25 +0000 Subject: [PATCH 05/11] feat: fix workflow names --- .github/workflows/release-check.yml | 2 +- .github/workflows/release.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release-check.yml b/.github/workflows/release-check.yml index 5350157b..30ff1f09 100644 --- a/.github/workflows/release-check.yml +++ b/.github/workflows/release-check.yml @@ -1,4 +1,4 @@ -name: Daily Release Check +name: Release Check on: # Allow manual trigger for testing diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 252c89fd..45f3aadd 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,4 +1,4 @@ -name: Daily Release +name: Release on: schedule: From c9b1adf3b6a690cd9f8faa046354eabe8da49481 Mon Sep 17 00:00:00 2001 From: David Soria Parra Date: Mon, 13 Jan 2025 11:47:41 +0000 Subject: [PATCH 06/11] use environments --- .github/workflows/release-check.yml | 25 ++++++++++++++++++++++++- .github/workflows/release.yml | 1 + 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release-check.yml b/.github/workflows/release-check.yml index 30ff1f09..d4fc9ed1 100644 --- a/.github/workflows/release-check.yml +++ b/.github/workflows/release-check.yml @@ -54,4 +54,27 @@ jobs: - name: Check release run: | - uv run --script scripts/release.py --dry-run "${{ matrix.directory }}" "${{ needs.prepare.outputs.last_release }}" + uv run --script scripts/release.py --dry-run "${{ matrix.directory }}" "${{ needs.prepare.outputs.last_release }}" | tee -a "$GITHUB_OUTPUT" + + check-tag: + needs: [prepare, check-release] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Simulate tag creation + run: | + if [ -s "$GITHUB_OUTPUT" ]; then + DATE=$(date +%Y.%m.%d) + echo "🔍 Dry run: Would create tag v${DATE} if this was a real release" + + echo "# Release ${DATE}" > notes.md + echo "" >> notes.md + echo "## Updated Packages" >> notes.md + while IFS= read -r line; do + echo "- ${line}" >> notes.md + done < "$GITHUB_OUTPUT" + + echo "🔍 Would create release with following notes:" + cat notes.md + fi diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 45f3aadd..d2d808b4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -33,6 +33,7 @@ jobs: release: needs: prepare runs-on: ubuntu-latest + environment: release strategy: matrix: directory: ${{ fromJson(needs.prepare.outputs.matrix) }} From 9f86ee599876c5540444bacd77cdc2ebe7dfb06e Mon Sep 17 00:00:00 2001 From: David Soria Parra Date: Mon, 13 Jan 2025 13:01:46 +0000 Subject: [PATCH 07/11] fix: fix scripts --- .github/workflows/release-check.yml | 18 ++++++++++++++---- .github/workflows/release.yml | 2 +- scripts/release.py | 2 +- 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/.github/workflows/release-check.yml b/.github/workflows/release-check.yml index d4fc9ed1..005f460c 100644 --- a/.github/workflows/release-check.yml +++ b/.github/workflows/release-check.yml @@ -30,6 +30,8 @@ jobs: check-release: needs: prepare runs-on: ubuntu-latest + outputs: + release: ${{ steps.check.outputs.release }} strategy: matrix: directory: ${{ fromJson(needs.prepare.outputs.matrix) }} @@ -53,8 +55,15 @@ jobs: run: uv python install - name: Check release + id: check run: | - uv run --script scripts/release.py --dry-run "${{ matrix.directory }}" "${{ needs.prepare.outputs.last_release }}" | tee -a "$GITHUB_OUTPUT" + output=$(uv run --script scripts/release.py --dry-run "${{ matrix.directory }}" "${{ needs.prepare.outputs.last_release }}" \ + | grep -o -E "[a-zA-Z0-9\-]+@[0-9]+\.[0-9]+\.[0-9]+" || true) + if [ ! -z "$output" ]; then + echo "release<> $GITHUB_OUTPUT + echo "$output" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + fi check-tag: needs: [prepare, check-release] @@ -64,7 +73,8 @@ jobs: - name: Simulate tag creation run: | - if [ -s "$GITHUB_OUTPUT" ]; then + echo "${{ needs.check-release.outputs.release }}" > packages.txt + if [ -s packages.txt ]; then DATE=$(date +%Y.%m.%d) echo "🔍 Dry run: Would create tag v${DATE} if this was a real release" @@ -72,8 +82,8 @@ jobs: echo "" >> notes.md echo "## Updated Packages" >> notes.md while IFS= read -r line; do - echo "- ${line}" >> notes.md - done < "$GITHUB_OUTPUT" + echo "- $line" >> notes.md + done < packages.txt echo "🔍 Would create release with following notes:" cat notes.md diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d2d808b4..d2d084de 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -64,7 +64,7 @@ jobs: env: NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} UV_PUBLISH_TOKEN: ${{ secrets.PYPI_TOKEN }} - run: uv run --script scripts/release.py "${{ matrix.directory }}" "${{ needs.prepare.outputs.last_release }}" >> "$GITHUB_OUTPUT" + run: uv run --script scripts/release.py "${{ matrix.directory }}" "${{ needs.prepare.outputs.last_release }}" >> "$GITHUB_OUTPUT" create-release: needs: [prepare, release] diff --git a/scripts/release.py b/scripts/release.py index 0853d36e..17329560 100755 --- a/scripts/release.py +++ b/scripts/release.py @@ -171,7 +171,7 @@ def main(directory: Path, git_hash: GitHash, dry_run: bool) -> int: if not dry_run: click.echo(f"{name}@{version}") else: - click.echo(f"🔍 Dry run: Would have published {name}@{version} if this was a real release") + click.echo(f"Dry run: Would have published {name}@{version}") return 0 except Exception as e: return 1 From 111806d6ae1cc80655cfef0d37ce5b7e0be35fed Mon Sep 17 00:00:00 2001 From: David Soria Parra Date: Mon, 13 Jan 2025 14:33:22 +0000 Subject: [PATCH 08/11] feat: use artifcats to collect mutliple job outputs into one A matrix job cant have multiple outputs directly, see https://github.com/orgs/community/discussions/17245. Use the artifact workaround. --- .github/workflows/release-check.yml | 113 ++++++++++++++++++++++------ 1 file changed, 91 insertions(+), 22 deletions(-) diff --git a/.github/workflows/release-check.yml b/.github/workflows/release-check.yml index 005f460c..63d7dfa0 100644 --- a/.github/workflows/release-check.yml +++ b/.github/workflows/release-check.yml @@ -18,20 +18,21 @@ jobs: - name: Find package directories id: set-matrix run: | + # Find all package.json and pyproject.toml files, excluding root DIRS=$(git ls-tree -r HEAD --name-only | grep -E "package.json|pyproject.toml" | xargs dirname | grep -v "^.$" | jq -R -s -c 'split("\n")[:-1]') echo "matrix=${DIRS}" >> $GITHUB_OUTPUT + echo "Found directories: ${DIRS}" - name: Get last release hash id: last-release run: | HASH=$(git rev-list --tags --max-count=1 || echo "HEAD~1") echo "hash=${HASH}" >> $GITHUB_OUTPUT + echo "Using last release hash: ${HASH}" check-release: needs: prepare runs-on: ubuntu-latest - outputs: - release: ${{ steps.check.outputs.release }} strategy: matrix: directory: ${{ fromJson(needs.prepare.outputs.matrix) }} @@ -45,46 +46,114 @@ jobs: - uses: astral-sh/setup-uv@v5 - name: Setup Node.js - if: endsWith(matrix.directory, 'package.json') + if: endsWith(matrix.directory, '/package.json') uses: actions/setup-node@v4 with: node-version: '18' - name: Setup Python - if: endsWith(matrix.directory, 'pyproject.toml') + if: endsWith(matrix.directory, '/pyproject.toml') run: uv python install - name: Check release id: check run: | - output=$(uv run --script scripts/release.py --dry-run "${{ matrix.directory }}" "${{ needs.prepare.outputs.last_release }}" \ - | grep -o -E "[a-zA-Z0-9\-]+@[0-9]+\.[0-9]+\.[0-9]+" || true) - if [ ! -z "$output" ]; then - echo "release<> $GITHUB_OUTPUT - echo "$output" >> $GITHUB_OUTPUT - echo "EOF" >> $GITHUB_OUTPUT + # Create unique hash for this directory + dir_hash=$(echo "${{ matrix.directory }}" | sha256sum | awk '{print $1}') + + # Run release check script with verbose output + echo "Running release check against last release: ${{ needs.prepare.outputs.last_release }}" + + # Run git diff first to show changes + echo "Changes since last release:" + git diff --name-only "${{ needs.prepare.outputs.last_release }}" -- "${{ matrix.directory }}" || true + + # Run the release check + output=$(uv run --script scripts/release.py --dry-run "${{ matrix.directory }}" "${{ needs.prepare.outputs.last_release }}" 2>&1) + exit_code=$? + + echo "Release check output (exit code: $exit_code):" + echo "$output" + + # Extract package info if successful + if [ $exit_code -eq 0 ]; then + pkg_info=$(echo "$output" | grep -o -E "[a-zA-Z0-9\-]+@[0-9]+\.[0-9]+\.[0-9]+" || true) + else + echo "Release check failed" + exit 1 fi + if [ ! -z "$pkg_info" ]; then + echo "Found package that needs release: $pkg_info" + + # Create outputs directory + mkdir -p ./outputs + + # Save both package info and full changes + echo "$pkg_info" > "./outputs/${dir_hash}_info" + echo "dir_hash=${dir_hash}" >> $GITHUB_OUTPUT + + # Log what we're saving + echo "Saved package info to ./outputs/${dir_hash}_info:" + cat "./outputs/${dir_hash}_info" + else + echo "No release needed for this package" + fi + + - name: Set artifact name + if: steps.check.outputs.dir_hash + id: artifact + run: | + # Replace forward slashes with dashes + SAFE_DIR=$(echo "${{ matrix.directory }}" | tr '/' '-') + echo "name=release-outputs-${SAFE_DIR}" >> $GITHUB_OUTPUT + + - uses: actions/upload-artifact@v4 + if: steps.check.outputs.dir_hash + with: + name: ${{ steps.artifact.outputs.name }} + path: ./outputs/${{ steps.check.outputs.dir_hash }}* + check-tag: needs: [prepare, check-release] runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + pattern: release-outputs-src-* + merge-multiple: true + path: outputs + - name: Simulate tag creation run: | - echo "${{ needs.check-release.outputs.release }}" > packages.txt - if [ -s packages.txt ]; then - DATE=$(date +%Y.%m.%d) - echo "🔍 Dry run: Would create tag v${DATE} if this was a real release" + if [ -d outputs ]; then + # Collect package info + find outputs -name "*_info" -exec cat {} \; > packages.txt - echo "# Release ${DATE}" > notes.md - echo "" >> notes.md - echo "## Updated Packages" >> notes.md - while IFS= read -r line; do - echo "- $line" >> notes.md - done < packages.txt + if [ -s packages.txt ]; then + DATE=$(date +%Y.%m.%d) + echo "🔍 Dry run: Would create tag v${DATE} if this was a real release" - echo "🔍 Would create release with following notes:" - cat notes.md + # Generate comprehensive release notes + { + echo "# Release ${DATE}" + echo "" + echo "## Updated Packages" + while IFS= read -r line; do + echo "- $line" + done < packages.txt + } > notes.md + + echo "🔍 Would create release with following notes:" + cat notes.md + + echo "🔍 Would create tag v${DATE} with the above release notes" + echo "🔍 Would create GitHub release from tag v${DATE}" + else + echo "No packages need release" + fi + else + echo "No release artifacts found" fi From 3d8c33cd9ac9b85458cdb58b628a747d7f8a3d31 Mon Sep 17 00:00:00 2001 From: David Soria Parra Date: Mon, 13 Jan 2025 20:01:18 +0000 Subject: [PATCH 09/11] feat: enhance release workflow with artifact collection and job output handling --- .github/workflows/release.yml | 119 +++++++++++++++++++++++++++------- 1 file changed, 95 insertions(+), 24 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d2d084de..588cacf5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -21,14 +21,17 @@ jobs: - name: Find package directories id: set-matrix run: | + # Find all package.json and pyproject.toml files, excluding root DIRS=$(git ls-tree -r HEAD --name-only | grep -E "package.json|pyproject.toml" | xargs dirname | grep -v "^.$" | jq -R -s -c 'split("\n")[:-1]') echo "matrix=${DIRS}" >> $GITHUB_OUTPUT + echo "Found directories: ${DIRS}" - name: Get last release hash id: last-release run: | HASH=$(git rev-list --tags --max-count=1 || echo "HEAD~1") echo "hash=${HASH}" >> $GITHUB_OUTPUT + echo "Using last release hash: ${HASH}" release: needs: prepare @@ -50,23 +53,76 @@ jobs: - uses: astral-sh/setup-uv@v5 - name: Setup Node.js - if: endsWith(matrix.directory, 'package.json') + if: endsWith(matrix.directory, '/package.json') uses: actions/setup-node@v4 with: node-version: '18' registry-url: 'https://registry.npmjs.org' - name: Setup Python - if: endsWith(matrix.directory, 'pyproject.toml') + if: endsWith(matrix.directory, '/pyproject.toml') run: uv python install - name: Release package + id: release env: NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} UV_PUBLISH_TOKEN: ${{ secrets.PYPI_TOKEN }} - run: uv run --script scripts/release.py "${{ matrix.directory }}" "${{ needs.prepare.outputs.last_release }}" >> "$GITHUB_OUTPUT" + run: | + # Create unique hash for this directory + dir_hash=$(echo "${{ matrix.directory }}" | sha256sum | awk '{print $1}') - create-release: + # Run git diff first to show changes + echo "Changes since last release:" + git diff --name-only "${{ needs.prepare.outputs.last_release }}" -- "${{ matrix.directory }}" || true + + # Run the release + output=$(uv run --script scripts/release.py "${{ matrix.directory }}" "${{ needs.prepare.outputs.last_release }}" 2>&1) + exit_code=$? + + echo "Release output (exit code: $exit_code):" + echo "$output" + + # Extract package info if successful + if [ $exit_code -eq 0 ]; then + pkg_info=$(echo "$output" | grep -o -E "[a-zA-Z0-9\-]+@[0-9]+\.[0-9]+\.[0-9]+" || true) + else + echo "Release failed" + exit 1 + fi + + if [ ! -z "$pkg_info" ]; then + echo "Released package: $pkg_info" + + # Create outputs directory + mkdir -p ./outputs + + # Save both package info and full changes + echo "$pkg_info" > "./outputs/${dir_hash}_info" + echo "dir_hash=${dir_hash}" >> $GITHUB_OUTPUT + + # Log what we're saving + echo "Saved package info to ./outputs/${dir_hash}_info:" + cat "./outputs/${dir_hash}_info" + else + echo "No release needed for this package" + fi + + - name: Set artifact name + if: steps.release.outputs.dir_hash + id: artifact + run: | + # Replace forward slashes with dashes + SAFE_DIR=$(echo "${{ matrix.directory }}" | tr '/' '-') + echo "name=release-outputs-${SAFE_DIR}" >> $GITHUB_OUTPUT + + - uses: actions/upload-artifact@v4 + if: steps.release.outputs.dir_hash + with: + name: ${{ steps.artifact.outputs.name }} + path: ./outputs/${{ steps.release.outputs.dir_hash }}* + + create-tag: needs: [prepare, release] runs-on: ubuntu-latest permissions: @@ -74,30 +130,45 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Create Release + - uses: actions/download-artifact@v4 + with: + pattern: release-outputs-src-* + merge-multiple: true + path: outputs + + - name: Create tag and release env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - # Check if there's output from release step - if [ -s "$GITHUB_OUTPUT" ]; then - DATE=$(date +%Y.%m.%d) + if [ -d outputs ]; then + # Collect package info + find outputs -name "*_info" -exec cat {} \; > packages.txt - # Create git tag - git tag -s -a -m"automated release v${DATE}" "v${DATE}" - git push origin "v${DATE}" + if [ -s packages.txt ]; then + DATE=$(date +%Y.%m.%d) + echo "Creating tag v${DATE}" - # Create release notes - echo "# Release ${DATE}" > notes.md - echo "" >> notes.md - echo "## Updated Packages" >> notes.md + # Generate comprehensive release notes + { + echo "# Release ${DATE}" + echo "" + echo "## Updated Packages" + while IFS= read -r line; do + echo "- $line" + done < packages.txt + } > notes.md - # Read updated packages from github output - while IFS= read -r line; do - echo "- ${line}" >> notes.md - done < "$GITHUB_OUTPUT" + # Create and push tag + git tag -a "v${DATE}" -m "Release ${DATE}" + git push origin "v${DATE}" - # Create GitHub release - gh release create "v${DATE}" \ - --title "Release ${DATE}" \ - --notes-file notes.md - fi + # Create GitHub release + gh release create "v${DATE}" \ + --title "Release ${DATE}" \ + --notes-file notes.md + else + echo "No packages need release" + fi + else + echo "No release artifacts found" + fi \ No newline at end of file From a5dbc1d3d3ae32cf4eda2972aa6d56105f37d022 Mon Sep 17 00:00:00 2001 From: David Soria Parra Date: Mon, 13 Jan 2025 20:07:29 +0000 Subject: [PATCH 10/11] fix: dont specify username --- scripts/release.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/release.py b/scripts/release.py index 17329560..ace528eb 100755 --- a/scripts/release.py +++ b/scripts/release.py @@ -135,7 +135,7 @@ def publish_package( # Build and publish to PyPI subprocess.run(["uv", "build"], cwd=path, check=True) subprocess.run( - ["uv", "publish", "--username", "__token__"], + ["uv", "publish"], cwd=path, check=True, ) From bedc11573e6561b7aabf103615abfbaee9728399 Mon Sep 17 00:00:00 2001 From: David Soria Parra Date: Mon, 13 Jan 2025 20:32:39 +0000 Subject: [PATCH 11/11] feat: improvements to release.yml --- .github/workflows/release.yml | 199 +++++++++++++--------------------- 1 file changed, 78 insertions(+), 121 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 588cacf5..4d135ad6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,31 +1,18 @@ -name: Release +name: Automatic Release Creation on: - schedule: - # Run every day at 9:00 UTC - - cron: '0 9 * * *' - # Allow manual trigger for testing workflow_dispatch: jobs: - prepare: + detect-last-release: runs-on: ubuntu-latest outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} last_release: ${{ steps.last-release.outputs.hash }} steps: - uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Find package directories - id: set-matrix - run: | - # Find all package.json and pyproject.toml files, excluding root - DIRS=$(git ls-tree -r HEAD --name-only | grep -E "package.json|pyproject.toml" | xargs dirname | grep -v "^.$" | jq -R -s -c 'split("\n")[:-1]') - echo "matrix=${DIRS}" >> $GITHUB_OUTPUT - echo "Found directories: ${DIRS}" - - name: Get last release hash id: last-release run: | @@ -33,142 +20,112 @@ jobs: echo "hash=${HASH}" >> $GITHUB_OUTPUT echo "Using last release hash: ${HASH}" - release: - needs: prepare + create-tag-name: runs-on: ubuntu-latest - environment: release - strategy: - matrix: - directory: ${{ fromJson(needs.prepare.outputs.matrix) }} - fail-fast: false - permissions: - contents: write - packages: write + outputs: + tag_name: ${{ steps.last-release.outputs.tag}} + steps: + - name: Get last release hash + id: last-release + run: | + DATE=$(date +%Y.%m.%d) + echo "tag=v${DATE}" >> $GITHUB_OUTPUT + echo "Using tag: v${DATE}" + detect-packages: + needs: [detect-last-release] + runs-on: ubuntu-latest + outputs: + packages: ${{ steps.find-packages.outputs.packages }} steps: - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: astral-sh/setup-uv@v5 + - name: Install uv + uses: astral-sh/setup-uv@v5 - - name: Setup Node.js - if: endsWith(matrix.directory, '/package.json') - uses: actions/setup-node@v4 - with: - node-version: '18' - registry-url: 'https://registry.npmjs.org' - - - name: Setup Python - if: endsWith(matrix.directory, '/pyproject.toml') - run: uv python install - - - name: Release package - id: release - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} - UV_PUBLISH_TOKEN: ${{ secrets.PYPI_TOKEN }} + - name: Find packages + id: find-packages + working-directory: src run: | - # Create unique hash for this directory - dir_hash=$(echo "${{ matrix.directory }}" | sha256sum | awk '{print $1}') + cat << 'EOF' > find_packages.py + import json + import os + import subprocess + from itertools import chain + from pathlib import Path - # Run git diff first to show changes - echo "Changes since last release:" - git diff --name-only "${{ needs.prepare.outputs.last_release }}" -- "${{ matrix.directory }}" || true + packages = [] - # Run the release - output=$(uv run --script scripts/release.py "${{ matrix.directory }}" "${{ needs.prepare.outputs.last_release }}" 2>&1) - exit_code=$? + print("Starting package detection...") + print(f"Using LAST_RELEASE: {os.environ['LAST_RELEASE']}") - echo "Release output (exit code: $exit_code):" - echo "$output" + # Find all directories containing package.json or pyproject.toml + paths = chain(Path('.').glob('*/package.json'), Path('.').glob('*/pyproject.toml')) + for path in paths: + print(f"\nChecking path: {path}") + # Check for changes in .py or .ts files + # Run git diff from the specific directory + cmd = ['git', 'diff', '--name-only', f'{os.environ["LAST_RELEASE"]}..HEAD', '--', '.'] + result = subprocess.run(cmd, capture_output=True, text=True, cwd=path.parent) - # Extract package info if successful - if [ $exit_code -eq 0 ]; then - pkg_info=$(echo "$output" | grep -o -E "[a-zA-Z0-9\-]+@[0-9]+\.[0-9]+\.[0-9]+" || true) - else - echo "Release failed" - exit 1 - fi + # Check if any .py or .ts files were changed + changed_files = result.stdout.strip().split('\n') + print(f"Changed files found: {changed_files}") - if [ ! -z "$pkg_info" ]; then - echo "Released package: $pkg_info" + has_changes = any(f.endswith(('.py', '.ts')) for f in changed_files if f) + if has_changes: + print(f"Adding package: {path.parent}") + packages.append(str(path.parent)) - # Create outputs directory - mkdir -p ./outputs + print(f"\nFinal packages list: {packages}") - # Save both package info and full changes - echo "$pkg_info" > "./outputs/${dir_hash}_info" - echo "dir_hash=${dir_hash}" >> $GITHUB_OUTPUT + # Write output + with open(os.environ['GITHUB_OUTPUT'], 'a') as f: + f.write(f"packages={json.dumps(packages)}\n") + EOF - # Log what we're saving - echo "Saved package info to ./outputs/${dir_hash}_info:" - cat "./outputs/${dir_hash}_info" - else - echo "No release needed for this package" - fi - - - name: Set artifact name - if: steps.release.outputs.dir_hash - id: artifact - run: | - # Replace forward slashes with dashes - SAFE_DIR=$(echo "${{ matrix.directory }}" | tr '/' '-') - echo "name=release-outputs-${SAFE_DIR}" >> $GITHUB_OUTPUT - - - uses: actions/upload-artifact@v4 - if: steps.release.outputs.dir_hash - with: - name: ${{ steps.artifact.outputs.name }} - path: ./outputs/${{ steps.release.outputs.dir_hash }}* + LAST_RELEASE=${{ needs.detect-last-release.outputs.last_release }} uv run --script --python 3.12 find_packages.py create-tag: - needs: [prepare, release] + needs: [detect-packages, create-tag-name] runs-on: ubuntu-latest permissions: contents: write steps: - uses: actions/checkout@v4 - - uses: actions/download-artifact@v4 - with: - pattern: release-outputs-src-* - merge-multiple: true - path: outputs - - - name: Create tag and release + - name: Create release env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - if [ -d outputs ]; then - # Collect package info - find outputs -name "*_info" -exec cat {} \; > packages.txt + # Configure git + git config --global user.name "GitHub Actions" + git config --global user.email "actions@github.com" - if [ -s packages.txt ]; then - DATE=$(date +%Y.%m.%d) - echo "Creating tag v${DATE}" + # Get packages array + PACKAGES='${{ needs.detect-packages.outputs.packages }}' - # Generate comprehensive release notes - { - echo "# Release ${DATE}" - echo "" - echo "## Updated Packages" - while IFS= read -r line; do - echo "- $line" - done < packages.txt - } > notes.md + if [ "$(echo "$PACKAGES" | jq 'length')" -gt 0 ]; then + # Generate comprehensive release notes + { + echo "# Release ${{ needs.create-tag-name.outputs.tag_name }}" + echo "" + echo "## Updated Packages" + echo "$PACKAGES" | jq -r '.[]' | while read -r package; do + echo "- $package" + done + } > notes.md - # Create and push tag - git tag -a "v${DATE}" -m "Release ${DATE}" - git push origin "v${DATE}" + # Create and push tag + git tag -a "${{ needs.create-tag-name.outputs.tag_name }}" -m "Release ${{ needs.create-tag-name.outputs.tag_name }}" + git push origin "${{ needs.create-tag-name.outputs.tag_name }}" - # Create GitHub release - gh release create "v${DATE}" \ - --title "Release ${DATE}" \ - --notes-file notes.md - else - echo "No packages need release" - fi + # Create GitHub release + gh release create "${{ needs.create-tag-name.outputs.tag_name }}" \ + --title "Release ${{ needs.create-tag-name.outputs.tag_name }}" \ + --notes-file notes.md else - echo "No release artifacts found" - fi \ No newline at end of file + echo "No packages need release" + fi