diff --git a/.github/workflows/build-citus-community-nightlies.yml b/.github/workflows/build-citus-community-nightlies.yml index e4773508..6e7820d0 100644 --- a/.github/workflows/build-citus-community-nightlies.yml +++ b/.github/workflows/build-citus-community-nightlies.yml @@ -33,14 +33,14 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v5 with: fetch-depth: 1 path: tools # This step is to fetch the images unanonymously to have higher bandwidth - name: Login to Docker Hub - uses: docker/login-action@v2 + uses: docker/login-action@v4 with: username: ${{ secrets.DOCKERHUB_USER_NAME }} password: ${{ secrets.DOCKERHUB_PASSWORD }} diff --git a/.github/workflows/citus-package-all-platforms-test.yml b/.github/workflows/citus-package-all-platforms-test.yml index a7692477..8ce8811d 100644 --- a/.github/workflows/citus-package-all-platforms-test.yml +++ b/.github/workflows/citus-package-all-platforms-test.yml @@ -1,8 +1,6 @@ name: Citus package all platforms tests env: - GH_TOKEN: ${{ secrets.GH_TOKEN }} - GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} PACKAGING_PASSPHRASE: ${{ secrets.PACKAGING_PASSPHRASE }} MICROSOFT_EMAIL: gindibay@microsoft.com USER_NAME: Gurkan Indibay @@ -36,8 +34,21 @@ jobs: PLATFORM: ${{ matrix.platform }} steps: + - name: Generate GitHub App token + id: app-token + uses: actions/create-github-app-token@v3 + with: + app-id: ${{ vars.GH_APP_ID }} + private-key: ${{ secrets.GH_APP_KEY }} + owner: citusdata + + - name: Export GitHub App token to environment + run: | + echo "GH_TOKEN=${{ steps.app-token.outputs.token }}" >> "$GITHUB_ENV" + echo "GITHUB_TOKEN=${{ steps.app-token.outputs.token }}" >> "$GITHUB_ENV" + - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v5 - name: Install dependencies run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 935cfd78..8596368d 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -25,16 +25,16 @@ jobs: # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v5 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v2 + uses: github/codeql-action/init@v3 with: languages: ${{ matrix.language }} - name: Autobuild - uses: github/codeql-action/autobuild@v2 + uses: github/codeql-action/autobuild@v3 - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 + uses: github/codeql-action/analyze@v3 diff --git a/.github/workflows/delete-packagecloud-packages.yml b/.github/workflows/delete-packagecloud-packages.yml index c9d55c53..ec0b6aa8 100644 --- a/.github/workflows/delete-packagecloud-packages.yml +++ b/.github/workflows/delete-packagecloud-packages.yml @@ -20,7 +20,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v5 - name: Install dependencies run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources diff --git a/.github/workflows/package-cloud-download-schedule.yml b/.github/workflows/package-cloud-download-schedule.yml index ab30e4ff..445bb89f 100644 --- a/.github/workflows/package-cloud-download-schedule.yml +++ b/.github/workflows/package-cloud-download-schedule.yml @@ -25,7 +25,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v5 - name: Install package dependencies run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources diff --git a/.github/workflows/package-tests.yml b/.github/workflows/package-tests.yml index ac160570..818c0d60 100644 --- a/.github/workflows/package-tests.yml +++ b/.github/workflows/package-tests.yml @@ -19,7 +19,7 @@ jobs: citus_version: ${{ steps.get-citus-version.outputs.citus_version }} steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v5 with: fetch-depth: 2 - name: Package version @@ -61,7 +61,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v5 - name: Install dependencies run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources diff --git a/.github/workflows/packaging-methods-tests.yml b/.github/workflows/packaging-methods-tests.yml index 7d53056f..11fcc14a 100644 --- a/.github/workflows/packaging-methods-tests.yml +++ b/.github/workflows/packaging-methods-tests.yml @@ -1,8 +1,5 @@ name: Packaging helper methods tests -env: - GH_TOKEN: ${{ secrets.GH_TOKEN }} - on: push: branches: @@ -15,8 +12,19 @@ jobs: runs-on: ubuntu-latest steps: + - name: Generate GitHub App token + id: app-token + uses: actions/create-github-app-token@v3 + with: + app-id: ${{ vars.GH_APP_ID }} + private-key: ${{ secrets.GH_APP_KEY }} + owner: citusdata + + - name: Export GitHub App token to environment + run: echo "GH_TOKEN=${{ steps.app-token.outputs.token }}" >> "$GITHUB_ENV" + - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v5 - name: Install package dependencies run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources diff --git a/.github/workflows/publish-docker-image-tests.yml b/.github/workflows/publish-docker-image-tests.yml index c14bc3d7..101ad201 100644 --- a/.github/workflows/publish-docker-image-tests.yml +++ b/.github/workflows/publish-docker-image-tests.yml @@ -13,7 +13,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v5 - name: Install package dependencies run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources diff --git a/.github/workflows/pypi-statistics-schedule.yml b/.github/workflows/pypi-statistics-schedule.yml index a8562ddc..6e10442a 100644 --- a/.github/workflows/pypi-statistics-schedule.yml +++ b/.github/workflows/pypi-statistics-schedule.yml @@ -20,7 +20,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v5 - name: Install package dependencies run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources diff --git a/.github/workflows/statistic-schedule.yml b/.github/workflows/statistic-schedule.yml index 3dcb4b22..d987383a 100644 --- a/.github/workflows/statistic-schedule.yml +++ b/.github/workflows/statistic-schedule.yml @@ -5,7 +5,6 @@ env: DB_PASSWORD: ${{ secrets.STATS_DB_PASSWORD }} DB_HOST_AND_PORT: ${{ secrets.STATS_DB_HOST_AND_PORT }} DB_NAME: ${{ secrets.STATS_DB_NAME }} - GH_TOKEN: ${{ secrets.GH_TOKEN }} on: schedule: - cron: "0 16 * * *" @@ -25,8 +24,19 @@ jobs: job_name: [docker_pull_citus, github_clone_citus, homebrew_citus] steps: + - name: Generate GitHub App token + id: app-token + uses: actions/create-github-app-token@v3 + with: + app-id: ${{ vars.GH_APP_ID }} + private-key: ${{ secrets.GH_APP_KEY }} + owner: citusdata + + - name: Export GitHub App token to environment + run: echo "GH_TOKEN=${{ steps.app-token.outputs.token }}" >> "$GITHUB_ENV" + - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v5 - name: Install package dependencies run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources diff --git a/.github/workflows/statistic-tests.yml b/.github/workflows/statistic-tests.yml index 5b9eb9ea..2f8c136a 100644 --- a/.github/workflows/statistic-tests.yml +++ b/.github/workflows/statistic-tests.yml @@ -5,7 +5,6 @@ env: DB_PASSWORD: ${{ secrets.STATS_DB_PASSWORD }} DB_HOST_AND_PORT: ${{ secrets.STATS_DB_HOST_AND_PORT }} DB_NAME: ${{ secrets.STATS_DB_NAME }} - GH_TOKEN: ${{ secrets.GH_TOKEN }} PACKAGE_CLOUD_API_TOKEN: ${{ secrets.PACKAGE_CLOUD_API_TOKEN }} PACKAGE_CLOUD_ADMIN_API_TOKEN: ${{ secrets.PACKAGE_CLOUD_ADMIN_API_TOKEN }} on: @@ -21,8 +20,19 @@ jobs: runs-on: ubuntu-latest steps: + - name: Generate GitHub App token + id: app-token + uses: actions/create-github-app-token@v3 + with: + app-id: ${{ vars.GH_APP_ID }} + private-key: ${{ secrets.GH_APP_KEY }} + owner: citusdata + + - name: Export GitHub App token to environment + run: echo "GH_TOKEN=${{ steps.app-token.outputs.token }}" >> "$GITHUB_ENV" + - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v5 - name: Install package dependencies run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources diff --git a/.github/workflows/tool-tests.yml b/.github/workflows/tool-tests.yml index 8759a7d5..a12e965f 100644 --- a/.github/workflows/tool-tests.yml +++ b/.github/workflows/tool-tests.yml @@ -1,7 +1,6 @@ name: Tool Tests env: - GH_TOKEN: ${{ secrets.GH_TOKEN }} MICROSOFT_EMAIL: gindibay@microsoft.com USER_NAME: Gurkan Indibay MAIN_BRANCH: all-citus @@ -19,7 +18,7 @@ jobs: steps: - name: Check out repository - uses: actions/checkout@v3 + uses: actions/checkout@v5 - name: Install all scripts run: make && sudo make install @@ -27,13 +26,24 @@ jobs: runs-on: ubuntu-latest steps: + - name: Generate GitHub App token + id: app-token + uses: actions/create-github-app-token@v3 + with: + app-id: ${{ vars.GH_APP_ID }} + private-key: ${{ secrets.GH_APP_KEY }} + owner: citusdata + + - name: Export GitHub App token to environment + run: echo "GH_TOKEN=${{ steps.app-token.outputs.token }}" >> "$GITHUB_ENV" + - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v5 with: fetch-depth: 0 - name: Set up Python 3.10 - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3.10" diff --git a/packaging_automation/citus_package.py b/packaging_automation/citus_package.py index 5acea6f9..59fee1c9 100644 --- a/packaging_automation/citus_package.py +++ b/packaging_automation/citus_package.py @@ -5,6 +5,7 @@ from enum import Enum from typing import Dict from typing import List +from typing import Optional from typing import Tuple import docker @@ -398,7 +399,7 @@ def get_docker_image_name(platform: str): @validate_parameters # disabled since this is related to parameter_validations library methods # pylint: disable=no-value-for-parameter -# pylint: disable= too-many-locals +# pylint: disable= too-many-locals, too-many-arguments def build_packages( github_token: non_empty(non_blank(str)), platform: non_empty(non_blank(str)), @@ -406,6 +407,7 @@ def build_packages( signing_credentials: SigningCredentials, input_output_parameters: InputOutputParameters, is_test: bool = False, + postgres_version: Optional[str] = None, ) -> None: os_name, os_version = decode_os_and_release(platform) release_versions, nightly_versions = get_postgres_versions( @@ -437,6 +439,13 @@ def build_packages( postgres_docker_extension_iterator = ["all"] else: postgres_docker_extension_iterator = postgress_versions_to_process + if postgres_version: + if postgres_version not in postgress_versions_to_process: + raise ValueError( + f"Requested postgres_version '{postgres_version}' is not in the " + f"{build_type.name} versions for '{platform}': {postgress_versions_to_process}" + ) + postgres_docker_extension_iterator = [postgres_version] docker_image_name = get_docker_image_name(platform) output_sub_folder = get_release_package_folder_name(os_name, os_version) diff --git a/packaging_automation/tests/test_citus_package.py b/packaging_automation/tests/test_citus_package.py index 7bf0a173..1dcf5434 100644 --- a/packaging_automation/tests/test_citus_package.py +++ b/packaging_automation/tests/test_citus_package.py @@ -2,6 +2,7 @@ import os import pathlib2 +import pytest from dotenv import dotenv_values from .test_utils import generate_new_gpg_key @@ -9,12 +10,14 @@ POSTGRES_VERSION_FILE, BuildType, InputOutputParameters, + PostgresVersionDockerImageType, SigningCredentials, build_packages, decode_os_and_release, get_build_platform, get_release_package_folder_name, get_postgres_versions, + platform_postgres_version_source, ) from ..common_tool_methods import ( define_rpm_public_key_to_machine, @@ -73,6 +76,7 @@ PLATFORM = get_build_platform( os.getenv("PLATFORM"), os.getenv("PACKAGING_IMAGE_PLATFORM") ) +POSTGRES_VERSION = os.getenv("POSTGRES_VERSION") PACKAGING_BRANCH_NAME = os.getenv("PACKAGING_BRANCH_NAME", "all-citus-unit-tests") @@ -105,6 +109,32 @@ def teardown_module(): def test_build_packages(): + # postgres_version only narrows the build for "multiple"-image platforms (rpm distros), + # where build_packages iterates one docker image per pg version. For "single"-image platforms + # (debian/ubuntu/pgxn) the iterator is always ["all"], so postgres_version is a no-op and every + # release version is built regardless of what is passed. Gate the filter-aware branches below on + # that distinction so the test's skip/count logic always matches what build_packages actually + # does, no matter what the external packaging matrix passes for a single-image platform. + os_name, _ = decode_os_and_release(PLATFORM) + version_filter_active = bool(POSTGRES_VERSION) and ( + platform_postgres_version_source[os_name] + == PostgresVersionDockerImageType.multiple + ) + # The packaging per-pg CI matrix enumerates pg{14..18} per rpm distro to drive + # update_image into building every {os}-pg{N} base image, but the rpm release set is only + # a subset (e.g. [15,16,17]). When POSTGRES_VERSION targets a version outside that set, there + # is nothing for this test to build/sign, so skip gracefully (skip == success) rather than + # letting build_packages raise. The image for that pg was still built by update_image, so + # push_images downstream still reseeds it. + if version_filter_active: + release_versions, _ = get_postgres_versions( + platform=PLATFORM, input_files_dir=PACKAGING_EXEC_FOLDER + ) + if POSTGRES_VERSION not in release_versions: + pytest.skip( + f"pg{POSTGRES_VERSION} not in release set {release_versions} for {PLATFORM}" + ) + delete_all_gpg_keys_by_name(TEST_GPG_KEY_NAME) delete_rpm_key_by_name(TEST_GPG_KEY_NAME) generate_new_gpg_key( @@ -130,9 +160,10 @@ def test_build_packages(): signing_credentials, input_output_parameters, is_test=True, + postgres_version=POSTGRES_VERSION, ) verify_rpm_signature_in_dir(BASE_OUTPUT_FOLDER) - os_name, os_version = decode_os_and_release(PLATFORM) + _, os_version = decode_os_and_release(PLATFORM) sub_folder = get_release_package_folder_name(os_name, os_version) release_output_folder = f"{BASE_OUTPUT_FOLDER}/{sub_folder}" # Regression guard for the sign_packages path-doubling bug: the build output folder and the @@ -151,9 +182,19 @@ def test_build_packages(): postgres_version_file_path = f"{PACKAGING_EXEC_FOLDER}/{POSTGRES_VERSION_FILE}" if PLATFORM != "pgxn": - assert len(os.listdir(release_output_folder)) == get_required_package_count( - input_files_dir=PACKAGING_EXEC_FOLDER, platform=PLATFORM - ) + # When POSTGRES_VERSION restricts the build to a single in-set version (multiple-image + # platforms only), only that version's packages are produced, so the expected count is the + # per-version package count (single_postgres_package_counts), not the full + # len(release_versions) * per-version count. When the filter is inactive — POSTGRES_VERSION + # empty/None, or a single-image platform where it is a no-op — keep the all-versions + # expectation. + if version_filter_active: + expected_package_count = single_postgres_package_counts[PLATFORM] + else: + expected_package_count = get_required_package_count( + input_files_dir=PACKAGING_EXEC_FOLDER, platform=PLATFORM + ) + assert len(os.listdir(release_output_folder)) == expected_package_count assert os.path.exists(postgres_version_file_path) config = dotenv_values(postgres_version_file_path) assert config["release_versions"] == "15,16,17" diff --git a/packaging_automation/tests/test_common_tool_methods.py b/packaging_automation/tests/test_common_tool_methods.py index 2cf7b98f..774857f3 100644 --- a/packaging_automation/tests/test_common_tool_methods.py +++ b/packaging_automation/tests/test_common_tool_methods.py @@ -4,10 +4,10 @@ from shutil import copyfile from datetime import timezone +from unittest.mock import MagicMock import pathlib2 -from github import Github from .test_utils import generate_new_gpg_key from ..common_tool_methods import ( @@ -46,7 +46,6 @@ str_array_to_str, ) -GITHUB_TOKEN = os.getenv("GH_TOKEN") BASE_PATH = pathlib2.Path(__file__).parents[1] TEST_BASE_PATH = pathlib2.Path(__file__).parent.absolute() TEST_GPG_KEY_NAME = "Citus Data " @@ -262,28 +261,66 @@ def test_prepend_line_in_file(): os.remove(test_file) +def _utc_date(date_str: str) -> datetime: + return datetime.strptime(date_str, "%Y.%m.%d").replace(tzinfo=timezone.utc) + + +def _mock_pull_request(number, merged_at, label_names=()): + pull_request = MagicMock() + pull_request.number = number + pull_request.merged_at = merged_at + labels = [] + for label_name in label_names: + label = MagicMock() + label.name = label_name + labels.append(label) + pull_request.labels = labels + return pull_request + + def test_getprs(): - # created at is not seen on Github. Should be checked on API result - g = Github(GITHUB_TOKEN) - repository = g.get_repo("citusdata/citus") + # get_prs_for_patch_release paginates every closed PR of the base repo through + # the live GitHub API. The hardcoded 2021 window forces walking citus's entire + # PR history, which intermittently trips a PyGithub redirect-loop error on deep + # pages. Mock the repository so the date-window filtering and merge-date sort are + # exercised deterministically and offline. + repository = MagicMock() + repository.get_pulls.return_value = [ + _mock_pull_request(4748, _utc_date("2021.02.26")), + _mock_pull_request(4750, _utc_date("2021.02.27")), + _mock_pull_request(4753, _utc_date("2021.02.28")), + _mock_pull_request(4760, _utc_date("2021.03.01")), + _mock_pull_request(4762, _utc_date("2021.03.01")), + _mock_pull_request(4769, _utc_date("2021.03.02")), + _mock_pull_request(4700, _utc_date("2021.02.20")), # before window + _mock_pull_request(4799, _utc_date("2021.03.10")), # after window + _mock_pull_request(4800, None), # not merged + ] prs = get_prs_for_patch_release( repository, - datetime.strptime("2021.02.26", "%Y.%m.%d").replace(tzinfo=timezone.utc), + _utc_date("2021.02.26"), "master", - datetime.strptime("2021.03.02", "%Y.%m.%d").replace(tzinfo=timezone.utc), + _utc_date("2021.03.02"), ) assert len(prs) == 6 assert prs[0].number == 4748 def test_getprs_with_backlog_label(): - g = Github(GITHUB_TOKEN) - repository = g.get_repo("citusdata/citus") + repository = MagicMock() + repository.get_pulls.return_value = [ + _mock_pull_request(4746, _utc_date("2021.02.25"), label_names=("backport",)), + _mock_pull_request(4740, _utc_date("2021.02.21"), label_names=("bug",)), + _mock_pull_request(4744, _utc_date("2021.02.24")), + _mock_pull_request(4730, _utc_date("2021.02.10"), label_names=("backport",)), + _mock_pull_request(4790, _utc_date("2021.03.05"), label_names=("backport",)), + _mock_pull_request(4795, None, label_names=("backport",)), + ] prs = get_prs_for_patch_release( repository, - datetime.strptime("2021.02.20", "%Y.%m.%d").replace(tzinfo=timezone.utc), + _utc_date("2021.02.20"), "master", - datetime.strptime("2021.02.27", "%Y.%m.%d").replace(tzinfo=timezone.utc), + _utc_date("2021.02.27"), ) prs_backlog = filter_prs_by_label(prs, "backport") assert len(prs_backlog) == 1