Tools GH App token migration (phase 1) + per-pg build_packages filter

.github/workflows/build-citus-community-nightlies.yml

@@ -33,14 +33,14 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v5
         with:
           fetch-depth: 1
           path: tools
 
       # This step is to fetch the images unanonymously to have higher bandwidth
       - name: Login to Docker Hub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v4
         with:
           username: ${{ secrets.DOCKERHUB_USER_NAME }}
           password: ${{ secrets.DOCKERHUB_PASSWORD }}

.github/workflows/citus-package-all-platforms-test.yml

@@ -1,8 +1,6 @@
 name: Citus package all platforms tests
 
 env:
-  GH_TOKEN: ${{ secrets.GH_TOKEN }}
-  GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
   PACKAGING_PASSPHRASE: ${{ secrets.PACKAGING_PASSPHRASE }}
   MICROSOFT_EMAIL: gindibay@microsoft.com
   USER_NAME: Gurkan Indibay
@@ -36,8 +34,21 @@ jobs:
       PLATFORM: ${{ matrix.platform }}
 
     steps:
+      - name: Generate GitHub App token
+        id: app-token
+        uses: actions/create-github-app-token@v3
+        with:
+          app-id: ${{ vars.GH_APP_ID }}
+          private-key: ${{ secrets.GH_APP_KEY }}
+          owner: citusdata
+
+      - name: Export GitHub App token to environment
+        run: |
+          echo "GH_TOKEN=${{ steps.app-token.outputs.token }}" >> "$GITHUB_ENV"
+          echo "GITHUB_TOKEN=${{ steps.app-token.outputs.token }}" >> "$GITHUB_ENV"
+
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v5
 
       - name: Install dependencies
         run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources

.github/workflows/codeql-analysis.yml

@@ -25,16 +25,16 @@ jobs:
         # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v5
 
       # Initializes the CodeQL tools for scanning.
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@v2
+        uses: github/codeql-action/init@v3
         with:
           languages: ${{ matrix.language }}
 
       - name: Autobuild
-        uses: github/codeql-action/autobuild@v2
+        uses: github/codeql-action/autobuild@v3
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v2
+        uses: github/codeql-action/analyze@v3

.github/workflows/delete-packagecloud-packages.yml

@@ -20,7 +20,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v5
 
       - name: Install dependencies
         run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources

.github/workflows/package-cloud-download-schedule.yml

@@ -25,7 +25,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v5
 
       - name: Install package dependencies
         run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources

.github/workflows/package-tests.yml

@@ -19,7 +19,7 @@ jobs:
       citus_version: ${{ steps.get-citus-version.outputs.citus_version }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v5
         with:
           fetch-depth: 2
       - name: Package version
@@ -61,7 +61,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v5
 
       - name: Install dependencies
         run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources

.github/workflows/packaging-methods-tests.yml

@@ -1,8 +1,5 @@
 name: Packaging helper methods tests
 
-env:
-  GH_TOKEN: ${{ secrets.GH_TOKEN }}
-
 on:
   push:
     branches:
@@ -15,8 +12,19 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
+      - name: Generate GitHub App token
+        id: app-token
+        uses: actions/create-github-app-token@v3
+        with:
+          app-id: ${{ vars.GH_APP_ID }}
+          private-key: ${{ secrets.GH_APP_KEY }}
+          owner: citusdata
+
+      - name: Export GitHub App token to environment
+        run: echo "GH_TOKEN=${{ steps.app-token.outputs.token }}" >> "$GITHUB_ENV"
+
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v5
 
       - name: Install package dependencies
         run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources

.github/workflows/publish-docker-image-tests.yml

@@ -13,7 +13,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v5
 
       - name: Install package dependencies
         run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources

.github/workflows/pypi-statistics-schedule.yml

@@ -20,7 +20,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v5
 
       - name: Install package dependencies
         run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources

.github/workflows/statistic-schedule.yml

@@ -5,7 +5,6 @@ env:
   DB_PASSWORD: ${{ secrets.STATS_DB_PASSWORD }}
   DB_HOST_AND_PORT: ${{ secrets.STATS_DB_HOST_AND_PORT }}
   DB_NAME: ${{ secrets.STATS_DB_NAME }}
-  GH_TOKEN: ${{ secrets.GH_TOKEN }}
 on:
   schedule:
     - cron: "0 16 * * *"
@@ -25,8 +24,19 @@ jobs:
         job_name: [docker_pull_citus, github_clone_citus, homebrew_citus]
 
     steps:
+      - name: Generate GitHub App token
+        id: app-token
+        uses: actions/create-github-app-token@v3
+        with:
+          app-id: ${{ vars.GH_APP_ID }}
+          private-key: ${{ secrets.GH_APP_KEY }}
+          owner: citusdata
+
+      - name: Export GitHub App token to environment
+        run: echo "GH_TOKEN=${{ steps.app-token.outputs.token }}" >> "$GITHUB_ENV"
+
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v5
 
       - name: Install package dependencies
         run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources

.github/workflows/statistic-tests.yml

@@ -5,7 +5,6 @@ env:
   DB_PASSWORD: ${{ secrets.STATS_DB_PASSWORD }}
   DB_HOST_AND_PORT: ${{ secrets.STATS_DB_HOST_AND_PORT }}
   DB_NAME: ${{ secrets.STATS_DB_NAME }}
-  GH_TOKEN: ${{ secrets.GH_TOKEN }}
   PACKAGE_CLOUD_API_TOKEN: ${{ secrets.PACKAGE_CLOUD_API_TOKEN }}
   PACKAGE_CLOUD_ADMIN_API_TOKEN: ${{ secrets.PACKAGE_CLOUD_ADMIN_API_TOKEN }}
 on:
@@ -21,8 +20,19 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
+      - name: Generate GitHub App token
+        id: app-token
+        uses: actions/create-github-app-token@v3
+        with:
+          app-id: ${{ vars.GH_APP_ID }}
+          private-key: ${{ secrets.GH_APP_KEY }}
+          owner: citusdata
+
+      - name: Export GitHub App token to environment
+        run: echo "GH_TOKEN=${{ steps.app-token.outputs.token }}" >> "$GITHUB_ENV"
+
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v5
 
       - name: Install package dependencies
         run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources

.github/workflows/tool-tests.yml

@@ -1,7 +1,6 @@
 name: Tool Tests
 
 env:
-  GH_TOKEN: ${{ secrets.GH_TOKEN }}
   MICROSOFT_EMAIL: gindibay@microsoft.com
   USER_NAME: Gurkan Indibay
   MAIN_BRANCH: all-citus
@@ -19,21 +18,32 @@ jobs:
 
     steps:
       - name: Check out repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v5
       - name: Install all scripts
         run: make && sudo make install
 
   unit_test_execution:
     runs-on: ubuntu-latest
 
     steps:
+      - name: Generate GitHub App token
+        id: app-token
+        uses: actions/create-github-app-token@v3
+        with:
+          app-id: ${{ vars.GH_APP_ID }}
+          private-key: ${{ secrets.GH_APP_KEY }}
+          owner: citusdata
+
+      - name: Export GitHub App token to environment
+        run: echo "GH_TOKEN=${{ steps.app-token.outputs.token }}" >> "$GITHUB_ENV"
+
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v5
         with:
           fetch-depth: 0
 
       - name: Set up Python 3.10
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: "3.10"
 

packaging_automation/citus_package.py

@@ -5,6 +5,7 @@
 from enum import Enum
 from typing import Dict
 from typing import List
+from typing import Optional
 from typing import Tuple
 
 import docker
@@ -398,14 +399,15 @@ def get_docker_image_name(platform: str):
 @validate_parameters
 # disabled since this is related to parameter_validations library methods
 # pylint: disable=no-value-for-parameter
-# pylint: disable= too-many-locals
+# pylint: disable= too-many-locals, too-many-arguments
 def build_packages(
     github_token: non_empty(non_blank(str)),
     platform: non_empty(non_blank(str)),
     build_type: BuildType,
     signing_credentials: SigningCredentials,
     input_output_parameters: InputOutputParameters,
     is_test: bool = False,
+    postgres_version: Optional[str] = None,
 ) -> None:
     os_name, os_version = decode_os_and_release(platform)
     release_versions, nightly_versions = get_postgres_versions(
@@ -437,6 +439,13 @@ def build_packages(
         postgres_docker_extension_iterator = ["all"]
     else:
         postgres_docker_extension_iterator = postgress_versions_to_process
+        if postgres_version:
+            if postgres_version not in postgress_versions_to_process:
+                raise ValueError(
+                    f"Requested postgres_version '{postgres_version}' is not in the "
+                    f"{build_type.name} versions for '{platform}': {postgress_versions_to_process}"
+                )
+            postgres_docker_extension_iterator = [postgres_version]
 
     docker_image_name = get_docker_image_name(platform)
     output_sub_folder = get_release_package_folder_name(os_name, os_version)

packaging_automation/tests/test_citus_package.py

@@ -2,19 +2,22 @@
 import os
 
 import pathlib2
+import pytest
 from dotenv import dotenv_values
 
 from .test_utils import generate_new_gpg_key
 from ..citus_package import (
     POSTGRES_VERSION_FILE,
     BuildType,
     InputOutputParameters,
+    PostgresVersionDockerImageType,
     SigningCredentials,
     build_packages,
     decode_os_and_release,
     get_build_platform,
     get_release_package_folder_name,
     get_postgres_versions,
+    platform_postgres_version_source,
 )
 from ..common_tool_methods import (
     define_rpm_public_key_to_machine,
@@ -73,6 +76,7 @@
 PLATFORM = get_build_platform(
     os.getenv("PLATFORM"), os.getenv("PACKAGING_IMAGE_PLATFORM")
 )
+POSTGRES_VERSION = os.getenv("POSTGRES_VERSION")
 PACKAGING_BRANCH_NAME = os.getenv("PACKAGING_BRANCH_NAME", "all-citus-unit-tests")
 
 
@@ -105,6 +109,32 @@ def teardown_module():
 
 
 def test_build_packages():
+    # postgres_version only narrows the build for "multiple"-image platforms (rpm distros),
+    # where build_packages iterates one docker image per pg version. For "single"-image platforms
+    # (debian/ubuntu/pgxn) the iterator is always ["all"], so postgres_version is a no-op and every
+    # release version is built regardless of what is passed. Gate the filter-aware branches below on
+    # that distinction so the test's skip/count logic always matches what build_packages actually
+    # does, no matter what the external packaging matrix passes for a single-image platform.
+    os_name, _ = decode_os_and_release(PLATFORM)
+    version_filter_active = bool(POSTGRES_VERSION) and (
+        platform_postgres_version_source[os_name]
+        == PostgresVersionDockerImageType.multiple
+    )
+    # The packaging per-pg CI matrix enumerates pg{14..18} per rpm distro to drive
+    # update_image into building every {os}-pg{N} base image, but the rpm release set is only
+    # a subset (e.g. [15,16,17]). When POSTGRES_VERSION targets a version outside that set, there
+    # is nothing for this test to build/sign, so skip gracefully (skip == success) rather than
+    # letting build_packages raise. The image for that pg was still built by update_image, so
+    # push_images downstream still reseeds it.
+    if version_filter_active:
+        release_versions, _ = get_postgres_versions(
+            platform=PLATFORM, input_files_dir=PACKAGING_EXEC_FOLDER
+        )
+        if POSTGRES_VERSION not in release_versions:
+            pytest.skip(
+                f"pg{POSTGRES_VERSION} not in release set {release_versions} for {PLATFORM}"
+            )
+
     delete_all_gpg_keys_by_name(TEST_GPG_KEY_NAME)
     delete_rpm_key_by_name(TEST_GPG_KEY_NAME)
     generate_new_gpg_key(
@@ -130,9 +160,10 @@ def test_build_packages():
         signing_credentials,
         input_output_parameters,
         is_test=True,
+        postgres_version=POSTGRES_VERSION,
     )
     verify_rpm_signature_in_dir(BASE_OUTPUT_FOLDER)
-    os_name, os_version = decode_os_and_release(PLATFORM)
+    _, os_version = decode_os_and_release(PLATFORM)
     sub_folder = get_release_package_folder_name(os_name, os_version)
     release_output_folder = f"{BASE_OUTPUT_FOLDER}/{sub_folder}"
     # Regression guard for the sign_packages path-doubling bug: the build output folder and the
@@ -151,9 +182,19 @@ def test_build_packages():
 
     postgres_version_file_path = f"{PACKAGING_EXEC_FOLDER}/{POSTGRES_VERSION_FILE}"
     if PLATFORM != "pgxn":
-        assert len(os.listdir(release_output_folder)) == get_required_package_count(
-            input_files_dir=PACKAGING_EXEC_FOLDER, platform=PLATFORM
-        )
+        # When POSTGRES_VERSION restricts the build to a single in-set version (multiple-image
+        # platforms only), only that version's packages are produced, so the expected count is the
+        # per-version package count (single_postgres_package_counts), not the full
+        # len(release_versions) * per-version count. When the filter is inactive — POSTGRES_VERSION
+        # empty/None, or a single-image platform where it is a no-op — keep the all-versions
+        # expectation.
+        if version_filter_active:
+            expected_package_count = single_postgres_package_counts[PLATFORM]
+        else:
+            expected_package_count = get_required_package_count(
+                input_files_dir=PACKAGING_EXEC_FOLDER, platform=PLATFORM
+            )
+        assert len(os.listdir(release_output_folder)) == expected_package_count
         assert os.path.exists(postgres_version_file_path)
         config = dotenv_values(postgres_version_file_path)
         assert config["release_versions"] == "15,16,17"