From 26de9fde7848d7408a1705b0e5b1cdbba8cac1d9 Mon Sep 17 00:00:00 2001 From: Steve Gontzes Date: Fri, 5 Jun 2026 16:58:28 -0400 Subject: [PATCH] Fix Windows S3 artifact uploads --- .github/workflows/release.yaml | 13 ++- Makefile | 1 + scripts/test-s3-release-uploads.ps1 | 100 ++++++++++++++++++ scripts/upload-release-artifacts.ps1 | 146 +++++++++++++++++++++++++++ 4 files changed, 253 insertions(+), 7 deletions(-) create mode 100644 scripts/test-s3-release-uploads.ps1 create mode 100644 scripts/upload-release-artifacts.ps1 diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 3833ff7..472a857 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -693,17 +693,16 @@ jobs: - name: Upload Windows artifacts to S3 working-directory: _workflows - shell: bash + shell: pwsh env: S3_BUCKET: ${{ env.S3_BUCKET }} S3_DIRECTORY: ${{ steps.s3-directory.outputs.S3_DIRECTORY }} run: | - set -euo pipefail - ./scripts/upload-release-artifacts.sh \ - --bucket "$S3_BUCKET" \ - --directory "$S3_DIRECTORY" \ - --base-dir "../_caller/dist" \ - --include-sbom-documents + ./scripts/upload-release-artifacts.ps1 ` + -Bucket $env:S3_BUCKET ` + -Directory $env:S3_DIRECTORY ` + -BaseDir "../_caller/dist" ` + -IncludeSbomDocuments - name: Set up Go for workflows tools uses: actions/setup-go@v6 diff --git a/Makefile b/Makefile index 1ec505b..82bd5d9 100644 --- a/Makefile +++ b/Makefile @@ -20,6 +20,7 @@ test-go: test-scripts: bash scripts/test-derive-iam-role-name.sh bash scripts/test-s3-release-uploads.sh + if command -v pwsh >/dev/null 2>&1; then pwsh -NoProfile -File scripts/test-s3-release-uploads.ps1; else echo "pwsh not found; skipping PowerShell S3 release upload tests"; fi .PHONY: workflow-validate workflow-validate: diff --git a/scripts/test-s3-release-uploads.ps1 b/scripts/test-s3-release-uploads.ps1 new file mode 100644 index 0000000..2b7f30d --- /dev/null +++ b/scripts/test-s3-release-uploads.ps1 @@ -0,0 +1,100 @@ +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" + +$ScriptDir = Split-Path -Parent $PSCommandPath +$RootDir = Split-Path -Parent $ScriptDir +$TempDir = Join-Path ([System.IO.Path]::GetTempPath()) ([System.Guid]::NewGuid().ToString()) +New-Item -ItemType Directory -Path $TempDir | Out-Null + +try { + $FakeAws = Join-Path $TempDir "aws.ps1" + $ArgsLog = Join-Path $TempDir "aws-args.log" + Set-Content -LiteralPath $FakeAws -Encoding utf8 -Value @' +param([Parameter(ValueFromRemainingArguments = $true)][string[]]$AwsArgs) + +foreach ($Arg in $AwsArgs) { + Add-Content -LiteralPath $env:AWS_ARGS_LOG -Value $Arg +} + +if ($AwsArgs.Count -ge 2 -and $AwsArgs[0] -eq "s3api" -and $AwsArgs[1] -eq "head-object") { + if ($env:FAKE_HEAD_OBJECT_JSON) { + Write-Output $env:FAKE_HEAD_OBJECT_JSON + exit 0 + } + exit 255 +} + +if ($AwsArgs.Count -ge 2 -and $AwsArgs[0] -eq "s3api" -and $AwsArgs[1] -eq "put-object" -and $env:FAKE_AWS_FAIL_PUT) { + Write-Error "put-object failed" + exit 254 +} + +exit 0 +'@ + + $DistDir = Join-Path $TempDir "dist" + New-Item -ItemType Directory -Path $DistDir | Out-Null + Set-Content -LiteralPath (Join-Path $DistDir "example.zip") -Value "zip" + Set-Content -LiteralPath (Join-Path $DistDir "example.zip.sig") -Value "sig" + Set-Content -LiteralPath (Join-Path $DistDir "example.zip.cert") -Value "cert" + Set-Content -LiteralPath (Join-Path $DistDir "example.zip.sbom.json") -Value "{}" + Set-Content -LiteralPath (Join-Path $DistDir "example.zip.sbom.sigstore.json") -Value "{}" + Set-Content -LiteralPath (Join-Path $DistDir "ignore.txt") -Value "ignore" + + $env:AWS_CLI = $FakeAws + $env:AWS_ARGS_LOG = $ArgsLog + Remove-Item Env:\FAKE_HEAD_OBJECT_JSON -ErrorAction SilentlyContinue + Remove-Item Env:\FAKE_AWS_FAIL_PUT -ErrorAction SilentlyContinue + + & (Join-Path $RootDir "scripts/upload-release-artifacts.ps1") ` + -Bucket "release-bucket" ` + -Directory "releases/ConductorOne/example/v1.2.3" ` + -BaseDir $DistDir ` + -IncludeSbomDocuments + + $Log = Get-Content -LiteralPath $ArgsLog + foreach ($Expected in @( + "releases/ConductorOne/example/v1.2.3/example.zip", + "releases/ConductorOne/example/v1.2.3/example.zip.sig", + "releases/ConductorOne/example/v1.2.3/example.zip.cert", + "releases/ConductorOne/example/v1.2.3/example.zip.sbom.json", + "releases/ConductorOne/example/v1.2.3/example.zip.sbom.sigstore.json", + "--if-none-match", + "*", + "--metadata")) { + if ($Log -notcontains $Expected) { + throw "missing expected AWS argument: ${Expected}" + } + } + if ($Log -contains "ignore.txt") { + throw "unexpected upload for ignore.txt" + } + + $FailDir = Join-Path $TempDir "fail-dist" + New-Item -ItemType Directory -Path $FailDir | Out-Null + Set-Content -LiteralPath (Join-Path $FailDir "failure.zip") -Value "zip" + Clear-Content -LiteralPath $ArgsLog + $env:FAKE_AWS_FAIL_PUT = "1" + + $Failed = $false + try { + & (Join-Path $RootDir "scripts/upload-release-artifacts.ps1") ` + -Bucket "release-bucket" ` + -Directory "releases/ConductorOne/example/v1.2.3" ` + -BaseDir $FailDir + } catch { + $Failed = $true + } + + if (-not $Failed) { + throw "put-object failure should fail the upload script" + } + + Write-Host "PowerShell S3 release upload tests passed" +} finally { + Remove-Item -Recurse -Force -LiteralPath $TempDir -ErrorAction SilentlyContinue + Remove-Item Env:\AWS_CLI -ErrorAction SilentlyContinue + Remove-Item Env:\AWS_ARGS_LOG -ErrorAction SilentlyContinue + Remove-Item Env:\FAKE_HEAD_OBJECT_JSON -ErrorAction SilentlyContinue + Remove-Item Env:\FAKE_AWS_FAIL_PUT -ErrorAction SilentlyContinue +} diff --git a/scripts/upload-release-artifacts.ps1 b/scripts/upload-release-artifacts.ps1 new file mode 100644 index 0000000..58a661c --- /dev/null +++ b/scripts/upload-release-artifacts.ps1 @@ -0,0 +1,146 @@ +param( + [Parameter(Mandatory = $true)] + [string]$Bucket, + + [Parameter(Mandatory = $true)] + [string]$Directory, + + [Parameter(Mandatory = $true)] + [string]$BaseDir, + + [switch]$IncludeSbomDocuments +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" + +$CacheControl = "public,max-age=31536000,immutable" +$AwsCli = if ($env:AWS_CLI) { $env:AWS_CLI } else { "aws" } + +function Get-ReleaseArtifactContentType { + param([Parameter(Mandatory = $true)][string]$Name) + + switch -Regex ($Name) { + "\.tar\.gz$" { + return "application/gzip" + } + "\.zip$" { + return "application/zip" + } + "\.msi$" { + return "application/x-msi" + } + "\.sig$" { + return "application/octet-stream" + } + "\.cert$" { + return "application/x-pem-file" + } + "\.sigstore\.json$" { + return "application/json" + } + "\.sbom\.json$" { + if ($IncludeSbomDocuments) { + return "application/json" + } + return $null + } + default { + return $null + } + } +} + +function Get-MetadataValue { + param( + [Parameter(Mandatory = $true)]$Metadata, + [Parameter(Mandatory = $true)][string]$Name + ) + + foreach ($Property in $Metadata.PSObject.Properties) { + if ($Property.Name -ieq $Name) { + return [string]$Property.Value + } + } + return $null +} + +function Invoke-AwsChecked { + param([Parameter(Mandatory = $true)][string[]]$Arguments) + + $Output = & $AwsCli @Arguments 2>&1 + $ExitCode = $LASTEXITCODE + if ($ExitCode -ne 0) { + $Message = ($Output | Out-String).Trim() + if ($Message) { + throw "aws $($Arguments -join ' ') failed with exit code ${ExitCode}: ${Message}" + } + throw "aws $($Arguments -join ' ') failed with exit code ${ExitCode}" + } + return $Output +} + +function Get-ExistingObject { + param( + [Parameter(Mandatory = $true)][string]$ObjectBucket, + [Parameter(Mandatory = $true)][string]$ObjectKey + ) + + $Output = & $AwsCli s3api head-object --bucket $ObjectBucket --key $ObjectKey 2>$null + if ($LASTEXITCODE -ne 0) { + return $null + } + + return (($Output | Out-String) | ConvertFrom-Json) +} + +if (-not (Test-Path -LiteralPath $BaseDir -PathType Container)) { + throw "Base directory not found: ${BaseDir}" +} + +$UploadCount = 0 +$Artifacts = Get-ChildItem -LiteralPath $BaseDir -File +foreach ($Artifact in $Artifacts) { + $ContentType = Get-ReleaseArtifactContentType -Name $Artifact.Name + if (-not $ContentType) { + continue + } + + $ObjectKey = "$Directory/$($Artifact.Name)" + $BodySha256 = (Get-FileHash -LiteralPath $Artifact.FullName -Algorithm SHA256).Hash.ToLowerInvariant() + $Existing = Get-ExistingObject -ObjectBucket $Bucket -ObjectKey $ObjectKey + if ($null -ne $Existing) { + $ExistingSha256 = if ($null -ne $Existing.Metadata) { + Get-MetadataValue -Metadata $Existing.Metadata -Name "sha256" + } else { + $null + } + + if ($ExistingSha256 -eq $BodySha256) { + Write-Host "S3 object already exists with matching sha256 metadata, skipping: s3://${Bucket}/${ObjectKey}" + $UploadCount++ + continue + } + + throw "S3 object already exists with different or missing sha256 metadata: s3://${Bucket}/${ObjectKey}" + } + + Write-Host "Uploading $($Artifact.Name) to S3 without overwrite..." + Invoke-AwsChecked -Arguments @( + "s3api", "put-object", + "--bucket", $Bucket, + "--key", $ObjectKey, + "--body", $Artifact.FullName, + "--cache-control", $CacheControl, + "--content-type", $ContentType, + "--metadata", "sha256=${BodySha256}", + "--if-none-match", "*" + ) | Out-Null + $UploadCount++ +} + +if ($UploadCount -eq 0) { + throw "No release artifacts found in ${BaseDir}" +} + +Write-Host "Uploaded ${UploadCount} release artifacts to S3"