From 00f276c05e2f99a26ff0a9fc9f47bfe3ec7c71c3 Mon Sep 17 00:00:00 2001 From: Dmitry Shibanov Date: Fri, 29 May 2020 15:24:03 +0300 Subject: [PATCH] add common helpers --- azure-pipelines/templates/test-job.yml | 9 +- config/node-manifest-config.json | 7 + helpers/CODE_OF_CONDUCT.md | 76 +++++++++ helpers/CONTRIBUTING.md | 31 ++++ helpers/LICENSE | 21 +++ helpers/README.md | 5 + helpers/SECURITY.md | 3 + helpers/clean-toolcache.ps1 | 13 ++ helpers/common-helpers.psm1 | 85 ++++++++++ helpers/nix-helpers.psm1 | 20 ++- .../generate-versions-manifest.ps1 | 158 ------------------ .../manifest-generator.ps1 | 35 ++++ .../manifest-utils.Tests.ps1 | 116 +++++++++++++ .../packages-generation/manifest-utils.psm1 | 77 +++++++++ .../pester-extensions.psm1 | 0 helpers/win-helpers.psm1 | 17 +- helpers/win-vs-env.psm1 | 48 ++++++ tests/Node.Tests.ps1 | 2 +- 18 files changed, 548 insertions(+), 175 deletions(-) create mode 100644 config/node-manifest-config.json create mode 100644 helpers/CODE_OF_CONDUCT.md create mode 100644 helpers/CONTRIBUTING.md create mode 100644 helpers/LICENSE create mode 100644 helpers/README.md create mode 100644 helpers/SECURITY.md create mode 100644 helpers/clean-toolcache.ps1 create mode 100644 helpers/common-helpers.psm1 delete mode 100644 helpers/packages-generation/generate-versions-manifest.ps1 create mode 100644 helpers/packages-generation/manifest-generator.ps1 create mode 100644 helpers/packages-generation/manifest-utils.Tests.ps1 create mode 100644 helpers/packages-generation/manifest-utils.psm1 rename helpers/{packages-generation => }/pester-extensions.psm1 (100%) create mode 100644 helpers/win-vs-env.psm1 diff --git a/azure-pipelines/templates/test-job.yml b/azure-pipelines/templates/test-job.yml index d823162..1205bcb 100644 --- a/azure-pipelines/templates/test-job.yml +++ b/azure-pipelines/templates/test-job.yml @@ -10,12 +10,9 @@ jobs: - task: PowerShell@2 displayName: Fully cleanup the toolcache directory before testing inputs: - TargetType: inline - script: | - $NodeToolcachePath = Join-Path -Path $env:AGENT_TOOLSDIRECTORY -ChildPath "node" - if (Test-Path $NodeToolcachePath) { - Remove-Item -Path $NodeToolcachePath -Recurse -Force - } + targetType: filePath + filePath: helpers/clean-toolcache.ps1 + arguments: -ToolName "node" - task: DownloadPipelineArtifact@2 inputs: diff --git a/config/node-manifest-config.json b/config/node-manifest-config.json new file mode 100644 index 0000000..765f52d --- /dev/null +++ b/config/node-manifest-config.json @@ -0,0 +1,7 @@ +{ + "regex": "node-\\d+\\.\\d+\\.\\d+-(\\w+)-(x\\d+)", + "groups": { + "arch": 2, + "platform": 1 + } +} \ No newline at end of file diff --git a/helpers/CODE_OF_CONDUCT.md b/helpers/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..517657b --- /dev/null +++ b/helpers/CODE_OF_CONDUCT.md @@ -0,0 +1,76 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to make participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, sex characteristics, gender identity and expression, +level of experience, education, socio-economic status, nationality, personal +appearance, race, religion, or sexual identity and orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or + advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic + address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies within all project spaces, and it also applies when +an individual is representing the project or its community in public spaces. +Examples of representing a project or community include using an official +project e-mail address, posting via an official social media account, or acting +as an appointed representative at an online or offline event. Representation of +a project may be further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting the project team at opensource@github.com. All +complaints will be reviewed and investigated and will result in a response that +is deemed necessary and appropriate to the circumstances. The project team is +obligated to maintain confidentiality with regard to the reporter of an incident. +Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, +available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see +https://www.contributor-covenant.org/faq \ No newline at end of file diff --git a/helpers/CONTRIBUTING.md b/helpers/CONTRIBUTING.md new file mode 100644 index 0000000..fbc5fc5 --- /dev/null +++ b/helpers/CONTRIBUTING.md @@ -0,0 +1,31 @@ +## Contributing + +[fork]: https://github.com/actions/versions-package-tools/fork +[pr]: https://github.com/actions/versions-package-tools/compare +[code-of-conduct]: CODE_OF_CONDUCT.md + +Hi there! We're thrilled that you'd like to contribute to this project. Your help is essential for keeping it great. + +Contributions to this project are [released](https://help.github.com/articles/github-terms-of-service/#6-contributions-under-repository-license) to the public under the [MIT](LICENSE.md). + +Please note that this project is released with a [Contributor Code of Conduct][code-of-conduct]. By participating in this project you agree to abide by its terms. + +## Submitting a pull request + +1. [Fork][fork] and clone the repository +1. Create a new branch: `git checkout -b my-branch-name` +1. Make your changes +1. Push to your fork and [submit a pull request][pr] +1. Make sure that checks in your pull request are green + +Here are a few things you can do that will increase the likelihood of your pull request being accepted: + +- Please include a summary of the change and which issue is fixed. Also include relevant motivation and context. +- Follow the style guide for [PowerShell](https://github.com/PoshCode/PowerShellPracticeAndStyle). +- Write [good commit messages](http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html). + +## Resources + +- [How to Contribute to Open Source](https://opensource.guide/how-to-contribute/) +- [Using Pull Requests](https://help.github.com/articles/about-pull-requests/) +- [GitHub Help](https://help.github.com) diff --git a/helpers/LICENSE b/helpers/LICENSE new file mode 100644 index 0000000..f42f5ab --- /dev/null +++ b/helpers/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2020 GitHub + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/helpers/README.md b/helpers/README.md new file mode 100644 index 0000000..d9d6ccf --- /dev/null +++ b/helpers/README.md @@ -0,0 +1,5 @@ +# Common tools for generation of packages in the actions/*-versions repositories +This repository contains PowerShell modules that are used to generate packages for Actions. The packages are consumed by the images generated through [actions/virtual-environments](https://github.com/actions/virtual-environments) and some of the setup-* Actions + +## Contribution +Contributions are welcome! See [Contributor's Guide](./CONTRIBUTING.md) for more details about contribution process and code structure diff --git a/helpers/SECURITY.md b/helpers/SECURITY.md new file mode 100644 index 0000000..f0b196f --- /dev/null +++ b/helpers/SECURITY.md @@ -0,0 +1,3 @@ +If you discover a security issue in this repo, please submit it through the [GitHub Security Bug Bounty](https://hackerone.com/github) + +Thanks for helping make GitHub Actions safe for everyone. diff --git a/helpers/clean-toolcache.ps1 b/helpers/clean-toolcache.ps1 new file mode 100644 index 0000000..67796de --- /dev/null +++ b/helpers/clean-toolcache.ps1 @@ -0,0 +1,13 @@ +param ( + [string] $ToolName +) + +$targetPath = $env:AGENT_TOOLSDIRECTORY +if ($ToolName) { + $targetPath = Join-Path $targetPath $ToolName +} + +if (Test-Path $targetPath) { + Get-ChildItem -Path $targetPath -Recurse | Where-Object { $_.LinkType -eq "SymbolicLink" } | ForEach-Object { $_.Delete() } + Remove-Item -Path $targetPath -Recurse -Force +} diff --git a/helpers/common-helpers.psm1 b/helpers/common-helpers.psm1 new file mode 100644 index 0000000..cbc7665 --- /dev/null +++ b/helpers/common-helpers.psm1 @@ -0,0 +1,85 @@ +<# +.SYNOPSIS +The execute command and print all output to the logs +#> +function Execute-Command { + [CmdletBinding()] + param( + [Parameter(Mandatory=$true)][string] $Command + ) + + Write-Debug "Execute $Command" + + try { + Invoke-Expression $Command | ForEach-Object { Write-Host $_ } + if ($LASTEXITCODE -ne 0) { throw "Exit code: $LASTEXITCODE"} + } + catch { + $errorMessage = "Error happened during command execution: $Command `n $_" + Write-Host $errorMessage + if ($ErrorActionPreference -ne "Continue") { + # avoid logging Azure DevOps issues in case of $ErrorActionPreference -eq Continue + Write-Host "##vso[task.logissue type=error;] $errorMessage" + } + } +} + +<# +.SYNOPSIS +Download file from url and return local path to file +#> +function Download-File { + param( + [Parameter(Mandatory=$true)] + [Uri]$Uri, + [Parameter(Mandatory=$true)] + [String]$OutputFolder + ) + + $targetFilename = [IO.Path]::GetFileName($Uri) + $targetFilepath = Join-Path $OutputFolder $targetFilename + + Write-Debug "Download source from $Uri to $OutFile" + try { + (New-Object System.Net.WebClient).DownloadFile($Uri, $targetFilepath) + return $targetFilepath + } catch { + Write-Host "Error during downloading file from '$Uri'" + "$_" + exit 1 + } +} + +<# +.SYNOPSIS +Generate file that contains the list of all files in particular directory +#> +function New-ToolStructureDump { + param( + [Parameter(Mandatory=$true)] + [String]$ToolPath, + [Parameter(Mandatory=$true)] + [String]$OutputFolder + ) + + $outputFile = Join-Path $OutputFolder "tools_structure.txt" + + $folderContent = Get-ChildItem -Path $ToolPath -Recurse | Sort-Object | Select-Object -Property FullName, Length + $folderContent | ForEach-Object { + $relativePath = $_.FullName.Replace($ToolPath, ""); + return "${relativePath}" + } | Out-File -FilePath $outputFile +} + +<# +.SYNOPSIS +Check if it is macOS / Ubuntu platform +#> +function IsNixPlatform { + param( + [Parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()] + [String]$Platform + ) + + return ($Platform -match "macos") -or ($Platform -match "ubuntu") -or ($Platform -match "linux") +} \ No newline at end of file diff --git a/helpers/nix-helpers.psm1 b/helpers/nix-helpers.psm1 index e6e3441..0f00e21 100644 --- a/helpers/nix-helpers.psm1 +++ b/helpers/nix-helpers.psm1 @@ -20,13 +20,25 @@ function Create-TarArchive { [String]$SourceFolder, [Parameter(Mandatory=$true)] [String]$ArchivePath, - [string]$CompressionType = "gz" + [string]$CompressionType = "gz", + [switch]$DereferenceSymlinks ) - $CompressionTypeArgument = If ([string]::IsNullOrWhiteSpace($CompressionType)) { "" } else { "--${CompressionType}" } + $arguments = @( + "-c", + "-f", $ArchivePath, + "." + ) + If ($CompressionType) { + $arguments += "--${CompressionType}" + } + + if ($DereferenceSymlinks) { + $arguments += "-h" + } Push-Location $SourceFolder - Write-Debug "tar -c $CompressionTypeArgument -f $ArchivePath ." - tar -c $CompressionTypeArgument -f $ArchivePath . + Write-Debug "tar $arguments" + tar @arguments Pop-Location } \ No newline at end of file diff --git a/helpers/packages-generation/generate-versions-manifest.ps1 b/helpers/packages-generation/generate-versions-manifest.ps1 deleted file mode 100644 index 52b652d..0000000 --- a/helpers/packages-generation/generate-versions-manifest.ps1 +++ /dev/null @@ -1,158 +0,0 @@ -<# -.SYNOPSIS -Generate versions manifest based on repository releases - -.DESCRIPTION -Versions manifest is needed to find the latest assets for particular version of tool -.PARAMETER GitHubRepositoryOwner -Required parameter. The organization which tool repository belongs -.PARAMETER GitHubRepositoryName -Optional parameter. The name of tool repository -.PARAMETER GitHubAccessToken -Required parameter. PAT Token to overcome GitHub API Rate limit -.PARAMETER OutputFile -Required parameter. File "*.json" where generated results will be saved -.PARAMETER PlatformMapFile -Optional parameter. Path to the json file with platform map -Structure example: -{ - "macos-1014": [ - { - "platform": "darwin", - "platform_version": "10.14" - }, ... - ], ... -} -#> - -param ( - [Parameter(Mandatory)] [string] $GitHubRepositoryOwner, - [Parameter(Mandatory)] [string] $GitHubRepositoryName, - [Parameter(Mandatory)] [string] $GitHubAccessToken, - [Parameter(Mandatory)] [string] $OutputFile, - [string] $PlatformMapFile -) - -Import-Module (Join-Path $PSScriptRoot "../github/github-api.psm1") - -if ($PlatformMapFile -and (Test-Path $PlatformMapFile)) { - $PlatformMap = Get-Content $PlatformMapFile -Raw | ConvertFrom-Json -AsHashtable -} else { - $PlatformMap = @{} -} - -function Get-FileNameWithoutExtension { - param ( - [Parameter(Mandatory)][string]$Filename - ) - - if ($Filename.EndsWith(".tar.gz")) { - $Filename = [IO.path]::GetFileNameWithoutExtension($Filename) - } - - return [IO.path]::GetFileNameWithoutExtension($Filename) -} - -function New-AssetItem { - param ( - [Parameter(Mandatory)][string]$Filename, - [Parameter(Mandatory)][string]$DownloadUrl, - [Parameter(Mandatory)][string]$Arch, - [Parameter(Mandatory)][string]$Platform, - [string]$PlatformVersion - ) - $asset = New-Object PSObject - - $asset | Add-Member -Name "filename" -Value $Filename -MemberType NoteProperty - $asset | Add-Member -Name "arch" -Value $Arch -MemberType NoteProperty - $asset | Add-Member -Name "platform" -Value $Platform -MemberType NoteProperty - if ($PlatformVersion) { $asset | Add-Member -Name "platform_version" -Value $PlatformVersion -MemberType NoteProperty } - $asset | Add-Member -Name "download_url" -Value $DownloadUrl -MemberType NoteProperty - - return $asset -} - -function Build-AssetsList { - param ( - [AllowEmptyCollection()] - [Parameter(Mandatory)][array]$ReleaseAssets - ) - - - $assets = @() - foreach($releaseAsset in $ReleaseAssets) { - $filename = Get-FileNameWithoutExtension -Filename $releaseAsset.name - $parts = $filename.Split("-") - $arch = $parts[-1] - $buildPlatform = [string]::Join("-", $parts[2..($parts.Length-2)]) - - if ($PlatformMap[$buildPlatform]) { - $PlatformMap[$buildPlatform] | ForEach-Object { - $assets += New-AssetItem -Filename $releaseAsset.name ` - -DownloadUrl $releaseAsset.browser_download_url ` - -Arch $arch ` - -Platform $_.platform ` - -PlatformVersion $_.platform_version - } - - } else { - $assets += New-AssetItem -Filename $releaseAsset.name ` - -DownloadUrl $releaseAsset.browser_download_url ` - -Arch $arch ` - -Platform $buildPlatform - } - } - - return $assets -} - -function Get-VersionFromRelease { - param ( - [Parameter(Mandatory)][object]$Release - ) - # Release name can contain additional information after ':' so filter it - [string]$releaseName = $Release.name.Split(':')[0] - [Version]$version = $null - if (![Version]::TryParse($releaseName, [ref]$version)) { - throw "Release '$($Release.id)' has invalid title '$($Release.name)'. It can't be parsed as version. ( $($Release.html_url) )" - } - - return $version -} - -function Build-VersionsManifest { - param ( - [Parameter(Mandatory)][array]$Releases - ) - - $Releases = $Releases | Sort-Object -Property "published_at" -Descending - - $versionsHash = @{} - foreach ($release in $Releases) { - if (($release.draft -eq $true) -or ($release.prerelease -eq $true)) { - continue - } - - [Version]$version = Get-VersionFromRelease $release - $versionKey = $version.ToString() - - if ($versionsHash.ContainsKey($versionKey)) { - continue - } - - $versionsHash.Add($versionKey, [PSCustomObject]@{ - version = $versionKey - stable = $true - release_url = $release.html_url - files = Build-AssetsList $release.assets - }) - } - - # Sort versions by descending - return $versionsHash.Values | Sort-Object -Property @{ Expression = { [Version]$_.version }; Descending = $true } -} - -$gitHubApi = Get-GitHubApi -AccountName $GitHubRepositoryOwner -ProjectName $GitHubRepositoryName -AccessToken $GitHubAccessToken -$releases = $gitHubApi.GetReleases() -$versionIndex = Build-VersionsManifest $releases -$versionIndex | ConvertTo-Json -Depth 5 | Out-File $OutputFile -Encoding UTF8NoBOM -Force diff --git a/helpers/packages-generation/manifest-generator.ps1 b/helpers/packages-generation/manifest-generator.ps1 new file mode 100644 index 0000000..85434d0 --- /dev/null +++ b/helpers/packages-generation/manifest-generator.ps1 @@ -0,0 +1,35 @@ +<# +.SYNOPSIS +Generate versions manifest based on repository releases + +.DESCRIPTION +Versions manifest is needed to find the latest assets for particular version of tool +.PARAMETER GitHubRepositoryOwner +Required parameter. The organization which tool repository belongs +.PARAMETER GitHubRepositoryName +Optional parameter. The name of tool repository +.PARAMETER GitHubAccessToken +Required parameter. PAT Token to overcome GitHub API Rate limit +.PARAMETER OutputFile +Required parameter. File "*.json" where generated results will be saved +.PARAMETER ConfigurationFile +Path to the json file with parsing configuration +#> + +param ( + [Parameter(Mandatory)] [string] $GitHubRepositoryOwner, + [Parameter(Mandatory)] [string] $GitHubRepositoryName, + [Parameter(Mandatory)] [string] $GitHubAccessToken, + [Parameter(Mandatory)] [string] $OutputFile, + [Parameter(Mandatory)] [string] $ConfigurationFile +) + +Import-Module (Join-Path $PSScriptRoot "../github/github-api.psm1") +Import-Module (Join-Path $PSScriptRoot "manifest-utils.psm1") -Force + +$configuration = Read-ConfigurationFile -Filepath $ConfigurationFile + +$gitHubApi = Get-GitHubApi -AccountName $GitHubRepositoryOwner -ProjectName $GitHubRepositoryName -AccessToken $GitHubAccessToken +$releases = $gitHubApi.GetReleases() +$versionIndex = Build-VersionsManifest -Releases $releases -Configuration $configuration +$versionIndex | ConvertTo-Json -Depth 5 | Out-File $OutputFile -Encoding UTF8NoBOM -Force diff --git a/helpers/packages-generation/manifest-utils.Tests.ps1 b/helpers/packages-generation/manifest-utils.Tests.ps1 new file mode 100644 index 0000000..f1aa785 --- /dev/null +++ b/helpers/packages-generation/manifest-utils.Tests.ps1 @@ -0,0 +1,116 @@ +Import-Module (Join-Path $PSScriptRoot "manifest-utils.psm1") -Force + +Describe "New-AssetItem" { + It "use regex to parse all values in correct order" { + $githubAsset = @{ name = "python-3.8.3-linux-16.04-x64.tar.gz"; browser_download_url = "long_url"; } + $configuration = @{ + regex = "python-\d+\.\d+\.\d+-(\w+)-([\w\.]+)?-?(x\d+)"; + groups = [PSCustomObject]@{ platform = 1; platform_version = 2; arch = 3; }; + } + $expectedOutput = [PSCustomObject]@{ + filename = "python-3.8.3-linux-16.04-x64.tar.gz"; platform = "linux"; platform_version = "16.04"; + arch = "x64"; download_url = "long_url"; + } + + $actualOutput = New-AssetItem -ReleaseAsset $githubAsset -Configuration $configuration + Assert-Equivalent -Actual $actualOutput -Expected $expectedOutput + } + + It "support constant values in groups" { + $githubAsset = @{ name = "python-3.8.3-linux-16.04-x64.tar.gz"; browser_download_url = "long_url"; } + $configuration = @{ + regex = "python-\d+\.\d+\.\d+-(\w+)-([\w\.]+)?-?(x\d+)"; + groups = [PSCustomObject]@{ platform = 1; platform_version = 2; arch = "x64"; } + } + $expectedOutput = [PSCustomObject]@{ + filename = "python-3.8.3-linux-16.04-x64.tar.gz"; platform = "linux"; platform_version = "16.04"; + arch = "x64"; download_url = "long_url"; + } + + $actualOutput = New-AssetItem -ReleaseAsset $githubAsset -Configuration $configuration + Assert-Equivalent -Actual $actualOutput -Expected $expectedOutput + } + + It "Skip empty groups" { + $githubAsset = @{ name = "python-3.8.3-win32-x64.zip"; browser_download_url = "long_url"; } + $configuration = @{ + regex = "python-\d+\.\d+\.\d+-(\w+)-([\w\.]+)?-?(x\d+)"; + groups = [PSCustomObject]@{ platform = 1; platform_version = 2; arch = 3; } + } + $expectedOutput = [PSCustomObject]@{ + filename = "python-3.8.3-win32-x64.zip"; platform = "win32"; + arch = "x64"; download_url = "long_url"; + } + + $actualOutput = New-AssetItem -ReleaseAsset $githubAsset -Configuration $configuration + Assert-Equivalent -Actual $actualOutput -Expected $expectedOutput + } +} + +Describe "Get-VersionFromRelease" { + It "clear version" { + $release = @{ name = "3.8.3" } + Get-VersionFromRelease -Release $release | Should -Be "3.8.3" + } + + It "version with title" { + $release = @{ name = "3.8.3: Release title" } + Get-VersionFromRelease -Release $release | Should -Be "3.8.3" + } +} + +Describe "Build-VersionsManifest" { + $assets = @( + @{ name = "python-3.8.3-linux-16.04-x64.tar.gz"; browser_download_url = "fake_url"; } + @{ name = "python-3.8.3-linux-18.04-x64.tar.gz"; browser_download_url = "fake_url"; } + ) + $configuration = @{ + regex = "python-\d+\.\d+\.\d+-(\w+)-([\w\.]+)?-?(x\d+)"; + groups = [PSCustomObject]@{ platform = 1; platform_version = 2; arch = "x64"; } + } + $expectedManifestFiles = @( + [PSCustomObject]@{ filename = "python-3.8.3-linux-16.04-x64.tar.gz"; arch = "x64"; platform = "linux"; platform_version = "16.04"; download_url = "fake_url" }, + [PSCustomObject]@{ filename = "python-3.8.3-linux-18.04-x64.tar.gz"; arch = "x64"; platform = "linux"; platform_version = "18.04"; download_url = "fake_url" } + ) + + It "build manifest with correct version order" { + $releases = @( + @{ name = "3.8.1"; draft = $false; prerelease = $false; html_url = "fake_html_url"; published_at = "2020-05-14T09:54:06Z"; assets = $assets }, + @{ name = "3.5.2: Hello"; draft = $false; prerelease = $false; html_url = "fake_html_url"; published_at = "2020-05-06T11:45:36Z"; assets = $assets }, + @{ name = "3.8.3: Release title"; draft = $false; prerelease = $false; html_url = "fake_html_url"; published_at = "2020-05-06T11:43:38Z"; assets = $assets } + ) + $expectedManifest = @( + [PSCustomObject]@{ version = "3.8.3"; stable = $true; release_url = "fake_html_url"; files = $expectedManifestFiles }, + [PSCustomObject]@{ version = "3.8.1"; stable = $true; release_url = "fake_html_url"; files = $expectedManifestFiles }, + [PSCustomObject]@{ version = "3.5.2"; stable = $true; release_url = "fake_html_url"; files = $expectedManifestFiles } + ) + $actualManifest = Build-VersionsManifest -Releases $releases -Configuration $configuration + Assert-Equivalent -Actual $actualManifest -Expected $expectedManifest + } + + It "Skip draft and prerelease" { + $releases = @( + @{ name = "3.8.1"; draft = $true; prerelease = $false; html_url = "fake_html_url"; published_at = "2020-05-14T09:54:06Z"; assets = $assets }, + @{ name = "3.5.2"; draft = $false; prerelease = $true; html_url = "fake_html_url"; published_at = "2020-05-06T11:45:36Z"; assets = $assets }, + @{ name = "3.8.3"; draft = $false; prerelease = $false; html_url = "fake_html_url"; published_at = "2020-05-06T11:43:38Z"; assets = $assets } + ) + $expectedManifest = @( + [PSCustomObject]@{ version = "3.8.3"; stable = $true; release_url = "fake_html_url"; files = $expectedManifestFiles } + ) + [array]$actualManifest = Build-VersionsManifest -Releases $releases -Configuration $configuration + Assert-Equivalent -Actual $actualManifest -Expected $expectedManifest + } + + It "take latest published release for each version" { + $releases = @( + @{ name = "3.8.1"; draft = $false; prerelease = $false; html_url = "fake_html_url1"; published_at = "2020-05-06T11:45:36Z"; assets = $assets }, + @{ name = "3.8.1"; draft = $false; prerelease = $false; html_url = "fake_html_url2"; published_at = "2020-05-14T09:54:06Z"; assets = $assets }, + @{ name = "3.8.1"; draft = $false; prerelease = $false; html_url = "fake_html_url3"; published_at = "2020-05-06T11:43:38Z"; assets = $assets } + ) + $expectedManifest = @( + [PSCustomObject]@{ version = "3.8.1"; stable = $true; release_url = "fake_html_url2"; files = $expectedManifestFiles } + ) + [array]$actualManifest = Build-VersionsManifest -Releases $releases -Configuration $configuration + Assert-Equivalent -Actual $actualManifest -Expected $expectedManifest + } +} \ No newline at end of file diff --git a/helpers/packages-generation/manifest-utils.psm1 b/helpers/packages-generation/manifest-utils.psm1 new file mode 100644 index 0000000..2d4ef00 --- /dev/null +++ b/helpers/packages-generation/manifest-utils.psm1 @@ -0,0 +1,77 @@ +function Read-ConfigurationFile { + param ([Parameter(Mandatory)][string]$Filepath) + return Get-Content $Filepath -Raw | ConvertFrom-Json +} + +function New-AssetItem { + param ( + [Parameter(Mandatory)][object]$ReleaseAsset, + [Parameter(Mandatory)][object]$Configuration + ) + $regexResult = [regex]::Match($ReleaseAsset.name, $Configuration.regex) + if (-not $regexResult.Success) { throw "Can't match asset filename '$($_.name)' to regex" } + + $result = New-Object PSObject + $result | Add-Member -Name "filename" -Value $ReleaseAsset.name -MemberType NoteProperty + $Configuration.groups.PSObject.Properties | ForEach-Object { + if (($_.Value).GetType().Name.StartsWith("Int")) { + $value = $regexResult.Groups[$_.Value].Value + } else { + $value = $_.Value + } + + if (-not ([string]::IsNullOrEmpty($value))) { + $result | Add-Member -Name $_.Name -Value $value -MemberType NoteProperty + } + } + + $result | Add-Member -Name "download_url" -Value $ReleaseAsset.browser_download_url -MemberType NoteProperty + return $result +} + +function Get-VersionFromRelease { + param ( + [Parameter(Mandatory)][object]$Release + ) + # Release name can contain additional information after ':' so filter it + [string]$releaseName = $Release.name.Split(':')[0] + [Version]$version = $null + if (![Version]::TryParse($releaseName, [ref]$version)) { + throw "Release '$($Release.id)' has invalid title '$($Release.name)'. It can't be parsed as version. ( $($Release.html_url) )" + } + + return $version +} + +function Build-VersionsManifest { + param ( + [Parameter(Mandatory)][array]$Releases, + [Parameter(Mandatory)][object]$Configuration + ) + + $Releases = $Releases | Sort-Object -Property "published_at" -Descending + + $versionsHash = @{} + foreach ($release in $Releases) { + if (($release.draft -eq $true) -or ($release.prerelease -eq $true)) { + continue + } + + [Version]$version = Get-VersionFromRelease $release + $versionKey = $version.ToString() + + if ($versionsHash.ContainsKey($versionKey)) { + continue + } + + $versionsHash.Add($versionKey, [PSCustomObject]@{ + version = $versionKey + stable = $true + release_url = $release.html_url + files = $release.assets | ForEach-Object { New-AssetItem -ReleaseAsset $_ -Configuration $Configuration } + }) + } + + # Sort versions by descending + return $versionsHash.Values | Sort-Object -Property @{ Expression = { [Version]$_.version }; Descending = $true } +} \ No newline at end of file diff --git a/helpers/packages-generation/pester-extensions.psm1 b/helpers/pester-extensions.psm1 similarity index 100% rename from helpers/packages-generation/pester-extensions.psm1 rename to helpers/pester-extensions.psm1 diff --git a/helpers/win-helpers.psm1 b/helpers/win-helpers.psm1 index 74a16f3..beac0b9 100644 --- a/helpers/win-helpers.psm1 +++ b/helpers/win-helpers.psm1 @@ -21,14 +21,19 @@ function Create-SevenZipArchive { [Parameter(Mandatory=$true)] [String]$ArchivePath, [String]$ArchiveType = "zip", - [String]$CompressionLevel = 5 + [String]$CompressionLevel = 5, + [switch]$IncludeSymlinks ) - $ArchiveTypeArgument = "-t${ArchiveType}" - $CompressionLevelArgument = "-mx=${CompressionLevel}" - + $ArchiveTypeArguments = @( + "-t${ArchiveType}", + "-mx=${CompressionLevel}" + ) + if ($IncludeSymlinks) { + $ArchiveTypeArguments += "-snl" + } Push-Location $SourceFolder - Write-Debug "7z a $ArchiveTypeArgument $CompressionLevelArgument $ArchivePath @$SourceFolder" - 7z a $ArchiveTypeArgument $CompressionLevelArgument $ArchivePath $SourceFolder\* + Write-Debug "7z a $ArchiveTypeArgument $ArchivePath @$SourceFolder" + 7z a @ArchiveTypeArguments $ArchivePath $SourceFolder\* Pop-Location } \ No newline at end of file diff --git a/helpers/win-vs-env.psm1 b/helpers/win-vs-env.psm1 new file mode 100644 index 0000000..1eb9093 --- /dev/null +++ b/helpers/win-vs-env.psm1 @@ -0,0 +1,48 @@ +### +# Visual Studio helper functions +### + +function Get-VSWhere { + $vswhere = "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe"; + + if (-not (Test-Path $vswhere )) { + [Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12 + $vswhere = ".\vswhere.exe" + $vswhereApiUri = "https://api.github.com/repos/Microsoft/vswhere/releases/latest" + $tag = (Invoke-RestMethod -Uri $vswhereApiUri)[0].tag_name + $vswhereUri = "https://github.com/Microsoft/vswhere/releases/download/$tag/vswhere.exe" + Invoke-WebRequest -Uri $vswhereUri -OutFile $vswhere | Out-Null + } + + return $vswhere +} + +function Invoke-Environment +{ + Param + ( + [Parameter(Mandatory)] + [string] + $Command + ) + + & "${env:COMSPEC}" /s /c "`"$Command`" -no_logo && set" | Foreach-Object { + if ($_ -match '^([^=]+)=(.*)') { + [System.Environment]::SetEnvironmentVariable($matches[1], $matches[2]) + } + } +} + +function Get-VSInstallationPath { + $vswhere = Get-VSWhere + $installationPath = & $vswhere -prerelease -legacy -latest -property installationPath + + return $installationPath +} + +function Invoke-VSDevEnvironment { + Write-Host "Invoke-VSDevEnvironment had been invoked" + $installationPath = Get-VSInstallationPath + $envFilepath = Join-Path $installationPath "Common7\Tools\vsdevcmd.bat" + Invoke-Environment -Command $envFilepath +} \ No newline at end of file diff --git a/tests/Node.Tests.ps1 b/tests/Node.Tests.ps1 index 8ff9cb7..3df4c21 100644 --- a/tests/Node.Tests.ps1 +++ b/tests/Node.Tests.ps1 @@ -3,7 +3,7 @@ param ( $Version ) -Import-Module (Join-Path $PSScriptRoot "../helpers/packages-generation/pester-extensions.psm1") +Import-Module (Join-Path $PSScriptRoot "../helpers/pester-extensions.psm1") function Get-UseNodeLogs { $logsFolderPath = Join-Path -Path $env:AGENT_HOMEDIRECTORY -ChildPath "_diag" | Join-Path -ChildPath "pages"