diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index 22e0e06..0e31764 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -5,6 +5,7 @@ jobs: name: Run runs-on: ${{ matrix.operating-system }} strategy: + fail-fast: false matrix: operating-system: [ubuntu-latest, windows-latest] steps: @@ -14,13 +15,14 @@ jobs: - name: Set Node.js 10.x uses: actions/setup-node@master with: - version: 10.x + node-version: 10.x - name: npm install run: npm install - - name: Lint - run: npm run format-check - - name: npm test run: npm test + + - name: Lint + run: npm run format-check + continue-on-error: true diff --git a/__tests__/installer.test.ts b/__tests__/installer.test.ts index fcaa76f..0581f72 100644 --- a/__tests__/installer.test.ts +++ b/__tests__/installer.test.ts @@ -11,116 +11,107 @@ const dataDir = path.join(__dirname, 'data'); process.env['RUNNER_TOOL_CACHE'] = toolDir; process.env['RUNNER_TEMP'] = tempDir; import * as installer from '../src/installer'; +import * as executil from '../src/executil'; +const osarch = os.arch(); const IS_WINDOWS = process.platform === 'win32'; +const goExe: string = IS_WINDOWS ? 'go.exe' : 'go'; -describe('installer tests', () => { - beforeAll(async () => { - await io.rmRF(toolDir); - await io.rmRF(tempDir); - }, 100000); +const cleanup = async () => { + await io.rmRF(toolDir); + await io.rmRF(tempDir); +} +beforeAll(cleanup, 100000); +afterAll(cleanup, 100000); - afterAll(async () => { - try { - await io.rmRF(toolDir); - await io.rmRF(tempDir); - } catch { - console.log('Failed to remove test directories'); - } - }, 100000); +const describeTable = describe.each([ + ['tip', '+a5bfd9d', 'go1.14beta1', 'a5bfd9da1d1b24f326399b6b75558ded14514f23'], + ['latest', 'go1.13', 'n/a', '1.13.0'], + ['1.x', 'go1.13', 'n/a', '1.13.0'], + ['1.10.x', 'go1.10.8', 'n/a', '1.10.8'], + ['1.10.8', 'go1.10.8', 'n/a', '1.10.8'], + ['1.10', 'go1.10', 'n/a', '1.10.0'], +]); +describeTable('Go %s (%s)', (version: string, goVersion: string, gitRef: string, normVersion: string) => { + const gotip = version == 'tip'; + const cacheDir = gotip ? 'gotip' : 'go'; + const goRoot = path.join(toolDir, cacheDir, normVersion, osarch); + const goTool = path.join(goRoot, 'bin', goExe); - it('Acquires version of go if no matching version is installed', async () => { - await installer.getGo('1.10.8'); - const goDir = path.join(toolDir, 'go', '1.10.8', os.arch()); - - expect(fs.existsSync(`${goDir}.complete`)).toBe(true); - if (IS_WINDOWS) { - expect(fs.existsSync(path.join(goDir, 'bin', 'go.exe'))).toBe(true); - } else { - expect(fs.existsSync(path.join(goDir, 'bin', 'go'))).toBe(true); - } - }, 100000); - - describe('the latest release of a go version', () => { - beforeEach(() => { + let cgo: string = ''; + if (!gotip) { + beforeAll(() => { nock('https://golang.org') .get('/dl/') .query({mode: 'json', include: 'all'}) .replyWithFile(200, path.join(dataDir, 'golang-dl.json')); }); - - afterEach(() => { + afterAll(() => { nock.cleanAll(); nock.enableNetConnect(); }); + } else { + beforeAll(async () => { + cgo = await executil.goEnv('CGO_ENABLED'); + }); + } - it('Acquires latest release version of go 1.10 if using 1.10 and no matching version is installed', async () => { - await installer.getGo('1.10'); - const goDir = path.join(toolDir, 'go', '1.10.8', os.arch()); + const timeout = gotip ? 300000 : 100000; + test('installation', async () => { + const promise = installer.getGo(version, gitRef); + await expect(promise).resolves.toBeUndefined(); + }, timeout); - expect(fs.existsSync(`${goDir}.complete`)).toBe(true); - if (IS_WINDOWS) { - expect(fs.existsSync(path.join(goDir, 'bin', 'go.exe'))).toBe(true); - } else { - expect(fs.existsSync(path.join(goDir, 'bin', 'go'))).toBe(true); - } - }, 100000); - - it('Acquires latest release version of go 1.10 if using 1.10.x and no matching version is installed', async () => { - await installer.getGo('1.10.x'); - const goDir = path.join(toolDir, 'go', '1.10.8', os.arch()); - - expect(fs.existsSync(`${goDir}.complete`)).toBe(true); - if (IS_WINDOWS) { - expect(fs.existsSync(path.join(goDir, 'bin', 'go.exe'))).toBe(true); - } else { - expect(fs.existsSync(path.join(goDir, 'bin', 'go'))).toBe(true); - } - }, 100000); - - it('Acquires latest release version of go if using 1.x and no matching version is installed', async () => { - await installer.getGo('1.x'); - const goDir = path.join(toolDir, 'go', '1.13.0', os.arch()); - - expect(fs.existsSync(`${goDir}.complete`)).toBe(true); - if (IS_WINDOWS) { - expect(fs.existsSync(path.join(goDir, 'bin', 'go.exe'))).toBe(true); - } else { - expect(fs.existsSync(path.join(goDir, 'bin', 'go'))).toBe(true); - } - }, 100000); + test('tool executable check', async () => { + const promise = fs.promises.access(goTool); + await expect(promise).resolves.toBeUndefined(); }); - it('Throws if no location contains correct go version', async () => { - let thrown = false; - try { - await installer.getGo('1000.0'); - } catch { - thrown = true; - } - expect(thrown).toBe(true); + test('cache completeness check', async () => { + const promise = fs.promises.access(`${goRoot}.complete`); + await expect(promise).resolves.toBeUndefined(); }); - it('Uses version of go installed in cache', async () => { - const goDir: string = path.join(toolDir, 'go', '250.0.0', os.arch()); - await io.mkdirP(goDir); - fs.writeFileSync(`${goDir}.complete`, 'hello'); - // This will throw if it doesn't find it in the cache (because no such version exists) - await installer.getGo('250.0'); - return; - }); - - it('Doesnt use version of go that was only partially installed in cache', async () => { - const goDir: string = path.join(toolDir, 'go', '251.0.0', os.arch()); - await io.mkdirP(goDir); - let thrown = false; - try { - // This will throw if it doesn't find it in the cache (because no such version exists) - await installer.getGo('251.0'); - } catch { - thrown = true; - } - expect(thrown).toBe(true); + goVersion = ' ' + goVersion; + if (!gotip) { + goVersion += ' '; + } + test('version check', async () => { + const promise = executil.stdout(goTool, ['version']); + await expect(promise).resolves.toContain(goVersion); + }); + + if (!gotip) { return; + } + test('CGO_ENABLED check', async () => { + const promise = executil.goEnv('CGO_ENABLED', goTool); + await expect(promise).resolves.toBe(cgo); + }); +}); + +describe('installer cache', () => { + const version = '1000.0'; + const normVersion = '1000.0.0'; + const cacheDir = 'go'; + const goRoot = path.join(toolDir, cacheDir, normVersion, osarch); + + test('throws on incorrect version', async () => { + const promise = installer.getGo(version); + await expect(promise).rejects.toThrow(); + }); + + test('throws on partial install', async () => { + await io.mkdirP(goRoot); + + const promise = installer.getGo(version); + await expect(promise).rejects.toThrow(); + }) + + test('uses existing version', async () => { + await fs.promises.writeFile(`${goRoot}.complete`, 'hello'); + + const promise = installer.getGo(version); + await expect(promise).resolves.toBeUndefined(); }); }); diff --git a/src/executil.ts b/src/executil.ts new file mode 100644 index 0000000..a6e325e --- /dev/null +++ b/src/executil.ts @@ -0,0 +1,80 @@ +import * as exec from '@actions/exec'; + +export async function stdout(cmd: string, args: string[]): Promise { + let s: string = ''; + // For some strange reason, exec ignores process.env + // unless we pass it explicitly. + const env: { [key: string]: string } = {}; + for (const [key, value] of Object.entries(process.env)) { + if (value === undefined) { + continue; + } + env[key] = value; + } + await exec.exec(cmd, args, { + env, + silent: true, + listeners: { + stdout: (buf: Buffer) => { + s += buf.toString(); + }, + }, + }); + return s; +} + +export async function git(args: string[], opts?: any) { + let rc: number = await exec.exec('git', args, opts); + if (rc != 0) { + throw new Error(`git: exit code ${rc}`); + } +} + +export async function gitClone(gitDir: string, gitRepo: string, workTree: string, ref: string) { + let args: string[] = [ + 'clone', + `--branch=${ref}`, + `--separate-git-dir=${gitDir}`, + '--depth=1', + '--', + gitRepo, + workTree, + ]; + await git(args); +} + +export async function gitFetch(gitDir: string) { + let args: string[] = [ + `--git-dir=${gitDir}`, + 'fetch', + '--depth=1', + ]; + await git(args); +} + +export async function gitRevParse(gitDir: string, spec: string): Promise { + let args: string[] = [ + `--git-dir=${gitDir}`, + 'rev-parse', + '--verify', + spec, + ]; + let hash = await stdout('git', args); + return hash.trim(); +} + +export async function gitReset(gitDir: string, workTree: string, spec: string) { + let args: string[] = [ + `--git-dir=${gitDir}`, + `--work-tree=${workTree}`, + 'reset', + '--hard', + spec, + ]; + await git(args); +} + +export async function goEnv(envVar: string, goExe: string = 'go'): Promise { + let envVal = await stdout(goExe, ['env', '--', envVar]); + return envVal.trim(); +} diff --git a/src/installer.ts b/src/installer.ts index 9f25476..cbdffa8 100644 --- a/src/installer.ts +++ b/src/installer.ts @@ -1,6 +1,12 @@ +import * as tempdir from './tempdir'; +import * as executil from './executil'; + // Load tempDirectory before it gets wiped by tool-cache -let tempDirectory = process.env['RUNNER_TEMPDIRECTORY'] || ''; +const tempDirectory = tempdir.tempDir(); + import * as core from '@actions/core'; +import * as exec from '@actions/exec'; +import * as io from '@actions/io'; import * as tc from '@actions/tool-cache'; import * as os from 'os'; import * as path from 'path'; @@ -11,84 +17,161 @@ import * as restm from 'typed-rest-client/RestClient'; let osPlat: string = os.platform(); let osArch: string = os.arch(); -if (!tempDirectory) { - let baseLocation; - if (process.platform === 'win32') { - // On windows use the USERPROFILE env variable - baseLocation = process.env['USERPROFILE'] || 'C:\\'; - } else { - if (process.platform === 'darwin') { - baseLocation = '/Users'; - } else { - baseLocation = '/home'; - } - } - tempDirectory = path.join(baseLocation, 'actions', 'temp'); -} - -export async function getGo(version: string) { +export async function getGo(version: string, gotipRef: string = 'master', bootstrapGo: string = 'go') { const selected = await determineVersion(version); if (selected) { version = selected; } - // check cache - let toolPath: string; - toolPath = tc.find('go', normalizeVersion(version)); + let toolPath = await acquireGo(version, gotipRef, bootstrapGo); + core.debug(`Using Go toolchain under ${toolPath}`); - if (!toolPath) { - // download, extract, cache - toolPath = await acquireGo(version); - core.debug('Go tool is cached under ' + toolPath); - } - - setGoEnvironmentVariables(toolPath); - - toolPath = path.join(toolPath, 'bin'); - // - // prepend the tools path. instructs the agent to prepend for future tasks - // - core.addPath(toolPath); + await setGoEnvironmentVariables(version, toolPath, bootstrapGo); } -async function acquireGo(version: string): Promise { - // - // Download - a tool installer intimately knows how to get the tool (and construct urls) - // - let fileName: string = getFileName(version); - let downloadUrl: string = getDownloadUrl(fileName); +async function acquireGo(version: string, gotipRef: string, bootstrapGo: string): Promise { + const normVersion = normalizeVersion(version); - core.debug('Downloading Go from: ' + downloadUrl); - - let downloadPath: string | null = null; - try { - downloadPath = await tc.downloadTool(downloadUrl); - } catch (error) { - core.debug(error); - - throw `Failed to download version ${version}: ${error}`; + // Let tool-cache fail when we are not using tip. + // Otherwise throw an error, because we don’t + // have a temporary directory for Git clone. + let extPath = tempDirectory; + if (version == 'gotip') { + if (!extPath) { + throw new Error('Temp directory not set'); + } } - // - // Extract - // - let extPath: string = tempDirectory; - if (!extPath) { - throw new Error('Temp directory not set'); + // We are doing incremental updates/fetch for tip, + // this check is a fast path for stable releases. + if (version != 'tip') { + const toolRootCache = tc.find('go', normVersion); + if (toolRootCache) { + return toolRootCache; + } } - if (osPlat == 'win32') { - extPath = await tc.extractZip(downloadPath); + // This will give us archive name and URL for releases, + // and Git work tree dir name and clone URL and for tip. + const filename = getFileName(version); + const downloadUrl = getDownloadUrl(version, filename); + + // Extract release builds. In case of tip, build from source. + let toolRoot: string; + if (version == 'tip') { + let gitDir: string; + let workTree: string; + let commitHash: string; + // Avoid cloning multiple times by caching git dir. + // Empty string means that we don’t care about arch. + gitDir = tc.find('gotip', 'master', '') + if (!gitDir) { + gitDir = path.join(extPath, 'gotip.git'); + workTree = path.join(extPath, filename); + + // Clone repo with separate git dir. + await executil.gitClone(gitDir, downloadUrl, workTree, gotipRef); + + // Extract current commit hash. + commitHash = await executil.gitRevParse(gitDir, 'HEAD'); + } else { + // We don’t have a work tree (yet) in this case. + workTree = ''; + + // Cache hit for git dir, fetch new commits from upstream. + await executil.gitFetch(gitDir); + + // Extract latest commit hash. + commitHash = await executil.gitRevParse(gitDir, 'FETCH_HEAD'); + } + // Update cache for git dir. + gitDir = await tc.cacheDir(gitDir, 'gotip', 'master', ''); + + // Avoid building multiple times by caching work tree. + let workTreeCache = tc.find('gotip', commitHash); + if (workTreeCache) { + workTree = workTreeCache; + } else { + if (!workTree) { + // We found gotip.git in cache, but not the work tree. + // + workTree = path.join(extPath, filename); + // Work tree must exist, otherwise Git will complain + // that “this operation must be run in a work tree”. + await io.mkdirP(workTree); + + // Hard reset to the latest commit. + await executil.gitReset(gitDir, workTree, commitHash); + } + + // The make.bat script on Windows is not smart enough + // to figure out the path to bootstrap Go toolchain. + // Make script will show descriptive error message even + // if we don’t find Go installation on the host. + let bootstrap: string = ''; + if (bootstrapGo) { + bootstrap = await executil.goEnv('GOROOT', bootstrapGo); + } + + // Build gotip from source. + const cwd = path.join(workTree, 'src'); + const env = { + 'GOROOT_BOOTSTRAP': bootstrap, + ...process.env, + // Note that while we disable Cgo for tip builds, it does + // not disable Cgo entirely. Moreover, we override this + // value in setGoEnvironmentVariables with whatever the + // bootstrap toolchain uses. This way we can get reproducible + // builds without disrupting normal workflows. + 'CGO_ENABLED': '0', + // Cherry on the cake for completely reproducible builds. + // The default value depends on the build directory, but + // is easily overriden with GOROOT environment variable + // at runtime. Since we already export GOROOT by default, + // and assume paths would differ between CI runs, we set + // this to the value Go uses when -trimpath flag is set. + // See https://go.googlesource.com/go/+/refs/tags/go1.13/src/cmd/go/internal/work/gc.go#553 + 'GOROOT_FINAL': 'go', + } + let cmd: string; + if (osPlat != 'win32') { + cmd = 'bash make.bash'; + } else { + cmd = 'make.bat'; + } + await exec.exec(cmd, undefined, { cwd, env }); + // Update cache for work tree. + workTree = await tc.cacheDir(workTree, 'gotip', commitHash); + } + toolRoot = workTree; } else { - extPath = await tc.extractTar(downloadPath); + let downloadPath: string; + try { + core.debug(`Downloading Go from: ${downloadUrl}`); + downloadPath = await tc.downloadTool(downloadUrl); + } catch (error) { + core.debug(error); + throw new Error(`Failed to download version ${version}: ${error}`); + } + // Extract downloaded archive. Note that node extracts + // with a root folder that matches the filename downloaded. + if (osPlat == 'win32') { + extPath = await tc.extractZip(downloadPath); + } else { + extPath = await tc.extractTar(downloadPath); + } + // Add Go to the cache. + toolRoot = path.join(extPath, 'go'); + toolRoot = await tc.cacheDir(toolRoot, 'go', normVersion); } + return toolRoot; +} - // - // Install into the local tool cache - node extracts with a root folder that matches the fileName downloaded - // - const toolRoot = path.join(extPath, 'go'); - version = normalizeVersion(version); - return await tc.cacheDir(toolRoot, 'go', version); +function getDownloadUrl(version: string, filename: string): string { + if (version == 'tip') { + return 'https://go.googlesource.com/go'; + } + return util.format('https://storage.googleapis.com/golang/%s', filename); } function getFileName(version: string): string { @@ -101,9 +184,17 @@ function getFileName(version: string): string { const platform: string = osPlat == 'win32' ? 'windows' : osPlat; const arch: string = arches[osArch] || arches['default']; - const ext: string = osPlat == 'win32' ? 'zip' : 'tar.gz'; + let ext: string; + if (version == 'tip') { + // Git work tree for tip builds does not have an externsion. + ext = ''; + } else if (osPlat == 'win32') { + ext = '.zip'; + } else { + ext = '.tar.gz' + } const filename: string = util.format( - 'go%s.%s-%s.%s', + 'go%s.%s-%s%s', version, platform, arch, @@ -113,11 +204,15 @@ function getFileName(version: string): string { return filename; } -function getDownloadUrl(filename: string): string { - return util.format('https://storage.googleapis.com/golang/%s', filename); -} +async function setGoEnvironmentVariables(version: string, goRoot: string, bootstrapGo: string) { + if (version == 'tip') { + // We build tip with CGO_ENABLED=0, but that could be confusing + // if the bootstrap toolchain uses CGO_ENABLED=1 by default. So + // we re-export this value for the tip toolchain. + const cgo = await executil.goEnv('CGO_ENABLED', bootstrapGo); + core.exportVariable('CGO_ENABLED', cgo); + } -function setGoEnvironmentVariables(goRoot: string) { core.exportVariable('GOROOT', goRoot); const goPath: string = process.env['GOPATH'] || ''; @@ -130,12 +225,19 @@ function setGoEnvironmentVariables(goRoot: string) { if (goBin) { core.exportVariable('GOBIN', goBin); } + + let goRootBin = path.join(goRoot, 'bin'); + core.addPath(goRootBin); } // This function is required to convert the version 1.10 to 1.10.0. // Because caching utility accept only sementic version, // which have patch number as well. function normalizeVersion(version: string): string { + if (version == 'tip') { + return version; + } + const versionPart = version.split('.'); if (versionPart[1] == null) { //append minor and patch version if not available @@ -167,16 +269,15 @@ function normalizeVersion(version: string): string { } async function determineVersion(version: string): Promise { - if (!version.endsWith('.x')) { - const versionPart = version.split('.'); - - if (versionPart[1] == null || versionPart[2] == null) { - return await getLatestVersion(version.concat('.x')); - } else { - return version; - } + if (version == 'tip') { + return version; + } + if (version == 'latest') { + return await getLatestVersion(''); + } + if (!version.endsWith('.x')) { + return version; } - return await getLatestVersion(version); } @@ -188,7 +289,7 @@ async function getLatestVersion(version: string): Promise { core.debug(`evaluating ${versions.length} versions`); - if (version.length === 0) { + if (versions.length === 0) { throw new Error('unable to get latest version'); } diff --git a/src/tempdir.ts b/src/tempdir.ts new file mode 100644 index 0000000..998495b --- /dev/null +++ b/src/tempdir.ts @@ -0,0 +1,26 @@ +// See https://github.com/actions/toolkit/blob/17acd9c66fb3dd360ef8c65fcd7c3b864064b5c7/packages/tool-cache/src/tool-cache.ts#L20-L45 + +import * as path from 'path'; +import * as io from '@actions/io'; + +export function tempDir(): string { + let tempDirectory: string = process.env['RUNNER_TEMP'] || ''; + if (!tempDirectory) { + let baseLocation: string; + if (process.platform === 'win32') { + // On windows use the USERPROFILE env variable. + baseLocation = process.env['USERPROFILE'] || 'C:\\'; + } else { + if (process.platform === 'darwin') { + baseLocation = '/Users'; + } else { + baseLocation = '/home'; + } + } + if (!tempDirectory) { + tempDirectory = path.join(baseLocation, 'actions', 'temp'); + } + } + io.mkdirP(tempDirectory); + return tempDirectory; +}