Add support for tip and latest aliases

.github/workflows/workflow.yml

@@ -5,6 +5,7 @@ jobs:
     name: Run
     runs-on: ${{ matrix.operating-system }}
     strategy:
+      fail-fast: false
       matrix:
         operating-system: [ubuntu-latest, windows-latest]
     steps:
@@ -14,13 +15,14 @@ jobs:
     - name: Set Node.js 10.x
       uses: actions/setup-node@master
       with:
-        version: 10.x
+        node-version: 10.x
 
     - name: npm install
       run: npm install
 
-    - name: Lint
-      run: npm run format-check
-
     - name: npm test
       run: npm test
+
+    - name: Lint
+      run: npm run format-check
+      continue-on-error: true

__tests__/installer.test.ts

@@ -11,116 +11,107 @@ const dataDir = path.join(__dirname, 'data');
 process.env['RUNNER_TOOL_CACHE'] = toolDir;
 process.env['RUNNER_TEMP'] = tempDir;
 import * as installer from '../src/installer';
+import * as executil from '../src/executil';
 
+const osarch = os.arch();
 const IS_WINDOWS = process.platform === 'win32';
+const goExe: string = IS_WINDOWS ? 'go.exe' : 'go';
 
-describe('installer tests', () => {
-  beforeAll(async () => {
-    await io.rmRF(toolDir);
-    await io.rmRF(tempDir);
-  }, 100000);
+const cleanup = async () => {
+  await io.rmRF(toolDir);
+  await io.rmRF(tempDir);
+}
+beforeAll(cleanup, 100000);
+afterAll(cleanup, 100000);
 
-  afterAll(async () => {
-    try {
-      await io.rmRF(toolDir);
-      await io.rmRF(tempDir);
-    } catch {
-      console.log('Failed to remove test directories');
-    }
-  }, 100000);
+const describeTable = describe.each([
+  ['tip',    '+a5bfd9d', 'go1.14beta1', 'a5bfd9da1d1b24f326399b6b75558ded14514f23'],
+  ['latest', 'go1.13',   'n/a',         '1.13.0'],
+  ['1.x',    'go1.13',   'n/a',         '1.13.0'],
+  ['1.10.x', 'go1.10.8', 'n/a',         '1.10.8'],
+  ['1.10.8', 'go1.10.8', 'n/a',         '1.10.8'],
+  ['1.10',   'go1.10',   'n/a',         '1.10.0'],
+]);
+describeTable('Go %s (%s)', (version: string, goVersion: string, gitRef: string, normVersion: string) => {
+  const gotip = version == 'tip';
+  const cacheDir = gotip ? 'gotip' : 'go';
+  const goRoot = path.join(toolDir, cacheDir, normVersion, osarch);
+  const goTool = path.join(goRoot, 'bin', goExe);
 
-  it('Acquires version of go if no matching version is installed', async () => {
-    await installer.getGo('1.10.8');
-    const goDir = path.join(toolDir, 'go', '1.10.8', os.arch());
-
-    expect(fs.existsSync(`${goDir}.complete`)).toBe(true);
-    if (IS_WINDOWS) {
-      expect(fs.existsSync(path.join(goDir, 'bin', 'go.exe'))).toBe(true);
-    } else {
-      expect(fs.existsSync(path.join(goDir, 'bin', 'go'))).toBe(true);
-    }
-  }, 100000);
-
-  describe('the latest release of a go version', () => {
-    beforeEach(() => {
+  let cgo: string = '';
+  if (!gotip) {
+    beforeAll(() => {
       nock('https://golang.org')
         .get('/dl/')
         .query({mode: 'json', include: 'all'})
         .replyWithFile(200, path.join(dataDir, 'golang-dl.json'));
     });
-
-    afterEach(() => {
+    afterAll(() => {
       nock.cleanAll();
       nock.enableNetConnect();
     });
+  } else {
+    beforeAll(async () => {
+      cgo = await executil.goEnv('CGO_ENABLED');
+    });
+  }
 
-    it('Acquires latest release version of go 1.10 if using 1.10 and no matching version is installed', async () => {
-      await installer.getGo('1.10');
-      const goDir = path.join(toolDir, 'go', '1.10.8', os.arch());
+  const timeout = gotip ? 300000 : 100000;
+  test('installation', async () => {
+    const promise = installer.getGo(version, gitRef);
+    await expect(promise).resolves.toBeUndefined();
+  }, timeout);
 
-      expect(fs.existsSync(`${goDir}.complete`)).toBe(true);
-      if (IS_WINDOWS) {
-        expect(fs.existsSync(path.join(goDir, 'bin', 'go.exe'))).toBe(true);
-      } else {
-        expect(fs.existsSync(path.join(goDir, 'bin', 'go'))).toBe(true);
-      }
-    }, 100000);
-
-    it('Acquires latest release version of go 1.10 if using 1.10.x and no matching version is installed', async () => {
-      await installer.getGo('1.10.x');
-      const goDir = path.join(toolDir, 'go', '1.10.8', os.arch());
-
-      expect(fs.existsSync(`${goDir}.complete`)).toBe(true);
-      if (IS_WINDOWS) {
-        expect(fs.existsSync(path.join(goDir, 'bin', 'go.exe'))).toBe(true);
-      } else {
-        expect(fs.existsSync(path.join(goDir, 'bin', 'go'))).toBe(true);
-      }
-    }, 100000);
-
-    it('Acquires latest release version of go if using 1.x and no matching version is installed', async () => {
-      await installer.getGo('1.x');
-      const goDir = path.join(toolDir, 'go', '1.13.0', os.arch());
-
-      expect(fs.existsSync(`${goDir}.complete`)).toBe(true);
-      if (IS_WINDOWS) {
-        expect(fs.existsSync(path.join(goDir, 'bin', 'go.exe'))).toBe(true);
-      } else {
-        expect(fs.existsSync(path.join(goDir, 'bin', 'go'))).toBe(true);
-      }
-    }, 100000);
+  test('tool executable check', async () => {
+    const promise = fs.promises.access(goTool);
+    await expect(promise).resolves.toBeUndefined();
   });
 
-  it('Throws if no location contains correct go version', async () => {
-    let thrown = false;
-    try {
-      await installer.getGo('1000.0');
-    } catch {
-      thrown = true;
-    }
-    expect(thrown).toBe(true);
+  test('cache completeness check', async () => {
+    const promise = fs.promises.access(`${goRoot}.complete`);
+    await expect(promise).resolves.toBeUndefined();
   });
 
-  it('Uses version of go installed in cache', async () => {
-    const goDir: string = path.join(toolDir, 'go', '250.0.0', os.arch());
-    await io.mkdirP(goDir);
-    fs.writeFileSync(`${goDir}.complete`, 'hello');
-    // This will throw if it doesn't find it in the cache (because no such version exists)
-    await installer.getGo('250.0');
-    return;
-  });
-
-  it('Doesnt use version of go that was only partially installed in cache', async () => {
-    const goDir: string = path.join(toolDir, 'go', '251.0.0', os.arch());
-    await io.mkdirP(goDir);
-    let thrown = false;
-    try {
-      // This will throw if it doesn't find it in the cache (because no such version exists)
-      await installer.getGo('251.0');
-    } catch {
-      thrown = true;
-    }
-    expect(thrown).toBe(true);
+  goVersion = ' ' + goVersion;
+  if (!gotip) {
+    goVersion += ' ';
+  }
+  test('version check', async () => {
+    const promise = executil.stdout(goTool, ['version']);
+    await expect(promise).resolves.toContain(goVersion);
+  });
+
+  if (!gotip) {
     return;
+  }
+  test('CGO_ENABLED check', async () => {
+    const promise = executil.goEnv('CGO_ENABLED', goTool);
+    await expect(promise).resolves.toBe(cgo);
+  });
+});
+
+describe('installer cache', () => {
+  const version = '1000.0';
+  const normVersion = '1000.0.0';
+  const cacheDir = 'go';
+  const goRoot = path.join(toolDir, cacheDir, normVersion, osarch);
+
+  test('throws on incorrect version', async () => {
+    const promise = installer.getGo(version);
+    await expect(promise).rejects.toThrow();
+  });
+
+  test('throws on partial install', async () => {
+    await io.mkdirP(goRoot);
+
+    const promise = installer.getGo(version);
+    await expect(promise).rejects.toThrow();
+  })
+
+  test('uses existing version', async () => {
+    await fs.promises.writeFile(`${goRoot}.complete`, 'hello');
+
+    const promise = installer.getGo(version);
+    await expect(promise).resolves.toBeUndefined();
   });
 });

src/executil.ts

@@ -0,0 +1,80 @@
+import * as exec from '@actions/exec';
+
+export async function stdout(cmd: string, args: string[]): Promise<string> {
+  let s: string = '';
+  // For some strange reason, exec ignores process.env
+  // unless we pass it explicitly.
+  const env: { [key: string]: string } = {};
+  for (const [key, value] of Object.entries(process.env)) {
+    if (value === undefined) {
+      continue;
+    }
+    env[key] = value;
+  }
+  await exec.exec(cmd, args, {
+    env,
+    silent: true,
+    listeners: {
+      stdout: (buf: Buffer) => {
+        s += buf.toString();
+      },
+    },
+  });
+  return s;
+}
+
+export async function git(args: string[], opts?: any) {
+  let rc: number = await exec.exec('git', args, opts);
+  if (rc != 0) {
+    throw new Error(`git: exit code ${rc}`);
+  }
+}
+
+export async function gitClone(gitDir: string, gitRepo: string, workTree: string, ref: string) {
+  let args: string[] = [
+    'clone',
+    `--branch=${ref}`,
+    `--separate-git-dir=${gitDir}`,
+    '--depth=1',
+    '--',
+    gitRepo,
+    workTree,
+  ];
+  await git(args);
+}
+
+export async function gitFetch(gitDir: string) {
+  let args: string[] = [
+    `--git-dir=${gitDir}`,
+    'fetch',
+    '--depth=1',
+  ];
+  await git(args);
+}
+
+export async function gitRevParse(gitDir: string, spec: string): Promise<string> {
+  let args: string[] = [
+    `--git-dir=${gitDir}`,
+    'rev-parse',
+    '--verify',
+    spec,
+  ];
+  let hash = await stdout('git', args);
+  return hash.trim();
+}
+
+export async function gitReset(gitDir: string, workTree: string, spec: string) {
+  let args: string[] = [
+    `--git-dir=${gitDir}`,
+    `--work-tree=${workTree}`,
+    'reset',
+    '--hard',
+    spec,
+  ];
+  await git(args);
+}
+
+export async function goEnv(envVar: string, goExe: string = 'go'): Promise<string> {
+  let envVal = await stdout(goExe, ['env', '--', envVar]);
+  return envVal.trim();
+}

src/installer.ts

@@ -1,6 +1,12 @@
+import * as tempdir from './tempdir';
+import * as executil from './executil';
+
 // Load tempDirectory before it gets wiped by tool-cache
-let tempDirectory = process.env['RUNNER_TEMPDIRECTORY'] || '';
+const tempDirectory = tempdir.tempDir();
+
 import * as core from '@actions/core';
+import * as exec from '@actions/exec';
+import * as io from '@actions/io';
 import * as tc from '@actions/tool-cache';
 import * as os from 'os';
 import * as path from 'path';
@@ -11,84 +17,161 @@ import * as restm from 'typed-rest-client/RestClient';
 let osPlat: string = os.platform();
 let osArch: string = os.arch();
 
-if (!tempDirectory) {
-  let baseLocation;
-  if (process.platform === 'win32') {
-    // On windows use the USERPROFILE env variable
-    baseLocation = process.env['USERPROFILE'] || 'C:\\';
-  } else {
-    if (process.platform === 'darwin') {
-      baseLocation = '/Users';
-    } else {
-      baseLocation = '/home';
-    }
-  }
-  tempDirectory = path.join(baseLocation, 'actions', 'temp');
-}
-
-export async function getGo(version: string) {
+export async function getGo(version: string, gotipRef: string = 'master', bootstrapGo: string = 'go') {
   const selected = await determineVersion(version);
   if (selected) {
     version = selected;
   }
 
-  // check cache
-  let toolPath: string;
-  toolPath = tc.find('go', normalizeVersion(version));
+  let toolPath = await acquireGo(version, gotipRef, bootstrapGo);
+  core.debug(`Using Go toolchain under ${toolPath}`);
 
-  if (!toolPath) {
-    // download, extract, cache
-    toolPath = await acquireGo(version);
-    core.debug('Go tool is cached under ' + toolPath);
-  }
-
-  setGoEnvironmentVariables(toolPath);
-
-  toolPath = path.join(toolPath, 'bin');
-  //
-  // prepend the tools path. instructs the agent to prepend for future tasks
-  //
-  core.addPath(toolPath);
+  await setGoEnvironmentVariables(version, toolPath, bootstrapGo);
 }
 
-async function acquireGo(version: string): Promise<string> {
-  //
-  // Download - a tool installer intimately knows how to get the tool (and construct urls)
-  //
-  let fileName: string = getFileName(version);
-  let downloadUrl: string = getDownloadUrl(fileName);
+async function acquireGo(version: string, gotipRef: string, bootstrapGo: string): Promise<string> {
+  const normVersion = normalizeVersion(version);
 
-  core.debug('Downloading Go from: ' + downloadUrl);
-
-  let downloadPath: string | null = null;
-  try {
-    downloadPath = await tc.downloadTool(downloadUrl);
-  } catch (error) {
-    core.debug(error);
-
-    throw `Failed to download version ${version}: ${error}`;
+  // Let tool-cache fail when we are not using tip.
+  // Otherwise throw an error, because we don’t
+  // have a temporary directory for Git clone.
+  let extPath = tempDirectory;
+  if (version == 'gotip') {
+    if (!extPath) {
+      throw new Error('Temp directory not set');
+    }
   }
 
-  //
-  // Extract
-  //
-  let extPath: string = tempDirectory;
-  if (!extPath) {
-    throw new Error('Temp directory not set');
+  // We are doing incremental updates/fetch for tip,
+  // this check is a fast path for stable releases.
+  if (version != 'tip') {
+    const toolRootCache = tc.find('go', normVersion);
+    if (toolRootCache) {
+      return toolRootCache;
+    }
   }
 
-  if (osPlat == 'win32') {
-    extPath = await tc.extractZip(downloadPath);
+  // This will give us archive name and URL for releases,
+  // and Git work tree dir name and clone URL and for tip.
+  const filename = getFileName(version);
+  const downloadUrl = getDownloadUrl(version, filename);
+
+  // Extract release builds. In case of tip, build from source.
+  let toolRoot: string;
+  if (version == 'tip') {
+      let gitDir: string;
+      let workTree: string;
+      let commitHash: string;
+      // Avoid cloning multiple times by caching git dir.
+      // Empty string means that we don’t care about arch.
+      gitDir = tc.find('gotip', 'master', '')
+      if (!gitDir) {
+        gitDir = path.join(extPath, 'gotip.git');
+        workTree = path.join(extPath, filename);
+
+        // Clone repo with separate git dir.
+        await executil.gitClone(gitDir, downloadUrl, workTree, gotipRef);
+
+        // Extract current commit hash.
+        commitHash = await executil.gitRevParse(gitDir, 'HEAD');
+      } else {
+        // We don’t have a work tree (yet) in this case.
+        workTree = '';
+
+        // Cache hit for git dir, fetch new commits from upstream.
+        await executil.gitFetch(gitDir);
+
+        // Extract latest commit hash.
+        commitHash = await executil.gitRevParse(gitDir, 'FETCH_HEAD');
+      }
+      // Update cache for git dir.
+      gitDir = await tc.cacheDir(gitDir, 'gotip', 'master', '');
+
+      // Avoid building multiple times by caching work tree.
+      let workTreeCache = tc.find('gotip', commitHash);
+      if (workTreeCache) {
+        workTree = workTreeCache;
+      } else {
+        if (!workTree) {
+          // We found gotip.git in cache, but not the work tree.
+          //
+          workTree = path.join(extPath, filename);
+          // Work tree must exist, otherwise Git will complain
+          // that “this operation must be run in a work tree”.
+          await io.mkdirP(workTree);
+
+          // Hard reset to the latest commit.
+          await executil.gitReset(gitDir, workTree, commitHash);
+        }
+
+        // The make.bat script on Windows is not smart enough
+        // to figure out the path to bootstrap Go toolchain.
+        // Make script will show descriptive error message even
+        // if we don’t find Go installation on the host.
+        let bootstrap: string = '';
+        if (bootstrapGo) {
+          bootstrap = await executil.goEnv('GOROOT', bootstrapGo);
+        }
+
+        // Build gotip from source.
+        const cwd = path.join(workTree, 'src');
+        const env = {
+          'GOROOT_BOOTSTRAP': bootstrap,
+          ...process.env,
+          // Note that while we disable Cgo for tip builds, it does
+          // not disable Cgo entirely. Moreover, we override this
+          // value in setGoEnvironmentVariables with whatever the
+          // bootstrap toolchain uses. This way we can get reproducible
+          // builds without disrupting normal workflows.
+          'CGO_ENABLED': '0',
+          // Cherry on the cake for completely reproducible builds.
+          // The default value depends on the build directory, but
+          // is easily overriden with GOROOT environment variable
+          // at runtime. Since we already export GOROOT by default,
+          // and assume paths would differ between CI runs, we set
+          // this to the value Go uses when -trimpath flag is set.
+          // See https://go.googlesource.com/go/+/refs/tags/go1.13/src/cmd/go/internal/work/gc.go#553
+          'GOROOT_FINAL': 'go',
+        }
+        let cmd: string;
+        if (osPlat != 'win32') {
+          cmd = 'bash make.bash';
+        } else {
+          cmd = 'make.bat';
+        }
+        await exec.exec(cmd, undefined, { cwd, env });
+        // Update cache for work tree.
+        workTree = await tc.cacheDir(workTree, 'gotip', commitHash);
+      }
+      toolRoot = workTree;
   } else {
-    extPath = await tc.extractTar(downloadPath);
+    let downloadPath: string;
+    try {
+      core.debug(`Downloading Go from: ${downloadUrl}`);
+      downloadPath = await tc.downloadTool(downloadUrl);
+    } catch (error) {
+      core.debug(error);
+      throw new Error(`Failed to download version ${version}: ${error}`);
+    }
+    // Extract downloaded archive. Note that node extracts
+    // with a root folder that matches the filename downloaded.
+    if (osPlat == 'win32') {
+      extPath = await tc.extractZip(downloadPath);
+    } else {
+      extPath = await tc.extractTar(downloadPath);
+    }
+    // Add Go to the cache.
+    toolRoot = path.join(extPath, 'go');
+    toolRoot = await tc.cacheDir(toolRoot, 'go', normVersion);
   }
+  return toolRoot;
+}
 
-  //
-  // Install into the local tool cache - node extracts with a root folder that matches the fileName downloaded
-  //
-  const toolRoot = path.join(extPath, 'go');
-  version = normalizeVersion(version);
-  return await tc.cacheDir(toolRoot, 'go', version);
+function getDownloadUrl(version: string, filename: string): string {
+  if (version == 'tip') {
+    return 'https://go.googlesource.com/go';
+  }
+  return util.format('https://storage.googleapis.com/golang/%s', filename);
 }
 
 function getFileName(version: string): string {
@@ -101,9 +184,17 @@ function getFileName(version: string): string {
 
   const platform: string = osPlat == 'win32' ? 'windows' : osPlat;
   const arch: string = arches[osArch] || arches['default'];
-  const ext: string = osPlat == 'win32' ? 'zip' : 'tar.gz';
+  let ext: string;
+  if (version == 'tip') {
+    // Git work tree for tip builds does not have an externsion.
+    ext = '';
+  } else if (osPlat == 'win32') {
+    ext = '.zip';
+  } else {
+    ext = '.tar.gz'
+  }
   const filename: string = util.format(
-    'go%s.%s-%s.%s',
+    'go%s.%s-%s%s',
     version,
     platform,
     arch,
@@ -113,11 +204,15 @@ function getFileName(version: string): string {
   return filename;
 }
 
-function getDownloadUrl(filename: string): string {
-  return util.format('https://storage.googleapis.com/golang/%s', filename);
-}
+async function setGoEnvironmentVariables(version: string, goRoot: string, bootstrapGo: string) {
+  if (version == 'tip') {
+    // We build tip with CGO_ENABLED=0, but that could be confusing
+    // if the bootstrap toolchain uses CGO_ENABLED=1 by default. So
+    // we re-export this value for the tip toolchain.
+    const cgo = await executil.goEnv('CGO_ENABLED', bootstrapGo);
+    core.exportVariable('CGO_ENABLED', cgo);
+  }
 
-function setGoEnvironmentVariables(goRoot: string) {
   core.exportVariable('GOROOT', goRoot);
 
   const goPath: string = process.env['GOPATH'] || '';
@@ -130,12 +225,19 @@ function setGoEnvironmentVariables(goRoot: string) {
   if (goBin) {
     core.exportVariable('GOBIN', goBin);
   }
+
+  let goRootBin = path.join(goRoot, 'bin');
+  core.addPath(goRootBin);
 }
 
 // This function is required to convert the version 1.10 to 1.10.0.
 // Because caching utility accept only sementic version,
 // which have patch number as well.
 function normalizeVersion(version: string): string {
+  if (version == 'tip') {
+    return version;
+  }
+
   const versionPart = version.split('.');
   if (versionPart[1] == null) {
     //append minor and patch version if not available
@@ -167,16 +269,15 @@ function normalizeVersion(version: string): string {
 }
 
 async function determineVersion(version: string): Promise<string> {
-  if (!version.endsWith('.x')) {
-    const versionPart = version.split('.');
-
-    if (versionPart[1] == null || versionPart[2] == null) {
-      return await getLatestVersion(version.concat('.x'));
-    } else {
-      return version;
-    }
+  if (version == 'tip') {
+    return version;
+  }
+  if (version == 'latest') {
+    return await getLatestVersion('');
+  }
+  if (!version.endsWith('.x')) {
+    return version;
   }
-
   return await getLatestVersion(version);
 }
 
@@ -188,7 +289,7 @@ async function getLatestVersion(version: string): Promise<string> {
 
   core.debug(`evaluating ${versions.length} versions`);
 
-  if (version.length === 0) {
+  if (versions.length === 0) {
     throw new Error('unable to get latest version');
   }
 

src/tempdir.ts

@@ -0,0 +1,26 @@
+// See https://github.com/actions/toolkit/blob/17acd9c66fb3dd360ef8c65fcd7c3b864064b5c7/packages/tool-cache/src/tool-cache.ts#L20-L45
+
+import * as path from 'path';
+import * as io from '@actions/io';
+
+export function tempDir(): string {
+  let tempDirectory: string = process.env['RUNNER_TEMP'] || '';
+  if (!tempDirectory) {
+    let baseLocation: string;
+    if (process.platform === 'win32') {
+      // On windows use the USERPROFILE env variable.
+      baseLocation = process.env['USERPROFILE'] || 'C:\\';
+    } else {
+      if (process.platform === 'darwin') {
+        baseLocation = '/Users';
+      } else {
+        baseLocation = '/home';
+      }
+    }
+    if (!tempDirectory) {
+      tempDirectory = path.join(baseLocation, 'actions', 'temp');
+    }
+  }
+  io.mkdirP(tempDirectory);
+  return tempDirectory;
+}