fallback to REST API to download repo (#104)

src/git-command-manager.ts

@@ -3,8 +3,13 @@ import * as exec from '@actions/exec'
 import * as fshelper from './fs-helper'
 import * as io from '@actions/io'
 import * as path from 'path'
+import * as retryHelper from './retry-helper'
 import {GitVersion} from './git-version'
 
+// Auth header not supported before 2.9
+// Wire protocol v2 not supported before 2.18
+export const MinimumGitVersion = new GitVersion('2.18')
+
 export interface IGitCommandManager {
   branchDelete(remote: boolean, branch: string): Promise<void>
   branchExists(remote: boolean, pattern: string): Promise<boolean>
@@ -150,22 +155,10 @@ class GitCommandManager {
       args.push(arg)
     }
 
-    let attempt = 1
-    const maxAttempts = 3
-    while (attempt <= maxAttempts) {
-      const allowAllExitCodes = attempt < maxAttempts
-      const output = await this.execGit(args, allowAllExitCodes)
-      if (output.exitCode === 0) {
-        break
-      }
-
-      const seconds = this.getRandomIntInclusive(1, 10)
-      core.warning(
-        `Git fetch failed with exit code ${output.exitCode}. Waiting ${seconds} seconds before trying again.`
-      )
-      await this.sleep(seconds * 1000)
-      attempt++
-    }
+    const that = this
+    await retryHelper.execute(async () => {
+      await that.execGit(args)
+    })
   }
 
   getWorkingDirectory(): string {
@@ -188,22 +181,10 @@ class GitCommandManager {
   async lfsFetch(ref: string): Promise<void> {
     const args = ['lfs', 'fetch', 'origin', ref]
 
-    let attempt = 1
-    const maxAttempts = 3
-    while (attempt <= maxAttempts) {
-      const allowAllExitCodes = attempt < maxAttempts
-      const output = await this.execGit(args, allowAllExitCodes)
-      if (output.exitCode === 0) {
-        break
-      }
-
-      const seconds = this.getRandomIntInclusive(1, 10)
-      core.warning(
-        `Git lfs fetch failed with exit code ${output.exitCode}. Waiting ${seconds} seconds before trying again.`
-      )
-      await this.sleep(seconds * 1000)
-      attempt++
-    }
+    const that = this
+    await retryHelper.execute(async () => {
+      await that.execGit(args)
+    })
   }
 
   async lfsInstall(): Promise<void> {
@@ -338,13 +319,9 @@ class GitCommandManager {
     }
 
     // Minimum git version
-    // Note:
-    // - Auth header not supported before 2.9
-    // - Wire protocol v2 not supported before 2.18
-    const minimumGitVersion = new GitVersion('2.18')
-    if (!gitVersion.checkMinimum(minimumGitVersion)) {
+    if (!gitVersion.checkMinimum(MinimumGitVersion)) {
       throw new Error(
-        `Minimum required git version is ${minimumGitVersion}. Your git ('${this.gitPath}') is ${gitVersion}`
+        `Minimum required git version is ${MinimumGitVersion}. Your git ('${this.gitPath}') is ${gitVersion}`
       )
     }
 
@@ -381,16 +358,6 @@ class GitCommandManager {
     core.debug(`Set git useragent to: ${gitHttpUserAgent}`)
     this.gitEnv['GIT_HTTP_USER_AGENT'] = gitHttpUserAgent
   }
-
-  private getRandomIntInclusive(minimum: number, maximum: number): number {
-    minimum = Math.floor(minimum)
-    maximum = Math.floor(maximum)
-    return Math.floor(Math.random() * (maximum - minimum + 1)) + minimum
-  }
-
-  private async sleep(milliseconds): Promise<void> {
-    return new Promise(resolve => setTimeout(resolve, milliseconds))
-  }
 }
 
 class GitOutput {

src/git-source-provider.ts

@@ -3,9 +3,11 @@ import * as coreCommand from '@actions/core/lib/command'
 import * as fs from 'fs'
 import * as fsHelper from './fs-helper'
 import * as gitCommandManager from './git-command-manager'
+import * as githubApiHelper from './github-api-helper'
 import * as io from '@actions/io'
 import * as path from 'path'
 import * as refHelper from './ref-helper'
+import * as stateHelper from './state-helper'
 import {IGitCommandManager} from './git-command-manager'
 
 const authConfigKey = `http.https://github.com/.extraheader`
@@ -23,6 +25,7 @@ export interface ISourceSettings {
 }
 
 export async function getSource(settings: ISourceSettings): Promise<void> {
+  // Repository URL
   core.info(
     `Syncing repository: ${settings.repositoryOwner}/${settings.repositoryName}`
   )
@@ -43,92 +46,92 @@ export async function getSource(settings: ISourceSettings): Promise<void> {
   }
 
   // Git command manager
-  core.info(`Working directory is '${settings.repositoryPath}'`)
-  const git = await gitCommandManager.CreateCommandManager(
-    settings.repositoryPath,
-    settings.lfs
-  )
+  const git = await getGitCommandManager(settings)
 
-  // Try prepare existing directory, otherwise recreate
-  if (
-    isExisting &&
-    !(await tryPrepareExistingDirectory(
+  // Prepare existing directory, otherwise recreate
+  if (isExisting) {
+    await prepareExistingDirectory(
       git,
       settings.repositoryPath,
       repositoryUrl,
       settings.clean
-    ))
-  ) {
-    // Delete the contents of the directory. Don't delete the directory itself
-    // since it may be the current working directory.
-    core.info(`Deleting the contents of '${settings.repositoryPath}'`)
-    for (const file of await fs.promises.readdir(settings.repositoryPath)) {
-      await io.rmRF(path.join(settings.repositoryPath, file))
-    }
-  }
-
-  // Initialize the repository
-  if (
-    !fsHelper.directoryExistsSync(path.join(settings.repositoryPath, '.git'))
-  ) {
-    await git.init()
-    await git.remoteAdd('origin', repositoryUrl)
-  }
-
-  // Disable automatic garbage collection
-  if (!(await git.tryDisableAutomaticGarbageCollection())) {
-    core.warning(
-      `Unable to turn off git automatic garbage collection. The git fetch operation may trigger garbage collection and cause a delay.`
     )
   }
 
-  // Remove possible previous extraheader
-  await removeGitConfig(git, authConfigKey)
+  if (!git) {
+    // Downloading using REST API
+    core.info(`The repository will be downloaded using the GitHub REST API`)
+    core.info(
+      `To create a local Git repository instead, add Git ${gitCommandManager.MinimumGitVersion} or higher to the PATH`
+    )
+    await githubApiHelper.downloadRepository(
+      settings.accessToken,
+      settings.repositoryOwner,
+      settings.repositoryName,
+      settings.ref,
+      settings.commit,
+      settings.repositoryPath
+    )
+  } else {
+    // Save state for POST action
+    stateHelper.setRepositoryPath(settings.repositoryPath)
 
-  // Add extraheader (auth)
-  const base64Credentials = Buffer.from(
-    `x-access-token:${settings.accessToken}`,
-    'utf8'
-  ).toString('base64')
-  core.setSecret(base64Credentials)
-  const authConfigValue = `AUTHORIZATION: basic ${base64Credentials}`
-  await git.config(authConfigKey, authConfigValue)
+    // Initialize the repository
+    if (
+      !fsHelper.directoryExistsSync(path.join(settings.repositoryPath, '.git'))
+    ) {
+      await git.init()
+      await git.remoteAdd('origin', repositoryUrl)
+    }
 
-  // LFS install
-  if (settings.lfs) {
-    await git.lfsInstall()
+    // Disable automatic garbage collection
+    if (!(await git.tryDisableAutomaticGarbageCollection())) {
+      core.warning(
+        `Unable to turn off git automatic garbage collection. The git fetch operation may trigger garbage collection and cause a delay.`
+      )
+    }
+
+    // Remove possible previous extraheader
+    await removeGitConfig(git, authConfigKey)
+
+    // Add extraheader (auth)
+    const base64Credentials = Buffer.from(
+      `x-access-token:${settings.accessToken}`,
+      'utf8'
+    ).toString('base64')
+    core.setSecret(base64Credentials)
+    const authConfigValue = `AUTHORIZATION: basic ${base64Credentials}`
+    await git.config(authConfigKey, authConfigValue)
+
+    // LFS install
+    if (settings.lfs) {
+      await git.lfsInstall()
+    }
+
+    // Fetch
+    const refSpec = refHelper.getRefSpec(settings.ref, settings.commit)
+    await git.fetch(settings.fetchDepth, refSpec)
+
+    // Checkout info
+    const checkoutInfo = await refHelper.getCheckoutInfo(
+      git,
+      settings.ref,
+      settings.commit
+    )
+
+    // LFS fetch
+    // Explicit lfs-fetch to avoid slow checkout (fetches one lfs object at a time).
+    // Explicit lfs fetch will fetch lfs objects in parallel.
+    if (settings.lfs) {
+      await git.lfsFetch(checkoutInfo.startPoint || checkoutInfo.ref)
+    }
+
+    // Checkout
+    await git.checkout(checkoutInfo.ref, checkoutInfo.startPoint)
+
+    // Dump some info about the checked out commit
+    await git.log1()
   }
-
-  // Fetch
-  const refSpec = refHelper.getRefSpec(settings.ref, settings.commit)
-  await git.fetch(settings.fetchDepth, refSpec)
-
-  // Checkout info
-  const checkoutInfo = await refHelper.getCheckoutInfo(
-    git,
-    settings.ref,
-    settings.commit
-  )
-
-  // LFS fetch
-  // Explicit lfs-fetch to avoid slow checkout (fetches one lfs object at a time).
-  // Explicit lfs fetch will fetch lfs objects in parallel.
-  if (settings.lfs) {
-    await git.lfsFetch(checkoutInfo.startPoint || checkoutInfo.ref)
-  }
-
-  // Checkout
-  await git.checkout(checkoutInfo.ref, checkoutInfo.startPoint)
-
-  // Dump some info about the checked out commit
-  await git.log1()
-
-  // Set intra-task state for cleanup
-  coreCommand.issueCommand(
-    'save-state',
-    {name: 'repositoryPath'},
-    settings.repositoryPath
-  )
 }
 
 export async function cleanup(repositoryPath: string): Promise<void> {
@@ -146,79 +149,110 @@ export async function cleanup(repositoryPath: string): Promise<void> {
   await removeGitConfig(git, authConfigKey)
 }
 
-async function tryPrepareExistingDirectory(
+async function getGitCommandManager(
+  settings: ISourceSettings
+): Promise<IGitCommandManager> {
+  core.info(`Working directory is '${settings.repositoryPath}'`)
+  let git = (null as unknown) as IGitCommandManager
+  try {
+    return await gitCommandManager.CreateCommandManager(
+      settings.repositoryPath,
+      settings.lfs
+    )
+  } catch (err) {
+    // Git is required for LFS
+    if (settings.lfs) {
+      throw err
+    }
+
+    // Otherwise fallback to REST API
+    return (null as unknown) as IGitCommandManager
+  }
+}
+
+async function prepareExistingDirectory(
   git: IGitCommandManager,
   repositoryPath: string,
   repositoryUrl: string,
   clean: boolean
-): Promise<boolean> {
+): Promise<void> {
+  let remove = false
+
+  // Check whether using git or REST API
+  if (!git) {
+    remove = true
+  }
   // Fetch URL does not match
-  if (
+  else if (
     !fsHelper.directoryExistsSync(path.join(repositoryPath, '.git')) ||
     repositoryUrl !== (await git.tryGetFetchUrl())
   ) {
-    return false
-  }
+    remove = true
+  } else {
+    // Delete any index.lock and shallow.lock left by a previously canceled run or crashed git process
+    const lockPaths = [
+      path.join(repositoryPath, '.git', 'index.lock'),
+      path.join(repositoryPath, '.git', 'shallow.lock')
+    ]
+    for (const lockPath of lockPaths) {
+      try {
+        await io.rmRF(lockPath)
+      } catch (error) {
+        core.debug(`Unable to delete '${lockPath}'. ${error.message}`)
+      }
+    }
 
-  // Delete any index.lock and shallow.lock left by a previously canceled run or crashed git process
-  const lockPaths = [
-    path.join(repositoryPath, '.git', 'index.lock'),
-    path.join(repositoryPath, '.git', 'shallow.lock')
-  ]
-  for (const lockPath of lockPaths) {
     try {
-      await io.rmRF(lockPath)
+      // Checkout detached HEAD
+      if (!(await git.isDetached())) {
+        await git.checkoutDetach()
+      }
+
+      // Remove all refs/heads/*
+      let branches = await git.branchList(false)
+      for (const branch of branches) {
+        await git.branchDelete(false, branch)
+      }
+
+      // Remove all refs/remotes/origin/* to avoid conflicts
+      branches = await git.branchList(true)
+      for (const branch of branches) {
+        await git.branchDelete(true, branch)
+      }
+
+      // Clean
+      if (clean) {
+        if (!(await git.tryClean())) {
+          core.debug(
+            `The clean command failed. This might be caused by: 1) path too long, 2) permission issue, or 3) file in use. For futher investigation, manually run 'git clean -ffdx' on the directory '${repositoryPath}'.`
+          )
+          remove = true
+        } else if (!(await git.tryReset())) {
+          remove = true
+        }
+
+        if (remove) {
+          core.warning(
+            `Unable to clean or reset the repository. The repository will be recreated instead.`
+          )
+        }
+      }
     } catch (error) {
-      core.debug(`Unable to delete '${lockPath}'. ${error.message}`)
-    }
-  }
-
-  try {
-    // Checkout detached HEAD
-    if (!(await git.isDetached())) {
-      await git.checkoutDetach()
-    }
-
-    // Remove all refs/heads/*
-    let branches = await git.branchList(false)
-    for (const branch of branches) {
-      await git.branchDelete(false, branch)
-    }
-
-    // Remove all refs/remotes/origin/* to avoid conflicts
-    branches = await git.branchList(true)
-    for (const branch of branches) {
-      await git.branchDelete(true, branch)
-    }
-  } catch (error) {
-    core.warning(
-      `Unable to prepare the existing repository. The repository will be recreated instead.`
-    )
-    return false
-  }
-
-  // Clean
-  if (clean) {
-    let succeeded = true
-    if (!(await git.tryClean())) {
-      core.debug(
-        `The clean command failed. This might be caused by: 1) path too long, 2) permission issue, or 3) file in use. For futher investigation, manually run 'git clean -ffdx' on the directory '${repositoryPath}'.`
-      )
-      succeeded = false
-    } else if (!(await git.tryReset())) {
-      succeeded = false
-    }
-
-    if (!succeeded) {
       core.warning(
-        `Unable to clean or reset the repository. The repository will be recreated instead.`
+        `Unable to prepare the existing repository. The repository will be recreated instead.`
       )
+      remove = true
     }
-
-    return succeeded
   }
 
-  return true
+  if (remove) {
+    // Delete the contents of the directory. Don't delete the directory itself
+    // since it might be the current working directory.
+    core.info(`Deleting the contents of '${repositoryPath}'`)
+    for (const file of await fs.promises.readdir(repositoryPath)) {
+      await io.rmRF(path.join(repositoryPath, file))
+    }
+  }
 }
 
 async function removeGitConfig(

src/github-api-helper.ts

@@ -0,0 +1,92 @@
+import * as assert from 'assert'
+import * as core from '@actions/core'
+import * as fs from 'fs'
+import * as github from '@actions/github'
+import * as io from '@actions/io'
+import * as path from 'path'
+import * as retryHelper from './retry-helper'
+import * as toolCache from '@actions/tool-cache'
+import {default as uuid} from 'uuid/v4'
+import {ReposGetArchiveLinkParams} from '@octokit/rest'
+
+const IS_WINDOWS = process.platform === 'win32'
+
+export async function downloadRepository(
+  accessToken: string,
+  owner: string,
+  repo: string,
+  ref: string,
+  commit: string,
+  repositoryPath: string
+): Promise<void> {
+  // Download the archive
+  let archiveData = await retryHelper.execute(async () => {
+    core.info('Downloading the archive')
+    return await downloadArchive(accessToken, owner, repo, ref, commit)
+  })
+
+  // Write archive to disk
+  core.info('Writing archive to disk')
+  const uniqueId = uuid()
+  const archivePath = path.join(repositoryPath, `${uniqueId}.tar.gz`)
+  await fs.promises.writeFile(archivePath, archiveData)
+  archiveData = Buffer.from('') // Free memory
+
+  // Extract archive
+  core.info('Extracting the archive')
+  const extractPath = path.join(repositoryPath, uniqueId)
+  await io.mkdirP(extractPath)
+  if (IS_WINDOWS) {
+    await toolCache.extractZip(archivePath, extractPath)
+  } else {
+    await toolCache.extractTar(archivePath, extractPath)
+  }
+  io.rmRF(archivePath)
+
+  // Determine the path of the repository content. The archive contains
+  // a top-level folder and the repository content is inside.
+  const archiveFileNames = await fs.promises.readdir(extractPath)
+  assert.ok(
+    archiveFileNames.length == 1,
+    'Expected exactly one directory inside archive'
+  )
+  const archiveVersion = archiveFileNames[0] // The top-level folder name includes the short SHA
+  core.info(`Resolved version ${archiveVersion}`)
+  const tempRepositoryPath = path.join(extractPath, archiveVersion)
+
+  // Move the files
+  for (const fileName of await fs.promises.readdir(tempRepositoryPath)) {
+    const sourcePath = path.join(tempRepositoryPath, fileName)
+    const targetPath = path.join(repositoryPath, fileName)
+    if (IS_WINDOWS) {
+      await io.cp(sourcePath, targetPath, {recursive: true}) // Copy on Windows (Windows Defender may have a lock)
+    } else {
+      await io.mv(sourcePath, targetPath)
+    }
+  }
+  io.rmRF(extractPath)
+}
+
+async function downloadArchive(
+  accessToken: string,
+  owner: string,
+  repo: string,
+  ref: string,
+  commit: string
+): Promise<Buffer> {
+  const octokit = new github.GitHub(accessToken)
+  const params: ReposGetArchiveLinkParams = {
+    owner: owner,
+    repo: repo,
+    archive_format: IS_WINDOWS ? 'zipball' : 'tarball',
+    ref: commit || ref
+  }
+  const response = await octokit.repos.getArchiveLink(params)
+  if (response.status != 200) {
+    throw new Error(
+      `Unexpected response from GitHub API. Status: ${response.status}, Data: ${response.data}`
+    )
+  }
+
+  return Buffer.from(response.data) // response.data is ArrayBuffer
+}

src/main.ts

@@ -3,8 +3,7 @@ import * as coreCommand from '@actions/core/lib/command'
 import * as gitSourceProvider from './git-source-provider'
 import * as inputHelper from './input-helper'
 import * as path from 'path'
-
-const cleanupRepositoryPath = process.env['STATE_repositoryPath'] as string
+import * as stateHelper from './state-helper'
 
 async function run(): Promise<void> {
   try {
@@ -31,14 +30,14 @@ async function run(): Promise<void> {
 
 async function cleanup(): Promise<void> {
   try {
-    await gitSourceProvider.cleanup(cleanupRepositoryPath)
+    await gitSourceProvider.cleanup(stateHelper.RepositoryPath)
   } catch (error) {
     core.warning(error.message)
   }
 }
 
 // Main
-if (!cleanupRepositoryPath) {
+if (!stateHelper.IsPost) {
   run()
 }
 // Post

src/retry-helper.ts

@@ -0,0 +1,61 @@
+import * as core from '@actions/core'
+
+const defaultMaxAttempts = 3
+const defaultMinSeconds = 10
+const defaultMaxSeconds = 20
+
+export class RetryHelper {
+  private maxAttempts: number
+  private minSeconds: number
+  private maxSeconds: number
+
+  constructor(
+    maxAttempts: number = defaultMaxAttempts,
+    minSeconds: number = defaultMinSeconds,
+    maxSeconds: number = defaultMaxSeconds
+  ) {
+    this.maxAttempts = maxAttempts
+    this.minSeconds = Math.floor(minSeconds)
+    this.maxSeconds = Math.floor(maxSeconds)
+    if (this.minSeconds > this.maxSeconds) {
+      throw new Error('min seconds should be less than or equal to max seconds')
+    }
+  }
+
+  async execute<T>(action: () => Promise<T>): Promise<T> {
+    let attempt = 1
+    while (attempt < this.maxAttempts) {
+      // Try
+      try {
+        return await action()
+      } catch (err) {
+        core.info(err.message)
+      }
+
+      // Sleep
+      const seconds = this.getSleepAmount()
+      core.info(`Waiting ${seconds} seconds before trying again`)
+      await this.sleep(seconds)
+      attempt++
+    }
+
+    // Last attempt
+    return await action()
+  }
+
+  private getSleepAmount(): number {
+    return (
+      Math.floor(Math.random() * (this.maxSeconds - this.minSeconds + 1)) +
+      this.minSeconds
+    )
+  }
+
+  private async sleep(seconds: number): Promise<void> {
+    return new Promise(resolve => setTimeout(resolve, seconds * 1000))
+  }
+}
+
+export async function execute<T>(action: () => Promise<T>): Promise<T> {
+  const retryHelper = new RetryHelper()
+  return await retryHelper.execute(action)
+}

src/state-helper.ts

@@ -0,0 +1,30 @@
+import * as core from '@actions/core'
+import * as coreCommand from '@actions/core/lib/command'
+
+/**
+ * Indicates whether the POST action is running
+ */
+export const IsPost = !!process.env['STATE_isPost']
+
+/**
+ * The repository path for the POST action. The value is empty during the MAIN action.
+ */
+export const RepositoryPath =
+  (process.env['STATE_repositoryPath'] as string) || ''
+
+/**
+ * Save the repository path so the POST action can retrieve the value.
+ */
+export function setRepositoryPath(repositoryPath: string) {
+  coreCommand.issueCommand(
+    'save-state',
+    {name: 'repositoryPath'},
+    repositoryPath
+  )
+}
+
+// Publish a variable so that when the POST action runs, it can determine it should run the cleanup logic.
+// This is necessary since we don't have a separate entry point.
+if (!IsPost) {
+  coreCommand.issueCommand('save-state', {name: 'isPost'}, 'true')
+}