Add support for sparse checkouts (#1369)

* Add support for sparse checkouts * sparse-checkout: optionally turn off cone mode While it _is_ true that cone mode is the default nowadays (mainly for performance reasons: code mode is much faster than non-cone mode), there _are_ legitimate use cases where non-cone mode is really useful. Let's add a flag to optionally disable cone mode. Signed-off-by: Johannes Schindelin <johannes.schindelin@gmx.de> * Verify minimum Git version for sparse checkout The `git sparse-checkout` command is available only since Git version v2.25.0. The `actions/checkout` Action actually supports older Git versions than that; As of time of writing, the minimum version is v2.18.0. Instead of raising this minimum version even for users who do not require a sparse checkout, only check for this minimum version specifically when a sparse checkout was asked for. Suggested-by: Tingluo Huang <tingluohuang@github.com> Signed-off-by: Johannes Schindelin <johannes.schindelin@gmx.de> * Support sparse checkout/LFS better Instead of fetching all the LFS objects present in the current revision in a sparse checkout, whether they are needed inside the sparse cone or not, let's instead only pull the ones that are actually needed. To do that, let's avoid running that preemptive `git lfs fetch` call in case of a sparse checkout. An alternative that was considered during the development of this patch (and ultimately rejected) was to use `git lfs pull --include <path>...`, but it turned out to be too inflexible because it requires exact paths, not the patterns that are available via the sparse checkout definition, and that risks running into command-line length limitations. Signed-off-by: Johannes Schindelin <johannes.schindelin@gmx.de> --------- Signed-off-by: Johannes Schindelin <johannes.schindelin@gmx.de> Co-authored-by: Daniel <daniel.fernandez@feverup.com>
2023-06-09 15:08:21 +02:00
parent f095bcc56b
commit d106d4669b
14 changed files with 395 additions and 31 deletions
--- a/src/git-command-manager.ts
+++ b/src/git-command-manager.ts
@@ -1,5 +1,6 @@
 import * as core from '@actions/core'
 import * as exec from '@actions/exec'
+import * as fs from 'fs'
 import * as fshelper from './fs-helper'
 import * as io from '@actions/io'
 import * as path from 'path'
@@ -16,6 +17,8 @@ export interface IGitCommandManager {
  branchDelete(remote: boolean, branch: string): Promise<void>
  branchExists(remote: boolean, pattern: string): Promise<boolean>
  branchList(remote: boolean): Promise<string[]>
+  sparseCheckout(sparseCheckout: string[]): Promise<void>
+  sparseCheckoutNonConeMode(sparseCheckout: string[]): Promise<void>
  checkout(ref: string, startPoint: string): Promise<void>
  checkoutDetach(): Promise<void>
  config(
@@ -25,7 +28,13 @@ export interface IGitCommandManager {
    add?: boolean
  ): Promise<void>
  configExists(configKey: string, globalConfig?: boolean): Promise<boolean>
-  fetch(refSpec: string[], fetchDepth?: number): Promise<void>
+  fetch(
+    refSpec: string[],
+    options: {
+      filter?: string
+      fetchDepth?: number
+    }
+  ): Promise<void>
  getDefaultBranch(repositoryUrl: string): Promise<string>
  getWorkingDirectory(): string
  init(): Promise<void>
@@ -52,9 +61,14 @@ export interface IGitCommandManager {

 export async function createCommandManager(
  workingDirectory: string,
-  lfs: boolean
+  lfs: boolean,
+  doSparseCheckout: boolean
 ): Promise<IGitCommandManager> {
-  return await GitCommandManager.createCommandManager(workingDirectory, lfs)
+  return await GitCommandManager.createCommandManager(
+    workingDirectory,
+    lfs,
+    doSparseCheckout
+  )
 }

 class GitCommandManager {
@@ -64,6 +78,7 @@ class GitCommandManager {
  }
  private gitPath = ''
  private lfs = false
+  private doSparseCheckout = false
  private workingDirectory = ''

  // Private constructor; use createCommandManager()
@@ -154,6 +169,27 @@ class GitCommandManager {
    return result
  }

+  async sparseCheckout(sparseCheckout: string[]): Promise<void> {
+    await this.execGit(['sparse-checkout', 'set', ...sparseCheckout])
+  }
+
+  async sparseCheckoutNonConeMode(sparseCheckout: string[]): Promise<void> {
+    await this.execGit(['config', 'core.sparseCheckout', 'true'])
+    const output = await this.execGit([
+      'rev-parse',
+      '--git-path',
+      'info/sparse-checkout'
+    ])
+    const sparseCheckoutPath = path.join(
+      this.workingDirectory,
+      output.stdout.trimRight()
+    )
+    await fs.promises.appendFile(
+      sparseCheckoutPath,
+      `\n${sparseCheckout.join('\n')}\n`
+    )
+  }
+
  async checkout(ref: string, startPoint: string): Promise<void> {
    const args = ['checkout', '--progress', '--force']
    if (startPoint) {
@@ -202,15 +238,23 @@ class GitCommandManager {
    return output.exitCode === 0
  }

-  async fetch(refSpec: string[], fetchDepth?: number): Promise<void> {
+  async fetch(
+    refSpec: string[],
+    options: {filter?: string; fetchDepth?: number}
+  ): Promise<void> {
    const args = ['-c', 'protocol.version=2', 'fetch']
    if (!refSpec.some(x => x === refHelper.tagsRefSpec)) {
      args.push('--no-tags')
    }

    args.push('--prune', '--progress', '--no-recurse-submodules')
-    if (fetchDepth && fetchDepth > 0) {
-      args.push(`--depth=${fetchDepth}`)
+
+    if (options.filter) {
+      args.push(`--filter=${options.filter}`)
+    }
+
+    if (options.fetchDepth && options.fetchDepth > 0) {
+      args.push(`--depth=${options.fetchDepth}`)
    } else if (
      fshelper.fileExistsSync(
        path.join(this.workingDirectory, '.git', 'shallow')
@@ -423,10 +467,15 @@ class GitCommandManager {

  static async createCommandManager(
    workingDirectory: string,
-    lfs: boolean
+    lfs: boolean,
+    doSparseCheckout: boolean
  ): Promise<GitCommandManager> {
    const result = new GitCommandManager()
-    await result.initializeCommandManager(workingDirectory, lfs)
+    await result.initializeCommandManager(
+      workingDirectory,
+      lfs,
+      doSparseCheckout
+    )
    return result
  }

@@ -476,7 +525,8 @@ class GitCommandManager {

  private async initializeCommandManager(
    workingDirectory: string,
-    lfs: boolean
+    lfs: boolean,
+    doSparseCheckout: boolean
  ): Promise<void> {
    this.workingDirectory = workingDirectory

@@ -539,6 +589,16 @@ class GitCommandManager {
      }
    }

+    this.doSparseCheckout = doSparseCheckout
+    if (this.doSparseCheckout) {
+      // The `git sparse-checkout` command was introduced in Git v2.25.0
+      const minimumGitSparseCheckoutVersion = new GitVersion('2.25')
+      if (!gitVersion.checkMinimum(minimumGitSparseCheckoutVersion)) {
+        throw new Error(
+          `Minimum Git version required for sparse checkout is ${minimumGitSparseCheckoutVersion}. Your git ('${this.gitPath}') is ${gitVersion}`
+        )
+      }
+    }
    // Set the user agent
    const gitHttpUserAgent = `git/${gitVersion} (github-actions-checkout)`
    core.debug(`Set git useragent to: ${gitHttpUserAgent}`)
--- a/src/git-source-provider.ts
+++ b/src/git-source-provider.ts
@@ -153,23 +153,26 @@ export async function getSource(settings: IGitSourceSettings): Promise<void> {

    // Fetch
    core.startGroup('Fetching the repository')
+    const fetchOptions: {filter?: string; fetchDepth?: number} = {}
+    if (settings.sparseCheckout) fetchOptions.filter = 'blob:none'
    if (settings.fetchDepth <= 0) {
      // Fetch all branches and tags
      let refSpec = refHelper.getRefSpecForAllHistory(
        settings.ref,
        settings.commit
      )
-      await git.fetch(refSpec)
+      await git.fetch(refSpec, fetchOptions)

      // When all history is fetched, the ref we're interested in may have moved to a different
      // commit (push or force push). If so, fetch again with a targeted refspec.
      if (!(await refHelper.testRef(git, settings.ref, settings.commit))) {
        refSpec = refHelper.getRefSpec(settings.ref, settings.commit)
-        await git.fetch(refSpec)
+        await git.fetch(refSpec, fetchOptions)
      }
    } else {
+      fetchOptions.fetchDepth = settings.fetchDepth
      const refSpec = refHelper.getRefSpec(settings.ref, settings.commit)
-      await git.fetch(refSpec, settings.fetchDepth)
+      await git.fetch(refSpec, fetchOptions)
    }
    core.endGroup()

@@ -185,12 +188,24 @@ export async function getSource(settings: IGitSourceSettings): Promise<void> {
    // LFS fetch
    // Explicit lfs-fetch to avoid slow checkout (fetches one lfs object at a time).
    // Explicit lfs fetch will fetch lfs objects in parallel.
-    if (settings.lfs) {
+    // For sparse checkouts, let `checkout` fetch the needed objects lazily.
+    if (settings.lfs && !settings.sparseCheckout) {
      core.startGroup('Fetching LFS objects')
      await git.lfsFetch(checkoutInfo.startPoint || checkoutInfo.ref)
      core.endGroup()
    }

+    // Sparse checkout
+    if (settings.sparseCheckout) {
+      core.startGroup('Setting up sparse checkout')
+      if (settings.sparseCheckoutConeMode) {
+        await git.sparseCheckout(settings.sparseCheckout)
+      } else {
+        await git.sparseCheckoutNonConeMode(settings.sparseCheckout)
+      }
+      core.endGroup()
+    }
+
    // Checkout
    core.startGroup('Checking out the ref')
    await git.checkout(checkoutInfo.ref, checkoutInfo.startPoint)
@@ -261,7 +276,11 @@ export async function cleanup(repositoryPath: string): Promise<void> {

  let git: IGitCommandManager
  try {
-    git = await gitCommandManager.createCommandManager(repositoryPath, false)
+    git = await gitCommandManager.createCommandManager(
+      repositoryPath,
+      false,
+      false
+    )
  } catch {
    return
  }
@@ -297,7 +316,8 @@ async function getGitCommandManager(
  try {
    return await gitCommandManager.createCommandManager(
      settings.repositoryPath,
-      settings.lfs
+      settings.lfs,
+      settings.sparseCheckout != null
    )
  } catch (err) {
    // Git is required for LFS
--- a/src/git-source-settings.ts
+++ b/src/git-source-settings.ts
@@ -29,6 +29,16 @@ export interface IGitSourceSettings {
   */
  clean: boolean

+  /**
+   * The array of folders to make the sparse checkout
+   */
+  sparseCheckout: string[]
+
+  /**
+   * Indicates whether to use cone mode in the sparse checkout (if any)
+   */
+  sparseCheckoutConeMode: boolean
+
  /**
   * The depth when fetching
   */
--- a/src/input-helper.ts
+++ b/src/input-helper.ts
@@ -82,6 +82,17 @@ export async function getInputs(): Promise<IGitSourceSettings> {
  result.clean = (core.getInput('clean') || 'true').toUpperCase() === 'TRUE'
  core.debug(`clean = ${result.clean}`)

+  // Sparse checkout
+  const sparseCheckout = core.getMultilineInput('sparse-checkout')
+  if (sparseCheckout.length) {
+    result.sparseCheckout = sparseCheckout
+    core.debug(`sparse checkout = ${result.sparseCheckout}`)
+  }
+
+  result.sparseCheckoutConeMode =
+    (core.getInput('sparse-checkout-cone-mode') || 'true').toUpperCase() ===
+    'TRUE'
+
  // Fetch depth
  result.fetchDepth = Math.floor(Number(core.getInput('fetch-depth') || '1'))
  if (isNaN(result.fetchDepth) || result.fetchDepth < 0) {