add support for submodules (#173)

2020-03-05 14:21:59 -05:00
parent 204620207c
commit 422dc45671
17 changed files with 915 additions and 220 deletions
--- a/src/git-auth-helper.ts
+++ b/src/git-auth-helper.ts
@@ -5,6 +5,7 @@ import * as fs from 'fs'
 import * as io from '@actions/io'
 import * as os from 'os'
 import * as path from 'path'
+import * as regexpHelper from './regexp-helper'
 import * as stateHelper from './state-helper'
 import {default as uuid} from 'uuid/v4'
 import {IGitCommandManager} from './git-command-manager'
@@ -12,11 +13,13 @@ import {IGitSourceSettings} from './git-source-settings'

 const IS_WINDOWS = process.platform === 'win32'
 const HOSTNAME = 'github.com'
-const EXTRA_HEADER_KEY = `http.https://${HOSTNAME}/.extraheader`

 export interface IGitAuthHelper {
  configureAuth(): Promise<void>
+  configureGlobalAuth(): Promise<void>
+  configureSubmoduleAuth(): Promise<void>
  removeAuth(): Promise<void>
+  removeGlobalAuth(): Promise<void>
 }

 export function createAuthHelper(
@@ -27,8 +30,12 @@ export function createAuthHelper(
 }

 class GitAuthHelper {
-  private git: IGitCommandManager
-  private settings: IGitSourceSettings
+  private readonly git: IGitCommandManager
+  private readonly settings: IGitSourceSettings
+  private readonly tokenConfigKey: string = `http.https://${HOSTNAME}/.extraheader`
+  private readonly tokenPlaceholderConfigValue: string
+  private temporaryHomePath = ''
+  private tokenConfigValue: string

  constructor(
    gitCommandManager: IGitCommandManager,
@@ -36,6 +43,15 @@ class GitAuthHelper {
  ) {
    this.git = gitCommandManager
    this.settings = gitSourceSettings || (({} as unknown) as IGitSourceSettings)
+
+    // Token auth header
+    const basicCredential = Buffer.from(
+      `x-access-token:${this.settings.authToken}`,
+      'utf8'
+    ).toString('base64')
+    core.setSecret(basicCredential)
+    this.tokenPlaceholderConfigValue = `AUTHORIZATION: basic ***`
+    this.tokenConfigValue = `AUTHORIZATION: basic ${basicCredential}`
  }

  async configureAuth(): Promise<void> {
@@ -46,48 +62,132 @@ class GitAuthHelper {
    await this.configureToken()
  }

+  async configureGlobalAuth(): Promise<void> {
+    // Create a temp home directory
+    const runnerTemp = process.env['RUNNER_TEMP'] || ''
+    assert.ok(runnerTemp, 'RUNNER_TEMP is not defined')
+    const uniqueId = uuid()
+    this.temporaryHomePath = path.join(runnerTemp, uniqueId)
+    await fs.promises.mkdir(this.temporaryHomePath, {recursive: true})
+
+    // Copy the global git config
+    const gitConfigPath = path.join(
+      process.env['HOME'] || os.homedir(),
+      '.gitconfig'
+    )
+    const newGitConfigPath = path.join(this.temporaryHomePath, '.gitconfig')
+    let configExists = false
+    try {
+      await fs.promises.stat(gitConfigPath)
+      configExists = true
+    } catch (err) {
+      if (err.code !== 'ENOENT') {
+        throw err
+      }
+    }
+    if (configExists) {
+      core.info(`Copying '${gitConfigPath}' to '${newGitConfigPath}'`)
+      await io.cp(gitConfigPath, newGitConfigPath)
+    } else {
+      await fs.promises.writeFile(newGitConfigPath, '')
+    }
+
+    // Configure the token
+    try {
+      core.info(
+        `Temporarily overriding HOME='${this.temporaryHomePath}' before making global git config changes`
+      )
+      this.git.setEnvironmentVariable('HOME', this.temporaryHomePath)
+      await this.configureToken(newGitConfigPath, true)
+    } catch (err) {
+      // Unset in case somehow written to the real global config
+      core.info(
+        'Encountered an error when attempting to configure token. Attempting unconfigure.'
+      )
+      await this.git.tryConfigUnset(this.tokenConfigKey, true)
+      throw err
+    }
+  }
+
+  async configureSubmoduleAuth(): Promise<void> {
+    if (this.settings.persistCredentials) {
+      // Configure a placeholder value. This approach avoids the credential being captured
+      // by process creation audit events, which are commonly logged. For more information,
+      // refer to https://docs.microsoft.com/en-us/windows-server/identity/ad-ds/manage/component-updates/command-line-process-auditing
+      const output = await this.git.submoduleForeach(
+        `git config "${this.tokenConfigKey}" "${this.tokenPlaceholderConfigValue}" && git config --local --show-origin --name-only --get-regexp remote.origin.url`,
+        this.settings.nestedSubmodules
+      )
+
+      // Replace the placeholder
+      const configPaths: string[] =
+        output.match(/(?<=(^|\n)file:)[^\t]+(?=\tremote\.origin\.url)/g) || []
+      for (const configPath of configPaths) {
+        core.debug(`Replacing token placeholder in '${configPath}'`)
+        this.replaceTokenPlaceholder(configPath)
+      }
+    }
+  }
+
  async removeAuth(): Promise<void> {
    await this.removeToken()
  }

-  private async configureToken(): Promise<void> {
+  async removeGlobalAuth(): Promise<void> {
+    core.info(`Unsetting HOME override`)
+    this.git.removeEnvironmentVariable('HOME')
+    await io.rmRF(this.temporaryHomePath)
+  }
+
+  private async configureToken(
+    configPath?: string,
+    globalConfig?: boolean
+  ): Promise<void> {
+    // Validate args
+    assert.ok(
+      (configPath && globalConfig) || (!configPath && !globalConfig),
+      'Unexpected configureToken parameter combinations'
+    )
+
+    // Default config path
+    if (!configPath && !globalConfig) {
+      configPath = path.join(this.git.getWorkingDirectory(), '.git', 'config')
+    }
+
    // Configure a placeholder value. This approach avoids the credential being captured
    // by process creation audit events, which are commonly logged. For more information,
    // refer to https://docs.microsoft.com/en-us/windows-server/identity/ad-ds/manage/component-updates/command-line-process-auditing
-    const placeholder = `AUTHORIZATION: basic ***`
-    await this.git.config(EXTRA_HEADER_KEY, placeholder)
-
-    // Determine the basic credential value
-    const basicCredential = Buffer.from(
-      `x-access-token:${this.settings.authToken}`,
-      'utf8'
-    ).toString('base64')
-    core.setSecret(basicCredential)
-
-    // Replace the value in the config file
-    const configPath = path.join(
-      this.git.getWorkingDirectory(),
-      '.git',
-      'config'
+    await this.git.config(
+      this.tokenConfigKey,
+      this.tokenPlaceholderConfigValue,
+      globalConfig
    )
+
+    // Replace the placeholder
+    await this.replaceTokenPlaceholder(configPath || '')
+  }
+
+  private async replaceTokenPlaceholder(configPath: string): Promise<void> {
+    assert.ok(configPath, 'configPath is not defined')
    let content = (await fs.promises.readFile(configPath)).toString()
-    const placeholderIndex = content.indexOf(placeholder)
+    const placeholderIndex = content.indexOf(this.tokenPlaceholderConfigValue)
    if (
      placeholderIndex < 0 ||
-      placeholderIndex != content.lastIndexOf(placeholder)
+      placeholderIndex != content.lastIndexOf(this.tokenPlaceholderConfigValue)
    ) {
-      throw new Error('Unable to replace auth placeholder in .git/config')
+      throw new Error(`Unable to replace auth placeholder in ${configPath}`)
    }
+    assert.ok(this.tokenConfigValue, 'tokenConfigValue is not defined')
    content = content.replace(
-      placeholder,
-      `AUTHORIZATION: basic ${basicCredential}`
+      this.tokenPlaceholderConfigValue,
+      this.tokenConfigValue
    )
    await fs.promises.writeFile(configPath, content)
  }

  private async removeToken(): Promise<void> {
    // HTTP extra header
-    await this.removeGitConfig(EXTRA_HEADER_KEY)
+    await this.removeGitConfig(this.tokenConfigKey)
  }

  private async removeGitConfig(configKey: string): Promise<void> {
@@ -98,5 +198,11 @@ class GitAuthHelper {
      // Load the config contents
      core.warning(`Failed to remove '${configKey}' from the git config`)
    }
+
+    const pattern = regexpHelper.escape(configKey)
+    await this.git.submoduleForeach(
+      `git config --local --name-only --get-regexp ${pattern} && git config --local --unset-all ${configKey} || :`,
+      true
+    )
  }
 }
--- a/src/git-command-manager.ts
+++ b/src/git-command-manager.ts
@@ -3,6 +3,7 @@ import * as exec from '@actions/exec'
 import * as fshelper from './fs-helper'
 import * as io from '@actions/io'
 import * as path from 'path'
+import * as regexpHelper from './regexp-helper'
 import * as retryHelper from './retry-helper'
 import {GitVersion} from './git-version'

@@ -16,8 +17,12 @@ export interface IGitCommandManager {
  branchList(remote: boolean): Promise<string[]>
  checkout(ref: string, startPoint: string): Promise<void>
  checkoutDetach(): Promise<void>
-  config(configKey: string, configValue: string): Promise<void>
-  configExists(configKey: string): Promise<boolean>
+  config(
+    configKey: string,
+    configValue: string,
+    globalConfig?: boolean
+  ): Promise<void>
+  configExists(configKey: string, globalConfig?: boolean): Promise<boolean>
  fetch(fetchDepth: number, refSpec: string[]): Promise<void>
  getWorkingDirectory(): string
  init(): Promise<void>
@@ -26,10 +31,14 @@ export interface IGitCommandManager {
  lfsInstall(): Promise<void>
  log1(): Promise<void>
  remoteAdd(remoteName: string, remoteUrl: string): Promise<void>
+  removeEnvironmentVariable(name: string): void
  setEnvironmentVariable(name: string, value: string): void
+  submoduleForeach(command: string, recursive: boolean): Promise<string>
+  submoduleSync(recursive: boolean): Promise<void>
+  submoduleUpdate(fetchDepth: number, recursive: boolean): Promise<void>
  tagExists(pattern: string): Promise<boolean>
  tryClean(): Promise<boolean>
-  tryConfigUnset(configKey: string): Promise<boolean>
+  tryConfigUnset(configKey: string, globalConfig?: boolean): Promise<boolean>
  tryDisableAutomaticGarbageCollection(): Promise<boolean>
  tryGetFetchUrl(): Promise<string>
  tryReset(): Promise<boolean>
@@ -124,16 +133,32 @@ class GitCommandManager {
    await this.execGit(args)
  }

-  async config(configKey: string, configValue: string): Promise<void> {
-    await this.execGit(['config', '--local', configKey, configValue])
+  async config(
+    configKey: string,
+    configValue: string,
+    globalConfig?: boolean
+  ): Promise<void> {
+    await this.execGit([
+      'config',
+      globalConfig ? '--global' : '--local',
+      configKey,
+      configValue
+    ])
  }

-  async configExists(configKey: string): Promise<boolean> {
-    const pattern = configKey.replace(/[^a-zA-Z0-9_]/g, x => {
-      return `\\${x}`
-    })
+  async configExists(
+    configKey: string,
+    globalConfig?: boolean
+  ): Promise<boolean> {
+    const pattern = regexpHelper.escape(configKey)
    const output = await this.execGit(
-      ['config', '--local', '--name-only', '--get-regexp', pattern],
+      [
+        'config',
+        globalConfig ? '--global' : '--local',
+        '--name-only',
+        '--get-regexp',
+        pattern
+      ],
      true
    )
    return output.exitCode === 0
@@ -208,10 +233,48 @@ class GitCommandManager {
    await this.execGit(['remote', 'add', remoteName, remoteUrl])
  }

+  removeEnvironmentVariable(name: string): void {
+    delete this.gitEnv[name]
+  }
+
  setEnvironmentVariable(name: string, value: string): void {
    this.gitEnv[name] = value
  }

+  async submoduleForeach(command: string, recursive: boolean): Promise<string> {
+    const args = ['submodule', 'foreach']
+    if (recursive) {
+      args.push('--recursive')
+    }
+    args.push(command)
+
+    const output = await this.execGit(args)
+    return output.stdout
+  }
+
+  async submoduleSync(recursive: boolean): Promise<void> {
+    const args = ['submodule', 'sync']
+    if (recursive) {
+      args.push('--recursive')
+    }
+
+    await this.execGit(args)
+  }
+
+  async submoduleUpdate(fetchDepth: number, recursive: boolean): Promise<void> {
+    const args = ['-c', 'protocol.version=2']
+    args.push('submodule', 'update', '--init', '--force')
+    if (fetchDepth > 0) {
+      args.push(`--depth=${fetchDepth}`)
+    }
+
+    if (recursive) {
+      args.push('--recursive')
+    }
+
+    await this.execGit(args)
+  }
+
  async tagExists(pattern: string): Promise<boolean> {
    const output = await this.execGit(['tag', '--list', pattern])
    return !!output.stdout.trim()
@@ -222,9 +285,17 @@ class GitCommandManager {
    return output.exitCode === 0
  }

-  async tryConfigUnset(configKey: string): Promise<boolean> {
+  async tryConfigUnset(
+    configKey: string,
+    globalConfig?: boolean
+  ): Promise<boolean> {
    const output = await this.execGit(
-      ['config', '--local', '--unset-all', configKey],
+      [
+        'config',
+        globalConfig ? '--global' : '--local',
+        '--unset-all',
+        configKey
+      ],
      true
    )
    return output.exitCode === 0
--- a/src/git-source-provider.ts
+++ b/src/git-source-provider.ts
@@ -61,63 +61,91 @@ export async function getSource(settings: IGitSourceSettings): Promise<void> {
      settings.commit,
      settings.repositoryPath
    )
-  } else {
-    // Save state for POST action
-    stateHelper.setRepositoryPath(settings.repositoryPath)
+    return
+  }

-    // Initialize the repository
-    if (
-      !fsHelper.directoryExistsSync(path.join(settings.repositoryPath, '.git'))
-    ) {
-      await git.init()
-      await git.remoteAdd('origin', repositoryUrl)
+  // Save state for POST action
+  stateHelper.setRepositoryPath(settings.repositoryPath)
+
+  // Initialize the repository
+  if (
+    !fsHelper.directoryExistsSync(path.join(settings.repositoryPath, '.git'))
+  ) {
+    await git.init()
+    await git.remoteAdd('origin', repositoryUrl)
+  }
+
+  // Disable automatic garbage collection
+  if (!(await git.tryDisableAutomaticGarbageCollection())) {
+    core.warning(
+      `Unable to turn off git automatic garbage collection. The git fetch operation may trigger garbage collection and cause a delay.`
+    )
+  }
+
+  const authHelper = gitAuthHelper.createAuthHelper(git, settings)
+  try {
+    // Configure auth
+    await authHelper.configureAuth()
+
+    // LFS install
+    if (settings.lfs) {
+      await git.lfsInstall()
    }

-    // Disable automatic garbage collection
-    if (!(await git.tryDisableAutomaticGarbageCollection())) {
-      core.warning(
-        `Unable to turn off git automatic garbage collection. The git fetch operation may trigger garbage collection and cause a delay.`
-      )
+    // Fetch
+    const refSpec = refHelper.getRefSpec(settings.ref, settings.commit)
+    await git.fetch(settings.fetchDepth, refSpec)
+
+    // Checkout info
+    const checkoutInfo = await refHelper.getCheckoutInfo(
+      git,
+      settings.ref,
+      settings.commit
+    )
+
+    // LFS fetch
+    // Explicit lfs-fetch to avoid slow checkout (fetches one lfs object at a time).
+    // Explicit lfs fetch will fetch lfs objects in parallel.
+    if (settings.lfs) {
+      await git.lfsFetch(checkoutInfo.startPoint || checkoutInfo.ref)
    }

-    const authHelper = gitAuthHelper.createAuthHelper(git, settings)
-    try {
-      // Configure auth
-      await authHelper.configureAuth()
+    // Checkout
+    await git.checkout(checkoutInfo.ref, checkoutInfo.startPoint)

-      // LFS install
-      if (settings.lfs) {
-        await git.lfsInstall()
+    // Submodules
+    if (settings.submodules) {
+      try {
+        // Temporarily override global config
+        await authHelper.configureGlobalAuth()
+
+        // Checkout submodules
+        await git.submoduleSync(settings.nestedSubmodules)
+        await git.submoduleUpdate(
+          settings.fetchDepth,
+          settings.nestedSubmodules
+        )
+        await git.submoduleForeach(
+          'git config --local gc.auto 0',
+          settings.nestedSubmodules
+        )
+
+        // Persist credentials
+        if (settings.persistCredentials) {
+          await authHelper.configureSubmoduleAuth()
+        }
+      } finally {
+        // Remove temporary global config override
+        await authHelper.removeGlobalAuth()
      }
+    }

-      // Fetch
-      const refSpec = refHelper.getRefSpec(settings.ref, settings.commit)
-      await git.fetch(settings.fetchDepth, refSpec)
-
-      // Checkout info
-      const checkoutInfo = await refHelper.getCheckoutInfo(
-        git,
-        settings.ref,
-        settings.commit
-      )
-
-      // LFS fetch
-      // Explicit lfs-fetch to avoid slow checkout (fetches one lfs object at a time).
-      // Explicit lfs fetch will fetch lfs objects in parallel.
-      if (settings.lfs) {
-        await git.lfsFetch(checkoutInfo.startPoint || checkoutInfo.ref)
-      }
-
-      // Checkout
-      await git.checkout(checkoutInfo.ref, checkoutInfo.startPoint)
-
-      // Dump some info about the checked out commit
-      await git.log1()
-    } finally {
-      // Remove auth
-      if (!settings.persistCredentials) {
-        await authHelper.removeAuth()
-      }
+    // Dump some info about the checked out commit
+    await git.log1()
+  } finally {
+    // Remove auth
+    if (!settings.persistCredentials) {
+      await authHelper.removeAuth()
    }
  }
 }
--- a/src/git-source-settings.ts
+++ b/src/git-source-settings.ts
@@ -7,6 +7,8 @@ export interface IGitSourceSettings {
  clean: boolean
  fetchDepth: number
  lfs: boolean
+  submodules: boolean
+  nestedSubmodules: boolean
  authToken: string
  persistCredentials: boolean
 }
--- a/src/input-helper.ts
+++ b/src/input-helper.ts
@@ -85,13 +85,6 @@ export function getInputs(): IGitSourceSettings {
  result.clean = (core.getInput('clean') || 'true').toUpperCase() === 'TRUE'
  core.debug(`clean = ${result.clean}`)

-  // Submodules
-  if (core.getInput('submodules')) {
-    throw new Error(
-      "The input 'submodules' is not supported in actions/checkout@v2"
-    )
-  }
-
  // Fetch depth
  result.fetchDepth = Math.floor(Number(core.getInput('fetch-depth') || '1'))
  if (isNaN(result.fetchDepth) || result.fetchDepth < 0) {
@@ -103,6 +96,19 @@ export function getInputs(): IGitSourceSettings {
  result.lfs = (core.getInput('lfs') || 'false').toUpperCase() === 'TRUE'
  core.debug(`lfs = ${result.lfs}`)

+  // Submodules
+  result.submodules = false
+  result.nestedSubmodules = false
+  const submodulesString = (core.getInput('submodules') || '').toUpperCase()
+  if (submodulesString == 'RECURSIVE') {
+    result.submodules = true
+    result.nestedSubmodules = true
+  } else if (submodulesString == 'TRUE') {
+    result.submodules = true
+  }
+  core.debug(`submodules = ${result.submodules}`)
+  core.debug(`recursive submodules = ${result.nestedSubmodules}`)
+
  // Auth token
  result.authToken = core.getInput('token')

--- a/src/regexp-helper.ts
+++ b/src/regexp-helper.ts
@@ -0,0 +1,5 @@
+export function escape(value: string): string {
+  return value.replace(/[^a-zA-Z0-9_]/g, x => {
+    return `\\${x}`
+  })
+}