command: add plugin-override opt to build/validate

The plugin-override flag aims to forcefully load a plugin by passing in
its path, ignoring completely the plugins discovered before, or those in
the required_plugins block.

command/build.go

@@ -87,7 +87,13 @@ func (c *BuildCommand) RunContext(buildCtx context.Context, cla *BuildArgs) int
 		return ret
 	}
 
-	diags := packerStarter.DetectPluginBinaries()
+	diags := packerStarter.DetectPluginBinaries(cla.getIgnoredPluginAccessors())
+	ret = writeDiags(c.Ui, nil, diags)
+	if ret != 0 {
+		return ret
+	}
+
+	diags = c.Meta.ProcessOverrides(&cla.MetaArgs)
 	ret = writeDiags(c.Ui, nil, diags)
 	if ret != 0 {
 		return ret
@@ -411,6 +417,7 @@ Options:
   -machine-readable             Produce machine-readable output.
   -on-error=[cleanup|abort|ask|run-cleanup-provisioner] If the build fails do: clean up (default), abort, ask, or run-cleanup-provisioner.
   -parallel-builds=1            Number of builds to run in parallel. 1 disables parallelization. 0 means no limit (Default: 0)
+  -plugin-override              Force loading a plugin from a binary. Format must be accessor=path (ex: -plugin-override "amazon=./path/to/plugin")
   -timestamp-ui                 Enable prefixing of each ui output with an RFC3339 timestamp.
   -var 'key=value'              Variable for templates, can be used multiple times.
   -var-file=path                JSON or HCL2 file containing user variables, can be used multiple times.

command/cli.go

@@ -57,6 +57,7 @@ func (ma *MetaArgs) AddFlagSets(fs *flag.FlagSet) {
 	fs.Var((*sliceflag.StringFlag)(&ma.Only), "only", "")
 	fs.Var((*sliceflag.StringFlag)(&ma.Except), "except", "")
 	fs.Var((*kvflag.Flag)(&ma.Vars), "var", "")
+	fs.Var((*kvflag.Flag)(&ma.PluginPathOverrides), "plugin-override", "use that to force loading a plugin from a binary. Format must be accessor=path")
 	fs.Var((*kvflag.StringSlice)(&ma.VarFiles), "var-file", "")
 	fs.Var(&ma.ConfigType, "config-type", "set to 'hcl2' to run in hcl2 mode when no file is passed.")
 }
@@ -75,6 +76,32 @@ type MetaArgs struct {
 	// WarnOnUndeclared does not have a common default, as the default varies per sub-command usage.
 	// Refer to individual command FlagSets for usage.
 	WarnOnUndeclaredVar bool
+
+	// PluginPathOverrides is the list of plugins to use instead of the ones
+	// discovered manually.
+	//
+	// Each plugin specified here will have precedence over the plugins
+	// that are loaded normally by Packer.
+	PluginPathOverrides map[string]string
+}
+
+// getIgnoredPluginAccessors returns a list of accessors to not try to install
+//
+// This is mostly useful for commands like validate or build, which load the
+// required plugins, and to not attempt to load/verify them if they are
+// overridden by the command-line args
+func (ma MetaArgs) getIgnoredPluginAccessors() []string {
+	overrides := len(ma.PluginPathOverrides)
+	if overrides == 0 {
+		return nil
+	}
+
+	ret := make([]string, 0, overrides)
+	for acc := range ma.PluginPathOverrides {
+		ret = append(ret, acc)
+	}
+
+	return ret
 }
 
 func (ba *BuildArgs) AddFlagSets(flags *flag.FlagSet) {

command/meta.go

@@ -8,7 +8,9 @@ import (
 	"flag"
 	"fmt"
 	"io"
+	"log"
 	"os"
+	"regexp"
 	"strings"
 
 	"github.com/hashicorp/hcl/v2"
@@ -40,6 +42,55 @@ type Meta struct {
 	Version    string
 }
 
+// markPluginAsNonBundled marks each plugin with the accessor as prefix as not bundled
+//
+// This is required as otherwise we may detect an overridden plugin as bundled,
+// since the marking happens before we try to override them.
+func markPluginAsNonBundled(pluginAccessor string) {
+	compoAcc := regexp.MustCompile(fmt.Sprintf(
+		"^packer-(builder|post-processor|provisioner|datasource)-%s",
+		pluginAccessor))
+	var toMarkAsNonBundled []string
+	for pluginComponent := range bundledStatus {
+		if compoAcc.MatchString(pluginComponent) {
+			toMarkAsNonBundled = append(toMarkAsNonBundled, pluginComponent)
+		}
+	}
+
+	for _, cmp := range toMarkAsNonBundled {
+		bundledStatus[cmp] = false
+	}
+}
+
+// ProcessOverrides takes a list of plugin overrides loaded from the
+// command-line arguments
+//
+// Each override must point to a valid, existing binary, and will be
+// used over any component previously loaded, either automatically, or
+// through the `DetectPluginBinaries` function.
+func (m *Meta) ProcessOverrides(cla *MetaArgs) hcl.Diagnostics {
+	log.Printf("Processing overrides")
+	var diags hcl.Diagnostics
+
+	for accessor, path := range cla.PluginPathOverrides {
+		log.Printf("Loading plugin %q from %q", accessor, path)
+		err := m.CoreConfig.Components.PluginConfig.DiscoverMultiPlugin(accessor, path)
+		if err != nil {
+			diags = diags.Append(&hcl.Diagnostic{
+				Severity: hcl.DiagError,
+				Summary:  "failed to locate overridden plugin",
+				Detail:   fmt.Sprintf("The plugin %q could not be loaded from provided path %q", accessor, path),
+			})
+
+			continue
+		}
+
+		markPluginAsNonBundled(accessor)
+	}
+
+	return diags
+}
+
 // Core returns the core for the given template given the configured
 // CoreConfig and user variables on this Meta.
 func (m *Meta) Core(tpl *template.Template, cla *MetaArgs) (*packer.Core, error) {

command/validate.go

@@ -65,7 +65,13 @@ func (c *ValidateCommand) RunContext(ctx context.Context, cla *ValidateArgs) int
 		return 0
 	}
 
-	diags := packerStarter.DetectPluginBinaries()
+	diags := packerStarter.DetectPluginBinaries(cla.getIgnoredPluginAccessors())
+	ret = writeDiags(c.Ui, nil, diags)
+	if ret != 0 {
+		return ret
+	}
+
+	diags = c.Meta.ProcessOverrides(&cla.MetaArgs)
 	ret = writeDiags(c.Ui, nil, diags)
 	if ret != 0 {
 		return ret
@@ -120,6 +126,7 @@ Options:
   -var-file=path                JSON or HCL2 file containing user variables, can be used multiple times.
   -no-warn-undeclared-var       Disable warnings for user variable files containing undeclared variables.
   -evaluate-datasources         Evaluate data sources during validation (HCL2 only, may incur costs); Defaults to false. 
+  -plugin-override              Force loading a plugin from a binary. Format must be accessor=path (ex: -plugin-override "amazon=./path/to/plugin")
 `
 
 	return strings.TrimSpace(helpText)

hcl2template/plugin.go

@@ -54,7 +54,17 @@ func (cfg *PackerConfig) PluginRequirements() (plugingetter.Requirements, hcl.Di
 	return reqs, diags
 }
 
-func (cfg *PackerConfig) DetectPluginBinaries() hcl.Diagnostics {
+func isIgnored(pluginAccessor string, ignoredAccessors []string) bool {
+	for _, acc := range ignoredAccessors {
+		if acc == pluginAccessor {
+			return true
+		}
+	}
+
+	return false
+}
+
+func (cfg *PackerConfig) DetectPluginBinaries(ignoreAccessors []string) hcl.Diagnostics {
 	opts := plugingetter.ListInstallationsOptions{
 		FromFolders: cfg.parser.PluginConfig.KnownPluginFolders,
 		BinaryInstallationOptions: plugingetter.BinaryInstallationOptions{
@@ -80,6 +90,12 @@ func (cfg *PackerConfig) DetectPluginBinaries() hcl.Diagnostics {
 	uninstalledPlugins := map[string]string{}
 
 	for _, pluginRequirement := range pluginReqs {
+		// If the plugin with the accessor is loaded directly, we won't
+		// try to load it from here.
+		if isIgnored(pluginRequirement.Accessor, ignoreAccessors) {
+			continue
+		}
+
 		sortedInstalls, err := pluginRequirement.ListInstallations(opts)
 		if err != nil {
 			diags = append(diags, &hcl.Diagnostic{

packer/core.go

@@ -134,7 +134,7 @@ func NewCore(c *CoreConfig) *Core {
 
 // DetectPluginBinaries is used to load required plugins from the template,
 // since it is unsupported in JSON, this is essentially a no-op.
-func (c *Core) DetectPluginBinaries() hcl.Diagnostics {
+func (c *Core) DetectPluginBinaries([]string) hcl.Diagnostics {
 	return nil
 }
 

packer/run_interfaces.go

@@ -43,7 +43,7 @@ type InitializeOptions struct {
 type PluginBinaryDetector interface {
 	// DetectPluginBinaries is used only for HCL2 templates, and loads required
 	// plugins if specified.
-	DetectPluginBinaries() hcl.Diagnostics
+	DetectPluginBinaries(ignore []string) hcl.Diagnostics
 }
 
 // The Handler handles all Packer things. This interface reflects the Packer