From 4eae2df2f1510d5ce405355a1b33bcdff7ae1052 Mon Sep 17 00:00:00 2001 From: Lucas Bajolet Date: Fri, 1 Sep 2023 12:02:18 -0400 Subject: [PATCH] hcl2template: break down datasource execution Since we want to allow for independently executing datasources when needed, we break down the logic to do so. Each datasource will now hold a list of dependencies. For now, these dependencies are all datasources themselves, but this may change in the future. Then, actual execution of the datasource is self-contained within a method of the structure, so that we can individually execute them. Finally, we replace the current sequential execution by a method of the PackerConfig with an approach akin to local variable evaluation, where we execute all the datasources one-by-one, and if they cannot be executed, move along. We retry this loop for as many times as we don't reach either a state where all the datasources have been executed; or when we reach a terminal state where datasources cannot be evaluated because of dependencies remaining, and no changes have occurred after a try. --- hcl2template/parser.go | 2 + hcl2template/types.datasource.go | 161 ++++++++++++++++++++++++++++ hcl2template/types.packer_config.go | 119 +------------------- 3 files changed, 164 insertions(+), 118 deletions(-) diff --git a/hcl2template/parser.go b/hcl2template/parser.go index 8f0883a22..d7c57ef19 100644 --- a/hcl2template/parser.go +++ b/hcl2template/parser.go @@ -370,6 +370,8 @@ func (cfg *PackerConfig) decodeDatasource(block *hcl.Block) hcl.Diagnostics { } cfg.Datasources[ref] = datasource + datasource.getDependencies() + return diags } diff --git a/hcl2template/types.datasource.go b/hcl2template/types.datasource.go index 920b8ac15..765401bd4 100644 --- a/hcl2template/types.datasource.go +++ b/hcl2template/types.datasource.go @@ -7,8 +7,10 @@ import ( "fmt" "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/hcldec" packersdk "github.com/hashicorp/packer-plugin-sdk/packer" hcl2shim "github.com/hashicorp/packer/hcl2template/shim" + "github.com/hashicorp/packer/packer" "github.com/zclconf/go-cty/cty" ) @@ -19,6 +21,9 @@ type DatasourceBlock struct { value cty.Value block *hcl.Block + + // dependencies is the list of datasources to execute before this one + dependencies []DatasourceRef } type DatasourceRef struct { @@ -35,6 +40,101 @@ func (data *DatasourceBlock) Ref() DatasourceRef { } } +func (ds *DatasourceBlock) getDependencies() { + var dependencies []DatasourceRef + + // Note: when looking at the expressions, we only need to care about + // attributes, as HCL2 expressions are not allowed in a block's labels. + vars := GetVarsByType(ds.block, "data") + for _, v := range vars { + // construct, backwards, the data source type and name we + // need to evaluate before this one can be evaluated. + dependencies = append(dependencies, DatasourceRef{ + Type: v[1].(hcl.TraverseAttr).Name, + Name: v[2].(hcl.TraverseAttr).Name, + }) + } + + ds.dependencies = dependencies +} + +const notReadyDataSourceError = "Dependencies not ready" + +// executed returns whether or not the datasource was executed +// +// Having a non-empty cty.Value object means this was filled-up after the +// datasource has been executed, so this is what we use for this test. +func (ds DatasourceBlock) executed() bool { + return ds.value != cty.Value{} +} + +// Execute starts the datasource and executes it immediately. +// +// If its dependencies are not ready for execution, this will return an error +// and will only execute when all its dependencies have executed. +func (ds *DatasourceBlock) Execute(cfg *PackerConfig, skipExecution bool) hcl.Diagnostics { + var diags hcl.Diagnostics + + ok := true + for _, depRef := range ds.dependencies { + dep := cfg.Datasources[depRef] + if dep == nil { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Nonexistent dependency referenced", + Detail: fmt.Sprintf("The referenced datasource %s.%s is not defined in the configuration, so this datasource won't be able to execute.", dep.Type, dep.Name), + Subject: &ds.block.DefRange, + }) + ok = false + continue + } + + if !dep.executed() { + ok = false + } + } + + if !ok { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: notReadyDataSourceError, + Detail: "At least one dependency for the datasource is not executed already", + Subject: &ds.block.DefRange, + }) + return diags + } + + // If we've gotten here, then it means ref doesn't seem to have any further + // dependencies we need to evaluate first. Evaluate it, with the cfg's full + // data source context. + datasource, diags := cfg.startDatasource(*ds) + if diags.HasErrors() { + return diags + } + + if skipExecution { + placeholderValue := cty.UnknownVal(hcldec.ImpliedType(datasource.OutputSpec())) + ds.value = placeholderValue + return diags + } + + opts, _ := decodeHCL2Spec(ds.block.Body, cfg.EvalContext(DatasourceContext, nil), datasource) + sp := packer.CheckpointReporter.AddSpan(ds.Type, "datasource", opts) + realValue, err := datasource.Execute() + sp.End(err) + if err != nil { + diags = append(diags, &hcl.Diagnostic{ + Summary: err.Error(), + Subject: &ds.block.DefRange, + Severity: hcl.DiagError, + }) + return diags + } + + ds.value = realValue + return diags +} + func (ds *Datasources) Values() (map[string]cty.Value, hcl.Diagnostics) { var diags hcl.Diagnostics res := map[string]cty.Value{} @@ -130,3 +230,64 @@ func (cfg *PackerConfig) startDatasource(ds DatasourceBlock) (packersdk.Datasour } return datasource, diags } + +// datasourcesDone checks whether all the datasources have been executed or not +func (cfg *PackerConfig) datasourcesDone() bool { + for _, ds := range cfg.Datasources { + if !ds.executed() { + return false + } + } + + return true +} + +func (cfg *PackerConfig) executeDatasources(skipExecution bool) hcl.Diagnostics { + // If we are done with datasources execution, we leave immediately + if cfg.datasourcesDone() { + return nil + } + + var outDiags hcl.Diagnostics + + foundSomething := false +outerDSEval: + for _, ds := range cfg.Datasources { + if ds.executed() { + continue + } + + diags := ds.Execute(cfg, skipExecution) + for _, diag := range diags { + if diag.Summary == notReadyDataSourceError { + // If we have a not ready error in the + // datasource list, we should attempt to run the + // rest, and eventually settle if we cannot move + // any further + continue outerDSEval + } + } + + foundSomething = true + + outDiags = append(outDiags, diags...) + } + + if !foundSomething { + return append(outDiags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "No datasource could be executed", + Detail: `While trying to recurisvely evaluating datasources, we could not find a next datasource to execute. +This is likely due to a cyclic dependency in your datasources`, + }) + } + + // If we couldn't execute a datasource for whatever reason, we leave + if outDiags.HasErrors() { + return outDiags + } + + // If we still found something to execute, we recursively execute the + // remainder of the datasources + return outDiags.Extend(cfg.executeDatasources(skipExecution)) +} diff --git a/hcl2template/types.packer_config.go b/hcl2template/types.packer_config.go index 9a2c159a7..6bb965785 100644 --- a/hcl2template/types.packer_config.go +++ b/hcl2template/types.packer_config.go @@ -10,7 +10,6 @@ import ( "github.com/gobwas/glob" hcl "github.com/hashicorp/hcl/v2" - "github.com/hashicorp/hcl/v2/hcldec" "github.com/hashicorp/hcl/v2/hclsyntax" packersdk "github.com/hashicorp/packer-plugin-sdk/packer" pkrfunction "github.com/hashicorp/packer/hcl2template/function" @@ -264,122 +263,6 @@ func (c *PackerConfig) evaluateLocalVariable(local *LocalBlock) hcl.Diagnostics return diags } -func (cfg *PackerConfig) evaluateDatasources(skipExecution bool) hcl.Diagnostics { - var diags hcl.Diagnostics - - dependencies := map[DatasourceRef][]DatasourceRef{} - for ref, ds := range cfg.Datasources { - if ds.value != (cty.Value{}) { - continue - } - // Pre-examine body of this data source to see if it uses another data - // source in any of its input expressions. If so, skip evaluating it for - // now, and add it to a list of datasources to evaluate again, later, - // with the datasources in its context. - dependencies[ref] = []DatasourceRef{} - - // Note: when looking at the expressions, we only need to care about - // attributes, as HCL2 expressions are not allowed in a block's labels. - vars := GetVarsByType(ds.block, "data") - for _, v := range vars { - // construct, backwards, the data source type and name we - // need to evaluate before this one can be evaluated. - dependsOn := DatasourceRef{ - Type: v[1].(hcl.TraverseAttr).Name, - Name: v[2].(hcl.TraverseAttr).Name, - } - dependencies[ref] = append(dependencies[ref], dependsOn) - } - } - - // Now that most of our data sources have been started and executed, we can - // try to execute the ones that depend on other data sources. - for ref := range dependencies { - _, moreDiags := cfg.recursivelyEvaluateDatasources(ref, dependencies, skipExecution, 0) - // Deduplicate diagnostics to prevent recursion messes. - cleanedDiags := map[string]*hcl.Diagnostic{} - for _, diag := range moreDiags { - cleanedDiags[diag.Summary] = diag - } - - for _, diag := range cleanedDiags { - diags = append(diags, diag) - } - } - - return diags -} - -func (cfg *PackerConfig) recursivelyEvaluateDatasources(ref DatasourceRef, dependencies map[DatasourceRef][]DatasourceRef, skipExecution bool, depth int) (map[DatasourceRef][]DatasourceRef, hcl.Diagnostics) { - var diags hcl.Diagnostics - var moreDiags hcl.Diagnostics - - if depth > 10 { - // Add a comment about recursion. - diags = append(diags, &hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "Max datasource recursion depth exceeded.", - Detail: "An error occured while recursively evaluating data " + - "sources. Either your data source depends on more than ten " + - "other data sources, or your data sources have a cyclic " + - "dependency. Please simplify your config to continue. ", - Subject: &(cfg.Datasources[ref]).block.DefRange, - }) - return dependencies, diags - } - - ds := cfg.Datasources[ref] - // Make sure everything ref depends on has already been evaluated. - for _, dep := range dependencies[ref] { - if _, ok := dependencies[dep]; ok { - depth += 1 - // If this dependency is not in the map, it means we've already - // launched and executed this datasource. Otherwise, it means - // we still need to run it. RECURSION TIME!! - dependencies, moreDiags = cfg.recursivelyEvaluateDatasources(dep, dependencies, skipExecution, depth) - diags = append(diags, moreDiags...) - if moreDiags.HasErrors() { - diags = append(diags, moreDiags...) - return dependencies, diags - } - } - } - // If we've gotten here, then it means ref doesn't seem to have any further - // dependencies we need to evaluate first. Evaluate it, with the cfg's full - // data source context. - datasource, startDiags := cfg.startDatasource(ds) - if startDiags.HasErrors() { - diags = append(diags, startDiags...) - return dependencies, diags - } - - if skipExecution { - placeholderValue := cty.UnknownVal(hcldec.ImpliedType(datasource.OutputSpec())) - ds.value = placeholderValue - cfg.Datasources[ref] = ds - return dependencies, diags - } - - opts, _ := decodeHCL2Spec(ds.block.Body, cfg.EvalContext(DatasourceContext, nil), datasource) - sp := packer.CheckpointReporter.AddSpan(ref.Type, "datasource", opts) - realValue, err := datasource.Execute() - sp.End(err) - if err != nil { - diags = append(diags, &hcl.Diagnostic{ - Summary: err.Error(), - Subject: &cfg.Datasources[ref].block.DefRange, - Severity: hcl.DiagError, - }) - return dependencies, diags - } - - ds.value = realValue - cfg.Datasources[ref] = ds - // remove ref from the dependencies map. - delete(dependencies, ref) - return dependencies, diags -} - // getCoreBuildProvisioners takes a list of provisioner block, starts according // provisioners and sends parsed HCL2 over to it. func (cfg *PackerConfig) getCoreBuildProvisioners(source SourceUseBlock, blocks []*ProvisionerBlock, ectx *hcl.EvalContext) ([]packer.CoreBuildProvisioner, hcl.Diagnostics) { @@ -799,7 +682,7 @@ func (p *PackerConfig) InspectConfig(opts packer.InspectConfigOptions) int { func (cfg *PackerConfig) Initialize(opts packer.InitializeOptions) hcl.Diagnostics { diags := cfg.InputVariables.ValidateValues() diags = append(diags, cfg.LocalVariables.ValidateValues()...) - diags = append(diags, cfg.evaluateDatasources(opts.SkipDatasourcesExecution)...) + diags = append(diags, cfg.executeDatasources(opts.SkipDatasourcesExecution)...) diags = append(diags, checkForDuplicateLocalDefinition(cfg.LocalBlocks)...) diags = append(diags, cfg.evaluateLocalVariables(cfg.LocalBlocks)...)