iSharkFly-Docs/Packer-Cn
2
0
Fork 0
mirror of https://github.com/hashicorp/packer.git synced 2026-09-20 06:51:39 -04:00
Code Issues Packages Projects Releases Wiki Activity

hcl2template: break down datasource execution

Browse Source 
Since we want to allow for independently executing datasources when
needed, we break down the logic to do so.

Each datasource will now hold a list of dependencies. For now, these
dependencies are all datasources themselves, but this may change in the
future.

Then, actual execution of the datasource is self-contained within a
method of the structure, so that we can individually execute them.

Finally, we replace the current sequential execution by a method of the
PackerConfig with an approach akin to local variable evaluation, where
we execute all the datasources one-by-one, and if they cannot be
executed, move along.
We retry this loop for as many times as we don't reach either a state
where all the datasources have been executed; or when we reach a
terminal state where datasources cannot be evaluated because of
dependencies remaining, and no changes have occurred after a try.
This commit is contained in:
Lucas Bajolet
2023-10-11 14:28:27 -04:00
parent 96a528be0a
commit 4eae2df2f1
3 changed files with 164 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files
hcl2template/parser.go
+2
View File
@@ -370,6 +370,8 @@ func (cfg *PackerConfig) decodeDatasource(block *hcl.Block) hcl.Diagnostics {
}
cfg.Datasources[ref] = datasource
datasource.getDependencies()
return diags
}
hcl2template/types.datasource.go
+161
View File
@@ -7,8 +7,10 @@ import (
"fmt"
"github.com/hashicorp/hcl/v2"
"github.com/hashicorp/hcl/v2/hcldec"
packersdk "github.com/hashicorp/packer-plugin-sdk/packer"
hcl2shim "github.com/hashicorp/packer/hcl2template/shim"
"github.com/hashicorp/packer/packer"
"github.com/zclconf/go-cty/cty"
)
@@ -19,6 +21,9 @@ type DatasourceBlock struct {
value cty.Value
block *hcl.Block
// dependencies is the list of datasources to execute before this one
dependencies []DatasourceRef
}
type DatasourceRef struct {
@@ -35,6 +40,101 @@ func (data *DatasourceBlock) Ref() DatasourceRef {
}
}
func (ds *DatasourceBlock) getDependencies() {
var dependencies []DatasourceRef
// Note: when looking at the expressions, we only need to care about
// attributes, as HCL2 expressions are not allowed in a block's labels.
vars := GetVarsByType(ds.block, "data")
for _, v := range vars {
// construct, backwards, the data source type and name we
// need to evaluate before this one can be evaluated.
dependencies = append(dependencies, DatasourceRef{
Type: v[1].(hcl.TraverseAttr).Name,
Name: v[2].(hcl.TraverseAttr).Name,
})
}
ds.dependencies = dependencies
}
const notReadyDataSourceError = "Dependencies not ready"
// executed returns whether or not the datasource was executed
//
// Having a non-empty cty.Value object means this was filled-up after the
// datasource has been executed, so this is what we use for this test.
func (ds DatasourceBlock) executed() bool {
return ds.value != cty.Value{}
}
// Execute starts the datasource and executes it immediately.
//
// If its dependencies are not ready for execution, this will return an error
// and will only execute when all its dependencies have executed.
func (ds *DatasourceBlock) Execute(cfg *PackerConfig, skipExecution bool) hcl.Diagnostics {
var diags hcl.Diagnostics
ok := true
for _, depRef := range ds.dependencies {
dep := cfg.Datasources[depRef]
if dep == nil {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Nonexistent dependency referenced",
Detail: fmt.Sprintf("The referenced datasource %s.%s is not defined in the configuration, so this datasource won't be able to execute.", dep.Type, dep.Name),
Subject: &ds.block.DefRange,
})
ok = false
continue
}
if !dep.executed() {
ok = false
}
}
if !ok {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: notReadyDataSourceError,
Detail: "At least one dependency for the datasource is not executed already",
Subject: &ds.block.DefRange,
})
return diags
}
// If we've gotten here, then it means ref doesn't seem to have any further
// dependencies we need to evaluate first. Evaluate it, with the cfg's full
// data source context.
datasource, diags := cfg.startDatasource(*ds)
if diags.HasErrors() {
return diags
}
if skipExecution {
placeholderValue := cty.UnknownVal(hcldec.ImpliedType(datasource.OutputSpec()))
ds.value = placeholderValue
return diags
}
opts, _ := decodeHCL2Spec(ds.block.Body, cfg.EvalContext(DatasourceContext, nil), datasource)
sp := packer.CheckpointReporter.AddSpan(ds.Type, "datasource", opts)
realValue, err := datasource.Execute()
sp.End(err)
if err != nil {
diags = append(diags, &hcl.Diagnostic{
Summary: err.Error(),
Subject: &ds.block.DefRange,
Severity: hcl.DiagError,
})
return diags
}
ds.value = realValue
return diags
}
func (ds *Datasources) Values() (map[string]cty.Value, hcl.Diagnostics) {
var diags hcl.Diagnostics
res := map[string]cty.Value{}
@@ -130,3 +230,64 @@ func (cfg *PackerConfig) startDatasource(ds DatasourceBlock) (packersdk.Datasour
}
return datasource, diags
}
// datasourcesDone checks whether all the datasources have been executed or not
func (cfg *PackerConfig) datasourcesDone() bool {
for _, ds := range cfg.Datasources {
if !ds.executed() {
return false
}
}
return true
}
func (cfg *PackerConfig) executeDatasources(skipExecution bool) hcl.Diagnostics {
// If we are done with datasources execution, we leave immediately
if cfg.datasourcesDone() {
return nil
}
var outDiags hcl.Diagnostics
foundSomething := false
outerDSEval:
for _, ds := range cfg.Datasources {
if ds.executed() {
continue
}
diags := ds.Execute(cfg, skipExecution)
for _, diag := range diags {
if diag.Summary == notReadyDataSourceError {
// If we have a not ready error in the
// datasource list, we should attempt to run the
// rest, and eventually settle if we cannot move
// any further
continue outerDSEval
}
}
foundSomething = true
outDiags = append(outDiags, diags...)
}
if !foundSomething {
return append(outDiags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "No datasource could be executed",
Detail: `While trying to recurisvely evaluating datasources, we could not find a next datasource to execute.
This is likely due to a cyclic dependency in your datasources`,
})
}
// If we couldn't execute a datasource for whatever reason, we leave
if outDiags.HasErrors() {
return outDiags
}
// If we still found something to execute, we recursively execute the
// remainder of the datasources
return outDiags.Extend(cfg.executeDatasources(skipExecution))
}
hcl2template/types.packer_config.go
+1 -118
View File
@@ -10,7 +10,6 @@ import (
"github.com/gobwas/glob"
hcl "github.com/hashicorp/hcl/v2"
"github.com/hashicorp/hcl/v2/hcldec"
"github.com/hashicorp/hcl/v2/hclsyntax"
packersdk "github.com/hashicorp/packer-plugin-sdk/packer"
pkrfunction "github.com/hashicorp/packer/hcl2template/function"
@@ -264,122 +263,6 @@ func (c *PackerConfig) evaluateLocalVariable(local *LocalBlock) hcl.Diagnostics
return diags
}
func (cfg *PackerConfig) evaluateDatasources(skipExecution bool) hcl.Diagnostics {
var diags hcl.Diagnostics
dependencies := map[DatasourceRef][]DatasourceRef{}
for ref, ds := range cfg.Datasources {
if ds.value != (cty.Value{}) {
continue
}
// Pre-examine body of this data source to see if it uses another data
// source in any of its input expressions. If so, skip evaluating it for
// now, and add it to a list of datasources to evaluate again, later,
// with the datasources in its context.
dependencies[ref] = []DatasourceRef{}
// Note: when looking at the expressions, we only need to care about
// attributes, as HCL2 expressions are not allowed in a block's labels.
vars := GetVarsByType(ds.block, "data")
for _, v := range vars {
// construct, backwards, the data source type and name we
// need to evaluate before this one can be evaluated.
dependsOn := DatasourceRef{
Type: v[1].(hcl.TraverseAttr).Name,
Name: v[2].(hcl.TraverseAttr).Name,
}
dependencies[ref] = append(dependencies[ref], dependsOn)
}
}
// Now that most of our data sources have been started and executed, we can
// try to execute the ones that depend on other data sources.
for ref := range dependencies {
_, moreDiags := cfg.recursivelyEvaluateDatasources(ref, dependencies, skipExecution, 0)
// Deduplicate diagnostics to prevent recursion messes.
cleanedDiags := map[string]*hcl.Diagnostic{}
for _, diag := range moreDiags {
cleanedDiags[diag.Summary] = diag
}
for _, diag := range cleanedDiags {
diags = append(diags, diag)
}
}
return diags
}
func (cfg *PackerConfig) recursivelyEvaluateDatasources(ref DatasourceRef, dependencies map[DatasourceRef][]DatasourceRef, skipExecution bool, depth int) (map[DatasourceRef][]DatasourceRef, hcl.Diagnostics) {
var diags hcl.Diagnostics
var moreDiags hcl.Diagnostics
if depth > 10 {
// Add a comment about recursion.
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Max datasource recursion depth exceeded.",
Detail: "An error occured while recursively evaluating data " +
"sources. Either your data source depends on more than ten " +
"other data sources, or your data sources have a cyclic " +
"dependency. Please simplify your config to continue. ",
Subject: &(cfg.Datasources[ref]).block.DefRange,
})
return dependencies, diags
}
ds := cfg.Datasources[ref]
// Make sure everything ref depends on has already been evaluated.
for _, dep := range dependencies[ref] {
if _, ok := dependencies[dep]; ok {
depth += 1
// If this dependency is not in the map, it means we've already
// launched and executed this datasource. Otherwise, it means
// we still need to run it. RECURSION TIME!!
dependencies, moreDiags = cfg.recursivelyEvaluateDatasources(dep, dependencies, skipExecution, depth)
diags = append(diags, moreDiags...)
if moreDiags.HasErrors() {
diags = append(diags, moreDiags...)
return dependencies, diags
}
}
}
// If we've gotten here, then it means ref doesn't seem to have any further
// dependencies we need to evaluate first. Evaluate it, with the cfg's full
// data source context.
datasource, startDiags := cfg.startDatasource(ds)
if startDiags.HasErrors() {
diags = append(diags, startDiags...)
return dependencies, diags
}
if skipExecution {
placeholderValue := cty.UnknownVal(hcldec.ImpliedType(datasource.OutputSpec()))
ds.value = placeholderValue
cfg.Datasources[ref] = ds
return dependencies, diags
}
opts, _ := decodeHCL2Spec(ds.block.Body, cfg.EvalContext(DatasourceContext, nil), datasource)
sp := packer.CheckpointReporter.AddSpan(ref.Type, "datasource", opts)
realValue, err := datasource.Execute()
sp.End(err)
if err != nil {
diags = append(diags, &hcl.Diagnostic{
Summary: err.Error(),
Subject: &cfg.Datasources[ref].block.DefRange,
Severity: hcl.DiagError,
})
return dependencies, diags
}
ds.value = realValue
cfg.Datasources[ref] = ds
// remove ref from the dependencies map.
delete(dependencies, ref)
return dependencies, diags
}
// getCoreBuildProvisioners takes a list of provisioner block, starts according
// provisioners and sends parsed HCL2 over to it.
func (cfg *PackerConfig) getCoreBuildProvisioners(source SourceUseBlock, blocks []*ProvisionerBlock, ectx *hcl.EvalContext) ([]packer.CoreBuildProvisioner, hcl.Diagnostics) {
@@ -799,7 +682,7 @@ func (p *PackerConfig) InspectConfig(opts packer.InspectConfigOptions) int {
func (cfg *PackerConfig) Initialize(opts packer.InitializeOptions) hcl.Diagnostics {
diags := cfg.InputVariables.ValidateValues()
diags = append(diags, cfg.LocalVariables.ValidateValues()...)
diags = append(diags, cfg.evaluateDatasources(opts.SkipDatasourcesExecution)...)
diags = append(diags, cfg.executeDatasources(opts.SkipDatasourcesExecution)...)
diags = append(diags, checkForDuplicateLocalDefinition(cfg.LocalBlocks)...)
diags = append(diags, cfg.evaluateLocalVariables(cfg.LocalBlocks)...)