empty commit

improve Implicit required plugin section

.gitattributes

@@ -6,6 +6,7 @@
 *.mdx text eol=lf
 *.ps1 text eol=lf
 *.hcl text eol=lf
+*.tmpl text eol=lf
 *.txt text eol=lf
 go.mod text eol=lf
 go.sum text eol=lf

.github/CONTRIBUTING.md

@@ -11,6 +11,12 @@ contribute to the project, read on. This document will cover what we're looking
 for. By addressing all the points we're looking for, it raises the chances we
 can quickly merge or address your contributions.
 
+When contributing in any way to the Packer project (new issue, PR, etc), please 
+be aware that our team identifies with many gender pronouns. Please remember to 
+use nonbinary pronouns (they/them) and gender neutral language ("Hello folks") 
+when addressing our team. For more reading on our code of conduct, please see the 
+[HashiCorp community guidelines](https://www.hashicorp.com/community-guidelines).
+
 ## Issues
 
 ### Reporting an Issue

.github/workflows/check-plugin-docs.js

@@ -0,0 +1,77 @@
+const fs = require("fs");
+const path = require("path");
+const fetchPluginDocs = require("../../website/components/remote-plugin-docs/utils/fetch-plugin-docs");
+
+const COLOR_RESET = "\x1b[0m";
+const COLOR_GREEN = "\x1b[32m";
+const COLOR_BLUE = "\x1b[34m";
+const COLOR_RED = "\x1b[31m";
+
+async function checkPluginDocs() {
+  const failureMessages = [];
+  const pluginsPath = "website/data/docs-remote-plugins.json";
+  const pluginsFile = fs.readFileSync(path.join(process.cwd(), pluginsPath));
+  const pluginEntries = JSON.parse(pluginsFile);
+  const entriesCount = pluginEntries.length;
+  console.log(`\nResolving plugin docs from ${entriesCount} repositories …`);
+  for (var i = 0; i < entriesCount; i++) {
+    const pluginEntry = pluginEntries[i];
+    const { title, repo, version } = pluginEntry;
+    console.log(`\n${COLOR_BLUE}${repo}${COLOR_RESET} | ${title}`);
+    console.log(`Fetching docs from release "${version}" …`);
+    try {
+      const undefinedProps = ["title", "repo", "version", "path"].filter(
+        (key) => typeof pluginEntry[key] == "undefined"
+      );
+      if (undefinedProps.length > 0) {
+        throw new Error(
+          `Failed to validate plugin docs config. Undefined configuration properties ${JSON.stringify(
+            undefinedProps
+          )} found for "${
+            title || pluginEntry.path || repo
+          }". In "website/data/docs-remote-plugins.json", please ensure the missing properties ${JSON.stringify(
+            undefinedProps
+          )} are defined. Additional information on this configuration can be found in "website/README.md".`
+        );
+      }
+      const docsMdxFiles = await fetchPluginDocs({ repo, tag: version });
+      const mdxFilesByComponent = docsMdxFiles.reduce((acc, mdxFile) => {
+        const componentType = mdxFile.filePath.split("/")[1];
+        if (!acc[componentType]) acc[componentType] = [];
+        acc[componentType].push(mdxFile);
+        return acc;
+      }, {});
+      console.log(`${COLOR_GREEN}Found valid docs:${COLOR_RESET}`);
+      Object.keys(mdxFilesByComponent).forEach((component) => {
+        const componentFiles = mdxFilesByComponent[component];
+        console.log(`  ${component}`);
+        componentFiles.forEach(({ filePath }) => {
+          const pathFromComponent = filePath.split("/").slice(2).join("/");
+          console.log(`  ├── ${pathFromComponent}`);
+        });
+      });
+    } catch (err) {
+      console.log(`${COLOR_RED}${err}${COLOR_RESET}`);
+      failureMessages.push(`\n${COLOR_RED}× ${repo}: ${COLOR_RESET}${err}`);
+    }
+  }
+
+  if (failureMessages.length === 0) {
+    console.log(
+      `\n---\n\n${COLOR_GREEN}Summary: Successfully resolved all plugin docs.`
+    );
+    pluginEntries.forEach((e) =>
+      console.log(`${COLOR_GREEN}✓ ${e.repo}${COLOR_RESET}`)
+    );
+    console.log("");
+  } else {
+    console.log(
+      `\n---\n\n${COLOR_RED}Summary: Failed to fetch docs for ${failureMessages.length} plugin(s):`
+    );
+    failureMessages.forEach((err) => console.log(err));
+    console.log("");
+    process.exit(1);
+  }
+}
+
+checkPluginDocs();

.github/workflows/check-plugin-docs.yml

@@ -0,0 +1,29 @@
+#
+# This GitHub action checks plugin repositories for valid docs.
+#
+# This provides a quick assessment on PRs of whether
+# there might be issues with docs in plugin repositories.
+#
+# This is intended to help debug Vercel build issues, which
+# may or may not be related to docs in plugin repositories.
+
+name: "website: Check plugin docs"
+on:
+  pull_request:
+    paths:
+      - "website/**"
+  schedule:
+    - cron: "45 0 * * *"
+
+jobs:
+  check-plugin-docs:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Setup Node
+        uses: actions/setup-node@v1
+      - name: Install Dependencies
+        run: npm i isomorphic-unfetch adm-zip gray-matter
+      - name: Fetch and validate plugin docs
+        run: node .github/workflows/check-plugin-docs.js

.github/workflows/linkchecker.yml

@@ -7,7 +7,7 @@ name: Check markdown links on modified website files
 jobs:
   vercel-deployment-poll:
     runs-on: ubuntu-latest
-    timeout-minutes: 3 #cancel job if no deployment is found within x minutes
+    timeout-minutes: 5 #cancel job if no deployment is found within x minutes
     outputs:
       url: ${{ steps.waitForVercelPreviewDeployment.outputs.url }}
     steps:

.hashibot.hcl

@@ -22,3 +22,21 @@ poll "closed_issue_locker" "locker" {
     If you have found a problem that seems similar to this, please open a new issue and complete the issue template so we can capture all the details necessary to investigate further.
   EOF
 }
+
+poll "label_issue_migrater" "remote_plugin_migrater" {
+  schedule                = "0 20 * * * *"
+  new_owner               = "hashicorp"
+  repo_prefix             = "packer-plugin-"
+  label_prefix            = "remote-plugin/"
+  excluded_label_prefixes  = ["communicator/"]
+  excluded_labels         = ["build", "core", "new-plugin-contribution", "website"]
+
+  issue_header     = <<-EOF
+    _This issue was originally opened by @${var.user} as ${var.repository}#${var.issue_number}. It was migrated here as a result of the [Packer plugin split](https://github.com/hashicorp/packer/issues/8610#issuecomment-770034737). The original body of the issue is below._
+
+    <hr>
+
+    EOF
+  migrated_comment = "This issue has been automatically migrated to ${var.repository}#${var.issue_number} because it looks like an issue with that plugin. If you believe this is _not_ an issue with the plugin, please reply to ${var.repository}#${var.issue_number}."
+}
+

CHANGELOG.md

@@ -1,30 +1,105 @@
 ## 1.7.1 (Upcoming)
 
+### NOTES:
+
+* builder/docker: Has been vendored in this release and will no longer be
+    updated with Packer core. In Packer v1.8.0 the plugin will be removed
+    entirely. The `docker` builder will continue to work as expected until
+    then, but for the latest offerings of the Docker plugin, users are
+    encourage to use the `packer init` command to install the latest release
+    version. For more details see [Installing Packer
+    Plugins](https://www.packer.io/docs/plugins#installing- plugins)
+* post-processor/docker-\*: Have been vendored in this release and will no
+    longer be updated with Packer core. In Packer v1.8.0 the plugin will be
+    removed entirely. The `docker` builder will continue to work as expected
+    until then, but for the latest offerings of the Docker plugin, users are
+    encourage to use the `packer init` command to install the latest release
+    version. For more details see [Installing Packer
+    Plugins](https://www.packer.io/docs/plugins#installing- plugins)
+* post-processor/exoscale-import: Has been vendored in this release and will no
+    longer be updated with Packer core. In Packer v1.8.0 the plugin will be
+    removed entirely. The `exoscale-import` post-processor will continue to
+    work as expected until then, but for the latest offerings of the Exoscale
+    plugin, users are encourage to use the `packer init` command to install the
+    latest release version. For more details see [Exoscale Plugin
+    Repostiroy](https://github.com/exoscale/packer-plugin-exoscale). [GH-10709]
+
 ### IMPROVEMENTS
 * builder/amazon: allow creation of ebs snapshots wihtout volumes. [GH-9591]
+* builder/amazon: Fix issue for multi-region AMI build that fail when
+    encrypting with KMS and sharing across accounts. [GH-10754]
+* builder/azure: Add client_cert_token_timeout option. [GH-10528]
+* builder/google: Make Windows password timeout configurable. [GH-10727]
+* builder/google: Update public GCP image project as gce-uefi-images are
+    deprecated. [GH-10724]
+* builder/qemu: Added firmware option. [GH-10683]
 * builder/scaleway: add support for timeout in shutdown step. [GH-10503]
+* builder/vagrant: Fix logging to be clearer when Vagrant builder overrides
+    values retrieved from vagrant's ssh_config call. [GH-10743]
+* builder/virtualbox:  Added ISO builder option to create additional disks.
+    [GH-10674]
+* builder/virtualbox: Add options for nested virtualisation and RTC time base.
+    [GH-10736]
 * builder/virtualbox: Add template options for chipset, firmware, nic, graphics
     controller, and audio controller. [GH-10671]
 * builder/virtualbox: Support for "virtio" storage and ISO drive. [GH-10632]
 * builder/vmware: Added "attach_snapshot" parameter to vmware vmx builder.
     [GH-10651]
+* command/fmt: Adding recursive flag to formatter to format subdirectories.
+    [GH-10457]
 * core: Change template parsing error to include warning about file extensions.
     [GH-10652]
+* core: Update to gopsutil v3.21.1 to allow builds to work for darwin arm64.
+    [GH-10697]
 * hcl2_upgrade: hcl2_upgrade command can now upgrade json var-files [GH-10676]
 
 ### BUG FIXES
+* buider/azure: Update builder to ensure a proper clean up Azure temporary
+    managed Os disks. [GH-10713]
 * builder/amazon: Update amazon SDK to fix an SSO login issue. [GH-10668]
 * builder/azure: Don't overwrite subscription id if unset. [GH-10659]
+* builder/azure: Set default for the parameter client_cert_token_timeout
+    [GH-10783]
+* builder/google: Add new configuration field `windows_password_timeout` to
+    allow user to set configurable timeouts. [GH-10727]
+* builder/hyperv: Make Packer respect winrm_host flag in winrm connect func.
+    [GH-10748]
+* builder/openstack: Make Packer respect winrm_host flag in winrm connect func.
+    [GH-10748]
 * builder/oracle-oci: Update Oracle Go SDK to fix issue with reading key file.
-    [GH-10560]
+    [GH-10560] [GH-10774]
+* builder/parallels: Make Packer respect winrm_host flag in winrm connect func.
+    [GH-10748]
+* builder/proxmox: Fixes issue when using `additional_iso_files` in HCL enabled
+    templates. [GH-10772]
+* builder/qemu: Make Packer respect winrm_host flag in winrm connect func.
+    [GH-10748]
+* builder/virtualbox: Make Packer respect winrm_host flag in winrm connect
+    func. [GH-10748]
 * builder/vmware: Added a fallback file check when trying to determine the
     network-mapping configuration. [GH-10543]
+* builder/vsphere: Fix issue where boot command would fail the build do to a
+    key typing error. This change will now retry to type the key on error
+    before giving up. [GH-10541]
+* core/hcl2_upgrade: Check for nil config map when provisioner/post-processor
+    doesn't have config. [GH-10730]
 * core/hcl2_upgrade: Make hcl2_upgrade command correctly translate
     pause_before. [GH-10654]
+* core/hcl2_upgrade: Make json variables using template engines get stored as
+    locals so they can be properly interpolated. [GH-10685]
+* core/init: Fixes issue where `packer init` was failing to install valid
+    plugins containing a 'v' within its name. [GH-10760]
+* core: Packer will now show a proper error message when failing to load the
+    contents of PACKER_CONFIG. [GH-10766]
+* core: Pin Packer to Golang 1.16 to fix code generation issues. [GH-10702]
 * core: Templates previously could not interpolate the environment variable
     PACKER_LOG_PATH. [GH-10660]
 * provisioner/chef-solo: HCL2 templates can support the json_string option.
     [GH-10655]
+* provisioner/inspec: Add new configuration field `valid_exit_codes` to allow
+    for non-zero exit codes. [GH-10723]
+* provisioner/salt-masterless: Update urls for the bootstrap scripts used by
+    salt-masterless provide. [GH-10755]
 
 ## 1.7.0 (February 17, 2021)
 

builder/amazon/common/access_config.go

@@ -150,7 +150,7 @@ type AccessConfig struct {
 	// environmental variable.
 	Token   string `mapstructure:"token" required:"false"`
 	session *session.Session
-	// Get credentials from Hashicorp Vault's aws secrets engine. You must
+	// Get credentials from HashiCorp Vault's aws secrets engine. You must
 	// already have created a role to use. For more information about
 	// generating credentials via the Vault engine, see the [Vault
 	// docs.](https://www.vaultproject.io/api/secret/aws#generate-credentials)

builder/azure/arm/builder_acc_test.go

@@ -20,12 +20,18 @@ package arm
 //   go test -v -timeout 90m -run TestBuilderAcc_.*
 
 import (
-	"testing"
-
+	"bytes"
+	"context"
+	"errors"
 	"fmt"
 	"os"
+	"strings"
+	"testing"
 
+	"github.com/Azure/go-autorest/autorest"
+	"github.com/Azure/go-autorest/autorest/azure/auth"
 	builderT "github.com/hashicorp/packer-plugin-sdk/acctest"
+	packersdk "github.com/hashicorp/packer-plugin-sdk/packer"
 )
 
 const DeviceLoginAcceptanceTest = "DEVICELOGIN_TEST"
@@ -96,10 +102,15 @@ func TestBuilderAcc_ManagedDisk_Linux_AzureCLI(t *testing.T) {
 		return
 	}
 
+	var b Builder
 	builderT.Test(t, builderT.TestCase{
-		PreCheck: func() { testAccPreCheck(t) },
-		Builder:  &Builder{},
+		PreCheck: func() { testAuthPreCheck(t) },
+		Builder:  &b,
 		Template: testBuilderAccManagedDiskLinuxAzureCLI,
+		Check: func([]packersdk.Artifact) error {
+			checkTemporaryGroupDeleted(t, &b)
+			return nil
+		},
 	})
 }
 
@@ -112,15 +123,108 @@ func TestBuilderAcc_Blob_Windows(t *testing.T) {
 }
 
 func TestBuilderAcc_Blob_Linux(t *testing.T) {
+	var b Builder
 	builderT.Test(t, builderT.TestCase{
-		PreCheck: func() { testAccPreCheck(t) },
-		Builder:  &Builder{},
+		PreCheck: func() { testAuthPreCheck(t) },
+		Builder:  &b,
 		Template: testBuilderAccBlobLinux,
+		Check: func([]packersdk.Artifact) error {
+			checkUnmanagedVHDDeleted(t, &b)
+			return nil
+		},
 	})
 }
 
 func testAccPreCheck(*testing.T) {}
 
+func testAuthPreCheck(t *testing.T) {
+	_, err := auth.NewAuthorizerFromEnvironment()
+	if err != nil {
+		t.Fatalf("failed to auth to azure: %s", err)
+	}
+}
+
+func checkTemporaryGroupDeleted(t *testing.T, b *Builder) {
+	ui := testUi()
+
+	spnCloud, spnKeyVault, err := b.getServicePrincipalTokens(ui.Say)
+	if err != nil {
+		t.Fatalf("failed getting azure tokens: %s", err)
+	}
+
+	ui.Message("Creating test Azure Resource Manager (ARM) client ...")
+	azureClient, err := NewAzureClient(
+		b.config.ClientConfig.SubscriptionID,
+		b.config.SharedGalleryDestination.SigDestinationSubscription,
+		b.config.ResourceGroupName,
+		b.config.StorageAccount,
+		b.config.ClientConfig.CloudEnvironment(),
+		b.config.SharedGalleryTimeout,
+		b.config.PollingDurationTimeout,
+		spnCloud,
+		spnKeyVault)
+
+	if err != nil {
+		t.Fatalf("failed to create azure client: %s", err)
+	}
+
+	// Validate resource group has been deleted
+	_, err = azureClient.GroupsClient.Get(context.Background(), b.config.tmpResourceGroupName)
+	if err == nil || !resourceNotFound(err) {
+		t.Fatalf("failed validating resource group deletion: %s", err)
+	}
+}
+
+func checkUnmanagedVHDDeleted(t *testing.T, b *Builder) {
+	ui := testUi()
+
+	spnCloud, spnKeyVault, err := b.getServicePrincipalTokens(ui.Say)
+	if err != nil {
+		t.Fatalf("failed getting azure tokens: %s", err)
+	}
+
+	azureClient, err := NewAzureClient(
+		b.config.ClientConfig.SubscriptionID,
+		b.config.SharedGalleryDestination.SigDestinationSubscription,
+		b.config.ResourceGroupName,
+		b.config.StorageAccount,
+		b.config.ClientConfig.CloudEnvironment(),
+		b.config.SharedGalleryTimeout,
+		b.config.PollingDurationTimeout,
+		spnCloud,
+		spnKeyVault)
+
+	if err != nil {
+		t.Fatalf("failed to create azure client: %s", err)
+	}
+
+	// validate temporary os blob was deleted
+	blob := azureClient.BlobStorageClient.GetContainerReference("images").GetBlobReference(b.config.tmpOSDiskName)
+	_, err = blob.BreakLease(nil)
+	if err != nil && !strings.Contains(err.Error(), "BlobNotFound") {
+		t.Fatalf("failed validating deletion of unmanaged vhd: %s", err)
+	}
+
+	// Validate resource group has been deleted
+	_, err = azureClient.GroupsClient.Get(context.Background(), b.config.tmpResourceGroupName)
+	if err == nil || !resourceNotFound(err) {
+		t.Fatalf("failed validating resource group deletion: %s", err)
+	}
+}
+
+func resourceNotFound(err error) bool {
+	derr := autorest.DetailedError{}
+	return errors.As(err, &derr) && derr.StatusCode == 404
+}
+
+func testUi() *packersdk.BasicUi {
+	return &packersdk.BasicUi{
+		Reader:      new(bytes.Buffer),
+		Writer:      new(bytes.Buffer),
+		ErrorWriter: new(bytes.Buffer),
+	}
+}
+
 const testBuilderAccManagedDiskWindows = `
 {
 	"variables": {
@@ -155,6 +259,7 @@ const testBuilderAccManagedDiskWindows = `
 	}]
 }
 `
+
 const testBuilderAccManagedDiskWindowsBuildResourceGroup = `
 {
 	"variables": {
@@ -287,6 +392,7 @@ const testBuilderAccManagedDiskLinux = `
 	}]
 }
 `
+
 const testBuilderAccManagedDiskLinuxDeviceLogin = `
 {
 	"variables": {
@@ -379,6 +485,7 @@ const testBuilderAccBlobLinux = `
 	}]
 }
 `
+
 const testBuilderAccManagedDiskLinuxAzureCLI = `
 {
 	"builders": [{
@@ -388,7 +495,8 @@ const testBuilderAccManagedDiskLinuxAzureCLI = `
 
 	  "managed_image_resource_group_name": "packer-acceptance-test",
 	  "managed_image_name": "testBuilderAccManagedDiskLinuxAzureCLI-{{timestamp}}",
-
+	  "temp_resource_group_name": "packer-acceptance-test-managed-cli",
+	  
 	  "os_type": "Linux",
 	  "image_publisher": "Canonical",
 	  "image_offer": "UbuntuServer",

builder/azure/arm/step_deploy_template.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"net/url"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/hashicorp/packer-plugin-sdk/multistep"
@@ -15,16 +16,17 @@ import (
 )
 
 type StepDeployTemplate struct {
-	client     *AzureClient
-	deploy     func(ctx context.Context, resourceGroupName string, deploymentName string) error
-	delete     func(ctx context.Context, deploymentName, resourceGroupName string) error
-	disk       func(ctx context.Context, resourceGroupName string, computeName string) (string, string, error)
-	deleteDisk func(ctx context.Context, imageType string, imageName string, resourceGroupName string) error
-	say        func(message string)
-	error      func(e error)
-	config     *Config
-	factory    templateFactoryFunc
-	name       string
+	client           *AzureClient
+	deploy           func(ctx context.Context, resourceGroupName string, deploymentName string) error
+	delete           func(ctx context.Context, deploymentName, resourceGroupName string) error
+	disk             func(ctx context.Context, resourceGroupName string, computeName string) (string, string, error)
+	deleteDisk       func(ctx context.Context, imageType string, imageName string, resourceGroupName string) error
+	deleteDeployment func(ctx context.Context, state multistep.StateBag) error
+	say              func(message string)
+	error            func(e error)
+	config           *Config
+	factory          templateFactoryFunc
+	name             string
 }
 
 func NewStepDeployTemplate(client *AzureClient, ui packersdk.Ui, config *Config, deploymentName string, factory templateFactoryFunc) *StepDeployTemplate {
@@ -41,6 +43,7 @@ func NewStepDeployTemplate(client *AzureClient, ui packersdk.Ui, config *Config,
 	step.delete = step.deleteDeploymentResources
 	step.disk = step.getImageDetails
 	step.deleteDisk = step.deleteImage
+	step.deleteDeployment = step.deleteDeploymentObject
 	return step
 }
 
@@ -58,32 +61,45 @@ func (s *StepDeployTemplate) Run(ctx context.Context, state multistep.StateBag)
 
 func (s *StepDeployTemplate) Cleanup(state multistep.StateBag) {
 	defer func() {
-		err := s.deleteTemplate(context.Background(), state)
+		err := s.deleteDeployment(context.Background(), state)
 		if err != nil {
-			s.say(s.client.LastError.Error())
+			s.say(err.Error())
 		}
 	}()
 
-	// Only clean up if this is an existing resource group that has been verified to exist.
-	// ArmIsResourceGroupCreated is set in step_create_resource_group to true, when Packer has verified that the resource group exists.
-	// ArmIsExistingResourceGroup is set to true when build_resource_group is set in the Packer configuration.
-	existingResourceGroup := state.Get(constants.ArmIsExistingResourceGroup).(bool)
-	resourceGroupCreated := state.Get(constants.ArmIsResourceGroupCreated).(bool)
-	if !existingResourceGroup || !resourceGroupCreated {
-		return
-	}
-
 	ui := state.Get("ui").(packersdk.Ui)
-	ui.Say("\nThe resource group was not created by Packer, deleting individual resources ...")
+	ui.Say("\nDeleting individual resources ...")
 
 	deploymentName := s.name
 	resourceGroupName := state.Get(constants.ArmResourceGroupName).(string)
-	err := s.deleteDeploymentResources(context.TODO(), deploymentName, resourceGroupName)
+	// Get image disk details before deleting the image; otherwise we won't be able to
+	// delete the disk as the image request will return a 404
+	computeName := state.Get(constants.ArmComputeName).(string)
+	imageType, imageName, err := s.disk(context.TODO(), resourceGroupName, computeName)
+
+	if err != nil && !strings.Contains(err.Error(), "ResourceNotFound") {
+		ui.Error(fmt.Sprintf("Could not retrieve OS Image details: %s", err))
+	}
+	err = s.delete(context.TODO(), deploymentName, resourceGroupName)
 	if err != nil {
 		s.reportIfError(err, resourceGroupName)
 	}
 
-	NewStepDeleteAdditionalDisks(s.client, ui).Run(context.TODO(), state)
+	// The disk was not found on the VM, this is an error.
+	if imageType == "" && imageName == "" {
+		ui.Error(fmt.Sprintf("Failed to find temporary OS disk on VM.  Please delete manually.\n\n"+
+			"VM Name: %s\n"+
+			"Error: %s", computeName, err))
+		return
+	}
+
+	ui.Say(fmt.Sprintf(" Deleting -> %s : '%s'", imageType, imageName))
+	err = s.deleteDisk(context.TODO(), imageType, imageName, resourceGroupName)
+	if err != nil {
+		ui.Error(fmt.Sprintf("Error deleting resource.  Please delete manually.\n\n"+
+			"Name: %s\n"+
+			"Error: %s", imageName, err))
+	}
 }
 
 func (s *StepDeployTemplate) deployTemplate(ctx context.Context, resourceGroupName string, deploymentName string) error {
@@ -106,7 +122,7 @@ func (s *StepDeployTemplate) deployTemplate(ctx context.Context, resourceGroupNa
 	return err
 }
 
-func (s *StepDeployTemplate) deleteTemplate(ctx context.Context, state multistep.StateBag) error {
+func (s *StepDeployTemplate) deleteDeploymentObject(ctx context.Context, state multistep.StateBag) error {
 	deploymentName := s.name
 	resourceGroupName := state.Get(constants.ArmResourceGroupName).(string)
 	ui := state.Get("ui").(packersdk.Ui)
@@ -222,6 +238,8 @@ func (s *StepDeployTemplate) deleteDeploymentResources(ctx context.Context, depl
 		return err
 	}
 
+	resources := map[string]string{}
+
 	for deploymentOperations.NotDone() {
 		deploymentOperation := deploymentOperations.Value()
 		// Sometimes an empty operation is added to the list by Azure
@@ -233,30 +251,47 @@ func (s *StepDeployTemplate) deleteDeploymentResources(ctx context.Context, depl
 		resourceName := *deploymentOperation.Properties.TargetResource.ResourceName
 		resourceType := *deploymentOperation.Properties.TargetResource.ResourceType
 
-		s.say(fmt.Sprintf(" -> %s : '%s'", resourceType, resourceName))
-
-		err = retry.Config{
-			Tries:      10,
-			RetryDelay: (&retry.Backoff{InitialBackoff: 10 * time.Second, MaxBackoff: 600 * time.Second, Multiplier: 2}).Linear,
-		}.Run(ctx, func(ctx context.Context) error {
-			err := deleteResource(ctx, s.client,
-				resourceType,
-				resourceName,
-				resourceGroupName)
-			if err != nil {
-				s.reportIfError(err, resourceName)
-			}
-			return nil
-		})
-		if err != nil {
-			s.reportIfError(err, resourceName)
-		}
+		s.say(fmt.Sprintf("Adding to deletion queue -> %s : '%s'", resourceType, resourceName))
+		resources[resourceType] = resourceName
 
 		if err = deploymentOperations.Next(); err != nil {
 			return err
 		}
 	}
 
+	var wg sync.WaitGroup
+	wg.Add(len(resources))
+
+	for resourceType, resourceName := range resources {
+		go func(resourceType, resourceName string) {
+			defer wg.Done()
+			retryConfig := retry.Config{
+				Tries:      10,
+				RetryDelay: (&retry.Backoff{InitialBackoff: 10 * time.Second, MaxBackoff: 600 * time.Second, Multiplier: 2}).Linear,
+			}
+
+			err = retryConfig.Run(ctx, func(ctx context.Context) error {
+				s.say(fmt.Sprintf("Attempting deletion -> %s : '%s'", resourceType, resourceName))
+				err := deleteResource(ctx, s.client,
+					resourceType,
+					resourceName,
+					resourceGroupName)
+				if err != nil {
+					s.say(fmt.Sprintf("Error deleting resource. Will retry.\n"+
+						"Name: %s\n"+
+						"Error: %s\n", resourceName, err.Error()))
+				}
+				return err
+			})
+			if err != nil {
+				s.reportIfError(err, resourceName)
+			}
+		}(resourceType, resourceName)
+	}
+
+	s.say("Waiting for deletion of all resources...")
+	wg.Wait()
+
 	return nil
 }
 

builder/azure/arm/step_deploy_template_test.go

@@ -6,6 +6,7 @@ import (
 	"testing"
 
 	"github.com/hashicorp/packer-plugin-sdk/multistep"
+	packersdk "github.com/hashicorp/packer-plugin-sdk/packer"
 	"github.com/hashicorp/packer/builder/azure/common/constants"
 )
 
@@ -108,11 +109,97 @@ func TestStepDeployTemplateDeleteImageShouldFailWithInvalidImage(t *testing.T) {
 	}
 }
 
+func TestStepDeployTemplateCleanupShouldDeleteManagedOSImageInExistingResourceGroup(t *testing.T) {
+	var deleteDiskCounter = 0
+	var testSubject = createTestStepDeployTemplateDeleteOSImage(&deleteDiskCounter)
+
+	stateBag := createTestStateBagStepDeployTemplate()
+	stateBag.Put(constants.ArmIsManagedImage, true)
+	stateBag.Put(constants.ArmIsExistingResourceGroup, true)
+	stateBag.Put(constants.ArmIsResourceGroupCreated, true)
+	stateBag.Put("ui", packersdk.TestUi(t))
+
+	testSubject.Cleanup(stateBag)
+	if deleteDiskCounter != 1 {
+		t.Fatalf("Expected DeployTemplate Cleanup to invoke deleteDisk 1 time, but invoked %d times", deleteDiskCounter)
+	}
+}
+
+func TestStepDeployTemplateCleanupShouldDeleteManagedOSImageInTemporaryResourceGroup(t *testing.T) {
+	var deleteDiskCounter = 0
+	var testSubject = createTestStepDeployTemplateDeleteOSImage(&deleteDiskCounter)
+
+	stateBag := createTestStateBagStepDeployTemplate()
+	stateBag.Put(constants.ArmIsManagedImage, true)
+	stateBag.Put(constants.ArmIsExistingResourceGroup, false)
+	stateBag.Put(constants.ArmIsResourceGroupCreated, true)
+	stateBag.Put("ui", packersdk.TestUi(t))
+
+	testSubject.Cleanup(stateBag)
+	if deleteDiskCounter != 1 {
+		t.Fatalf("Expected DeployTemplate Cleanup to invoke deleteDisk 1 times, but invoked %d times", deleteDiskCounter)
+	}
+}
+
+func TestStepDeployTemplateCleanupShouldDeleteVHDOSImageInExistingResourceGroup(t *testing.T) {
+	var deleteDiskCounter = 0
+	var testSubject = createTestStepDeployTemplateDeleteOSImage(&deleteDiskCounter)
+
+	stateBag := createTestStateBagStepDeployTemplate()
+	stateBag.Put(constants.ArmIsManagedImage, false)
+	stateBag.Put(constants.ArmIsExistingResourceGroup, true)
+	stateBag.Put(constants.ArmIsResourceGroupCreated, true)
+	stateBag.Put("ui", packersdk.TestUi(t))
+
+	testSubject.Cleanup(stateBag)
+	if deleteDiskCounter != 1 {
+		t.Fatalf("Expected DeployTemplate Cleanup to invoke deleteDisk 1 time, but invoked %d times", deleteDiskCounter)
+	}
+}
+
+func TestStepDeployTemplateCleanupShouldVHDOSImageInTemporaryResourceGroup(t *testing.T) {
+	var deleteDiskCounter = 0
+	var testSubject = createTestStepDeployTemplateDeleteOSImage(&deleteDiskCounter)
+
+	stateBag := createTestStateBagStepDeployTemplate()
+	stateBag.Put(constants.ArmIsManagedImage, false)
+	stateBag.Put(constants.ArmIsExistingResourceGroup, false)
+	stateBag.Put(constants.ArmIsResourceGroupCreated, true)
+	stateBag.Put("ui", packersdk.TestUi(t))
+
+	testSubject.Cleanup(stateBag)
+	if deleteDiskCounter != 1 {
+		t.Fatalf("Expected DeployTemplate Cleanup to invoke deleteDisk 1 times, but invoked %d times", deleteDiskCounter)
+	}
+}
+
 func createTestStateBagStepDeployTemplate() multistep.StateBag {
 	stateBag := new(multistep.BasicStateBag)
 
 	stateBag.Put(constants.ArmDeploymentName, "Unit Test: DeploymentName")
 	stateBag.Put(constants.ArmResourceGroupName, "Unit Test: ResourceGroupName")
+	stateBag.Put(constants.ArmComputeName, "Unit Test: ComputeName")
 
 	return stateBag
 }
+
+func createTestStepDeployTemplateDeleteOSImage(deleteDiskCounter *int) *StepDeployTemplate {
+	return &StepDeployTemplate{
+		deploy: func(context.Context, string, string) error { return nil },
+		say:    func(message string) {},
+		error:  func(e error) {},
+		deleteDisk: func(ctx context.Context, imageType string, imageName string, resourceGroupName string) error {
+			*deleteDiskCounter++
+			return nil
+		},
+		disk: func(ctx context.Context, resourceGroupName, computeName string) (string, string, error) {
+			return "Microsoft.Compute/disks", "", nil
+		},
+		delete: func(ctx context.Context, deploymentName, resourceGroupName string) error {
+			return nil
+		},
+		deleteDeployment: func(ctx context.Context, state multistep.StateBag) error {
+			return nil
+		},
+	}
+}

builder/azure/common/client/config.go

@@ -165,7 +165,7 @@ func (c Config) Validate(errs *packersdk.MultiError) {
 		if _, err := os.Stat(c.ClientCertPath); err != nil {
 			errs = packersdk.MultiErrorAppend(errs, fmt.Errorf("client_cert_path is not an accessible file: %v", err))
 		}
-		if c.ClientCertExpireTimeout < 5*time.Minute {
+		if c.ClientCertExpireTimeout != 0 && c.ClientCertExpireTimeout < 5*time.Minute {
 			errs = packersdk.MultiErrorAppend(errs, fmt.Errorf("client_cert_token_timeout will expire within 5 minutes, please set a value greater than 5 minutes"))
 		}
 		return

builder/cloudstack/builder.go

@@ -60,12 +60,7 @@ func (b *Builder) Run(ctx context.Context, ui packersdk.Ui, hook packersdk.Hook)
 	// Build the steps.
 	steps := []multistep.Step{
 		&stepPrepareConfig{},
-		&commonsteps.StepHTTPServer{
-			HTTPDir:     b.config.HTTPDir,
-			HTTPPortMin: b.config.HTTPPortMin,
-			HTTPPortMax: b.config.HTTPPortMax,
-			HTTPAddress: b.config.HTTPAddress,
-		},
+		commonsteps.HTTPServerFromHTTPConfig(&b.config.HTTPConfig),
 		&stepKeypair{
 			Debug:        b.config.PackerDebug,
 			Comm:         &b.config.Comm,

builder/cloudstack/config.hcl2spec.go

@@ -19,6 +19,7 @@ type FlatConfig struct {
 	PackerUserVars            map[string]string `mapstructure:"packer_user_variables" cty:"packer_user_variables" hcl:"packer_user_variables"`
 	PackerSensitiveVars       []string          `mapstructure:"packer_sensitive_variables" cty:"packer_sensitive_variables" hcl:"packer_sensitive_variables"`
 	HTTPDir                   *string           `mapstructure:"http_directory" cty:"http_directory" hcl:"http_directory"`
+	HTTPContent               map[string]string `mapstructure:"http_content" cty:"http_content" hcl:"http_content"`
 	HTTPPortMin               *int              `mapstructure:"http_port_min" cty:"http_port_min" hcl:"http_port_min"`
 	HTTPPortMax               *int              `mapstructure:"http_port_max" cty:"http_port_max" hcl:"http_port_max"`
 	HTTPAddress               *string           `mapstructure:"http_bind_address" cty:"http_bind_address" hcl:"http_bind_address"`
@@ -135,6 +136,7 @@ func (*FlatConfig) HCL2Spec() map[string]hcldec.Spec {
 		"packer_user_variables":        &hcldec.AttrSpec{Name: "packer_user_variables", Type: cty.Map(cty.String), Required: false},
 		"packer_sensitive_variables":   &hcldec.AttrSpec{Name: "packer_sensitive_variables", Type: cty.List(cty.String), Required: false},
 		"http_directory":               &hcldec.AttrSpec{Name: "http_directory", Type: cty.String, Required: false},
+		"http_content":                 &hcldec.AttrSpec{Name: "http_content", Type: cty.Map(cty.String), Required: false},
 		"http_port_min":                &hcldec.AttrSpec{Name: "http_port_min", Type: cty.Number, Required: false},
 		"http_port_max":                &hcldec.AttrSpec{Name: "http_port_max", Type: cty.Number, Required: false},
 		"http_bind_address":            &hcldec.AttrSpec{Name: "http_bind_address", Type: cty.String, Required: false},

builder/hyperv/iso/builder.go

@@ -211,12 +211,7 @@ func (b *Builder) Run(ctx context.Context, ui packersdk.Ui, hook packersdk.Hook)
 			Directories: b.config.FloppyConfig.FloppyDirectories,
 			Label:       b.config.FloppyConfig.FloppyLabel,
 		},
-		&commonsteps.StepHTTPServer{
-			HTTPDir:     b.config.HTTPDir,
-			HTTPPortMin: b.config.HTTPPortMin,
-			HTTPPortMax: b.config.HTTPPortMax,
-			HTTPAddress: b.config.HTTPAddress,
-		},
+		commonsteps.HTTPServerFromHTTPConfig(&b.config.HTTPConfig),
 		&hypervcommon.StepCreateSwitch{
 			SwitchName: b.config.SwitchName,
 		},

builder/hyperv/iso/builder.hcl2spec.go

@@ -19,6 +19,7 @@ type FlatConfig struct {
 	PackerUserVars                 map[string]string `mapstructure:"packer_user_variables" cty:"packer_user_variables" hcl:"packer_user_variables"`
 	PackerSensitiveVars            []string          `mapstructure:"packer_sensitive_variables" cty:"packer_sensitive_variables" hcl:"packer_sensitive_variables"`
 	HTTPDir                        *string           `mapstructure:"http_directory" cty:"http_directory" hcl:"http_directory"`
+	HTTPContent                    map[string]string `mapstructure:"http_content" cty:"http_content" hcl:"http_content"`
 	HTTPPortMin                    *int              `mapstructure:"http_port_min" cty:"http_port_min" hcl:"http_port_min"`
 	HTTPPortMax                    *int              `mapstructure:"http_port_max" cty:"http_port_max" hcl:"http_port_max"`
 	HTTPAddress                    *string           `mapstructure:"http_bind_address" cty:"http_bind_address" hcl:"http_bind_address"`
@@ -141,6 +142,7 @@ func (*FlatConfig) HCL2Spec() map[string]hcldec.Spec {
 		"packer_user_variables":            &hcldec.AttrSpec{Name: "packer_user_variables", Type: cty.Map(cty.String), Required: false},
 		"packer_sensitive_variables":       &hcldec.AttrSpec{Name: "packer_sensitive_variables", Type: cty.List(cty.String), Required: false},
 		"http_directory":                   &hcldec.AttrSpec{Name: "http_directory", Type: cty.String, Required: false},
+		"http_content":                     &hcldec.AttrSpec{Name: "http_content", Type: cty.Map(cty.String), Required: false},
 		"http_port_min":                    &hcldec.AttrSpec{Name: "http_port_min", Type: cty.Number, Required: false},
 		"http_port_max":                    &hcldec.AttrSpec{Name: "http_port_max", Type: cty.Number, Required: false},
 		"http_bind_address":                &hcldec.AttrSpec{Name: "http_bind_address", Type: cty.String, Required: false},

builder/hyperv/vmcx/builder.go

@@ -251,12 +251,7 @@ func (b *Builder) Run(ctx context.Context, ui packersdk.Ui, hook packersdk.Hook)
 			Directories: b.config.FloppyConfig.FloppyDirectories,
 			Label:       b.config.FloppyConfig.FloppyLabel,
 		},
-		&commonsteps.StepHTTPServer{
-			HTTPDir:     b.config.HTTPDir,
-			HTTPPortMin: b.config.HTTPPortMin,
-			HTTPPortMax: b.config.HTTPPortMax,
-			HTTPAddress: b.config.HTTPAddress,
-		},
+		commonsteps.HTTPServerFromHTTPConfig(&b.config.HTTPConfig),
 		&hypervcommon.StepCreateSwitch{
 			SwitchName: b.config.SwitchName,
 		},

builder/hyperv/vmcx/builder.hcl2spec.go

@@ -19,6 +19,7 @@ type FlatConfig struct {
 	PackerUserVars                 map[string]string `mapstructure:"packer_user_variables" cty:"packer_user_variables" hcl:"packer_user_variables"`
 	PackerSensitiveVars            []string          `mapstructure:"packer_sensitive_variables" cty:"packer_sensitive_variables" hcl:"packer_sensitive_variables"`
 	HTTPDir                        *string           `mapstructure:"http_directory" cty:"http_directory" hcl:"http_directory"`
+	HTTPContent                    map[string]string `mapstructure:"http_content" cty:"http_content" hcl:"http_content"`
 	HTTPPortMin                    *int              `mapstructure:"http_port_min" cty:"http_port_min" hcl:"http_port_min"`
 	HTTPPortMax                    *int              `mapstructure:"http_port_max" cty:"http_port_max" hcl:"http_port_max"`
 	HTTPAddress                    *string           `mapstructure:"http_bind_address" cty:"http_bind_address" hcl:"http_bind_address"`
@@ -143,6 +144,7 @@ func (*FlatConfig) HCL2Spec() map[string]hcldec.Spec {
 		"packer_user_variables":            &hcldec.AttrSpec{Name: "packer_user_variables", Type: cty.Map(cty.String), Required: false},
 		"packer_sensitive_variables":       &hcldec.AttrSpec{Name: "packer_sensitive_variables", Type: cty.List(cty.String), Required: false},
 		"http_directory":                   &hcldec.AttrSpec{Name: "http_directory", Type: cty.String, Required: false},
+		"http_content":                     &hcldec.AttrSpec{Name: "http_content", Type: cty.Map(cty.String), Required: false},
 		"http_port_min":                    &hcldec.AttrSpec{Name: "http_port_min", Type: cty.Number, Required: false},
 		"http_port_max":                    &hcldec.AttrSpec{Name: "http_port_max", Type: cty.Number, Required: false},
 		"http_bind_address":                &hcldec.AttrSpec{Name: "http_bind_address", Type: cty.String, Required: false},

builder/oracle/oci/config.go

@@ -253,7 +253,7 @@ func (c *Config) Prepare(raws ...interface{}) error {
 
 		if _, err := configProvider.PrivateRSAKey(); err != nil {
 			errs = packersdk.MultiErrorAppend(
-				errs, errors.New("'key_file' must be specified"))
+				errs, fmt.Errorf("'key_file' must be correctly specified. %w", err))
 		}
 
 		c.configProvider = configProvider

builder/osc/bsu/step_create_omi.go

@@ -32,6 +32,9 @@ func (s *stepCreateOMI) Run(ctx context.Context, state multistep.StateBag) multi
 		ImageName:           omiName,
 		BlockDeviceMappings: config.BlockDevices.BuildOscOMIDevices(),
 	}
+	if config.OMIDescription != "" {
+		createOpts.Description = config.OMIDescription
+	}
 
 	resp, _, err := oscconn.ImageApi.CreateImage(context.Background(), &osc.CreateImageOpts{
 		CreateImageRequest: optional.NewInterface(createOpts),

builder/osc/bsusurrogate/step_register_omi.go

@@ -37,6 +37,9 @@ func (s *StepRegisterOMI) Run(ctx context.Context, state multistep.StateBag) mul
 		BlockDeviceMappings: blockDevices,
 	}
 
+	if config.OMIDescription != "" {
+		registerOpts.Description = config.OMIDescription
+	}
 	registerResp, _, err := oscconn.ImageApi.CreateImage(context.Background(), &osc.CreateImageOpts{
 		CreateImageRequest: optional.NewInterface(registerOpts),
 	})

builder/osc/chroot/step_create_omi.go

@@ -75,6 +75,10 @@ func (s *StepCreateOMI) Run(ctx context.Context, state multistep.StateBag) multi
 		registerOpts = buildRegisterOpts(config, image, newMappings)
 	}
 
+	if config.OMIDescription != "" {
+		registerOpts.Description = config.OMIDescription
+	}
+
 	registerResp, _, err := osconn.ImageApi.CreateImage(context.Background(), &osc.CreateImageOpts{
 		CreateImageRequest: optional.NewInterface(registerOpts),
 	})

builder/parallels/iso/builder.go

@@ -209,12 +209,7 @@ func (b *Builder) Run(ctx context.Context, ui packersdk.Ui, hook packersdk.Hook)
 			Directories: b.config.FloppyConfig.FloppyDirectories,
 			Label:       b.config.FloppyConfig.FloppyLabel,
 		},
-		&commonsteps.StepHTTPServer{
-			HTTPDir:     b.config.HTTPDir,
-			HTTPPortMin: b.config.HTTPPortMin,
-			HTTPPortMax: b.config.HTTPPortMax,
-			HTTPAddress: b.config.HTTPAddress,
-		},
+		commonsteps.HTTPServerFromHTTPConfig(&b.config.HTTPConfig),
 		new(stepCreateVM),
 		new(stepCreateDisk),
 		new(stepSetBootOrder),

builder/parallels/iso/builder.hcl2spec.go

@@ -19,6 +19,7 @@ type FlatConfig struct {
 	PackerUserVars            map[string]string `mapstructure:"packer_user_variables" cty:"packer_user_variables" hcl:"packer_user_variables"`
 	PackerSensitiveVars       []string          `mapstructure:"packer_sensitive_variables" cty:"packer_sensitive_variables" hcl:"packer_sensitive_variables"`
 	HTTPDir                   *string           `mapstructure:"http_directory" cty:"http_directory" hcl:"http_directory"`
+	HTTPContent               map[string]string `mapstructure:"http_content" cty:"http_content" hcl:"http_content"`
 	HTTPPortMin               *int              `mapstructure:"http_port_min" cty:"http_port_min" hcl:"http_port_min"`
 	HTTPPortMax               *int              `mapstructure:"http_port_max" cty:"http_port_max" hcl:"http_port_max"`
 	HTTPAddress               *string           `mapstructure:"http_bind_address" cty:"http_bind_address" hcl:"http_bind_address"`
@@ -126,6 +127,7 @@ func (*FlatConfig) HCL2Spec() map[string]hcldec.Spec {
 		"packer_user_variables":        &hcldec.AttrSpec{Name: "packer_user_variables", Type: cty.Map(cty.String), Required: false},
 		"packer_sensitive_variables":   &hcldec.AttrSpec{Name: "packer_sensitive_variables", Type: cty.List(cty.String), Required: false},
 		"http_directory":               &hcldec.AttrSpec{Name: "http_directory", Type: cty.String, Required: false},
+		"http_content":                 &hcldec.AttrSpec{Name: "http_content", Type: cty.Map(cty.String), Required: false},
 		"http_port_min":                &hcldec.AttrSpec{Name: "http_port_min", Type: cty.Number, Required: false},
 		"http_port_max":                &hcldec.AttrSpec{Name: "http_port_max", Type: cty.Number, Required: false},
 		"http_bind_address":            &hcldec.AttrSpec{Name: "http_bind_address", Type: cty.String, Required: false},

builder/proxmox/clone/config.hcl2spec.go

@@ -20,6 +20,7 @@ type FlatConfig struct {
 	PackerUserVars            map[string]string                  `mapstructure:"packer_user_variables" cty:"packer_user_variables" hcl:"packer_user_variables"`
 	PackerSensitiveVars       []string                           `mapstructure:"packer_sensitive_variables" cty:"packer_sensitive_variables" hcl:"packer_sensitive_variables"`
 	HTTPDir                   *string                            `mapstructure:"http_directory" cty:"http_directory" hcl:"http_directory"`
+	HTTPContent               map[string]string                  `mapstructure:"http_content" cty:"http_content" hcl:"http_content"`
 	HTTPPortMin               *int                               `mapstructure:"http_port_min" cty:"http_port_min" hcl:"http_port_min"`
 	HTTPPortMax               *int                               `mapstructure:"http_port_max" cty:"http_port_max" hcl:"http_port_max"`
 	HTTPAddress               *string                            `mapstructure:"http_bind_address" cty:"http_bind_address" hcl:"http_bind_address"`
@@ -81,6 +82,7 @@ type FlatConfig struct {
 	SkipCertValidation        *bool                              `mapstructure:"insecure_skip_tls_verify" cty:"insecure_skip_tls_verify" hcl:"insecure_skip_tls_verify"`
 	Username                  *string                            `mapstructure:"username" cty:"username" hcl:"username"`
 	Password                  *string                            `mapstructure:"password" cty:"password" hcl:"password"`
+	Token                     *string                            `mapstructure:"token" cty:"token" hcl:"token"`
 	Node                      *string                            `mapstructure:"node" cty:"node" hcl:"node"`
 	Pool                      *string                            `mapstructure:"pool" cty:"pool" hcl:"pool"`
 	VMName                    *string                            `mapstructure:"vm_name" cty:"vm_name" hcl:"vm_name"`
@@ -129,6 +131,7 @@ func (*FlatConfig) HCL2Spec() map[string]hcldec.Spec {
 		"packer_user_variables":        &hcldec.AttrSpec{Name: "packer_user_variables", Type: cty.Map(cty.String), Required: false},
 		"packer_sensitive_variables":   &hcldec.AttrSpec{Name: "packer_sensitive_variables", Type: cty.List(cty.String), Required: false},
 		"http_directory":               &hcldec.AttrSpec{Name: "http_directory", Type: cty.String, Required: false},
+		"http_content":                 &hcldec.AttrSpec{Name: "http_content", Type: cty.Map(cty.String), Required: false},
 		"http_port_min":                &hcldec.AttrSpec{Name: "http_port_min", Type: cty.Number, Required: false},
 		"http_port_max":                &hcldec.AttrSpec{Name: "http_port_max", Type: cty.Number, Required: false},
 		"http_bind_address":            &hcldec.AttrSpec{Name: "http_bind_address", Type: cty.String, Required: false},
@@ -190,6 +193,7 @@ func (*FlatConfig) HCL2Spec() map[string]hcldec.Spec {
 		"insecure_skip_tls_verify":     &hcldec.AttrSpec{Name: "insecure_skip_tls_verify", Type: cty.Bool, Required: false},
 		"username":                     &hcldec.AttrSpec{Name: "username", Type: cty.String, Required: false},
 		"password":                     &hcldec.AttrSpec{Name: "password", Type: cty.String, Required: false},
+		"token":                        &hcldec.AttrSpec{Name: "token", Type: cty.String, Required: false},
 		"node":                         &hcldec.AttrSpec{Name: "node", Type: cty.String, Required: false},
 		"pool":                         &hcldec.AttrSpec{Name: "pool", Type: cty.String, Required: false},
 		"vm_name":                      &hcldec.AttrSpec{Name: "vm_name", Type: cty.String, Required: false},

builder/proxmox/common/builder.go

@@ -2,7 +2,6 @@ package proxmox
 
 import (
 	"context"
-	"crypto/tls"
 	"errors"
 	"fmt"
 
@@ -35,15 +34,7 @@ type Builder struct {
 
 func (b *Builder) Run(ctx context.Context, ui packersdk.Ui, hook packersdk.Hook, state multistep.StateBag) (packersdk.Artifact, error) {
 	var err error
-	tlsConfig := &tls.Config{
-		InsecureSkipVerify: b.config.SkipCertValidation,
-	}
-	b.proxmoxClient, err = proxmox.NewClient(b.config.proxmoxURL.String(), nil, tlsConfig)
-	if err != nil {
-		return nil, err
-	}
-
-	err = b.proxmoxClient.Login(b.config.Username, b.config.Password, "")
+	b.proxmoxClient, err = newProxmoxClient(b.config)
 	if err != nil {
 		return nil, err
 	}
@@ -61,12 +52,7 @@ func (b *Builder) Run(ctx context.Context, ui packersdk.Ui, hook packersdk.Hook,
 		&stepStartVM{
 			vmCreator: b.vmCreator,
 		},
-		&commonsteps.StepHTTPServer{
-			HTTPDir:     b.config.HTTPDir,
-			HTTPPortMin: b.config.HTTPPortMin,
-			HTTPPortMax: b.config.HTTPPortMax,
-			HTTPAddress: b.config.HTTPAddress,
-		},
+		commonsteps.HTTPServerFromHTTPConfig(&b.config.HTTPConfig),
 		&stepTypeBootCommand{
 			BootConfig: b.config.BootConfig,
 			Ctx:        b.config.Ctx,

builder/proxmox/common/client.go

@@ -0,0 +1,37 @@
+package proxmox
+
+import (
+	"crypto/tls"
+	"log"
+	"time"
+
+	"github.com/Telmate/proxmox-api-go/proxmox"
+)
+
+const defaultTaskTimeout = 30 * time.Second
+
+func newProxmoxClient(config Config) (*proxmox.Client, error) {
+	tlsConfig := &tls.Config{
+		InsecureSkipVerify: config.SkipCertValidation,
+	}
+
+	client, err := proxmox.NewClient(config.proxmoxURL.String(), nil, tlsConfig, int(defaultTaskTimeout.Seconds()))
+	if err != nil {
+		return nil, err
+	}
+
+	if config.Token != "" {
+		// configure token auth
+		log.Print("using token auth")
+		client.SetAPIToken(config.Username, config.Token)
+	} else {
+		// fallback to login if not using tokens
+		log.Print("using password auth")
+		err = client.Login(config.Username, config.Password, "")
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return client, nil
+}

builder/proxmox/common/client_test.go

@@ -0,0 +1,91 @@
+package proxmox
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"testing"
+
+	"github.com/Telmate/proxmox-api-go/proxmox"
+	"github.com/stretchr/testify/require"
+)
+
+func TestTokenAuth(t *testing.T) {
+	mockAPI := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
+		if req.Header.Get("Authorization") != "PVEAPIToken=dummy@vmhost!test-token=ac5293bf-15e2-477f-b04c-a6dfa7a46b80" {
+			rw.WriteHeader(http.StatusUnauthorized)
+			return
+		}
+	}))
+	defer mockAPI.Close()
+
+	pmURL, _ := url.Parse(mockAPI.URL)
+	config := Config{
+		proxmoxURL:         pmURL,
+		SkipCertValidation: false,
+		Username:           "dummy@vmhost!test-token",
+		Password:           "not-used",
+		Token:              "ac5293bf-15e2-477f-b04c-a6dfa7a46b80",
+	}
+
+	client, err := newProxmoxClient(config)
+	require.NoError(t, err)
+
+	ref := proxmox.NewVmRef(110)
+	ref.SetNode("node1")
+	ref.SetVmType("qemu")
+	err = client.Sendkey(ref, "ping")
+	require.NoError(t, err)
+}
+
+func TestLogin(t *testing.T) {
+	mockAPI := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
+		// mock ticketing api
+		if req.Method == http.MethodPost && req.URL.Path == "/access/ticket" {
+			body, _ := ioutil.ReadAll(req.Body)
+			values, _ := url.ParseQuery(string(body))
+			user := values.Get("username")
+			pass := values.Get("password")
+			if user != "dummy@vmhost" || pass != "correct-horse-battery-staple" {
+				rw.WriteHeader(http.StatusUnauthorized)
+				return
+			}
+
+			_ = json.NewEncoder(rw).Encode(map[string]interface{}{
+				"data": map[string]string{
+					"username":            user,
+					"ticket":              "dummy-ticket",
+					"CSRFPreventionToken": "random-token",
+				},
+			})
+			return
+		}
+
+		// validate ticket
+		if val, err := req.Cookie("PVEAuthCookie"); err != nil || val.Value != "dummy-ticket" {
+			rw.WriteHeader(http.StatusUnauthorized)
+			return
+		}
+	}))
+	defer mockAPI.Close()
+
+	pmURL, _ := url.Parse(mockAPI.URL)
+	config := Config{
+		proxmoxURL:         pmURL,
+		SkipCertValidation: false,
+		Username:           "dummy@vmhost",
+		Password:           "correct-horse-battery-staple",
+		Token:              "",
+	}
+
+	client, err := newProxmoxClient(config)
+	require.NoError(t, err)
+
+	ref := proxmox.NewVmRef(110)
+	ref.SetNode("node1")
+	ref.SetVmType("qemu")
+	err = client.Sendkey(ref, "ping")
+	require.NoError(t, err)
+}

builder/proxmox/common/config.go

@@ -35,6 +35,7 @@ type Config struct {
 	SkipCertValidation bool   `mapstructure:"insecure_skip_tls_verify"`
 	Username           string `mapstructure:"username"`
 	Password           string `mapstructure:"password"`
+	Token              string `mapstructure:"token"`
 	Node               string `mapstructure:"node"`
 	Pool               string `mapstructure:"pool"`
 
@@ -73,8 +74,8 @@ type additionalISOsConfig struct {
 	ISOFile               string `mapstructure:"iso_file"`
 	ISOStoragePool        string `mapstructure:"iso_storage_pool"`
 	Unmount               bool   `mapstructure:"unmount"`
-	ShouldUploadISO       bool
-	DownloadPathKey       string
+	ShouldUploadISO       bool   `mapstructure-to-hcl2:",skip"`
+	DownloadPathKey       string `mapstructure-to-hcl2:",skip"`
 }
 
 type nicConfig struct {
@@ -135,6 +136,9 @@ func (c *Config) Prepare(upper interface{}, raws ...interface{}) ([]string, []st
 	if c.Password == "" {
 		c.Password = os.Getenv("PROXMOX_PASSWORD")
 	}
+	if c.Token == "" {
+		c.Token = os.Getenv("PROXMOX_TOKEN")
+	}
 	if c.BootKeyInterval == 0 && os.Getenv(bootcommand.PackerKeyEnv) != "" {
 		var err error
 		c.BootKeyInterval, err = time.ParseDuration(os.Getenv(bootcommand.PackerKeyEnv))
@@ -220,8 +224,8 @@ func (c *Config) Prepare(upper interface{}, raws ...interface{}) ([]string, []st
 	if c.Username == "" {
 		errs = packersdk.MultiErrorAppend(errs, errors.New("username must be specified"))
 	}
-	if c.Password == "" {
-		errs = packersdk.MultiErrorAppend(errs, errors.New("password must be specified"))
+	if c.Password == "" && c.Token == "" {
+		errs = packersdk.MultiErrorAppend(errs, errors.New("password or token must be specified"))
 	}
 	if c.ProxmoxURLRaw == "" {
 		errs = packersdk.MultiErrorAppend(errs, errors.New("proxmox_url must be specified"))

builder/proxmox/common/config.hcl2spec.go

@@ -19,6 +19,7 @@ type FlatConfig struct {
 	PackerUserVars            map[string]string          `mapstructure:"packer_user_variables" cty:"packer_user_variables" hcl:"packer_user_variables"`
 	PackerSensitiveVars       []string                   `mapstructure:"packer_sensitive_variables" cty:"packer_sensitive_variables" hcl:"packer_sensitive_variables"`
 	HTTPDir                   *string                    `mapstructure:"http_directory" cty:"http_directory" hcl:"http_directory"`
+	HTTPContent               map[string]string          `mapstructure:"http_content" cty:"http_content" hcl:"http_content"`
 	HTTPPortMin               *int                       `mapstructure:"http_port_min" cty:"http_port_min" hcl:"http_port_min"`
 	HTTPPortMax               *int                       `mapstructure:"http_port_max" cty:"http_port_max" hcl:"http_port_max"`
 	HTTPAddress               *string                    `mapstructure:"http_bind_address" cty:"http_bind_address" hcl:"http_bind_address"`
@@ -80,6 +81,7 @@ type FlatConfig struct {
 	SkipCertValidation        *bool                      `mapstructure:"insecure_skip_tls_verify" cty:"insecure_skip_tls_verify" hcl:"insecure_skip_tls_verify"`
 	Username                  *string                    `mapstructure:"username" cty:"username" hcl:"username"`
 	Password                  *string                    `mapstructure:"password" cty:"password" hcl:"password"`
+	Token                     *string                    `mapstructure:"token" cty:"token" hcl:"token"`
 	Node                      *string                    `mapstructure:"node" cty:"node" hcl:"node"`
 	Pool                      *string                    `mapstructure:"pool" cty:"pool" hcl:"pool"`
 	VMName                    *string                    `mapstructure:"vm_name" cty:"vm_name" hcl:"vm_name"`
@@ -126,6 +128,7 @@ func (*FlatConfig) HCL2Spec() map[string]hcldec.Spec {
 		"packer_user_variables":        &hcldec.AttrSpec{Name: "packer_user_variables", Type: cty.Map(cty.String), Required: false},
 		"packer_sensitive_variables":   &hcldec.AttrSpec{Name: "packer_sensitive_variables", Type: cty.List(cty.String), Required: false},
 		"http_directory":               &hcldec.AttrSpec{Name: "http_directory", Type: cty.String, Required: false},
+		"http_content":                 &hcldec.AttrSpec{Name: "http_content", Type: cty.Map(cty.String), Required: false},
 		"http_port_min":                &hcldec.AttrSpec{Name: "http_port_min", Type: cty.Number, Required: false},
 		"http_port_max":                &hcldec.AttrSpec{Name: "http_port_max", Type: cty.Number, Required: false},
 		"http_bind_address":            &hcldec.AttrSpec{Name: "http_bind_address", Type: cty.String, Required: false},
@@ -187,6 +190,7 @@ func (*FlatConfig) HCL2Spec() map[string]hcldec.Spec {
 		"insecure_skip_tls_verify":     &hcldec.AttrSpec{Name: "insecure_skip_tls_verify", Type: cty.Bool, Required: false},
 		"username":                     &hcldec.AttrSpec{Name: "username", Type: cty.String, Required: false},
 		"password":                     &hcldec.AttrSpec{Name: "password", Type: cty.String, Required: false},
+		"token":                        &hcldec.AttrSpec{Name: "token", Type: cty.String, Required: false},
 		"node":                         &hcldec.AttrSpec{Name: "node", Type: cty.String, Required: false},
 		"pool":                         &hcldec.AttrSpec{Name: "pool", Type: cty.String, Required: false},
 		"vm_name":                      &hcldec.AttrSpec{Name: "vm_name", Type: cty.String, Required: false},
@@ -226,8 +230,6 @@ type FlatadditionalISOsConfig struct {
 	ISOFile         *string  `mapstructure:"iso_file" cty:"iso_file" hcl:"iso_file"`
 	ISOStoragePool  *string  `mapstructure:"iso_storage_pool" cty:"iso_storage_pool" hcl:"iso_storage_pool"`
 	Unmount         *bool    `mapstructure:"unmount" cty:"unmount" hcl:"unmount"`
-	ShouldUploadISO *bool    `cty:"should_upload_iso" hcl:"should_upload_iso"`
-	DownloadPathKey *string  `cty:"download_path_key" hcl:"download_path_key"`
 }
 
 // FlatMapstructure returns a new FlatadditionalISOsConfig.
@@ -251,8 +253,6 @@ func (*FlatadditionalISOsConfig) HCL2Spec() map[string]hcldec.Spec {
 		"iso_file":             &hcldec.AttrSpec{Name: "iso_file", Type: cty.String, Required: false},
 		"iso_storage_pool":     &hcldec.AttrSpec{Name: "iso_storage_pool", Type: cty.String, Required: false},
 		"unmount":              &hcldec.AttrSpec{Name: "unmount", Type: cty.Bool, Required: false},
-		"should_upload_iso":    &hcldec.AttrSpec{Name: "should_upload_iso", Type: cty.Bool, Required: false},
-		"download_path_key":    &hcldec.AttrSpec{Name: "download_path_key", Type: cty.String, Required: false},
 	}
 	return s
 }

builder/proxmox/iso/config.hcl2spec.go

@@ -20,6 +20,7 @@ type FlatConfig struct {
 	PackerUserVars            map[string]string                  `mapstructure:"packer_user_variables" cty:"packer_user_variables" hcl:"packer_user_variables"`
 	PackerSensitiveVars       []string                           `mapstructure:"packer_sensitive_variables" cty:"packer_sensitive_variables" hcl:"packer_sensitive_variables"`
 	HTTPDir                   *string                            `mapstructure:"http_directory" cty:"http_directory" hcl:"http_directory"`
+	HTTPContent               map[string]string                  `mapstructure:"http_content" cty:"http_content" hcl:"http_content"`
 	HTTPPortMin               *int                               `mapstructure:"http_port_min" cty:"http_port_min" hcl:"http_port_min"`
 	HTTPPortMax               *int                               `mapstructure:"http_port_max" cty:"http_port_max" hcl:"http_port_max"`
 	HTTPAddress               *string                            `mapstructure:"http_bind_address" cty:"http_bind_address" hcl:"http_bind_address"`
@@ -81,6 +82,7 @@ type FlatConfig struct {
 	SkipCertValidation        *bool                              `mapstructure:"insecure_skip_tls_verify" cty:"insecure_skip_tls_verify" hcl:"insecure_skip_tls_verify"`
 	Username                  *string                            `mapstructure:"username" cty:"username" hcl:"username"`
 	Password                  *string                            `mapstructure:"password" cty:"password" hcl:"password"`
+	Token                     *string                            `mapstructure:"token" cty:"token" hcl:"token"`
 	Node                      *string                            `mapstructure:"node" cty:"node" hcl:"node"`
 	Pool                      *string                            `mapstructure:"pool" cty:"pool" hcl:"pool"`
 	VMName                    *string                            `mapstructure:"vm_name" cty:"vm_name" hcl:"vm_name"`
@@ -135,6 +137,7 @@ func (*FlatConfig) HCL2Spec() map[string]hcldec.Spec {
 		"packer_user_variables":        &hcldec.AttrSpec{Name: "packer_user_variables", Type: cty.Map(cty.String), Required: false},
 		"packer_sensitive_variables":   &hcldec.AttrSpec{Name: "packer_sensitive_variables", Type: cty.List(cty.String), Required: false},
 		"http_directory":               &hcldec.AttrSpec{Name: "http_directory", Type: cty.String, Required: false},
+		"http_content":                 &hcldec.AttrSpec{Name: "http_content", Type: cty.Map(cty.String), Required: false},
 		"http_port_min":                &hcldec.AttrSpec{Name: "http_port_min", Type: cty.Number, Required: false},
 		"http_port_max":                &hcldec.AttrSpec{Name: "http_port_max", Type: cty.Number, Required: false},
 		"http_bind_address":            &hcldec.AttrSpec{Name: "http_bind_address", Type: cty.String, Required: false},
@@ -196,6 +199,7 @@ func (*FlatConfig) HCL2Spec() map[string]hcldec.Spec {
 		"insecure_skip_tls_verify":     &hcldec.AttrSpec{Name: "insecure_skip_tls_verify", Type: cty.Bool, Required: false},
 		"username":                     &hcldec.AttrSpec{Name: "username", Type: cty.String, Required: false},
 		"password":                     &hcldec.AttrSpec{Name: "password", Type: cty.String, Required: false},
+		"token":                        &hcldec.AttrSpec{Name: "token", Type: cty.String, Required: false},
 		"node":                         &hcldec.AttrSpec{Name: "node", Type: cty.String, Required: false},
 		"pool":                         &hcldec.AttrSpec{Name: "pool", Type: cty.String, Required: false},
 		"vm_name":                      &hcldec.AttrSpec{Name: "vm_name", Type: cty.String, Required: false},

builder/qemu/builder.go

@@ -96,12 +96,7 @@ func (b *Builder) Run(ctx context.Context, ui packersdk.Ui, hook packersdk.Hook)
 			QemuImgArgs:     b.config.QemuImgArgs,
 		},
 		new(stepHTTPIPDiscover),
-		&commonsteps.StepHTTPServer{
-			HTTPDir:     b.config.HTTPDir,
-			HTTPPortMin: b.config.HTTPPortMin,
-			HTTPPortMax: b.config.HTTPPortMax,
-			HTTPAddress: b.config.HTTPAddress,
-		},
+		commonsteps.HTTPServerFromHTTPConfig(&b.config.HTTPConfig),
 		&stepPortForward{
 			CommunicatorType: b.config.CommConfig.Comm.Type,
 			NetBridge:        b.config.NetBridge,

builder/qemu/config.hcl2spec.go

@@ -19,6 +19,7 @@ type FlatConfig struct {
 	PackerUserVars            map[string]string `mapstructure:"packer_user_variables" cty:"packer_user_variables" hcl:"packer_user_variables"`
 	PackerSensitiveVars       []string          `mapstructure:"packer_sensitive_variables" cty:"packer_sensitive_variables" hcl:"packer_sensitive_variables"`
 	HTTPDir                   *string           `mapstructure:"http_directory" cty:"http_directory" hcl:"http_directory"`
+	HTTPContent               map[string]string `mapstructure:"http_content" cty:"http_content" hcl:"http_content"`
 	HTTPPortMin               *int              `mapstructure:"http_port_min" cty:"http_port_min" hcl:"http_port_min"`
 	HTTPPortMax               *int              `mapstructure:"http_port_max" cty:"http_port_max" hcl:"http_port_max"`
 	HTTPAddress               *string           `mapstructure:"http_bind_address" cty:"http_bind_address" hcl:"http_bind_address"`
@@ -153,6 +154,7 @@ func (*FlatConfig) HCL2Spec() map[string]hcldec.Spec {
 		"packer_user_variables":        &hcldec.AttrSpec{Name: "packer_user_variables", Type: cty.Map(cty.String), Required: false},
 		"packer_sensitive_variables":   &hcldec.AttrSpec{Name: "packer_sensitive_variables", Type: cty.List(cty.String), Required: false},
 		"http_directory":               &hcldec.AttrSpec{Name: "http_directory", Type: cty.String, Required: false},
+		"http_content":                 &hcldec.AttrSpec{Name: "http_content", Type: cty.Map(cty.String), Required: false},
 		"http_port_min":                &hcldec.AttrSpec{Name: "http_port_min", Type: cty.Number, Required: false},
 		"http_port_max":                &hcldec.AttrSpec{Name: "http_port_max", Type: cty.Number, Required: false},
 		"http_bind_address":            &hcldec.AttrSpec{Name: "http_bind_address", Type: cty.String, Required: false},

builder/qemu/step_run.go

@@ -298,6 +298,7 @@ func (s *stepRun) applyUserOverrides(defaultArgs map[string]interface{}, config
 			HTTPIP      string
 			HTTPPort    int
 			HTTPDir     string
+			HTTPContent map[string]string
 			OutputDir   string
 			Name        string
 			SSHHostPort int
@@ -308,6 +309,7 @@ func (s *stepRun) applyUserOverrides(defaultArgs map[string]interface{}, config
 			HTTPIP:      httpIp,
 			HTTPPort:    httpPort,
 			HTTPDir:     config.HTTPDir,
+			HTTPContent: config.HTTPContent,
 			OutputDir:   config.OutputDir,
 			Name:        config.VMName,
 			SSHHostPort: commHostPort,

builder/vagrant/builder.hcl2spec.go

@@ -19,6 +19,7 @@ type FlatConfig struct {
 	PackerUserVars            map[string]string `mapstructure:"packer_user_variables" cty:"packer_user_variables" hcl:"packer_user_variables"`
 	PackerSensitiveVars       []string          `mapstructure:"packer_sensitive_variables" cty:"packer_sensitive_variables" hcl:"packer_sensitive_variables"`
 	HTTPDir                   *string           `mapstructure:"http_directory" cty:"http_directory" hcl:"http_directory"`
+	HTTPContent               map[string]string `mapstructure:"http_content" cty:"http_content" hcl:"http_content"`
 	HTTPPortMin               *int              `mapstructure:"http_port_min" cty:"http_port_min" hcl:"http_port_min"`
 	HTTPPortMax               *int              `mapstructure:"http_port_max" cty:"http_port_max" hcl:"http_port_max"`
 	HTTPAddress               *string           `mapstructure:"http_bind_address" cty:"http_bind_address" hcl:"http_bind_address"`
@@ -128,6 +129,7 @@ func (*FlatConfig) HCL2Spec() map[string]hcldec.Spec {
 		"packer_user_variables":        &hcldec.AttrSpec{Name: "packer_user_variables", Type: cty.Map(cty.String), Required: false},
 		"packer_sensitive_variables":   &hcldec.AttrSpec{Name: "packer_sensitive_variables", Type: cty.List(cty.String), Required: false},
 		"http_directory":               &hcldec.AttrSpec{Name: "http_directory", Type: cty.String, Required: false},
+		"http_content":                 &hcldec.AttrSpec{Name: "http_content", Type: cty.Map(cty.String), Required: false},
 		"http_port_min":                &hcldec.AttrSpec{Name: "http_port_min", Type: cty.Number, Required: false},
 		"http_port_max":                &hcldec.AttrSpec{Name: "http_port_max", Type: cty.Number, Required: false},
 		"http_bind_address":            &hcldec.AttrSpec{Name: "http_bind_address", Type: cty.String, Required: false},

builder/virtualbox/iso/builder.go

@@ -406,12 +406,7 @@ func (b *Builder) Run(ctx context.Context, ui packersdk.Ui, hook packersdk.Hook)
 			Label: b.config.CDConfig.CDLabel,
 		},
 		new(vboxcommon.StepHTTPIPDiscover),
-		&commonsteps.StepHTTPServer{
-			HTTPDir:     b.config.HTTPDir,
-			HTTPPortMin: b.config.HTTPPortMin,
-			HTTPPortMax: b.config.HTTPPortMax,
-			HTTPAddress: b.config.HTTPAddress,
-		},
+		commonsteps.HTTPServerFromHTTPConfig(&b.config.HTTPConfig),
 		&vboxcommon.StepSshKeyPair{
 			Debug:        b.config.PackerDebug,
 			DebugKeyPath: fmt.Sprintf("%s.pem", b.config.PackerBuildName),

builder/virtualbox/iso/builder.hcl2spec.go

@@ -19,6 +19,7 @@ type FlatConfig struct {
 	PackerUserVars            map[string]string `mapstructure:"packer_user_variables" cty:"packer_user_variables" hcl:"packer_user_variables"`
 	PackerSensitiveVars       []string          `mapstructure:"packer_sensitive_variables" cty:"packer_sensitive_variables" hcl:"packer_sensitive_variables"`
 	HTTPDir                   *string           `mapstructure:"http_directory" cty:"http_directory" hcl:"http_directory"`
+	HTTPContent               map[string]string `mapstructure:"http_content" cty:"http_content" hcl:"http_content"`
 	HTTPPortMin               *int              `mapstructure:"http_port_min" cty:"http_port_min" hcl:"http_port_min"`
 	HTTPPortMax               *int              `mapstructure:"http_port_max" cty:"http_port_max" hcl:"http_port_max"`
 	HTTPAddress               *string           `mapstructure:"http_bind_address" cty:"http_bind_address" hcl:"http_bind_address"`
@@ -162,6 +163,7 @@ func (*FlatConfig) HCL2Spec() map[string]hcldec.Spec {
 		"packer_user_variables":        &hcldec.AttrSpec{Name: "packer_user_variables", Type: cty.Map(cty.String), Required: false},
 		"packer_sensitive_variables":   &hcldec.AttrSpec{Name: "packer_sensitive_variables", Type: cty.List(cty.String), Required: false},
 		"http_directory":               &hcldec.AttrSpec{Name: "http_directory", Type: cty.String, Required: false},
+		"http_content":                 &hcldec.AttrSpec{Name: "http_content", Type: cty.Map(cty.String), Required: false},
 		"http_port_min":                &hcldec.AttrSpec{Name: "http_port_min", Type: cty.Number, Required: false},
 		"http_port_max":                &hcldec.AttrSpec{Name: "http_port_max", Type: cty.Number, Required: false},
 		"http_bind_address":            &hcldec.AttrSpec{Name: "http_bind_address", Type: cty.String, Required: false},

builder/virtualbox/ovf/builder.go

@@ -65,12 +65,7 @@ func (b *Builder) Run(ctx context.Context, ui packersdk.Ui, hook packersdk.Hook)
 			Label: b.config.CDConfig.CDLabel,
 		},
 		new(vboxcommon.StepHTTPIPDiscover),
-		&commonsteps.StepHTTPServer{
-			HTTPDir:     b.config.HTTPDir,
-			HTTPPortMin: b.config.HTTPPortMin,
-			HTTPPortMax: b.config.HTTPPortMax,
-			HTTPAddress: b.config.HTTPAddress,
-		},
+		commonsteps.HTTPServerFromHTTPConfig(&b.config.HTTPConfig),
 		&vboxcommon.StepSshKeyPair{
 			Debug:        b.config.PackerDebug,
 			DebugKeyPath: fmt.Sprintf("%s.pem", b.config.PackerBuildName),

builder/virtualbox/ovf/config.hcl2spec.go

@@ -19,6 +19,7 @@ type FlatConfig struct {
 	PackerUserVars            map[string]string `mapstructure:"packer_user_variables" cty:"packer_user_variables" hcl:"packer_user_variables"`
 	PackerSensitiveVars       []string          `mapstructure:"packer_sensitive_variables" cty:"packer_sensitive_variables" hcl:"packer_sensitive_variables"`
 	HTTPDir                   *string           `mapstructure:"http_directory" cty:"http_directory" hcl:"http_directory"`
+	HTTPContent               map[string]string `mapstructure:"http_content" cty:"http_content" hcl:"http_content"`
 	HTTPPortMin               *int              `mapstructure:"http_port_min" cty:"http_port_min" hcl:"http_port_min"`
 	HTTPPortMax               *int              `mapstructure:"http_port_max" cty:"http_port_max" hcl:"http_port_max"`
 	HTTPAddress               *string           `mapstructure:"http_bind_address" cty:"http_bind_address" hcl:"http_bind_address"`
@@ -138,6 +139,7 @@ func (*FlatConfig) HCL2Spec() map[string]hcldec.Spec {
 		"packer_user_variables":        &hcldec.AttrSpec{Name: "packer_user_variables", Type: cty.Map(cty.String), Required: false},
 		"packer_sensitive_variables":   &hcldec.AttrSpec{Name: "packer_sensitive_variables", Type: cty.List(cty.String), Required: false},
 		"http_directory":               &hcldec.AttrSpec{Name: "http_directory", Type: cty.String, Required: false},
+		"http_content":                 &hcldec.AttrSpec{Name: "http_content", Type: cty.Map(cty.String), Required: false},
 		"http_port_min":                &hcldec.AttrSpec{Name: "http_port_min", Type: cty.Number, Required: false},
 		"http_port_max":                &hcldec.AttrSpec{Name: "http_port_max", Type: cty.Number, Required: false},
 		"http_bind_address":            &hcldec.AttrSpec{Name: "http_bind_address", Type: cty.String, Required: false},

builder/virtualbox/vm/builder.go

@@ -64,12 +64,7 @@ func (b *Builder) Run(ctx context.Context, ui packersdk.Ui, hook packersdk.Hook)
 			KeepRegistered: b.config.KeepRegistered,
 		},
 		new(vboxcommon.StepHTTPIPDiscover),
-		&commonsteps.StepHTTPServer{
-			HTTPDir:     b.config.HTTPDir,
-			HTTPPortMin: b.config.HTTPPortMin,
-			HTTPPortMax: b.config.HTTPPortMax,
-			HTTPAddress: b.config.HTTPAddress,
-		},
+		commonsteps.HTTPServerFromHTTPConfig(&b.config.HTTPConfig),
 		&vboxcommon.StepDownloadGuestAdditions{
 			GuestAdditionsMode:   b.config.GuestAdditionsMode,
 			GuestAdditionsURL:    b.config.GuestAdditionsURL,

builder/virtualbox/vm/config.hcl2spec.go

@@ -19,6 +19,7 @@ type FlatConfig struct {
 	PackerUserVars            map[string]string `mapstructure:"packer_user_variables" cty:"packer_user_variables" hcl:"packer_user_variables"`
 	PackerSensitiveVars       []string          `mapstructure:"packer_sensitive_variables" cty:"packer_sensitive_variables" hcl:"packer_sensitive_variables"`
 	HTTPDir                   *string           `mapstructure:"http_directory" cty:"http_directory" hcl:"http_directory"`
+	HTTPContent               map[string]string `mapstructure:"http_content" cty:"http_content" hcl:"http_content"`
 	HTTPPortMin               *int              `mapstructure:"http_port_min" cty:"http_port_min" hcl:"http_port_min"`
 	HTTPPortMax               *int              `mapstructure:"http_port_max" cty:"http_port_max" hcl:"http_port_max"`
 	HTTPAddress               *string           `mapstructure:"http_bind_address" cty:"http_bind_address" hcl:"http_bind_address"`
@@ -136,6 +137,7 @@ func (*FlatConfig) HCL2Spec() map[string]hcldec.Spec {
 		"packer_user_variables":        &hcldec.AttrSpec{Name: "packer_user_variables", Type: cty.Map(cty.String), Required: false},
 		"packer_sensitive_variables":   &hcldec.AttrSpec{Name: "packer_sensitive_variables", Type: cty.List(cty.String), Required: false},
 		"http_directory":               &hcldec.AttrSpec{Name: "http_directory", Type: cty.String, Required: false},
+		"http_content":                 &hcldec.AttrSpec{Name: "http_content", Type: cty.Map(cty.String), Required: false},
 		"http_port_min":                &hcldec.AttrSpec{Name: "http_port_min", Type: cty.Number, Required: false},
 		"http_port_max":                &hcldec.AttrSpec{Name: "http_port_max", Type: cty.Number, Required: false},
 		"http_bind_address":            &hcldec.AttrSpec{Name: "http_bind_address", Type: cty.String, Required: false},

builder/vmware/iso/builder.go

@@ -117,12 +117,7 @@ func (b *Builder) Run(ctx context.Context, ui packersdk.Ui, hook packersdk.Hook)
 		},
 		&vmwcommon.StepSuppressMessages{},
 		&vmwcommon.StepHTTPIPDiscover{},
-		&commonsteps.StepHTTPServer{
-			HTTPDir:     b.config.HTTPDir,
-			HTTPPortMin: b.config.HTTPPortMin,
-			HTTPPortMax: b.config.HTTPPortMax,
-			HTTPAddress: b.config.HTTPAddress,
-		},
+		commonsteps.HTTPServerFromHTTPConfig(&b.config.HTTPConfig),
 		&vmwcommon.StepConfigureVNC{
 			Enabled:            !b.config.DisableVNC && !b.config.VNCOverWebsocket,
 			VNCBindAddress:     b.config.VNCBindAddress,

builder/vmware/iso/config.hcl2spec.go

@@ -19,6 +19,7 @@ type FlatConfig struct {
 	PackerUserVars            map[string]string `mapstructure:"packer_user_variables" cty:"packer_user_variables" hcl:"packer_user_variables"`
 	PackerSensitiveVars       []string          `mapstructure:"packer_sensitive_variables" cty:"packer_sensitive_variables" hcl:"packer_sensitive_variables"`
 	HTTPDir                   *string           `mapstructure:"http_directory" cty:"http_directory" hcl:"http_directory"`
+	HTTPContent               map[string]string `mapstructure:"http_content" cty:"http_content" hcl:"http_content"`
 	HTTPPortMin               *int              `mapstructure:"http_port_min" cty:"http_port_min" hcl:"http_port_min"`
 	HTTPPortMax               *int              `mapstructure:"http_port_max" cty:"http_port_max" hcl:"http_port_max"`
 	HTTPAddress               *string           `mapstructure:"http_bind_address" cty:"http_bind_address" hcl:"http_bind_address"`
@@ -167,6 +168,7 @@ func (*FlatConfig) HCL2Spec() map[string]hcldec.Spec {
 		"packer_user_variables":          &hcldec.AttrSpec{Name: "packer_user_variables", Type: cty.Map(cty.String), Required: false},
 		"packer_sensitive_variables":     &hcldec.AttrSpec{Name: "packer_sensitive_variables", Type: cty.List(cty.String), Required: false},
 		"http_directory":                 &hcldec.AttrSpec{Name: "http_directory", Type: cty.String, Required: false},
+		"http_content":                   &hcldec.AttrSpec{Name: "http_content", Type: cty.Map(cty.String), Required: false},
 		"http_port_min":                  &hcldec.AttrSpec{Name: "http_port_min", Type: cty.Number, Required: false},
 		"http_port_max":                  &hcldec.AttrSpec{Name: "http_port_max", Type: cty.Number, Required: false},
 		"http_bind_address":              &hcldec.AttrSpec{Name: "http_bind_address", Type: cty.String, Required: false},

builder/vmware/vmx/builder.go

@@ -108,12 +108,7 @@ func (b *Builder) Run(ctx context.Context, ui packersdk.Ui, hook packersdk.Hook)
 		},
 		&vmwcommon.StepSuppressMessages{},
 		&vmwcommon.StepHTTPIPDiscover{},
-		&commonsteps.StepHTTPServer{
-			HTTPDir:     b.config.HTTPDir,
-			HTTPPortMin: b.config.HTTPPortMin,
-			HTTPPortMax: b.config.HTTPPortMax,
-			HTTPAddress: b.config.HTTPAddress,
-		},
+		commonsteps.HTTPServerFromHTTPConfig(&b.config.HTTPConfig),
 		&vmwcommon.StepUploadVMX{
 			RemoteType: b.config.RemoteType,
 		},

builder/vmware/vmx/config.hcl2spec.go

@@ -19,6 +19,7 @@ type FlatConfig struct {
 	PackerUserVars            map[string]string `mapstructure:"packer_user_variables" cty:"packer_user_variables" hcl:"packer_user_variables"`
 	PackerSensitiveVars       []string          `mapstructure:"packer_sensitive_variables" cty:"packer_sensitive_variables" hcl:"packer_sensitive_variables"`
 	HTTPDir                   *string           `mapstructure:"http_directory" cty:"http_directory" hcl:"http_directory"`
+	HTTPContent               map[string]string `mapstructure:"http_content" cty:"http_content" hcl:"http_content"`
 	HTTPPortMin               *int              `mapstructure:"http_port_min" cty:"http_port_min" hcl:"http_port_min"`
 	HTTPPortMax               *int              `mapstructure:"http_port_max" cty:"http_port_max" hcl:"http_port_max"`
 	HTTPAddress               *string           `mapstructure:"http_bind_address" cty:"http_bind_address" hcl:"http_bind_address"`
@@ -149,6 +150,7 @@ func (*FlatConfig) HCL2Spec() map[string]hcldec.Spec {
 		"packer_user_variables":          &hcldec.AttrSpec{Name: "packer_user_variables", Type: cty.Map(cty.String), Required: false},
 		"packer_sensitive_variables":     &hcldec.AttrSpec{Name: "packer_sensitive_variables", Type: cty.List(cty.String), Required: false},
 		"http_directory":                 &hcldec.AttrSpec{Name: "http_directory", Type: cty.String, Required: false},
+		"http_content":                   &hcldec.AttrSpec{Name: "http_content", Type: cty.Map(cty.String), Required: false},
 		"http_port_min":                  &hcldec.AttrSpec{Name: "http_port_min", Type: cty.Number, Required: false},
 		"http_port_max":                  &hcldec.AttrSpec{Name: "http_port_max", Type: cty.Number, Required: false},
 		"http_bind_address":              &hcldec.AttrSpec{Name: "http_bind_address", Type: cty.String, Required: false},

builder/vsphere/clone/builder.go

@@ -89,12 +89,7 @@ func (b *Builder) Run(ctx context.Context, ui packersdk.Ui, hook packersdk.Hook)
 				HTTPIP:  b.config.BootConfig.HTTPIP,
 				Network: b.config.WaitIpConfig.GetIPNet(),
 			},
-			&commonsteps.StepHTTPServer{
-				HTTPDir:     b.config.HTTPDir,
-				HTTPPortMin: b.config.HTTPPortMin,
-				HTTPPortMax: b.config.HTTPPortMax,
-				HTTPAddress: b.config.HTTPAddress,
-			},
+			commonsteps.HTTPServerFromHTTPConfig(&b.config.HTTPConfig),
 			&common.StepSshKeyPair{
 				Debug:        b.config.PackerDebug,
 				DebugKeyPath: fmt.Sprintf("%s.pem", b.config.PackerBuildName),

builder/vsphere/clone/config.hcl2spec.go

@@ -20,6 +20,7 @@ type FlatConfig struct {
 	PackerUserVars                  map[string]string                           `mapstructure:"packer_user_variables" cty:"packer_user_variables" hcl:"packer_user_variables"`
 	PackerSensitiveVars             []string                                    `mapstructure:"packer_sensitive_variables" cty:"packer_sensitive_variables" hcl:"packer_sensitive_variables"`
 	HTTPDir                         *string                                     `mapstructure:"http_directory" cty:"http_directory" hcl:"http_directory"`
+	HTTPContent                     map[string]string                           `mapstructure:"http_content" cty:"http_content" hcl:"http_content"`
 	HTTPPortMin                     *int                                        `mapstructure:"http_port_min" cty:"http_port_min" hcl:"http_port_min"`
 	HTTPPortMax                     *int                                        `mapstructure:"http_port_max" cty:"http_port_max" hcl:"http_port_max"`
 	HTTPAddress                     *string                                     `mapstructure:"http_bind_address" cty:"http_bind_address" hcl:"http_bind_address"`
@@ -159,6 +160,7 @@ func (*FlatConfig) HCL2Spec() map[string]hcldec.Spec {
 		"packer_user_variables":          &hcldec.AttrSpec{Name: "packer_user_variables", Type: cty.Map(cty.String), Required: false},
 		"packer_sensitive_variables":     &hcldec.AttrSpec{Name: "packer_sensitive_variables", Type: cty.List(cty.String), Required: false},
 		"http_directory":                 &hcldec.AttrSpec{Name: "http_directory", Type: cty.String, Required: false},
+		"http_content":                   &hcldec.AttrSpec{Name: "http_content", Type: cty.Map(cty.String), Required: false},
 		"http_port_min":                  &hcldec.AttrSpec{Name: "http_port_min", Type: cty.Number, Required: false},
 		"http_port_max":                  &hcldec.AttrSpec{Name: "http_port_max", Type: cty.Number, Required: false},
 		"http_bind_address":              &hcldec.AttrSpec{Name: "http_bind_address", Type: cty.String, Required: false},

builder/vsphere/common/step_boot_command.go

@@ -99,7 +99,19 @@ func (s *StepBootCommand) Run(ctx context.Context, state multistep.StateBag) mul
 			Shift:    shift,
 		})
 		if err != nil {
-			return fmt.Errorf("error typing a boot command (code, down) `%d, %t`: %w", code, down, err)
+			// retry once if error
+			ui.Error(fmt.Errorf("error typing a boot command (code, down) `%d, %t`: %w", code, down, err).Error())
+			ui.Say("trying key input again")
+			time.Sleep(s.Config.BootGroupInterval)
+			_, err = vm.TypeOnKeyboard(driver.KeyInput{
+				Scancode: code,
+				Ctrl:     keyCtrl,
+				Alt:      keyAlt,
+				Shift:    shift,
+			})
+			if err != nil {
+				return fmt.Errorf("error typing a boot command (code, down) `%d, %t`: %w", code, down, err)
+			}
 		}
 		return nil
 	}

builder/vsphere/iso/builder.go

@@ -92,12 +92,7 @@ func (b *Builder) Run(ctx context.Context, ui packersdk.Ui, hook packersdk.Hook)
 			HTTPIP:  b.config.BootConfig.HTTPIP,
 			Network: b.config.WaitIpConfig.GetIPNet(),
 		},
-		&commonsteps.StepHTTPServer{
-			HTTPDir:     b.config.HTTPDir,
-			HTTPPortMin: b.config.HTTPPortMin,
-			HTTPPortMax: b.config.HTTPPortMax,
-			HTTPAddress: b.config.HTTPAddress,
-		},
+		commonsteps.HTTPServerFromHTTPConfig(&b.config.HTTPConfig),
 		&common.StepRun{
 			Config:   &b.config.RunConfig,
 			SetOrder: true,

builder/vsphere/iso/config.hcl2spec.go

@@ -20,6 +20,7 @@ type FlatConfig struct {
 	PackerUserVars                  map[string]string                           `mapstructure:"packer_user_variables" cty:"packer_user_variables" hcl:"packer_user_variables"`
 	PackerSensitiveVars             []string                                    `mapstructure:"packer_sensitive_variables" cty:"packer_sensitive_variables" hcl:"packer_sensitive_variables"`
 	HTTPDir                         *string                                     `mapstructure:"http_directory" cty:"http_directory" hcl:"http_directory"`
+	HTTPContent                     map[string]string                           `mapstructure:"http_content" cty:"http_content" hcl:"http_content"`
 	HTTPPortMin                     *int                                        `mapstructure:"http_port_min" cty:"http_port_min" hcl:"http_port_min"`
 	HTTPPortMax                     *int                                        `mapstructure:"http_port_max" cty:"http_port_max" hcl:"http_port_max"`
 	HTTPAddress                     *string                                     `mapstructure:"http_bind_address" cty:"http_bind_address" hcl:"http_bind_address"`
@@ -161,6 +162,7 @@ func (*FlatConfig) HCL2Spec() map[string]hcldec.Spec {
 		"packer_user_variables":          &hcldec.AttrSpec{Name: "packer_user_variables", Type: cty.Map(cty.String), Required: false},
 		"packer_sensitive_variables":     &hcldec.AttrSpec{Name: "packer_sensitive_variables", Type: cty.List(cty.String), Required: false},
 		"http_directory":                 &hcldec.AttrSpec{Name: "http_directory", Type: cty.String, Required: false},
+		"http_content":                   &hcldec.AttrSpec{Name: "http_content", Type: cty.Map(cty.String), Required: false},
 		"http_port_min":                  &hcldec.AttrSpec{Name: "http_port_min", Type: cty.Number, Required: false},
 		"http_port_max":                  &hcldec.AttrSpec{Name: "http_port_max", Type: cty.Number, Required: false},
 		"http_bind_address":              &hcldec.AttrSpec{Name: "http_bind_address", Type: cty.String, Required: false},

command/hcl2_upgrade.go

@@ -8,7 +8,9 @@ import (
 	"os"
 	"path/filepath"
 	"reflect"
+	"regexp"
 	"sort"
+	"strconv"
 	"strings"
 	texttemplate "text/template"
 
@@ -120,6 +122,7 @@ var (
 	amazonSecretsManagerMap = map[string]map[string]interface{}{}
 	localsVariableMap       = map[string]string{}
 	timestamp               = false
+	isotime                 = false
 )
 
 type BlockParser interface {
@@ -269,18 +272,18 @@ func (uc UnhandleableArgumentError) Error() string {
 # Visit %s for more infos.`, uc.Call, uc.Correspondance, uc.Docs)
 }
 
+func fallbackReturn(err error, s []byte) []byte {
+	if strings.Contains(err.Error(), "unhandled") {
+		return append([]byte(fmt.Sprintf("\n# %s\n", err)), s...)
+	}
+
+	return append([]byte(fmt.Sprintf("\n# could not parse template for following block: %q\n", err)), s...)
+}
+
 // transposeTemplatingCalls executes parts of blocks as go template files and replaces
 // their result with their hcl2 variant. If something goes wrong the template
 // containing the go template string is returned.
 func transposeTemplatingCalls(s []byte) []byte {
-	fallbackReturn := func(err error) []byte {
-		if strings.Contains(err.Error(), "unhandled") {
-			return append([]byte(fmt.Sprintf("\n# %s\n", err)), s...)
-		}
-
-		return append([]byte(fmt.Sprintf("\n# could not parse template for following block: %q\n", err)), s...)
-	}
-
 	funcErrors := &multierror.Error{
 		ErrorFormat: func(es []error) string {
 			if len(es) == 1 {
@@ -336,8 +339,14 @@ func transposeTemplatingCalls(s []byte) []byte {
 			return "${local.timestamp}"
 		},
 		"isotime": func(a ...string) string {
-			timestamp = true
-			return "${local.timestamp}"
+			if len(a) == 0 {
+				// returns rfc3339 formatted string.
+				return "${timestamp()}"
+			}
+			// otherwise a valid isotime func has one input.
+			isotime = true
+			return fmt.Sprintf("${legacy_isotime(\"%s\")}", a[0])
+
 		},
 		"user": func(in string) string {
 			if _, ok := localsVariableMap[in]; ok {
@@ -430,14 +439,29 @@ func transposeTemplatingCalls(s []byte) []byte {
 		Parse(string(s))
 
 	if err != nil {
-		return fallbackReturn(err)
+		if strings.Contains(err.Error(), "unexpected \"\\\\\" in operand") {
+			// This error occurs if the operand in the text template used
+			// escaped quoting \" instead of bactick quoting `
+			// Create a regex to do a string replace on this block, to fix
+			// quoting.
+			q := fixQuoting(string(s))
+			unquoted := []byte(q)
+			tpl, err = texttemplate.New("hcl2_upgrade").
+				Funcs(funcMap).
+				Parse(string(unquoted))
+			if err != nil {
+				return fallbackReturn(err, unquoted)
+			}
+		} else {
+			return fallbackReturn(err, s)
+		}
 	}
 
 	str := &bytes.Buffer{}
 	// PASSTHROUGHS is a map of variable-specific golang text template fields
 	// that should remain in the text template format.
 	if err := tpl.Execute(str, PASSTHROUGHS); err != nil {
-		return fallbackReturn(err)
+		return fallbackReturn(err, s)
 	}
 
 	out := str.Bytes()
@@ -454,14 +478,6 @@ func transposeTemplatingCalls(s []byte) []byte {
 // In variableTransposeTemplatingCalls the definition of aws_secretsmanager function will create a data source
 // with the same name as the variable.
 func variableTransposeTemplatingCalls(s []byte) (isLocal bool, body []byte) {
-	fallbackReturn := func(err error) []byte {
-		if strings.Contains(err.Error(), "unhandled") {
-			return append([]byte(fmt.Sprintf("\n# %s\n", err)), s...)
-		}
-
-		return append([]byte(fmt.Sprintf("\n# could not parse template for following block: %q\n", err)), s...)
-	}
-
 	setIsLocal := func(a ...string) string {
 		isLocal = true
 		return ""
@@ -503,14 +519,29 @@ func variableTransposeTemplatingCalls(s []byte) (isLocal bool, body []byte) {
 		Parse(string(s))
 
 	if err != nil {
-		return isLocal, fallbackReturn(err)
+		if strings.Contains(err.Error(), "unexpected \"\\\\\" in operand") {
+			// This error occurs if the operand in the text template used
+			// escaped quoting \" instead of bactick quoting `
+			// Create a regex to do a string replace on this block, to fix
+			// quoting.
+			q := fixQuoting(string(s))
+			unquoted := []byte(q)
+			tpl, err = texttemplate.New("hcl2_upgrade").
+				Funcs(funcMap).
+				Parse(string(unquoted))
+			if err != nil {
+				return isLocal, fallbackReturn(err, unquoted)
+			}
+		} else {
+			return isLocal, fallbackReturn(err, s)
+		}
 	}
 
 	str := &bytes.Buffer{}
 	// PASSTHROUGHS is a map of variable-specific golang text template fields
 	// that should remain in the text template format.
 	if err := tpl.Execute(str, PASSTHROUGHS); err != nil {
-		return isLocal, fallbackReturn(err)
+		return isLocal, fallbackReturn(err, s)
 	}
 
 	return isLocal, str.Bytes()
@@ -675,12 +706,49 @@ type VariableParser struct {
 	localsOut       []byte
 }
 
+func makeLocal(variable *template.Variable, sensitive bool, localBody *hclwrite.Body, localsContent *hclwrite.File, hasLocals *bool) []byte {
+	if sensitive {
+		// Create Local block because this is sensitive
+		sensitiveLocalContent := hclwrite.NewEmptyFile()
+		body := sensitiveLocalContent.Body()
+		body.AppendNewline()
+		sensitiveLocalBody := body.AppendNewBlock("local", []string{variable.Key}).Body()
+		sensitiveLocalBody.SetAttributeValue("sensitive", cty.BoolVal(true))
+		sensitiveLocalBody.SetAttributeValue("expression", hcl2shim.HCL2ValueFromConfigValue(variable.Default))
+		localsVariableMap[variable.Key] = "local"
+		return sensitiveLocalContent.Bytes()
+	}
+	localBody.SetAttributeValue(variable.Key, hcl2shim.HCL2ValueFromConfigValue(variable.Default))
+	localsVariableMap[variable.Key] = "locals"
+	*hasLocals = true
+	return []byte{}
+}
+
+func makeVariable(variable *template.Variable, sensitive bool) []byte {
+	variablesContent := hclwrite.NewEmptyFile()
+	variablesBody := variablesContent.Body()
+	variablesBody.AppendNewline()
+	variableBody := variablesBody.AppendNewBlock("variable", []string{variable.Key}).Body()
+	variableBody.SetAttributeRaw("type", hclwrite.Tokens{&hclwrite.Token{Bytes: []byte("string")}})
+
+	if variable.Default != "" || !variable.Required {
+		shimmed := hcl2shim.HCL2ValueFromConfigValue(variable.Default)
+		variableBody.SetAttributeValue("default", shimmed)
+	}
+	if sensitive {
+		variableBody.SetAttributeValue("sensitive", cty.BoolVal(true))
+	}
+
+	return variablesContent.Bytes()
+}
+
 func (p *VariableParser) Parse(tpl *template.Template) error {
-	// OutPut Locals and Local blocks
+	// Output Locals and Local blocks
 	localsContent := hclwrite.NewEmptyFile()
 	localsBody := localsContent.Body()
 	localsBody.AppendNewline()
 	localBody := localsBody.AppendNewBlock("locals", nil).Body()
+	hasLocals := false
 
 	if len(p.variablesOut) == 0 {
 		p.variablesOut = []byte{}
@@ -700,47 +768,34 @@ func (p *VariableParser) Parse(tpl *template.Template) error {
 		})
 	}
 
-	hasLocals := false
 	for _, variable := range variables {
-		variablesContent := hclwrite.NewEmptyFile()
-		variablesBody := variablesContent.Body()
-		variablesBody.AppendNewline()
-		variableBody := variablesBody.AppendNewBlock("variable", []string{variable.Key}).Body()
-		variableBody.SetAttributeRaw("type", hclwrite.Tokens{&hclwrite.Token{Bytes: []byte("string")}})
+		// Create new HCL2 "variables" block, and populate the "value"
+		// field with the "Default" value from the JSON variable.
 
-		if variable.Default != "" || !variable.Required {
-			variableBody.SetAttributeValue("default", hcl2shim.HCL2ValueFromConfigValue(variable.Default))
-		}
+		// Interpolate Jsonval first as an hcl variable to determine if it is
+		// a local.
+		isLocal, _ := variableTransposeTemplatingCalls([]byte(variable.Default))
 		sensitive := false
 		if isSensitiveVariable(variable.Key, tpl.SensitiveVariables) {
 			sensitive = true
-			variableBody.SetAttributeValue("sensitive", cty.BoolVal(true))
 		}
-		isLocal, out := variableTransposeTemplatingCalls(variablesContent.Bytes())
+		// Create final HCL block and append.
 		if isLocal {
-			if sensitive {
-				// Create Local block because this is sensitive
-				localContent := hclwrite.NewEmptyFile()
-				body := localContent.Body()
-				body.AppendNewline()
-				localBody := body.AppendNewBlock("local", []string{variable.Key}).Body()
-				localBody.SetAttributeValue("sensitive", cty.BoolVal(true))
-				localBody.SetAttributeValue("expression", hcl2shim.HCL2ValueFromConfigValue(variable.Default))
-				p.localsOut = append(p.localsOut, transposeTemplatingCalls(localContent.Bytes())...)
-				localsVariableMap[variable.Key] = "local"
-				continue
+			sensitiveBlocks := makeLocal(variable, sensitive, localBody, localsContent, &hasLocals)
+			if len(sensitiveBlocks) > 0 {
+				p.localsOut = append(p.localsOut, transposeTemplatingCalls(sensitiveBlocks)...)
 			}
-			localBody.SetAttributeValue(variable.Key, hcl2shim.HCL2ValueFromConfigValue(variable.Default))
-			localsVariableMap[variable.Key] = "locals"
-			hasLocals = true
 			continue
 		}
+		varbytes := makeVariable(variable, sensitive)
+		_, out := variableTransposeTemplatingCalls(varbytes)
 		p.variablesOut = append(p.variablesOut, out...)
 	}
 
-	if hasLocals {
+	if hasLocals == true {
 		p.localsOut = append(p.localsOut, transposeTemplatingCalls(localsContent.Bytes())...)
 	}
+
 	return nil
 }
 
@@ -771,6 +826,9 @@ func (p *LocalsParser) Write(out *bytes.Buffer) {
 		}
 		fmt.Fprintln(out, `locals { timestamp = regex_replace(timestamp(), "[- TZ:]", "") }`)
 	}
+	if isotime {
+		fmt.Fprintln(out, `# The "legacy_isotime" function has been provided for backwards compatability, but we recommend switching to the timestamp and formatdate functions.`)
+	}
 	if len(p.LocalsOut) > 0 {
 		if p.WithAnnotations {
 			out.Write([]byte(localsVarHeader))
@@ -1158,6 +1216,7 @@ var PASSTHROUGHS = map[string]string{"NVME_Present": "{{ .NVME_Present }}",
 	"WinRMPassword":              "{{ .WinRMPassword }}",
 	"DefaultOrganizationID":      "{{ .DefaultOrganizationID }}",
 	"HTTPDir":                    "{{ .HTTPDir }}",
+	"HTTPContent":                "{{ .HTTPContent }}",
 	"SegmentPath":                "{{ .SegmentPath }}",
 	"NewVHDSizeBytes":            "{{ .NewVHDSizeBytes }}",
 	"CTyp":                       "{{ .CTyp }}",
@@ -1221,3 +1280,23 @@ var PASSTHROUGHS = map[string]string{"NVME_Present": "{{ .NVME_Present }}",
 	"ProviderVagrantfile":        "{{ .ProviderVagrantfile }}",
 	"Sound_Present":              "{{ .Sound_Present }}",
 }
+
+func fixQuoting(old string) string {
+	// This regex captures golang template functions that use escaped quotes:
+	// {{ env \"myvar\" }}
+	re := regexp.MustCompile(`{{\s*\w*\s*(\\".*\\")\s*}}`)
+
+	body := re.ReplaceAllFunc([]byte(old), func(s []byte) []byte {
+		// Get the capture group
+		group := re.ReplaceAllString(string(s), `$1`)
+
+		unquoted, err := strconv.Unquote(fmt.Sprintf("\"%s\"", group))
+		if err != nil {
+			return s
+		}
+		return []byte(strings.Replace(string(s), group, unquoted, 1))
+
+	})
+
+	return string(body)
+}

command/hcl2_upgrade_test.go

@@ -29,6 +29,7 @@ func Test_hcl2_upgrade(t *testing.T) {
 		{folder: "variables-only", flags: []string{}},
 		{folder: "variables-with-variables", flags: []string{}},
 		{folder: "complete-variables-with-template-engine", flags: []string{}},
+		{folder: "escaping", flags: []string{}},
 	}
 
 	for _, tc := range tc {
@@ -46,8 +47,8 @@ func Test_hcl2_upgrade(t *testing.T) {
 			if err != nil {
 				t.Fatalf("%v %s", err, bs)
 			}
-			expected := mustBytes(ioutil.ReadFile(expectedPath))
-			actual := mustBytes(ioutil.ReadFile(outputPath))
+			expected := string(mustBytes(ioutil.ReadFile(expectedPath)))
+			actual := string(mustBytes(ioutil.ReadFile(outputPath)))
 
 			if diff := cmp.Diff(expected, actual); diff != "" {
 				t.Fatalf("unexpected output: %s", diff)

command/init.go

@@ -121,13 +121,56 @@ func (c *InitCommand) RunContext(buildCtx context.Context, cla *InitArgs) int {
 			Getters:                   getters,
 		})
 		if err != nil {
-			c.Ui.Error(err.Error())
-			ret = 1
+			if pluginRequirement.Implicit {
+				msg := fmt.Sprintf(`
+Warning! At least one component used in your config file(s) has moved out of 
+Packer into the %q plugin.
+For that reason, Packer init tried to install the latest version of the %s 
+plugin. Unfortunately, this failed :
+%s`,
+					pluginRequirement.Identifier,
+					pluginRequirement.Identifier.Type,
+					err)
+				c.Ui.Say(msg)
+			} else {
+				c.Ui.Error(err.Error())
+				ret = 1
+			}
 		}
 		if newInstall != nil {
+			if pluginRequirement.Implicit {
+				msg := fmt.Sprintf("Installed implicitly required plugin %s %s in %q", pluginRequirement.Identifier, newInstall.Version, newInstall.BinaryPath)
+				ui.Say(msg)
+
+				warn := fmt.Sprintf(`
+Warning, at least one component used in your config file(s) has moved out of 
+Packer into the %[2]q plugin and is now being implicitly required. 
+For more details on implicitly required plugins see https://packer.io/docs/commands/init#implicit-required-plugin
+
+To avoid any backward incompatible changes with your
+config file you may want to lock the plugin version by pasting the following to your config:
+
+packer {
+  required_plugins {
+    %[1]s = {
+      source  = "%[2]s"
+      version = "~> %[3]s"
+    }
+  }
+}
+`,
+					pluginRequirement.Identifier.Type,
+					pluginRequirement.Identifier,
+					newInstall.Version,
+				)
+				ui.Error(warn)
+				continue
+			}
 			msg := fmt.Sprintf("Installed plugin %s %s in %q", pluginRequirement.Identifier, newInstall.Version, newInstall.BinaryPath)
 			ui.Say(msg)
+
 		}
+
 	}
 	return ret
 }

command/test-fixtures/hcl2_upgrade/complete-variables-with-template-engine/expected.pkr.hcl

@@ -5,6 +5,7 @@ variable "env_test" {
 }
 
 locals { timestamp = regex_replace(timestamp(), "[- TZ:]", "") }
+# The "legacy_isotime" function has been provided for backwards compatability, but we recommend switching to the timestamp and formatdate functions.
 
 # 5 errors occurred upgrading the following block:
 # unhandled "lower" call:
@@ -33,7 +34,7 @@ locals { timestamp = regex_replace(timestamp(), "[- TZ:]", "") }
 # Visit https://www.packer.io/docs/templates/hcl_templates/functions/string/upper for more infos.
 locals {
   build_timestamp = "${local.timestamp}"
-  iso_datetime    = "${local.timestamp}"
+  iso_datetime    = "${legacy_isotime("2006-01-02T15:04:05Z07:00")}"
   lower           = "{{ lower `HELLO` }}"
   pwd             = "${path.cwd}"
   replace         = "{{ replace `b` `c` `ababa` 2 }}"

command/test-fixtures/hcl2_upgrade/complete/expected.pkr.hcl

@@ -198,7 +198,7 @@ build {
   # Please manually upgrade to use custom validation rules, `replace(string, substring, replacement)` or `regex_replace(string, substring, replacement)`
   # Visit https://packer.io/docs/templates/hcl_templates/variables#custom-validation-rules , https://www.packer.io/docs/templates/hcl_templates/functions/string/replace or https://www.packer.io/docs/templates/hcl_templates/functions/string/regex_replace for more infos.
   provisioner "shell" {
-    inline = ["echo mybuild-{{ clean_resource_name `${local.timestamp}` }}"]
+    inline = ["echo mybuild-{{ clean_resource_name `${timestamp()}` }}"]
   }
 
 

command/test-fixtures/hcl2_upgrade/escaping/expected.pkr.hcl

@@ -0,0 +1,31 @@
+
+variable "conf" {
+  type    = string
+  default = "${env("ONE")}-${env("ANOTHER")}-${env("BACKTICKED")}"
+}
+
+variable "manyspaces" {
+  type    = string
+  default = "${env("ASDFASDF")}"
+}
+
+variable "nospaces" {
+  type    = string
+  default = "${env("SOMETHING")}"
+}
+
+locals { timestamp = regex_replace(timestamp(), "[- TZ:]", "") }
+# The "legacy_isotime" function has been provided for backwards compatability, but we recommend switching to the timestamp and formatdate functions.
+
+source "null" "autogenerated_1" {
+  communicator = "none"
+}
+
+build {
+  sources = ["source.null.autogenerated_1"]
+
+  provisioner "shell-local" {
+    inline = ["echo ${var.conf}-${local.timestamp}-${legacy_isotime("01-02-2006")}"]
+  }
+
+}

command/test-fixtures/hcl2_upgrade/escaping/input.json

@@ -0,0 +1,21 @@
+{
+	"variables": {
+		"conf": "{{ env \"ONE\" }}-{{env \"ANOTHER\"}}-{{ env  `BACKTICKED` }}",
+		"nospaces": "{{env \"SOMETHING\"}}",
+		"manyspaces": "{{   env \"ASDFASDF\"}}"
+	},
+	"builders": [
+		{
+			"type": "null",
+			"communicator": "none"
+		}
+	],
+	"provisioners": [
+		{
+			"type": "shell-local",
+			"inline": [
+				"echo {{user \"conf\"}}-{{timestamp}}-{{isotime \"01-02-2006\"}}"
+			]
+		}
+	]
+}

command/test-fixtures/hcl2_upgrade/without-annotations/expected.pkr.hcl

@@ -151,7 +151,7 @@ build {
   # Please manually upgrade to use custom validation rules, `replace(string, substring, replacement)` or `regex_replace(string, substring, replacement)`
   # Visit https://packer.io/docs/templates/hcl_templates/variables#custom-validation-rules , https://www.packer.io/docs/templates/hcl_templates/functions/string/replace or https://www.packer.io/docs/templates/hcl_templates/functions/string/regex_replace for more infos.
   provisioner "shell" {
-    inline = ["echo mybuild-{{ clean_resource_name `${local.timestamp}` }}"]
+    inline = ["echo mybuild-{{ clean_resource_name `${timestamp()}` }}"]
   }
 
 

examples/hcl/linux/source.parallels-iso.pkr.hcl

@@ -2,7 +2,7 @@
 source "parallels-iso" "base-ubuntu-amd64" {
   boot_wait              = "10s"
   guest_os_type          = "ubuntu"
-  http_directory         = local.http_directory
+  http_content           = local.http_directory_content
   parallels_tools_flavor = "lin"
   prlctl_version_file    = ".prlctl_version"
   shutdown_command       = "echo 'vagrant' | sudo -S shutdown -P now"

examples/hcl/linux/source.virtualbox-iso.pkr.hcl

@@ -1,7 +1,7 @@
 source "virtualbox-iso" "base-ubuntu-amd64" {
     headless                = var.headless
     guest_os_type           = "Ubuntu_64"
-    http_directory          = local.http_directory
+    http_content            = local.http_directory_content
     shutdown_command        = "echo 'vagrant' | sudo -S shutdown -P now"
     ssh_username            = "vagrant"
     ssh_password            = "vagrant"

examples/hcl/linux/variables.18.04.pkr.hcl

@@ -1,9 +1,9 @@
 variable "ubuntu_1804_version" {
-  default = "18.04.4"
+  default = "18.04.5"
 }
 
 locals {
-  iso_url_ubuntu_1804           = "http://cdimage.ubuntu.com/ubuntu/releases/18.04/release/ubuntu-18.04.4-server-amd64.iso"
+  iso_url_ubuntu_1804           = "http://cdimage.ubuntu.com/ubuntu/releases/18.04/release/ubuntu-${var.ubuntu_1804_version}-server-amd64.iso"
   iso_checksum_url_ubuntu_1804  = "http://cdimage.ubuntu.com/ubuntu/releases/18.04/release/SHA256SUMS"
   ubuntu_1804_boot_command      = [
     "<esc><wait>",

examples/hcl/linux/variables.common.pkr.hcl

@@ -20,4 +20,10 @@ locals {
   // value. This validates that the http directory exists even before starting
   // any builder/provisioner.
   http_directory = dirname(convert(fileset(".", "etc/http/*"), list(string))[0])
+  http_directory_content = {
+    "/alpine-answers"           = file("${local.http_directory}/alpine-answers"),
+    "/alpine-setup.sh"          = file("${local.http_directory}/alpine-setup.sh"),
+    "/preseed_hardcoded_ip.cfg" = file("${local.http_directory}/preseed_hardcoded_ip.cfg"),
+    "/preseed.cfg"              = file("${local.http_directory}/preseed.cfg"),
+  }
 }

formatted.pkr.hcl

@@ -1,39 +0,0 @@
-source "amazon-ebs" "example" {
-  communicator = "none"
-  source_ami = "potato"
-  ami_name = "potato"
-  instance_type = "potato"
-}
-
-build {
-  name = "my-provisioners-are-cooler"
-  sources = ["source.amazon-ebs.example"]
-
-  provisioner "comment-that-works" {
-
-  }
-}
-
-packer {
-  required_plugins {
-    comment = {
-      source  = "sylviamoss/comment"
-      version = "v0.2.15"
-    }
-    comment-that-works = {
-      source  = "sylviamoss/comment"
-      version = "v0.2.19"
-    }
-  }
-}
-
-build {
-  sources = ["source.amazon-ebs.example"]
-
-  provisioner "comment-my-provisioner" {
-
-  }
-  provisioner "shell-local" {
-    inline = ["yo"]
-  }
-}

go.mod

@@ -12,7 +12,7 @@ require (
 	github.com/Azure/go-autorest/autorest/to v0.3.0
 	github.com/ChrisTrenkamp/goxpath v0.0.0-20170922090931-c385f95c6022
 	github.com/NaverCloudPlatform/ncloud-sdk-go-v2 v1.1.0
-	github.com/Telmate/proxmox-api-go v0.0.0-20200715182505-ec97c70ba887
+	github.com/Telmate/proxmox-api-go v0.0.0-20210320143302-fea68269e6b0
 	github.com/aliyun/alibaba-cloud-sdk-go v0.0.0-20190418113227-25233c783f4e
 	github.com/aliyun/aliyun-oss-go-sdk v0.0.0-20170113022742-e6dbea820a9f
 	github.com/antihax/optional v1.0.0
@@ -51,7 +51,7 @@ require (
 	github.com/hashicorp/go-version v1.2.0
 	github.com/hashicorp/hcl/v2 v2.8.0
 	github.com/hashicorp/packer-plugin-docker v0.0.2
-	github.com/hashicorp/packer-plugin-sdk v0.0.14
+	github.com/hashicorp/packer-plugin-sdk v0.1.1-0.20210323111710-5a7ab7f7a1c0
 	github.com/hashicorp/vault/api v1.0.4
 	github.com/hetznercloud/hcloud-go v1.15.1
 	github.com/hyperonecom/h1-client-go v0.0.0-20191203060043-b46280e4c4a4
@@ -64,6 +64,7 @@ require (
 	github.com/mitchellh/cli v1.1.0
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/mitchellh/go-vnc v0.0.0-20150629162542-723ed9867aed
+	github.com/mitchellh/gox v1.0.1 // indirect
 	github.com/mitchellh/mapstructure v1.4.0
 	github.com/mitchellh/panicwrap v1.0.0
 	github.com/mitchellh/prefixedio v0.0.0-20151214002211-6e6954073784

go.sum

@@ -81,8 +81,9 @@ github.com/NaverCloudPlatform/ncloud-sdk-go-v2 v1.1.0 h1:0nxjOH7NurPGUWNG5BCrASW
 github.com/NaverCloudPlatform/ncloud-sdk-go-v2 v1.1.0/go.mod h1:P+3VS0ETiQPyWOx3vB/oeC8J3qd7jnVZLYAFwWgGRt8=
 github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk=
 github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
-github.com/Telmate/proxmox-api-go v0.0.0-20200715182505-ec97c70ba887 h1:Q65o4V0g/KR1sSUZIMf4m1rShb7f1tVHuEt30hfnh2A=
 github.com/Telmate/proxmox-api-go v0.0.0-20200715182505-ec97c70ba887/go.mod h1:OGWyIMJ87/k/GCz8CGiWB2HOXsOVDM6Lpe/nFPkC4IQ=
+github.com/Telmate/proxmox-api-go v0.0.0-20210320143302-fea68269e6b0 h1:LeBf+Ex12uqA6dWZp73Qf3dzpV/LvB9SRmHgPBwnXrQ=
+github.com/Telmate/proxmox-api-go v0.0.0-20210320143302-fea68269e6b0/go.mod h1:ayPkdmEKnlssqLQ9K1BE1jlsaYhXVwkoduXI30oQF0I=
 github.com/abdullin/seq v0.0.0-20160510034733-d5467c17e7af h1:DBNMBMuMiWYu0b+8KMJuWmfCkcxl09JwdlqwDZZ6U14=
 github.com/abdullin/seq v0.0.0-20160510034733-d5467c17e7af/go.mod h1:5Jv4cbFiHJMsVxt52+i0Ha45fjshj6wxYr1r19tB9bw=
 github.com/agext/levenshtein v1.2.1 h1:QmvMAjj2aEICytGiWzmxoE0x2KZvE0fvmqMOfy2tjT8=
@@ -393,6 +394,7 @@ github.com/hashicorp/packer v1.6.7-0.20210125170305-539638b0f951/go.mod h1:Z3eun
 github.com/hashicorp/packer v1.6.7-0.20210126105722-aef4ced967ec/go.mod h1:2+Vo/c/fA+TD9yFc/h9jQMFm4yG+IymQIr0OdJJOPiE=
 github.com/hashicorp/packer v1.6.7-0.20210208125835-f616955ebcb6/go.mod h1:7f5ZpTTRG53rQ58BcTADuTnpiBcB3wapuxl4sF2sGMM=
 github.com/hashicorp/packer v1.6.7-0.20210217093213-201869d627bf/go.mod h1:+EWPPcqee4h8S/y913Dnta1eJkgiqsGXBQgB75A2qV0=
+github.com/hashicorp/packer v1.7.0/go.mod h1:3KRJcwOctl2JaAGpQMI1bWQRArfWNWqcYjO6AOsVVGQ=
 github.com/hashicorp/packer-plugin-docker v0.0.2 h1:j/hQTogaN2pZfZohlZTRu5YvNZg2/qtYYHkxPBxv2Oo=
 github.com/hashicorp/packer-plugin-docker v0.0.2/go.mod h1:A2p9qztS4n88KsNF+qBM7BWw2HndW636GpFIjNSvbKM=
 github.com/hashicorp/packer-plugin-sdk v0.0.6/go.mod h1:Nvh28f+Jmpp2rcaN79bULTouNkGNDRfHckhHKTAXtyU=
@@ -402,8 +404,9 @@ github.com/hashicorp/packer-plugin-sdk v0.0.7-0.20210122130548-45a6ca0a9365/go.m
 github.com/hashicorp/packer-plugin-sdk v0.0.10-0.20210126105622-8e1648006d93/go.mod h1:AtWQLNfpn7cgH2SmZ1PTedwqNOhiPvzcuKfH5sDvIQ0=
 github.com/hashicorp/packer-plugin-sdk v0.0.11/go.mod h1:GNb0WNs7zibb8vzUZce1As64z2AW0FEMwhe2J7/NW5I=
 github.com/hashicorp/packer-plugin-sdk v0.0.12/go.mod h1:hs82OYeufirGG6KRENMpjBWomnIlte99X6wXAPThJ5I=
-github.com/hashicorp/packer-plugin-sdk v0.0.14 h1:42WOZLmIbAYYC1WXxtlrQZN+fFdysVvTmj3jtoI6gOU=
 github.com/hashicorp/packer-plugin-sdk v0.0.14/go.mod h1:tNb3XzJPnjMl3QuUdKmF47B5ImerdTakalHzUAvW0aw=
+github.com/hashicorp/packer-plugin-sdk v0.1.1-0.20210323111710-5a7ab7f7a1c0 h1:nw4RqF7C4jUWGo5PGDG4dSclU+G/vXyVBHIu5j7akd4=
+github.com/hashicorp/packer-plugin-sdk v0.1.1-0.20210323111710-5a7ab7f7a1c0/go.mod h1:1d3nqB9LUsXMQaNUiL67Q+WYEtjsVcLNTX8ikVlpBrc=
 github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
 github.com/hashicorp/serf v0.9.2 h1:yJoyfZXo4Pk2p/M/viW+YLibBFiIbKoP79gu7kDAFP0=
 github.com/hashicorp/serf v0.9.2/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk=
@@ -526,6 +529,7 @@ github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7/go.mod h1:ZX
 github.com/mitchellh/go-wordwrap v1.0.0 h1:6GlHJ/LTGMrIJbwgdqdl2eEH8o+Exx/0m8ir9Gns0u4=
 github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo=
 github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=
+github.com/mitchellh/gox v1.0.1 h1:x0jD3dcHk9a9xPSDN6YEL4xL6Qz0dvNYm8yZqui5chI=
 github.com/mitchellh/gox v1.0.1/go.mod h1:ED6BioOGXMswlXa2zxfh/xdd5QhwYliBFn9V18Ap4z4=
 github.com/mitchellh/iochan v1.0.0 h1:C+X3KsSTLFVBr/tK1eYN/vs4rJcvsiLU338UhYPJWeY=
 github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=

hcl2template/common_test.go

@@ -20,8 +20,8 @@ import (
 
 const lockedVersion = "v1.5.0"
 
-func getBasicParser() *Parser {
-	return &Parser{
+func getBasicParser(opts ...getParserOption) *Parser {
+	parser := &Parser{
 		CorePackerVersion:       version.Must(version.NewSemver(lockedVersion)),
 		CorePackerVersionString: lockedVersion,
 		Parser:                  hclparse.NewParser(),
@@ -44,8 +44,14 @@ func getBasicParser() *Parser {
 			},
 		},
 	}
+	for _, configure := range opts {
+		configure(parser)
+	}
+	return parser
 }
 
+type getParserOption func(*Parser)
+
 type parseTestArgs struct {
 	filename string
 	vars     map[string]string
@@ -338,7 +344,7 @@ var cmpOpts = []cmp.Option{
 		PackerConfig{},
 		Variable{},
 		SourceBlock{},
-		Datasource{},
+		DatasourceBlock{},
 		ProvisionerBlock{},
 		PostProcessorBlock{},
 		packer.CoreBuild{},

hcl2template/components.go

@@ -0,0 +1,11 @@
+package hcl2template
+
+// ComponentKind helps enumerate what kind of components exist in this Package.
+type ComponentKind int
+
+const (
+	Builder ComponentKind = iota
+	Provisioner
+	PostProcessor
+	Datasource
+)

hcl2template/function/datetime.go

@@ -1,12 +1,22 @@
 package function
 
 import (
+	"fmt"
 	"time"
 
 	"github.com/zclconf/go-cty/cty"
 	"github.com/zclconf/go-cty/cty/function"
 )
 
+// InitTime is the UTC time when this package was initialized. It is
+// used as the timestamp for all configuration templates so that they
+// match for a single build.
+var InitTime time.Time
+
+func init() {
+	InitTime = time.Now().UTC()
+}
+
 // TimestampFunc constructs a function that returns a string representation of the current date and time.
 var TimestampFunc = function.New(&function.Spec{
 	Params: []function.Parameter{},
@@ -24,3 +34,44 @@ var TimestampFunc = function.New(&function.Spec{
 func Timestamp() (cty.Value, error) {
 	return TimestampFunc.Call([]cty.Value{})
 }
+
+// LegacyIsotimeFunc constructs a function that returns a string representation
+// of the current date and time using golang's datetime formatting.
+var LegacyIsotimeFunc = function.New(&function.Spec{
+	Params: []function.Parameter{},
+	VarParam: &function.Parameter{
+		Name: "format",
+		Type: cty.String,
+	},
+	Type: function.StaticReturnType(cty.String),
+	Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) {
+		if len(args) > 1 {
+			return cty.StringVal(""), fmt.Errorf("too many values, 1 needed: %v", args)
+		} else if len(args) == 0 {
+			return cty.StringVal(InitTime.Format(time.RFC3339)), nil
+		}
+		format := args[0].AsString()
+		return cty.StringVal(InitTime.Format(format)), nil
+	},
+})
+
+// LegacyIsotimeFunc returns a string representation of the current date and
+// time using the given format string. The format string follows golang's
+// datetime formatting. See
+// https://www.packer.io/docs/templates/legacy_json_templates/engine#isotime-function-format-reference
+// for more details.
+//
+// This function has been provided to create backwards compatability with
+// Packer's legacy JSON templates. However, we recommend that you upgrade your
+// HCL Packer template to use Timestamp and FormatDate together as soon as is
+// convenient.
+//
+// Please note that if you are using a large number of builders, provisioners
+// or post-processors, the isotime may be slightly different for each one
+// because it is from when the plugin is launched not the initial Packer
+// process. In order to avoid this and make the timestamp consistent across all
+// plugins, set it as a user variable and then access the user variable within
+// your plugins.
+func LegacyIsotime(format cty.Value) (cty.Value, error) {
+	return LegacyIsotimeFunc.Call([]cty.Value{format})
+}

hcl2template/function/datetime_test.go

@@ -0,0 +1,64 @@
+package function
+
+import (
+	"fmt"
+	"regexp"
+	"testing"
+	"time"
+
+	"github.com/zclconf/go-cty/cty"
+)
+
+// HCL template usage example:
+//
+// locals {
+//   emptyformat  = legacy_isotime()
+//   golangformat = legacy_isotime("01-02-2006")
+// }
+
+func TestLegacyIsotime_empty(t *testing.T) {
+	got, err := LegacyIsotimeFunc.Call([]cty.Value{})
+	if err != nil {
+		t.Fatalf("unexpected error: %s", err)
+	}
+
+	_, err = time.Parse(time.RFC3339, got.AsString())
+	if err != nil {
+		t.Fatalf("Didn't get an RFC3339 string from empty case: %s", err)
+	}
+
+}
+
+func TestLegacyIsotime_inputs(t *testing.T) {
+	tests := []struct {
+		Value cty.Value
+		Want  string
+	}{
+		{
+			cty.StringVal("01-02-2006"),
+			`^\d{2}-\d{2}-\d{4}$`,
+		},
+		{
+			cty.StringVal("Mon Jan 02, 2006"),
+			`^(Mon|Tue|Wed|Thu|Fri|Sat|Sun){1} (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec){1} \d{2}, \d{4}$`,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(fmt.Sprintf("legacy_isotime(%#v)", test.Value), func(t *testing.T) {
+			got, err := LegacyIsotime(test.Value)
+
+			if err != nil {
+				t.Fatalf("unexpected error: %s", err)
+			}
+			re, err := regexp.Compile(test.Want)
+			if err != nil {
+				t.Fatalf("Bad regular expression test string: %#v", err)
+			}
+
+			if !re.MatchString(got.AsString()) {
+				t.Errorf("wrong result\ngot:  %#v\nwant: %#v", got, test.Want)
+			}
+		})
+	}
+}

hcl2template/function/templatefile.go

@@ -0,0 +1,140 @@
+package function
+
+import (
+	"fmt"
+
+	"github.com/hashicorp/go-cty-funcs/filesystem"
+	"github.com/hashicorp/hcl/v2"
+	"github.com/hashicorp/hcl/v2/hclsyntax"
+	"github.com/zclconf/go-cty/cty"
+	"github.com/zclconf/go-cty/cty/function"
+)
+
+// MakeTemplateFileFunc constructs a function that takes a file path and
+// an arbitrary object of named values and attempts to render the referenced
+// file as a template using HCL template syntax.
+//
+// The template itself may recursively call other functions so a callback
+// must be provided to get access to those functions. The template cannot,
+// however, access any variables defined in the scope: it is restricted only to
+// those variables provided in the second function argument.
+//
+// As a special exception, a referenced template file may not recursively call
+// the templatefile function, since that would risk the same file being
+// included into itself indefinitely.
+func MakeTemplateFileFunc(baseDir string, funcsCb func() map[string]function.Function) function.Function {
+
+	params := []function.Parameter{
+		{
+			Name: "path",
+			Type: cty.String,
+		},
+		{
+			Name: "vars",
+			Type: cty.DynamicPseudoType,
+		},
+	}
+
+	loadTmpl := func(fn string) (hcl.Expression, error) {
+		// We re-use File here to ensure the same filename interpretation
+		// as it does, along with its other safety checks.
+		tmplVal, err := filesystem.File(baseDir, cty.StringVal(fn))
+		if err != nil {
+			return nil, err
+		}
+
+		expr, diags := hclsyntax.ParseTemplate([]byte(tmplVal.AsString()), fn, hcl.Pos{Line: 1, Column: 1})
+		if diags.HasErrors() {
+			return nil, diags
+		}
+
+		return expr, nil
+	}
+
+	renderTmpl := func(expr hcl.Expression, varsVal cty.Value) (cty.Value, error) {
+		if varsTy := varsVal.Type(); !(varsTy.IsMapType() || varsTy.IsObjectType()) {
+			return cty.DynamicVal, function.NewArgErrorf(1, "invalid vars value: must be a map") // or an object, but we don't strongly distinguish these most of the time
+		}
+
+		ctx := &hcl.EvalContext{
+			Variables: varsVal.AsValueMap(),
+		}
+
+		// We require all of the variables to be valid HCL identifiers, because
+		// otherwise there would be no way to refer to them in the template
+		// anyway. Rejecting this here gives better feedback to the user
+		// than a syntax error somewhere in the template itself.
+		for n := range ctx.Variables {
+			if !hclsyntax.ValidIdentifier(n) {
+				// This error message intentionally doesn't describe _all_ of
+				// the different permutations that are technically valid as an
+				// HCL identifier, but rather focuses on what we might
+				// consider to be an "idiomatic" variable name.
+				return cty.DynamicVal, function.NewArgErrorf(1, "invalid template variable name %q: must start with a letter, followed by zero or more letters, digits, and underscores", n)
+			}
+		}
+
+		// We'll pre-check references in the template here so we can give a
+		// more specialized error message than HCL would by default, so it's
+		// clearer that this problem is coming from a templatefile call.
+		for _, traversal := range expr.Variables() {
+			root := traversal.RootName()
+			if _, ok := ctx.Variables[root]; !ok {
+				return cty.DynamicVal, function.NewArgErrorf(1, "vars map does not contain key %q, referenced at %s", root, traversal[0].SourceRange())
+			}
+		}
+
+		givenFuncs := funcsCb() // this callback indirection is to avoid chicken/egg problems
+		funcs := make(map[string]function.Function, len(givenFuncs))
+		for name, fn := range givenFuncs {
+			if name == "templatefile" {
+				// We stub this one out to prevent recursive calls.
+				funcs[name] = function.New(&function.Spec{
+					Params: params,
+					Type: func(args []cty.Value) (cty.Type, error) {
+						return cty.NilType, fmt.Errorf("cannot recursively call templatefile from inside templatefile call")
+					},
+				})
+				continue
+			}
+			funcs[name] = fn
+		}
+		ctx.Functions = funcs
+
+		val, diags := expr.Value(ctx)
+		if diags.HasErrors() {
+			return cty.DynamicVal, diags
+		}
+		return val, nil
+	}
+
+	return function.New(&function.Spec{
+		Params: params,
+		Type: func(args []cty.Value) (cty.Type, error) {
+			if !(args[0].IsKnown() && args[1].IsKnown()) {
+				return cty.DynamicPseudoType, nil
+			}
+
+			// We'll render our template now to see what result type it
+			// produces. A template consisting only of a single interpolation
+			// can potentially return any type.
+			expr, err := loadTmpl(args[0].AsString())
+			if err != nil {
+				return cty.DynamicPseudoType, err
+			}
+
+			// This is safe even if args[1] contains unknowns because the HCL
+			// template renderer itself knows how to short-circuit those.
+			val, err := renderTmpl(expr, args[1])
+			return val.Type(), err
+		},
+		Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) {
+			expr, err := loadTmpl(args[0].AsString())
+			if err != nil {
+				return cty.DynamicVal, err
+			}
+			return renderTmpl(expr, args[1])
+		},
+	})
+
+}

hcl2template/function/templatefile_test.go

@@ -0,0 +1,151 @@
+package function
+
+import (
+	"fmt"
+	"path/filepath"
+	"testing"
+
+	"github.com/hashicorp/go-cty-funcs/filesystem"
+	"github.com/zclconf/go-cty/cty"
+	"github.com/zclconf/go-cty/cty/function"
+	"github.com/zclconf/go-cty/cty/function/stdlib"
+)
+
+func TestTemplateFile(t *testing.T) {
+	tests := []struct {
+		Path cty.Value
+		Vars cty.Value
+		Want cty.Value
+		Err  string
+	}{
+		{
+			cty.StringVal("testdata/hello.txt"),
+			cty.EmptyObjectVal,
+			cty.StringVal("Hello World"),
+			``,
+		},
+		{
+			cty.StringVal("testdata/icon.png"),
+			cty.EmptyObjectVal,
+			cty.NilVal,
+			`contents of testdata/icon.png are not valid UTF-8; use the filebase64 function to obtain the Base64 encoded contents or the other file functions (e.g. filemd5, filesha256) to obtain file hashing results instead`,
+		},
+		{
+			cty.StringVal("testdata/missing"),
+			cty.EmptyObjectVal,
+			cty.NilVal,
+			`no file exists at ` + filepath.Clean("testdata/missing"),
+		},
+		{
+			cty.StringVal("testdata/hello.tmpl"),
+			cty.MapVal(map[string]cty.Value{
+				"name": cty.StringVal("Jodie"),
+			}),
+			cty.StringVal("Hello, Jodie!"),
+			``,
+		},
+		{
+			cty.StringVal("testdata/hello.tmpl"),
+			cty.MapVal(map[string]cty.Value{
+				"name!": cty.StringVal("Jodie"),
+			}),
+			cty.NilVal,
+			`invalid template variable name "name!": must start with a letter, followed by zero or more letters, digits, and underscores`,
+		},
+		{
+			cty.StringVal("testdata/hello.tmpl"),
+			cty.ObjectVal(map[string]cty.Value{
+				"name": cty.StringVal("Jimbo"),
+			}),
+			cty.StringVal("Hello, Jimbo!"),
+			``,
+		},
+		{
+			cty.StringVal("testdata/hello.tmpl"),
+			cty.EmptyObjectVal,
+			cty.NilVal,
+			`vars map does not contain key "name", referenced at testdata/hello.tmpl:1,10-14`,
+		},
+		{
+			cty.StringVal("testdata/func.tmpl"),
+			cty.ObjectVal(map[string]cty.Value{
+				"list": cty.ListVal([]cty.Value{
+					cty.StringVal("a"),
+					cty.StringVal("b"),
+					cty.StringVal("c"),
+				}),
+			}),
+			cty.StringVal("The items are a, b, c"),
+			``,
+		},
+		{
+			cty.StringVal("testdata/recursive.tmpl"),
+			cty.MapValEmpty(cty.String),
+			cty.NilVal,
+			`testdata/recursive.tmpl:1,3-16: Error in function call; Call to function "templatefile" failed: cannot recursively call templatefile from inside templatefile call.`,
+		},
+		{
+			cty.StringVal("testdata/list.tmpl"),
+			cty.ObjectVal(map[string]cty.Value{
+				"list": cty.ListVal([]cty.Value{
+					cty.StringVal("a"),
+					cty.StringVal("b"),
+					cty.StringVal("c"),
+				}),
+			}),
+			cty.StringVal("- a\n- b\n- c\n"),
+			``,
+		},
+		{
+			cty.StringVal("testdata/list.tmpl"),
+			cty.ObjectVal(map[string]cty.Value{
+				"list": cty.True,
+			}),
+			cty.NilVal,
+			`testdata/list.tmpl:1,13-17: Iteration over non-iterable value; A value of type bool cannot be used as the collection in a 'for' expression.`,
+		},
+		{
+			cty.StringVal("testdata/bare.tmpl"),
+			cty.ObjectVal(map[string]cty.Value{
+				"val": cty.True,
+			}),
+			cty.True, // since this template contains only an interpolation, its true value shines through
+			``,
+		},
+	}
+
+	templateFileFn := MakeTemplateFileFunc(".", func() map[string]function.Function {
+		return map[string]function.Function{
+			"join":         stdlib.JoinFunc,
+			"templatefile": filesystem.MakeFileFunc(".", false), // just a placeholder, since templatefile itself overrides this
+		}
+	})
+
+	for _, test := range tests {
+		t.Run(fmt.Sprintf("TemplateFile(%#v, %#v)", test.Path, test.Vars), func(t *testing.T) {
+			got, err := templateFileFn.Call([]cty.Value{test.Path, test.Vars})
+
+			if argErr, ok := err.(function.ArgError); ok {
+				if argErr.Index < 0 || argErr.Index > 1 {
+					t.Errorf("ArgError index %d is out of range for templatefile (must be 0 or 1)", argErr.Index)
+				}
+			}
+
+			if test.Err != "" {
+				if err == nil {
+					t.Fatal("succeeded; want error")
+				}
+				if got, want := err.Error(), test.Err; got != want {
+					t.Errorf("wrong error\ngot:  %s\nwant: %s", got, want)
+				}
+				return
+			} else if err != nil {
+				t.Fatalf("unexpected error: %s", err)
+			}
+
+			if !got.RawEquals(test.Want) {
+				t.Errorf("wrong result\ngot:  %#v\nwant: %#v", got, test.Want)
+			}
+		})
+	}
+}

hcl2template/function/testdata/bare.tmpl

@@ -0,0 +1 @@
+${val}

hcl2template/function/testdata/func.tmpl

@@ -0,0 +1 @@
+The items are ${join(", ", list)}

hcl2template/function/testdata/hello.tmpl

@@ -0,0 +1 @@
+Hello, ${name}!

hcl2template/function/testdata/hello.txt

@@ -0,0 +1 @@
+Hello World

hcl2template/function/testdata/list.tmpl

@@ -0,0 +1,3 @@
+%{ for x in list ~}
+- ${x}
+%{ endfor ~}

hcl2template/function/testdata/recursive.tmpl

@@ -0,0 +1 @@
+${templatefile("recursive.tmpl", {})}

hcl2template/functions.go

@@ -68,6 +68,7 @@ func Functions(basedir string) map[string]function.Function {
 		"jsondecode":         stdlib.JSONDecodeFunc,
 		"jsonencode":         stdlib.JSONEncodeFunc,
 		"keys":               stdlib.KeysFunc,
+		"legacy_isotime":     pkrfunction.LegacyIsotimeFunc,
 		"length":             pkrfunction.LengthFunc,
 		"log":                stdlib.LogFunc,
 		"lookup":             stdlib.LookupFunc,
@@ -117,6 +118,12 @@ func Functions(basedir string) map[string]function.Function {
 		"zipmap":             stdlib.ZipmapFunc,
 	}
 
+	funcs["templatefile"] = pkrfunction.MakeTemplateFileFunc(basedir, func() map[string]function.Function {
+		// The templatefile function prevents recursive calls to itself
+		// by copying this map and overwriting the "templatefile" entry.
+		return funcs
+	})
+
 	return funcs
 }
 

hcl2template/plugin.go

@@ -41,6 +41,7 @@ func (cfg *PackerConfig) PluginRequirements() (plugingetter.Requirements, hcl.Di
 				Accessor:           name,
 				Identifier:         block.Type,
 				VersionConstraints: block.Requirement.Required,
+				Implicit:           block.PluginDependencyReason == PluginDependencyImplicit,
 			})
 			uniq[name] = block
 		}

hcl2template/types.datasource.go

@@ -11,8 +11,8 @@ import (
 	"github.com/zclconf/go-cty/cty"
 )
 
-// DataBlock references an HCL 'data' block.
-type Datasource struct {
+// DatasourceBlock references an HCL 'data' block.
+type DatasourceBlock struct {
 	Type string
 	Name string
 
@@ -25,9 +25,9 @@ type DatasourceRef struct {
 	Name string
 }
 
-type Datasources map[DatasourceRef]Datasource
+type Datasources map[DatasourceRef]DatasourceBlock
 
-func (data *Datasource) Ref() DatasourceRef {
+func (data *DatasourceBlock) Ref() DatasourceRef {
 	return DatasourceRef{
 		Type: data.Type,
 		Name: data.Name,
@@ -124,9 +124,9 @@ func (cfg *PackerConfig) startDatasource(dataSourceStore packer.DatasourceStore,
 	return datasource, diags
 }
 
-func (p *Parser) decodeDataBlock(block *hcl.Block) (*Datasource, hcl.Diagnostics) {
+func (p *Parser) decodeDataBlock(block *hcl.Block) (*DatasourceBlock, hcl.Diagnostics) {
 	var diags hcl.Diagnostics
-	r := &Datasource{
+	r := &DatasourceBlock{
 		Type:  block.Labels[0],
 		Name:  block.Labels[1],
 		block: block,

hcl2template/types.packer_config_test.go

@@ -131,7 +131,7 @@ func TestParser_complete(t *testing.T) {
 					},
 				},
 				Datasources: Datasources{
-					DatasourceRef{Type: "amazon-ami", Name: "test"}: Datasource{
+					DatasourceRef{Type: "amazon-ami", Name: "test"}: DatasourceBlock{
 						Type:  "amazon-ami",
 						Name:  "test",
 						value: cty.StringVal("foo"),

hcl2template/types.required_plugins.go

@@ -6,6 +6,7 @@ import (
 	"github.com/hashicorp/go-version"
 	"github.com/hashicorp/hcl/v2"
 	"github.com/hashicorp/packer/hcl2template/addrs"
+	"github.com/hashicorp/packer/packer"
 	"github.com/zclconf/go-cty/cty"
 )
 
@@ -41,9 +42,13 @@ func (cfg *PackerConfig) decodeRequiredPluginsBlock(f *hcl.File) hcl.Diagnostics
 }
 
 func (cfg *PackerConfig) decodeImplicitRequiredPluginsBlocks(f *hcl.File) hcl.Diagnostics {
-	// when a plugin is used but not defined in the required plugin blocks, it
-	// is 'implicitly used'. Here we read common configuration blocks to try to
-	// guess plugins.
+	// when a plugin is used but not available it should be 'implicitly
+	// required'. Here we read common configuration blocks to try to guess
+	// plugin usages.
+
+	// decodeRequiredPluginsBlock needs to be called before
+	// decodeImplicitRequiredPluginsBlocks; otherwise all required plugins will
+	// be implicitly required too.
 
 	var diags hcl.Diagnostics
 
@@ -51,14 +56,112 @@ func (cfg *PackerConfig) decodeImplicitRequiredPluginsBlocks(f *hcl.File) hcl.Di
 	diags = append(diags, moreDiags...)
 
 	for _, block := range content.Blocks {
+
 		switch block.Type {
 		case sourceLabel:
-			// TODO
+			diags = append(diags, cfg.decodeImplicitRequiredPluginsBlock(Builder, block)...)
+		case dataSourceLabel:
+			diags = append(diags, cfg.decodeImplicitRequiredPluginsBlock(Datasource, block)...)
+		case buildLabel:
+			content, _, moreDiags := block.Body.PartialContent(buildSchema)
+			diags = append(diags, moreDiags...)
+			for _, block := range content.Blocks {
+
+				switch block.Type {
+				case buildProvisionerLabel:
+					diags = append(diags, cfg.decodeImplicitRequiredPluginsBlock(Provisioner, block)...)
+				case buildPostProcessorLabel:
+					diags = append(diags, cfg.decodeImplicitRequiredPluginsBlock(PostProcessor, block)...)
+				case buildPostProcessorsLabel:
+					content, _, moreDiags := block.Body.PartialContent(postProcessorsSchema)
+					diags = append(diags, moreDiags...)
+					for _, block := range content.Blocks {
+
+						switch block.Type {
+						case buildPostProcessorLabel:
+							diags = append(diags, cfg.decodeImplicitRequiredPluginsBlock(PostProcessor, block)...)
+						}
+					}
+				}
+			}
+
 		}
 	}
 	return diags
 }
 
+func (cfg *PackerConfig) decodeImplicitRequiredPluginsBlock(k ComponentKind, block *hcl.Block) hcl.Diagnostics {
+	if len(block.Labels) == 0 {
+		// malformed block ? Let's not panic :)
+		return nil
+	}
+	// Currently all block types are `type "component-kind" ["name"] {`
+	// this makes this simple.
+	componentName := block.Labels[0]
+
+	store := map[ComponentKind]packer.BasicStore{
+		Builder:       cfg.parser.PluginConfig.Builders,
+		PostProcessor: cfg.parser.PluginConfig.PostProcessors,
+		Provisioner:   cfg.parser.PluginConfig.Provisioners,
+		Datasource:    cfg.parser.PluginConfig.DataSources,
+	}[k]
+	if store.Has(componentName) {
+		// If any core or pre-loaded plugin defines the `happycloud-uploader`
+		// pp, skip. This happens for core and manually installed plugins, as
+		// they will be listed in the PluginConfig before parsing any HCL.
+		return nil
+	}
+
+	redirect := map[ComponentKind]map[string]string{
+		Builder:       cfg.parser.PluginConfig.BuilderRedirects,
+		PostProcessor: cfg.parser.PluginConfig.PostProcessorRedirects,
+		Provisioner:   cfg.parser.PluginConfig.ProvisionerRedirects,
+		Datasource:    cfg.parser.PluginConfig.DatasourceRedirects,
+	}[k][componentName]
+
+	if redirect == "" {
+		// no known redirect for this component
+		return nil
+	}
+
+	redirectAddr, diags := addrs.ParsePluginSourceString(redirect)
+	if diags.HasErrors() {
+		// This should never happen, since the map is manually filled.
+		return diags
+	}
+
+	for _, req := range cfg.Packer.RequiredPlugins {
+		if _, found := req.RequiredPlugins[redirectAddr.Type]; found {
+			// This could happen if a plugin was forked. For example, I forked
+			// the github.com/hashicorp/happycloud plugin into
+			// github.com/azr/happycloud that is required in my config file; and
+			// am using the `happycloud-uploader` pp component from it. In that
+			// case - and to avoid miss-requires - we won't implicitly import
+			// any other `happycloud` plugin.
+			return nil
+		}
+	}
+
+	cfg.implicitlyRequirePlugin(redirectAddr)
+	return nil
+}
+
+func (cfg *PackerConfig) implicitlyRequirePlugin(plugin *addrs.Plugin) {
+	cfg.Packer.RequiredPlugins = append(cfg.Packer.RequiredPlugins, &RequiredPlugins{
+		RequiredPlugins: map[string]*RequiredPlugin{
+			plugin.Type: {
+				Name:   plugin.Type,
+				Source: plugin.String(),
+				Type:   plugin,
+				Requirement: VersionConstraint{
+					Required: nil, // means latest
+				},
+				PluginDependencyReason: PluginDependencyImplicit,
+			},
+		},
+	})
+}
+
 // RequiredPlugin represents a declaration of a dependency on a particular
 // Plugin version or source.
 type RequiredPlugin struct {
@@ -71,8 +174,24 @@ type RequiredPlugin struct {
 	Type        *addrs.Plugin
 	Requirement VersionConstraint
 	DeclRange   hcl.Range
+	PluginDependencyReason
 }
 
+// PluginDependencyReason is an enumeration of reasons why a dependency might be
+// present.
+type PluginDependencyReason int
+
+const (
+	// PluginDependencyExplicit means that there is an explicit
+	// "required_plugin" block in the configuration.
+	PluginDependencyExplicit PluginDependencyReason = iota
+
+	// PluginDependencyImplicit means that there is no explicit
+	// "required_plugin" block but there is at least one resource that uses this
+	// plugin.
+	PluginDependencyImplicit
+)
+
 type RequiredPlugins struct {
 	RequiredPlugins map[string]*RequiredPlugin
 	DeclRange       hcl.Range

hcl2template/types.required_plugins_test.go

@@ -0,0 +1,363 @@
+package hcl2template
+
+import (
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/hashicorp/go-version"
+	"github.com/hashicorp/packer/hcl2template/addrs"
+)
+
+func TestPackerConfig_required_plugin_parse(t *testing.T) {
+
+	tests := []struct {
+		name           string
+		cfg            PackerConfig
+		requirePlugins string
+		restOfTemplate string
+		wantDiags      bool
+		wantConfig     PackerConfig
+	}{
+		{"required_plugin", PackerConfig{parser: getBasicParser()}, `
+		packer {
+			required_plugins {
+				amazon = {
+					source  = "github.com/hashicorp/amazon"
+					version = "~> v1.2.3"
+				}
+			}
+		} `, `
+		source "amazon-ebs" "example" {
+		}
+		`, false, PackerConfig{
+			Packer: struct {
+				VersionConstraints []VersionConstraint
+				RequiredPlugins    []*RequiredPlugins
+			}{
+				RequiredPlugins: []*RequiredPlugins{
+					{RequiredPlugins: map[string]*RequiredPlugin{
+						"amazon": {
+							Name:   "amazon",
+							Source: "github.com/hashicorp/amazon",
+							Type:   &addrs.Plugin{Hostname: "github.com", Namespace: "hashicorp", Type: "amazon"},
+							Requirement: VersionConstraint{
+								Required: mustVersionConstraints(version.NewConstraint("~> v1.2.3")),
+							},
+							PluginDependencyReason: PluginDependencyExplicit,
+						},
+					}},
+				},
+			},
+		}},
+		{"required_plugin_forked_no_redirect", PackerConfig{parser: getBasicParser()}, `
+		packer {
+			required_plugins {
+				amazon = {
+					source  = "github.com/azr/amazon"
+					version = "~> v1.2.3"
+				}
+			}
+		} `, `
+		source "amazon-chroot" "example" {
+		}
+		`, false, PackerConfig{
+			Packer: struct {
+				VersionConstraints []VersionConstraint
+				RequiredPlugins    []*RequiredPlugins
+			}{
+				RequiredPlugins: []*RequiredPlugins{
+					{RequiredPlugins: map[string]*RequiredPlugin{
+						"amazon": {
+							Name:   "amazon",
+							Source: "github.com/azr/amazon",
+							Type:   &addrs.Plugin{Hostname: "github.com", Namespace: "azr", Type: "amazon"},
+							Requirement: VersionConstraint{
+								Required: mustVersionConstraints(version.NewConstraint("~> v1.2.3")),
+							},
+							PluginDependencyReason: PluginDependencyExplicit,
+						},
+					}},
+				},
+			},
+		}},
+		{"required_plugin_forked", PackerConfig{
+			parser: getBasicParser(func(p *Parser) {
+				p.PluginConfig.BuilderRedirects = map[string]string{
+					"amazon-chroot": "github.com/hashicorp/amazon",
+				}
+			},
+			)}, `
+		packer {
+			required_plugins {
+				amazon = {
+					source  = "github.com/azr/amazon"
+					version = "~> v1.2.3"
+				}
+			}
+		} `, `
+		source "amazon-chroot" "example" {
+		}
+		`, false, PackerConfig{
+			Packer: struct {
+				VersionConstraints []VersionConstraint
+				RequiredPlugins    []*RequiredPlugins
+			}{
+				RequiredPlugins: []*RequiredPlugins{
+					{RequiredPlugins: map[string]*RequiredPlugin{
+						"amazon": {
+							Name:   "amazon",
+							Source: "github.com/azr/amazon",
+							Type:   &addrs.Plugin{Hostname: "github.com", Namespace: "azr", Type: "amazon"},
+							Requirement: VersionConstraint{
+								Required: mustVersionConstraints(version.NewConstraint("~> v1.2.3")),
+							},
+							PluginDependencyReason: PluginDependencyExplicit,
+						},
+					}},
+				},
+			},
+		}},
+		{"missing-required-plugin-for-pre-defined-builder", PackerConfig{
+			parser: getBasicParser(func(p *Parser) {
+				p.PluginConfig.BuilderRedirects = map[string]string{
+					"amazon-ebs": "github.com/hashicorp/amazon",
+				}
+			},
+			)},
+			`
+			packer {
+			}`, `
+			# amazon-ebs is mocked in getBasicParser()
+			source "amazon-ebs" "example" {
+			}
+			`,
+			false,
+			PackerConfig{
+				Packer: struct {
+					VersionConstraints []VersionConstraint
+					RequiredPlugins    []*RequiredPlugins
+				}{
+					RequiredPlugins: nil,
+				},
+			}},
+		{"missing-required-plugin-for-builder", PackerConfig{
+			parser: getBasicParser(func(p *Parser) {
+				p.PluginConfig.BuilderRedirects = map[string]string{
+					"amazon-chroot": "github.com/hashicorp/amazon",
+				}
+			},
+			)},
+			`
+			packer {
+			}`, `
+			source "amazon-chroot" "example" {
+			}
+			`,
+			false,
+			PackerConfig{
+				Packer: struct {
+					VersionConstraints []VersionConstraint
+					RequiredPlugins    []*RequiredPlugins
+				}{
+					RequiredPlugins: []*RequiredPlugins{
+						{RequiredPlugins: map[string]*RequiredPlugin{
+							"amazon": {
+								Name:   "amazon",
+								Source: "github.com/hashicorp/amazon",
+								Type:   &addrs.Plugin{Hostname: "github.com", Namespace: "hashicorp", Type: "amazon"},
+								Requirement: VersionConstraint{
+									Required: nil,
+								},
+								PluginDependencyReason: PluginDependencyImplicit,
+							},
+						}},
+					},
+				},
+			}},
+		{"missing-required-plugin-for-provisioner", PackerConfig{
+			parser: getBasicParser(func(p *Parser) {
+				p.PluginConfig.ProvisionerRedirects = map[string]string{
+					"ansible-local": "github.com/ansible/ansible",
+				}
+			},
+			)},
+			`
+			packer {
+			}`, `
+			build {
+				provisioner "ansible-local" {}
+			}
+			`,
+			false,
+			PackerConfig{
+				Packer: struct {
+					VersionConstraints []VersionConstraint
+					RequiredPlugins    []*RequiredPlugins
+				}{
+					RequiredPlugins: []*RequiredPlugins{
+						{RequiredPlugins: map[string]*RequiredPlugin{
+							"ansible": {
+								Name:   "ansible",
+								Source: "github.com/ansible/ansible",
+								Type:   &addrs.Plugin{Hostname: "github.com", Namespace: "ansible", Type: "ansible"},
+								Requirement: VersionConstraint{
+									Required: nil,
+								},
+								PluginDependencyReason: PluginDependencyImplicit,
+							},
+						}},
+					},
+				},
+			}},
+		{"missing-required-plugin-for-post-processor", PackerConfig{
+			parser: getBasicParser(func(p *Parser) {
+				p.PluginConfig.PostProcessorRedirects = map[string]string{
+					"docker-push": "github.com/hashicorp/docker",
+				}
+			},
+			)},
+			`
+			packer {
+			}`, `
+			build {
+				post-processor "docker-push" {}
+			}
+			`,
+			false,
+			PackerConfig{
+				Packer: struct {
+					VersionConstraints []VersionConstraint
+					RequiredPlugins    []*RequiredPlugins
+				}{
+					RequiredPlugins: []*RequiredPlugins{
+						{RequiredPlugins: map[string]*RequiredPlugin{
+							"docker": {
+								Name:   "docker",
+								Source: "github.com/hashicorp/docker",
+								Type:   &addrs.Plugin{Hostname: "github.com", Namespace: "hashicorp", Type: "docker"},
+								Requirement: VersionConstraint{
+									Required: nil,
+								},
+								PluginDependencyReason: PluginDependencyImplicit,
+							},
+						}},
+					},
+				},
+			}},
+		{"missing-required-plugin-for-nested-post-processor", PackerConfig{
+			parser: getBasicParser(func(p *Parser) {
+				p.PluginConfig.PostProcessorRedirects = map[string]string{
+					"docker-push": "github.com/hashicorp/docker",
+				}
+			},
+			)},
+			`
+			packer {
+			}`, `
+			build {
+				post-processors {
+					post-processor "docker-push" {
+					}
+				}
+			}
+			`,
+			false,
+			PackerConfig{
+				Packer: struct {
+					VersionConstraints []VersionConstraint
+					RequiredPlugins    []*RequiredPlugins
+				}{
+					RequiredPlugins: []*RequiredPlugins{
+						{RequiredPlugins: map[string]*RequiredPlugin{
+							"docker": {
+								Name:   "docker",
+								Source: "github.com/hashicorp/docker",
+								Type:   &addrs.Plugin{Hostname: "github.com", Namespace: "hashicorp", Type: "docker"},
+								Requirement: VersionConstraint{
+									Required: nil,
+								},
+								PluginDependencyReason: PluginDependencyImplicit,
+							},
+						}},
+					},
+				},
+			}},
+
+		{"required-plugin-renamed", PackerConfig{
+			parser: getBasicParser(func(p *Parser) {
+				p.PluginConfig.BuilderRedirects = map[string]string{
+					"amazon-chroot": "github.com/hashicorp/amazon",
+				}
+			},
+			)},
+			`
+			packer {
+				required_plugins {
+					amazon-v1 = {
+						source  = "github.com/hashicorp/amazon"
+						version = "~> v1.0"
+					}
+				}
+			}`, `
+			source "amazon-v1-chroot" "example" {
+			}
+			source "amazon-chroot" "example" {
+			}
+			`,
+			false,
+			PackerConfig{
+				Packer: struct {
+					VersionConstraints []VersionConstraint
+					RequiredPlugins    []*RequiredPlugins
+				}{
+					RequiredPlugins: []*RequiredPlugins{
+						{RequiredPlugins: map[string]*RequiredPlugin{
+							"amazon-v1": {
+								Name:   "amazon-v1",
+								Source: "github.com/hashicorp/amazon",
+								Type:   &addrs.Plugin{Hostname: "github.com", Namespace: "hashicorp", Type: "amazon"},
+								Requirement: VersionConstraint{
+									Required: mustVersionConstraints(version.NewConstraint("~> v1.0")),
+								},
+								PluginDependencyReason: PluginDependencyExplicit,
+							},
+						}},
+						{RequiredPlugins: map[string]*RequiredPlugin{
+							"amazon": {
+								Name:   "amazon",
+								Source: "github.com/hashicorp/amazon",
+								Type:   &addrs.Plugin{Hostname: "github.com", Namespace: "hashicorp", Type: "amazon"},
+								Requirement: VersionConstraint{
+									Required: nil,
+								},
+								PluginDependencyReason: PluginDependencyImplicit,
+							},
+						}},
+					},
+				},
+			}},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			cfg := tt.cfg
+			file, diags := cfg.parser.ParseHCL([]byte(tt.requirePlugins), "required_plugins.pkr.hcl")
+			if len(diags) > 0 {
+				t.Fatal(diags)
+			}
+			if diags := cfg.decodeRequiredPluginsBlock(file); len(diags) > 0 {
+				t.Fatal(diags)
+			}
+
+			rest, diags := cfg.parser.ParseHCL([]byte(tt.restOfTemplate), "rest.pkr.hcl")
+			if len(diags) > 0 {
+				t.Fatal(diags)
+			}
+			if gotDiags := cfg.decodeImplicitRequiredPluginsBlocks(rest); (len(gotDiags) > 0) != tt.wantDiags {
+				t.Fatal(gotDiags)
+			}
+			if diff := cmp.Diff(tt.wantConfig, cfg, cmpOpts...); diff != "" {
+				t.Errorf("PackerConfig.inferImplicitRequiredPluginFromBlocks() unexpected PackerConfig: %v", diff)
+			}
+		})
+	}
+}

main.go

@@ -152,7 +152,7 @@ func wrappedMain() int {
 	// passed into commands like `packer build`
 	config, err := loadConfig()
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "Error loading configuration: \n\n%s\n", err)
+		fmt.Fprintf(os.Stdout, "%s Error loading configuration: \n\n%s\n", ErrorPrefix, err)
 		return 1
 	}
 
@@ -166,7 +166,7 @@ func wrappedMain() int {
 
 	cacheDir, err := packersdk.CachePath()
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "Error preparing cache directory: \n\n%s\n", err)
+		fmt.Fprintf(os.Stdout, "%s Error preparing cache directory: \n\n%s\n", ErrorPrefix, err)
 		return 1
 	}
 	log.Printf("[INFO] Setting cache directory: %s", cacheDir)
@@ -187,7 +187,7 @@ func wrappedMain() int {
 		// Set this so that we don't get colored output in our machine-
 		// readable UI.
 		if err := os.Setenv("PACKER_NO_COLOR", "1"); err != nil {
-			fmt.Fprintf(os.Stderr, "Packer failed to initialize UI: %s\n", err)
+			fmt.Fprintf(os.Stdout, "%s Packer failed to initialize UI: %s\n", ErrorPrefix, err)
 			return 1
 		}
 	} else {
@@ -202,13 +202,13 @@ func wrappedMain() int {
 			currentPID := os.Getpid()
 			backgrounded, err := checkProcess(currentPID)
 			if err != nil {
-				fmt.Fprintf(os.Stderr, "cannot determine if process is in "+
-					"background: %s\n", err)
+				fmt.Fprintf(os.Stdout, "%s cannot determine if process is in "+
+					"background: %s\n", ErrorPrefix, err)
 			}
 			if backgrounded {
-				fmt.Fprint(os.Stderr, "Running in background, not using a TTY\n")
+				fmt.Fprintf(os.Stdout, "%s Running in background, not using a TTY\n", ErrorPrefix)
 			} else if TTY, err := openTTY(); err != nil {
-				fmt.Fprintf(os.Stderr, "No tty available: %s\n", err)
+				fmt.Fprintf(os.Stdout, "%s No tty available: %s\n", ErrorPrefix, err)
 			} else {
 				basicUi.TTY = TTY
 				basicUi.PB = &packer.UiProgressBar{}
@@ -246,7 +246,7 @@ func wrappedMain() int {
 	}
 
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "Error executing CLI: %s\n", err)
+		fmt.Fprintf(os.Stdout, "%s Error executing CLI: %s\n", ErrorPrefix, err)
 		return 1
 	}
 
@@ -297,6 +297,67 @@ func loadConfig() (*config, error) {
 		PluginMinPort:      10000,
 		PluginMaxPort:      25000,
 		KnownPluginFolders: packer.PluginFolders("."),
+
+		// BuilderRedirects
+		BuilderRedirects: map[string]string{
+
+			// "amazon-chroot":       "github.com/hashicorp/amazon",
+			// "amazon-ebs":          "github.com/hashicorp/amazon",
+			// "amazon-ebssurrogate": "github.com/hashicorp/amazon",
+			// "amazon-ebsvolume":    "github.com/hashicorp/amazon",
+			// "amazon-instance":     "github.com/hashicorp/amazon",
+
+			// "azure-arm":    "github.com/hashicorp/azure",
+			// "azure-chroot": "github.com/hashicorp/azure",
+			// "dtl":          "github.com/hashicorp/azure",
+
+			// "docker": "github.com/hashicorp/docker",
+
+			// "googlecompute": "github.com/hashicorp/googlecompute",
+
+			// "parallels-iso": "github.com/hashicorp/parallels",
+			// "parallels-pvm": "github.com/hashicorp/parallels",
+
+			// "qemu": "github.com/hashicorp/qemu",
+
+			// "vagrant": "github.com/hashicorp/vagrant",
+
+			// "virtualbox-iso": "github.com/hashicorp/virtualbox",
+			// "virtualbox-ovf": "github.com/hashicorp/virtualbox",
+			// "virtualbox-vm":  "github.com/hashicorp/virtualbox",
+
+			// "vmware-iso": "github.com/hashicorp/vmware",
+			// "vmware-vmx": "github.com/hashicorp/vmware",
+
+			// "vsphere-iso":   "github.com/hashicorp/vsphere",
+			// "vsphere-clone": "github.com/hashicorp/vsphere",
+		},
+		DatasourceRedirects: map[string]string{
+			// "amazon-ami": "github.com/hashicorp/amazon",
+		},
+		ProvisionerRedirects: map[string]string{
+			// "ansible":       "github.com/hashicorp/ansible",
+			// "ansible-local": "github.com/hashicorp/ansible",
+
+			// "azure-dtlartifact": "github.com/hashicorp/azure",
+		},
+		PostProcessorRedirects: map[string]string{
+			// "amazon-import": "github.com/hashicorp/amazon",
+
+			// "docker-import": "github.com/hashicorp/docker",
+			// "docker-push":   "github.com/hashicorp/docker",
+			// "docker-save":   "github.com/hashicorp/docker",
+			// "docker-tag":    "github.com/hashicorp/docker",
+
+			// "googlecompute-export": "github.com/hashicorp/googlecompute",
+			// "googlecompute-import": "github.com/hashicorp/googlecompute",
+
+			// "vagrant":       "github.com/hashicorp/vagrant",
+			// "vagrant-cloud": "github.com/hashicorp/vagrant",
+
+			// "vsphere":          "github.com/hashicorp/vsphere",
+			// "vsphere-template": "github.com/hashicorp/vsphere",
+		},
 	}
 	if err := config.Plugins.Discover(); err != nil {
 		return nil, err

packer/plugin-getter/plugins.go

@@ -38,6 +38,9 @@ type Requirement struct {
 	// VersionConstraints as defined by user. Empty ( to be avoided ) means
 	// highest found version.
 	VersionConstraints version.Constraints
+
+	// was this require implicitly guessed ?
+	Implicit bool
 }
 
 type BinaryInstallationOptions struct {

packer/plugin.go

@@ -24,6 +24,23 @@ type PluginConfig struct {
 	Provisioners       ProvisionerSet
 	PostProcessors     PostProcessorSet
 	DataSources        DatasourceSet
+
+	// Redirects are only set when a plugin was completely moved out; they allow
+	// telling where a plugin has moved by checking if a known component of this
+	// plugin is used. For example implicitly require the
+	// github.com/hashicorp/amazon plugin if it was moved out and the
+	// "amazon-ebs" plugin is used, but not found.
+	//
+	// Redirects will be bypassed if the redirected components are already found
+	// in their corresponding sets (Builders, Provisioners, PostProcessors,
+	// DataSources). That is, for example, if you manually put a single
+	// component plugin in the plugins folder.
+	//
+	// Example BuilderRedirects: "amazon-ebs" => "github.com/hashicorp/amazon"
+	BuilderRedirects       map[string]string
+	DatasourceRedirects    map[string]string
+	ProvisionerRedirects   map[string]string
+	PostProcessorRedirects map[string]string
 }
 
 // PACKERSPACE is used to represent the spaces that separate args for a command

scripts/build.sh

@@ -126,7 +126,7 @@ if [ -n "${PACKER_DEV+x}" ]; then
 fi
 
 export CGO_ENABLED=0
-set +e
+
 ${GOX:?command not found} \
     -os="${XC_OS:-$ALL_XC_OS}" \
     -arch="${XC_ARCH:-$ALL_XC_ARCH}" \
@@ -134,7 +134,6 @@ ${GOX:?command not found} \
     -ldflags "${GOLDFLAGS}" \
     -output "pkg/{{.OS}}_{{.Arch}}/packer" \
     .
-set -e
 
 # trim GOPATH to first element
 IFS="${PATHSEP}"

vendor/github.com/Telmate/proxmox-api-go/proxmox/client.go

@@ -20,9 +20,6 @@ import (
 	"time"
 )
 
-// TaskTimeout - default async task call timeout in seconds
-const TaskTimeout = 300
-
 // TaskStatusCheckInterval - time between async checks in seconds
 const TaskStatusCheckInterval = 2
 
@@ -30,11 +27,12 @@ const exitStatusSuccess = "OK"
 
 // Client - URL, user and password to specifc Proxmox node
 type Client struct {
-	session  *Session
-	ApiUrl   string
-	Username string
-	Password string
-	Otp      string
+	session     *Session
+	ApiUrl      string
+	Username    string
+	Password    string
+	Otp         string
+	TaskTimeout int
 }
 
 // VmRef - virtual machine ref parts
@@ -86,15 +84,27 @@ func NewVmRef(vmId int) (vmr *VmRef) {
 	return
 }
 
-func NewClient(apiUrl string, hclient *http.Client, tls *tls.Config) (client *Client, err error) {
+func NewClient(apiUrl string, hclient *http.Client, tls *tls.Config, taskTimeout int) (client *Client, err error) {
 	var sess *Session
 	sess, err = NewSession(apiUrl, hclient, tls)
 	if err == nil {
-		client = &Client{session: sess, ApiUrl: apiUrl}
+		client = &Client{session: sess, ApiUrl: apiUrl, TaskTimeout: taskTimeout}
 	}
 	return client, err
 }
 
+// SetAPIToken specifies a pair of user identifier and token UUID to use
+// for authenticating API calls.
+// If this is set, a ticket from calling `Login` will not be used.
+//
+// - `userID` is expected to be in the form `USER@REALM!TOKENID`
+// - `token` is just the UUID you get when initially creating the token
+//
+// See https://pve.proxmox.com/wiki/User_Management#pveum_tokens
+func (c *Client) SetAPIToken(userID, token string) {
+	c.session.SetAPIToken(userID, token)
+}
+
 func (c *Client) Login(username string, password string, otp string) (err error) {
 	c.Username = username
 	c.Password = password
@@ -214,6 +224,40 @@ func (c *Client) GetVmConfig(vmr *VmRef) (vmConfig map[string]interface{}, err e
 	return
 }
 
+func (c *Client) GetStorageStatus(vmr *VmRef, storageName string) (storageStatus map[string]interface{}, err error) {
+	err = c.CheckVmRef(vmr)
+	if err != nil {
+		return nil, err
+	}
+	var data map[string]interface{}
+	url := fmt.Sprintf("/nodes/%s/storage/%s/status", vmr.node, storageName)
+	err = c.GetJsonRetryable(url, &data, 3)
+	if err != nil {
+		return nil, err
+	}
+	if data["data"] == nil {
+		return nil, errors.New("Storage STATUS not readable")
+	}
+	storageStatus = data["data"].(map[string]interface{})
+	return
+}
+
+func (c *Client) GetStorageContent(vmr *VmRef, storageName string) (data map[string]interface{}, err error) {
+	err = c.CheckVmRef(vmr)
+	if err != nil {
+		return nil, err
+	}
+	url := fmt.Sprintf("/nodes/%s/storage/%s/content", vmr.node, storageName)
+	err = c.GetJsonRetryable(url, &data, 3)
+	if err != nil {
+		return nil, err
+	}
+	if data["data"] == nil {
+		return nil, errors.New("Storage Content not readable")
+	}
+	return
+}
+
 func (c *Client) GetVmSpiceProxy(vmr *VmRef) (vmSpiceProxy map[string]interface{}, err error) {
 	err = c.CheckVmRef(vmr)
 	if err != nil {
@@ -343,7 +387,7 @@ func (c *Client) WaitForCompletion(taskResponse map[string]interface{}) (waitExi
 	}
 	waited := 0
 	taskUpid := taskResponse["data"].(string)
-	for waited < TaskTimeout {
+	for waited < c.TaskTimeout {
 		exitStatus, statErr := c.GetTaskExitstatus(taskUpid)
 		if statErr != nil {
 			if statErr != io.ErrUnexpectedEOF { // don't give up on ErrUnexpectedEOF
@@ -421,6 +465,10 @@ func (c *Client) ResumeVm(vmr *VmRef) (exitStatus string, err error) {
 }
 
 func (c *Client) DeleteVm(vmr *VmRef) (exitStatus string, err error) {
+	return c.DeleteVmParams(vmr, nil)
+}
+
+func (c *Client) DeleteVmParams(vmr *VmRef, params map[string]interface{}) (exitStatus string, err error) {
 	err = c.CheckVmRef(vmr)
 	if err != nil {
 		return "", err
@@ -442,9 +490,10 @@ func (c *Client) DeleteVm(vmr *VmRef) (exitStatus string, err error) {
 		}
 	}
 
+	reqbody := ParamsToBody(params)
 	url := fmt.Sprintf("/nodes/%s/%s/%d", vmr.node, vmr.vmType, vmr.vmId)
 	var taskResponse map[string]interface{}
-	_, err = c.session.RequestJSON("DELETE", url, nil, nil, nil, &taskResponse)
+	_, err = c.session.RequestJSON("DELETE", url, nil, nil, &reqbody, &taskResponse)
 	exitStatus, err = c.WaitForCompletion(taskResponse)
 	return
 }
@@ -523,6 +572,61 @@ func (c *Client) CloneQemuVm(vmr *VmRef, vmParams map[string]interface{}) (exitS
 	return
 }
 
+func (c *Client) CreateQemuSnapshot(vmr *VmRef, snapshotName string) (exitStatus string, err error) {
+	err = c.CheckVmRef(vmr)
+	snapshotParams := map[string]interface{}{
+		"snapname": snapshotName,
+	}
+	reqbody := ParamsToBody(snapshotParams)
+	if err != nil {
+		return "", err
+	}
+	url := fmt.Sprintf("/nodes/%s/%s/%d/snapshot/", vmr.node, vmr.vmType, vmr.vmId)
+	resp, err := c.session.Post(url, nil, nil, &reqbody)
+	if err == nil {
+		taskResponse, err := ResponseJSON(resp)
+		if err != nil {
+			return "", err
+		}
+		exitStatus, err = c.WaitForCompletion(taskResponse)
+	}
+	return
+}
+
+func (c *Client) DeleteQemuSnapshot(vmr *VmRef, snapshotName string) (exitStatus string, err error) {
+	err = c.CheckVmRef(vmr)
+	if err != nil {
+		return "", err
+	}
+	url := fmt.Sprintf("/nodes/%s/%s/%d/snapshot/%s", vmr.node, vmr.vmType, vmr.vmId, snapshotName)
+	resp, err := c.session.Delete(url, nil, nil)
+	if err == nil {
+		taskResponse, err := ResponseJSON(resp)
+		if err != nil {
+			return "", err
+		}
+		exitStatus, err = c.WaitForCompletion(taskResponse)
+	}
+	return
+}
+
+func (c *Client) ListQemuSnapshot(vmr *VmRef) (taskResponse map[string]interface{}, exitStatus string, err error) {
+	err = c.CheckVmRef(vmr)
+	if err != nil {
+		return nil, "", err
+	}
+	url := fmt.Sprintf("/nodes/%s/%s/%d/snapshot/", vmr.node, vmr.vmType, vmr.vmId)
+	resp, err := c.session.Get(url, nil, nil)
+	if err == nil {
+		taskResponse, err := ResponseJSON(resp)
+		if err != nil {
+			return nil, "", err
+		}
+		return taskResponse, "", nil
+	}
+	return
+}
+
 func (c *Client) RollbackQemuVm(vmr *VmRef, snapshot string) (exitStatus string, err error) {
 	err = c.CheckVmRef(vmr)
 	if err != nil {
@@ -581,14 +685,24 @@ func (c *Client) MigrateNode(vmr *VmRef, newTargetNode string, online bool) (exi
 	return nil, err
 }
 
+// ResizeQemuDisk allows the caller to increase the size of a disk by the indicated number of gigabytes
 func (c *Client) ResizeQemuDisk(vmr *VmRef, disk string, moreSizeGB int) (exitStatus interface{}, err error) {
+	size := fmt.Sprintf("+%dG", moreSizeGB)
+	return c.ResizeQemuDiskRaw(vmr, disk, size)
+}
+
+// ResizeQemuDiskRaw allows the caller to provide the raw resize string to be send to proxmox.
+// See the proxmox API documentation for full information, but the short version is if you prefix
+// your desired size with a '+' character it will ADD size to the disk.  If you just specify the size by
+// itself it will do an absolute resizing to the specified size. Permitted suffixes are K, M, G, T
+// to indicate order of magnitude (kilobyte, megabyte, etc). Decrease of disk size is not permitted.
+func (c *Client) ResizeQemuDiskRaw(vmr *VmRef, disk string, size string) (exitStatus interface{}, err error) {
 	// PUT
 	//disk:virtio0
 	//size:+2G
 	if disk == "" {
 		disk = "virtio0"
 	}
-	size := fmt.Sprintf("+%dG", moreSizeGB)
 	reqbody := ParamsToBody(map[string]interface{}{"disk": disk, "size": size})
 	url := fmt.Sprintf("/nodes/%s/%s/%d/resize", vmr.node, vmr.vmType, vmr.vmId)
 	resp, err := c.session.Put(url, nil, nil, &reqbody)
@@ -602,6 +716,20 @@ func (c *Client) ResizeQemuDisk(vmr *VmRef, disk string, moreSizeGB int) (exitSt
 	return
 }
 
+func (c *Client) MoveLxcDisk(vmr *VmRef, disk string, storage string) (exitStatus interface{}, err error) {
+	reqbody := ParamsToBody(map[string]interface{}{"disk": disk, "storage": storage, "delete": true})
+	url := fmt.Sprintf("/nodes/%s/%s/%d/move_volume", vmr.node, vmr.vmType, vmr.vmId)
+	resp, err := c.session.Post(url, nil, nil, &reqbody)
+	if err == nil {
+		taskResponse, err := ResponseJSON(resp)
+		if err != nil {
+			return nil, err
+		}
+		exitStatus, err = c.WaitForCompletion(taskResponse)
+	}
+	return
+}
+
 func (c *Client) MoveQemuDisk(vmr *VmRef, disk string, storage string) (exitStatus interface{}, err error) {
 	if disk == "" {
 		disk = "virtio0"
@@ -661,7 +789,7 @@ func (c *Client) CreateVMDisk(
 			return err
 		}
 		if diskName, containsData := taskResponse["data"]; !containsData || diskName != fullDiskName {
-			return errors.New(fmt.Sprintf("Cannot create VM disk %s", fullDiskName))
+			return errors.New(fmt.Sprintf("Cannot create VM disk %s - %s", fullDiskName, diskName))
 		}
 	} else {
 		return err
@@ -680,7 +808,7 @@ func (c *Client) createVMDisks(
 	for deviceName, deviceConf := range vmParams {
 		rxStorageModels := `(ide|sata|scsi|virtio)\d+`
 		if matched, _ := regexp.MatchString(rxStorageModels, deviceName); matched {
-			deviceConfMap := ParseConf(deviceConf.(string), ",", "=")
+			deviceConfMap := ParsePMConf(deviceConf.(string), "")
 			// This if condition to differentiate between `disk` and `cdrom`.
 			if media, containsFile := deviceConfMap["media"]; containsFile && media == "disk" {
 				fullDiskName := deviceConfMap["file"].(string)
@@ -722,6 +850,135 @@ func (c *Client) DeleteVMDisks(
 	return nil
 }
 
+// VzDump - Create backup
+func (c *Client) VzDump(vmr *VmRef, params map[string]interface{}) (exitStatus interface{}, err error) {
+	err = c.CheckVmRef(vmr)
+	if err != nil {
+		return nil, err
+	}
+	reqbody := ParamsToBody(params)
+	url := fmt.Sprintf("/nodes/%s/vzdump", vmr.node)
+	resp, err := c.session.Post(url, nil, nil, &reqbody)
+	if err == nil {
+		taskResponse, err := ResponseJSON(resp)
+		if err != nil {
+			return nil, err
+		}
+		exitStatus, err = c.WaitForCompletion(taskResponse)
+	}
+	return
+}
+
+// CreateVNCProxy - Creates a TCP VNC proxy connections
+func (c *Client) CreateVNCProxy(vmr *VmRef, params map[string]interface{}) (vncProxyRes map[string]interface{}, err error) {
+	err = c.CheckVmRef(vmr)
+	if err != nil {
+		return nil, err
+	}
+	reqbody := ParamsToBody(params)
+	url := fmt.Sprintf("/nodes/%s/qemu/%d/vncproxy", vmr.node, vmr.vmId)
+	resp, err := c.session.Post(url, nil, nil, &reqbody)
+	if err != nil {
+		return nil, err
+	}
+	vncProxyRes, err = ResponseJSON(resp)
+	if err != nil {
+		return nil, err
+	}
+	if vncProxyRes["data"] == nil {
+		return nil, errors.New("VNC Proxy not readable")
+	}
+	vncProxyRes = vncProxyRes["data"].(map[string]interface{})
+	return
+}
+
+// GetExecStatus - Gets the status of the given pid started by the guest-agent
+func (c *Client) GetExecStatus(vmr *VmRef, pid string) (status map[string]interface{}, err error) {
+	err = c.CheckVmRef(vmr)
+	if err != nil {
+		return nil, err
+	}
+	err = c.GetJsonRetryable(fmt.Sprintf("/nodes/%s/%s/%d/agent/exec-status?pid=%s", vmr.node, vmr.vmType, vmr.vmId, pid), &status, 3)
+	if err == nil {
+		status = status["data"].(map[string]interface{})
+	}
+	return
+}
+
+// SetQemuFirewallOptions - Set Firewall options.
+func (c *Client) SetQemuFirewallOptions(vmr *VmRef, fwOptions map[string]interface{}) (exitStatus interface{}, err error) {
+	err = c.CheckVmRef(vmr)
+	if err != nil {
+		return nil, err
+	}
+	reqbody := ParamsToBody(fwOptions)
+	url := fmt.Sprintf("/nodes/%s/qemu/%d/firewall/options", vmr.node, vmr.vmId)
+	resp, err := c.session.Put(url, nil, nil, &reqbody)
+	if err == nil {
+		taskResponse, err := ResponseJSON(resp)
+		if err != nil {
+			return nil, err
+		}
+		exitStatus, err = c.WaitForCompletion(taskResponse)
+	}
+	return
+}
+
+// GetQemuFirewallOptions - Get VM firewall options.
+func (c *Client) GetQemuFirewallOptions(vmr *VmRef) (firewallOptions map[string]interface{}, err error) {
+	err = c.CheckVmRef(vmr)
+	if err != nil {
+		return nil, err
+	}
+	url := fmt.Sprintf("/nodes/%s/qemu/%d/firewall/options", vmr.node, vmr.vmId)
+	resp, err := c.session.Get(url, nil, nil)
+	if err == nil {
+		firewallOptions, err := ResponseJSON(resp)
+		if err != nil {
+			return nil, err
+		}
+		return firewallOptions, nil
+	}
+	return
+}
+
+// CreateQemuIPSet - Create new IPSet
+func (c *Client) CreateQemuIPSet(vmr *VmRef, params map[string]interface{}) (exitStatus interface{}, err error) {
+	err = c.CheckVmRef(vmr)
+	if err != nil {
+		return nil, err
+	}
+	reqbody := ParamsToBody(params)
+	url := fmt.Sprintf("/nodes/%s/qemu/%d/firewall/ipset", vmr.node, vmr.vmId)
+	resp, err := c.session.Post(url, nil, nil, &reqbody)
+	if err == nil {
+		taskResponse, err := ResponseJSON(resp)
+		if err != nil {
+			return nil, err
+		}
+		exitStatus, err = c.WaitForCompletion(taskResponse)
+	}
+	return
+}
+
+// GetQemuIPSet - List IPSets
+func (c *Client) GetQemuIPSet(vmr *VmRef) (ipsets map[string]interface{}, err error) {
+	err = c.CheckVmRef(vmr)
+	if err != nil {
+		return nil, err
+	}
+	url := fmt.Sprintf("/nodes/%s/qemu/%d/firewall/ipset", vmr.node, vmr.vmId)
+	resp, err := c.session.Get(url, nil, nil)
+	if err == nil {
+		ipsets, err := ResponseJSON(resp)
+		if err != nil {
+			return nil, err
+		}
+		return ipsets, nil
+	}
+	return
+}
+
 func (c *Client) Upload(node string, storage string, contentType string, filename string, file io.Reader) error {
 	var doStreamingIO bool
 	var fileSize int64

vendor/github.com/Telmate/proxmox-api-go/proxmox/config_lxc.go

@@ -4,7 +4,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"log"
 	"strconv"
 	"strings"
 )
@@ -36,7 +35,7 @@ type configLxc struct {
 	Pool               string      `json:"pool,omitempty"`
 	Protection         bool        `json:"protection"`
 	Restore            bool        `json:"restore,omitempty"`
-	RootFs             string      `json:"rootfs,omitempty"`
+	RootFs             QemuDevice  `json:"rootfs,omitempty"`
 	SearchDomain       string      `json:"searchdomain,omitempty"`
 	SSHPublicKeys      string      `json:"ssh-public-keys,omitempty"`
 	Start              bool        `json:"start"`
@@ -47,6 +46,7 @@ type configLxc struct {
 	Tty                int         `json:"tty"`
 	Unique             bool        `json:"unique,omitempty"`
 	Unprivileged       bool        `json:"unprivileged"`
+	Tags               string      `json:"tags"`
 	Unused             []string    `json:"unused,omitempty"`
 }
 
@@ -72,12 +72,7 @@ func NewConfigLxc() configLxc {
 func NewConfigLxcFromJson(io io.Reader) (config configLxc, err error) {
 	config = NewConfigLxc()
 	err = json.NewDecoder(io).Decode(config)
-	if err != nil {
-		log.Fatal(err)
-		return config, err
-	}
-	log.Println(config)
-	return
+	return config, err
 }
 
 func NewConfigLxcFromApi(vmr *VmRef, client *Client) (config *configLxc, err error) {
@@ -85,7 +80,6 @@ func NewConfigLxcFromApi(vmr *VmRef, client *Client) (config *configLxc, err err
 	var lxcConfig map[string]interface{}
 	lxcConfig, err = client.GetVmConfig(vmr)
 	if err != nil {
-		log.Fatal(err)
 		return nil, err
 	}
 
@@ -106,7 +100,7 @@ func NewConfigLxcFromApi(vmr *VmRef, client *Client) (config *configLxc, err err
 	if _, isSet := lxcConfig["console"]; isSet {
 		console = Itob(int(lxcConfig["console"].(float64)))
 	}
-	cores := 1
+	cores := 0
 	if _, isSet := lxcConfig["cores"]; isSet {
 		cores = int(lxcConfig["cores"].(float64))
 	}
@@ -158,6 +152,12 @@ func NewConfigLxcFromApi(vmr *VmRef, client *Client) (config *configLxc, err err
 		memory = int(lxcConfig["memory"].(float64))
 	}
 
+	// add rootfs
+	rootfs := QemuDevice{}
+	if rootfsStr, isSet := lxcConfig["rootfs"]; isSet {
+		rootfs = ParsePMConf(rootfsStr.(string), "volume")
+	}
+
 	// add mountpoints
 	mpNames := []string{}
 
@@ -168,17 +168,14 @@ func NewConfigLxcFromApi(vmr *VmRef, client *Client) (config *configLxc, err err
 	}
 
 	for _, mpName := range mpNames {
-		mpConfStr := lxcConfig[mpName]
-		mpConfList := strings.Split(mpConfStr.(string), ",")
+		mpConfStr := lxcConfig[mpName].(string)
+		mpConfMap := ParseLxcDisk(mpConfStr)
 
+		// add mp id
 		id := rxDeviceID.FindStringSubmatch(mpName)
 		mpID, _ := strconv.Atoi(id[0])
-		// add mp id
-		mpConfMap := QemuDevice{
-			"id": mpID,
-		}
-		// add rest of device config
-		mpConfMap.readDeviceConfig(mpConfList)
+		mpConfMap["slot"] = mpID
+
 		// prepare empty mountpoint map
 		if config.Mountpoints == nil {
 			config.Mountpoints = QemuDevices{}
@@ -215,6 +212,13 @@ func NewConfigLxcFromApi(vmr *VmRef, client *Client) (config *configLxc, err err
 		}
 		// add rest of device config
 		nicConfMap.readDeviceConfig(nicConfList)
+
+		if nicConfMap["firewall"] == 1 {
+			nicConfMap["firewall"] = true
+		} else if nicConfMap["firewall"] == 0 {
+			nicConfMap["firewall"] = false
+		}
+
 		// prepare empty network map
 		if config.Networks == nil {
 			config.Networks = QemuDevices{}
@@ -237,10 +241,6 @@ func NewConfigLxcFromApi(vmr *VmRef, client *Client) (config *configLxc, err err
 	if _, isSet := lxcConfig["protection"]; isSet {
 		protection = Itob(int(lxcConfig["protection"].(float64)))
 	}
-	rootfs := ""
-	if _, isSet := lxcConfig["rootfs"]; isSet {
-		rootfs = lxcConfig["rootfs"].(string)
-	}
 	searchdomain := ""
 	if _, isSet := lxcConfig["searchdomain"]; isSet {
 		searchdomain = lxcConfig["searchdomain"].(string)
@@ -265,6 +265,10 @@ func NewConfigLxcFromApi(vmr *VmRef, client *Client) (config *configLxc, err err
 	if _, isset := lxcConfig["unprivileged"]; isset {
 		unprivileged = Itob(int(lxcConfig["unprivileged"].(float64)))
 	}
+	tags := ""
+	if _, isSet := lxcConfig["tags"]; isSet {
+		tags = lxcConfig["tags"].(string)
+	}
 	var unused []string
 	if _, isset := lxcConfig["unused"]; isset {
 		unused = lxcConfig["unused"].([]string)
@@ -294,6 +298,7 @@ func NewConfigLxcFromApi(vmr *VmRef, client *Client) (config *configLxc, err err
 	config.Tty = tty
 	config.Unprivileged = unprivileged
 	config.Unused = unused
+	config.Tags = tags
 
 	return
 }
@@ -301,123 +306,21 @@ func NewConfigLxcFromApi(vmr *VmRef, client *Client) (config *configLxc, err err
 // create LXC container using the Proxmox API
 func (config configLxc) CreateLxc(vmr *VmRef, client *Client) (err error) {
 	vmr.SetVmType("lxc")
-
-	// convert config to map
-	params, _ := json.Marshal(&config)
-	var paramMap map[string]interface{}
-	json.Unmarshal(params, &paramMap)
-
-	// build list of features
-	// add features as parameter list to lxc parameters
-	// this overwrites the orginal formatting with a
-	// comma separated list of "key=value" pairs
-	featuresParam := QemuDeviceParam{}
-	featuresParam = featuresParam.createDeviceParam(config.Features, nil)
-	if len(featuresParam) > 0 {
-		paramMap["features"] = strings.Join(featuresParam, ",")
-	}
-
-	// build list of mountpoints
-	// this does the same as for the feature list
-	// except that there can be multiple of these mountpoint sets
-	// and each mountpoint set comes with a new id
-	for mpID, mpConfMap := range config.Mountpoints {
-		mpConfParam := QemuDeviceParam{}
-		mpConfParam = mpConfParam.createDeviceParam(mpConfMap, nil)
-
-		// add mp to lxc parameters
-		mpName := fmt.Sprintf("mp%v", mpID)
-		paramMap[mpName] = strings.Join(mpConfParam, ",")
-	}
-
-	// build list of network parameters
-	for nicID, nicConfMap := range config.Networks {
-		nicConfParam := QemuDeviceParam{}
-		nicConfParam = nicConfParam.createDeviceParam(nicConfMap, nil)
-
-		// add nic to lxc parameters
-		nicName := fmt.Sprintf("net%v", nicID)
-		paramMap[nicName] = strings.Join(nicConfParam, ",")
-	}
-
-	// build list of unused volumes for sake of completenes,
-	// even if it is not recommended to change these volumes manually
-	for volID, vol := range config.Unused {
-		// add volume to lxc parameters
-		volName := fmt.Sprintf("unused%v", volID)
-		paramMap[volName] = vol
-	}
-
-	// now that we concatenated the key value parameter
-	// list for the networks, mountpoints and unused volumes,
-	// remove the original keys, since the Proxmox API does
-	// not know how to handle this key
-	delete(paramMap, "networks")
-	delete(paramMap, "mountpoints")
-	delete(paramMap, "unused")
+	paramMap := config.mapToAPIParams()
 
 	// amend vmid
 	paramMap["vmid"] = vmr.vmId
 
 	exitStatus, err := client.CreateLxcContainer(vmr.node, paramMap)
 	if err != nil {
-		return fmt.Errorf("Error creating LXC container: %v, error status: %s (params: %v)", err, exitStatus, params)
+		params, _ := json.Marshal(&paramMap)
+		return fmt.Errorf("Error creating LXC container: %v, error status: %s (params: %v)", err, exitStatus, string(params))
 	}
 	return
 }
 
 func (config configLxc) UpdateConfig(vmr *VmRef, client *Client) (err error) {
-	// convert config to map
-	params, _ := json.Marshal(&config)
-	var paramMap map[string]interface{}
-	json.Unmarshal(params, &paramMap)
-
-	// build list of features
-	// add features as parameter list to lxc parameters
-	// this overwrites the orginal formatting with a
-	// comma separated list of "key=value" pairs
-	featuresParam := QemuDeviceParam{}
-	featuresParam = featuresParam.createDeviceParam(config.Features, nil)
-	paramMap["features"] = strings.Join(featuresParam, ",")
-
-	// build list of mountpoints
-	// this does the same as for the feature list
-	// except that there can be multiple of these mountpoint sets
-	// and each mountpoint set comes with a new id
-	for mpID, mpConfMap := range config.Mountpoints {
-		mpConfParam := QemuDeviceParam{}
-		mpConfParam = mpConfParam.createDeviceParam(mpConfMap, nil)
-
-		// add mp to lxc parameters
-		mpName := fmt.Sprintf("mp%v", mpID)
-		paramMap[mpName] = strings.Join(mpConfParam, ",")
-	}
-
-	// build list of network parameters
-	for nicID, nicConfMap := range config.Networks {
-		nicConfParam := QemuDeviceParam{}
-		nicConfParam = nicConfParam.createDeviceParam(nicConfMap, nil)
-
-		// add nic to lxc parameters
-		nicName := fmt.Sprintf("net%v", nicID)
-		paramMap[nicName] = strings.Join(nicConfParam, ",")
-	}
-
-	// build list of unused volumes for sake of completenes,
-	// even if it is not recommended to change these volumes manually
-	for volID, vol := range config.Unused {
-		// add volume to lxc parameters
-		volName := fmt.Sprintf("unused%v", volID)
-		paramMap[volName] = vol
-	}
-
-	// now that we concatenated the key value parameter
-	// list for the networks, mountpoints and unused volumes,
-	// remove the original keys, since the Proxmox API does
-	// not know how to handle this key
-	delete(paramMap, "networks")
-	delete(paramMap, "mountpoints")
-	delete(paramMap, "unused")
+	paramMap := config.mapToAPIParams()
 
 	// delete parameters wich are not supported in updated operations
 	delete(paramMap, "pool")
@@ -425,6 +328,7 @@ func (config configLxc) UpdateConfig(vmr *VmRef, client *Client) (err error) {
 	delete(paramMap, "password")
 	delete(paramMap, "ostemplate")
 	delete(paramMap, "start")
+
 	// even though it is listed as a PUT option in the API documentation
 	// we remove it here because "it should not be modified manually";
 	// also, error "500 unable to modify read-only option: 'unprivileged'"
@@ -433,3 +337,77 @@ func (config configLxc) UpdateConfig(vmr *VmRef, client *Client) (err error) {
 	_, err = client.SetLxcConfig(vmr, paramMap)
 	return err
 }
+
+func ParseLxcDisk(diskStr string) QemuDevice {
+	disk := ParsePMConf(diskStr, "volume")
+
+	// add features, if any
+	if mountoptions, isSet := disk["mountoptions"]; isSet {
+		moList := strings.Split(mountoptions.(string), ";")
+		moMap := map[string]bool{}
+		for _, mo := range moList {
+			moMap[mo] = true
+		}
+		disk["mountoptions"] = moMap
+	}
+
+	storageName, fileName := ParseSubConf(disk["volume"].(string), ":")
+	disk["storage"] = storageName
+	disk["file"] = fileName
+
+	return disk
+}
+
+func (config configLxc) mapToAPIParams() map[string]interface{} {
+	// convert config to map
+	params, _ := json.Marshal(&config)
+	var paramMap map[string]interface{}
+	json.Unmarshal(params, &paramMap)
+
+	// build list of features
+	// add features as parameter list to lxc parameters
+	// this overwrites the orginal formatting with a
+	// comma separated list of "key=value" pairs
+	paramMap["features"] = formatDeviceParam(config.Features)
+
+	// format rootfs params as expected
+	if rootfs := config.RootFs; rootfs != nil {
+		paramMap["rootfs"] = FormatDiskParam(rootfs)
+	}
+
+	// build list of mountpoints
+	// this does the same as for the feature list
+	// except that there can be multiple of these mountpoint sets
+	// and each mountpoint set comes with a new id
+	for _, mpConfMap := range config.Mountpoints {
+		// add mp to lxc parameters
+		mpID := mpConfMap["slot"]
+		mpName := fmt.Sprintf("mp%v", mpID)
+		paramMap[mpName] = FormatDiskParam(mpConfMap)
+	}
+
+	// build list of network parameters
+	for nicID, nicConfMap := range config.Networks {
+		// add nic to lxc parameters
+		nicName := fmt.Sprintf("net%v", nicID)
+		paramMap[nicName] = formatDeviceParam(nicConfMap)
+	}
+
+	// build list of unused volumes for sake of completeness,
+	// even if it is not recommended to change these volumes manually
+	for volID, vol := range config.Unused {
+		// add volume to lxc parameters
+		volName := fmt.Sprintf("unused%v", volID)
+		paramMap[volName] = vol
+	}
+
+	// now that we concatenated the key value parameter
+	// list for the networks, mountpoints and unused volumes,
+	// remove the original keys, since the Proxmox API does
+	// not know how to handle this key
+	delete(paramMap, "networks")
+	delete(paramMap, "mountpoints")
+	delete(paramMap, "unused")
+
+	return paramMap
+}

vendor/github.com/Telmate/proxmox-api-go/proxmox/config_qemu.go

@@ -15,6 +15,10 @@ import (
 	"time"
 )
 
+// Currently ZFS local, LVM, Ceph RBD, CephFS, Directory and virtio-scsi-pci are considered.
+// Other formats are not verified, but could be added if they're needed.
+const rxStorageTypes = `(zfspool|lvm|rbd|cephfs|dir|virtio-scsi-pci)`
+
 type (
 	QemuDevices     map[int]map[string]interface{}
 	QemuDevice      map[string]interface{}
@@ -23,33 +27,35 @@ type (
 
 // ConfigQemu - Proxmox API QEMU options
 type ConfigQemu struct {
-	VmID         int         `json:"vmid"`
-	Name         string      `json:"name"`
-	Description  string      `json:"desc"`
-	Pool         string      `json:"pool,omitempty"`
-	Bios         string      `json:"bios"`
-	Onboot       bool        `json:"onboot"`
-	Agent        int         `json:"agent"`
-	Memory       int         `json:"memory"`
-	Balloon      int         `json:"balloon"`
-	QemuOs       string      `json:"os"`
-	QemuCores    int         `json:"cores"`
-	QemuSockets  int         `json:"sockets"`
-	QemuVcpus    int         `json:"vcpus"`
-	QemuCpu      string      `json:"cpu"`
-	QemuNuma     bool        `json:"numa"`
-	QemuKVM      bool        `json:"kvm"`
-	Hotplug      string      `json:"hotplug"`
-	QemuIso      string      `json:"iso"`
-	FullClone    *int        `json:"fullclone"`
-	Boot         string      `json:"boot"`
-	BootDisk     string      `json:"bootdisk,omitempty"`
-	Scsihw       string      `json:"scsihw,omitempty"`
-	QemuDisks    QemuDevices `json:"disk"`
-	QemuVga      QemuDevice  `json:"vga,omitempty"`
-	QemuNetworks QemuDevices `json:"network"`
-	QemuSerials  QemuDevices `json:"serial,omitempty"`
-	HaState      string      `json:"hastate,omitempty"`
+	VmID            int         `json:"vmid"`
+	Name            string      `json:"name"`
+	Description     string      `json:"desc"`
+	Pool            string      `json:"pool,omitempty"`
+	Bios            string      `json:"bios"`
+	Onboot          bool        `json:"onboot"`
+	Agent           int         `json:"agent"`
+	Memory          int         `json:"memory"`
+	Balloon         int         `json:"balloon"`
+	QemuOs          string      `json:"os"`
+	QemuCores       int         `json:"cores"`
+	QemuSockets     int         `json:"sockets"`
+	QemuVcpus       int         `json:"vcpus"`
+	QemuCpu         string      `json:"cpu"`
+	QemuNuma        bool        `json:"numa"`
+	QemuKVM         bool        `json:"kvm"`
+	Hotplug         string      `json:"hotplug"`
+	QemuIso         string      `json:"iso"`
+	FullClone       *int        `json:"fullclone"`
+	Boot            string      `json:"boot"`
+	BootDisk        string      `json:"bootdisk,omitempty"`
+	Scsihw          string      `json:"scsihw,omitempty"`
+	QemuDisks       QemuDevices `json:"disk"`
+	QemuUnusedDisks QemuDevices `json:"unused_disk"`
+	QemuVga         QemuDevice  `json:"vga,omitempty"`
+	QemuNetworks    QemuDevices `json:"network"`
+	QemuSerials     QemuDevices `json:"serial,omitempty"`
+	HaState         string      `json:"hastate,omitempty"`
+	Tags            string      `json:"tags"`
 
 	// Deprecated single disk.
 	DiskSize    float64 `json:"diskGB"`
@@ -100,6 +106,7 @@ func (config ConfigQemu) CreateVm(vmr *VmRef, client *Client) (err error) {
 		"memory":      config.Memory,
 		"boot":        config.Boot,
 		"description": config.Description,
+		"tags":        config.Tags,
 	}
 
 	if config.Bios != "" {
@@ -127,7 +134,10 @@ func (config ConfigQemu) CreateVm(vmr *VmRef, client *Client) (err error) {
 	}
 
 	// Create disks config.
-	config.CreateQemuDisksParams(vmr.vmId, params, false)
+	err = config.CreateQemuDisksParams(vmr.vmId, params, false)
+	if err != nil {
+		log.Printf("[ERROR] %q", err)
+	}
 
 	// Create vga config.
 	vgaParam := QemuDeviceParam{}
@@ -137,17 +147,26 @@ func (config ConfigQemu) CreateVm(vmr *VmRef, client *Client) (err error) {
 	}
 
 	// Create networks config.
-	config.CreateQemuNetworksParams(vmr.vmId, params)
+	err = config.CreateQemuNetworksParams(vmr.vmId, params)
+	if err != nil {
+		log.Printf("[ERROR] %q", err)
+	}
 
 	// Create serial interfaces
-	config.CreateQemuSerialsParams(vmr.vmId, params)
+	err = config.CreateQemuSerialsParams(vmr.vmId, params)
+	if err != nil {
+		log.Printf("[ERROR] %q", err)
+	}
 
 	exitStatus, err := client.CreateQemuVm(vmr.node, params)
 	if err != nil {
 		return fmt.Errorf("Error creating VM: %v, error status: %s (params: %v)", err, exitStatus, params)
 	}
 
-	client.UpdateVMHA(vmr, config.HaState)
+	_, err = client.UpdateVMHA(vmr, config.HaState)
+	if err != nil {
+		log.Printf("[ERROR] %q", err)
+	}
 
 	return
 }
@@ -210,6 +229,7 @@ func (config ConfigQemu) UpdateConfig(vmr *VmRef, client *Client) (err error) {
 	configParams := map[string]interface{}{
 		"name":        config.Name,
 		"description": config.Description,
+		"tags":        config.Tags,
 		"onboot":      config.Onboot,
 		"agent":       config.Agent,
 		"sockets":     config.QemuSockets,
@@ -253,8 +273,16 @@ func (config ConfigQemu) UpdateConfig(vmr *VmRef, client *Client) (err error) {
 	configParamsDisk := map[string]interface{}{
 		"vmid": vmr.vmId,
 	}
-	config.CreateQemuDisksParams(vmr.vmId, configParamsDisk, false)
-	client.createVMDisks(vmr.node, configParamsDisk)
+	// TODO keep going if error=
+	err = config.CreateQemuDisksParams(vmr.vmId, configParamsDisk, false)
+	if err != nil {
+		log.Printf("[ERROR] %q", err)
+	}
+	// TODO keep going if error=
+	_, err = client.createVMDisks(vmr.node, configParamsDisk)
+	if err != nil {
+		log.Printf("[ERROR] %q", err)
+	}
 	//Copy the disks to the global configParams
 	for key, value := range configParamsDisk {
 		//vmid is only required in createVMDisks
@@ -264,7 +292,10 @@ func (config ConfigQemu) UpdateConfig(vmr *VmRef, client *Client) (err error) {
 	}
 
 	// Create networks config.
-	config.CreateQemuNetworksParams(vmr.vmId, configParams)
+	err = config.CreateQemuNetworksParams(vmr.vmId, configParams)
+	if err != nil {
+		log.Printf("[ERROR] %q", err)
+	}
 
 	// Create vga config.
 	vgaParam := QemuDeviceParam{}
@@ -276,7 +307,10 @@ func (config ConfigQemu) UpdateConfig(vmr *VmRef, client *Client) (err error) {
 	}
 
 	// Create serial interfaces
-	config.CreateQemuSerialsParams(vmr.vmId, configParams)
+	err = config.CreateQemuSerialsParams(vmr.vmId, configParams)
+	if err != nil {
+		log.Printf("[ERROR] %q", err)
+	}
 
 	// cloud-init options
 	if config.CIuser != "" {
@@ -321,7 +355,10 @@ func (config ConfigQemu) UpdateConfig(vmr *VmRef, client *Client) (err error) {
 		return err
 	}
 
-	client.UpdateVMHA(vmr, config.HaState)
+	_, err = client.UpdateVMHA(vmr, config.HaState)
+	if err != nil {
+		log.Printf("[ERROR] %q", err)
+	}
 
 	_, err = client.UpdateVMPool(vmr, config.Pool)
 
@@ -340,13 +377,14 @@ func NewConfigQemuFromJson(io io.Reader) (config *ConfigQemu, err error) {
 }
 
 var (
-	rxIso        = regexp.MustCompile(`(.*?),media`)
-	rxDeviceID   = regexp.MustCompile(`\d+`)
-	rxDiskName   = regexp.MustCompile(`(virtio|scsi)\d+`)
-	rxDiskType   = regexp.MustCompile(`\D+`)
-	rxNicName    = regexp.MustCompile(`net\d+`)
-	rxMpName     = regexp.MustCompile(`mp\d+`)
-	rxSerialName = regexp.MustCompile(`serial\d+`)
+	rxIso            = regexp.MustCompile(`(.*?),media`)
+	rxDeviceID       = regexp.MustCompile(`\d+`)
+	rxDiskName       = regexp.MustCompile(`(virtio|scsi)\d+`)
+	rxDiskType       = regexp.MustCompile(`\D+`)
+	rxUnusedDiskName = regexp.MustCompile(`^(unused)\d+`)
+	rxNicName        = regexp.MustCompile(`net\d+`)
+	rxMpName         = regexp.MustCompile(`mp\d+`)
+	rxSerialName     = regexp.MustCompile(`serial\d+`)
 )
 
 func NewConfigQemuFromApi(vmr *VmRef, client *Client) (config *ConfigQemu, err error) {
@@ -387,6 +425,10 @@ func NewConfigQemuFromApi(vmr *VmRef, client *Client) (config *ConfigQemu, err e
 	if _, isSet := vmConfig["description"]; isSet {
 		description = vmConfig["description"].(string)
 	}
+	tags := ""
+	if _, isSet := vmConfig["tags"]; isSet {
+		tags = vmConfig["tags"].(string)
+	}
 	bios := "seabios"
 	if _, isSet := vmConfig["bios"]; isSet {
 		bios = vmConfig["bios"].(string)
@@ -466,28 +508,30 @@ func NewConfigQemuFromApi(vmr *VmRef, client *Client) (config *ConfigQemu, err e
 	}
 
 	config = &ConfigQemu{
-		Name:         name,
-		Description:  strings.TrimSpace(description),
-		Bios:         bios,
-		Onboot:       onboot,
-		Agent:        agent,
-		QemuOs:       ostype,
-		Memory:       int(memory),
-		QemuCores:    int(cores),
-		QemuSockets:  int(sockets),
-		QemuCpu:      cpu,
-		QemuNuma:     numa,
-		QemuKVM:      kvm,
-		Hotplug:      hotplug,
-		QemuVlanTag:  -1,
-		Boot:         boot,
-		BootDisk:     bootdisk,
-		Scsihw:       scsihw,
-		HaState:      hastate,
-		QemuDisks:    QemuDevices{},
-		QemuVga:      QemuDevice{},
-		QemuNetworks: QemuDevices{},
-		QemuSerials:  QemuDevices{},
+		Name:            name,
+		Description:     strings.TrimSpace(description),
+		Tags:            strings.TrimSpace(tags),
+		Bios:            bios,
+		Onboot:          onboot,
+		Agent:           agent,
+		QemuOs:          ostype,
+		Memory:          int(memory),
+		QemuCores:       int(cores),
+		QemuSockets:     int(sockets),
+		QemuCpu:         cpu,
+		QemuNuma:        numa,
+		QemuKVM:         kvm,
+		Hotplug:         hotplug,
+		QemuVlanTag:     -1,
+		Boot:            boot,
+		BootDisk:        bootdisk,
+		Scsihw:          scsihw,
+		HaState:         hastate,
+		QemuDisks:       QemuDevices{},
+		QemuUnusedDisks: QemuDevices{},
+		QemuVga:         QemuDevice{},
+		QemuNetworks:    QemuDevices{},
+		QemuSerials:     QemuDevices{},
 	}
 
 	if balloon >= 1 {
@@ -533,32 +577,66 @@ func NewConfigQemuFromApi(vmr *VmRef, client *Client) (config *ConfigQemu, err e
 	// Add disks.
 	diskNames := []string{}
 
-	for k, _ := range vmConfig {
+	for k := range vmConfig {
 		if diskName := rxDiskName.FindStringSubmatch(k); len(diskName) > 0 {
 			diskNames = append(diskNames, diskName[0])
 		}
 	}
 
 	for _, diskName := range diskNames {
-		diskConfStr := vmConfig[diskName]
-		diskConfList := strings.Split(diskConfStr.(string), ",")
+		diskConfStr := vmConfig[diskName].(string)
 
-		//
 		id := rxDeviceID.FindStringSubmatch(diskName)
 		diskID, _ := strconv.Atoi(id[0])
 		diskType := rxDiskType.FindStringSubmatch(diskName)[0]
-		storageName, fileName := ParseSubConf(diskConfList[0], ":")
 
-		//
-		diskConfMap := QemuDevice{
-			"id":      diskID,
-			"type":    diskType,
-			"storage": storageName,
-			"file":    fileName,
+		diskConfMap := ParsePMConf(diskConfStr, "volume")
+		diskConfMap["slot"] = diskID
+		diskConfMap["type"] = diskType
+
+		storageName, fileName := ParseSubConf(diskConfMap["volume"].(string), ":")
+		diskConfMap["storage"] = storageName
+		diskConfMap["file"] = fileName
+
+		filePath := diskConfMap["volume"]
+
+		// Get disk format
+		storageContent, err := client.GetStorageContent(vmr, storageName)
+		if err != nil {
+			log.Fatal(err)
+			return nil, err
 		}
+		var storageFormat string
+		contents := storageContent["data"].([]interface{})
+		for content := range contents {
+			storageContentMap := contents[content].(map[string]interface{})
+			if storageContentMap["volid"] == filePath {
+				storageFormat = storageContentMap["format"].(string)
+				break
+			}
+		}
+		diskConfMap["format"] = storageFormat
 
-		// Add rest of device config.
-		diskConfMap.readDeviceConfig(diskConfList[1:])
+		// Get storage type for disk
+		var storageStatus map[string]interface{}
+		storageStatus, err = client.GetStorageStatus(vmr, storageName)
+		if err != nil {
+			log.Fatal(err)
+			return nil, err
+		}
+		storageType := storageStatus["type"]
+
+		diskConfMap["storage_type"] = storageType
+
+		// Convert to gigabytes if disk size was received in terabytes
+		sizeIsInTerabytes, err := regexp.MatchString("[0-9]+T", diskConfMap["size"].(string))
+		if err != nil {
+			log.Fatal(err)
+			return nil, err
+		}
+		if sizeIsInTerabytes {
+			diskConfMap["size"] = fmt.Sprintf("%.0fG", DiskSizeGB(diskConfMap["size"]))
+		}
 
 		// And device config to disks map.
 		if len(diskConfMap) > 0 {
@@ -566,11 +644,49 @@ func NewConfigQemuFromApi(vmr *VmRef, client *Client) (config *ConfigQemu, err e
 		}
 	}
 
+	// Add unused disks
+	// unused0:local:100/vm-100-disk-1.qcow2
+	unusedDiskNames := []string{}
+	for k := range vmConfig {
+		// look for entries from the config in the format "unusedX:<storagepath>" where X is an integer
+		if unusedDiskName := rxUnusedDiskName.FindStringSubmatch(k); len(unusedDiskName) > 0 {
+			unusedDiskNames = append(unusedDiskNames, unusedDiskName[0])
+		}
+	}
+
+	fmt.Println(fmt.Sprintf("unusedDiskNames: %v", unusedDiskNames))
+	for _, unusedDiskName := range unusedDiskNames {
+		unusedDiskConfStr := vmConfig[unusedDiskName].(string)
+		finalDiskConfMap := QemuDevice{}
+
+		// parse "unused0" to get the id '0' as an int
+		id := rxDeviceID.FindStringSubmatch(unusedDiskName)
+		diskID, err := strconv.Atoi(id[0])
+		if err != nil {
+			return nil, errors.New(fmt.Sprintf("Unable to parse unused disk id from input string '%v' tried to convert '%v' to integer.", unusedDiskName, diskID))
+		}
+		finalDiskConfMap["slot"] = diskID
+
+		// parse the attributes from the unused disk
+		// extract the storage and file path from the unused disk entry
+		parsedUnusedDiskMap := ParsePMConf(unusedDiskConfStr, "storage+file")
+		storageName, fileName := ParseSubConf(parsedUnusedDiskMap["storage+file"].(string), ":")
+		finalDiskConfMap["storage"] = storageName
+		finalDiskConfMap["file"] = fileName
+
+		config.QemuUnusedDisks[diskID] = finalDiskConfMap
+	}
+
 	//Display
 	if vga, isSet := vmConfig["vga"]; isSet {
 		vgaList := strings.Split(vga.(string), ",")
 		vgaMap := QemuDevice{}
-		vgaMap.readDeviceConfig(vgaList)
+
+		// TODO: keep going if error?
+		err = vgaMap.readDeviceConfig(vgaList)
+		if err != nil {
+			log.Printf("[ERROR] %q", err)
+		}
 		if len(vgaMap) > 0 {
 			config.QemuVga = vgaMap
 		}
@@ -579,7 +695,7 @@ func NewConfigQemuFromApi(vmr *VmRef, client *Client) (config *ConfigQemu, err e
 	// Add networks.
 	nicNames := []string{}
 
-	for k, _ := range vmConfig {
+	for k := range vmConfig {
 		if nicName := rxNicName.FindStringSubmatch(k); len(nicName) > 0 {
 			nicNames = append(nicNames, nicName[0])
 		}
@@ -601,7 +717,15 @@ func NewConfigQemuFromApi(vmr *VmRef, client *Client) (config *ConfigQemu, err e
 		}
 
 		// Add rest of device config.
-		nicConfMap.readDeviceConfig(nicConfList[1:])
+		err = nicConfMap.readDeviceConfig(nicConfList[1:])
+		if err != nil {
+			log.Printf("[ERROR] %q", err)
+		}
+		if nicConfMap["firewall"] == 1 {
+			nicConfMap["firewall"] = true
+		} else if nicConfMap["firewall"] == 0 {
+			nicConfMap["firewall"] = false
+		}
 
 		// And device config to networks.
 		if len(nicConfMap) > 0 {
@@ -612,7 +736,7 @@ func NewConfigQemuFromApi(vmr *VmRef, client *Client) (config *ConfigQemu, err e
 	// Add serials
 	serialNames := []string{}
 
-	for k, _ := range vmConfig {
+	for k := range vmConfig {
 		if serialName := rxSerialName.FindStringSubmatch(k); len(serialName) > 0 {
 			serialNames = append(serialNames, serialName[0])
 		}
@@ -773,6 +897,52 @@ func SendKeysString(vmr *VmRef, client *Client, keys string) (err error) {
 	return nil
 }
 
+// Given a QemuDevice, return a param string to give to ProxMox
+func formatDeviceParam(device QemuDevice) string {
+	deviceConfParams := QemuDeviceParam{}
+	deviceConfParams = deviceConfParams.createDeviceParam(device, nil)
+	return strings.Join(deviceConfParams, ",")
+}
+
+// Given a QemuDevice (represesting a disk), return a param string to give to ProxMox
+func FormatDiskParam(disk QemuDevice) string {
+	diskConfParam := QemuDeviceParam{}
+
+	if volume, ok := disk["volume"]; ok && volume != "" {
+		diskConfParam = append(diskConfParam, volume.(string))
+		diskConfParam = append(diskConfParam, fmt.Sprintf("size=%v", disk["size"]))
+	} else {
+		volumeInit := fmt.Sprintf("%v:%v", disk["storage"], DiskSizeGB(disk["size"]))
+		diskConfParam = append(diskConfParam, volumeInit)
+	}
+
+	// Set cache if not none (default).
+	if cache, ok := disk["cache"]; ok && cache != "none" {
+		diskCache := fmt.Sprintf("cache=%v", disk["cache"])
+		diskConfParam = append(diskConfParam, diskCache)
+	}
+
+	// Mountoptions
+	if mountoptions, ok := disk["mountoptions"]; ok {
+		options := []string{}
+		for opt, enabled := range mountoptions.(map[string]interface{}) {
+			if enabled.(bool) {
+				options = append(options, opt)
+			}
+		}
+		diskMountOpts := fmt.Sprintf("mountoptions=%v", strings.Join(options, ";"))
+		diskConfParam = append(diskConfParam, diskMountOpts)
+	}
+
+	// Keys that are not used as real/direct conf.
+	ignoredKeys := []string{"key", "slot", "type", "storage", "file", "size", "cache", "volume", "container", "vm", "mountoptions", "storage_type", "format"}
+
+	// Rest of config.
+	diskConfParam = diskConfParam.createDeviceParam(disk, ignoredKeys)
+
+	return strings.Join(diskConfParam, ",")
+}
+
 // Create parameters for each Nic device.
 func (c ConfigQemu) CreateQemuNetworksParams(vmID int, params map[string]interface{}) error {
 
@@ -873,57 +1043,41 @@ func (c ConfigQemu) CreateQemuDisksParams(
 
 	// For new style with multi disk device.
 	for diskID, diskConfMap := range c.QemuDisks {
-
 		// skip the first disk for clones (may not always be right, but a template probably has at least 1 disk)
 		if diskID == 0 && cloned {
 			continue
 		}
-		diskConfParam := QemuDeviceParam{
-			"media=disk",
-		}
 
 		// Device name.
 		deviceType := diskConfMap["type"].(string)
 		qemuDiskName := deviceType + strconv.Itoa(diskID)
 
-		// Set disk storage.
-		// Disk size.
-		diskSizeGB := fmt.Sprintf("size=%v", diskConfMap["size"])
-		diskConfParam = append(diskConfParam, diskSizeGB)
-
-		// Disk name.
-		var diskFile string
-		// Currently ZFS local, LVM, Ceph RBD, CephFS and Directory are considered.
-		// Other formats are not verified, but could be added if they're needed.
-		rxStorageTypes := `(zfspool|lvm|rbd|cephfs)`
-		storageType := diskConfMap["storage_type"].(string)
-		if matched, _ := regexp.MatchString(rxStorageTypes, storageType); matched {
-			diskFile = fmt.Sprintf("file=%v:vm-%v-disk-%v", diskConfMap["storage"], vmID, diskID)
-		} else {
-			diskFile = fmt.Sprintf("file=%v:%v/vm-%v-disk-%v.%v", diskConfMap["storage"], vmID, vmID, diskID, diskConfMap["format"])
-		}
-		diskConfParam = append(diskConfParam, diskFile)
-
-		// Set cache if not none (default).
-		if diskConfMap["cache"].(string) != "none" {
-			diskCache := fmt.Sprintf("cache=%v", diskConfMap["cache"])
-			diskConfParam = append(diskConfParam, diskCache)
-		}
-
-		// Keys that are not used as real/direct conf.
-		ignoredKeys := []string{"id", "type", "storage", "storage_type", "size", "cache"}
-
-		// Rest of config.
-		diskConfParam = diskConfParam.createDeviceParam(diskConfMap, ignoredKeys)
-
 		// Add back to Qemu prams.
-		params[qemuDiskName] = strings.Join(diskConfParam, ",")
+		params[qemuDiskName] = FormatDiskParam(diskConfMap)
+	}
+
+	return nil
+}
+
+// Create parameters for serial interface
+func (c ConfigQemu) CreateQemuSerialsParams(
+	vmID int,
+	params map[string]interface{},
+) error {
+
+	// For new style with multi disk device.
+	for serialID, serialConfMap := range c.QemuSerials {
+		// Device name.
+		deviceType := serialConfMap["type"].(string)
+		qemuSerialName := "serial" + strconv.Itoa(serialID)
+
+		// Add back to Qemu prams.
+		params[qemuSerialName] = deviceType
 	}
 
 	return nil
 }
 
-// Create the parameters for each device that will be sent to Proxmox API.
 func (p QemuDeviceParam) createDeviceParam(
 	deviceConfMap QemuDevice,
 	ignoredKeys []string,
@@ -964,43 +1118,6 @@ func (c ConfigQemu) String() string {
 	return string(jsConf)
 }
 
-// Create parameters for serial interface
-func (c ConfigQemu) CreateQemuSerialsParams(
-	vmID int,
-	params map[string]interface{},
-) error {
-
-	// For new style with multi disk device.
-	for serialID, serialConfMap := range c.QemuSerials {
-		// Device name.
-		deviceType := serialConfMap["type"].(string)
-		qemuSerialName := "serial" + strconv.Itoa(serialID)
-
-		// Add back to Qemu prams.
-		params[qemuSerialName] = deviceType
-	}
-
-	return nil
-}
-
-// NextId - Get next free VMID
-func (c *Client) NextId() (id int, err error) {
-	var data map[string]interface{}
-	_, err = c.session.GetJSON("/cluster/nextid", nil, nil, &data)
-	if err != nil {
-		return -1, err
-	}
-	if data["data"] == nil || data["errors"] != nil {
-		return -1, fmt.Errorf(data["errors"].(string))
-	}
-
-	i, err := strconv.Atoi(data["data"].(string))
-	if err != nil {
-		return -1, err
-	}
-	return i, nil
-}
-
 // VMIdExists - If you pass an VMID that exists it will raise an error otherwise it will return the vmID
 func (c *Client) VMIdExists(vmID int) (id int, err error) {
 	_, err = c.session.Get(fmt.Sprintf("/cluster/nextid?vmid=%d", vmID), nil, nil)

vendor/github.com/Telmate/proxmox-api-go/proxmox/session.go

@@ -28,6 +28,7 @@ type Session struct {
 	ApiUrl     string
 	AuthTicket string
 	CsrfToken  string
+	AuthToken  string // Combination of user, realm, token ID and UUID
 	Headers    http.Header
 }
 
@@ -108,6 +109,11 @@ func TypedResponse(resp *http.Response, v interface{}) error {
 	return nil
 }
 
+func (s *Session) SetAPIToken(userID, token string) {
+	auth := fmt.Sprintf("%s=%s", userID, token)
+	s.AuthToken = auth
+}
+
 func (s *Session) Login(username string, password string, otp string) (err error) {
 	reqUser := map[string]interface{}{"username": username, "password": password}
 	if otp != "" {
@@ -150,7 +156,9 @@ func (s *Session) NewRequest(method, url string, headers *http.Header, body io.R
 	if headers != nil {
 		req.Header = *headers
 	}
-	if s.AuthTicket != "" {
+	if s.AuthToken != "" {
+		req.Header.Add("Authorization", "PVEAPIToken="+s.AuthToken)
+	} else if s.AuthTicket != "" {
 		req.Header.Add("Cookie", "PVEAuthCookie="+s.AuthTicket)
 		req.Header.Add("CSRFPreventionToken", s.CsrfToken)
 	}

vendor/github.com/Telmate/proxmox-api-go/proxmox/util.go

@@ -1,6 +1,7 @@
 package proxmox
 
 import (
+	"regexp"
 	"strconv"
 	"strings"
 )
@@ -51,12 +52,57 @@ func ParseConf(
 	kvString string,
 	confSeparator string,
 	subConfSeparator string,
+	implicitFirstKey string,
 ) QemuDevice {
 	var confMap = QemuDevice{}
 	confList := strings.Split(kvString, confSeparator)
+
+	if implicitFirstKey != "" {
+		if !strings.Contains(confList[0], "=") {
+			confMap[implicitFirstKey] = confList[0]
+			confList = confList[1:]
+		}
+	}
+
 	for _, item := range confList {
 		key, value := ParseSubConf(item, subConfSeparator)
 		confMap[key] = value
 	}
 	return confMap
 }
+
+func ParsePMConf(
+	kvString string,
+	implicitFirstKey string,
+) QemuDevice {
+	return ParseConf(kvString, ",", "=", implicitFirstKey)
+}
+
+// Convert a disk-size string to a GB float
+func DiskSizeGB(dcSize interface{}) float64 {
+	var diskSize float64
+	switch dcSize.(type) {
+	case string:
+		diskString := strings.ToUpper(dcSize.(string))
+		re := regexp.MustCompile("([0-9]+)([A-Z]*)")
+		diskArray := re.FindStringSubmatch(diskString)
+
+		diskSize, _ = strconv.ParseFloat(diskArray[1], 64)
+
+		if len(diskArray) >= 3 {
+			switch diskArray[2] {
+			case "T", "TB":
+				diskSize *= 1024
+			case "G", "GB":
+				//Nothing to do
+			case "M", "MB":
+				diskSize /= 1024
+			case "K", "KB":
+				diskSize /= 1048576
+			}
+		}
+	case float64:
+		diskSize = dcSize.(float64)
+	}
+	return diskSize
+}

vendor/github.com/hashicorp/packer-plugin-sdk/didyoumean/name_suggestion.go

@@ -0,0 +1,24 @@
+package didyoumean
+
+import (
+	"github.com/agext/levenshtein"
+)
+
+// NameSuggestion tries to find a name from the given slice of suggested names
+// that is close to the given name and returns it if found. If no suggestion is
+// close enough, returns the empty string.
+//
+// The suggestions are tried in order, so earlier suggestions take precedence if
+// the given string is similar to two or more suggestions.
+//
+// This function is intended to be used with a relatively-small number of
+// suggestions. It's not optimized for hundreds or thousands of them.
+func NameSuggestion(given string, suggestions []string) string {
+	for _, suggestion := range suggestions {
+		dist := levenshtein.Distance(given, suggestion, nil)
+		if dist < 3 { // threshold determined experimentally
+			return suggestion
+		}
+	}
+	return ""
+}

vendor/github.com/hashicorp/packer-plugin-sdk/multistep/commonsteps/http_config.go

@@ -23,6 +23,15 @@ type HTTPConfig struct {
 	// started. The address and port of the HTTP server will be available as
 	// variables in `boot_command`. This is covered in more detail below.
 	HTTPDir string `mapstructure:"http_directory"`
+	// Key/Values to serve using an HTTP server. http_content works like and
+	// conflicts with http_directory The keys represent the paths and the values
+	// contents. This is useful for hosting kickstart files and so on. By
+	// default this is empty, which means no HTTP server will be started. The
+	// address and port of the HTTP server will be available as variables in
+	// `boot_command`. This is covered in more detail below. Example: Setting
+	// `"foo/bar"="baz"`, will allow you to http get on
+	// `http://{http_ip}:{http_port}/foo/bar`.
+	HTTPContent map[string]string `mapstructure:"http_content"`
 	// These are the minimum and maximum port to use for the HTTP server
 	// started to serve the `http_directory`. Because Packer often runs in
 	// parallel, Packer will choose a randomly available port in this range to
@@ -66,5 +75,10 @@ func (c *HTTPConfig) Prepare(ctx *interpolate.Context) []error {
 			errors.New("http_port_min must be less than http_port_max"))
 	}
 
+	if len(c.HTTPContent) > 0 && len(c.HTTPDir) > 0 {
+		errs = append(errs,
+			errors.New("http_content cannot be used in conjunction with http_dir. Consider using the file function to load file in memory and serve them with http_content: https://www.packer.io/docs/templates/hcl_templates/functions/file/file"))
+	}
+
 	return errs
 }

vendor/github.com/hashicorp/packer-plugin-sdk/multistep/commonsteps/step_http_server.go

@@ -3,14 +3,28 @@ package commonsteps
 import (
 	"context"
 	"fmt"
-
+	"log"
 	"net/http"
+	"os"
+	"path"
+	"sort"
 
+	"github.com/hashicorp/packer-plugin-sdk/didyoumean"
 	"github.com/hashicorp/packer-plugin-sdk/multistep"
 	"github.com/hashicorp/packer-plugin-sdk/net"
 	packersdk "github.com/hashicorp/packer-plugin-sdk/packer"
 )
 
+func HTTPServerFromHTTPConfig(cfg *HTTPConfig) *StepHTTPServer {
+	return &StepHTTPServer{
+		HTTPDir:     cfg.HTTPDir,
+		HTTPContent: cfg.HTTPContent,
+		HTTPPortMin: cfg.HTTPPortMin,
+		HTTPPortMax: cfg.HTTPPortMax,
+		HTTPAddress: cfg.HTTPAddress,
+	}
+}
+
 // This step creates and runs the HTTP server that is serving files from the
 // directory specified by the 'http_directory` configuration parameter in the
 // template.
@@ -22,6 +36,7 @@ import (
 //   http_port int - The port the HTTP server started on.
 type StepHTTPServer struct {
 	HTTPDir     string
+	HTTPContent map[string]string
 	HTTPPortMin int
 	HTTPPortMax int
 	HTTPAddress string
@@ -29,16 +44,58 @@ type StepHTTPServer struct {
 	l *net.Listener
 }
 
+func (s *StepHTTPServer) Handler() http.Handler {
+	if s.HTTPDir != "" {
+		return http.FileServer(http.Dir(s.HTTPDir))
+	}
+
+	return MapServer(s.HTTPContent)
+}
+
+type MapServer map[string]string
+
+func (s MapServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	path := path.Clean(r.URL.Path)
+	content, found := s[path]
+	if !found {
+		paths := make([]string, 0, len(s))
+		for k := range s {
+			paths = append(paths, k)
+		}
+		sort.Strings(paths)
+		err := fmt.Sprintf("%s not found.", path)
+		if sug := didyoumean.NameSuggestion(path, paths); sug != "" {
+			err += fmt.Sprintf(" Did you mean %q?", sug)
+		}
+
+		http.Error(w, err, http.StatusNotFound)
+		return
+	}
+
+	if _, err := w.Write([]byte(content)); err != nil {
+		// log err in case the file couldn't be 100% transferred for example.
+		log.Printf("http_content serve error: %v", err)
+	}
+}
+
 func (s *StepHTTPServer) Run(ctx context.Context, state multistep.StateBag) multistep.StepAction {
 	ui := state.Get("ui").(packersdk.Ui)
 
-	if s.HTTPDir == "" {
+	if s.HTTPDir == "" && len(s.HTTPContent) == 0 {
 		state.Put("http_port", 0)
 		return multistep.ActionContinue
 	}
 
+	if s.HTTPDir != "" {
+		if _, err := os.Stat(s.HTTPDir); err != nil {
+			err := fmt.Errorf("Error finding %q: %s", s.HTTPDir, err)
+			state.Put("error", err)
+			ui.Error(err.Error())
+			return multistep.ActionHalt
+		}
+	}
+
 	// Find an available TCP port for our HTTP server
-	var httpAddr string
 	var err error
 	s.l, err = net.ListenRangeConfig{
 		Min:     s.HTTPPortMin,
@@ -57,8 +114,7 @@ func (s *StepHTTPServer) Run(ctx context.Context, state multistep.StateBag) mult
 	ui.Say(fmt.Sprintf("Starting HTTP server on port %d", s.l.Port))
 
 	// Start the HTTP server and run it in the background
-	fileServer := http.FileServer(http.Dir(s.HTTPDir))
-	server := &http.Server{Addr: httpAddr, Handler: fileServer}
+	server := &http.Server{Addr: "", Handler: s.Handler()}
 	go server.Serve(s.l)
 
 	// Save the address into the state so it can be accessed in the future
@@ -67,9 +123,20 @@ func (s *StepHTTPServer) Run(ctx context.Context, state multistep.StateBag) mult
 	return multistep.ActionContinue
 }
 
-func (s *StepHTTPServer) Cleanup(multistep.StateBag) {
+func (s *StepHTTPServer) Cleanup(state multistep.StateBag) {
 	if s.l != nil {
+		ui := state.Get("ui").(packersdk.Ui)
+
 		// Close the listener so that the HTTP server stops
-		s.l.Close()
+		if err := s.l.Close(); err != nil {
+			err = fmt.Errorf("Failed closing http server on port %d: %w", s.l.Port, err)
+			ui.Error(err.Error())
+			// Here this error should be shown to the UI but it won't
+			// specifically stop Packer from terminating successfully. It could
+			// cause a "Listen leak" if it happenned a lot. Though Listen will
+			// try other ports if one is already used. In the case we want to
+			// Listen on only one port, the next Listen call could fail or be
+			// longer than expected.
+		}
 	}
 }

vendor/github.com/hashicorp/packer-plugin-sdk/template/funcs.go

@@ -56,7 +56,10 @@ func Vault(path string, key string) (string, error) {
 		}
 
 		// neither v1 nor v2 proudced a valid value
-		return "", fmt.Errorf("Vault data was empty at the given path. Warnings: %s", strings.Join(secret.Warnings, "; "))
+		return "", fmt.Errorf("Vault data was empty at the given path. Check "+
+			"the Vault function docs for help: "+
+			"https://www.packer.io/docs/templates/hcl_templates/functions/contextual/vault. "+
+			"Original warnings from Vault call: %s", strings.Join(secret.Warnings, "; "))
 	}
 
 	if val, ok := data.(map[string]interface{})[key]; ok {

vendor/github.com/hashicorp/packer-plugin-sdk/version/version.go

@@ -13,12 +13,12 @@ import (
 var GitCommit string
 
 // Package version helps plugin creators set and track the sdk version using
-var Version = "0.0.14"
+var Version = "0.1.1"
 
 // A pre-release marker for the version. If this is "" (empty string)
 // then it means that it is a final release. Otherwise, this is a pre-release
 // such as "dev" (in development), "beta", "rc1", etc.
-var VersionPrerelease = ""
+var VersionPrerelease = "dev"
 
 // SDKVersion is used by the plugin set to allow Packer to recognize
 // what version of the sdk the plugin is.

vendor/modules.txt

@@ -78,7 +78,7 @@ github.com/NaverCloudPlatform/ncloud-sdk-go-v2/ncloud
 github.com/NaverCloudPlatform/ncloud-sdk-go-v2/services/server
 # github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d
 github.com/StackExchange/wmi
-# github.com/Telmate/proxmox-api-go v0.0.0-20200715182505-ec97c70ba887
+# github.com/Telmate/proxmox-api-go v0.0.0-20210320143302-fea68269e6b0
 ## explicit
 github.com/Telmate/proxmox-api-go/proxmox
 # github.com/agext/levenshtein v1.2.1
@@ -428,7 +428,7 @@ github.com/hashicorp/packer-plugin-docker/post-processor/docker-import
 github.com/hashicorp/packer-plugin-docker/post-processor/docker-push
 github.com/hashicorp/packer-plugin-docker/post-processor/docker-save
 github.com/hashicorp/packer-plugin-docker/post-processor/docker-tag
-# github.com/hashicorp/packer-plugin-sdk v0.0.14
+# github.com/hashicorp/packer-plugin-sdk v0.1.1-0.20210323111710-5a7ab7f7a1c0
 ## explicit
 github.com/hashicorp/packer-plugin-sdk/acctest
 github.com/hashicorp/packer-plugin-sdk/acctest/provisioneracc
@@ -440,6 +440,7 @@ github.com/hashicorp/packer-plugin-sdk/common
 github.com/hashicorp/packer-plugin-sdk/communicator
 github.com/hashicorp/packer-plugin-sdk/communicator/ssh
 github.com/hashicorp/packer-plugin-sdk/communicator/sshkey
+github.com/hashicorp/packer-plugin-sdk/didyoumean
 github.com/hashicorp/packer-plugin-sdk/filelock
 github.com/hashicorp/packer-plugin-sdk/guestexec
 github.com/hashicorp/packer-plugin-sdk/hcl2helper
@@ -570,6 +571,8 @@ github.com/mitchellh/go-testing-interface
 github.com/mitchellh/go-vnc
 # github.com/mitchellh/go-wordwrap v1.0.0
 github.com/mitchellh/go-wordwrap
+# github.com/mitchellh/gox v1.0.1
+## explicit
 # github.com/mitchellh/iochan v1.0.0
 github.com/mitchellh/iochan
 # github.com/mitchellh/mapstructure v1.4.0

website/README.md

@@ -238,14 +238,22 @@ $ terraform apply
 
 <!-- END: editing-markdown -->
 
-<!-- BEGIN: editing-docs-sidebars -->
-<!-- Generated text, do not edit directly -->
+<!--
+
+NOTE: The "Editing Navigation Sidebars" section is forked from editing-docs-sidebars.
+
+We plan on rolling these changes back into our "readme partials" source once all docs sites
+have been transitioned to the JSON navigation format. See MKTG_032 for details:
+
+https://docs.google.com/document/d/1kYvbyd6njHFSscoE1dtDNHQ3U8IzaMdcjOS0jg87rHg/
+
+-->
 
 ## Editing Navigation Sidebars
 
-The structure of the sidebars are controlled by files in the [`/data` directory](data). For example, [this file](data/docs-navigation.js) controls the **docs** sidebar. Within the `data` folder, any file with `-navigation` after it controls the navigation for the given section.
+The structure of the sidebars are controlled by files in the [`/data` directory](data). For example, [data/docs-nav-data.json](data/docs-nav-data.json) controls the **docs** sidebar. Within the `data` folder, any file with `-nav-data` after it controls the navigation for the given section.
 
-The sidebar uses a simple recursive data structure to represent _files_ and _directories_. A file is represented by a string, and a directory is represented by an object. The sidebar is meant to reflect the structure of the docs within the filesystem while also allowing custom ordering. Let's look at an example. First, here's our example folder structure:
+The sidebar uses a simple recursive data structure to represent _files_ and _directories_. The sidebar is meant to reflect the structure of the docs within the filesystem while also allowing custom ordering. Let's look at an example. First, here's our example folder structure:
 
 ```text
 .
@@ -259,36 +267,55 @@ The sidebar uses a simple recursive data structure to represent _files_ and _dir
 │           └── nested-file.mdx
 ```
 
-Here's how this folder structure could be represented as a sidebar navigation, in this example it would be the file `website/data/docs-navigation.js`:
+Here's how this folder structure could be represented as a sidebar navigation, in this example it would be the file `website/data/docs-nav-data.json`:
 
-```js
-export default {
-  category: 'directory',
-  content: [
-    'file',
-    'another-file',
-    {
-      category: 'nested-directory',
-      content: ['nested-file'],
-    },
-  ],
-}
+```json
+[
+  {
+    "title": "Directory",
+    "routes": [
+      {
+        "title": "Overview",
+        "path": "directory"
+      },
+      {
+        "title": "File",
+        "path": "directory/file"
+      },
+      {
+        "title": "Another File",
+        "path": "directory/another-file"
+      },
+      {
+        "title": "Nested Directory",
+        "routes": [
+          {
+            "title": "Overview",
+            "path": "directory/nested-directory"
+          },
+          {
+            "title": "Nested File",
+            "path": "directory/nested-directory/nested-file"
+          }
+        ]
+      }
+    ]
+  }
+]
 ```
 
-- `category` values will be **directory names** within the `pages/<section>` directory
-- `content` values will be **file names** within their appropriately nested directory
-
 A couple more important notes:
 
-- Within this data structure, ordering does not matter, but hierarchy does. So while you could put `file` and `another-file` in any order, or even leave one or both of them out, you could not decide to un-nest the `nested-directory` object without also un-nesting it in the filesystem.
-- The `sidebar_title` frontmatter property on each `mdx` page is responsible for displaying the human-readable page name in the navigation.
-- _By default_, every directory/category must have an `index.mdx` file. This file will be automatically added to the navigation as "Overview", and its `sidebar_title` property will set the human-readable name of the entire category.
+- Within this data structure, ordering is flexible, but hierarchy is not. The structure of the sidebar must correspond to the structure of the content directory. So while you could put `file` and `another-file` in any order in the sidebar, or even leave one or both of them out, you could not decide to un-nest the `nested-directory` object without also un-nesting it in the filesystem.
+- The `title` property on each node in the `nav-data` tree is the human-readable name in the navigation.
+- The `path` property on each leaf node in the `nav-data` tree is the URL path where the `.mdx` document will be rendered, and the
+- Note that "index" files must be explicitly added. These will be automatically resolved, so the `path` value should be, as above, `directory` rather than `directory/index`. A common convention is to set the `title` of an "index" node to be `"Overview"`.
 
 Below we will discuss a couple of more unusual but still helpful patterns.
 
 ### Index-less Categories
 
-Sometimes you may want to include a category but not have a need for an index page for the category. This can be accomplished, but a human-readable category name needs to be set manually, since the category name is normally pulled from the `sidebar_title` property of the index page. Here's an example of how an index-less category might look:
+Sometimes you may want to include a category but not have a need for an index page for the category. This can be accomplished, but as with other branch and leaf nodes, a human-readable `title` needs to be set manually. Here's an example of how an index-less category might look:
 
 ```text
 .
@@ -297,36 +324,95 @@ Sometimes you may want to include a category but not have a need for an index pa
 │       └── file.mdx
 ```
 
-```js
-// website/data/docs-navigation.js
-export default {
-  category: 'indexless-category',
-  name: 'Indexless Category',
-  content: ['file'],
-}
+```json
+// website/data/docs-nav-data.json
+[
+  {
+    "title": "Indexless Category",
+    "routes": [
+      {
+        "title": "File",
+        "path": "indexless-category/file"
+      }
+    ]
+  }
+]
 ```
 
-The addition of the `name` property to a category object is all it takes to be able to skip the index file.
-
 ### Custom or External Links
 
 Sometimes you may have a need to include a link that is not directly to a file within the docs hierarchy. This can also be supported using a different pattern. For example:
 
-```js
-export default {
-  category: 'directory',
-  content: [
-    'file',
-    'another-file',
-    { title: 'Tao of HashiCorp', href: 'https://www.hashicorp.com/tao-of-hashicorp' }
-    }
-  ]
-}
+```json
+[
+  {
+    "name": "Directory",
+    "routes": [
+      {
+        "title": "File",
+        "path": "directory/file"
+      },
+      {
+        "title": "Another File",
+        "path": "directory/another-file"
+      },
+      {
+        "title": "Tao of HashiCorp",
+        "href": "https://www.hashicorp.com/tao-of-hashicorp"
+      }
+    ]
+  }
+]
 ```
 
 If the link provided in the `href` property is external, it will display a small icon indicating this. If it's internal, it will appear the same way as any other direct file link.
 
-<!-- END: editing-docs-sidebars -->
+### Plugin Docs
+
+Plugin documentation may be located within the `packer` repository, or split out into separate `packer-plugin-` repositories. For plugin docs within the `packer` repository, the process for authoring files and managing sidebar data is identical to the process for other documentation.
+
+For plugins in separate repositories, additional configuration is required.
+
+#### Setting up remote plugin docs
+
+Some setup is required to include docs from remote plugin repositories on the [packer.io/docs](https://www.packer.io/docs) site.
+
+1. The plugin repository needs to include a `docs.zip` asset in its release
+2. The `packer` repository must have a corresponding entry in `website/data/docs-remote-plugins.json` which points to the plugin repository.
+
+The `docs.zip` release asset is expected to be generated as part of the standard release process for `packer-plugin-*` repositories. Additional details on this process can be found in [the `packer-plugin-scaffolding` `README`](https://github.com/hashicorp/packer-plugin-scaffolding#registering-documentation-on-packerio).
+
+The `docs-remote-plugins.json` file contains an array of entries. Each entry points to a plugin repository. The `{ title, path, repo, version }` properties are required for each entry.
+
+```json5
+[
+  {
+    // ALL FIELDS ARE REQUIRED.
+    // "title" sets the human-readable title shown in navigation
+    title: 'Scaffolding',
+    // "path" sets the URL subpath under the component URL (eg `docs/builders`)
+    path: 'scaffolding',
+    // "repo" points to the plugin repo, in the format "organization/repo-name"
+    // if the organization == hashicorp, the plugin docs will be labelled "official".
+    // for all other organizations or users, plugin docs will be labelled "community".
+    repo: 'hashicorp/packer-plugin-scaffolding',
+    // "version" is used to fetch "docs.zip" from the matching tagged release.
+    // version: "latest" is permitted, but please be aware that it
+    // may fetch incompatible or unintended versions of plugin docs.
+    // if version is NOT "latest", and if "docs.zip" is unavailable, then
+    // we fall back to fetching docs from the source "{version}.zip"
+    version: 'v0.0.5',
+  },
+]
+```
+
+#### Updating remote plugin docs
+
+Documentation from plugin repositories is fetched and rendered every time the Packer website builds. So, to update plugin documentation on the live site:
+
+1. In the plugin repository, publish a new release that includes a `docs.zip` release asset
+2. In the `packer` repository, update `website/data/docs-remote-plugins.json` to ensure the corresponding entry points to the correct release `version` (which should correspond to the release's tag name). This may not be necessary if the `version` is set to `"latest"`.
+3. Rebuild the website. This will happen automatically on commits to `stable-website`. In exceptional cases, the site can also be [manually re-deployed through Vercel](https://vercel.com/hashicorp/packer).
 
 <!-- BEGIN: releases -->
 <!-- Generated text, do not edit directly -->
@@ -374,8 +460,18 @@ You may customize the parameters in any way you'd like. To remove a prerelease f
 
 <!-- END: releases -->
 
-<!-- BEGIN: redirects -->
-<!-- Generated text, do not edit directly -->
+<!--
+
+NOTE: The "Redirects" section is forked from redirects.
+
+There are minor changes related to sidebar navigation format changes.
+
+We plan on rolling these changes back into our "readme partials" source once all docs sites
+have been transitioned to the JSON navigation format. See MKTG_032 for details:
+
+https://docs.google.com/document/d/1kYvbyd6njHFSscoE1dtDNHQ3U8IzaMdcjOS0jg87rHg/
+
+-->
 
 ## Link Validation
 
@@ -406,7 +502,7 @@ There are a couple important caveats with redirects. First, redirects are applie
 
 Second, redirects do not apply to client-side navigation. By default, all links in the navigation and docs sidebar will navigate purely on the client side, which makes navigation through the docs significantly faster, especially for those with low-end devices and/or weak internet connections. In the future, we plan to convert all internal links within docs pages to behave this way as well. This means that if there is a link on this website to a given piece of content that has changed locations in some way, we need to also _directly change existing links to the content_. This way, if a user clicks a link that navigates on the client side, or if they hit the url directly and the page renders from the server side, either one will work perfectly.
 
-Let's look at an example. Say you have a page called `/docs/foo` which needs to be moved to `/docs/nested/foo`. Additionally, this is a page that has been around for a while and we know there are links into `/docs/foo.html` left over from our previous website structure. First, we move the page, then adjust the docs sidenav, in `data/docs-navigation.js`. Find the category the page is in, and move it into the appropriate subcategory. Next, we add to `_redirects` as such:
+Let's look at an example. Say you have a page called `/docs/foo` which needs to be moved to `/docs/nested/foo`. Additionally, this is a page that has been around for a while and we know there are links into `/docs/foo.html` left over from our previous website structure. First, we move the page, then adjust the docs sidenav, in `data/docs-nav-data.json`. Find the category the page is in, and move it into the appropriate subcategory. Next, we add to `_redirects` as such:
 
 ```
 /foo       /nested/foo  301!
@@ -415,26 +511,36 @@ Let's look at an example. Say you have a page called `/docs/foo` which needs to
 
 Finally, we run a global search for internal links to `/foo`, and make sure to adjust them to be `/nested/foo` - this is to ensure that client-side navigation still works correctly. _Adding a redirect alone is not enough_.
 
-One more example - let's say that content is being moved to an external website. A common example is guides moving to `learn.hashicorp.com`. In this case, we take all the same steps, except that we need to make a different type of change to the `docs-navigation` file. If previously the structure looked like:
+One more example - let's say that content is being moved to an external website. A common example is guides moving to `learn.hashicorp.com`. In this case, we take all the same steps, except that we need to make a different type of change to the `docs-nav-data` file. If previously the structure looked like:
 
-```js
-{
-  category: 'docs',
-  content: [
-    'foo'
-  ]
-}
+```json
+[
+  {
+    "name": "Docs",
+    "routes": [
+      {
+        "title": "Foo",
+        "path": "docs/foo"
+      }
+    ]
+  }
+]
 ```
 
-If we no longer want the link to be in the side nav, we can simply remove it. If we do still want the link in the side nav, but pointing to an external destnation, we need to slightly change the structure as such:
+If we no longer want the link to be in the side nav, we can simply remove it. If we do still want the link in the side nav, but pointing to an external destination, we need to slightly change the structure as such:
 
-```js
-{
-  category: 'docs',
-  content: [
-    { title: 'Foo Title', href: 'https://learn.hashicorp.com/<product>/foo' }
-  ]
-}
+```json
+[
+  {
+    "name": "Docs",
+    "routes": [
+      {
+        "title": "Foo",
+        "href": "https://learn.hashicorp.com/<product>/foo"
+      }
+    ]
+  }
+]
 ```
 
 As the majority of items in the side nav are internal links, the structure makes it as easy as possible to represent these links. This alternate syntax is the most concise manner than an external link can be represented. External links can be used anywhere within the docs sidenav.