diff --git a/common/config.go b/common/config.go index 24c6d4233..70ebc8452 100644 --- a/common/config.go +++ b/common/config.go @@ -9,6 +9,8 @@ import ( "runtime" "strings" "time" + + "github.com/hashicorp/packer/packer" ) // PackerKeyEnv is used to specify the key interval (delay) between keystrokes @@ -41,7 +43,7 @@ func SupportedProtocol(u *url.URL) bool { // build a dummy NewDownloadClient since this is the only place that valid // protocols are actually exposed. - cli := NewDownloadClient(&DownloadConfig{}, nil) + cli := NewDownloadClient(&DownloadConfig{}, new(packer.NoopUi)) // Iterate through each downloader to see if a protocol was found. ok := false @@ -173,7 +175,7 @@ func FileExistsLocally(original string) bool { // First create a dummy downloader so we can figure out which // protocol to use. - cli := NewDownloadClient(&DownloadConfig{}, nil) + cli := NewDownloadClient(&DownloadConfig{}, new(packer.NoopUi)) d, ok := cli.config.DownloaderMap[u.Scheme] if !ok { return false diff --git a/common/download.go b/common/download.go index f0e88420b..78f31f25d 100644 --- a/common/download.go +++ b/common/download.go @@ -10,19 +10,17 @@ import ( "errors" "fmt" "hash" + "io" "log" + "net/http" "net/url" "os" "path" + "path/filepath" "runtime" "strings" -) -// imports related to each Downloader implementation -import ( - "io" - "net/http" - "path/filepath" + "github.com/hashicorp/packer/packer" ) // DownloadConfig is the configuration given to instantiate a new @@ -58,8 +56,7 @@ type DownloadConfig struct { // A DownloadClient helps download, verify checksums, etc. type DownloadClient struct { - config *DownloadConfig - downloader Downloader + config *DownloadConfig } // HashForType returns the Hash implementation for the given string @@ -81,33 +78,24 @@ func HashForType(t string) hash.Hash { // NewDownloadClient returns a new DownloadClient for the given // configuration. -func NewDownloadClient(c *DownloadConfig, bar ProgressBar) *DownloadClient { - const mtu = 1500 /* ethernet */ - 20 /* ipv4 */ - 20 /* tcp */ - - // If bar is nil, then use a dummy progress bar that doesn't do anything - if bar == nil { - bar = GetDummyProgressBar() - } - +func NewDownloadClient(c *DownloadConfig, ui packer.Ui) *DownloadClient { // Create downloader map if it hasn't been specified already. if c.DownloaderMap == nil { c.DownloaderMap = map[string]Downloader{ - "file": &FileDownloader{progress: bar, bufferSize: nil}, - "http": &HTTPDownloader{progress: bar, userAgent: c.UserAgent}, - "https": &HTTPDownloader{progress: bar, userAgent: c.UserAgent}, - "smb": &SMBDownloader{progress: bar, bufferSize: nil}, + "file": &FileDownloader{Ui: ui, bufferSize: nil}, + "http": &HTTPDownloader{Ui: ui, userAgent: c.UserAgent}, + "https": &HTTPDownloader{Ui: ui, userAgent: c.UserAgent}, + "smb": &SMBDownloader{Ui: ui, bufferSize: nil}, } } return &DownloadClient{config: c} } -// A downloader implements the ability to transfer a file, and cancel or resume -// it. +// Downloader defines what capabilities a downloader should have. type Downloader interface { Resume() Cancel() - Progress() uint64 - Total() uint64 + ProgressBar() packer.ProgressBar } // A LocalDownloader is responsible for converting a uri to a local path @@ -150,17 +138,17 @@ func (d *DownloadClient) Get() (string, error) { var finalPath string var ok bool - d.downloader, ok = d.config.DownloaderMap[u.Scheme] + downloader, ok := d.config.DownloaderMap[u.Scheme] if !ok { return "", fmt.Errorf("No downloader for scheme: %s", u.Scheme) } - remote, ok := d.downloader.(RemoteDownloader) + remote, ok := downloader.(RemoteDownloader) if !ok { - return "", fmt.Errorf("Unable to treat uri scheme %s as a Downloader. : %T", u.Scheme, d.downloader) + return "", fmt.Errorf("Unable to treat uri scheme %s as a Downloader. : %T", u.Scheme, downloader) } - local, ok := d.downloader.(LocalDownloader) + local, ok := downloader.(LocalDownloader) if !ok && !d.config.CopyFile { d.config.CopyFile = true } @@ -233,11 +221,9 @@ func (d *DownloadClient) VerifyChecksum(path string) (bool, error) { // HTTPDownloader is an implementation of Downloader that downloads // files over HTTP. type HTTPDownloader struct { - current uint64 - total uint64 userAgent string - progress ProgressBar + Ui packer.Ui } func (d *HTTPDownloader) Cancel() { @@ -256,8 +242,7 @@ func (d *HTTPDownloader) Download(dst *os.File, src *url.URL) error { return err } - // Reset our progress - d.current = 0 + var current int64 // Make the request. We first make a HEAD request so we can check // if the server supports range queries. If the server/URL doesn't @@ -301,7 +286,7 @@ func (d *HTTPDownloader) Download(dst *os.File, src *url.URL) error { if _, err = dst.Seek(0, os.SEEK_END); err == nil { req.Header.Set("Range", fmt.Sprintf("bytes=%d-", fi.Size())) - d.current = uint64(fi.Size()) + current = fi.Size() } } } @@ -328,23 +313,22 @@ func (d *HTTPDownloader) Download(dst *os.File, src *url.URL) error { return fmt.Errorf("HTTP error: %s", err.Error()) } - d.total = d.current + uint64(resp.ContentLength) + total := current + resp.ContentLength - bar := d.progress - bar.SetTotal64(int64(d.total)) - progressBar := bar.Start() - progressBar.Set64(int64(d.current)) + bar := d.ProgressBar() + bar.Start(total) + defer bar.Finish() + bar.Add(current) + + body := bar.NewProxyReader(resp.Body) var buffer [4096]byte for { - n, err := resp.Body.Read(buffer[:]) + n, err := body.Read(buffer[:]) if err != nil && err != io.EOF { return err } - d.current += uint64(n) - progressBar.Set64(int64(d.current)) - if _, werr := dst.Write(buffer[:n]); werr != nil { return werr } @@ -353,36 +337,16 @@ func (d *HTTPDownloader) Download(dst *os.File, src *url.URL) error { break } } - progressBar.Finish() return nil } -func (d *HTTPDownloader) Progress() uint64 { - return d.current -} - -func (d *HTTPDownloader) Total() uint64 { - return d.total -} - // FileDownloader is an implementation of Downloader that downloads // files using the regular filesystem. type FileDownloader struct { bufferSize *uint - active bool - current uint64 - total uint64 - - progress ProgressBar -} - -func (d *FileDownloader) Progress() uint64 { - return d.current -} - -func (d *FileDownloader) Total() uint64 { - return d.total + active bool + Ui packer.Ui } func (d *FileDownloader) Cancel() { @@ -449,7 +413,6 @@ func (d *FileDownloader) Download(dst *os.File, src *url.URL) error { } /* download the file using the operating system's facilities */ - d.current = 0 d.active = true f, err := os.Open(realpath) @@ -463,44 +426,37 @@ func (d *FileDownloader) Download(dst *os.File, src *url.URL) error { if err != nil { return err } - d.total = uint64(fi.Size()) - bar := d.progress - bar.SetTotal64(int64(d.total)) - progressBar := bar.Start() - progressBar.Set64(int64(d.current)) + bar := d.ProgressBar() + + bar.Start(fi.Size()) + defer bar.Finish() + fProxy := bar.NewProxyReader(f) // no bufferSize specified, so copy synchronously. if d.bufferSize == nil { - var n int64 - n, err = io.Copy(dst, f) + _, err = io.Copy(dst, fProxy) d.active = false - d.current += uint64(n) - progressBar.Set64(int64(d.current)) - // use a goro in case someone else wants to enable cancel/resume } else { errch := make(chan error) go func(d *FileDownloader, r io.Reader, w io.Writer, e chan error) { for d.active { - n, err := io.CopyN(w, r, int64(*d.bufferSize)) + _, err := io.CopyN(w, r, int64(*d.bufferSize)) if err != nil { break } - d.current += uint64(n) - progressBar.Set64(int64(d.current)) } d.active = false e <- err - }(d, f, dst, errch) + }(d, fProxy, dst, errch) // ...and we spin until it's done err = <-errch } - progressBar.Finish() - f.Close() + return err } @@ -509,19 +465,8 @@ func (d *FileDownloader) Download(dst *os.File, src *url.URL) error { type SMBDownloader struct { bufferSize *uint - active bool - current uint64 - total uint64 - - progress ProgressBar -} - -func (d *SMBDownloader) Progress() uint64 { - return d.current -} - -func (d *SMBDownloader) Total() uint64 { - return d.total + active bool + Ui packer.Ui } func (d *SMBDownloader) Cancel() { @@ -570,7 +515,6 @@ func (d *SMBDownloader) Download(dst *os.File, src *url.URL) error { } /* Open up the "\\"-prefixed path using the Windows filesystem */ - d.current = 0 d.active = true f, err := os.Open(realpath) @@ -584,43 +528,39 @@ func (d *SMBDownloader) Download(dst *os.File, src *url.URL) error { if err != nil { return err } - d.total = uint64(fi.Size()) - bar := d.progress - bar.SetTotal64(int64(d.total)) - progressBar := bar.Start() - progressBar.Set64(int64(d.current)) + bar := d.ProgressBar() + + bar.Start(fi.Size()) + defer bar.Finish() + fProxy := bar.NewProxyReader(f) // no bufferSize specified, so copy synchronously. if d.bufferSize == nil { - var n int64 - n, err = io.Copy(dst, f) + _, err = io.Copy(dst, fProxy) d.active = false - d.current += uint64(n) - progressBar.Set64(int64(d.current)) - // use a goro in case someone else wants to enable cancel/resume } else { errch := make(chan error) go func(d *SMBDownloader, r io.Reader, w io.Writer, e chan error) { for d.active { - n, err := io.CopyN(w, r, int64(*d.bufferSize)) + _, err := io.CopyN(w, r, int64(*d.bufferSize)) if err != nil { break } - d.current += uint64(n) - progressBar.Set64(int64(d.current)) } d.active = false e <- err - }(d, f, dst, errch) + }(d, fProxy, dst, errch) // ...and as usual we spin until it's done err = <-errch } - progressBar.Finish() - f.Close() return err } + +func (d *HTTPDownloader) ProgressBar() packer.ProgressBar { return d.Ui.ProgressBar() } +func (d *FileDownloader) ProgressBar() packer.ProgressBar { return d.Ui.ProgressBar() } +func (d *SMBDownloader) ProgressBar() packer.ProgressBar { return d.Ui.ProgressBar() } diff --git a/common/download_test.go b/common/download_test.go index 153e8daf2..dd41d6cc1 100644 --- a/common/download_test.go +++ b/common/download_test.go @@ -12,6 +12,8 @@ import ( "runtime" "strings" "testing" + + "github.com/hashicorp/packer/packer" ) func TestDownloadClientVerifyChecksum(t *testing.T) { @@ -36,7 +38,7 @@ func TestDownloadClientVerifyChecksum(t *testing.T) { Checksum: checksum, } - d := NewDownloadClient(config, nil) + d := NewDownloadClient(config, new(packer.NoopUi)) result, err := d.VerifyChecksum(tf.Name()) if err != nil { t.Fatalf("Verify err: %s", err) @@ -59,7 +61,7 @@ func TestDownloadClient_basic(t *testing.T) { Url: ts.URL + "/basic.txt", TargetPath: tf.Name(), CopyFile: true, - }, nil) + }, new(packer.NoopUi)) path, err := client.Get() if err != nil { @@ -95,7 +97,7 @@ func TestDownloadClient_checksumBad(t *testing.T) { Hash: HashForType("md5"), Checksum: checksum, CopyFile: true, - }, nil) + }, new(packer.NoopUi)) if _, err := client.Get(); err == nil { t.Fatal("should error") @@ -121,7 +123,7 @@ func TestDownloadClient_checksumGood(t *testing.T) { Hash: HashForType("md5"), Checksum: checksum, CopyFile: true, - }, nil) + }, new(packer.NoopUi)) path, err := client.Get() if err != nil { @@ -153,7 +155,7 @@ func TestDownloadClient_checksumNoDownload(t *testing.T) { Hash: HashForType("md5"), Checksum: checksum, CopyFile: true, - }, nil) + }, new(packer.NoopUi)) path, err := client.Get() if err != nil { t.Fatalf("err: %s", err) @@ -183,7 +185,7 @@ func TestDownloadClient_notFound(t *testing.T) { client := NewDownloadClient(&DownloadConfig{ Url: ts.URL + "/not-found.txt", TargetPath: tf.Name(), - }, nil) + }, new(packer.NoopUi)) if _, err := client.Get(); err == nil { t.Fatal("should error") @@ -211,7 +213,7 @@ func TestDownloadClient_resume(t *testing.T) { Url: ts.URL, TargetPath: tf.Name(), CopyFile: true, - }, nil) + }, new(packer.NoopUi)) path, err := client.Get() if err != nil { @@ -273,7 +275,7 @@ func TestDownloadClient_usesDefaultUserAgent(t *testing.T) { CopyFile: true, } - client := NewDownloadClient(config, nil) + client := NewDownloadClient(config, new(packer.NoopUi)) _, err = client.Get() if err != nil { t.Fatal(err) @@ -306,7 +308,7 @@ func TestDownloadClient_setsUserAgent(t *testing.T) { CopyFile: true, } - client := NewDownloadClient(config, nil) + client := NewDownloadClient(config, new(packer.NoopUi)) _, err = client.Get() if err != nil { t.Fatal(err) @@ -405,7 +407,7 @@ func TestDownloadFileUrl(t *testing.T) { CopyFile: false, } - client := NewDownloadClient(config, nil) + client := NewDownloadClient(config, new(packer.NoopUi)) // Verify that we fail to match the checksum _, err = client.Get() @@ -436,7 +438,7 @@ func SimulateFileUriDownload(t *testing.T, uri string) (string, error) { } // go go go - client := NewDownloadClient(config, nil) + client := NewDownloadClient(config, new(packer.NoopUi)) path, err := client.Get() // ignore any non-important checksum errors if it's not a unc path diff --git a/common/progress.go b/common/progress.go deleted file mode 100644 index 83b6d0e2b..000000000 --- a/common/progress.go +++ /dev/null @@ -1,145 +0,0 @@ -package common - -import ( - "fmt" - "github.com/cheggaaa/pb" - "github.com/hashicorp/packer/helper/multistep" - "github.com/hashicorp/packer/packer" - "github.com/hashicorp/packer/packer/rpc" - "log" - "reflect" - "time" -) - -// This is the arrow from packer/ui.go -> TargetedUI.prefixLines -const targetedUIArrowText = "==>" - -// The ProgressBar interface is used for abstracting cheggaaa's progress- -// bar, or any other progress bar. If a UI does not support a progress- -// bar, then it must return a null progress bar. -const ( - DefaultProgressBarWidth = 80 -) - -type ProgressBar = *pb.ProgressBar - -// Figure out the terminal dimensions and use it to calculate the available rendering space -func calculateProgressBarWidth(length int) int { - // If the UI's width is signed, then this is an interface that doesn't really benefit from a progress bar - if length < 0 { - log.Println("Refusing to render progress-bar for unsupported UI.") - return length - } - - // Figure out the terminal width if possible - width, _, err := GetTerminalDimensions() - if err != nil { - newerr := fmt.Errorf("Unable to determine terminal dimensions: %v", err) - log.Printf("Using default width (%d) for progress-bar due to error: %s", DefaultProgressBarWidth, newerr) - return DefaultProgressBarWidth - } - - // If the terminal width is smaller than the requested length, then complain - if width < length { - newerr := fmt.Errorf("Terminal width (%d) is smaller than UI message width (%d).", width, length) - log.Printf("Using default width (%d) for progress-bar due to error: %s", DefaultProgressBarWidth, newerr) - return DefaultProgressBarWidth - } - - // Otherwise subtract the minimum length and return it - return width - length -} - -// Get a progress bar with the default appearance -func GetDefaultProgressBar() ProgressBar { - bar := pb.New64(0) - bar.ShowPercent = true - bar.ShowCounters = true - bar.ShowSpeed = false - bar.ShowBar = true - bar.ShowTimeLeft = false - bar.ShowFinalTime = false - bar.SetUnits(pb.U_BYTES) - bar.Format("[=>-]") - bar.SetRefreshRate(5 * time.Second) - return bar -} - -// Return a dummy progress bar that doesn't do anything -func GetDummyProgressBar() ProgressBar { - bar := pb.New64(0) - bar.ManualUpdate = true - return bar -} - -// Given a packer.Ui, calculate the number of characters that a packer.Ui will -// prefix a message with. Then we can use this to calculate the progress bar's width. -func calculateUiPrefixLength(ui packer.Ui) int { - var recursiveCalculateUiPrefixLength func(packer.Ui, int) int - - // Define a recursive closure that traverses through all the known packer.Ui types - // and aggregates the length of the message prefix from each particular type - recursiveCalculateUiPrefixLength = func(ui packer.Ui, agg int) int { - switch ui.(type) { - - case *packer.ColoredUi: - // packer.ColoredUi is simply a wrapper around .Ui - u := ui.(*packer.ColoredUi) - return recursiveCalculateUiPrefixLength(u.Ui, agg) - - case *packer.TargetedUI: - // A TargetedUI adds the .Target and an arrow by default - u := ui.(*packer.TargetedUI) - res := fmt.Sprintf("%s %s: ", targetedUIArrowText, u.Target) - return recursiveCalculateUiPrefixLength(u.Ui, agg+len(res)) - - case *packer.BasicUi: - // The standard BasicUi appends only a newline - return agg + len("\n") - - // packer.rpc.Ui returns 0 here to trigger the hack described later - case *rpc.Ui: - return 0 - - case *packer.MachineReadableUi: - // MachineReadableUi doesn't emit anything...like at all - return 0 - } - - log.Printf("Calculating the message prefix length for packer.Ui type (%T) is not implemented. Using the current aggregated length of %d.", ui, agg) - return agg - } - return recursiveCalculateUiPrefixLength(ui, 0) -} - -func GetPackerConfigFromStateBag(state multistep.StateBag) *PackerConfig { - config := state.Get("config") - rConfig := reflect.Indirect(reflect.ValueOf(config)) - iPackerConfig := rConfig.FieldByName("PackerConfig").Interface() - packerConfig := iPackerConfig.(PackerConfig) - return &packerConfig -} - -func GetProgressBar(ui packer.Ui, config *PackerConfig) ProgressBar { - // Figure out the prefix length by quering the UI - uiPrefixLength := calculateUiPrefixLength(ui) - - // hack to deal with packer.rpc.Ui courtesy of @Swampdragons - if _, ok := ui.(*rpc.Ui); uiPrefixLength == 0 && config != nil && ok { - res := fmt.Sprintf("%s %s: \n", targetedUIArrowText, config.PackerBuildName) - uiPrefixLength = len(res) - } - - // Now we can use the prefix length to calculate the progress bar width - width := calculateProgressBarWidth(uiPrefixLength) - - log.Printf("ProgressBar: Using progress bar width: %d\n", width) - - // Get a default progress bar and set some output defaults - bar := GetDefaultProgressBar() - bar.SetWidth(width) - bar.Callback = func(message string) { - ui.Message(message) - } - return bar -} diff --git a/common/progress_test.go b/common/progress_test.go deleted file mode 100644 index 028492f88..000000000 --- a/common/progress_test.go +++ /dev/null @@ -1,142 +0,0 @@ -package common - -import ( - "github.com/hashicorp/packer/packer" - "testing" -) - -// test packer.Ui implementation to verify that progress bar is being written -type testProgressBarUi struct { - messageCalled bool - messageMessage string -} - -func (u *testProgressBarUi) Say(string) {} -func (u *testProgressBarUi) Error(string) {} -func (u *testProgressBarUi) Machine(string, ...string) {} - -func (u *testProgressBarUi) Ask(string) (string, error) { - return "", nil -} -func (u *testProgressBarUi) Message(message string) { - u.messageCalled = true - u.messageMessage = message -} - -// ..and now let's begin our actual tests -func TestCalculateUiPrefixLength_Unknown(t *testing.T) { - ui := &testProgressBarUi{} - - expected := 0 - if res := calculateUiPrefixLength(ui); res != expected { - t.Fatalf("calculateUiPrefixLength should have returned a length of %d", expected) - } -} - -func TestCalculateUiPrefixLength_BasicUi(t *testing.T) { - ui := &packer.BasicUi{} - - expected := 1 - if res := calculateUiPrefixLength(ui); res != expected { - t.Fatalf("calculateUiPrefixLength should have returned a length of %d", expected) - } -} - -func TestCalculateUiPrefixLength_TargetedUI(t *testing.T) { - ui := &packer.TargetedUI{} - ui.Target = "TestTarget" - arrowText := "==>" - - expected := len(arrowText + " " + ui.Target + ": ") - if res := calculateUiPrefixLength(ui); res != expected { - t.Fatalf("calculateUiPrefixLength should have returned a length of %d", expected) - } -} - -func TestCalculateUiPrefixLength_TargetedUIWrappingBasicUi(t *testing.T) { - ui := &packer.TargetedUI{} - ui.Target = "TestTarget" - ui.Ui = &packer.BasicUi{} - arrowText := "==>" - - expected := len(arrowText + " " + ui.Target + ": " + "\n") - if res := calculateUiPrefixLength(ui); res != expected { - t.Fatalf("calculateUiPrefixLength should have returned a length of %d", expected) - } -} - -func TestCalculateUiPrefixLength_TargetedUIWrappingMachineUi(t *testing.T) { - ui := &packer.TargetedUI{} - ui.Target = "TestTarget" - ui.Ui = &packer.MachineReadableUi{} - - expected := 0 - if res := calculateUiPrefixLength(ui); res != expected { - t.Fatalf("calculateUiPrefixLength should have returned a length of %d", expected) - } -} -func TestDefaultProgressBar(t *testing.T) { - var callbackCalled bool - - // Initialize the default progress bar - bar := GetDefaultProgressBar() - bar.Callback = func(state string) { - callbackCalled = true - t.Logf("TestDefaultProgressBar emitted %#v", state) - } - bar.SetTotal64(1) - - // Set it off - progressBar := bar.Start() - progressBar.Set64(1) - - // Check to see that the callback was hit - if !callbackCalled { - t.Fatalf("TestDefaultProgressBar.Callback should be called") - } -} - -func TestDummyProgressBar(t *testing.T) { - var callbackCalled bool - - // Initialize the dummy progress bar - bar := GetDummyProgressBar() - bar.Callback = func(state string) { - callbackCalled = true - t.Logf("TestDummyProgressBar emitted %#v", state) - } - bar.SetTotal64(1) - - // Now we can go - progressBar := bar.Start() - progressBar.Set64(1) - - // Check to see that the callback was hit - if callbackCalled { - t.Fatalf("TestDummyProgressBar.Callback should not be called") - } -} - -func TestUiProgressBar(t *testing.T) { - - ui := &testProgressBarUi{} - - // Initialize the Ui progress bar - bar := GetProgressBar(ui, nil) - bar.SetTotal64(1) - - // Ensure that callback has been set to something - if bar.Callback == nil { - t.Fatalf("TestUiProgressBar.Callback should be initialized") - } - - // Now we can go - progressBar := bar.Start() - progressBar.Set64(1) - - // Check to see that the callback was hit - if !ui.messageCalled { - t.Fatalf("TestUiProgressBar.messageCalled should be called") - } - t.Logf("TestUiProgressBar emitted %#v", ui.messageMessage) -} diff --git a/common/step_download.go b/common/step_download.go index c55e12632..61f7baf85 100644 --- a/common/step_download.go +++ b/common/step_download.go @@ -63,9 +63,6 @@ func (s *StepDownload) Run(_ context.Context, state multistep.StateBag) multiste ui.Say(fmt.Sprintf("Retrieving %s", s.Description)) - // Get a progress bar from the ui so we can hand it off to the download client - bar := GetProgressBar(ui, GetPackerConfigFromStateBag(state)) - // First try to use any already downloaded file // If it fails, proceed to regular download logic @@ -99,7 +96,7 @@ func (s *StepDownload) Run(_ context.Context, state multistep.StateBag) multiste } downloadConfigs[i] = config - if match, _ := NewDownloadClient(config, bar).VerifyChecksum(config.TargetPath); match { + if match, _ := NewDownloadClient(config, ui).VerifyChecksum(config.TargetPath); match { ui.Message(fmt.Sprintf("Found already downloaded, initial checksum matched, no download needed: %s", url)) finalPath = config.TargetPath break @@ -143,11 +140,8 @@ func (s *StepDownload) download(config *DownloadConfig, state multistep.StateBag var path string ui := state.Get("ui").(packer.Ui) - // Get a progress bar and hand it off to the download client - bar := GetProgressBar(ui, GetPackerConfigFromStateBag(state)) - - // Create download client with config and progress bar - download := NewDownloadClient(config, bar) + // Create download client with config + download := NewDownloadClient(config, ui) downloadCompleteCh := make(chan error, 1) go func() { @@ -159,7 +153,6 @@ func (s *StepDownload) download(config *DownloadConfig, state multistep.StateBag for { select { case err := <-downloadCompleteCh: - bar.Finish() if err != nil { return "", err, true @@ -174,7 +167,6 @@ func (s *StepDownload) download(config *DownloadConfig, state multistep.StateBag case <-time.After(1 * time.Second): if _, ok := state.GetOk(multistep.StateCancelled); ok { - bar.Finish() ui.Say("Interrupt received. Cancelling download...") return "", nil, false } diff --git a/packer/progressbar.go b/packer/progressbar.go new file mode 100644 index 000000000..e2b229d07 --- /dev/null +++ b/packer/progressbar.go @@ -0,0 +1,147 @@ +package packer + +import ( + "fmt" + "io" + "sync" + "sync/atomic" + + "github.com/cheggaaa/pb" +) + +// ProgressBar allows to graphically display +// a self refreshing progress bar. +type ProgressBar interface { + Start(total int64) + Add(current int64) + NewProxyReader(r io.Reader) (proxy io.Reader) + Finish() +} + +// StackableProgressBar is a progress bar that +// allows to track multiple downloads at once. +// Every call to Start increments a counter that +// will display the number of current loadings. +// Every call to Start will add total to an internal +// total that is the total displayed. +// First call to Start will start a goroutine +// that is waiting for every download to be finished. +// Last call to Finish triggers a cleanup. +// When all active downloads are finished +// StackableProgressBar will clean itself to a default +// state. +type StackableProgressBar struct { + mtx sync.Mutex // locks in Start & Finish + BasicProgressBar + items int32 + total int64 + + started bool +} + +var _ ProgressBar = new(StackableProgressBar) + +func (spb *StackableProgressBar) start() { + spb.BasicProgressBar.ProgressBar = pb.New(0) + spb.BasicProgressBar.ProgressBar.SetUnits(pb.U_BYTES) + + spb.BasicProgressBar.ProgressBar.Start() + spb.started = true +} + +func (spb *StackableProgressBar) Start(total int64) { + spb.mtx.Lock() + + spb.total += total + spb.items++ + + if !spb.started { + spb.start() + } + spb.SetTotal64(spb.total) + spb.prefix() + spb.mtx.Unlock() +} + +func (spb *StackableProgressBar) prefix() { + spb.BasicProgressBar.ProgressBar.Prefix(fmt.Sprintf("%d items: ", atomic.LoadInt32(&spb.items))) +} + +func (spb *StackableProgressBar) Finish() { + spb.mtx.Lock() + + spb.items-- + if spb.items == 0 { + // slef cleanup + spb.BasicProgressBar.ProgressBar.Finish() + spb.BasicProgressBar.ProgressBar = nil + spb.started = false + spb.total = 0 + return + } + spb.prefix() + spb.mtx.Unlock() +} + +// BasicProgressBar is packer's basic progress bar. +// Current implementation will always try to keep +// itself at the bottom of a terminal. +type BasicProgressBar struct { + *pb.ProgressBar +} + +var _ ProgressBar = new(BasicProgressBar) + +func (bpb *BasicProgressBar) Start(total int64) { + bpb.SetTotal64(total) + bpb.ProgressBar.Start() +} + +func (bpb *BasicProgressBar) Add(current int64) { + bpb.ProgressBar.Add64(current) +} +func (bpb *BasicProgressBar) NewProxyReader(r io.Reader) io.Reader { + return &ProxyReader{ + Reader: r, + ProgressBar: bpb, + } +} +func (bpb *BasicProgressBar) NewProxyReadCloser(r io.ReadCloser) io.ReadCloser { + return &ProxyReader{ + Reader: r, + ProgressBar: bpb, + } +} + +// NoopProgressBar is a silent progress bar. +type NoopProgressBar struct { +} + +var _ ProgressBar = new(NoopProgressBar) + +func (npb *NoopProgressBar) Start(int64) {} +func (npb *NoopProgressBar) Add(int64) {} +func (npb *NoopProgressBar) Finish() {} +func (npb *NoopProgressBar) NewProxyReader(r io.Reader) io.Reader { return r } +func (npb *NoopProgressBar) NewProxyReadCloser(r io.ReadCloser) io.ReadCloser { return r } + +// ProxyReader implements io.ReadCloser but sends +// count of read bytes to a progress bar +type ProxyReader struct { + io.Reader + ProgressBar +} + +func (r *ProxyReader) Read(p []byte) (n int, err error) { + n, err = r.Reader.Read(p) + r.ProgressBar.Add(int64(n)) + return +} + +// Close the reader if it implements io.Closer +func (r *ProxyReader) Close() (err error) { + if closer, ok := r.Reader.(io.Closer); ok { + return closer.Close() + } + return +} diff --git a/packer/rpc/server.go b/packer/rpc/server.go index 8982ec0e7..dff8854e2 100644 --- a/packer/rpc/server.go +++ b/packer/rpc/server.go @@ -114,7 +114,8 @@ func (s *Server) RegisterProvisioner(p packer.Provisioner) { func (s *Server) RegisterUi(ui packer.Ui) { s.server.RegisterName(DefaultUiEndpoint, &UiServer{ - ui: ui, + ui: ui, + register: s.server.RegisterName, }) } diff --git a/packer/rpc/ui.go b/packer/rpc/ui.go index 1c6356a65..c9d3ef176 100644 --- a/packer/rpc/ui.go +++ b/packer/rpc/ui.go @@ -1,6 +1,7 @@ package rpc import ( + "io" "log" "net/rpc" @@ -14,10 +15,13 @@ type Ui struct { endpoint string } +var _ packer.Ui = new(Ui) + // UiServer wraps a packer.Ui implementation and makes it exportable // as part of a Golang RPC server. type UiServer struct { - ui packer.Ui + ui packer.Ui + register func(name string, rcvr interface{}) error } // The arguments sent to Ui.Machine @@ -60,6 +64,31 @@ func (u *Ui) Say(message string) { } } +func (u *Ui) ProgressBar() packer.ProgressBar { + if err := u.client.Call("Ui.ProgressBar", new(interface{}), new(interface{})); err != nil { + log.Printf("Error in Ui RPC call: %s", err) + } + return u // Ui is also a progress bar !! +} + +var _ packer.ProgressBar = new(Ui) + +func (pb *Ui) Start(total int64) { + pb.client.Call("Ui.Start", total, new(interface{})) +} + +func (pb *Ui) Add(current int64) { + pb.client.Call("Ui.Add", current, new(interface{})) +} + +func (pb *Ui) Finish() { + pb.client.Call("Ui.Finish", nil, new(interface{})) +} + +func (pb *Ui) NewProxyReader(r io.Reader) io.Reader { + return &packer.ProxyReader{Reader: r, ProgressBar: pb} +} + func (u *UiServer) Ask(query string, reply *string) (err error) { *reply, err = u.ui.Ask(query) return @@ -91,3 +120,26 @@ func (u *UiServer) Say(message *string, reply *interface{}) error { *reply = nil return nil } + +func (u *UiServer) ProgressBar(_ *string, reply *interface{}) error { + // No-op for now, this function might be + // used in the future if we want to use + // different progress bars with identifiers. + u.ui.ProgressBar() + return nil +} + +func (pb *UiServer) Finish(_ string, _ *interface{}) error { + pb.ui.ProgressBar().Finish() + return nil +} + +func (pb *UiServer) Start(total int64, _ *interface{}) error { + pb.ui.ProgressBar().Start(total) + return nil +} + +func (pb *UiServer) Add(current int64, _ *interface{}) error { + pb.ui.ProgressBar().Add(current) + return nil +} diff --git a/packer/rpc/ui_test.go b/packer/rpc/ui_test.go index 4dd2d7036..078b85e44 100644 --- a/packer/rpc/ui_test.go +++ b/packer/rpc/ui_test.go @@ -1,8 +1,11 @@ package rpc import ( + "io" "reflect" "testing" + + "github.com/hashicorp/packer/packer" ) type testUi struct { @@ -17,6 +20,12 @@ type testUi struct { messageMessage string sayCalled bool sayMessage string + + progressBarCalled bool + progressBarStartCalled bool + progressBarAddCalled bool + progressBarFinishCalled bool + progressBarNewProxyReaderCalled bool } func (u *testUi) Ask(query string) (string, error) { @@ -46,6 +55,28 @@ func (u *testUi) Say(message string) { u.sayMessage = message } +func (u *testUi) ProgressBar() packer.ProgressBar { + u.progressBarCalled = true + return u +} + +func (u *testUi) Start(int64) { + u.progressBarStartCalled = true +} + +func (u *testUi) Add(int64) { + u.progressBarAddCalled = true +} + +func (u *testUi) Finish() { + u.progressBarFinishCalled = true +} + +func (u *testUi) NewProxyReader(r io.Reader) io.Reader { + u.progressBarNewProxyReaderCalled = true + return r +} + func TestUiRPC(t *testing.T) { // Create the UI to test ui := new(testUi) @@ -88,6 +119,26 @@ func TestUiRPC(t *testing.T) { t.Fatalf("bad: %#v", ui.errorMessage) } + bar := uiClient.ProgressBar() + if ui.progressBarCalled != true { + t.Errorf("ProgressBar not called.") + } + + bar.Start(100) + if ui.progressBarStartCalled != true { + t.Errorf("progressBar.Start not called.") + } + + bar.Add(1) + if ui.progressBarAddCalled != true { + t.Errorf("progressBar.Add not called.") + } + + bar.Finish() + if ui.progressBarFinishCalled != true { + t.Errorf("progressBar.Finish not called.") + } + uiClient.Machine("foo", "bar", "baz") if !ui.machineCalled { t.Fatal("machine should be called") diff --git a/packer/ui.go b/packer/ui.go index c16a2eae0..340c56827 100644 --- a/packer/ui.go +++ b/packer/ui.go @@ -37,8 +37,20 @@ type Ui interface { Message(string) Error(string) Machine(string, ...string) + ProgressBar() ProgressBar } +type NoopUi struct{} + +var _ Ui = new(NoopUi) + +func (*NoopUi) Ask(string) (string, error) { return "", errors.New("this is a noop ui") } +func (*NoopUi) Say(string) { return } +func (*NoopUi) Message(string) { return } +func (*NoopUi) Error(string) { return } +func (*NoopUi) Machine(string, ...string) { return } +func (*NoopUi) ProgressBar() ProgressBar { return new(NoopProgressBar) } + // ColoredUi is a UI that is colored using terminal colors. type ColoredUi struct { Color UiColor @@ -46,6 +58,8 @@ type ColoredUi struct { Ui Ui } +var _ Ui = new(ColoredUi) + // TargetedUI is a UI that wraps another UI implementation and modifies // the output to indicate a specific target. Specifically, all Say output // is prefixed with the target name. Message output is not prefixed but @@ -56,6 +70,8 @@ type TargetedUI struct { Ui Ui } +var _ Ui = new(TargetedUI) + // The BasicUI is a UI that reads and writes from a standard Go reader // and writer. It is safe to be called from multiple goroutines. Machine // readable output is simply logged for this UI. @@ -66,6 +82,13 @@ type BasicUi struct { l sync.Mutex interrupted bool scanner *bufio.Scanner + StackableProgressBar +} + +var _ Ui = new(BasicUi) + +func (bu *BasicUi) ProgressBar() ProgressBar { + return &bu.StackableProgressBar } // MachineReadableUi is a UI that only outputs machine-readable output @@ -74,6 +97,8 @@ type MachineReadableUi struct { Writer io.Writer } +var _ Ui = new(MachineReadableUi) + func (u *ColoredUi) Ask(query string) (string, error) { return u.Ui.Ask(u.colorize(query, u.Color, true)) } @@ -100,6 +125,10 @@ func (u *ColoredUi) Machine(t string, args ...string) { u.Ui.Machine(t, args...) } +func (u *ColoredUi) ProgressBar() ProgressBar { + return u.Ui.ProgressBar() //TODO(adrien): color me +} + func (u *ColoredUi) colorize(message string, color UiColor, bold bool) string { if !u.supportsColors() { return message @@ -153,6 +182,10 @@ func (u *TargetedUI) Machine(t string, args ...string) { u.Ui.Machine(fmt.Sprintf("%s,%s", u.Target, t), args...) } +func (u *TargetedUI) ProgressBar() ProgressBar { + return u.Ui.ProgressBar() +} + func (u *TargetedUI) prefixLines(arrow bool, message string) string { arrowText := "==>" if !arrow { @@ -305,3 +338,7 @@ func (u *MachineReadableUi) Machine(category string, args ...string) { } } } + +func (u *MachineReadableUi) ProgressBar() ProgressBar { + return new(NoopProgressBar) +} diff --git a/provisioner/ansible-local/provisioner_test.go b/provisioner/ansible-local/provisioner_test.go index e72c6c01a..741bba235 100644 --- a/provisioner/ansible-local/provisioner_test.go +++ b/provisioner/ansible-local/provisioner_test.go @@ -8,11 +8,12 @@ import ( "testing" "fmt" + "os/exec" + "github.com/hashicorp/packer/builder/docker" "github.com/hashicorp/packer/packer" "github.com/hashicorp/packer/provisioner/file" "github.com/hashicorp/packer/template" - "os/exec" ) func TestProvisioner_Impl(t *testing.T) { @@ -132,7 +133,7 @@ func TestProvisionerProvision_PlaybookFiles(t *testing.T) { } comm := &communicatorMock{} - if err := p.Provision(&uiStub{}, comm); err != nil { + if err := p.Provision(new(packer.NoopUi), comm); err != nil { t.Fatalf("err: %s", err) } @@ -166,7 +167,7 @@ func TestProvisionerProvision_PlaybookFilesWithPlaybookDir(t *testing.T) { } comm := &communicatorMock{} - if err := p.Provision(&uiStub{}, comm); err != nil { + if err := p.Provision(new(packer.NoopUi), comm); err != nil { t.Fatalf("err: %s", err) } diff --git a/provisioner/ansible-local/ui_stub.go b/provisioner/ansible-local/ui_stub.go deleted file mode 100644 index 4faa2a215..000000000 --- a/provisioner/ansible-local/ui_stub.go +++ /dev/null @@ -1,15 +0,0 @@ -package ansiblelocal - -type uiStub struct{} - -func (su *uiStub) Ask(string) (string, error) { - return "", nil -} - -func (su *uiStub) Error(string) {} - -func (su *uiStub) Machine(string, ...string) {} - -func (su *uiStub) Message(string) {} - -func (su *uiStub) Say(msg string) {} diff --git a/provisioner/ansible/adapter_test.go b/provisioner/ansible/adapter_test.go index 2ab2d350b..29667cbe2 100644 --- a/provisioner/ansible/adapter_test.go +++ b/provisioner/ansible/adapter_test.go @@ -24,7 +24,7 @@ func TestAdapter_Serve(t *testing.T) { config := &ssh.ServerConfig{} - ui := new(ui) + ui := new(packer.NoopUi) sut := newAdapter(done, &l, config, "", newUi(ui), communicator{}) go func() { @@ -93,36 +93,6 @@ func (a addr) String() string { return "test" } -type ui int - -func (u *ui) Ask(s string) (string, error) { - *u++ - return s, nil -} - -func (u *ui) Say(s string) { - *u++ - log.Println(s) -} - -func (u *ui) Message(s string) { - *u++ - log.Println(s) -} - -func (u *ui) Error(s string) { - *u++ - log.Println(s) -} - -func (u *ui) Machine(s1 string, s2 ...string) { - *u++ - log.Println(s1) - for _, s := range s2 { - log.Println(s) - } -} - type communicator struct{} func (c communicator) Start(*packer.RemoteCmd) error { diff --git a/provisioner/ansible/provisioner.go b/provisioner/ansible/provisioner.go index 574ec2b39..d2528218d 100644 --- a/provisioner/ansible/provisioner.go +++ b/provisioner/ansible/provisioner.go @@ -612,3 +612,7 @@ func (ui *Ui) Machine(t string, args ...string) { ui.ui.Machine(t, args...) <-ui.sem } + +func (ui *Ui) ProgressBar() packer.ProgressBar { + return new(packer.NoopProgressBar) +} diff --git a/provisioner/file/provisioner.go b/provisioner/file/provisioner.go index 13e2aa8e6..1be9f1885 100644 --- a/provisioner/file/provisioner.go +++ b/provisioner/file/provisioner.go @@ -127,12 +127,12 @@ func (p *Provisioner) ProvisionDownload(ui packer.Ui, comm packer.Communicator) defer f.Close() // Get a default progress bar - pb := common.GetProgressBar(ui, &p.config.PackerConfig) - bar := pb.Start() - defer bar.Finish() + pb := packer.NoopProgressBar{} + pb.Start(0) // TODO: find size ? Remove ? + defer pb.Finish() // Create MultiWriter for the current progress - pf := io.MultiWriter(f, bar) + pf := io.MultiWriter(f) // Download the file if err = comm.Download(src, pf); err != nil { @@ -176,8 +176,8 @@ func (p *Provisioner) ProvisionUpload(ui packer.Ui, comm packer.Communicator) er } // Get a default progress bar - pb := common.GetProgressBar(ui, &p.config.PackerConfig) - bar := pb.Start() + bar := ui.ProgressBar() + bar.Start(info.Size()) defer bar.Finish() // Create ProxyReader for the current progress diff --git a/provisioner/file/provisioner_test.go b/provisioner/file/provisioner_test.go index 175082aae..732bba3ff 100644 --- a/provisioner/file/provisioner_test.go +++ b/provisioner/file/provisioner_test.go @@ -1,6 +1,7 @@ package file import ( + "bytes" "io/ioutil" "os" "path/filepath" @@ -99,27 +100,6 @@ func TestProvisionerPrepare_EmptyDestination(t *testing.T) { } } -type stubUi struct { - sayMessages string -} - -func (su *stubUi) Ask(string) (string, error) { - return "", nil -} - -func (su *stubUi) Error(string) { -} - -func (su *stubUi) Machine(string, ...string) { -} - -func (su *stubUi) Message(string) { -} - -func (su *stubUi) Say(msg string) { - su.sayMessages += msg -} - func TestProvisionerProvision_SendsFile(t *testing.T) { var p Provisioner tf, err := ioutil.TempFile("", "packer") @@ -141,18 +121,21 @@ func TestProvisionerProvision_SendsFile(t *testing.T) { t.Fatalf("err: %s", err) } - ui := &stubUi{} + b := bytes.NewBuffer(nil) + ui := &packer.BasicUi{ + Writer: b, + } comm := &packer.MockCommunicator{} err = p.Provision(ui, comm) if err != nil { t.Fatalf("should successfully provision: %s", err) } - if !strings.Contains(ui.sayMessages, tf.Name()) { + if !strings.Contains(b.String(), tf.Name()) { t.Fatalf("should print source filename") } - if !strings.Contains(ui.sayMessages, "something") { + if !strings.Contains(b.String(), "something") { t.Fatalf("should print destination filename") } @@ -197,18 +180,21 @@ func TestProvisionDownloadMkdirAll(t *testing.T) { if err := p.Prepare(config); err != nil { t.Fatalf("err: %s", err) } - ui := &stubUi{} + b := bytes.NewBuffer(nil) + ui := &packer.BasicUi{ + Writer: b, + } comm := &packer.MockCommunicator{} err = p.ProvisionDownload(ui, comm) if err != nil { t.Fatalf("should successfully provision: %s", err) } - if !strings.Contains(ui.sayMessages, tf.Name()) { + if !strings.Contains(b.String(), tf.Name()) { t.Fatalf("should print source filename") } - if !strings.Contains(ui.sayMessages, "something") { + if !strings.Contains(b.String(), "something") { t.Fatalf("should print destination filename") }