From 1681c410ddbfac5f4dfb755aa6c21ea3631b2c26 Mon Sep 17 00:00:00 2001 From: Yury Semikhatsky Date: Tue, 12 Apr 2022 08:33:40 -0700 Subject: [PATCH] feat: large file uploads (#891) --- .../playwright/impl/BrowserContextImpl.java | 7 ++ .../microsoft/playwright/impl/Connection.java | 3 + .../playwright/impl/ElementHandleImpl.java | 25 ++++++- .../playwright/impl/FileChooserImpl.java | 5 +- .../microsoft/playwright/impl/FrameImpl.java | 22 +++++-- .../playwright/impl/Serialization.java | 8 +++ .../com/microsoft/playwright/impl/Utils.java | 48 ++++++++++++++ .../playwright/impl/WritableStream.java | 39 +++++++++++ .../playwright/MultipartFormData.java | 66 +++++++++++++++++++ .../playwright/TestBrowserTypeConnect.java | 42 ++++++++++++ .../playwright/TestPageSetInputFiles.java | 47 +++++++++++++ .../src/test/resources/input/fileupload.html | 4 +- 12 files changed, 304 insertions(+), 12 deletions(-) create mode 100644 playwright/src/main/java/com/microsoft/playwright/impl/WritableStream.java create mode 100644 playwright/src/test/java/com/microsoft/playwright/MultipartFormData.java diff --git a/playwright/src/main/java/com/microsoft/playwright/impl/BrowserContextImpl.java b/playwright/src/main/java/com/microsoft/playwright/impl/BrowserContextImpl.java index 0da52c78..79d6e6c6 100644 --- a/playwright/src/main/java/com/microsoft/playwright/impl/BrowserContextImpl.java +++ b/playwright/src/main/java/com/microsoft/playwright/impl/BrowserContextImpl.java @@ -560,4 +560,11 @@ class BrowserContextImpl extends ChannelOwner implements BrowserContext { } listeners.notify(EventType.CLOSE, this); } + + WritableStream createTempFile(String name) { + JsonObject params = new JsonObject(); + params.addProperty("name", name); + JsonObject json = sendMessage("createTempFile", params).getAsJsonObject(); + return connection.getExistingObject(json.getAsJsonObject("writableStream").get("guid").getAsString()); + } } diff --git a/playwright/src/main/java/com/microsoft/playwright/impl/Connection.java b/playwright/src/main/java/com/microsoft/playwright/impl/Connection.java index d7aef088..4dd2bfad 100644 --- a/playwright/src/main/java/com/microsoft/playwright/impl/Connection.java +++ b/playwright/src/main/java/com/microsoft/playwright/impl/Connection.java @@ -302,6 +302,9 @@ public class Connection { case "Worker": result = new WorkerImpl(parent, type, guid, initializer); break; + case "WritableStream": + result = new WritableStream(parent, type, guid, initializer); + break; default: throw new PlaywrightException("Unknown type " + type); } diff --git a/playwright/src/main/java/com/microsoft/playwright/impl/ElementHandleImpl.java b/playwright/src/main/java/com/microsoft/playwright/impl/ElementHandleImpl.java index fc9449c9..95b131ee 100644 --- a/playwright/src/main/java/com/microsoft/playwright/impl/ElementHandleImpl.java +++ b/playwright/src/main/java/com/microsoft/playwright/impl/ElementHandleImpl.java @@ -33,7 +33,8 @@ import java.util.Base64; import java.util.List; import static com.microsoft.playwright.impl.Serialization.*; -import static com.microsoft.playwright.impl.Utils.convertType; +import static com.microsoft.playwright.impl.Utils.*; +import static com.microsoft.playwright.impl.Utils.addLargeFileUploadParams; import static com.microsoft.playwright.options.ScreenshotType.JPEG; import static com.microsoft.playwright.options.ScreenshotType.PNG; @@ -299,7 +300,7 @@ public class ElementHandleImpl extends JSHandleImpl implements ElementHandle { } @Override - public Frame ownerFrame() { + public FrameImpl ownerFrame() { return withLogging("ElementHandle.ownerFrame", () -> { JsonObject json = sendMessage("ownerFrame").getAsJsonObject(); if (!json.has("frame")) { @@ -455,7 +456,24 @@ public class ElementHandleImpl extends JSHandleImpl implements ElementHandle { @Override public void setInputFiles(Path[] files, SetInputFilesOptions options) { - setInputFiles(Utils.toFilePayloads(files), options); + withLogging("ElementHandle.setInputFiles", () -> setInputFilesImpl(files, options)); + } + + void setInputFilesImpl(Path[] files, SetInputFilesOptions options) { + FrameImpl frame = ownerFrame(); + if (frame == null) { + throw new Error("Cannot set input files to detached element"); + } + if (hasLargeFile(files)) { + if (options == null) { + options = new SetInputFilesOptions(); + } + JsonObject params = gson().toJsonTree(options).getAsJsonObject(); + addLargeFileUploadParams(files, params, frame.page().context()); + sendMessage("setInputFilePaths", params); + } else { + setInputFilesImpl(Utils.toFilePayloads(files), options); + } } @Override @@ -469,6 +487,7 @@ public class ElementHandleImpl extends JSHandleImpl implements ElementHandle { } void setInputFilesImpl(FilePayload[] files, SetInputFilesOptions options) { + checkFilePayloadSize(files); if (options == null) { options = new SetInputFilesOptions(); } diff --git a/playwright/src/main/java/com/microsoft/playwright/impl/FileChooserImpl.java b/playwright/src/main/java/com/microsoft/playwright/impl/FileChooserImpl.java index 0d5f06e6..ddfb66ea 100644 --- a/playwright/src/main/java/com/microsoft/playwright/impl/FileChooserImpl.java +++ b/playwright/src/main/java/com/microsoft/playwright/impl/FileChooserImpl.java @@ -58,7 +58,8 @@ class FileChooserImpl implements FileChooser { @Override public void setFiles(Path[] files, SetFilesOptions options) { - setFiles(Utils.toFilePayloads(files), options); + page.withLogging("FileChooser.setInputFiles", + () -> element.setInputFilesImpl(files, convertType(options, ElementHandle.SetInputFilesOptions.class))); } @Override @@ -69,6 +70,6 @@ class FileChooserImpl implements FileChooser { @Override public void setFiles(FilePayload[] files, SetFilesOptions options) { page.withLogging("FileChooser.setInputFiles", - () -> element.setInputFilesImpl(files, convertType(options, ElementHandle.SetInputFilesOptions.class))); + () -> element.setInputFilesImpl(files, convertType(options, ElementHandle.SetInputFilesOptions.class))); } } diff --git a/playwright/src/main/java/com/microsoft/playwright/impl/FrameImpl.java b/playwright/src/main/java/com/microsoft/playwright/impl/FrameImpl.java index a6485605..047be9ca 100644 --- a/playwright/src/main/java/com/microsoft/playwright/impl/FrameImpl.java +++ b/playwright/src/main/java/com/microsoft/playwright/impl/FrameImpl.java @@ -23,6 +23,7 @@ import com.microsoft.playwright.*; import com.microsoft.playwright.options.*; import java.io.IOException; +import java.io.OutputStream; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; @@ -31,7 +32,7 @@ import java.util.function.Consumer; import java.util.function.Predicate; import java.util.regex.Pattern; -import static com.microsoft.playwright.impl.Utils.convertType; +import static com.microsoft.playwright.impl.Utils.*; import static com.microsoft.playwright.options.WaitUntilState.*; import static com.microsoft.playwright.impl.Serialization.*; @@ -686,21 +687,32 @@ public class FrameImpl extends ChannelOwner implements Frame { withLogging("Frame.setInputFiles", () -> setInputFilesImpl(selector, files, options)); } + void setInputFilesImpl(String selector, Path[] files, SetInputFilesOptions options) { + if (hasLargeFile(files)) { + if (options == null) { + options = new SetInputFilesOptions(); + } + JsonObject params = gson().toJsonTree(options).getAsJsonObject(); + addLargeFileUploadParams(files, params, page.context()); + params.addProperty("selector", selector); + sendMessage("setInputFilePaths", params); + } else { + setInputFilesImpl(selector, Utils.toFilePayloads(files), options); + } + } + @Override public void setInputFiles(String selector, FilePayload files, SetInputFilesOptions options) { setInputFiles(selector, new FilePayload[]{files}, options); } - void setInputFilesImpl(String selector, Path[] files, SetInputFilesOptions options) { - setInputFiles(selector, Utils.toFilePayloads(files), options); - } - @Override public void setInputFiles(String selector, FilePayload[] files, SetInputFilesOptions options) { withLogging("Frame.setInputFiles", () -> setInputFilesImpl(selector, files, options)); } void setInputFilesImpl(String selector, FilePayload[] files, SetInputFilesOptions options) { + checkFilePayloadSize(files); if (options == null) { options = new SetInputFilesOptions(); } diff --git a/playwright/src/main/java/com/microsoft/playwright/impl/Serialization.java b/playwright/src/main/java/com/microsoft/playwright/impl/Serialization.java index 36078a9a..c1d47220 100644 --- a/playwright/src/main/java/com/microsoft/playwright/impl/Serialization.java +++ b/playwright/src/main/java/com/microsoft/playwright/impl/Serialization.java @@ -212,6 +212,14 @@ class Serialization { } } + static JsonArray toJsonArray(Path[] files) { + JsonArray jsonFiles = new JsonArray(); + for (Path p : files) { + jsonFiles.add(p.toString()); + } + return jsonFiles; + } + static JsonArray toJsonArray(FilePayload[] files) { JsonArray jsonFiles = new JsonArray(); for (FilePayload p : files) { diff --git a/playwright/src/main/java/com/microsoft/playwright/impl/Utils.java b/playwright/src/main/java/com/microsoft/playwright/impl/Utils.java index b961702d..f4aa518a 100644 --- a/playwright/src/main/java/com/microsoft/playwright/impl/Utils.java +++ b/playwright/src/main/java/com/microsoft/playwright/impl/Utils.java @@ -16,6 +16,8 @@ package com.microsoft.playwright.impl; +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; import com.microsoft.playwright.PlaywrightException; import com.microsoft.playwright.options.FilePayload; import com.microsoft.playwright.options.HttpHeader; @@ -23,6 +25,7 @@ import com.microsoft.playwright.options.HttpHeader; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; import java.lang.reflect.Field; import java.lang.reflect.Modifier; import java.nio.file.Files; @@ -30,6 +33,8 @@ import java.nio.file.Path; import java.util.*; import java.util.regex.Pattern; +import static com.microsoft.playwright.impl.Serialization.toJsonArray; + class Utils { static T convertType(F f, Class t) { if (f == null) { @@ -144,6 +149,49 @@ class Utils { return mimeType; } + static final int maxUplodBufferSize = 50 * 1024 * 1024; + + static boolean hasLargeFile(Path[] files) { + for (Path file: files) { + try { + if (Files.size(file)> maxUplodBufferSize) { + return true; + } + } catch (IOException e) { + throw new PlaywrightException("Cannot get file size.", e); + } + } + return false; + } + + static void addLargeFileUploadParams(Path[] files, JsonObject params, BrowserContextImpl context) { + if (context.browser().isRemote) { + List streams = new ArrayList<>(); + JsonArray jsonStreams = new JsonArray(); + for (Path path : files) { + WritableStream temp = context.createTempFile(path.getFileName().toString()); + streams.add(temp); + try (OutputStream out = temp.stream()) { + Files.copy(path, out); + } catch (IOException e) { + throw new PlaywrightException("Failed to copy file to remote server.", e); + } + jsonStreams.add(temp.toProtocol()); + } + params.add("streams", jsonStreams); + } else { + params.add("localPaths", toJsonArray(files)); + } + } + + static void checkFilePayloadSize(FilePayload[] files) { + for (FilePayload file: files) { + if (file.buffer.length > maxUplodBufferSize) { + throw new PlaywrightException("Cannot set buffer larger than 50Mb, please write it to a file and pass its path instead."); + } + } + } + static FilePayload[] toFilePayloads(Path[] files) { List payloads = new ArrayList<>(); for (Path file : files) { diff --git a/playwright/src/main/java/com/microsoft/playwright/impl/WritableStream.java b/playwright/src/main/java/com/microsoft/playwright/impl/WritableStream.java new file mode 100644 index 00000000..e67229c6 --- /dev/null +++ b/playwright/src/main/java/com/microsoft/playwright/impl/WritableStream.java @@ -0,0 +1,39 @@ +package com.microsoft.playwright.impl; + +import com.google.gson.JsonObject; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Base64; + +class WritableStream extends ChannelOwner { + WritableStream(ChannelOwner parent, String type, String guid, JsonObject initializer) { + super(parent, type, guid, initializer); + } + + OutputStream stream() { + return new OutputStream() { + @Override + public void write(int b) throws IOException { + write(new byte[] { (byte) b }); + } + + @Override + public void write(byte[] b, int off, int len) throws IOException { + JsonObject params = new JsonObject(); + ByteBuffer buffer = ByteBuffer.wrap(b, off, len); + ByteBuffer encoded = Base64.getEncoder().encode(buffer); + params.addProperty("binary", new String(encoded.array(), StandardCharsets.UTF_8)); + sendMessage("write", params); + } + }; + } + + JsonObject toProtocol() { + JsonObject json = new JsonObject(); + json.addProperty("guid", guid); + return json; + } +} diff --git a/playwright/src/test/java/com/microsoft/playwright/MultipartFormData.java b/playwright/src/test/java/com/microsoft/playwright/MultipartFormData.java new file mode 100644 index 00000000..6b3a2932 --- /dev/null +++ b/playwright/src/test/java/com/microsoft/playwright/MultipartFormData.java @@ -0,0 +1,66 @@ +package com.microsoft.playwright; + +import com.sun.net.httpserver.HttpExchange; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class MultipartFormData { + static MultipartFormData parseRequest(HttpExchange exchange) throws IOException { + ByteArrayOutputStream bodyBytes = new ByteArrayOutputStream(); + try (OutputStream output = bodyBytes) { + Utils.copy(exchange.getRequestBody(), output); + } + String body = new String(bodyBytes.toByteArray(), StandardCharsets.UTF_8); + String contentType = exchange.getRequestHeaders().get("content-type").get(0); + Matcher matcher = Pattern.compile("boundary=(.*)$").matcher(contentType); + if (!matcher.find()) { + throw new RuntimeException("Boundary not found!"); + } + String boundary = matcher.group(1); + return new MultipartFormData(body, boundary); + } + + static class Field { + final String filename; + final String content; + + Field(String filename, String content) { + this.filename = filename; + this.content = content; + } + } + + final List fields = new ArrayList<>(); + + MultipartFormData(String body, String boundary) { + String[] parts = Pattern.compile("--" + boundary + "(--)?\r\n", Pattern.MULTILINE).split(body); + for (String part : parts) { + if (part.trim().length() == 0) { + continue; + } + String[] headersAndContent = Pattern.compile("\r\n\r\n", Pattern.MULTILINE).split(part); + if (headersAndContent.length != 2) { + throw new RuntimeException("Unexpected format: " + part); + } + String headers = headersAndContent[0]; + String filename = null; + for (String header: Pattern.compile("\r\n", Pattern.MULTILINE).split(headers)) { + Matcher matcher = Pattern.compile("content-disposition: .*filename=\"([^\"]+)\"", Pattern.CASE_INSENSITIVE).matcher(header); + if (!matcher.find()) { + continue; + } + filename = matcher.group(1); + } + String content = headersAndContent[1]; + content = content.substring(0, content.length() - "\r\n".length()); + fields.add(new Field(filename, content)); + } + } +} diff --git a/playwright/src/test/java/com/microsoft/playwright/TestBrowserTypeConnect.java b/playwright/src/test/java/com/microsoft/playwright/TestBrowserTypeConnect.java index 96602443..af5ac0d7 100644 --- a/playwright/src/test/java/com/microsoft/playwright/TestBrowserTypeConnect.java +++ b/playwright/src/test/java/com/microsoft/playwright/TestBrowserTypeConnect.java @@ -30,6 +30,8 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.*; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; import static com.microsoft.playwright.Utils.*; @@ -518,4 +520,44 @@ public class TestBrowserTypeConnect extends TestBase { assertEquals(200, response.status()); assertEquals("{\"foo\": \"bar\"}\n", response.text()); } + + @Test + void shouldUploadLargeFile(@TempDir Path tmpDir) throws IOException, ExecutionException, InterruptedException { + Assumptions.assumeTrue(3 <= (Runtime.getRuntime().maxMemory() >> 30), "Fails if max heap size is < 3Gb"); + page.navigate(server.PREFIX + "/input/fileupload.html"); + Path uploadFile = tmpDir.resolve("200MB.zip"); + String str = String.join("", Collections.nCopies(4 * 1024, "A")); + + try (Writer stream = new OutputStreamWriter(Files.newOutputStream(uploadFile))) { + for (int i = 0; i < 50 * 1024; i++) { + stream.write(str); + } + } + Locator input = page.locator("input[type='file']"); + JSHandle events = input.evaluateHandle("e => {\n" + + " const events = [];\n" + + " e.addEventListener('input', () => events.push('input'));\n" + + " e.addEventListener('change', () => events.push('change'));\n" + + " return events;\n" + + " }"); + input.setInputFiles(uploadFile); + assertEquals("200MB.zip", input.evaluate("e => e.files[0].name")); + assertEquals(asList("input", "change"), events.evaluate("e => e")); + CompletableFuture formData = new CompletableFuture<>(); + server.setRoute("/upload", exchange -> { + try { + MultipartFormData multipartFormData = MultipartFormData.parseRequest(exchange); + formData.complete(multipartFormData); + } catch (Exception e) { + e.printStackTrace(); + formData.completeExceptionally(e); + } + exchange.sendResponseHeaders(200, -1); + }); + page.click("input[type=submit]", new Page.ClickOptions().setTimeout(90_000)); + List fields = formData.get().fields; + assertEquals(1, fields.size()); + assertEquals("200MB.zip", fields.get(0).filename); + assertEquals(200 * 1024 * 1024, fields.get(0).content.length()); + } } diff --git a/playwright/src/test/java/com/microsoft/playwright/TestPageSetInputFiles.java b/playwright/src/test/java/com/microsoft/playwright/TestPageSetInputFiles.java index 7971068c..72911448 100644 --- a/playwright/src/test/java/com/microsoft/playwright/TestPageSetInputFiles.java +++ b/playwright/src/test/java/com/microsoft/playwright/TestPageSetInputFiles.java @@ -17,15 +17,22 @@ package com.microsoft.playwright; import com.microsoft.playwright.options.FilePayload; +import org.junit.jupiter.api.Assumptions; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import java.io.*; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.time.Duration; import java.time.Instant; import java.util.ArrayList; +import java.util.Collections; import java.util.List; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; import java.util.function.Consumer; import static java.util.Arrays.asList; @@ -49,6 +56,46 @@ public class TestPageSetInputFiles extends TestBase { "}", input)); } + @Test + void shouldUploadLargeFile(@TempDir Path tmpDir) throws IOException, ExecutionException, InterruptedException { + Assumptions.assumeTrue(3 <= (Runtime.getRuntime().maxMemory() >> 30), "Fails if max heap size is < 3Gb"); + page.navigate(server.PREFIX + "/input/fileupload.html"); + Path uploadFile = tmpDir.resolve("200MB.zip"); + String str = String.join("", Collections.nCopies(4 * 1024, "A")); + + try (Writer stream = new OutputStreamWriter(Files.newOutputStream(uploadFile))) { + for (int i = 0; i < 50 * 1024; i++) { + stream.write(str); + } + } + Locator input = page.locator("input[type='file']"); + JSHandle events = input.evaluateHandle("e => {\n" + + " const events = [];\n" + + " e.addEventListener('input', () => events.push('input'));\n" + + " e.addEventListener('change', () => events.push('change'));\n" + + " return events;\n" + + " }"); + input.setInputFiles(uploadFile); + assertEquals("200MB.zip", input.evaluate("e => e.files[0].name")); + assertEquals(asList("input", "change"), events.evaluate("e => e")); + CompletableFuture formData = new CompletableFuture<>(); + server.setRoute("/upload", exchange -> { + try { + MultipartFormData multipartFormData = MultipartFormData.parseRequest(exchange); + formData.complete(multipartFormData); + } catch (Exception e) { + e.printStackTrace(); + formData.completeExceptionally(e); + } + exchange.sendResponseHeaders(200, -1); + }); + page.click("input[type=submit]", new Page.ClickOptions().setTimeout(90_000)); + List fields = formData.get().fields; + assertEquals(1, fields.size()); + assertEquals("200MB.zip", fields.get(0).filename); + assertEquals(200 * 1024 * 1024, fields.get(0).content.length()); + } + @Test void shouldWork() { page.setContent(""); diff --git a/playwright/src/test/resources/input/fileupload.html b/playwright/src/test/resources/input/fileupload.html index 85d2c7ce..6486e5c2 100644 --- a/playwright/src/test/resources/input/fileupload.html +++ b/playwright/src/test/resources/input/fileupload.html @@ -4,8 +4,8 @@ File upload test -
- + +