honeymoose/playwright-java
1
0
Fork 0
mirror of synced 2026-08-05 15:06:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: large file uploads (#891)

Browse Source
This commit is contained in:
Yury Semikhatsky
2022-04-12 08:33:40 -07:00
committed by GitHub
parent 9f6860539a
commit 1681c410dd
12 changed files with 304 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
playwright/src/main/java/com/microsoft/playwright/impl/BrowserContextImpl.java
+7
View File
@@ -560,4 +560,11 @@ class BrowserContextImpl extends ChannelOwner implements BrowserContext {
}
listeners.notify(EventType.CLOSE, this);
}
WritableStream createTempFile(String name) {
JsonObject params = new JsonObject();
params.addProperty("name", name);
JsonObject json = sendMessage("createTempFile", params).getAsJsonObject();
return connection.getExistingObject(json.getAsJsonObject("writableStream").get("guid").getAsString());
}
}
playwright/src/main/java/com/microsoft/playwright/impl/Connection.java
+3
View File
@@ -302,6 +302,9 @@ public class Connection {
case "Worker":
result = new WorkerImpl(parent, type, guid, initializer);
break;
case "WritableStream":
result = new WritableStream(parent, type, guid, initializer);
break;
default:
throw new PlaywrightException("Unknown type " + type);
}
playwright/src/main/java/com/microsoft/playwright/impl/ElementHandleImpl.java
+22 -3
View File
@@ -33,7 +33,8 @@ import java.util.Base64;
import java.util.List;
import static com.microsoft.playwright.impl.Serialization.*;
import static com.microsoft.playwright.impl.Utils.convertType;
import static com.microsoft.playwright.impl.Utils.*;
import static com.microsoft.playwright.impl.Utils.addLargeFileUploadParams;
import static com.microsoft.playwright.options.ScreenshotType.JPEG;
import static com.microsoft.playwright.options.ScreenshotType.PNG;
@@ -299,7 +300,7 @@ public class ElementHandleImpl extends JSHandleImpl implements ElementHandle {
}
@Override
public Frame ownerFrame() {
public FrameImpl ownerFrame() {
return withLogging("ElementHandle.ownerFrame", () -> {
JsonObject json = sendMessage("ownerFrame").getAsJsonObject();
if (!json.has("frame")) {
@@ -455,7 +456,24 @@ public class ElementHandleImpl extends JSHandleImpl implements ElementHandle {
@Override
public void setInputFiles(Path[] files, SetInputFilesOptions options) {
setInputFiles(Utils.toFilePayloads(files), options);
withLogging("ElementHandle.setInputFiles", () -> setInputFilesImpl(files, options));
}
void setInputFilesImpl(Path[] files, SetInputFilesOptions options) {
FrameImpl frame = ownerFrame();
if (frame == null) {
throw new Error("Cannot set input files to detached element");
}
if (hasLargeFile(files)) {
if (options == null) {
options = new SetInputFilesOptions();
}
JsonObject params = gson().toJsonTree(options).getAsJsonObject();
addLargeFileUploadParams(files, params, frame.page().context());
sendMessage("setInputFilePaths", params);
} else {
setInputFilesImpl(Utils.toFilePayloads(files), options);
}
}
@Override
@@ -469,6 +487,7 @@ public class ElementHandleImpl extends JSHandleImpl implements ElementHandle {
}
void setInputFilesImpl(FilePayload[] files, SetInputFilesOptions options) {
checkFilePayloadSize(files);
if (options == null) {
options = new SetInputFilesOptions();
}
playwright/src/main/java/com/microsoft/playwright/impl/FileChooserImpl.java
+3 -2
View File
@@ -58,7 +58,8 @@ class FileChooserImpl implements FileChooser {
@Override
public void setFiles(Path[] files, SetFilesOptions options) {
setFiles(Utils.toFilePayloads(files), options);
page.withLogging("FileChooser.setInputFiles",
() -> element.setInputFilesImpl(files, convertType(options, ElementHandle.SetInputFilesOptions.class)));
}
@Override
@@ -69,6 +70,6 @@ class FileChooserImpl implements FileChooser {
@Override
public void setFiles(FilePayload[] files, SetFilesOptions options) {
page.withLogging("FileChooser.setInputFiles",
() -> element.setInputFilesImpl(files, convertType(options, ElementHandle.SetInputFilesOptions.class)));
() -> element.setInputFilesImpl(files, convertType(options, ElementHandle.SetInputFilesOptions.class)));
}
}
playwright/src/main/java/com/microsoft/playwright/impl/FrameImpl.java
+17 -5
View File
@@ -23,6 +23,7 @@ import com.microsoft.playwright.*;
import com.microsoft.playwright.options.*;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
@@ -31,7 +32,7 @@ import java.util.function.Consumer;
import java.util.function.Predicate;
import java.util.regex.Pattern;
import static com.microsoft.playwright.impl.Utils.convertType;
import static com.microsoft.playwright.impl.Utils.*;
import static com.microsoft.playwright.options.WaitUntilState.*;
import static com.microsoft.playwright.impl.Serialization.*;
@@ -686,21 +687,32 @@ public class FrameImpl extends ChannelOwner implements Frame {
withLogging("Frame.setInputFiles", () -> setInputFilesImpl(selector, files, options));
}
void setInputFilesImpl(String selector, Path[] files, SetInputFilesOptions options) {
if (hasLargeFile(files)) {
if (options == null) {
options = new SetInputFilesOptions();
}
JsonObject params = gson().toJsonTree(options).getAsJsonObject();
addLargeFileUploadParams(files, params, page.context());
params.addProperty("selector", selector);
sendMessage("setInputFilePaths", params);
} else {
setInputFilesImpl(selector, Utils.toFilePayloads(files), options);
}
}
@Override
public void setInputFiles(String selector, FilePayload files, SetInputFilesOptions options) {
setInputFiles(selector, new FilePayload[]{files}, options);
}
void setInputFilesImpl(String selector, Path[] files, SetInputFilesOptions options) {
setInputFiles(selector, Utils.toFilePayloads(files), options);
}
@Override
public void setInputFiles(String selector, FilePayload[] files, SetInputFilesOptions options) {
withLogging("Frame.setInputFiles", () -> setInputFilesImpl(selector, files, options));
}
void setInputFilesImpl(String selector, FilePayload[] files, SetInputFilesOptions options) {
checkFilePayloadSize(files);
if (options == null) {
options = new SetInputFilesOptions();
}
playwright/src/main/java/com/microsoft/playwright/impl/Serialization.java
+8
View File
@@ -212,6 +212,14 @@ class Serialization {
}
}
static JsonArray toJsonArray(Path[] files) {
JsonArray jsonFiles = new JsonArray();
for (Path p : files) {
jsonFiles.add(p.toString());
}
return jsonFiles;
}
static JsonArray toJsonArray(FilePayload[] files) {
JsonArray jsonFiles = new JsonArray();
for (FilePayload p : files) {
playwright/src/main/java/com/microsoft/playwright/impl/Utils.java
+48
View File
@@ -16,6 +16,8 @@
package com.microsoft.playwright.impl;
import com.google.gson.JsonArray;
import com.google.gson.JsonObject;
import com.microsoft.playwright.PlaywrightException;
import com.microsoft.playwright.options.FilePayload;
import com.microsoft.playwright.options.HttpHeader;
@@ -23,6 +25,7 @@ import com.microsoft.playwright.options.HttpHeader;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.lang.reflect.Field;
import java.lang.reflect.Modifier;
import java.nio.file.Files;
@@ -30,6 +33,8 @@ import java.nio.file.Path;
import java.util.*;
import java.util.regex.Pattern;
import static com.microsoft.playwright.impl.Serialization.toJsonArray;
class Utils {
static <F, T> T convertType(F f, Class<T> t) {
if (f == null) {
@@ -144,6 +149,49 @@ class Utils {
return mimeType;
}
static final int maxUplodBufferSize = 50 * 1024 * 1024;
static boolean hasLargeFile(Path[] files) {
for (Path file: files) {
try {
if (Files.size(file)> maxUplodBufferSize) {
return true;
}
} catch (IOException e) {
throw new PlaywrightException("Cannot get file size.", e);
}
}
return false;
}
static void addLargeFileUploadParams(Path[] files, JsonObject params, BrowserContextImpl context) {
if (context.browser().isRemote) {
List<WritableStream> streams = new ArrayList<>();
JsonArray jsonStreams = new JsonArray();
for (Path path : files) {
WritableStream temp = context.createTempFile(path.getFileName().toString());
streams.add(temp);
try (OutputStream out = temp.stream()) {
Files.copy(path, out);
} catch (IOException e) {
throw new PlaywrightException("Failed to copy file to remote server.", e);
}
jsonStreams.add(temp.toProtocol());
}
params.add("streams", jsonStreams);
} else {
params.add("localPaths", toJsonArray(files));
}
}
static void checkFilePayloadSize(FilePayload[] files) {
for (FilePayload file: files) {
if (file.buffer.length > maxUplodBufferSize) {
throw new PlaywrightException("Cannot set buffer larger than 50Mb, please write it to a file and pass its path instead.");
}
}
}
static FilePayload[] toFilePayloads(Path[] files) {
List<FilePayload> payloads = new ArrayList<>();
for (Path file : files) {
playwright/src/main/java/com/microsoft/playwright/impl/WritableStream.java
+39
View File
@@ -0,0 +1,39 @@
package com.microsoft.playwright.impl;
import com.google.gson.JsonObject;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.util.Base64;
class WritableStream extends ChannelOwner {
WritableStream(ChannelOwner parent, String type, String guid, JsonObject initializer) {
super(parent, type, guid, initializer);
}
OutputStream stream() {
return new OutputStream() {
@Override
public void write(int b) throws IOException {
write(new byte[] { (byte) b });
}
@Override
public void write(byte[] b, int off, int len) throws IOException {
JsonObject params = new JsonObject();
ByteBuffer buffer = ByteBuffer.wrap(b, off, len);
ByteBuffer encoded = Base64.getEncoder().encode(buffer);
params.addProperty("binary", new String(encoded.array(), StandardCharsets.UTF_8));
sendMessage("write", params);
}
};
}
JsonObject toProtocol() {
JsonObject json = new JsonObject();
json.addProperty("guid", guid);
return json;
}
}
playwright/src/test/java/com/microsoft/playwright/MultipartFormData.java
+66
View File
@@ -0,0 +1,66 @@
package com.microsoft.playwright;
import com.sun.net.httpserver.HttpExchange;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class MultipartFormData {
static MultipartFormData parseRequest(HttpExchange exchange) throws IOException {
ByteArrayOutputStream bodyBytes = new ByteArrayOutputStream();
try (OutputStream output = bodyBytes) {
Utils.copy(exchange.getRequestBody(), output);
}
String body = new String(bodyBytes.toByteArray(), StandardCharsets.UTF_8);
String contentType = exchange.getRequestHeaders().get("content-type").get(0);
Matcher matcher = Pattern.compile("boundary=(.*)$").matcher(contentType);
if (!matcher.find()) {
throw new RuntimeException("Boundary not found!");
}
String boundary = matcher.group(1);
return new MultipartFormData(body, boundary);
}
static class Field {
final String filename;
final String content;
Field(String filename, String content) {
this.filename = filename;
this.content = content;
}
}
final List<Field> fields = new ArrayList<>();
MultipartFormData(String body, String boundary) {
String[] parts = Pattern.compile("--" + boundary + "(--)?\r\n", Pattern.MULTILINE).split(body);
for (String part : parts) {
if (part.trim().length() == 0) {
continue;
}
String[] headersAndContent = Pattern.compile("\r\n\r\n", Pattern.MULTILINE).split(part);
if (headersAndContent.length != 2) {
throw new RuntimeException("Unexpected format: " + part);
}
String headers = headersAndContent[0];
String filename = null;
for (String header: Pattern.compile("\r\n", Pattern.MULTILINE).split(headers)) {
Matcher matcher = Pattern.compile("content-disposition: .*filename=\"([^\"]+)\"", Pattern.CASE_INSENSITIVE).matcher(header);
if (!matcher.find()) {
continue;
}
filename = matcher.group(1);
}
String content = headersAndContent[1];
content = content.substring(0, content.length() - "\r\n".length());
fields.add(new Field(filename, content));
}
}
}
playwright/src/test/java/com/microsoft/playwright/TestBrowserTypeConnect.java
+42
View File
@@ -30,6 +30,8 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.*;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import java.util.stream.Collectors;
import static com.microsoft.playwright.Utils.*;
@@ -518,4 +520,44 @@ public class TestBrowserTypeConnect extends TestBase {
assertEquals(200, response.status());
assertEquals("{\"foo\": \"bar\"}\n", response.text());
}
@Test
void shouldUploadLargeFile(@TempDir Path tmpDir) throws IOException, ExecutionException, InterruptedException {
Assumptions.assumeTrue(3 <= (Runtime.getRuntime().maxMemory() >> 30), "Fails if max heap size is < 3Gb");
page.navigate(server.PREFIX + "/input/fileupload.html");
Path uploadFile = tmpDir.resolve("200MB.zip");
String str = String.join("", Collections.nCopies(4 * 1024, "A"));
try (Writer stream = new OutputStreamWriter(Files.newOutputStream(uploadFile))) {
for (int i = 0; i < 50 * 1024; i++) {
stream.write(str);
}
}
Locator input = page.locator("input[type='file']");
JSHandle events = input.evaluateHandle("e => {\n" +
" const events = [];\n" +
" e.addEventListener('input', () => events.push('input'));\n" +
" e.addEventListener('change', () => events.push('change'));\n" +
" return events;\n" +
" }");
input.setInputFiles(uploadFile);
assertEquals("200MB.zip", input.evaluate("e => e.files[0].name"));
assertEquals(asList("input", "change"), events.evaluate("e => e"));
CompletableFuture<MultipartFormData> formData = new CompletableFuture<>();
server.setRoute("/upload", exchange -> {
try {
MultipartFormData multipartFormData = MultipartFormData.parseRequest(exchange);
formData.complete(multipartFormData);
} catch (Exception e) {
e.printStackTrace();
formData.completeExceptionally(e);
}
exchange.sendResponseHeaders(200, -1);
});
page.click("input[type=submit]", new Page.ClickOptions().setTimeout(90_000));
List<MultipartFormData.Field> fields = formData.get().fields;
assertEquals(1, fields.size());
assertEquals("200MB.zip", fields.get(0).filename);
assertEquals(200 * 1024 * 1024, fields.get(0).content.length());
}
}
playwright/src/test/java/com/microsoft/playwright/TestPageSetInputFiles.java
+47
View File
@@ -17,15 +17,22 @@
package com.microsoft.playwright;
import com.microsoft.playwright.options.FilePayload;
import org.junit.jupiter.api.Assumptions;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import java.io.*;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.function.Consumer;
import static java.util.Arrays.asList;
@@ -49,6 +56,46 @@ public class TestPageSetInputFiles extends TestBase {
"}", input));
}
@Test
void shouldUploadLargeFile(@TempDir Path tmpDir) throws IOException, ExecutionException, InterruptedException {
Assumptions.assumeTrue(3 <= (Runtime.getRuntime().maxMemory() >> 30), "Fails if max heap size is < 3Gb");
page.navigate(server.PREFIX + "/input/fileupload.html");
Path uploadFile = tmpDir.resolve("200MB.zip");
String str = String.join("", Collections.nCopies(4 * 1024, "A"));
try (Writer stream = new OutputStreamWriter(Files.newOutputStream(uploadFile))) {
for (int i = 0; i < 50 * 1024; i++) {
stream.write(str);
}
}
Locator input = page.locator("input[type='file']");
JSHandle events = input.evaluateHandle("e => {\n" +
" const events = [];\n" +
" e.addEventListener('input', () => events.push('input'));\n" +
" e.addEventListener('change', () => events.push('change'));\n" +
" return events;\n" +
" }");
input.setInputFiles(uploadFile);
assertEquals("200MB.zip", input.evaluate("e => e.files[0].name"));
assertEquals(asList("input", "change"), events.evaluate("e => e"));
CompletableFuture<MultipartFormData> formData = new CompletableFuture<>();
server.setRoute("/upload", exchange -> {
try {
MultipartFormData multipartFormData = MultipartFormData.parseRequest(exchange);
formData.complete(multipartFormData);
} catch (Exception e) {
e.printStackTrace();
formData.completeExceptionally(e);
}
exchange.sendResponseHeaders(200, -1);
});
page.click("input[type=submit]", new Page.ClickOptions().setTimeout(90_000));
List<MultipartFormData.Field> fields = formData.get().fields;
assertEquals(1, fields.size());
assertEquals("200MB.zip", fields.get(0).filename);
assertEquals(200 * 1024 * 1024, fields.get(0).content.length());
}
@Test
void shouldWork() {
page.setContent("<input type=file>");
playwright/src/test/resources/input/fileupload.html
+2 -2
View File
@@ -4,8 +4,8 @@
<title>File upload test</title>
</head>
<body>
<form action="/input/fileupload.html">
<input type="file">
<form action="/upload" method="post" enctype="multipart/form-data">
<input type="file" name="file1">
<input type="submit">
</form>
</body>