Skip to content

Commit b65e657

Browse files
Yavor16Yavor16
committed
Allow special characters in the url
1 parent f81be7e commit b65e657

File tree

4 files changed

+182
-34
lines changed

4 files changed

+182
-34
lines changed

‎multiapps-controller-api/src/main/java/org/cloudfoundry/multiapps/controller/api/model/FileUrl.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import com.fasterxml.jackson.annotation.JsonProperty;
44
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
55
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
6+
import org.cloudfoundry.multiapps.common.Nullable;
67
import org.immutables.value.Value;
78

89
@Value.Immutable
@@ -14,5 +15,9 @@ public interface FileUrl {
1415
@JsonProperty("file_url")
1516
String getFileUrl();
1617

18+
@Nullable
19+
@Value.Parameter
20+
@JsonProperty("credentials")
21+
UserCredentials getUserCredentials();
1722
//this could potentially contain a TLS certificate as well, if the remote endpoint is a custom registry/repository
1823
}

‎multiapps-controller-api/src/main/java/org/cloudfoundry/multiapps/controller/api/model/UserCredentials.java‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
package org.cloudfoundry.multiapps.controller.api.model;
2+
3+
import org.cloudfoundry.multiapps.common.Nullable;
4+
import org.immutables.value.Value;
5+
6+
import com.fasterxml.jackson.annotation.JsonProperty;
7+
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
8+
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
9+
10+
@Value.Immutable
11+
@JsonSerialize(as = ImmutableUserCredentials.class)
12+
@JsonDeserialize(as = ImmutableUserCredentials.class)
13+
public interface UserCredentials {
14+
15+
@Nullable
16+
@Value.Parameter
17+
@JsonProperty("username")
18+
String getUsername();
19+
20+
@Nullable
21+
@Value.Parameter
22+
@JsonProperty("password")
23+
String getPassword();
24+
//this could potentially contain a TLS certificate as well, if the remote endpoint is a custom registry/repository
25+
}

‎multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/api/impl/FilesApiServiceImpl.java‎

Lines changed: 53 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,13 @@
55
import java.io.InputStream;
66
import java.math.BigInteger;
77
import java.net.URI;
8+
import java.net.URLDecoder;
89
import java.net.http.HttpClient;
910
import java.net.http.HttpClient.Redirect;
1011
import java.net.http.HttpRequest;
1112
import java.net.http.HttpResponse;
1213
import java.net.http.HttpResponse.BodyHandlers;
14+
import java.nio.charset.StandardCharsets;
1315
import java.text.MessageFormat;
1416
import java.time.Duration;
1517
import java.time.LocalDateTime;
@@ -24,9 +26,6 @@
2426
import java.util.concurrent.atomic.AtomicLong;
2527
import java.util.stream.Collectors;
2628

27-
import jakarta.inject.Inject;
28-
import jakarta.inject.Named;
29-
3029
import org.apache.commons.io.IOUtils;
3130
import org.apache.commons.io.input.ProxyInputStream;
3231
import org.cloudfoundry.multiapps.common.SLException;
@@ -36,6 +35,7 @@
3635
import org.cloudfoundry.multiapps.controller.api.model.FileUrl;
3736
import org.cloudfoundry.multiapps.controller.api.model.ImmutableAsyncUploadResult;
3837
import org.cloudfoundry.multiapps.controller.api.model.ImmutableFileMetadata;
38+
import org.cloudfoundry.multiapps.controller.api.model.UserCredentials;
3939
import org.cloudfoundry.multiapps.controller.client.util.CheckedSupplier;
4040
import org.cloudfoundry.multiapps.controller.client.util.ResilientOperationExecutor;
4141
import org.cloudfoundry.multiapps.controller.core.auditlogging.FilesApiServiceAuditLog;
@@ -69,6 +69,9 @@
6969
import org.springframework.web.multipart.MultipartFile;
7070
import org.springframework.web.multipart.MultipartHttpServletRequest;
7171

72+
import jakarta.inject.Inject;
73+
import jakarta.inject.Named;
74+
7275
@Named
7376
public class FilesApiServiceImpl implements FilesApiService {
7477

@@ -77,6 +80,11 @@ public class FilesApiServiceImpl implements FilesApiService {
7780
private static final int INPUT_STREAM_BUFFER_SIZE = 16 * 1024;
7881
private static final Duration HTTP_CONNECT_TIMEOUT = Duration.ofMinutes(10);
7982
private static final String RETRY_AFTER_SECONDS = "30";
83+
private static final String USERNAME_PASSWORD_URL_FORMAT = "{0}:{1}";
84+
static {
85+
System.setProperty(Constants.RETRY_LIMIT_PROPERTY, "0");
86+
}
87+
8088
private final CachedMap<String, AtomicLong> jobCounters = new CachedMap<>(Duration.ofHours(1));
8189
private final CachedMap<String, Future<?>> runningTasks = new CachedMap<>(Duration.ofHours(1));
8290
private final ResilientOperationExecutor resilientOperationExecutor = getResilientOperationExecutor();
@@ -97,10 +105,6 @@ public class FilesApiServiceImpl implements FilesApiService {
97105
@Inject
98106
private ExecutorService fileStorageThreadPool;
99107

100-
static {
101-
System.setProperty(Constants.RETRY_LIMIT_PROPERTY, "0");
102-
}
103-
104108
@Override
105109
public ResponseEntity<List<FileMetadata>> getFiles(String spaceGuid, String namespace) {
106110
try {
@@ -156,7 +160,7 @@ public ResponseEntity<Void> startUploadFromUrl(String spaceGuid, String namespac
156160
deleteAsyncJobEntry(existingJob);
157161
}
158162
}
159-
return triggerUploadFromUrl(spaceGuid, namespace, urlWithoutUserInfo, decodedUrl);
163+
return triggerUploadFromUrl(spaceGuid, namespace, urlWithoutUserInfo, decodedUrl, fileUrl.getUserCredentials());
160164
}
161165

162166
private String getLocationHeader(String spaceGuid, String jobId) {
@@ -289,12 +293,14 @@ private void deleteAsyncJobEntry(AsyncUploadJobEntry entry) {
289293
}
290294
}
291295

292-
private ResponseEntity<Void> triggerUploadFromUrl(String spaceGuid, String namespace, String urlWithoutUserInfo, String decodedUrl) {
296+
private ResponseEntity<Void> triggerUploadFromUrl(String spaceGuid, String namespace, String urlWithoutUserInfo, String decodedUrl,
297+
UserCredentials userCredentials) {
293298
var entry = createJobEntry(spaceGuid, namespace, urlWithoutUserInfo);
294299
LOGGER.debug(Messages.CREATING_ASYNC_UPLOAD_JOB, urlWithoutUserInfo, entry.getId());
295300
uploadJobService.add(entry);
296301
try {
297-
Future<?> runningTask = deployFromUrlExecutor.submit(() -> uploadFileFromUrl(entry, spaceGuid, namespace, decodedUrl));
302+
Future<?> runningTask = deployFromUrlExecutor.submit(() -> uploadFileFromUrl(entry, spaceGuid, namespace, decodedUrl,
303+
userCredentials));
298304
runningTasks.put(entry.getId(), runningTask);
299305
} catch (RejectedExecutionException ignored) {
300306
LOGGER.debug(Messages.ASYNC_UPLOAD_JOB_REJECTED, entry.getId());
@@ -345,7 +351,8 @@ private AsyncUploadResult createErrorResult(String error, AsyncUploadResult.Clie
345351
.build();
346352
}
347353

348-
private void uploadFileFromUrl(AsyncUploadJobEntry jobEntry, String spaceGuid, String namespace, String fileUrl) {
354+
private void uploadFileFromUrl(AsyncUploadJobEntry jobEntry, String spaceGuid, String namespace, String fileUrl,
355+
UserCredentials userCredentials) {
349356
var counter = new AtomicLong(0);
350357
jobCounters.put(jobEntry.getId(), counter);
351358
LOGGER.info(Messages.STARTING_DOWNLOAD_OF_MTAR, jobEntry.getUrl());
@@ -358,7 +365,8 @@ private void uploadFileFromUrl(AsyncUploadJobEntry jobEntry, String spaceGuid, S
358365
FileEntry fileEntry = resilientOperationExecutor.execute((CheckedSupplier<FileEntry>) () -> doUploadFileFromUrl(spaceGuid,
359366
namespace,
360367
fileUrl,
361-
counter));
368+
counter,
369+
userCredentials));
362370
LOGGER.trace(Messages.UPLOADED_MTAR_FROM_REMOTE_ENDPOINT_AND_JOB_ID, jobEntry.getUrl(), jobEntry.getId(),
363371
ChronoUnit.MILLIS.between(startTime, LocalDateTime.now()));
364372
var descriptor = fileService.processFileContent(spaceGuid, fileEntry.getId(), this::extractDeploymentDescriptor);
@@ -376,14 +384,16 @@ private void uploadFileFromUrl(AsyncUploadJobEntry jobEntry, String spaceGuid, S
376384
}
377385
}
378386

379-
private FileEntry doUploadFileFromUrl(String spaceGuid, String namespace, String fileUrl, AtomicLong counter) throws Exception {
387+
private FileEntry doUploadFileFromUrl(String spaceGuid, String namespace, String fileUrl, AtomicLong counter,
388+
UserCredentials userCredentials)
389+
throws Exception {
380390
if (!UriUtil.isUrlSecure(fileUrl)) {
381391
throw new SLException(Messages.MTAR_ENDPOINT_NOT_SECURE);
382392
}
383393
UriUtil.validateUrl(fileUrl);
384394
HttpClient client = buildHttpClient(fileUrl);
385395

386-
HttpResponse<InputStream> response = callRemoteEndpointWithRetry(client, fileUrl);
396+
HttpResponse<InputStream> response = callRemoteEndpointWithRetry(client, fileUrl, userCredentials);
387397
long fileSize = response.headers()
388398
.firstValueAsLong(Constants.CONTENT_LENGTH)
389399
.orElseThrow(() -> new SLException(Messages.FILE_URL_RESPONSE_DID_NOT_RETURN_CONTENT_LENGTH));
@@ -411,10 +421,11 @@ private FileEntry doUploadFileFromUrl(String spaceGuid, String namespace, String
411421
}
412422
}
413423

414-
private HttpResponse<InputStream> callRemoteEndpointWithRetry(HttpClient client, String decodedUrl) throws Exception {
424+
public HttpResponse<InputStream> callRemoteEndpointWithRetry(HttpClient client, String decodedUrl, UserCredentials userCredentials)
425+
throws Exception {
415426
return resilientOperationExecutor.execute((CheckedSupplier<HttpResponse<InputStream>>) () -> {
416-
var request = buildFetchFileRequest(decodedUrl);
417-
LOGGER.debug(Messages.CALLING_REMOTE_MTAR_ENDPOINT, request.uri());
427+
var request = buildFetchFileRequest(decodedUrl, userCredentials);
428+
LOGGER.debug(Messages.CALLING_REMOTE_MTAR_ENDPOINT, getMaskedUri(urlDecodeUrl(decodedUrl)));
418429
var response = client.send(request, BodyHandlers.ofInputStream());
419430
if (response.statusCode() / 100 != 2) {
420431
String error = readErrorBodyFromResponse(response);
@@ -424,13 +435,26 @@ private HttpResponse<InputStream> callRemoteEndpointWithRetry(HttpClient client,
424435
UriUtil.stripUserInfo(decodedUrl));
425436
throw new SLException(errorMessage);
426437
}
427-
throw new SLException(MessageFormat.format(Messages.ERROR_FROM_REMOTE_MTAR_ENDPOINT, request.uri(), response.statusCode(),
428-
error));
438+
throw new SLException(MessageFormat.format(Messages.ERROR_FROM_REMOTE_MTAR_ENDPOINT, getMaskedUri(urlDecodeUrl(decodedUrl)),
439+
response.statusCode(), error));
429440
}
430441
return response;
431442
});
432443
}
433444

445+
private String getMaskedUri(String url) {
446+
if (url.contains("@")) {
447+
return url.substring(url.lastIndexOf("@"))
448+
.replace("@", "...");
449+
} else {
450+
return url;
451+
}
452+
}
453+
454+
private String urlDecodeUrl(String url) {
455+
return URLDecoder.decode(url, StandardCharsets.UTF_8);
456+
}
457+
434458
private void resetCounterOnRetry(AtomicLong counter) {
435459
counter.set(0);
436460
}
@@ -443,14 +467,21 @@ protected HttpClient buildHttpClient(String decodedUrl) {
443467
.build();
444468
}
445469

446-
private HttpRequest buildFetchFileRequest(String decodedUrl) {
470+
private HttpRequest buildFetchFileRequest(String decodedUrl, UserCredentials userCredentials) {
447471
var builder = HttpRequest.newBuilder()
448472
.GET()
449473
.timeout(Duration.ofMinutes(15));
450474
var uri = URI.create(decodedUrl);
451475
var userInfo = uri.getUserInfo();
452-
if (userInfo != null) {
453-
builder.uri(URI.create(decodedUrl.replace(userInfo + "@", "")));
476+
if (userCredentials != null) {
477+
builder.uri(uri);
478+
String userCredentialsUrlFormat = MessageFormat.format(USERNAME_PASSWORD_URL_FORMAT, userCredentials.getUsername(),
479+
userCredentials.getPassword());
480+
String encodedAuth = Base64.getEncoder()
481+
.encodeToString(userCredentialsUrlFormat.getBytes());
482+
builder.header(HttpHeaders.AUTHORIZATION, "Basic " + encodedAuth);
483+
} else if (userInfo != null) {
484+
builder.uri(URI.create(decodedUrl.replace(uri.getRawUserInfo() + "@", "")));
454485
String encodedAuth = Base64.getEncoder()
455486
.encodeToString(userInfo.getBytes());
456487
builder.header(HttpHeaders.AUTHORIZATION, "Basic " + encodedAuth);

0 commit comments

Comments
 (0)