Skip to content

Commit f209ead

Browse files
Yavor16Yavor16
committed
Allow special characters in the url
1 parent 0b3f947 commit f209ead

File tree

4 files changed

+83
-28
lines changed

4 files changed

+83
-28
lines changed

‎multiapps-controller-api/src/main/java/org/cloudfoundry/multiapps/controller/api/model/FileUrl.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import com.fasterxml.jackson.annotation.JsonProperty;
44
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
55
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
6+
import org.cloudfoundry.multiapps.common.Nullable;
67
import org.immutables.value.Value;
78

89
@Value.Immutable
@@ -14,5 +15,9 @@ public interface FileUrl {
1415
@JsonProperty("file_url")
1516
String getFileUrl();
1617

18+
@Nullable
19+
@Value.Parameter
20+
@JsonProperty("credentials")
21+
UserCredentials getUserCredentials();
1722
//this could potentially contain a TLS certificate as well, if the remote endpoint is a custom registry/repository
1823
}

‎multiapps-controller-api/src/main/java/org/cloudfoundry/multiapps/controller/api/model/UserCredentials.java‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
package org.cloudfoundry.multiapps.controller.api.model;
2+
3+
import org.cloudfoundry.multiapps.common.Nullable;
4+
import org.immutables.value.Value;
5+
6+
import com.fasterxml.jackson.annotation.JsonProperty;
7+
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
8+
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
9+
10+
@Value.Immutable
11+
@JsonSerialize(as = ImmutableUserCredentials.class)
12+
@JsonDeserialize(as = ImmutableUserCredentials.class)
13+
public interface UserCredentials {
14+
15+
@Nullable
16+
@Value.Parameter
17+
@JsonProperty("username")
18+
String getUsername();
19+
20+
@Nullable
21+
@Value.Parameter
22+
@JsonProperty("password")
23+
String getPassword();
24+
//this could potentially contain a TLS certificate as well, if the remote endpoint is a custom registry/repository
25+
}

‎multiapps-controller-web/src/main/java/org/cloudfoundry/multiapps/controller/web/api/impl/FilesApiServiceImpl.java‎

Lines changed: 47 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -24,9 +24,6 @@
2424
import java.util.concurrent.atomic.AtomicLong;
2525
import java.util.stream.Collectors;
2626

27-
import jakarta.inject.Inject;
28-
import jakarta.inject.Named;
29-
3027
import org.apache.commons.io.IOUtils;
3128
import org.apache.commons.io.input.ProxyInputStream;
3229
import org.cloudfoundry.multiapps.common.SLException;
@@ -36,6 +33,7 @@
3633
import org.cloudfoundry.multiapps.controller.api.model.FileUrl;
3734
import org.cloudfoundry.multiapps.controller.api.model.ImmutableAsyncUploadResult;
3835
import org.cloudfoundry.multiapps.controller.api.model.ImmutableFileMetadata;
36+
import org.cloudfoundry.multiapps.controller.api.model.UserCredentials;
3937
import org.cloudfoundry.multiapps.controller.client.util.CheckedSupplier;
4038
import org.cloudfoundry.multiapps.controller.client.util.ResilientOperationExecutor;
4139
import org.cloudfoundry.multiapps.controller.core.auditlogging.FilesApiServiceAuditLog;
@@ -69,6 +67,9 @@
6967
import org.springframework.web.multipart.MultipartFile;
7068
import org.springframework.web.multipart.MultipartHttpServletRequest;
7169

70+
import jakarta.inject.Inject;
71+
import jakarta.inject.Named;
72+
7273
@Named
7374
public class FilesApiServiceImpl implements FilesApiService {
7475

@@ -77,6 +78,11 @@ public class FilesApiServiceImpl implements FilesApiService {
7778
private static final int INPUT_STREAM_BUFFER_SIZE = 16 * 1024;
7879
private static final Duration HTTP_CONNECT_TIMEOUT = Duration.ofMinutes(10);
7980
private static final String RETRY_AFTER_SECONDS = "30";
81+
private static final String USERNAME_PASSWORD_URL_FORAMT = "{0}:{1}";
82+
static {
83+
System.setProperty(Constants.RETRY_LIMIT_PROPERTY, "0");
84+
}
85+
8086
private final CachedMap<String, AtomicLong> jobCounters = new CachedMap<>(Duration.ofHours(1));
8187
private final CachedMap<String, Future<?>> runningTasks = new CachedMap<>(Duration.ofHours(1));
8288
private final ResilientOperationExecutor resilientOperationExecutor = getResilientOperationExecutor();
@@ -97,10 +103,6 @@ public class FilesApiServiceImpl implements FilesApiService {
97103
@Inject
98104
private ExecutorService fileStorageThreadPool;
99105

100-
static {
101-
System.setProperty(Constants.RETRY_LIMIT_PROPERTY, "0");
102-
}
103-
104106
@Override
105107
public ResponseEntity<List<FileMetadata>> getFiles(String spaceGuid, String namespace) {
106108
try {
@@ -156,7 +158,7 @@ public ResponseEntity<Void> startUploadFromUrl(String spaceGuid, String namespac
156158
deleteAsyncJobEntry(existingJob);
157159
}
158160
}
159-
return triggerUploadFromUrl(spaceGuid, namespace, urlWithoutUserInfo, decodedUrl);
161+
return triggerUploadFromUrl(spaceGuid, namespace, urlWithoutUserInfo, decodedUrl, fileUrl.getUserCredentials());
160162
}
161163

162164
private String getLocationHeader(String spaceGuid, String jobId) {
@@ -289,12 +291,14 @@ private void deleteAsyncJobEntry(AsyncUploadJobEntry entry) {
289291
}
290292
}
291293

292-
private ResponseEntity<Void> triggerUploadFromUrl(String spaceGuid, String namespace, String urlWithoutUserInfo, String decodedUrl) {
294+
private ResponseEntity<Void> triggerUploadFromUrl(String spaceGuid, String namespace, String urlWithoutUserInfo, String decodedUrl,
295+
UserCredentials userCredentials) {
293296
var entry = createJobEntry(spaceGuid, namespace, urlWithoutUserInfo);
294297
LOGGER.debug(Messages.CREATING_ASYNC_UPLOAD_JOB, urlWithoutUserInfo, entry.getId());
295298
uploadJobService.add(entry);
296299
try {
297-
Future<?> runningTask = deployFromUrlExecutor.submit(() -> uploadFileFromUrl(entry, spaceGuid, namespace, decodedUrl));
300+
Future<?> runningTask = deployFromUrlExecutor.submit(() -> uploadFileFromUrl(entry, spaceGuid, namespace, decodedUrl,
301+
userCredentials));
298302
runningTasks.put(entry.getId(), runningTask);
299303
} catch (RejectedExecutionException ignored) {
300304
LOGGER.debug(Messages.ASYNC_UPLOAD_JOB_REJECTED, entry.getId());
@@ -345,7 +349,8 @@ private AsyncUploadResult createErrorResult(String error, AsyncUploadResult.Clie
345349
.build();
346350
}
347351

348-
private void uploadFileFromUrl(AsyncUploadJobEntry jobEntry, String spaceGuid, String namespace, String fileUrl) {
352+
private void uploadFileFromUrl(AsyncUploadJobEntry jobEntry, String spaceGuid, String namespace, String fileUrl,
353+
UserCredentials userCredentials) {
349354
var counter = new AtomicLong(0);
350355
jobCounters.put(jobEntry.getId(), counter);
351356
LOGGER.info(Messages.STARTING_DOWNLOAD_OF_MTAR, jobEntry.getUrl());
@@ -358,7 +363,8 @@ private void uploadFileFromUrl(AsyncUploadJobEntry jobEntry, String spaceGuid, S
358363
FileEntry fileEntry = resilientOperationExecutor.execute((CheckedSupplier<FileEntry>) () -> doUploadFileFromUrl(spaceGuid,
359364
namespace,
360365
fileUrl,
361-
counter));
366+
counter,
367+
userCredentials));
362368
LOGGER.trace(Messages.UPLOADED_MTAR_FROM_REMOTE_ENDPOINT_AND_JOB_ID, jobEntry.getUrl(), jobEntry.getId(),
363369
ChronoUnit.MILLIS.between(startTime, LocalDateTime.now()));
364370
var descriptor = fileService.processFileContent(spaceGuid, fileEntry.getId(), this::extractDeploymentDescriptor);
@@ -376,14 +382,16 @@ private void uploadFileFromUrl(AsyncUploadJobEntry jobEntry, String spaceGuid, S
376382
}
377383
}
378384

379-
private FileEntry doUploadFileFromUrl(String spaceGuid, String namespace, String fileUrl, AtomicLong counter) throws Exception {
385+
private FileEntry doUploadFileFromUrl(String spaceGuid, String namespace, String fileUrl, AtomicLong counter,
386+
UserCredentials userCredentials)
387+
throws Exception {
380388
if (!UriUtil.isUrlSecure(fileUrl)) {
381389
throw new SLException(Messages.MTAR_ENDPOINT_NOT_SECURE);
382390
}
383391
UriUtil.validateUrl(fileUrl);
384392
HttpClient client = buildHttpClient(fileUrl);
385393

386-
HttpResponse<InputStream> response = callRemoteEndpointWithRetry(client, fileUrl);
394+
HttpResponse<InputStream> response = callRemoteEndpointWithRetry(client, fileUrl, userCredentials);
387395
long fileSize = response.headers()
388396
.firstValueAsLong(Constants.CONTENT_LENGTH)
389397
.orElseThrow(() -> new SLException(Messages.FILE_URL_RESPONSE_DID_NOT_RETURN_CONTENT_LENGTH));
@@ -411,10 +419,11 @@ private FileEntry doUploadFileFromUrl(String spaceGuid, String namespace, String
411419
}
412420
}
413421

414-
private HttpResponse<InputStream> callRemoteEndpointWithRetry(HttpClient client, String decodedUrl) throws Exception {
422+
private HttpResponse<InputStream> callRemoteEndpointWithRetry(HttpClient client, String decodedUrl, UserCredentials userCredentials)
423+
throws Exception {
415424
return resilientOperationExecutor.execute((CheckedSupplier<HttpResponse<InputStream>>) () -> {
416-
var request = buildFetchFileRequest(decodedUrl);
417-
LOGGER.debug(Messages.CALLING_REMOTE_MTAR_ENDPOINT, request.uri());
425+
var request = buildFetchFileRequest(decodedUrl, userCredentials);
426+
LOGGER.debug(Messages.CALLING_REMOTE_MTAR_ENDPOINT, getUriFromAtSign(request.uri()));
418427
var response = client.send(request, BodyHandlers.ofInputStream());
419428
if (response.statusCode() / 100 != 2) {
420429
String error = readErrorBodyFromResponse(response);
@@ -424,13 +433,22 @@ private HttpResponse<InputStream> callRemoteEndpointWithRetry(HttpClient client,
424433
UriUtil.stripUserInfo(decodedUrl));
425434
throw new SLException(errorMessage);
426435
}
427-
throw new SLException(MessageFormat.format(Messages.ERROR_FROM_REMOTE_MTAR_ENDPOINT, request.uri(), response.statusCode(),
428-
error));
436+
throw new SLException(MessageFormat.format(Messages.ERROR_FROM_REMOTE_MTAR_ENDPOINT, getUriFromAtSign(request.uri()),
437+
response.statusCode(), error));
429438
}
430439
return response;
431440
});
432441
}
433442

443+
private String getUriFromAtSign(URI uri) {
444+
String uriString = uri.toString();
445+
if (uriString.contains("@")) {
446+
return uriString.substring(uriString.lastIndexOf("@"));
447+
} else {
448+
return uriString;
449+
}
450+
}
451+
434452
private void resetCounterOnRetry(AtomicLong counter) {
435453
counter.set(0);
436454
}
@@ -443,14 +461,21 @@ protected HttpClient buildHttpClient(String decodedUrl) {
443461
.build();
444462
}
445463

446-
private HttpRequest buildFetchFileRequest(String decodedUrl) {
464+
private HttpRequest buildFetchFileRequest(String decodedUrl, UserCredentials userCredentials) {
447465
var builder = HttpRequest.newBuilder()
448466
.GET()
449467
.timeout(Duration.ofMinutes(15));
450468
var uri = URI.create(decodedUrl);
451469
var userInfo = uri.getUserInfo();
452-
if (userInfo != null) {
453-
builder.uri(URI.create(decodedUrl.replace(userInfo + "@", "")));
470+
if (userCredentials != null) {
471+
builder.uri(uri);
472+
String userCredentialsUrlFormat = MessageFormat.format(USERNAME_PASSWORD_URL_FORAMT, userCredentials.getUsername(),
473+
userCredentials.getPassword());
474+
String encodedAuth = Base64.getEncoder()
475+
.encodeToString(userCredentialsUrlFormat.getBytes());
476+
builder.header(HttpHeaders.AUTHORIZATION, "Basic " + encodedAuth);
477+
} else if (userInfo != null) {
478+
builder.uri(URI.create(decodedUrl.replace(uri.getRawUserInfo() + "@", "")));
454479
String encodedAuth = Base64.getEncoder()
455480
.encodeToString(userInfo.getBytes());
456481
builder.header(HttpHeaders.AUTHORIZATION, "Basic " + encodedAuth);

‎multiapps-controller-web/src/test/java/org/cloudfoundry/multiapps/controller/web/api/impl/FilesApiServiceImplTest.java‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -231,7 +231,7 @@ void testUploadFileFromUrl() throws Exception {
231231
when(future.isDone()).thenReturn(true);
232232
prepareAsyncExecutor(future);
233233

234-
ResponseEntity<Void> startUploadResponse = testedClass.startUploadFromUrl(SPACE_GUID, NAMESPACE, ImmutableFileUrl.of(FILE_URL));
234+
ResponseEntity<Void> startUploadResponse = testedClass.startUploadFromUrl(SPACE_GUID, NAMESPACE, ImmutableFileUrl.of(FILE_URL, null));
235235

236236
assertEquals(startUploadResponse.getStatusCode(), HttpStatus.ACCEPTED);
237237

@@ -322,7 +322,7 @@ void testFileUrlDoesntReturnContentLength() throws Exception {
322322
when(future.isDone()).thenReturn(true);
323323
prepareAsyncExecutor(future);
324324

325-
ResponseEntity<Void> startUploadResponse = testedClass.startUploadFromUrl(SPACE_GUID, NAMESPACE, ImmutableFileUrl.of(FILE_URL));
325+
ResponseEntity<Void> startUploadResponse = testedClass.startUploadFromUrl(SPACE_GUID, NAMESPACE, ImmutableFileUrl.of(FILE_URL, null));
326326

327327
assertEquals(startUploadResponse.getStatusCode(), HttpStatus.ACCEPTED);
328328

@@ -348,12 +348,12 @@ void testUploadFromUrlWhenThereIsValidExistingJob() {
348348
Future<?> runningTask = mock(Future.class);
349349
prepareAsyncExecutor(runningTask);
350350
when(uploadJobService.update(any(), any())).thenReturn(jobEntry);
351-
ResponseEntity<Void> firstUpload = testedClass.startUploadFromUrl(SPACE_GUID, NAMESPACE, ImmutableFileUrl.of(FILE_URL));
351+
ResponseEntity<Void> firstUpload = testedClass.startUploadFromUrl(SPACE_GUID, NAMESPACE, ImmutableFileUrl.of(FILE_URL, null));
352352
String locationHeader = firstUpload.getHeaders()
353353
.getFirst(org.springframework.http.HttpHeaders.LOCATION);
354354
String createdJobId = locationHeader.substring(locationHeader.lastIndexOf("/") + 1);
355355
when(jobEntry.getId()).thenReturn(createdJobId);
356-
ResponseEntity<Void> secondUpload = testedClass.startUploadFromUrl(SPACE_GUID, NAMESPACE, ImmutableFileUrl.of(FILE_URL));
356+
ResponseEntity<Void> secondUpload = testedClass.startUploadFromUrl(SPACE_GUID, NAMESPACE, ImmutableFileUrl.of(FILE_URL, null));
357357
assertEquals(HttpStatus.SEE_OTHER, secondUpload.getStatusCode());
358358
}
359359

@@ -385,7 +385,7 @@ void testFileUrlReturnsContentLengthAboveMaxUploadSize() throws Exception {
385385
when(future.isDone()).thenReturn(true);
386386
prepareAsyncExecutor(future);
387387

388-
ResponseEntity<Void> startUploadResponse = testedClass.startUploadFromUrl(SPACE_GUID, NAMESPACE, ImmutableFileUrl.of(FILE_URL));
388+
ResponseEntity<Void> startUploadResponse = testedClass.startUploadFromUrl(SPACE_GUID, NAMESPACE, ImmutableFileUrl.of(FILE_URL, null));
389389

390390
assertEquals(startUploadResponse.getStatusCode(), HttpStatus.ACCEPTED);
391391

@@ -432,7 +432,7 @@ void testUploadFileWithInvalidUrl(String url) throws Exception {
432432
.encodeToString(url.getBytes(StandardCharsets.UTF_8));
433433

434434
ResponseEntity<Void> startUploadResponse = testedClass.startUploadFromUrl(SPACE_GUID, NAMESPACE,
435-
ImmutableFileUrl.of(invalidFileUrl));
435+
ImmutableFileUrl.of(invalidFileUrl, null));
436436

437437
assertEquals(startUploadResponse.getStatusCode(), HttpStatus.ACCEPTED);
438438

0 commit comments

Comments
 (0)