Skip to content

Commit c9d2303

Browse files
DaanHooglandDaanHoogland
committed
Merge release branch 4.18 to main
* 4.18: test,refactor: fix test_project_resources cleanup (#8097) kvm: fix direct download template size (#8093)
2 parents e437d10 + 065abe2 commit c9d2303

6 files changed

Lines changed: 81 additions & 78 deletions

File tree

‎core/src/main/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloader.java‎

Lines changed: 27 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -19,29 +19,6 @@
1919

2020
package org.apache.cloudstack.direct.download;
2121

22-
import com.cloud.utils.Pair;
23-
import com.cloud.utils.exception.CloudRuntimeException;
24-
import com.cloud.utils.script.Script;
25-
import com.cloud.utils.storage.QCOW2Utils;
26-
import org.apache.cloudstack.utils.security.SSLUtils;
27-
import org.apache.commons.httpclient.HttpStatus;
28-
import org.apache.commons.io.IOUtils;
29-
import org.apache.http.Header;
30-
import org.apache.http.HttpEntity;
31-
import org.apache.http.client.methods.CloseableHttpResponse;
32-
import org.apache.commons.collections.MapUtils;
33-
import org.apache.http.client.config.RequestConfig;
34-
import org.apache.http.client.methods.HttpGet;
35-
import org.apache.http.client.methods.HttpHead;
36-
import org.apache.http.client.methods.HttpUriRequest;
37-
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
38-
import org.apache.http.impl.client.CloseableHttpClient;
39-
import org.apache.http.impl.client.HttpClients;
40-
import org.apache.http.util.EntityUtils;
41-
42-
import javax.net.ssl.HttpsURLConnection;
43-
import javax.net.ssl.SSLContext;
44-
import javax.net.ssl.TrustManager;
4522
import java.io.ByteArrayInputStream;
4623
import java.io.File;
4724
import java.io.FileInputStream;
@@ -60,6 +37,32 @@
6037
import java.util.List;
6138
import java.util.Map;
6239

40+
import javax.net.ssl.HttpsURLConnection;
41+
import javax.net.ssl.SSLContext;
42+
import javax.net.ssl.TrustManager;
43+
44+
import org.apache.cloudstack.utils.security.SSLUtils;
45+
import org.apache.commons.collections.MapUtils;
46+
import org.apache.commons.httpclient.HttpStatus;
47+
import org.apache.commons.io.IOUtils;
48+
import org.apache.http.Header;
49+
import org.apache.http.HttpEntity;
50+
import org.apache.http.client.config.RequestConfig;
51+
import org.apache.http.client.methods.CloseableHttpResponse;
52+
import org.apache.http.client.methods.HttpGet;
53+
import org.apache.http.client.methods.HttpHead;
54+
import org.apache.http.client.methods.HttpUriRequest;
55+
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
56+
import org.apache.http.impl.client.CloseableHttpClient;
57+
import org.apache.http.impl.client.HttpClients;
58+
import org.apache.http.util.EntityUtils;
59+
60+
import com.cloud.utils.Pair;
61+
import com.cloud.utils.UriUtils;
62+
import com.cloud.utils.exception.CloudRuntimeException;
63+
import com.cloud.utils.script.Script;
64+
import com.cloud.utils.storage.QCOW2Utils;
65+
6366
public class HttpsDirectTemplateDownloader extends DirectTemplateDownloaderImpl {
6467

6568
protected CloseableHttpClient httpsClient;
@@ -183,8 +186,7 @@ public Long getRemoteFileSize(String url, String format) {
183186
SSLContext context = getSSLContext();
184187
urlConnection.setSSLSocketFactory(context.getSocketFactory());
185188
urlConnection.connect();
186-
boolean isCompressed = !url.endsWith("qcow2");
187-
return QCOW2Utils.getVirtualSize(urlObj.openStream(), isCompressed);
189+
return QCOW2Utils.getVirtualSize(urlObj.openStream(), UriUtils.isUrlForCompressedFile(url));
188190
} catch (IOException e) {
189191
throw new CloudRuntimeException(String.format("Cannot obtain qcow2 virtual size due to: %s", e.getMessage()), e);
190192
}

‎test/integration/component/test_project_resources.py‎

Lines changed: 34 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -551,66 +551,59 @@ def setUpClass(cls):
551551
cls.hypervisor = cls.testClient.getHypervisorInfo()
552552
if cls.hypervisor.lower() in ['lxc']:
553553
raise unittest.SkipTest("create template from volume is not supported on %s" % cls.hypervisor.lower())
554+
cls._cleanup = []
554555

555556
cls.template = get_template(
556-
cls.api_client,
557-
cls.zone.id,
558-
cls.services["ostype"]
559-
)
557+
cls.api_client,
558+
cls.zone.id,
559+
cls.services["ostype"]
560+
)
560561
cls.services["server"]["zoneid"] = cls.zone.id
561562

562563
# Create Domains, Account etc
563564
cls.domain = Domain.create(
564-
cls.api_client,
565-
cls.services["domain"]
566-
)
565+
cls.api_client,
566+
cls.services["domain"]
567+
)
568+
cls._cleanup.append(cls.domain)
567569

568570
cls.account = Account.create(
569-
cls.api_client,
570-
cls.services["account"],
571-
domainid=cls.domain.id
572-
)
571+
cls.api_client,
572+
cls.services["account"],
573+
domainid=cls.domain.id
574+
)
575+
cls._cleanup.append(cls.account)
573576
cls.user = Account.create(
574-
cls.api_client,
575-
cls.services["account"],
576-
domainid=cls.domain.id
577-
)
577+
cls.api_client,
578+
cls.services["account"],
579+
domainid=cls.domain.id
580+
)
581+
cls._cleanup.append(cls.user)
578582
# Create project as a domain admin
579583
cls.project = Project.create(
580-
cls.api_client,
581-
cls.services["project"],
582-
account=cls.account.name,
583-
domainid=cls.account.domainid
584-
)
584+
cls.api_client,
585+
cls.services["project"],
586+
account=cls.account.name,
587+
domainid=cls.account.domainid
588+
)
589+
cls._cleanup.append(cls.project)
585590
cls.services["account"] = cls.account.name
586591

587592
# Create Service offering and disk offerings etc
588593
cls.service_offering = ServiceOffering.create(
589-
cls.api_client,
590-
cls.services["service_offering"]
591-
)
594+
cls.api_client,
595+
cls.services["service_offering"]
596+
)
597+
cls._cleanup.append(cls.service_offering)
592598
cls.userapiclient = cls.testClient.getUserApiClient(
593-
UserName=cls.account.name,
594-
DomainName=cls.domain.name
595-
)
596-
597-
cls._cleanup = [
598-
cls.project,
599-
cls.service_offering,
600-
cls.account,
601-
cls.user,
602-
cls.domain
603-
]
599+
UserName=cls.account.name,
600+
DomainName=cls.domain.name
601+
)
604602
return
605603

606604
@classmethod
607605
def tearDownClass(cls):
608-
try:
609-
#Cleanup resources used
610-
cleanup_resources(cls.api_client, cls._cleanup)
611-
except Exception as e:
612-
raise Exception("Warning: Exception during cleanup : %s" % e)
613-
return
606+
super(TestTemplates, cls).tearDownClass()
614607

615608
def setUp(self):
616609
self.apiclient = self.testClient.getApiClient()
@@ -619,12 +612,7 @@ def setUp(self):
619612
return
620613

621614
def tearDown(self):
622-
try:
623-
#Clean up, terminate the created instance, volumes and snapshots
624-
cleanup_resources(self.apiclient, self.cleanup)
625-
except Exception as e:
626-
raise Exception("Warning: Exception during cleanup : %s" % e)
627-
return
615+
super(TestTemplates, self).tearDown()
628616

629617
@attr(tags=["advanced", "basic", "sg", "eip", "advancedns"], required_hardware="false")
630618
def test_04_public_private_template_use_in_project(self):

‎utils/src/main/java/com/cloud/utils/UriUtils.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -408,14 +408,14 @@ public static List<String> getMetalinkUrls(String metalinkUrl) {
408408
return urls;
409409
}
410410

411-
public static final Set<String> COMMPRESSION_FORMATS = ImmutableSet.of("zip", "bz2", "gz");
411+
public static final Set<String> COMPRESSION_FORMATS = ImmutableSet.of("zip", "bz2", "gz");
412412

413413
public static final Set<String> buildExtensionSet(boolean metalink, String... baseExtensions) {
414414
final ImmutableSet.Builder<String> builder = ImmutableSet.builder();
415415

416416
for (String baseExtension : baseExtensions) {
417417
builder.add("." + baseExtension);
418-
for (String format : COMMPRESSION_FORMATS) {
418+
for (String format : COMPRESSION_FORMATS) {
419419
builder.add("." + baseExtension + "." + format);
420420
}
421421
}
@@ -647,4 +647,8 @@ private static UriInfo getRbdUrlInfo(String url) {
647647
throw new CloudRuntimeException(url + " is not a valid uri for RBD");
648648
}
649649
}
650+
651+
public static boolean isUrlForCompressedFile(String url) {
652+
return UriUtils.COMPRESSION_FORMATS.stream().anyMatch(f -> url.toLowerCase().endsWith(f));
653+
}
650654
}

‎utils/src/main/java/com/cloud/utils/storage/QCOW2Utils.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
import org.apache.log4j.Logger;
3333

3434
import com.cloud.utils.NumbersUtil;
35+
import com.cloud.utils.UriUtils;
3536

3637
public final class QCOW2Utils {
3738
public static final Logger LOGGER = Logger.getLogger(QCOW2Utils.class.getName());
@@ -114,7 +115,7 @@ private static long getVirtualSizeFromInputStream(InputStream inputStream) throw
114115
public static long getVirtualSize(String urlStr) {
115116
try {
116117
URL url = new URL(urlStr);
117-
return getVirtualSizeFromInputStream(url.openStream());
118+
return getVirtualSize(url.openStream(), UriUtils.isUrlForCompressedFile(urlStr));
118119
} catch (MalformedURLException e) {
119120
LOGGER.warn("Failed to validate for qcow2, malformed URL: " + urlStr + ", error: " + e.getMessage());
120121
throw new IllegalArgumentException("Invalid URL: " + urlStr);

‎utils/src/main/java/org/apache/cloudstack/utils/imagestore/ImageStoreUtil.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,7 @@ public static boolean isCorrectExtension(String path, String format) {
101101

102102
public static boolean isCompressedExtension(String path) {
103103
final String lowerCasePath = path.toLowerCase();
104-
return UriUtils.COMMPRESSION_FORMATS
104+
return UriUtils.COMPRESSION_FORMATS
105105
.stream()
106106
.map(extension -> "." + extension)
107107
.anyMatch(lowerCasePath::endsWith);

‎utils/src/test/java/com/cloud/utils/UriUtilsTest.java‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,13 +19,13 @@
1919

2020
package com.cloud.utils;
2121

22-
import org.junit.Assert;
23-
import org.junit.Test;
24-
2522
import java.util.Arrays;
2623
import java.util.Collections;
2724
import java.util.List;
2825

26+
import org.junit.Assert;
27+
import org.junit.Test;
28+
2929
public class UriUtilsTest {
3030
@Test
3131
public void encodeURIComponent() {
@@ -265,4 +265,12 @@ public void testGetUriInfoIpv6() {
265265
testGetUriInfoInternal(url11, host);
266266
testGetUriInfoInternal(url12, host);
267267
}
268+
269+
@Test
270+
public void testIsUrlForCompressedFile() {
271+
Assert.assertTrue(UriUtils.isUrlForCompressedFile("https://abc.com/xyz.bz2"));
272+
Assert.assertTrue(UriUtils.isUrlForCompressedFile("http://abc.com/xyz.zip"));
273+
Assert.assertTrue(UriUtils.isUrlForCompressedFile("https://abc.com/xyz.gz"));
274+
Assert.assertFalse(UriUtils.isUrlForCompressedFile("http://abc.com/xyz.qcow2"));
275+
}
268276
}

0 commit comments

Comments
 (0)