Skip to content

SYNPY-731 #571

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
zimingd merged 7 commits into from
Jul 10, 2018
Merged

SYNPY-731 #571

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
5a426ed
remove redundant test
kimyen Jul 9, 2018
9666d2f
update sftp server
kimyen Jul 9, 2018
6c171d5
better folder names
kimyen Jul 10, 2018
f1f611d
clean up
kimyen Jul 10, 2018
0cff55d
ignore hostly
kimyen Jul 10, 2018
1b2ae55
Revert "ignore hostly"
kimyen Jul 10, 2018
e641b62
update sftp server endpoint
kimyen Jul 10, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .travis.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ matrix:

addons:
#This is used for SFTP integration tests
ssh_known_hosts: ec2-54-212-85-156.us-west-2.compute.amazonaws.com
ssh_known_hosts: ec2-18-209-45-78.compute-1.amazonaws.com

before_install:
- |
Expand Down
2 changes: 1 addition & 1 deletion appveyor.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ install:
- "%PYTHON%\\python.exe setup.py install"

before_test:
- IF NOT DEFINED APPVEYOR_PULL_REQUEST_NUMBER ( ssh-keyscan -t ecdsa ec2-54-212-85-156.us-west-2.compute.amazonaws.com >> %userprofile%\\.ssh\\known_hosts )
- IF NOT DEFINED APPVEYOR_PULL_REQUEST_NUMBER ( ssh-keyscan -t ecdsa ec2-18-209-45-78.compute-1.amazonaws.com >> %userprofile%\\.ssh\\known_hosts )

test_script:
- cmd: "%PYTHON%\\python.exe -m nose -s tests\\unit"
Expand Down
Binary file modified test.synapseConfig.enc
View file
Open in desktop
Binary file not shown.
24 changes: 0 additions & 24 deletions tests/integration/test_download.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,30 +76,6 @@ def test_resume_partial_download():
assert_true(filecmp.cmp(original_file, path), "File comparison failed")


def test_ftp_download():
"""Test downloading an Entity that points to a file on an FTP server. """

# Another test with an external reference. This is because we only need to test FTP download; not upload.
# Also so we don't have to maintain an FTP server just for this purpose.
# Make an entity that points to an FTP server file.
entity = File(parent=project['id'], name='1KB.zip')
fileHandle = {'externalURL': 'ftp://speedtest.tele2.net/1KB.zip',
"fileName": entity.name,
"contentType": "application/zip",
"contentMd5": '0f343b0931126a20f133d67c2b018a3b',
"contentSize": 1024,
"concreteType": "org.sagebionetworks.repo.model.file.ExternalFileHandle"}
fileHandle = syn.restPOST('/externalFileHandle', json.dumps(fileHandle), syn.fileHandleEndpoint)
entity.dataFileHandleId = fileHandle['id']
entity = syn.store(entity)

# Download the entity and check that MD5 matches expected
FTPfile = syn.get(entity.id, downloadLocation=os.getcwd(), downloadFile=True)
assert_equals(FTPfile.md5, utils.md5_for_file(FTPfile.path).hexdigest())
schedule_for_cleanup(entity)
os.remove(FTPfile.path)


def test_http_download__range_request_error():
# SYNPY-525

Expand Down
15 changes: 9 additions & 6 deletions tests/integration/test_sftp_upload.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,15 +26,18 @@
import integration
from integration import schedule_for_cleanup

SFTP_SERVER_PREFIX = "sftp://ec2-18-209-45-78.compute-1.amazonaws.com"
SFTP_USER_HOME_PATH = "/home/sftpuser"

DESTINATIONS = [{"uploadType": "SFTP",
"description": 'EC2 subfolder A',
"supportsSubfolders": True,
"url": "sftp://ec2-54-212-85-156.us-west-2.compute.amazonaws.com/public/pythonClientIntegration/test%20space",
"url": SFTP_SERVER_PREFIX + SFTP_USER_HOME_PATH + "/folder_A",
"banner": "Uploading file to EC2\n"},
{"uploadType": "SFTP",
"supportsSubfolders": True,
"description": 'EC2 subfolder B',
"url": "sftp://ec2-54-212-85-156.us-west-2.compute.amazonaws.com/public/pythonClientIntegration/another_location",
"url": SFTP_SERVER_PREFIX + SFTP_USER_HOME_PATH + "/folder_B",
"banner": "Uploading file to EC2 version 2\n"}
]

Expand Down Expand Up @@ -78,10 +81,10 @@ def test_synStore_sftpIntegration():

def test_synGet_sftpIntegration():
# Create file by uploading directly to sftp and creating entity from URL
serverURL = 'sftp://ec2-54-212-85-156.us-west-2.compute.amazonaws.com/public/'+str(uuid.uuid1())
serverURL = SFTP_SERVER_PREFIX + SFTP_USER_HOME_PATH + '/test_synGet_sftpIntegration/' + str(uuid.uuid1())
filepath = utils.make_bogus_binary_file(1*MB - 777771)

username, password = syn._getUserCredentials(serverURL)
username, password = syn._getUserCredentials(SFTP_SERVER_PREFIX)

url = SFTPWrapper.upload_file(filepath, url=serverURL, username=username, password=password)
file = syn.store(File(path=url, parent=project, synapseStore=False))
Expand All @@ -92,12 +95,12 @@ def test_synGet_sftpIntegration():

def test_utils_sftp_upload_and_download():
"""Tries to upload a file to an sftp file """
serverURL = 'sftp://ec2-54-212-85-156.us-west-2.compute.amazonaws.com/public/'+str(uuid.uuid1())
serverURL = SFTP_SERVER_PREFIX + SFTP_USER_HOME_PATH + '/test_utils_sftp_upload_and_download/' + str(uuid.uuid1())
filepath = utils.make_bogus_binary_file(1*MB - 777771)

tempdir = tempfile.mkdtemp()

username, password = syn._getUserCredentials(serverURL)
username, password = syn._getUserCredentials(SFTP_SERVER_PREFIX)

try:
url = SFTPWrapper.upload_file(filepath, url=serverURL, username=username, password=password)
Expand Down