-
Notifications
You must be signed in to change notification settings - Fork 2.4k
Expand file tree
/
Copy pathfetch_strategy.py
More file actions
1838 lines (1453 loc) · 62.4 KB
/
fetch_strategy.py
File metadata and controls
1838 lines (1453 loc) · 62.4 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
# Copyright Spack Project Developers. See COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
"""
Fetch strategies are used to download source code into a staging area
in order to build it. They need to define the following methods:
``fetch()``
This should attempt to download/check out source from somewhere.
``check()``
Apply a checksum to the downloaded source code, e.g. for an archive.
May not do anything if the fetch method was safe to begin with.
``expand()``
Expand (e.g., an archive) downloaded file to source, with the
standard stage source path as the destination directory.
``reset()``
Restore original state of downloaded code. Used by clean commands.
This may just remove the expanded source and re-expand an archive,
or it may run something like git reset ``--hard``.
``archive()``
Archive a source directory, e.g. for creating a mirror.
"""
import copy
import functools
import hashlib
import http.client
import os
import re
import shutil
import sys
import time
import urllib.parse
import urllib.request
from pathlib import PurePath
from typing import Callable, List, Mapping, Optional, Type
import spack.config
import spack.error
import spack.llnl.url
import spack.llnl.util.filesystem as fs
import spack.llnl.util.tty as tty
import spack.oci.opener
import spack.util.archive
import spack.util.crypto as crypto
import spack.util.executable
import spack.util.git
import spack.util.url as url_util
import spack.util.web as web_util
import spack.version
from spack.llnl.string import comma_and, quote
from spack.llnl.util.filesystem import get_single_file, mkdirp, symlink, temp_cwd, working_dir
from spack.util.compression import decompressor_for
from spack.util.executable import CommandNotFoundError, Executable, which
#: List of all fetch strategies, created by FetchStrategy metaclass.
all_strategies: List[Type["FetchStrategy"]] = []
def _needs_stage(fun):
"""Many methods on fetch strategies require a stage to be set
using set_stage(). This decorator adds a check for self.stage."""
@functools.wraps(fun)
def wrapper(self, *args, **kwargs):
if not self.stage:
raise NoStageError(fun)
return fun(self, *args, **kwargs)
return wrapper
def _ensure_one_stage_entry(stage_path):
"""Ensure there is only one stage entry in the stage path."""
stage_entries = os.listdir(stage_path)
assert len(stage_entries) == 1
return os.path.join(stage_path, stage_entries[0])
def fetcher(cls):
"""Decorator used to register fetch strategies."""
all_strategies.append(cls)
return cls
class FetchStrategy:
"""Superclass of all fetch strategies."""
#: The URL attribute must be specified either at the package class
#: level, or as a keyword argument to ``version()``. It is used to
#: distinguish fetchers for different versions in the package DSL.
url_attr: Optional[str] = None
#: Optional attributes can be used to distinguish fetchers when :
#: classes have multiple ``url_attrs`` at the top-level.
# optional attributes in version() args.
optional_attrs: List[str] = []
def __init__(self, **kwargs):
# The stage is initialized late, so that fetch strategies can be
# constructed at package construction time. This is where things
# will be fetched.
self.stage = None
# Enable or disable caching for this strategy based on
# 'no_cache' option from version directive.
self.cache_enabled = not kwargs.pop("no_cache", False)
self.package = None
def set_package(self, package):
self.package = package
# Subclasses need to implement these methods
def fetch(self):
"""Fetch source code archive or repo.
Returns:
bool: True on success, False on failure.
"""
def check(self):
"""Checksum the archive fetched by this FetchStrategy."""
def expand(self):
"""Expand the downloaded archive into the stage source path."""
def reset(self):
"""Revert to freshly downloaded state.
For archive files, this may just re-expand the archive.
"""
def archive(self, destination):
"""Create an archive of the downloaded data for a mirror.
For downloaded files, this should preserve the checksum of the
original file. For repositories, it should just create an
expandable tarball out of the downloaded repository.
"""
@property
def cachable(self):
"""Whether fetcher is capable of caching the resource it retrieves.
This generally is determined by whether the resource is
identifiably associated with a specific package version.
Returns:
bool: True if can cache, False otherwise.
"""
def source_id(self):
"""A unique ID for the source.
It is intended that a human could easily generate this themselves using
the information available to them in the Spack package.
The returned value is added to the content which determines the full
hash for a package using :class:`str`.
"""
raise NotImplementedError
def mirror_id(self):
"""This is a unique ID for a source that is intended to help identify
reuse of resources across packages.
It is unique like source-id, but it does not include the package name
and is not necessarily easy for a human to create themselves.
"""
raise NotImplementedError
def __str__(self): # Should be human readable URL.
return "FetchStrategy.__str___"
@classmethod
def matches(cls, args):
"""Predicate that matches fetch strategies to arguments of
the version directive.
Args:
args: arguments of the version directive
"""
return cls.url_attr in args
@fetcher
class BundleFetchStrategy(FetchStrategy):
"""
Fetch strategy associated with bundle, or no-code, packages.
Having a basic fetch strategy is a requirement for executing post-install
hooks. Consequently, this class provides the API but does little more
than log messages.
TODO: Remove this class by refactoring resource handling and the link
between composite stages and composite fetch strategies (see #11981).
"""
#: There is no associated URL keyword in ``version()`` for no-code
#: packages but this property is required for some strategy-related
#: functions (e.g., check_pkg_attributes).
url_attr = ""
def fetch(self):
"""Simply report success -- there is no code to fetch."""
return True
@property
def cachable(self):
"""Report False as there is no code to cache."""
return False
def source_id(self):
"""BundlePackages don't have a source id."""
return ""
def mirror_id(self):
"""BundlePackages don't have a mirror id."""
def _format_speed(total_bytes: int, elapsed: float) -> str:
"""Return a human-readable average download speed string."""
elapsed = 1 if elapsed <= 0 else elapsed # avoid divide by zero
speed = total_bytes / elapsed
if speed >= 1e9:
return f"{speed / 1e9:6.1f} GB/s"
elif speed >= 1e6:
return f"{speed / 1e6:6.1f} MB/s"
elif speed >= 1e3:
return f"{speed / 1e3:6.1f} KB/s"
return f"{speed:6.1f} B/s"
def _format_bytes(total_bytes: int) -> str:
"""Return a human-readable total bytes string."""
if total_bytes >= 1e9:
return f"{total_bytes / 1e9:7.2f} GB"
elif total_bytes >= 1e6:
return f"{total_bytes / 1e6:7.2f} MB"
elif total_bytes >= 1e3:
return f"{total_bytes / 1e3:7.2f} KB"
return f"{total_bytes:7.2f} B"
class FetchProgress:
#: Characters to rotate in the spinner.
spinner = ["|", "/", "-", "\\"]
def __init__(
self,
total_bytes: Optional[int] = None,
enabled: bool = True,
get_time: Callable[[], float] = time.time,
) -> None:
"""Initialize a FetchProgress instance.
Args:
total_bytes: Total number of bytes to download, if known.
enabled: Whether to print progress information.
get_time: Function to get the current time."""
#: Number of bytes downloaded so far.
self.current_bytes = 0
#: Delta time between progress prints
self.delta = 0.1
#: Whether to print progress information.
self.enabled = enabled
#: Function to get the current time.
self.get_time = get_time
#: Time of last progress print to limit output
self.last_printed = 0.0
#: Time of start of download
self.start_time = get_time() if enabled else 0.0
#: Total number of bytes to download, if known.
self.total_bytes = total_bytes if total_bytes and total_bytes > 0 else 0
#: Index of spinner character to print (used if total bytes is unknown)
self.index = 0
@classmethod
def from_headers(
cls,
headers: Mapping[str, str],
enabled: bool = True,
get_time: Callable[[], float] = time.time,
) -> "FetchProgress":
"""Create a FetchProgress instance from HTTP headers."""
# headers.get is case-insensitive if it's from a HTTPResponse object.
content_length = headers.get("Content-Length")
try:
total_bytes = int(content_length) if content_length else None
except ValueError:
total_bytes = None
return cls(total_bytes=total_bytes, enabled=enabled, get_time=get_time)
def advance(self, num_bytes: int, out=sys.stdout) -> None:
if not self.enabled:
return
self.current_bytes += num_bytes
self.print(out=out)
def print(self, final: bool = False, out=sys.stdout) -> None:
if not self.enabled:
return
current_time = self.get_time()
if self.last_printed + self.delta < current_time or final:
self.last_printed = current_time
# print a newline if this is the final update
maybe_newline = "\n" if final else ""
# if we know the total bytes, show a percentage, otherwise a spinner
if self.total_bytes > 0:
percentage = min(100 * self.current_bytes / self.total_bytes, 100.0)
percent_or_spinner = f"[{percentage:3.0f}%] "
else:
# only show the spinner if we are not at 100%
if final:
percent_or_spinner = "[100%] "
else:
percent_or_spinner = f"[ {self.spinner[self.index]} ] "
self.index = (self.index + 1) % len(self.spinner)
print(
f"\r {percent_or_spinner}{_format_bytes(self.current_bytes)} "
f"@ {_format_speed(self.current_bytes, current_time - self.start_time)}"
f"{maybe_newline}",
end="",
flush=True,
file=out,
)
@fetcher
class URLFetchStrategy(FetchStrategy):
"""URLFetchStrategy pulls source code from a URL for an archive, check the
archive against a checksum, and decompresses the archive.
The destination for the resulting file(s) is the standard stage path.
"""
url_attr = "url"
# these are checksum types. The generic 'checksum' is deprecated for
# specific hash names, but we need it for backward compatibility
optional_attrs = [*crypto.hashes.keys(), "checksum"]
def __init__(self, *, url: str, checksum: Optional[str] = None, **kwargs) -> None:
super().__init__(**kwargs)
self.url = url
self.mirrors = kwargs.get("mirrors", [])
# digest can be set as the first argument, or from an explicit
# kwarg by the hash name.
self.digest: Optional[str] = checksum
for h in self.optional_attrs:
if h in kwargs:
self.digest = kwargs[h]
self.expand_archive: bool = kwargs.get("expand", True)
self.extra_options: dict = kwargs.get("fetch_options", {})
self._curl: Optional[Executable] = None
self.extension: Optional[str] = kwargs.get("extension", None)
self._effective_url: Optional[str] = None
@property
def curl(self) -> Executable:
if not self._curl:
self._curl = web_util.require_curl()
return self._curl
def source_id(self):
return self.digest
def mirror_id(self):
if not self.digest:
return None
# The filename is the digest. A directory is also created based on
# truncating the digest to avoid creating a directory with too many
# entries
return os.path.sep.join(["archive", self.digest[:2], self.digest])
@property
def candidate_urls(self):
return [self.url] + (self.mirrors or [])
@_needs_stage
def fetch(self):
if self.archive_file:
tty.debug(f"Already downloaded {self.archive_file}")
return
errors: List[Exception] = []
for url in self.candidate_urls:
try:
self._fetch_from_url(url)
break
except FailedDownloadError as e:
errors.extend(e.exceptions)
else:
raise FailedDownloadError(*errors)
if not self.archive_file:
raise FailedDownloadError(
RuntimeError(f"Missing archive {self.archive_file} after fetching")
)
def _fetch_from_url(self, url):
fetch_method = spack.config.get("config:url_fetch_method", "urllib")
if fetch_method.startswith("curl"):
return self._fetch_curl(url, config_args=fetch_method.split()[1:])
else:
return self._fetch_urllib(url)
def _check_headers(self, headers):
# Check if we somehow got an HTML file rather than the archive we
# asked for. We only look at the last content type, to handle
# redirects properly.
content_types = re.findall(r"Content-Type:[^\r\n]+", headers, flags=re.IGNORECASE)
if content_types and "text/html" in content_types[-1]:
msg = (
f"The contents of {self.archive_file or 'the archive'} fetched from {self.url} "
" looks like HTML. This can indicate a broken URL, or an internet gateway issue."
)
if self._effective_url != self.url:
msg += f" The URL redirected to {self._effective_url}."
tty.warn(msg)
@_needs_stage
def _fetch_urllib(self, url, chunk_size=65536, retries=5):
"""Fetch a URL using urllib, with retries on transient errors and progress reporting."""
save_file = self.stage.save_filename
part_file = save_file + ".part"
request = urllib.request.Request(
url, headers={"User-Agent": web_util.SPACK_USER_AGENT, "Accept": "*/*"}
)
response_headers_str = None
for attempt in range(retries):
try:
with web_util.urlopen(request) as response:
tty.verbose(f"Fetching {url}")
progress = FetchProgress.from_headers(
response.headers, enabled=sys.stdout.isatty()
)
with open(part_file, "wb") as f:
while True:
chunk = response.read(chunk_size)
if not chunk:
break
f.write(chunk)
progress.advance(len(chunk))
progress.print(final=True)
# Capture metadata before context manager closes the connection
if isinstance(response, http.client.HTTPResponse):
self._effective_url = response.geturl()
response_headers_str = str(response.headers)
os.replace(part_file, save_file)
break # success: exit retry loop
except Exception as e:
# clean up archive on failure.
if self.archive_file:
os.remove(self.archive_file)
if os.path.lexists(part_file):
os.remove(part_file)
# Raise if this was the last attempt, or if the error was not transient.
if (attempt + 1 == retries) or not web_util.is_transient_error(e):
raise FailedDownloadError(e) from e
tty.debug(f"Retrying fetch (attempt {attempt + 1}): {e}")
time.sleep(2**attempt)
# Save the redirected URL for error messages. Sometimes we're redirected to an arbitrary
# mirror that is broken, leading to spurious download failures. In that case it's helpful
# for users to know which URL was actually fetched.
self._check_headers(response_headers_str)
@_needs_stage
def _fetch_curl(self, url, config_args=[]):
save_file = None
partial_file = None
if self.stage.save_filename:
save_file = self.stage.save_filename
partial_file = self.stage.save_filename + ".part"
tty.verbose(f"Fetching {url}")
if partial_file:
save_args = [
"-C",
"-", # continue partial downloads
"-o",
partial_file,
] # use a .part file
else:
save_args = ["-O"]
timeout = 0
cookie_args = []
if self.extra_options:
cookie = self.extra_options.get("cookie")
if cookie:
cookie_args.append("-j") # junk cookies
cookie_args.append("-b") # specify cookie
cookie_args.append(cookie)
timeout = self.extra_options.get("timeout")
base_args = web_util.base_curl_fetch_args(url, timeout)
curl_args = config_args + save_args + base_args + cookie_args
# Run curl but grab the mime type from the http headers
curl = self.curl
with working_dir(self.stage.path):
headers = curl(*curl_args, output=str, fail_on_error=False)
if curl.returncode != 0:
# clean up archive on failure.
if self.archive_file:
os.remove(self.archive_file)
if partial_file and os.path.lexists(partial_file):
os.remove(partial_file)
try:
web_util.check_curl_code(curl.returncode)
except spack.error.FetchError as e:
raise FailedDownloadError(e) from e
self._check_headers(headers)
if save_file and (partial_file is not None):
fs.rename(partial_file, save_file)
@property # type: ignore # decorated properties unsupported in mypy
@_needs_stage
def archive_file(self):
"""Path to the source archive within this stage directory."""
return self.stage.archive_file
@property
def cachable(self):
return self.cache_enabled and bool(self.digest)
@_needs_stage
def expand(self):
if not self.expand_archive:
tty.debug(
"Staging unexpanded archive {0} in {1}".format(
self.archive_file, self.stage.source_path
)
)
if not self.stage.expanded:
mkdirp(self.stage.source_path)
dest = os.path.join(self.stage.source_path, os.path.basename(self.archive_file))
shutil.move(self.archive_file, dest)
return
tty.debug("Staging archive: {0}".format(self.archive_file))
if not self.archive_file:
raise NoArchiveFileError(
"Couldn't find archive file", "Failed on expand() for URL %s" % self.url
)
# TODO: replace this by mime check.
if not self.extension:
self.extension = spack.llnl.url.determine_url_file_extension(self.url)
if self.stage.expanded:
tty.debug("Source already staged to %s" % self.stage.source_path)
return
decompress = decompressor_for(self.archive_file, self.extension)
# Below we assume that the command to decompress expand the
# archive in the current working directory
with fs.exploding_archive_catch(self.stage):
decompress(self.archive_file)
def archive(self, destination):
"""Just moves this archive to the destination."""
if not self.archive_file:
raise NoArchiveFileError("Cannot call archive() before fetching.")
web_util.push_to_url(
self.archive_file, url_util.path_to_file_url(destination), keep_original=True
)
@_needs_stage
def check(self):
"""Check the downloaded archive against a checksum digest.
No-op if this stage checks code out of a repository."""
if not self.digest:
raise NoDigestError(f"Attempt to check {self.__class__.__name__} with no digest.")
verify_checksum(self.archive_file, self.digest, self.url, self._effective_url)
@_needs_stage
def reset(self):
"""
Removes the source path if it exists, then re-expands the archive.
"""
if not self.archive_file:
raise NoArchiveFileError(
f"Tried to reset {self.__class__.__name__} before fetching",
f"Failed on reset() for URL{self.url}",
)
# Remove everything but the archive from the stage
for filename in os.listdir(self.stage.path):
abspath = os.path.join(self.stage.path, filename)
if abspath != self.archive_file:
shutil.rmtree(abspath, ignore_errors=True)
# Expand the archive again
self.expand()
def __repr__(self):
return f"{self.__class__.__name__}<{self.url}>"
def __str__(self):
return self.url
@fetcher
class CacheURLFetchStrategy(URLFetchStrategy):
"""The resource associated with a cache URL may be out of date."""
@_needs_stage
def fetch(self):
path = url_util.file_url_string_to_path(self.url)
# check whether the cache file exists.
if not os.path.isfile(path):
raise NoCacheError(f"No cache of {path}")
# remove old symlink if one is there.
filename = self.stage.save_filename
if os.path.lexists(filename):
os.remove(filename)
# Symlink to local cached archive.
symlink(path, filename)
# Remove link if checksum fails, or subsequent fetchers will assume they don't need to
# download.
if self.digest:
try:
self.check()
except ChecksumError:
os.remove(self.archive_file)
raise
# Notify the user how we fetched.
tty.msg(f"Using cached archive: {path}")
class OCIRegistryFetchStrategy(URLFetchStrategy):
def __init__(self, *, url: str, checksum: Optional[str] = None, **kwargs):
super().__init__(url=url, checksum=checksum, **kwargs)
self._urlopen = kwargs.get("_urlopen", spack.oci.opener.urlopen)
@_needs_stage
def fetch(self):
file = self.stage.save_filename
if os.path.lexists(file):
os.remove(file)
try:
response = self._urlopen(self.url)
tty.verbose(f"Fetching {self.url}")
with open(file, "wb") as f:
shutil.copyfileobj(response, f)
except OSError as e:
# clean up archive on failure.
if self.archive_file:
os.remove(self.archive_file)
if os.path.lexists(file):
os.remove(file)
raise FailedDownloadError(e) from e
class VCSFetchStrategy(FetchStrategy):
"""Superclass for version control system fetch strategies.
Like all fetchers, VCS fetchers are identified by the attributes
passed to the ``version`` directive. The optional_attrs for a VCS
fetch strategy represent types of revisions, e.g. tags, branches,
commits, etc.
The required attributes (git, svn, etc.) are used to specify the URL
and to distinguish a VCS fetch strategy from a URL fetch strategy.
"""
def __init__(self, **kwargs):
super().__init__(**kwargs)
# Set a URL based on the type of fetch strategy.
self.url = kwargs.get(self.url_attr, None)
if not self.url:
raise ValueError(f"{self.__class__} requires {self.url_attr} argument.")
for attr in self.optional_attrs:
setattr(self, attr, kwargs.get(attr, None))
@_needs_stage
def check(self):
tty.debug(f"No checksum needed when fetching with {self.url_attr}")
@_needs_stage
def expand(self):
tty.debug(f"Source fetched with {self.url_attr} is already expanded.")
@_needs_stage
def archive(self, destination, *, exclude: Optional[str] = None):
assert spack.llnl.url.extension_from_path(destination) == "tar.gz"
assert self.stage.source_path.startswith(self.stage.path)
# We need to prepend this dir name to every entry of the tarfile
top_level_dir = PurePath(self.stage.srcdir or os.path.basename(self.stage.source_path))
with working_dir(self.stage.source_path), spack.util.archive.gzip_compressed_tarfile(
destination
) as (tar, _, _):
spack.util.archive.reproducible_tarfile_from_prefix(
tar=tar,
prefix=".",
skip=lambda entry: entry.name == exclude,
path_to_name=lambda path: (top_level_dir / PurePath(path)).as_posix(),
)
def __str__(self):
return f"VCS: {self.url}"
def __repr__(self):
return f"{self.__class__}<{self.url}>"
@fetcher
class GoFetchStrategy(VCSFetchStrategy):
"""Fetch strategy that employs the ``go get`` infrastructure.
Use like this in a package::
version("name", go="github.com/monochromegane/the_platinum_searcher/...")
Go get does not natively support versions, they can be faked with git.
The fetched source will be moved to the standard stage sourcepath directory
during the expand step.
"""
url_attr = "go"
def __init__(self, **kwargs):
# Discards the keywords in kwargs that may conflict with the next
# call to __init__
forwarded_args = copy.copy(kwargs)
forwarded_args.pop("name", None)
super().__init__(**forwarded_args)
self._go = None
@property
def go_version(self):
vstring = self.go("version", output=str).split(" ")[2]
return spack.version.Version(vstring)
@property
def go(self):
if not self._go:
self._go = which("go", required=True)
return self._go
@_needs_stage
def fetch(self):
tty.debug("Getting go resource: {0}".format(self.url))
with working_dir(self.stage.path):
try:
os.mkdir("go")
except OSError:
pass
env = dict(os.environ)
env["GOPATH"] = os.path.join(os.getcwd(), "go")
self.go("get", "-v", "-d", self.url, env=env)
def archive(self, destination):
super().archive(destination, exclude=".git")
@_needs_stage
def expand(self):
tty.debug("Source fetched with %s is already expanded." % self.url_attr)
# Move the directory to the well-known stage source path
repo_root = _ensure_one_stage_entry(self.stage.path)
shutil.move(repo_root, self.stage.source_path)
@_needs_stage
def reset(self):
with working_dir(self.stage.source_path):
self.go("clean")
def __str__(self):
return "[go] %s" % self.url
@fetcher
class GitFetchStrategy(VCSFetchStrategy):
"""
Fetch strategy that gets source code from a git repository.
Use like this in a package::
version("name", git="https://github.com/project/repo.git")
Optionally, you can provide a branch, or commit to check out, e.g.::
version("1.1", git="https://github.com/project/repo.git", tag="v1.1")
You can use these three optional attributes in addition to ``git``:
* ``branch``: Particular branch to build from (default is the repository's default branch)
* ``tag``: Particular tag to check out
* ``commit``: Particular commit hash in the repo
Repositories are cloned into the standard stage source path directory.
"""
url_attr = "git"
optional_attrs = [
"tag",
"branch",
"commit",
"submodules",
"get_full_repo",
"submodules_delete",
"git_sparse_paths",
"skip_checkout",
]
def __init__(self, **kwargs):
self.commit: Optional[str] = None
self.tag: Optional[str] = None
self.branch: Optional[str] = None
# Discards the keywords in kwargs that may conflict with the next call
# to __init__
forwarded_args = copy.copy(kwargs)
forwarded_args.pop("name", None)
super().__init__(**forwarded_args)
self._git = None
self.submodules = kwargs.get("submodules", False)
self.submodules_delete = kwargs.get("submodules_delete", False)
self.get_full_repo = kwargs.get("get_full_repo", False)
self.git_sparse_paths = kwargs.get("git_sparse_paths", None)
# skipping checkout with a blobless clone is an efficient way to traverse meta-data
# see https://bhupesh.me/minimalist-guide-git-clone/
self.skip_checkout = kwargs.get("skip_checkout", False)
@property
def git_version(self):
return GitFetchStrategy.version_from_git(self.git)
@staticmethod
def version_from_git(git_exe):
"""Given a git executable, return the Version (this will fail if
the output cannot be parsed into a valid Version).
"""
version_string = ".".join(map(str, git_exe.version))
return spack.version.Version(version_string)
@property
def git(self):
if not self._git:
try:
self._git = spack.util.git.git(required=True)
except CommandNotFoundError as exc:
tty.error(str(exc))
raise
# Disable advice for a quieter fetch
# https://github.com/git/git/blob/master/Documentation/RelNotes/1.7.2.txt
if self.git_version >= spack.version.Version("1.7.2"):
self._git.add_default_arg("-c", "advice.detachedHead=false")
# If the user asked for insecure fetching, make that work
# with git as well.
if not spack.config.get("config:verify_ssl"):
self._git.add_default_env("GIT_SSL_NO_VERIFY", "true")
return self._git
@property
def cachable(self):
return self.cache_enabled and bool(self.commit)
def source_id(self):
# TODO: tree-hash would secure download cache and mirrors, commit only secures checkouts.
# TODO(psakiev): Tree-hash is part of the commit SHA computation, question comment validity
return self.commit
def mirror_id(self):
if self.commit:
provenance_id = self.commit
repo_path = urllib.parse.urlparse(self.url).path
if self.git_sparse_paths:
sparse_paths = []
if callable(self.git_sparse_paths):
sparse_paths.extend(self.git_sparse_paths())
else:
sparse_paths.extend(self.git_sparse_paths)
sparse_string = "_".join(sparse_paths)
sparse_hash = hashlib.sha1(sparse_string.encode("utf-8")).hexdigest()
provenance_id = f"{provenance_id}_{sparse_hash}"
result = os.path.sep.join(["git", repo_path, provenance_id])
return result
def _repo_info(self):
args = ""
if self.commit:
args = f" at commit {self.commit}"
elif self.tag:
args = f" at tag {self.tag}"
elif self.branch:
args = f" on branch {self.branch}"
return f"{self.url}{args}"
@_needs_stage
def fetch(self):
if self.stage.expanded:
tty.debug(f"Already fetched {self.stage.source_path}")
return
self._clone_src()
self.submodule_operations()
def bare_clone(self, dest: str) -> None:
"""
Execute a bare clone for metadata only
Requires a destination since bare cloning does not provide source
and shouldn't be used for staging.
"""
# Default to spack source path
tty.debug(f"Cloning git repository: {self._repo_info()}")
git = self.git
debug = spack.config.get("config:debug")
# We don't need to worry about which commit/branch/tag is checked out
clone_args = ["clone", "--bare"]
if not debug:
clone_args.append("--quiet")
clone_args.extend([self.url, dest])
git(*clone_args)
def _clone_src(self) -> None:
"""Clone a repository to a path using git."""
# Default to spack source path
dest = self.stage.source_path
tty.debug(f"Cloning git repository: {self._repo_info()}")
depth = None if self.get_full_repo else 1
name = self.package.name if self.package else None
checkout_ref = self.commit or self.tag or self.branch
fetch_ref = self.tag or self.branch
kwargs = {"debug": spack.config.get("config:debug"), "git_exe": self.git, "dest": name}
# TODO(psakievich) The use of the minimal clone need clearer justification via package API
# or something. There is a trade space of storage minimization vs available git information
# that grows to non-trivial proportions for larger projects
minimal_clone = self.commit and name and not self.get_full_repo
with temp_cwd(ignore_cleanup_errors=True):
if minimal_clone:
try:
spack.util.git.git_init_fetch(self.url, self.commit, depth, **kwargs)
except spack.util.executable.ProcessError:
spack.util.git.git_clone(
self.url, fetch_ref, self.get_full_repo, depth, **kwargs
)
else:
spack.util.git.git_clone(self.url, fetch_ref, self.get_full_repo, depth, **kwargs)
repo_name = get_single_file(".")
kwargs["dest"] = repo_name
if not self.skip_checkout:
spack.util.git.git_checkout(checkout_ref, self.git_sparse_paths, **kwargs)
if self.stage:
self.stage.srcdir = repo_name
shutil.copytree(repo_name, dest, symlinks=True)
return
def submodule_operations(self):