-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathyahooscrape.sh
More file actions
executable file
·60 lines (48 loc) · 1.23 KB
/
yahooscrape.sh
File metadata and controls
executable file
·60 lines (48 loc) · 1.23 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
#!/bin/bash
#yahoo! finance stock symbol scraper
#requires 'links'
#which pages?
markets=(
trending-tickers
most-active
gainers
losers
etfs
commodities
world-indices
currencies
mutualfunds
bonds
options/highest-implied-volatility
options/highest-open-interest
sector/ms_basic_materials
sector/ms_communication_services
sector/ms_consumer_cyclical
sector/ms_consumer_defensive
sector/ms_energy
sector/ms_financial_services
sector/ms_healthcare
sector/ms_industrials
sector/ms_real_estate
sector/ms_technology
sector/ms_utilities
)
#get data, cut table and print it
getf() {
D="$(links -dump -width 300 "https://finance.yahoo.com/$m?offset=$off&count=$count" | sed -n '/Symbol.*Name/,/^$/p')"
#print table
printf '%s\n' "$D"
#print stats (address and items per page on stderr)
printf '%s items: %s\n' "https://finance.yahoo.com/$m?offset=$off&count=$count" "$(wc -l <<<"$D")" 1>&2
#sleep 0.2
}
#scrape loop
for m in "${markets[@]}"; do
count=100; off=0
getf
while (( $(wc -l <<<"$D") > count )); do
off=$((count+off))
getf
done
done #| grep -v 'Symbol.*Name' | sed -Ee 's/^(\]|\[|\s|\t)*([A-Z]*)/\2/' -e 's/[0-9]+\.[0-9].*$//' | sed 's/\s/\t/'
#uncomment above line to process tables further