Python - yahoo finance download all s&p 500 stocks

余生颓废 提交于 2020-01-24 13:54:27

问题


I am trying to run the code from here to download all S&P 500 stocks:

https://pythonprogramming.net/sp500-company-price-data-python-programming-for-finance/

import bs4 as bs
import datetime as dt
import os
import pandas_datareader.data as web
import pickle
import requests


def save_sp500_tickers():
    resp = requests.get('http://en.wikipedia.org/wiki/List_of_S%26P_500_companies')
    soup = bs.BeautifulSoup(resp.text, 'lxml')
    table = soup.find('table', {'class': 'wikitable sortable'})
    tickers = []
    for row in table.findAll('tr')[1:]:
        ticker = row.findAll('td')[0].text
        tickers.append(ticker)
    with open("sp500tickers.pickle", "wb") as f:
        pickle.dump(tickers, f)
    return tickers


# save_sp500_tickers()
def get_data_from_yahoo(reload_sp500=False):
    if reload_sp500:
        tickers = save_sp500_tickers()
    else:
        with open("sp500tickers.pickle", "rb") as f:
            tickers = pickle.load(f)
    if not os.path.exists('stock_dfs'):
        os.makedirs('stock_dfs')

    start = dt.datetime(2010, 1, 1)
    end = dt.datetime.now()
    for ticker in tickers:
        # just in case your connection breaks, we'd like to save our progress!
        if not os.path.exists('stock_dfs/{}.csv'.format(ticker)):
            df = web.DataReader(ticker, 'morningstar', start, end)
            df.reset_index(inplace=True)
            df.set_index("Date", inplace=True)
            df = df.drop("Symbol", axis=1)
            df.to_csv('stock_dfs/{}.csv'.format(ticker))
        else:
            print('Already have {}'.format(ticker))


get_data_from_yahoo()

but I am getting this error:

Traceback (most recent call last):
  File "main.py", line 46, in <module>
    get_data_from_yahoo()
  File "main.py", line 27, in get_data_from_yahoo
    with open("sp500tickers.pickle", "rb") as f:
FileNotFoundError: [Errno 2] No such file or directory: 'sp500tickers.pickle'

how can I fix it?


回答1:


Apparently you did not save/create sp500tickers.pickle, which is done by the function save_sp500_tickers().

However, I get another error:

NotImplementedError: data_source='morningstar' is not implemented

The code is not stable as it seems. However, with some modification, it is easy to retrieve the data:

import bs4 as bs
import requests
import yfinance as yf
import datetime

resp = requests.get('http://en.wikipedia.org/wiki/List_of_S%26P_500_companies')
soup = bs.BeautifulSoup(resp.text, 'lxml')
table = soup.find('table', {'class': 'wikitable sortable'})
tickers = []
for row in table.findAll('tr')[1:]:
    ticker = row.findAll('td')[0].text
    tickers.append(ticker)

tickers = [s.replace('\n', '') for s in tickers]
start = datetime.datetime(2019,1,1)
end = datetime.datetime(2019,7,17)
data = yf.download(tickers, start=start, end=end)
print(data)

Some companies are delisted as it appears, but I get the output:

[*********************100%***********************]  501 of 505 downloaded
           Adj Close                                                                                     ...   Volume

                   A    AAL     AAP    AAPL   ABBV    ABC    ABMD    ABT     ACN    ADBE     ADI    ADM  ...     WYNN      XEC      XEL      XLNX       XOM     XRAY       XRX      XYL      YUM      ZBH     ZION      ZTS
Date                                                                                                     ...

2018-12-31     67.03  31.72  157.28  155.41  85.99  72.99  325.04  71.15  139.25  226.24   84.50  39.90  ...  2609800   983600  4485400   1433400  15807000  1664300   3462600   663800  1657000  1389700  2575600  1485200
2019-01-02     65.27  32.08  157.74  155.58  83.23  73.04  309.96  68.37  138.84  224.57   84.59  39.89  ...  4174400  1194000  4476100   2478400  16727200  2899400   3726900   990900  1743400  1152100  2234100  2665600
2019-01-03     62.87  29.69  162.66  140.09  80.49  71.98  302.29  65.14  134.10  215.70   79.48  39.71  ...  2885100  1230400  5287600   2666200  13866100  2382100   3390100  1243100  2680300  1166100  2078400  2390900
2019-01-04     65.04  31.65  158.63  146.07  83.08  72.81  313.44  67.00  139.31  226.19   81.41  40.63  ...  3007200  1709400  5535600   3011700  16043600  2787800   4557100   970900  2142700  1580400  2370500  3383500
2019-01-07     66.42  32.55  160.96  145.74  84.29  73.33  314.80  68.00  139.80  229.26   81.92  40.83  ...  3143800  1217200  4208100   2826100  10844200  2362800   3293100  1341600  2561100   900300  2459700  2360800
2019-01-08     67.40  32.02  159.84  148.52  84.68  74.12  318.42  67.19  143.33  232.68   83.92  41.59  ...  2255700  1499900  3128300   3172500  11439000  2277600   3552600   912800  2604700  3259000  2819200  2250300
2019-01-09     68.81  33.01  164.24  151.04  81.91  74.87  336.40  68.31  144.05  235.43   86.04  41.95  ...  2193000  1958500  3625600   2754500  13473500  2723600   3510300  1164400  2279700  1513300  2700300  3158400
2019-01-10     69.45  31.65  164.88  151.52  82.27  74.56  335.13  67.95  145.19  237.77   87.23  42.24  ...  2465400  2014700  3903700   2291500  13151100  1893200   3332600   826900  2726100  1485200  2063000  1837500
2019-01-11     69.93  31.41  159.24  150.04  82.37  75.11  323.80  68.20  144.41  237.55   88.66  42.04  ...  1806000  1266500  3491900   2667300   9790100  2064300   4390700   736400  2346600   938800  2074100  1393900
2019-01-14     69.31  31.32  158.54  147.78  80.03  75.40  315.25  67.47  143.12  234.56   86.14  41.74  ...  3692200  1673200  4068400   1570900  11012400  2124800   3320900   705800  1409200  1271100  2116200  1592700
2019-01-15     70.15  31.71  160.10  150.80  80.73  75.80  325.61  69.64  145.06  242.36   86.25  42.35  ...  2516900  1629100  3072400   1971800  10298300  2006800   2826100   607300  1532600  1037000  1748000  2277900
2019-01-16     69.58  32.44  162.25  152.65  80.78  75.84  324.13  69.00  146.04  241.95   85.65  42.17  ...  2316600  1594300  3397100   2085100  12595600  1448400   3347900   607400  1457200   959600  2826500  3880600
2019-01-17     70.37  33.17  166.09  153.55  82.33  74.64  330.32  69.69  147.26  244.40   87.43  42.28  ...  2545800  1270400  3164500   2446100  10141700  1208200   2993000   594100  2031700  1244000  3989600  2711400
2019-01-18     71.47  33.55  167.42  154.50  84.51  78.35  333.74  70.58  148.58  247.51   89.94  43.11  ...  2714200   914500  3451000   3477400  15759200  1868300   4487000  1062700  2041800  2101200  4259600  3059000
2019-01-22     70.87  32.34  163.94  151.03  83.71  78.20  336.13  70.65  147.11  243.85   87.75  42.43  ...  2014200  1790200  6609100   2971500  15035600  1613100   4434100  1441700  1947100  1252800  4301100  2782900
2019-01-23     71.19  31.26  164.14  151.64  83.51  77.86  336.98  69.09  146.57  245.14   87.48  42.57  ...  1807500  1046300  2956100   4098300  13187200  2446600   4027400  1095100  1551700   887900  5894200  2073800
2019-01-24     71.72  33.25  156.13  150.44  81.09  77.21  348.50  70.03  148.28  242.56   93.28  42.53  ...  1792000   842200  2887300  19931600  11272500  1364600   2768500  1195400  1983200   764400  3144700  2883600
2019-01-25     73.18  34.55  153.04  155.42  76.05  78.15  349.37  70.31  149.96  244.95   96.15  42.58  ...  2043900  1318000  3463100   8754100   8996200  1774800   3449400   948900  1501900  1151600  2842500  4021400
2019-01-28     72.80  36.12  154.68  153.99  72.84  78.60  336.90  68.24  150.34  241.85   95.83  43.02  ...  2271400  1016100  2412000   7988500  13678400  1664400   3423400   684000  1075300   974500  2613100  2471800
2019-01-29     73.09  35.84  155.21  152.39  73.15  78.03  330.38  68.87  150.13  238.27   95.50  43.06  ...  1689800   922900  3156000   3905300  10513200  2016500  10519900   992400  1292600   961700  1810200  1842100
2019-01-30     74.96  35.89  158.86  162.80  74.65  77.04  339.59  70.84  152.17  242.67   97.47  43.53  ...  3583900  1076600  3221500   5057100  11524700  1494000   6826900  1368800  1554200  1076600  2328600  2097300
2019-01-31     75.57  35.33  159.02  163.98  75.81  81.79  351.07  72.12  151.64  247.82   97.33  43.73  ...  3719800  1419300  6841500   3660000  19876700  1442100   5970100  1682100  2682500  1303500  3905100  2620000
2019-02-01     75.56  35.67  158.86  164.06  76.01  82.17  351.08  71.51  152.81  247.38   98.98  43.72  ...  2990800  1232700  3570900   4524000  22968700  1617000   4348000  1294800  1496600  2744500  2423300  2111900
2019-02-04     75.55  36.33  161.17  168.72  74.15  82.85  330.41  71.65  154.60  254.74   98.15  43.33  ...  1210900   952300  2968900   5315000  15483800  2307300   2915200   998100  1671500  2140200  2684900  1977000
2019-02-05     75.82  36.58  161.28  171.60  75.24  84.15  344.28  72.25  155.36  254.82   98.84  40.76  ...  1946600   635300  2775700   3437700  12786700  2143800   5402700  1044400  1888200  1583100  3755200  3263200
2019-02-06     75.81  36.41  157.04  171.66  74.67  85.00  344.78  72.63  155.20  254.35  100.82  40.78  ...  2418700   761600  1904700   5066400  10866900   912600   3613900   992500  1743300   966600  2015400  1968900
2019-02-07     74.85  35.82  162.94  168.41  74.75  84.17  341.61  71.89  153.25  253.74   99.07  40.32  ...  2345600  1508100  2313800   4973300  14418900  1324600   3059400   794800  2653900  1985000  3289800  1594400
2019-02-08     75.32  35.66  162.22  168.61  75.22  82.71  341.21  72.47  153.65  257.00   99.17  40.67  ...  2181200  1649200  2075200   2388100  11959500   952800   2570000   979500  1961300  1318200  2887100  2082700
2019-02-11     75.60  36.24  164.94  167.64  75.32  82.06  354.92  72.68  153.39  258.39  100.17  40.92  ...  1314000  1219900  2413400   2677900  10434800  1096100   2591800   719400  2468900  2094200  1826100  3361100
2019-02-12     76.58  35.41  167.70  169.08  75.47  84.16  358.08  74.18  154.68  261.37  102.45  41.24  ...  1924200   928700  3126600   4731600  12599200  2668400   3945900  1123400  2086800  2036700  2656200  2626400
...              ...    ...     ...     ...    ...    ...     ...    ...     ...     ...     ...    ...  ...      ...      ...      ...       ...       ...      ...       ...      ...      ...      ...      ...      ...
2019-06-04     67.66  28.92  154.49  178.42  73.43  81.45  267.06  76.86  177.21  268.71  100.47  39.26  ...  3236300  1066500  2441800   4133200  10423000  3158200   3416200   899000  2501700  1091200  2669300  3712100
2019-06-05     68.05  30.16  154.49  181.30  73.72  80.89  268.80  78.08  178.79  272.86   99.37  38.90  ...  2106800  1444800  3101500   3021400  10617800  1626200   1859300  1014700  1356200  1242800  1886700  3224100
2019-06-06     68.86  30.18  154.78  183.97  73.73  80.99  269.19  79.47  179.63  274.80   99.47  39.34  ...  1589100  1474900  1642200   2789300  11738900  1445500   2033500   740800  1598300  1201900  1898200  3123400
2019-06-07     69.22  30.71  155.23  188.86  74.08  82.71  267.87  80.12  182.14  278.16  100.19  39.21  ...  1545700  1541300  2857600   2548700   9095500   944700   1537900   606400  1318200   758900  2360400  1964800
2019-06-10     69.99  30.55  153.40  191.28  73.62  83.98  272.43  80.64  183.65  280.34  106.02  39.69  ...  2718500  2050700  2051200   3689600   7760400  1301500   1310200   358600  1159500   711500  1866300  2730000
2019-06-11     69.79  30.37  152.05  193.49  74.79  83.93  262.03  80.49  182.59  275.99  104.64  39.79  ...  1954800  1060500  1832100   3571200   9415100  1152800   1778100   708000  1801400   754800  2591600  1706300
2019-06-12     69.99  30.89  150.42  192.88  75.17  84.59  254.54  81.32  183.80  276.84  103.85  40.34  ...  1094500  1902300  1870000   2239600   7920800  1032300   1806800   372500  1646400   785100  1547300  1907300
2019-06-13     70.00  32.87  150.49  192.84  75.53  85.64  255.53  81.66  184.16  276.30  105.25  40.63  ...  1619300  1709300  2787700   2606900   8986400   786100   1883300   422000  1173500   819000  1978300  1790000
2019-06-14     69.67  32.46  151.68  191.44  75.28  84.26  250.96  81.59  184.18  274.28  103.50  40.41  ...  1525300  1530600  3163100   3032800   8575900   652900   1470600   503900  1088300  1326500  1421300  1790100
2019-06-17     69.99  32.34  151.62  192.58  75.25  84.55  256.15  81.42  183.11  275.60  104.57  40.36  ...  1037900  1395200  2721500   1910500   9355700   756700   1893200   534100   825000   933100  1452700  1726700
2019-06-18     71.60  32.30  154.71  197.11  74.86  84.66  256.47  82.78  183.71  276.78  109.39  40.82  ...  2536200  1274200  4299700   5151100  10091500   781800   2244100  1012000  1050900   916600  2115000  2572900
2019-06-19     72.33  32.99  153.65  196.53  74.25  85.26  256.30  83.68  183.16  291.21  110.48  40.33  ...  1582300  1739400  3489700   2631900   9757400   993500   1863300   871900   927600  1135400  1725900  2026700
2019-06-20     73.00  31.93  153.21  198.11  74.95  84.77  260.33  84.29  185.72  302.11  112.00  40.91  ...  1379100  1846500  3489700   2948900  16229700  2493500   2057400  1048500  1122600  1479100  1866200  3083300
2019-06-21     73.02  31.36  153.18  197.43  75.37  83.68  256.04  84.34  184.77  299.33  113.10  40.70  ...  1431800  2162900  5244200   4518400  23108100  4626900   3293000  1431400  2078400  2131600  3085700  3468800
2019-06-24     72.82  31.61  152.27  197.24  75.05  82.75  253.17  84.06  185.31  299.87  112.34  40.35  ...  1255100  1173500  3051500   1974600  10011800  1419800   1562000   708500  1024000   843900  1404100  1875700
2019-06-25     72.20  31.05  154.44  194.25  62.86  83.50  257.06  83.59  184.11  287.97  109.73  40.04  ...  1922700  1156800  3847700   2829400   9740000  1357000   1940400   844000  1540700   884300  1412500  1669200
2019-06-26     71.83  31.25  149.87  198.45  65.06  83.00  251.62  82.32  182.53  288.72  110.32  40.12  ...  1638400  1185300  6044500   3186100  11365500  1355300   1697500   536500  1513200  1609700  1460200  1743100
2019-06-27     72.87  31.93  151.65  198.39  66.97  84.84  254.93  83.04  182.22  293.23  112.28  40.19  ...  1339800  1231900  2978000   4137900   8147800   981700   1136600   396100  1611100  1139900  1442300  1348800
2019-06-28     74.35  32.39  154.08  196.58  69.57  84.47  260.49  83.45  183.98  294.65  112.31  40.42  ...  1848300  1835300  3793600   4524900  14781300  2231700   2775300  2211800  3871200  2577800  2737100  2414100
2019-07-01     75.62  32.66  156.53  200.19  70.22  84.80  265.02  83.84  186.07  300.97  114.43  41.01  ...  3571500  1895800  2503800   4280500  11437900  1684300   2492700   769400  1786700  1213800  1528100  1470700
2019-07-02     75.45  31.97  159.73  201.36  71.02  84.67  261.49  84.30  187.79  301.39  112.94  41.10  ...  1429500  1799700  2088000   3113500  10475400   947100   1543800  1011900  1584100  1147600  1757300  1568100
2019-07-03     76.64  32.50  160.57  203.03  71.72  86.78  261.33  84.98  190.15  305.70  112.60  41.48  ...  1154300   558500  1363400   2627900   5246100   641900    805900   420700  1049200   766100  1033700  1052000
2019-07-05     75.59  32.24  160.56  202.85  69.83  87.15  261.47  84.76  190.57  303.98  112.30  41.55  ...  1144600   775800  1838500   1747400   6880800   971600   1339300   654900  1281000   910300  1456800  1762400
2019-07-08     73.68  32.57  158.13  198.67  68.14  87.02  257.78  84.43  189.50  302.92  111.27  41.45  ...  1215100  1383900  1765000   1431600   7876100  2162200   1571300  1347600  1251000  1596500  2010200  1267300
2019-07-09     73.29  32.14  155.44  199.88  68.16  87.41  257.30  84.61  190.27  304.20  112.72  41.38  ...  1734800  1151300  2394600   2081400   7863000  1634000   1819800  1005700  1116500   913100  1558800  2722000
2019-07-10     73.21  32.72  156.37  201.85  68.55  88.47  255.81  84.72  190.89  306.98  113.44  41.41  ...  3005800  2842900  2468800   2062600   8077400  1112900   2126200   642600  1123400   632600  1404700  1763700
2019-07-11     71.59  32.81  156.23  200.38  68.16  89.69  266.09  85.11  193.84  307.60  113.35  41.34  ...  1817500  2559400  3103300   1880000   6404000  1465500   1943900   602800   904400   885000  1500800  1588000
2019-07-12     71.50  33.55  158.52  201.92  68.26  89.02  262.69  83.58  195.37  309.42  115.62  41.53  ...  1689200   999500  2064100   2183500   6139000  1515300   1452500   800500  1457600   644100  1565400  1473400
2019-07-15     70.40  33.38  158.86  203.82  68.25  89.10  260.58  83.48  194.48  308.76  116.44  41.35  ...  2650300  1607200  2198200   1407300   7116400  1160600   1403000   485800  1164300  1188100  1415200  1255200
2019-07-16     69.80  33.99  161.70  203.12  67.18  88.17  253.75  82.83  193.42  305.70  114.88  40.44  ...  2256800  3076900  3085300   2050600  10197400  1554000   1836000   646500   920700  1099400  1508700  1214600

[136 rows x 3030 columns]


来源:https://stackoverflow.com/questions/58890570/python-yahoo-finance-download-all-sp-500-stocks

易学教程内所有资源均来自网络或用户发布的内容,如有违反法律规定的内容欢迎反馈
该文章没有解决你所遇到的问题?点击提问,说说你的问题,让更多的人一起探讨吧!