Merge pull request #703 from addisonlynch/refactor-reader-config

BUG/ENH: Refactor date handling

Author: bashtage

docs/source/remote_data.rst

@@ -140,7 +140,9 @@ Historical Time Series Data
 Through the
 `Alpha Vantage <https://www.alphavantage.co/documentation>`__ Time Series
 endpoints, it is possible to obtain historical equities data for individual
-symbols. The following endpoints are available:
+symbols. For daily, weekly, and monthly frequencies, 20+ years of historical data is available. The past 3-5 days of intraday data is also available.
+
+The following endpoints are available:
 
 * ``av-intraday`` - Intraday Time Series
 * ``av-daily`` - Daily Time Series
@@ -591,7 +593,7 @@ example is to download 'Trade Union Density' data which set code is 'TUD'.
     import pandas_datareader.data as web
     import datetime
 
-    df = web.DataReader('TUD', 'oecd', end=datetime.datetime(2012, 1, 1))
+    df = web.DataReader('TUD', 'oecd')
 
     df.columns
 

docs/source/whatsnew/v0.8.0.txt

@@ -31,6 +31,7 @@ Enhancements
 - Added testing on Python 3.7 (:issue:`667`)
 - Allow IEX to read less than 1 year of data (:issue:`649`)
 - Allow data download from Poland using stooq (:issue:`597`)
+- All time series readers now use a rolling default starting date (most are 5 years before the current date. Intraday readers are 3-5 days from the current date)
 
 .. _whatsnew_080.api_breaking:
 

pandas_datareader/_utils.py

@@ -16,9 +16,16 @@ class RemoteDataError(IOError):
 
 def _sanitize_dates(start, end):
     """
-    Return (datetime_start, datetime_end) tuple
-    if start is None - default is 2010/01/01
+    Return (timestamp_start, timestamp_end) tuple
+    if start is None - default is 5 years before the current date
     if end is None - default is today
+
+    Parameters
+    ----------
+    start: str, int, date, datetime, timestamp
+        Desired start date
+    end: str, int, date, datetime, timestamp
+        Desired end date
     """
     if is_number(start):
         # regard int as year
@@ -30,9 +37,17 @@ def _sanitize_dates(start, end):
     end = to_datetime(end)
 
     if start is None:
-        start = dt.datetime(2010, 1, 1)
+        # default to 5 years before today
+        today = dt.date.today()
+        start = today - dt.timedelta(days=365 * 5)
     if end is None:
-        end = dt.datetime.today()
+        # default to today
+        end = dt.date.today()
+    try:
+        start = to_datetime(start)
+        end = to_datetime(end)
+    except (TypeError, ValueError):
+        raise ValueError("Invalid date format.")
     if start > end:
         raise ValueError("start must be an earlier date than end")
     return start, end

pandas_datareader/av/time_series.py

@@ -1,4 +1,4 @@
-from datetime import datetime
+import datetime as dt
 
 from pandas_datareader.av import AlphaVantage
 
@@ -13,11 +13,12 @@ class AVTimeSeriesReader(AlphaVantage):
     ----------
     symbols : string
         Single stock symbol (ticker)
-    start : string, (defaults to '1/1/2010')
-        Starting date, timestamp. Parses many different kind of date
-        representations (e.g., 'JAN-01-2010', '1/1/10', 'Jan, 1, 1980')
-    end : string, (defaults to today)
-        Ending date, timestamp. Same format as starting date.
+    start : string, int, date, datetime, timestamp
+        Starting date. Parses many different kind of date
+        representations (e.g., 'JAN-01-2010', '1/1/10', 'Jan, 1, 1980'). Defaults to
+        20 years before current date.
+    end : string, int, date, datetime, timestamp
+        Ending date
     retry_count : int, default 3
         Number of times to retry query request.
     pause : int, default 0.1
@@ -52,6 +53,7 @@ def __init__(
         chunksize=25,
         api_key=None,
     ):
+        self._func = function
         super(AVTimeSeriesReader, self).__init__(
             symbols=symbols,
             start=start,
@@ -62,19 +64,26 @@ def __init__(
             api_key=api_key,
         )
 
-        self._func = function
+    @property
+    def default_start_date(self):
+        d_days = 3 if self.intraday else 365 * 20
+        return dt.datetime.today() - dt.timedelta(days=d_days)
 
     @property
     def function(self):
         return self._func
 
+    @property
+    def intraday(self):
+        return True if self.function == "TIME_SERIES_INTRADAY" else False
+
     @property
     def output_size(self):
         """ Used to limit the size of the Alpha Vantage query when
         possible.
         """
-        delta = datetime.now() - self.start
-        return "full" if delta.days > 80 else "compact"
+        delta = dt.datetime.now() - self.start
+        return "compact" if delta.days < 80 and not self.intraday else "full"
 
     @property
     def data_key(self):
@@ -88,7 +97,7 @@ def params(self):
             "apikey": self.api_key,
             "outputsize": self.output_size,
         }
-        if self.function == "TIME_SERIES_INTRADAY":
+        if self.intraday:
             p.update({"interval": "1min"})
         return p
 

pandas_datareader/base.py

@@ -1,3 +1,4 @@
+import datetime
 import time
 import warnings
 
@@ -27,11 +28,11 @@ class _BaseReader(object):
     ----------
     symbols : {str, List[str]}
         String symbol of like of symbols
-    start : string, (defaults to '1/1/2010')
-        Starting date, timestamp. Parses many different kind of date
+    start : string, int, date, datetime, timestamp
+        Starting date. Parses many different kind of date
         representations (e.g., 'JAN-01-2010', '1/1/10', 'Jan, 1, 1980')
-    end : string, (defaults to today)
-        Ending date, timestamp. Same format as starting date.
+    end : string, int, date, datetime, timestamp
+        Ending date
     retry_count : int, default 3
         Number of times to retry query request.
     pause : float, default 0.1
@@ -59,7 +60,7 @@ def __init__(
 
         self.symbols = symbols
 
-        start, end = _sanitize_dates(start, end)
+        start, end = _sanitize_dates(start or self.default_start_date, end)
         self.start = start
         self.end = end
 
@@ -76,6 +77,12 @@ def close(self):
         """Close network session"""
         self.session.close()
 
+    @property
+    def default_start_date(self):
+        """Default start date for reader. Defaults to 5 years before current date"""
+        today = datetime.date.today()
+        return today - datetime.timedelta(days=365 * 5)
+
     @property
     def url(self):
         """API URL"""

pandas_datareader/iex/daily.py

@@ -27,11 +27,12 @@ class IEXDailyReader(_DailyBaseReader):
     symbols : string, array-like object (list, tuple, Series), or DataFrame
         Single stock symbol (ticker), array-like object of symbols or
         DataFrame with index containing stock symbols.
-    start : string, (defaults to '1/1/2010')
-        Starting date, timestamp. Parses many different kind of date
-        representations (e.g., 'JAN-01-2010', '1/1/10', 'Jan, 1, 1980')
-    end : string, (defaults to today)
-        Ending date, timestamp. Same format as starting date.
+    start : string, int, date, datetime, timestamp
+        Starting date. Parses many different kind of date
+        representations (e.g., 'JAN-01-2010', '1/1/10', 'Jan, 1, 1980'). Defaults to
+        15 years before current date
+    end : string, int, date, datetime, timestamp
+        Ending date
     retry_count : int, default 3
         Number of times to retry query request.
     pause : int, default 0.1
@@ -80,6 +81,11 @@ def __init__(
             chunksize=chunksize,
         )
 
+    @property
+    def default_start_date(self):
+        today = datetime.date.today()
+        return today - datetime.timedelta(days=365 * 15)
+
     @property
     def url(self):
         """API URL"""

pandas_datareader/moex.py

@@ -15,11 +15,12 @@ class MoexReader(_DailyBaseReader):
     symbols : str, an array-like object (list, tuple, Series), or a DataFrame
         A single stock symbol (secid), an array-like object of symbols or
         a DataFrame with an index containing stock symbols.
-    start : str, (defaults to '1/1/2010')
-        The starting date, timestamp. Parses many different kind of date
-        representations (e.g., 'JAN-01-2010', '1/1/10', 'Jan, 1, 1980')
-    end : str, (defaults to today)
-        The ending date, timestamp. Same format as starting date.
+    start : string, int, date, datetime, timestamp
+        Starting date. Parses many different kind of date
+        representations (e.g., 'JAN-01-2010', '1/1/10', 'Jan, 1, 1980'). Defaults to
+        20 years before current date.
+    end : string, int, date, datetime, timestamp
+        Ending date
     retry_count : int, default 3
         The number of times to retry query request.
     pause : int, default 0.1

pandas_datareader/quandl.py

@@ -23,11 +23,12 @@ class QuandlReader(_DailyBaseReader):
         Beware of ambiguous symbols (different securities per country)!
         Note: Cannot use more than a single string because of the inflexible
         way the URL is composed of url and _get_params in the superclass
-    start : string
-        Starting date, timestamp. Parses many different kind of date
-        representations (e.g., 'JAN-01-2010', '1/1/10', 'Jan, 1, 1980')
-    end : string, (defaults to today)
-        Ending date, timestamp. Same format as starting date.
+    start : string, int, date, datetime, timestamp
+        Starting date. Parses many different kind of date
+        representations (e.g., 'JAN-01-2010', '1/1/10', 'Jan, 1, 1980'). Defaults to
+        20 years before current date.
+    end : string, int, date, datetime, timestamp
+        Ending date
     retry_count : int, default 3
         Number of times to retry query request.
     pause : int, default 0.1

pandas_datareader/stooq.py

@@ -12,8 +12,12 @@ class StooqDailyReader(_DailyBaseReader):
     symbols : string, array-like object (list, tuple, Series), or DataFrame
         Single stock symbol (ticker), array-like object of symbols or
         DataFrame with index containing stock symbols.
-    start: string, date which to start interval at YYYYMMDD.
-    end: string, date which to end interval at YYYYMMDD.
+    start : string, int, date, datetime, timestamp
+        Starting date. Parses many different kind of date
+        representations (e.g., 'JAN-01-2010', '1/1/10', 'Jan, 1, 1980'). Defaults to
+        20 years before current date.
+    end : string, int, date, datetime, timestamp
+        Ending date
     retry_count : int, default 3
         Number of times to retry query request.
     pause : int, default 0.1

pandas_datareader/tests/test_base.py

@@ -1,3 +1,4 @@
+import datetime as dt
 import pytest
 import requests
 
@@ -26,6 +27,10 @@ def test_invalid_format(self):
             b._format = "IM_NOT_AN_IMPLEMENTED_TYPE"
             b._read_one_data("a", None)
 
+    def test_default_start_date(self):
+        b = base._BaseReader([])
+        assert b.default_start_date == dt.date.today() - dt.timedelta(days=365 * 5)
+
 
 class TestDailyBaseReader(object):
     def test_get_params(self):

pandas_datareader/tests/test_utils.py

@@ -0,0 +1,45 @@
+import datetime as dt
+import pandas as pd
+import pytest
+
+from pandas_datareader._utils import _sanitize_dates
+
+
+class TestUtils(object):
+    @pytest.mark.parametrize(
+        "input_date",
+        [
+            "2019-01-01",
+            "JAN-01-2010",
+            dt.datetime(2019, 1, 1),
+            dt.date(2019, 1, 1),
+            pd.Timestamp(2019, 1, 1),
+        ],
+    )
+    def test_sanitize_dates(self, input_date):
+        expected_start = pd.to_datetime(input_date)
+        expected_end = pd.to_datetime(dt.date.today())
+        result = _sanitize_dates(input_date, None)
+        assert result == (expected_start, expected_end)
+
+    def test_sanitize_dates_int(self):
+        start_int = 2018
+        end_int = 2019
+        expected_start = pd.to_datetime(dt.datetime(start_int, 1, 1))
+        expected_end = pd.to_datetime(dt.datetime(end_int, 1, 1))
+        assert _sanitize_dates(start_int, end_int) == (expected_start, expected_end)
+
+    def test_sanitize_invalid_dates(self):
+        with pytest.raises(ValueError):
+            _sanitize_dates(2019, 2018)
+
+        with pytest.raises(ValueError):
+            _sanitize_dates("2019-01-01", "2018-01-01")
+
+        with pytest.raises(ValueError):
+            _sanitize_dates("20199", None)
+
+    def test_sanitize_dates_defaults(self):
+        default_start = pd.to_datetime(dt.date.today() - dt.timedelta(days=365 * 5))
+        default_end = pd.to_datetime(dt.date.today())
+        assert _sanitize_dates(None, None) == (default_start, default_end)

pandas_datareader/tiingo.py

@@ -34,11 +34,12 @@ class TiingoIEXHistoricalReader(_BaseReader):
         ----------
         symbols : {str, List[str]}
             String symbol of like of symbols
-        start : str, (defaults to '1/1/2010')
-            Starting date, timestamp. Parses many different kind of date
-            representations (e.g., 'JAN-01-2010', '1/1/10', 'Jan, 1, 1980')
-        end : str, (defaults to today)
-            Ending date, timestamp. Same format as starting date.
+        start : string, int, date, datetime, timestamp
+            Starting date. Parses many different kind of date
+            representations (e.g., 'JAN-01-2010', '1/1/10', 'Jan, 1, 1980'). Defaults to
+            20 years before current date.
+        end : string, int, date, datetime, timestamp
+            Ending date
         retry_count : int, default 3
             Number of times to retry query request.
         pause : float, default 0.1

pandas_datareader/tsp.py

@@ -12,11 +12,12 @@ class TSPReader(_BaseReader):
     symbols : str, array-like object (list, tuple, Series), or DataFrame
         Single stock symbol (ticker), array-like object of symbols or
         DataFrame with index containing stock symbols.
-    start : str, (defaults to '1/1/2010')
-        Starting date, timestamp. Parses many different kind of date
-        representations (e.g., 'JAN-01-2010', '1/1/10', 'Jan, 1, 1980')
-    end : str, (defaults to today)
-        Ending date, timestamp. Same format as starting date.
+    start : string, int, date, datetime, timestamp
+        Starting date. Parses many different kind of date
+        representations (e.g., 'JAN-01-2010', '1/1/10', 'Jan, 1, 1980'). Defaults to
+        20 years before current date.
+    end : string, int, date, datetime, timestamp
+        Ending date
     retry_count : int, default 3
         Number of times to retry query request.
     pause : int, default 0.1

pandas_datareader/yahoo/daily.py

@@ -22,11 +22,12 @@ class YahooDailyReader(_DailyBaseReader):
     symbols : string, array-like object (list, tuple, Series), or DataFrame
         Single stock symbol (ticker), array-like object of symbols or
         DataFrame with index containing stock symbols.
-    start : string, (defaults to '1/1/2010')
-        Starting date, timestamp. Parses many different kind of date
-        representations (e.g., 'JAN-01-2010', '1/1/10', 'Jan, 1, 1980')
-    end : string, (defaults to today)
-        Ending date, timestamp. Same format as starting date.
+    start : string, int, date, datetime, timestamp
+        Starting date. Parses many different kind of date
+        representations (e.g., 'JAN-01-2010', '1/1/10', 'Jan, 1, 1980'). Defaults to
+        5 years before current date.
+    end : string, int, date, datetime, timestamp
+        Ending date
     retry_count : int, default 3
         Number of times to retry query request.
     pause : int, default 0.1