Merge branch 'master' into fix/combine-first

Author: Prabakaran Kumaresshan

.github/workflows/pre-commit.yml

@@ -0,0 +1,14 @@
+name: pre-commit
+
+on:
+  pull_request:
+  push:
+    branches: [master]
+
+jobs:
+  pre-commit:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/setup-python@v2
+    - uses: pre-commit/[email protected]

.github/workflows/stale-pr.yml

@@ -2,7 +2,7 @@ name: "Stale PRs"
 on:
   schedule:
   # * is a special character in YAML so you have to quote this string
-  - cron: "0 */6 * * *"
+  - cron: "0 0 * * *"
 
 jobs:
   stale:
@@ -11,11 +11,11 @@ jobs:
     - uses: actions/stale@v3
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
-        stale-pr-message: "This pull request is stale because it has been open for thirty days with no activity."
+        stale-pr-message: "This pull request is stale because it has been open for thirty days with no activity. Please update or respond to this comment if you're still interested in working on this."
         skip-stale-pr-message: false
         stale-pr-label: "Stale"
-        exempt-pr-labels: "Needs Review,Blocked"
+        exempt-pr-labels: "Needs Review,Blocked,Needs Discussion"
         days-before-stale: 30
         days-before-close: -1
-        remove-stale-when-updated: true
-        debug-only: true
+        remove-stale-when-updated: false
+        debug-only: false

.pre-commit-config.yaml

@@ -1,32 +1,45 @@
 repos:
 -   repo: https://github.com/python/black
-    rev: 19.10b0
+    rev: 20.8b1
     hooks:
     -   id: black
-        language_version: python3
 -   repo: https://gitlab.com/pycqa/flake8
-    rev: 3.7.7
+    rev: 3.8.3
     hooks:
     -   id: flake8
-        language: python_venv
         additional_dependencies: [flake8-comprehensions>=3.1.0]
     -   id: flake8
         name: flake8-pyx
-        language: python_venv
         files: \.(pyx|pxd)$
         types:
           - file
         args: [--append-config=flake8/cython.cfg]
     -   id: flake8
         name: flake8-pxd
-        language: python_venv
         files: \.pxi\.in$
         types:
           - file
         args: [--append-config=flake8/cython-template.cfg]
--   repo: https://github.com/pre-commit/mirrors-isort
-    rev: v4.3.21
+-   repo: https://github.com/PyCQA/isort
+    rev: 5.2.2
     hooks:
     -   id: isort
-        language: python_venv
         exclude: ^pandas/__init__\.py$|^pandas/core/api\.py$
+-   repo: https://github.com/asottile/pyupgrade
+    rev: v2.7.2
+    hooks:
+    -   id: pyupgrade
+        args: [--py37-plus]
+-   repo: https://github.com/pre-commit/pygrep-hooks
+    rev: v1.6.0
+    hooks:
+      - id: rst-backticks
+-   repo: local
+    hooks:
+    -   id: pip_to_conda
+        name: Generate pip dependency from conda
+        description: This hook checks if the conda environment.yml and requirements-dev.txt are equal
+        language: system
+        entry: python -m scripts.generate_pip_deps_from_conda
+        files: ^(environment.yml|requirements-dev.txt)$
+        pass_filenames: false

.travis.yml

@@ -1,6 +1,15 @@
 language: python
 python: 3.7
 
+addons:
+  apt:
+    update: true
+    packages:
+      - xvfb
+
+services:
+  - xvfb
+
 # To turn off cached cython files and compiler cache
 # set NOCACHE-true
 # To delete caches go to https://travis-ci.org/OWNER/REPOSITORY/caches or run
@@ -10,46 +19,43 @@ cache:
   ccache: true
   directories:
     - $HOME/.cache # cython cache
-    - $HOME/.ccache # compiler cache
 
 env:
   global:
-    # Variable for test workers
     - PYTEST_WORKERS="auto"
     # create a github personal access token
     # cd pandas-dev/pandas
     # travis encrypt 'PANDAS_GH_TOKEN=personal_access_token' -r pandas-dev/pandas
     - secure: "EkWLZhbrp/mXJOx38CHjs7BnjXafsqHtwxPQrqWy457VDFWhIY1DMnIR/lOWG+a20Qv52sCsFtiZEmMfUjf0pLGXOqurdxbYBGJ7/ikFLk9yV2rDwiArUlVM9bWFnFxHvdz9zewBH55WurrY4ShZWyV+x2dWjjceWG5VpWeI6sA="
 
 git:
-  # for cloning
   depth: false
 
 matrix:
   fast_finish: true
 
   include:
-    # In allowed failures
     - dist: bionic
       python: 3.9-dev
       env:
         - JOB="3.9-dev" PATTERN="(not slow and not network and not clipboard)"
+
     - env:
         - JOB="3.8" ENV_FILE="ci/deps/travis-38.yaml" PATTERN="(not slow and not network and not clipboard)"
 
     - env:
         - JOB="3.7" ENV_FILE="ci/deps/travis-37.yaml" PATTERN="(not slow and not network and not clipboard)"
 
-    - arch: arm64
-      env:
-        - JOB="3.7, arm64" PYTEST_WORKERS=8 ENV_FILE="ci/deps/travis-37-arm64.yaml" PATTERN="(not slow and not network and not clipboard)"
-
     - env:
         - JOB="3.7, locale" ENV_FILE="ci/deps/travis-37-locale.yaml" PATTERN="((not slow and not network and not clipboard) or (single and db))" LOCALE_OVERRIDE="zh_CN.UTF-8" SQL="1"
       services:
         - mysql
         - postgresql
 
+    - arch: arm64
+      env:
+        - JOB="3.7, arm64" PYTEST_WORKERS=1 ENV_FILE="ci/deps/travis-37-arm64.yaml" PATTERN="(not slow and not network and not clipboard and not arm_slow)"
+
     - env:
         # Enabling Deprecations when running tests
         # PANDAS_TESTING_MODE="deprecate" causes DeprecationWarning messages to be displayed in the logs
@@ -58,13 +64,12 @@ matrix:
       services:
         - mysql
         - postgresql
+
   allow_failures:
+    # Moved to allowed_failures 2020-09-29 due to timeouts https://github.com/pandas-dev/pandas/issues/36719
     - arch: arm64
       env:
-        - JOB="3.7, arm64" PYTEST_WORKERS=8 ENV_FILE="ci/deps/travis-37-arm64.yaml" PATTERN="(not slow and not network and not clipboard)"
-    - dist: bionic
-      env:
-        - JOB="3.9-dev" PATTERN="(not slow and not network and not clipboard)"
+        - JOB="3.7, arm64" PYTEST_WORKERS=1 ENV_FILE="ci/deps/travis-37-arm64.yaml" PATTERN="(not slow and not network and not clipboard and not arm_slow)"
 
 
 before_install:
@@ -78,12 +83,6 @@ before_install:
   - uname -a
   - git --version
   - ./ci/check_git_tags.sh
-  # Because travis runs on Google Cloud and has a /etc/boto.cfg,
-  # it breaks moto import, see:
-  # https://github.com/spulec/moto/issues/1771
-  # https://github.com/boto/boto/issues/3741
-  # This overrides travis and tells it to look nowhere.
-  - export BOTO_CONFIG=/dev/null
 
 install:
   - echo "install start"

asv_bench/asv.conf.json

@@ -39,7 +39,7 @@
     // followed by the pip installed packages).
     "matrix": {
         "numpy": [],
-        "Cython": ["0.29.16"],
+        "Cython": ["0.29.21"],
         "matplotlib": [],
         "sqlalchemy": [],
         "scipy": [],

asv_bench/benchmarks/arithmetic.py

@@ -125,7 +125,7 @@ def setup(self, op):
         arr1 = np.random.randn(n_rows, int(n_cols / 2)).astype("f8")
         arr2 = np.random.randn(n_rows, int(n_cols / 2)).astype("f4")
         df = pd.concat(
-            [pd.DataFrame(arr1), pd.DataFrame(arr2)], axis=1, ignore_index=True,
+            [pd.DataFrame(arr1), pd.DataFrame(arr2)], axis=1, ignore_index=True
         )
         # should already be the case, but just to be sure
         df._consolidate_inplace()

asv_bench/benchmarks/frame_methods.py

@@ -219,6 +219,46 @@ def time_to_html_mixed(self):
         self.df2.to_html()
 
 
+class ToNumpy:
+    def setup(self):
+        N = 10000
+        M = 10
+        self.df_tall = DataFrame(np.random.randn(N, M))
+        self.df_wide = DataFrame(np.random.randn(M, N))
+        self.df_mixed_tall = self.df_tall.copy()
+        self.df_mixed_tall["foo"] = "bar"
+        self.df_mixed_tall[0] = period_range("2000", periods=N)
+        self.df_mixed_tall[1] = range(N)
+        self.df_mixed_wide = self.df_wide.copy()
+        self.df_mixed_wide["foo"] = "bar"
+        self.df_mixed_wide[0] = period_range("2000", periods=M)
+        self.df_mixed_wide[1] = range(M)
+
+    def time_to_numpy_tall(self):
+        self.df_tall.to_numpy()
+
+    def time_to_numpy_wide(self):
+        self.df_wide.to_numpy()
+
+    def time_to_numpy_mixed_tall(self):
+        self.df_mixed_tall.to_numpy()
+
+    def time_to_numpy_mixed_wide(self):
+        self.df_mixed_wide.to_numpy()
+
+    def time_values_tall(self):
+        self.df_tall.values
+
+    def time_values_wide(self):
+        self.df_wide.values
+
+    def time_values_mixed_tall(self):
+        self.df_mixed_tall.values
+
+    def time_values_mixed_wide(self):
+        self.df_mixed_wide.values
+
+
 class Repr:
     def setup(self):
         nrows = 10000

asv_bench/benchmarks/strings.py

@@ -13,13 +13,20 @@ class Construction:
     param_names = ["dtype"]
 
     def setup(self, dtype):
-        self.data = tm.rands_array(nchars=10 ** 5, size=10)
+        self.series_arr = tm.rands_array(nchars=10, size=10 ** 5)
+        self.frame_arr = self.series_arr.reshape((50_000, 2)).copy()
 
-    def time_construction(self, dtype):
-        Series(self.data, dtype=dtype)
+    def time_series_construction(self, dtype):
+        Series(self.series_arr, dtype=dtype)
 
-    def peakmem_construction(self, dtype):
-        Series(self.data, dtype=dtype)
+    def peakmem_series_construction(self, dtype):
+        Series(self.series_arr, dtype=dtype)
+
+    def time_frame_construction(self, dtype):
+        DataFrame(self.frame_arr, dtype=dtype)
+
+    def peakmem_frame_construction(self, dtype):
+        DataFrame(self.frame_arr, dtype=dtype)
 
 
 class Methods:

asv_bench/benchmarks/timeseries.py

@@ -263,6 +263,29 @@ def time_lookup_and_cleanup(self):
         self.ts.index._cleanup()
 
 
+class ToDatetimeFromIntsFloats:
+    def setup(self):
+        self.ts_sec = Series(range(1521080307, 1521685107), dtype="int64")
+        self.ts_sec_float = self.ts_sec.astype("float64")
+
+        self.ts_nanosec = 1_000_000 * self.ts_sec
+        self.ts_nanosec_float = self.ts_nanosec.astype("float64")
+
+    # speed of int64 and float64 paths should be comparable
+
+    def time_nanosec_int64(self):
+        to_datetime(self.ts_nanosec, unit="ns")
+
+    def time_nanosec_float64(self):
+        to_datetime(self.ts_nanosec_float, unit="ns")
+
+    def time_sec_int64(self):
+        to_datetime(self.ts_sec, unit="s")
+
+    def time_sec_float64(self):
+        to_datetime(self.ts_sec_float, unit="s")
+
+
 class ToDatetimeYYYYMMDD:
     def setup(self):
         rng = date_range(start="1/1/2000", periods=10000, freq="D")

ci/build39.sh

@@ -1,10 +1,8 @@
 #!/bin/bash -e
 # Special build for python3.9 until numpy puts its own wheels up
 
-sudo apt-get install build-essential gcc xvfb
 pip install --no-deps -U pip wheel setuptools
-pip install numpy python-dateutil pytz pytest pytest-xdist hypothesis
-pip install cython --pre # https://github.com/cython/cython/issues/3395
+pip install cython numpy python-dateutil pytz pytest pytest-xdist hypothesis
 
 python setup.py build_ext -inplace
 python -m pip install --no-build-isolation -e .

ci/code_checks.sh

@@ -335,7 +335,7 @@ if [[ -z "$CHECK" || "$CHECK" == "doctests" ]]; then
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
     MSG='Doctests strings.py' ; echo $MSG
-    pytest -q --doctest-modules pandas/core/strings.py
+    pytest -q --doctest-modules pandas/core/strings/
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
     # Directories

ci/deps/azure-37-32bit.yaml

@@ -21,6 +21,6 @@ dependencies:
   # see comment above
   - pip
   - pip:
-    - cython>=0.29.16
+    - cython>=0.29.21
     - numpy>=1.16.5
     - pytest>=5.0.1

ci/deps/azure-37-locale.yaml

@@ -6,7 +6,7 @@ dependencies:
   - python=3.7.*
 
   # tools
-  - cython>=0.29.16
+  - cython>=0.29.21
   - pytest>=5.0.1
   - pytest-xdist>=1.21
   - pytest-asyncio

ci/deps/azure-37-locale_slow.yaml

@@ -6,7 +6,7 @@ dependencies:
   - python=3.7.*
 
   # tools
-  - cython>=0.29.16
+  - cython>=0.29.21
   - pytest>=5.0.1
   - pytest-xdist>=1.21
   - hypothesis>=3.58.0

ci/deps/azure-37-minimum_versions.yaml

@@ -5,7 +5,7 @@ dependencies:
   - python=3.7.1
 
   # tools
-  - cython=0.29.16
+  - cython=0.29.21
   - pytest=5.0.1
   - pytest-xdist>=1.21
   - hypothesis>=3.58.0
@@ -20,7 +20,7 @@ dependencies:
   - numexpr=2.6.8
   - numpy=1.16.5
   - openpyxl=2.6.0
-  - pytables=3.4.4
+  - pytables=3.5.1
   - python-dateutil=2.7.3
   - pytz=2017.3
   - pyarrow=0.15

ci/deps/azure-37-slow.yaml

@@ -6,7 +6,7 @@ dependencies:
   - python=3.7.*
 
   # tools
-  - cython>=0.29.16
+  - cython>=0.29.21
   - pytest>=5.0.1
   - pytest-xdist>=1.21
   - hypothesis>=3.58.0

ci/deps/azure-38-locale.yaml

@@ -5,7 +5,7 @@ dependencies:
   - python=3.8.*
 
   # tools
-  - cython>=0.29.16
+  - cython>=0.29.21
   - pytest>=5.0.1
   - pytest-xdist>=1.21
   - pytest-asyncio>=0.12.0