Merge branch 'master' into feature/allow-setting-read-concern-queryset

Author: bagerard

.gitignore

@@ -1,8 +1,15 @@
-.*
 !.gitignore
 *~
 *.py[co]
 .*.sw[po]
+.cache/
+.coverage
+.coveragerc
+.env
+.idea/
+.pytest_cache/
+.tox/
+.eggs/
 *.egg
 docs/.build
 docs/_build
@@ -13,8 +20,6 @@ env/
 .settings
 .project
 .pydevproject
-tests/test_bugfix.py
 htmlcov/
 venv
 venv3
-scratchpad

.landscape.yml

@@ -5,17 +5,12 @@ pylint:
 
     options:
         additional-builtins:
-            # add xrange and long as valid built-ins. In Python 3, xrange is
-            # translated into range and long is translated into int via 2to3 (see
-            # "use_2to3" in setup.py). This should be removed when we drop Python
-            # 2 support (which probably won't happen any time soon).
-            - xrange
+            # add long as valid built-ins.
             - long
 
 pyflakes:
     disable:
-        # undefined variables are already covered by pylint (and exclude
-        # xrange & long)
+        # undefined variables are already covered by pylint (and exclude long)
         - F821
 
 ignore-paths:

.pre-commit-config.yaml

@@ -0,0 +1,12 @@
+fail_fast: false
+repos:
+    - repo: https://github.com/ambv/black
+      rev: 19.10b0
+      hooks:
+        - id: black
+    - repo: https://gitlab.com/pycqa/flake8
+      rev: 3.8.0a2
+      hooks:
+        - id: flake8
+          additional_dependencies:
+            - flake8-import-order

.travis.yml

@@ -1,13 +1,10 @@
 # For full coverage, we'd have to test all supported Python, MongoDB, and
 # PyMongo combinations. However, that would result in an overly long build
 # with a very large number of jobs, hence we only test a subset of all the
-# combinations:
-# * MongoDB v3.4 & the latest PyMongo v3.x is currently the "main" setup,
-#   tested against Python v2.7, v3.5, v3.6, v3.7, v3.8, PyPy and PyPy3.
-# * Besides that, we test the lowest actively supported Python/MongoDB/PyMongo
-#   combination: MongoDB v3.4, PyMongo v3.4, Python v2.7.
-# * MongoDB v3.6 is tested against Python v3.6, and PyMongo v3.6, v3.7, v3.8.
-#
+# combinations.
+# * Python3.7, MongoDB v3.4 & the latest PyMongo v3.x is currently the "main" setup,
+# Other combinations are tested. See below for the details or check the travis jobs
+
 # We should periodically check MongoDB Server versions supported by MongoDB
 # Inc., add newly released versions to the test matrix, and remove versions
 # which have reached their End of Life. See:
@@ -16,19 +13,15 @@
 #
 # Reminder: Update README.rst if you change MongoDB versions we test.
 
-
 language: python
+dist: xenial
 python:
-- 2.7
 - 3.5
 - 3.6
 - 3.7
 - 3.8
-- pypy
 - pypy3
 
-dist: xenial
-
 env:
   global:
     - MONGODB_3_4=3.4.17
@@ -39,6 +32,8 @@ env:
     - PYMONGO_3_6=3.6
     - PYMONGO_3_9=3.9
     - PYMONGO_3_10=3.10
+
+    - MAIN_PYTHON_VERSION=3.7
   matrix:
     - MONGODB=${MONGODB_3_4} PYMONGO=${PYMONGO_3_10}
 
@@ -47,8 +42,6 @@ matrix:
   fast_finish: true
 
   include:
-  - python: 2.7
-    env: MONGODB=${MONGODB_3_4} PYMONGO=${PYMONGO_3_4}
   - python: 3.7
     env: MONGODB=${MONGODB_3_6} PYMONGO=${PYMONGO_3_6}
   - python: 3.7
@@ -66,29 +59,23 @@ install:
   # Install Python dependencies.
   - pip install --upgrade pip
   - pip install coveralls
-  - pip install flake8 flake8-import-order
-  - pip install tox         # tox 3.11.0 has requirement virtualenv>=14.0.0
-  - pip install virtualenv  # virtualenv>=14.0.0 has dropped Python 3.2 support (and pypy3 is based on py32)
+  - pip install pre-commit
+  - pip install tox
   # tox dryrun to setup the tox venv (we run a mock test).
   - tox -e $(echo py$TRAVIS_PYTHON_VERSION-mg$PYMONGO | tr -d . | sed -e 's/pypypy/pypy/') -- -a "-k=test_ci_placeholder"
-  # Install black for Python v3.7 only.
-  - if [[ $TRAVIS_PYTHON_VERSION == '3.7' ]]; then pip install black; fi
 
 before_script:
   - mkdir ${PWD}/mongodb-linux-x86_64-${MONGODB}/data
   - ${PWD}/mongodb-linux-x86_64-${MONGODB}/bin/mongod --dbpath ${PWD}/mongodb-linux-x86_64-${MONGODB}/data --logpath ${PWD}/mongodb-linux-x86_64-${MONGODB}/mongodb.log --fork
-  - if [[ $TRAVIS_PYTHON_VERSION == '2.7' ]]; then flake8 .; else echo "flake8 only runs on py27"; fi   # Run flake8 for Python 2.7 only
-  - if [[ $TRAVIS_PYTHON_VERSION == '3.7' ]]; then black --check .; else echo "black only runs on py37"; fi   # Run black for Python 3.7 only
+  # Run pre-commit hooks (black, flake8, etc) on entire codebase
+  - if [[ $TRAVIS_PYTHON_VERSION == $MAIN_PYTHON_VERSION ]]; then pre-commit run -a; else echo "pre-commit checks only runs on py37"; fi
   - mongo --eval 'db.version();'    # Make sure mongo is awake
 
 script:
   - tox -e $(echo py$TRAVIS_PYTHON_VERSION-mg$PYMONGO | tr -d . | sed -e 's/pypypy/pypy/') -- -a "--cov=mongoengine"
 
-# For now only submit coveralls for Python v2.7. Python v3.x currently shows
-# 0% coverage. That's caused by 'use_2to3', which builds the py3-compatible
-# code in a separate dir and runs tests on that.
 after_success:
-- if [[ $TRAVIS_PYTHON_VERSION == '2.7' ]]; then coveralls --verbose; else echo "coveralls only sent for py27"; fi
+- - if [[ $TRAVIS_PYTHON_VERSION == $MAIN_PYTHON_VERSION ]]; then coveralls --verbose; else echo "coveralls only sent for py37"; fi
 
 notifications:
   irc: irc.freenode.org#mongoengine
@@ -110,11 +97,11 @@ deploy:
   distributions: "sdist bdist_wheel"
 
   # Only deploy on tagged commits (aka GitHub releases) and only for the parent
-  # repo's builds running Python v2.7 along with PyMongo v3.x and MongoDB v3.4.
+  # repo's builds running Python v3.7 along with PyMongo v3.x and MongoDB v3.4.
   # We run Travis against many different Python, PyMongo, and MongoDB versions
   # and we don't want the deploy to occur multiple times).
   on:
     tags: true
     repo: MongoEngine/mongoengine
     condition: ($PYMONGO = ${PYMONGO_3_10}) && ($MONGODB = ${MONGODB_3_4})
-    python: 2.7
+    python: 3.7

CONTRIBUTING.rst

@@ -20,19 +20,43 @@ post to the `user group <http://groups.google.com/group/mongoengine-users>`
 Supported Interpreters
 ----------------------
 
-MongoEngine supports CPython 2.7 and newer. Language
-features not supported by all interpreters can not be used.
-The codebase is written in python 2 so you must be using python 2
-when developing new features. Compatibility of the library with Python 3
-relies on the 2to3 package that gets executed as part of the installation
-build. You should ensure that your code is properly converted by
-`2to3 <http://docs.python.org/library/2to3.html>`_.
+MongoEngine supports CPython 3.5 and newer as well as Pypy3.
+Language features not supported by all interpreters can not be used.
+
+Python3 codebase
+----------------------
+
+Since 0.20, the codebase is exclusively Python 3.
+
+Earlier versions were exclusively Python2, and were relying on 2to3 to support Python3 installs.
+Travis runs the tests against the main Python 3.x versions.
+
 
 Style Guide
 -----------
 
-MongoEngine uses `black <https://github.com/python/black>`_ for code
-formatting.
+MongoEngine's codebase is formatted with `black <https://github.com/python/black>`_, other tools like
+flake8 are also used. Those tools will run as part of the CI and will fail in case the code is not formatted properly.
+
+To install all development tools, simply run the following commands:
+
+.. code-block:: console
+
+    $ python -m pip install -r requirements-dev.txt
+
+
+You can install `pre-commit <https://pre-commit.com/>`_ into your git hooks,
+to automatically check and fix any formatting issue before creating a
+git commit.
+
+To enable ``pre-commit`` simply run:
+
+.. code-block:: console
+
+    $ pre-commit install
+
+See the ``.pre-commit-config.yaml`` configuration file for more information
+on how it works.
 
 Testing
 -------

README.rst

@@ -34,21 +34,22 @@ with MongoDB version > 4.0.
 Installation
 ============
 We recommend the use of `virtualenv <https://virtualenv.pypa.io/>`_ and of
-`pip <https://pip.pypa.io/>`_. You can then use ``pip install -U mongoengine``.
+`pip <https://pip.pypa.io/>`_. You can then use ``python -m pip install -U mongoengine``.
 You may also have `setuptools <http://peak.telecommunity.com/DevCenter/setuptools>`_
 and thus you can use ``easy_install -U mongoengine``. Another option is
 `pipenv <https://docs.pipenv.org/>`_. You can then use ``pipenv install mongoengine``
 to both create the virtual environment and install the package. Otherwise, you can
 download the source from `GitHub <http://github.com/MongoEngine/mongoengine>`_ and
 run ``python setup.py install``.
 
+The support for Python2 was dropped with MongoEngine 0.20.0
+
 Dependencies
 ============
-All of the dependencies can easily be installed via `pip <https://pip.pypa.io/>`_.
+All of the dependencies can easily be installed via `python -m pip <https://pip.pypa.io/>`_.
 At the very least, you'll need these two packages to use MongoEngine:
 
 - pymongo>=3.4
-- six>=1.10.0
 
 If you utilize a ``DateTimeField``, you might also use a more flexible date parser:
 
@@ -58,6 +59,10 @@ If you need to use an ``ImageField`` or ``ImageGridFsProxy``:
 
 - Pillow>=2.0.0
 
+If you need to use signals:
+
+- blinker>=1.3
+
 Examples
 ========
 Some simple examples of what MongoEngine code looks like:
@@ -125,7 +130,7 @@ installed in your environment and then:
 .. code-block:: shell
 
     # Install tox
-    $ pip install tox
+    $ python -m pip install tox
     # Run the test suites
     $ tox
 

benchmarks/test_inserts.py

@@ -4,12 +4,14 @@
 def main():
     setup = """
 from pymongo import MongoClient
+
 connection = MongoClient()
 connection.drop_database('mongoengine_benchmark_test')
 """
 
     stmt = """
 from pymongo import MongoClient
+
 connection = MongoClient()
 
 db = connection.mongoengine_benchmark_test
@@ -56,6 +58,7 @@ def main():
 
     setup = """
 from pymongo import MongoClient
+
 connection = MongoClient()
 connection.drop_database('mongoengine_benchmark_test')
 connection.close()

docs/changelog.rst

@@ -6,9 +6,21 @@ Changelog
 Development
 ===========
 - (Fill this out as you fix issues and develop your features).
+- ATTENTION: Drop support for Python2
 - Add Mongo 4.0 to Travis
+- Fix error when setting a string as a ComplexDateTimeField #2253
+- Bump development Status classifier to Production/Stable #2232
+- Improve Queryset.get to avoid confusing MultipleObjectsReturned message in case multiple match are found #630
 - Fixed a bug causing inaccurate query results, while combining ``__raw__`` and regular filters for the same field #2264
-- Add support for the `elemMatch` projection operator in .fields (e.g BlogPost.objects.fields(elemMatch__comments="test")) #2267
+- Add support for the `elemMatch` projection operator in .fields() (e.g BlogPost.objects.fields(elemMatch__comments="test")) #2267
+- DictField validate failed without default connection (bug introduced in 0.19.0) #2239
+- Remove methods deprecated years ago:
+    - name parameter in Field constructor e.g `StringField(name="...")`, was replaced by db_field
+    - Queryset.slave_okay() was deprecated since pymongo3
+    - dropDups was dropped with MongoDB3
+    - ``Queryset._ensure_indexes`` and ``Queryset.ensure_indexes``, the right method to use is ``Document.ensure_indexes``
+- Added pre-commit #2212
+- Renamed requirements-lint.txt to requirements-dev.txt #2212
 
 Changes in 0.19.1
 =================

docs/django.rst

@@ -13,7 +13,7 @@ Help Wanted!
 
 The MongoEngine team is looking for help contributing and maintaining a new
 Django extension for MongoEngine! If you have Django experience and would like
-to help contribute to the project, please get in touch on the 
-`mailing list <http://groups.google.com/group/mongoengine-users>`_ or by 
+to help contribute to the project, please get in touch on the
+`mailing list <http://groups.google.com/group/mongoengine-users>`_ or by
 simply contributing on
 `GitHub <https://github.com/MongoEngine/django-mongoengine>`_.

docs/faq.rst

@@ -10,4 +10,3 @@ If this is a requirement for your project, check the alternative:  `uMongo`_ and
 
 .. _uMongo: https://umongo.readthedocs.io/
 .. _MotorEngine: https://motorengine.readthedocs.io/
-