Merge branch 'master' of github.com:MongoEngine/mongoengine into negative_indexes_in_list

Author: cp-bagerard

.travis.yml

@@ -32,10 +32,11 @@ env:
   global:
     - MONGODB_3_4=3.4.17
     - MONGODB_3_6=3.6.12
+    - PYMONGO_3_9=3.9
     - PYMONGO_3_6=3.6
     - PYMONGO_3_4=3.4
   matrix:
-    - MONGODB=${MONGODB_3_4} PYMONGO=${PYMONGO_3_6}
+    - MONGODB=${MONGODB_3_4} PYMONGO=${PYMONGO_3_9}
 
 matrix:
 
@@ -47,6 +48,8 @@ matrix:
     env: MONGODB=${MONGODB_3_4} PYMONGO=${PYMONGO_3_4}
   - python: 3.7
     env: MONGODB=${MONGODB_3_6} PYMONGO=${PYMONGO_3_6}
+  - python: 3.7
+    env: MONGODB=${MONGODB_3_6} PYMONGO=${PYMONGO_3_9}
 
 
 install:
@@ -60,8 +63,8 @@ install:
   - pip install flake8 flake8-import-order
   - pip install tox         # tox 3.11.0 has requirement virtualenv>=14.0.0
   - pip install virtualenv  # virtualenv>=14.0.0 has dropped Python 3.2 support (and pypy3 is based on py32)
-  # Install the tox venv.
-  - tox -e $(echo py$TRAVIS_PYTHON_VERSION-mg$PYMONGO | tr -d . | sed -e 's/pypypy/pypy/') -- -e test
+  # tox dryrun to setup the tox venv (we run a mock test).
+  - tox -e $(echo py$TRAVIS_PYTHON_VERSION-mg$PYMONGO | tr -d . | sed -e 's/pypypy/pypy/') -- -a "-k=test_ci_placeholder"
   # Install black for Python v3.7 only.
   - if [[ $TRAVIS_PYTHON_VERSION == '3.7' ]]; then pip install black; fi
 
@@ -73,13 +76,13 @@ before_script:
   - mongo --eval 'db.version();'    # Make sure mongo is awake
 
 script:
-  - tox -e $(echo py$TRAVIS_PYTHON_VERSION-mg$PYMONGO | tr -d . | sed -e 's/pypypy/pypy/') -- --with-coverage
+  - tox -e $(echo py$TRAVIS_PYTHON_VERSION-mg$PYMONGO | tr -d . | sed -e 's/pypypy/pypy/') -- -a "--cov=mongoengine"
 
 # For now only submit coveralls for Python v2.7. Python v3.x currently shows
 # 0% coverage. That's caused by 'use_2to3', which builds the py3-compatible
 # code in a separate dir and runs tests on that.
 after_success:
-- if [[ $TRAVIS_PYTHON_VERSION == '2.7' ]]; then coveralls --verbose; fi
+- if [[ $TRAVIS_PYTHON_VERSION == '2.7' ]]; then coveralls --verbose; else echo "coveralls only sent for py27"; fi
 
 notifications:
   irc: irc.freenode.org#mongoengine

AUTHORS

@@ -252,3 +252,5 @@ that much better:
  * Paulo Amaral (https://github.com/pauloAmaral)
  * Gaurav Dadhania (https://github.com/GVRV)
  * Yurii Andrieiev (https://github.com/yandrieiev)
+ * Filip Kucharczyk (https://github.com/Pacu2)
+ * Eric Timmons (https://github.com/daewok)

CONTRIBUTING.rst

@@ -54,7 +54,7 @@ General Guidelines
   should adapt to the breaking change in docs/upgrade.rst.
 - Write inline documentation for new classes and methods.
 - Write tests and make sure they pass (make sure you have a mongod
-  running on the default port, then execute ``python setup.py nosetests``
+  running on the default port, then execute ``python setup.py test``
   from the cmd line to run the test suite).
 - Ensure tests pass on all supported Python, PyMongo, and MongoDB versions.
   You can test various Python and PyMongo versions locally by executing

README.rst

@@ -116,7 +116,8 @@ Some simple examples of what MongoEngine code looks like:
 Tests
 =====
 To run the test suite, ensure you are running a local instance of MongoDB on
-the standard port and have ``nose`` installed. Then, run ``python setup.py nosetests``.
+the standard port and have ``pytest`` installed. Then, run ``python setup.py test``
+or simply ``pytest``.
 
 To run the test suite on every supported Python and PyMongo version, you can
 use ``tox``. You'll need to make sure you have each supported Python version
@@ -129,16 +130,14 @@ installed in your environment and then:
     # Run the test suites
     $ tox
 
-If you wish to run a subset of tests, use the nosetests convention:
+If you wish to run a subset of tests, use the pytest convention:
 
 .. code-block:: shell
 
     # Run all the tests in a particular test file
-    $ python setup.py nosetests --tests tests/fields/fields.py
+    $ pytest tests/fields/test_fields.py
     # Run only particular test class in that file
-    $ python setup.py nosetests --tests tests/fields/fields.py:FieldTest
-    # Use the -s option if you want to print some debug statements or use pdb
-    $ python setup.py nosetests --tests tests/fields/fields.py:FieldTest -s
+    $ pytest tests/fields/test_fields.py::TestField
 
 Community
 =========

docs/changelog.rst

@@ -6,6 +6,8 @@ Changelog
 Development
 ===========
 - (Fill this out as you fix issues and develop your features).
+- Documentation improvements:
+    - Documented how `pymongo.monitoring` can be used to log all queries issued by MongoEngine to the driver.
 - BREAKING CHANGE: ``class_check`` and ``read_preference`` keyword arguments are no longer available when filtering a ``QuerySet``. #2112
     - Instead of ``Doc.objects(foo=bar, read_preference=...)`` use ``Doc.objects(foo=bar).read_preference(...)``.
     - Instead of ``Doc.objects(foo=bar, class_check=False)`` use ``Doc.objects(foo=bar).clear_cls_query(...)``.
@@ -15,9 +17,18 @@ Development
     - If you catch/use ``MongoEngineConnectionError`` in your code, you'll have to rename it.
 - BREAKING CHANGE: Positional arguments when instantiating a document are no longer supported. #2103
     - From now on keyword arguments (e.g. ``Doc(field_name=value)``) are required.
+- BREAKING CHANGE: A ``LazyReferenceField`` is now stored in the ``_data`` field of its parent as a ``DBRef``, ``Document``, or ``EmbeddedDocument`` (``ObjectId`` is no longer allowed). #2182
+- DEPRECATION: ``Q.empty`` & ``QNode.empty`` are marked as deprecated and will be removed in a next version of MongoEngine. #2210
+    - Added ability to check if Q or QNode are empty by parsing them to bool.
+    - Instead of ``Q(name="John").empty`` use ``not Q(name="John")``.
+- Improve error message related to InvalidDocumentError #2180
 - Fix updating/modifying/deleting/reloading a document that's sharded by a field with ``db_field`` specified. #2125
 - ``ListField`` now accepts an optional ``max_length`` parameter. #2110
+- Switch from nosetest to pytest as test runner #2114
 - The codebase is now formatted using ``black``. #2109
+- In bulk write insert, the detailed error message would raise in exception.
+- Added ability to compare Q and Q operations #2204
+- Added ability to use a db alias on query_counter #2194
 
 Changes in 0.18.2
 =================

docs/conf.py

@@ -11,7 +11,8 @@
 # All configuration values have a default; values that are commented out
 # serve to show the default.
 
-import sys, os
+import os
+import sys
 
 import sphinx_rtd_theme
 

docs/faq.rst

@@ -0,0 +1,13 @@
+==========================
+Frequently Asked Questions
+==========================
+
+Does MongoEngine support asynchronous drivers (Motor, TxMongo)?
+---------------------------------------------------------------
+
+No, MongoEngine is exclusively based on PyMongo and isn't designed to support other driver.
+If this is a requirement for your project, check the alternative:  `uMongo`_ and `MotorEngine`_.
+
+.. _uMongo: https://umongo.readthedocs.io/
+.. _MotorEngine: https://motorengine.readthedocs.io/
+

docs/guide/defining-documents.rst

@@ -352,7 +352,7 @@ Its value can take any of the following constants:
   Deletion is denied if there still exist references to the object being
   deleted.
 :const:`mongoengine.NULLIFY`
-  Any object's fields still referring to the object being deleted are removed
+  Any object's fields still referring to the object being deleted are set to None
   (using MongoDB's "unset" operation), effectively nullifying the relationship.
 :const:`mongoengine.CASCADE`
   Any object containing fields that are referring to the object being deleted
@@ -744,7 +744,7 @@ Document inheritance
 
 To create a specialised type of a :class:`~mongoengine.Document` you have
 defined, you may subclass it and add any extra fields or methods you may need.
-As this is new class is not a direct subclass of
+As this new class is not a direct subclass of
 :class:`~mongoengine.Document`, it will not be stored in its own collection; it
 will use the same collection as its superclass uses. This allows for more
 convenient and efficient retrieval of related documents -- all you need do is
@@ -767,6 +767,27 @@ document.::
           Setting :attr:`allow_inheritance` to True should also be used in
           :class:`~mongoengine.EmbeddedDocument` class in case you need to subclass it
 
+When it comes to querying using :attr:`.objects()`, querying `Page.objects()` will query
+both `Page` and `DatedPage` whereas querying `DatedPage` will only query the `DatedPage` documents.
+Behind the scenes, MongoEngine deals with inheritance by adding a :attr:`_cls` attribute that contains
+the class name in every documents. When a document is loaded, MongoEngine checks
+it's :attr:`_cls` attribute and use that class to construct the instance.::
+
+    Page(title='a funky title').save()
+    DatedPage(title='another title', date=datetime.utcnow()).save()
+
+    print(Page.objects().count())         # 2
+    print(DatedPage.objects().count())    # 1
+
+    # print documents in their native form
+    # we remove 'id' to avoid polluting the output with unnecessary detail
+    qs = Page.objects.exclude('id').as_pymongo()
+    print(list(qs))
+    # [
+    #   {'_cls': u 'Page', 'title': 'a funky title'},
+    #   {'_cls': u 'Page.DatedPage', 'title': u 'another title', 'date': datetime.datetime(2019, 12, 13, 20, 16, 59, 993000)}
+    # ]
+
 Working with existing data
 --------------------------
 As MongoEngine no longer defaults to needing :attr:`_cls`, you can quickly and

docs/guide/index.rst

@@ -13,4 +13,5 @@ User Guide
    gridfs
    signals
    text-indexes
+   logging-monitoring
    mongomock

docs/guide/logging-monitoring.rst

@@ -0,0 +1,80 @@
+==================
+Logging/Monitoring
+==================
+
+It is possible to use `pymongo.monitoring <https://api.mongodb.com/python/current/api/pymongo/monitoring.html>`_ to monitor
+the driver events (e.g: queries, connections, etc). This can be handy if you want to monitor the queries issued by
+MongoEngine to the driver.
+
+To use `pymongo.monitoring` with MongoEngine, you need to make sure that you are registering the listeners
+**before** establishing the database connection (i.e calling `connect`):
+
+The following snippet provides a basic logging of all command events:
+
+.. code-block:: python
+
+    import logging
+    from pymongo import monitoring
+    from mongoengine import *
+
+    log = logging.getLogger()
+    log.setLevel(logging.DEBUG)
+    logging.basicConfig(level=logging.DEBUG)
+
+
+    class CommandLogger(monitoring.CommandListener):
+
+        def started(self, event):
+            log.debug("Command {0.command_name} with request id "
+                     "{0.request_id} started on server "
+                     "{0.connection_id}".format(event))
+
+        def succeeded(self, event):
+            log.debug("Command {0.command_name} with request id "
+                     "{0.request_id} on server {0.connection_id} "
+                     "succeeded in {0.duration_micros} "
+                     "microseconds".format(event))
+
+        def failed(self, event):
+            log.debug("Command {0.command_name} with request id "
+                     "{0.request_id} on server {0.connection_id} "
+                     "failed in {0.duration_micros} "
+                     "microseconds".format(event))
+
+    monitoring.register(CommandLogger())
+
+
+    class Jedi(Document):
+        name = StringField()
+
+
+    connect()
+
+
+    log.info('GO!')
+
+    log.info('Saving an item through MongoEngine...')
+    Jedi(name='Obi-Wan Kenobii').save()
+
+    log.info('Querying through MongoEngine...')
+    obiwan = Jedi.objects.first()
+
+    log.info('Updating through MongoEngine...')
+    obiwan.name = 'Obi-Wan Kenobi'
+    obiwan.save()
+
+
+Executing this prints the following output::
+
+    INFO:root:GO!
+    INFO:root:Saving an item through MongoEngine...
+    DEBUG:root:Command insert with request id 1681692777 started on server ('localhost', 27017)
+    DEBUG:root:Command insert with request id 1681692777 on server ('localhost', 27017) succeeded in 562 microseconds
+    INFO:root:Querying through MongoEngine...
+    DEBUG:root:Command find with request id 1714636915 started on server ('localhost', 27017)
+    DEBUG:root:Command find with request id 1714636915 on server ('localhost', 27017) succeeded in 341 microseconds
+    INFO:root:Updating through MongoEngine...
+    DEBUG:root:Command update with request id 1957747793 started on server ('localhost', 27017)
+    DEBUG:root:Command update with request id 1957747793 on server ('localhost', 27017) succeeded in 455 microseconds
+
+More details can of course be obtained by checking the `event` argument from the `CommandListener`.