Merge branch 'master' into fix_2484

Author: janste63

.github/workflows/github-actions.yml

@@ -11,9 +11,10 @@ on:
     tags:
       - 'v[0-9]+\.[0-9]+\.[0-9]+*'
 env:
-  MONGODB_3_4: 3.4.19
-  MONGODB_3_6: 3.6.13
-  MONGODB_4_0: 4.0.13
+  MONGODB_3_6: 3.6.14
+  MONGODB_4_0: 4.0.23
+  MONGODB_4_2: 4.2
+  MONGODB_4_4: 4.4
 
   PYMONGO_3_4: 3.4
   PYMONGO_3_6: 3.6
@@ -47,14 +48,14 @@ jobs:
         MONGODB: [$MONGODB_4_0]
         PYMONGO: [$PYMONGO_3_11]
         include:
-          - python-version: 3.7
-            MONGODB: $MONGODB_3_4
-            PYMONGO: $PYMONGO_3_6
           - python-version: 3.7
             MONGODB: $MONGODB_3_6
             PYMONGO: $PYMONGO_3_9
           - python-version: 3.7
-            MONGODB: $MONGODB_3_6
+            MONGODB: $MONGODB_4_2
+            PYMONGO: $PYMONGO_3_6
+          - python-version: 3.7
+            MONGODB: $MONGODB_4_4
             PYMONGO: $PYMONGO_3_11
     steps:
     - uses: actions/checkout@v2

.github/workflows/install_mongo.sh

@@ -2,7 +2,17 @@
 
 MONGODB=$1
 
+# Mongo > 4.0 follows different name convention for download links
 mongo_build=mongodb-linux-x86_64-${MONGODB}
+
+if [[ "$MONGODB" == *"4.2"* ]]; then
+  mongo_build=mongodb-linux-x86_64-ubuntu1804-v${MONGODB}-latest
+elif [[ "$MONGODB" == *"4.4"* ]]; then
+  mongo_build=mongodb-linux-x86_64-ubuntu1804-v${MONGODB}-latest
+fi
+
 wget http://fastdl.mongodb.org/linux/$mongo_build.tgz
 tar xzf $mongo_build.tgz
-${PWD}/$mongo_build/bin/mongod --version
+
+mongodb_dir=$(find ${PWD}/ -type d -name "mongodb-linux-x86_64*")
+$mongodb_dir/bin/mongod --version

.github/workflows/start_mongo.sh

@@ -2,7 +2,8 @@
 
 MONGODB=$1
 
-mongodb_dir=${PWD}/mongodb-linux-x86_64-${MONGODB}
+mongodb_dir=$(find ${PWD}/ -type d -name "mongodb-linux-x86_64*")
+
 mkdir $mongodb_dir/data
 $mongodb_dir/bin/mongod --dbpath $mongodb_dir/data --logpath $mongodb_dir/mongodb.log --fork
 mongo --eval 'db.version();'    # Make sure mongo is awake

docs/changelog.rst

@@ -1,13 +1,20 @@
 
+
 =========
 Changelog
 =========
 
 Development
 ===========
 - (Fill this out as you fix issues and develop your features).
-- Bugfix: manually setting SequenceField in DynamicDocument doesn't increment the counter #2471
 - Bug fix: ignore LazyReferenceFields when clearing _changed_fields #2484
+- Improve connection doc #2481
+
+Changes in 0.23.0
+=================
+- Bugfix: manually setting SequenceField in DynamicDocument doesn't increment the counter #2471
+- Add MongoDB 4.2 and 4.4 to CI
+- Add support for allowDiskUse on querysets #2468
 
 Changes in 0.22.1
 =================

docs/guide/connecting.rst

@@ -5,7 +5,7 @@ Connecting to MongoDB
 =====================
 
 Connections in MongoEngine are registered globally and are identified with aliases.
-If no `alias` is provided during the connection, it will use "default" as alias.
+If no ``alias`` is provided during the connection, it will use "default" as alias.
 
 To connect to a running instance of :program:`mongod`, use the :func:`~mongoengine.connect`
 function. The first argument is the name of the database to connect to::
@@ -14,27 +14,66 @@ function. The first argument is the name of the database to connect to::
     connect('project1')
 
 By default, MongoEngine assumes that the :program:`mongod` instance is running
-on **localhost** on port **27017**. If MongoDB is running elsewhere, you should
-provide the :attr:`host` and :attr:`port` arguments to
-:func:`~mongoengine.connect`::
+on **localhost** on port **27017**.
 
-    connect('project1', host='192.168.1.35', port=12345)
+If MongoDB is running elsewhere, you need to provide details on how to connect. There are two ways of
+doing this. Using a connection string in URI format (**this is the preferred method**) or individual attributes
+provided as keyword arguments.
 
-If the database requires authentication, :attr:`username`, :attr:`password`
-and :attr:`authentication_source` arguments should be provided::
+Connect with URI string
+=======================
+
+When using a connection string in URI format you should specify the connection details
+as the :attr:`host` to :func:`~mongoengine.connect`. In a web application context for instance, the URI
+is typically read from the config file::
+
+        connect(host="mongodb://127.0.0.1:27017/my_db")
+
+If the database requires authentication, you can specify it in the
+URI. As each database can have its own users configured, you need to tell MongoDB
+where to look for the user you are working with, that's what the ``?authSource=admin`` bit
+of the MongoDB connection string is for::
+
+    # Connects to 'my_db' database by authenticating
+    # with given credentials against the 'admin' database (by default as authSource isn't provided)
+    connect(host="mongodb://my_user:[email protected]:27017/my_db")
+
+    # Equivalent to previous connection but explicitly states that
+    # it should use admin as the authentication source database
+    connect(host="mongodb://my_user:my_password@hostname:port/my_db?authSource=admin")
+
+    # Connects to 'my_db' database by authenticating
+    # with given credentials against that same database
+    connect(host="mongodb://my_user:[email protected]:27017/my_db?authSource=my_db")
+
+The URI string can also be used to configure advanced parameters like ssl, replicaSet, etc. For more
+information or example about URI string, you can refer to the `official doc <https://docs.mongodb.com/manual/reference/connection-string/>`_::
+
+    connect(host="mongodb://my_user:[email protected]:27017/my_db?authSource=admin&ssl=true&replicaSet=globaldb")
+
+.. note:: URI containing SRV records (e.g "mongodb+srv://server.example.com/") can be used as well
 
-    connect('project1', username='webapp', password='pwd123', authentication_source='admin')
+Connect with keyword attributes
+===============================
 
-URI style connections are also supported -- just supply the URI as
-the :attr:`host` to
-:func:`~mongoengine.connect`::
+The second option for specifying the connection details is to provide the information as keyword
+attributes to :func:`~mongoengine.connect`::
 
-    connect('project1', host='mongodb://localhost/database_name')
+    connect('my_db', host='127.0.0.1', port=27017)
 
-.. note:: URI containing SRV records (e.g mongodb+srv://server.example.com/) can be used as well as the :attr:`host`
+If the database requires authentication, :attr:`username`, :attr:`password`
+and :attr:`authentication_source` arguments should be provided::
+
+    connect('my_db', username='my_user', password='my_password', authentication_source='admin')
+
+The set of attributes that :func:`~mongoengine.connect` recognizes includes but is not limited to:
+:attr:`host`, :attr:`port`, :attr:`read_preference`, :attr:`username`, :attr:`password`, :attr:`authentication_source`, :attr:`authentication_mechanism`,
+:attr:`replicaset`, :attr:`tls`, etc. Most of the parameters accepted by `pymongo.MongoClient <https://pymongo.readthedocs.io/en/stable/api/pymongo/mongo_client.html#pymongo.mongo_client.MongoClient>`_
+can be used with :func:`~mongoengine.connect` and will simply be forwarded when instantiating the `pymongo.MongoClient`.
 
 .. note:: Database, username and password from URI string overrides
-    corresponding parameters in :func:`~mongoengine.connect`: ::
+    corresponding parameters in :func:`~mongoengine.connect`, this should
+    obviously be avoided: ::
 
         connect(
             db='test',
@@ -43,28 +82,19 @@ the :attr:`host` to
             host='mongodb://admin:qwerty@localhost/production'
         )
 
-    will establish connection to ``production`` database using
-    ``admin`` username and ``12345`` password.
+    will establish connection to ``production`` database using ``admin`` username and ``qwerty`` password.
 
 .. note:: Calling :func:`~mongoengine.connect` without argument will establish
     a connection to the "test" database by default
 
-Replica Sets
-============
-
-MongoEngine supports connecting to replica sets::
-
-    from mongoengine import connect
-
-    # Regular connect
-    connect('dbname', replicaset='rs-name')
-
-    # MongoDB URI-style connect
-    connect(host='mongodb://localhost/dbname?replicaSet=rs-name')
+Read Preferences
+================
 
-Read preferences are supported through the connection or via individual
+As stated above, Read preferences are supported through the connection but also via individual
 queries by passing the read_preference ::
 
+    from pymongo import ReadPreference
+
     Bar.objects().read_preference(ReadPreference.PRIMARY)
     Bar.objects(read_preference=ReadPreference.PRIMARY)
 

docs/guide/defining-documents.rst

@@ -290,12 +290,12 @@ as the constructor's argument::
         content = StringField()
 
 
-.. _one-to-many-with-listfields:
+.. _many-to-many-with-listfields:
 
-One to Many with ListFields
+Many to Many with ListFields
 '''''''''''''''''''''''''''
 
-If you are implementing a one to many relationship via a list of references,
+If you are implementing a many to many relationship via a list of references,
 then the references are stored as DBRefs and to query you need to pass an
 instance of the object to the query::
 

mongoengine/__init__.py

@@ -28,7 +28,7 @@
 )
 
 
-VERSION = (0, 22, 1)
+VERSION = (0, 23, 0)
 
 
 def get_version():

mongoengine/base/fields.py

@@ -267,6 +267,17 @@ def __init__(self, field=None, **kwargs):
         self.field = field
         super().__init__(**kwargs)
 
+    @staticmethod
+    def _lazy_load_refs(instance, name, ref_values, *, max_depth):
+        _dereference = _import_class("DeReference")()
+        documents = _dereference(
+            ref_values,
+            max_depth=max_depth,
+            instance=instance,
+            name=name,
+        )
+        return documents
+
     def __get__(self, instance, owner):
         """Descriptor to automatically dereference references."""
         if instance is None:
@@ -284,19 +295,15 @@ def __get__(self, instance, owner):
             or isinstance(self.field, (GenericReferenceField, ReferenceField))
         )
 
-        _dereference = _import_class("DeReference")()
-
         if (
             instance._initialised
             and dereference
             and instance._data.get(self.name)
             and not getattr(instance._data[self.name], "_dereferenced", False)
         ):
-            instance._data[self.name] = _dereference(
-                instance._data.get(self.name),
-                max_depth=1,
-                instance=instance,
-                name=self.name,
+            ref_values = instance._data.get(self.name)
+            instance._data[self.name] = self._lazy_load_refs(
+                ref_values=ref_values, instance=instance, name=self.name, max_depth=1
             )
             if hasattr(instance._data[self.name], "_dereferenced"):
                 instance._data[self.name]._dereferenced = True
@@ -322,7 +329,9 @@ def __get__(self, instance, owner):
             and isinstance(value, (BaseList, BaseDict))
             and not value._dereferenced
         ):
-            value = _dereference(value, max_depth=1, instance=instance, name=self.name)
+            value = self._lazy_load_refs(
+                ref_values=value, instance=instance, name=self.name, max_depth=1
+            )
             value._dereferenced = True
             instance._data[self.name] = value