How to get ReferenceField data in mongoengine?

Question

I'm have a problem that query set retrieving oid in json, and I would like to retrieve actual username of that User collection I have below:

class User(db.Document):
    username      = db.StringField(required=True)
    password_hash = db.StringField()
    is_admin      = db.IntField(default=0)

class Message(db.EmbeddedDocument):
    to_users     = db.ListField(db.ReferenceField(User))
    created_at   = db.DateTimeField(default=datetime.now)
    is_read      = db.BooleanField(default=False)
    body         = db.StringField(required=True)

class Inbox(db.Document):
    from_user    = db.ReferenceField(User, required=True)
    subject      = db.StringField(max_length=255, required=True)
    created_at   = db.DateTimeField(default=datetime.now)
    messages     = db.ListField(db.EmbeddedDocumentField(Message))

username = User().get_username()
username = User.objects(username=username).first()

inbox = Inbox.objects.filter(messages__to_users__in=[username]).only('from_user', 'subject', 'created_at').to_json()

And that's the result which I get:

[{"created_at": {"$date": 1401593024844}, "from_user": {"$oid": "538ad45fb43fdd69076d3e64"}, "subject": "test"}]

Would be nice to have something like this:

[{"created_at": {"$date": 1401593024844}, "from_user": {"username": "holms"}, "subject": "test"}]

or somehow similar like "from_user" : {"User": {"username":"holms"}}

UPDATE

Above answer works for the ReferenceField that is in Document, but what about EmbeddedDocument references? Let's say I'll add "messages" to only()?

In [97]: inbox = Inbox.objects(messages__to_users__in=[username]).only('from_user', 'subject', 'created_at', 'messages').select_related()

Currently I've found the way by mapping dictionary which is quite painful.

In [125]: for entry in inbox:
    item = dict(subject=entry.subject, messages=map(lambda msg: dict(body=msg.body, is_read=msg.is_read, to_users=map(lambda usr: usr.username, msg.to_users)), entry.messages), from_user=entry.from_user.username)
    result.append(item)
   .....:

In [126]: result
Out[126]:
[
 {'from_user': u'holms',
  'messages': [{'body': u'test body',
    'is_read': False,
    'to_users': [u'holms']}],
  'subject': u'test'}]

UPDATE 2 Seems to be this done the trick

 57     def to_json(self):
 58         data = self.to_mongo() # get pymongo representation
 59         data["from_user"] = {"User": {"username": self.from_user.username} }
 60
 61         for key, reply in enumerate(data["messages"]):
 62             for user in self.messages[key].to_users:
 63                 print user.username
 64             reply["to_users"] = {"User": {"username": user.username} }
 65
 66         return json_util.dumps(data)

and you get this :)

    {
       "_id":{
          "$oid":"538ad500b43fdd690e2eefb5"
       },
       "from_user":{
          "User":{
             "username":"holms"
          }
       },
       "subject":"test",
       "created_at":{
          "$date":1401593024844
       },
       "messages":[
          {
             "to_users":{
                "User":{
                   "username":"holms"
                }
             },
             "created_at":{
                "$date":1401593024845
             },
             "is_read":false,
             "body":"test body"
          }
       ]

    }

Solution

Currently this is not supported directly as MongoEngine only supports mongodb's extended json syntax.

All mongoengine does under the covers is use pymongo's json_utils to dump the data. Theres no reason why you can't use it explicitly eg:

    from bson import json_util

    class Inbox(db.Document):
        from_user    = db.ReferenceField(User, required=True)
        subject      = db.StringField(max_length=255, required=True)
        created_at   = db.DateTimeField(default=datetime.now)
        messages     = db.ListField(db.EmbeddedDocumentField(Message))


        def to_json(self):
            data = self.to_mongo() // get the pymongo representation of the document
            data["from_user"] = {"User": {"username": self.from_user.username}}
            return json_util.dumps(data)

            User.drop_collection()

    ...

    Inbox.drop_collection()

    ross = User(username="Ross").save()
    Inbox(from_user=ross, subject="Mongoengine should make json easier").save()

    doc = Inbox.objects.only('from_user', 'subject', 'created_at').get()
    print doc.to_json()

    {"_id": {"$oid": "538c3d71c3d384172fe35393"}, 
     "from_user": {"User": {"username": "Ross"}}, 
     "subject": "Mongoengine should make json easier", 
     "created_at": {"$date": 1401703297198}, "messages": []}

Updated

Example with custom queryset:

   from bson import json_util

    class CustomQuerySet(QuerySet):
         def to_json(self):
            return "[%s]" % (",".join([doc.to_json() for doc in self]))

    class Inbox(Document):
        from_user    = ReferenceField(User, required=True)
        subject      = StringField(max_length=255, required=True)
        created_at   = DateTimeField(default=datetime.now)
        messages     = ListField(EmbeddedDocumentField(Message))

        meta = {'queryset_class': CustomQuerySet}

        def to_json(self):
            data = self.to_mongo()
            data["from_user"] = {"User": {"username": self.from_user.username}}
            return json_util.dumps(data)

  ...
  ipdb> Inbox.objects.only('from_user', 'subject', 'created_at').to_json()
        '[{"_id": {"$oid": "538d84cbc3d3843eeeb5dbbe"}, 
           "from_user": {"User": {"username": "Ross"}}, 
           "subject": "Mongoengine should make json easier", 
           "created_at": {"$date": 1401787099246}, "messages": []}]'