How to fetch/scan all items from `AWS dynamodb` using node.js

Question

How to fetch/scan all items from AWS dynamodb using node.js. I am posting my code here.

var docClient = new aws.DynamoDB.DocumentClient();
    var params = {
    TableName:"users",
    KeyConditionExpression:"user_status=:status",
    ExpressionAttributeValues: {
        ":status": "Y"
    }
    };

    var queryExecute = function(callback) {
        docClient.query(params,function(err,result) {
            if(err) {
                console.log(err)
                callback(err);
                } else {
                console.log(result);

                if(result.LastEvaluatedKey) {
                    params.ExclusiveStartKey = result.LastEvaluatedKey;
                    queryExecute(callback);
                    } else {
                        callback(err,items);
                    }
                }
            });
        }
        queryExecute(callback); 

This is giving me below error.

ValidationException: Query condition missed key schema element: `user_id`.

Here primary key is user_id. I don't want to use it with my query condition, because I need to set a value if I mentioned primary key in KeyConditionExpression. May be I am wrong. However please suggest me a good way to fetch all items from dynamodb, which is having user_status = "Y"

Solution 1

If you would like to get the data from DynamoDB without using Hash key value, you need to use Scan API.

Note: The Scan API reads all the items in the table to get the results. So, it is a costly operation in DynamoDB.

Alternate Approach : Use GSI

Scan Code for the above sceanario:-

var docClient = new AWS.DynamoDB.DocumentClient();

var params = {
    TableName: "users",
    FilterExpression: "#user_status = :user_status_val",
    ExpressionAttributeNames: {
        "#user_status": "user_status",
    },
    ExpressionAttributeValues: { ":user_status_val": 'somestatus' }

};

docClient.scan(params, onScan);
var count = 0;

function onScan(err, data) {
    if (err) {
        console.error("Unable to scan the table. Error JSON:", JSON.stringify(err, null, 2));
    } else {        
        console.log("Scan succeeded.");
        data.Items.forEach(function(itemdata) {
           console.log("Item :", ++count,JSON.stringify(itemdata));
        });

        // continue scanning if we have more items
        if (typeof data.LastEvaluatedKey != "undefined") {
            console.log("Scanning for more...");
            params.ExclusiveStartKey = data.LastEvaluatedKey;
            docClient.scan(params, onScan);
        }
    }
}

Solution 2

This is working for me:

export const scanTable = async (tableName) => {
    const params = {
        TableName: tableName,
    };

    const scanResults = [];
    const items;
    do{
        items =  await documentClient.scan(params).promise();
        items.Items.forEach((item) => scanResults.push(item));
        params.ExclusiveStartKey  = items.LastEvaluatedKey;
    }while(typeof items.LastEvaluatedKey !== "undefined");
    
    return scanResults;

};

Solution 3

AWS documentation example didn't work for me. @Hank approach did the trick.

Using handler inside a lambda:

const AWS = require('aws-sdk');
const docClient = new AWS.DynamoDB.DocumentClient({
    // optional tuning - 50% faster(cold) / 20% faster(hot)
    apiVersion: '2012-08-10',
    sslEnabled: false,
    paramValidation: false,
    convertResponseTypes: false
});

const tableName = 'series';

exports.handler = async (event, context, callback) => {
    let params = { TableName: tableName };

    let scanResults = [];
    let items;

    do {
        items = await docClient.scan(params).promise();
        items.Items.forEach((item) => scanResults.push(item));
        params.ExclusiveStartKey = items.LastEvaluatedKey;
    } while (typeof items.LastEvaluatedKey != "undefined");

    callback(null, scanResults);
};

Solution 4

Using Promises and async

const aws = require('aws-sdk');
aws.config.update({ region: 'us-east-1' });
const documentClient = new aws.DynamoDB.DocumentClient();

const scanAll = async (params) => {
  let lastEvaluatedKey = 'dummy'; // string must not be empty
  const itemsAll = [];
  while (lastEvaluatedKey) {
    const data = await documentClient.scan(params).promise();
    itemsAll.push(...data.Items);
    lastEvaluatedKey = data.LastEvaluatedKey;
    if (lastEvaluatedKey) {
      params.ExclusiveStartKey = lastEvaluatedKey;
    }
  }
  return itemsAll;
}

Use like this

const itemsAll = scanAll(params);

The code is the same for query (just replace scan with query)

Solution 5

I use promises like this:

let AWS = require('aws-sdk');
let docClient = new AWS.DynamoDB.DocumentClient();

async function dbRead(params) {
    let promise = docClient.scan(params).promise();
    let result = await promise;
    let data = result.Items;
    if (result.LastEvaluatedKey) {
        params.ExclusiveStartKey = result.LastEvaluatedKey;
        data = data.concat(await dbRead(params));
    }
    return data;
}

and to use it:

let params = {
  TableName: 'Table'
};
let data = await dbRead(params);

View more solutions

Comments

[Vingtoft]

Scan only scans up to 1MB of data, so this code will not work on databases larger than 1MB.

[notionquest]

The code has a recursive call until LastEvaluatedKey is undefined. So, it should work.

[Vingtoft]

Sorry I missed that!

[Jeremy Souffir]

if (typeof data.LastEvaluatedKey != "undefined") You can just do: if (data.LastEvaluatedKey)

[Juha Kervinen]

While this seems to work, I think it would be a lot better if this answer would actually return the data, instead of just printing it to the console. It's not necessarily easy for people new to JS or functional programming to come up with a solution from here which actually returns the data (via callbak) to the calling function.

[David White]

note: this example and the next still works if you have more data provided then a single scan can handle. Be aware of the dynamodb memory limitations.

[Sandeep]

Hi, i followed this and i am getting results back. only issue is LastEvaluatedKey returned in query is always same for me.

[Eric Dela Cruz]

Agree with @JuhaKervinen, it's better to return data. Hank's answer does that and is much more concise.

[backdesk]

In order to understand recursion, you must first understand recursion.

[Gabriel]

This solution is great for those of us who need a synchronous approach using recursion.

[MSOACC]

This works but some best-practice improvements: make scanResults a const, change != to !==, make items a const and declare it inside the loop with a separate variable outside the do loop for storing the LastEvaluatedKey.

[Dandry]

I like this approach. Thanks!

[kolodi]

You cant assign to constant "items" inside the loop, make it "let" instead. Also you can use better syntax to add multiple items in array: scanResults.push(...items.Items), here you must also check if items.Items is not undefined before pushing.

[Jarrett]

May i ask what is the line params.ExclusiveStartKey = items.LastEvaluateKey for?

[pho_pho]

@Jarrett that is for pagination of results, and allows you to keep track of where to start from for the next scan operation. Scan in dDB will return up to 1 MB of data, or the number of records specified by the Limit parameter. If the table contains more records, then you will need to grab ExclusiveStartKey and pass it in as LastEvaluatedKey for the next query to start scanning from that location. docs.aws.amazon.com/amazondynamodb/latest/APIReference/…

[Sh4dy]

I'm getting an error:- Property 'ExclusiveStartKey' does not exist on type '{ TableName: any; }'. params.ExclusiveStartKey = items.LastEvaluatedKey; How are you referring to a key which is not declared in the params object?

[Manjunath]

Just in case someone is looking for this scenario. You may be using TypeScript to get this error, you may create an interface or a custom type and make ExclusiveStartKey as optional prop and then have this interface for params, this should fix the issue.