Scanning over a Store

The Scan family of APIs are very similar to List but they allow you to list Items across your entire Store. This can be useful scenarios such as:

• Migrations and backfills that need to operate on every Item
• Custom exporters to other datastores
• Auditing/validation workflows
• Deleting unwanted data
• Building global aggregations (e.g. compute the top X blog posts by comments, or counting the number of items meeting some criteria)

Be warned that these operations can be slow and expensive, especially on large Stores. You should use them sparingly and consider using List instead if you can. These results of a Scan operation are not guaranteed to be in any particular order and Items with multiple key paths will only be returned once with their primary key path.

Beginning a Scan

Just like for a List operation, you begin by calling BeginScan, with your desired parameters. Then you can continue to retrieve more Items by calling ContinueScan with the token returned by BeginScan.

For this example we’ll use the schema defined in Example: Movies Schema, which defines these key paths (among others):

Item Type	Key Path Template
Movie	/movie-:id
Actor	/actor-:id

Go

package main

import (
  "context"
  "fmt"
  "os"
  "slices"
  "strconv"
  "time"

  "github.com/google/uuid"

  "github.com/StatelyCloud/go-sdk/stately"
  // This is the code you generated from schema
  "github.com/StatelyCloud/stately/go-sdk-sample/schema"
)

func sampleScan(
  ctx context.Context,
  client stately.Client,
) (*stately.ListToken, error) {
  iter, err := client.BeginScan(
    ctx,
    stately.ScanOptions{ItemTypes: []string{"Movie", "Actor"}},
  )
  if err != nil {
    return nil, err
  }

  for iter.Next() {
    item := iter.Value()
    switch v := item.(type) {
    case *schema.Movie:
      fmt.Printf("Movie Title: %s\n", v.GetTitle())
    case *schema.Actor:
      fmt.Printf("Actor Name: %s\n", v.GetName())
    }
  }
  // When we've exhausted the iterator, we'll get a token that we
  // can use to fetch the next page of items.
  return iter.Token()
}

Ruby

require 'bundler/setup'
require_relative 'schema/stately'
require 'byebug'


def sample_scan(client)
  begin_scan_result, token = client.begin_scan(item_types: ['Movie', 'Actor'])

  begin_scan_result.each do |item|
    case item
    when StatelyDB::Types::Movie
      puts "[Movie] title: #{item.title}"
    when StatelyDB::Types::Actor
      puts "[Actor] name: #{item.name}"
    end
  end

  return token
end

Python

from __future__ import annotations

from typing import TYPE_CHECKING

from statelydb import ListToken, SyncChangedItem, SyncDeletedItem, SyncReset, key_path

from .schema import Actor, Change, Character, Client, Movie


async def sample_scan(client: Client) -> None:
    scan_resp = await client.begin_scan(item_types=[Movie, Actor])

    async for item in scan_resp:
        if isinstance(item, Movie):
            print(f"[Movie] title: {item.title}")
        elif isinstance(item, Actor):
            print(f"[Actor] name: {item.name}")

    # When we've exhausted the iterator, we'll get a token that we can
    # use to fetch the next page of items.
    return scan_resp.token

TypeScript

import { createClient, DatabaseClient, Movie } from "./schema/index.js";
import { keyPath, ListToken } from "@stately-cloud/client";

async function sampleScan(client: DatabaseClient): Promise<ListToken> {
  let iter = client.beginScan({
    itemTypes: ["Movie", "Actor"],
  });
  for await (const item of iter) {
    if (client.isType(item, "Movie")) {
      console.log("Movie:", item.title);
    } else if (client.isType(item, "Actor")) {
      console.log("Actor:", item.name);
    }
  }
  return iter.token!;
}

CLI

stately item scan \
  --store-id <store-id-goes-here> \
  --item-types Movie,Actor

Using the List Token to Continue

The result from BeginScan includes a list token which you can use to continue in the ContinueScan. Read more about list tokens in Using the List Token to Continue. token.canSync will always be set to false for Scan operations.

Go

package main

import (
  "context"
  "fmt"
  "os"
  "slices"
  "strconv"
  "time"

  "github.com/google/uuid"

  "github.com/StatelyCloud/go-sdk/stately"
  // This is the code you generated from schema
  "github.com/StatelyCloud/stately/go-sdk-sample/schema"
)

func sampleContinueScan(
  ctx context.Context,
  client stately.Client,
  token *stately.ListToken,
) (*stately.ListToken, error) {
  iter, err := client.ContinueScan(ctx, token.Data)
  if err != nil {
    return nil, err
  }
  for iter.Next() {
    item := iter.Value()
    switch v := item.(type) {
    case *schema.Character:
      fmt.Printf("Character Name: %s\n", v.GetName())
    case *schema.Actor:
      fmt.Printf("Actor Name: %s\n", v.GetName())
    }
  }
  // You could save the token to call ContinueScan later.
  return iter.Token()
}

Ruby

require 'bundler/setup'
require_relative 'schema/stately'
require 'byebug'


def sample_continue_list(client, token)
  # Fetch the next page of items
  continue_scan_result, token = client.continue_scan(token)

  continue_scan_result.each do |item|
    case item
    when StatelyDB::Types::Movie
      puts "[Movie] title: #{item.title}"
    when StatelyDB::Types::Actor
      puts "[Actor] name: #{item.name}"
    end
  end

  # You could save the token to call ContinueScan later.
  return token
end

Python

from __future__ import annotations

from typing import TYPE_CHECKING

from statelydb import ListToken, SyncChangedItem, SyncDeletedItem, SyncReset, key_path

from .schema import Actor, Change, Character, Client, Movie


async def sample_continue_scan(client: Client, token: str) -> ListToken:
    # Fetch the next page of items
    continue_scan_result = await client.continue_scan(token)

    # Print out the paths of the next batch of listed items
    async for item in continue_scan_result:
        if isinstance(item, Movie):
            print(f"[Movie] title: {item.title}")
        elif isinstance(item, Actor):
            print(f"[Actor] name: {item.name}")

    # You could save the token to call ContinueScan later.
    return continue_scan_result.token

TypeScript

import { createClient, DatabaseClient, Movie } from "./schema/index.js";
import { keyPath, ListToken } from "@stately-cloud/client";

async function sampleContinueScan(
  client: DatabaseClient,
  token: ListToken,
): Promise<ListToken> {
  // You can call `collect` on the iterator to pull
  // all the items into an Array.
  const { items, token: newToken } = await client
    .continueScan(token)
    .collect();

  for (const item of items) {
    if (client.isType(item, "Movie")) {
      console.log("Movie:", item.title);
    } else if (client.isType(item, "Actor")) {
      console.log("Actor:", item.name);
    }
  }
  // You could save the token to call ContinueScan later.
  return newToken;
}

Filtering

You can pass a filter to BeginScan to only retrieve Items that match the filter. We currently support filtering by Item Type.

Limits

Pass a limit to BeginScan to limit the max number of items to retrieve. If limit is set to 0 then the first page of results will be returned which may be empty because all the results were filtered out. Be sure to check token.canContinue to see if there are more results to fetch.

Segmentation

Because a Scan operation can be slow and expensive, you can segment the operation into smaller chunks by passing a totalSegments and segmentIndex parameter to BeginScan. This will allow you to run multiple Scan operations in parallel, each responsible for a different segment of the Store. You can split your scan into up to 1000000 segments.

Listing Across Client Upgrades

Just like for List operations, you are not able to use a list token across client versions.