To set up Drizzle ORM in NestJS with each type of databases, follow these steps:

1. Install Dependencies

Install Drizzle ORM, driver of the database that you wish to use and its NestJS integration module using your favorite package manager.

PlanetScale

npm install drizzle-orm
npm install @planetscale/database
npm install @knaadh/nestjs-drizzle-planetscale

Turso

npm install drizzle-orm
npm install @libsql/client
npm install @knaadh/nestjs-drizzle-turso

MySQL2

npm install drizzle-orm
npm install mysql2
npm install @knaadh/nestjs-drizzle-mysql2

Node-Postgres

npm install drizzle-orm
npm install pg
npm install @knaadh/nestjs-drizzle-pg

Better-SQLite3

npm install drizzle-orm
npm install better-sqlite3
npm install @knaadh/nestjs-drizzle-better-sqlite3

2. Create SQL Schema File

In Drizzle ORM, you need to define your database schema in TypeScript. You can declare your schema in a single schema.ts file or group them logically in multiple files. The schema declaration includes tables, indexes, constraints, foreign keys, and enums as explained here. Here’s an example of declaring tables in a schema.ts file for each of the database types.

MySQL

import { mysqlTable, serial, varchar } from 'drizzle-orm/mysql-core';

export const books = mysqlTable('Books', {
  id: serial('id').primaryKey(),
  name: varchar('name', { length: 256 }),
});

export const authors = mysqlTable('Authors', {
  id: serial('id').primaryKey(),
  name: varchar('name', { length: 256 }),
});

PostgreSQL

import { pgTable, serial, varchar } from 'drizzle-orm/pg-core';

export const books = pgTable('Books', {
  id: serial('id').primaryKey(),
  name: varchar('name', { length: 256 }),
});

export const authors = pgTable('Authors', {
  id: serial('id').primaryKey(),
  name: varchar('name', { length: 256 }),
});

SQLite

import { sqliteTable, text, integer } from 'drizzle-orm/sqlite-core';

export const books = sqliteTable('Books', {
  id: integer('id').primaryKey(),
  name: text('name'),
});

export const authors = sqliteTable('Authors', {
  id: integer('id').primaryKey(),
  name: text('name'),
});

3. Configure the Module:

Import the integration module of the respective database driver into the root AppModule and configure it with your credentials, schema file and injection tag as shown in the sample code below for each of the drivers.

PlanetScale

import { Module } from '@nestjs/common';
import { AppController } from './app.controller';
import { AppService } from './app.service';
import * as schema from '../db/schema';
import { DrizzlePlanetScaleModule } from '@knaadh/nestjs-drizzle-planetscale';

@Module({
  imports: [
    DrizzlePlanetScaleModule.register({
      tag: 'DB_PROD',
      planetscale: {
        config: {
          username: 'PLANETSCALE_USERNAME',
          password: 'PLANETSCALE_PASSWORD',
          host: 'PLANETSCALE_HOST',
        },
      },
      config: { schema: { ...schema } },
    }),
  ],
  controllers: [AppController],
  providers: [AppService],
})
export class AppModule {}

Turso

import { Module } from '@nestjs/common';
import { AppController } from './app.controller';
import { AppService } from './app.service';
import * as schema from '../db/schema';
import { DrizzleTursoModule } from '@knaadh/nestjs-drizzle-turso';

@Module({
  imports: [
    DrizzleTursoModule.register({
      tag: 'DB_PROD',
      turso: {
        config: {
          url: 'DATABASE_URL',
          authToken: 'DATABASE_AUTH_TOKEN',
        },
      },
      config: { schema: { ...schema } },
    }),
  ],
  controllers: [AppController],
  providers: [AppService],
})
export class AppModule {}

MySQL2

import { Module } from '@nestjs/common';
import { AppController } from './app.controller';
import { AppService } from './app.service';
import * as schema from '../db/schema';
import { DrizzleMySqlModule } from '@knaadh/nestjs-drizzle-mysql2';

@Module({
  imports: [
    DrizzleMySqlModule.register({
      tag: 'DB_PROD',
      mysql: {
        connection: 'client',
        config: {
          host: DATABASE_HOST,
          user: DATABASE_USER,
          password : DATABASE_PASSWORD,
          database: DATABASE_NAME,
        },
      },
      config: { schema: { ...schema }, mode: 'default' },
    }),
  ],
  controllers: [AppController],
  providers: [AppService],
})
export class AppModule {}

Node-Postgres

import { Module } from '@nestjs/common';
import { AppController } from './app.controller';
import { AppService } from './app.service';
import * as schema from '../db/schema';
import { DrizzlePGModule } from '@knaadh/nestjs-drizzle-pg';

@Module({
  imports: [
    DrizzlePGModule.register({
      tag: 'DB_PROD',
      pg: {
        connection: 'client',
        config: {
          connectionString: DATABASE_CONNECTION_URL,
        },
      },
      config: { schema: { ...schema } },
    }),
  ],
  controllers: [AppController],
  providers: [AppService],
})
export class AppModule {}

Better-SQLite3

import { Module } from '@nestjs/common';
import { AppController } from './app.controller';
import { AppService } from './app.service';
import * as schema from '../db/schema';
import { DrizzleBetterSQLiteModule } from '@knaadh/nestjs-drizzle-better-sqlite3';

@Module({
  imports: [
    DrizzleBetterSQLiteModule.register({
      tag: 'DB_PROD',
      sqlite3: {
        filename: DATABASE_FILE,
      },
      config: { schema: { ...schema } },
    }),
  ],
  controllers: [AppController],
  providers: [AppService],
})
export class AppModule {}

4. Inject Drizzle Service:

You can inject the Drizzle instance anywhere using the tag specified in the module configuration of its respective database driver.

PlanetScale

import { Inject, Injectable } from '@nestjs/common';
import * as schema from '../db/schema';
import { PlanetScaleDatabase } from 'drizzle-orm/planetscale-serverless';
@Injectable()
export class AppService {
  constructor(
    @Inject('DB_PROD') private drizzleProd: PlanetScaleDatabase<typeof schema>
  ) {}
  async getData() {
    const authors = await this.drizzleProd.query.authors.findMany();
    return authors;
  }
}

Turso

import { Inject, Injectable } from '@nestjs/common';
import * as schema from '../db/schema';
import { LibSQLDatabase } from 'drizzle-orm/libsql';
@Injectable()
export class AppService {
  constructor(
    @Inject('DB_PROD') private drizzleProd: LibSQLDatabase<typeof schema>
  ) {}
  async getData() {
    const authors = await this.drizzleProd.query.authors.findMany();
    return authors;
  }
}

MySQL2

import { Inject, Injectable } from '@nestjs/common';
import * as schema from '../db/schema';
import { MySql2Database } from 'drizzle-orm/mysql2';
@Injectable()
export class AppService {
  constructor(
    @Inject('DB_PROD') private drizzleProd: MySql2Database<typeof schema>
  ) {}
  async getData() {
    const authors = await this.drizzleProd.query.authors.findMany();
    return authors;
  }
}

Node-Postgres

import { Inject, Injectable } from '@nestjs/common';
import * as schema from '../db/schema';
import { NodePgDatabase } from 'drizzle-orm/node-postgres';
@Injectable()
export class AppService {
  constructor(
    @Inject('DB_PROD') private drizzleProd: NodePgDatabase<typeof schema>
  ) {}
  async getData() {
    const authors = await this.drizzleProd.query.authors.findMany();
    return authors;
  }
}

Better-SQLite3

import { Inject, Injectable } from '@nestjs/common';
import * as schema from '../db/schema';
import { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3';
@Injectable()
export class AppService {
  constructor(
    @Inject('DB_PROD') private drizzleProd: BetterSQLite3Database<typeof schema>
  ) {}
  async getData() {
    const authors = await this.drizzleProd.query.authors.findMany();
    return authors;
  }
}

Note: You can connect as many databases as you want, provided the tag should always be different for each database module.

The post How to use Drizzle ORM with NestJS appeared first on Mithle.sh.

Benchmarks

For benchmarking, we are using the artist table from the MusicBrainz database. We are using Knex-Nest – pagination included Knex module for NestJS – for benchmarking.

Offset Pagination

 async getOffsetPagingBenchmarks(
    from = 1,
    to = 10,
    perPage = 50,
    total?: number
  ) {
    let avgQueryTimeOffset = 0;
    for (let index = from; index <= to; index += 1) {
      const tag = `Page ${index}`;
      const start = hrtime.bigint();
      const query = this.knexPg('artist')
        .select('id', 'name')
        .orderBy('id', 'asc');
      await KnexPagination.offsetPaginate({
        query: query,
        perPage: perPage,
        goToPage: index,
        count: total,
      });
      const end = hrtime.bigint();
      const latency = round(Number(end - start) / 1000000);
      avgQueryTimeOffset += latency;
      console.log(`${tag}: ${latency} ms`);
    }
    avgQueryTimeOffset = round(avgQueryTimeOffset / 10);
    console.log(`Average (offset): ${avgQueryTimeOffset} ms`);
    return avgQueryTimeOffset;
  }

Cursor Pagination

 async getCursorPagingBenchmarks(
    from = 'start',
    numOfPages = 10,
    perPage = 50
  ) {
    let avgQueryTimeCursor = 0;
    let cursor = null;
    let index = 0;
    while (numOfPages > 0) {
      const tag = `Page ${index++}`;
      const start = hrtime.bigint();
      const query = this.knexPg('artist').select('id', 'name');
      const result = await KnexPagination.cursorPaginate({
        query: query,
        perPage: perPage,
        cursor: {
          key: 'id',
          order: 'asc',
          value: cursor,
          direction: from === 'start' ? 'next' : 'prev',
        },
      });
      if (result.pagination.cursor) {
        cursor = result.pagination.cursor.next_cursor;
      }
      const end = hrtime.bigint();
      const latency = round(Number(end - start) / 1000000);
      avgQueryTimeCursor += latency;
      console.log(`${tag}: ${latency} ms`);
      numOfPages -= 1;
    }
    avgQueryTimeCursor = round(avgQueryTimeCursor / 10);
    console.log(`Average (cursor): ${avgQueryTimeCursor} ms`);
    return avgQueryTimeCursor;
  }

Let us fetch the first 10 pages with 50 records per page and see what results we get:

async getBenchmarks() {
    return {
      'Average Time (Offset)': (await this.getOffsetPagingBenchmarks()) + ' ms',
      'Average Time (Cursor)': (await this.getCursorPagingBenchmarks()) + ' ms',
    };
  }

{
    "Average Time (Offset)": "96.49 ms",
    "Average Time (Cursor)": "0.7 ms"
}

As you can see, offset pagination is almost 137x times slower than cursor pagination even when we are just fetching the first ten pages. The reason is that for each page a count query is executed and its cost is proportional to the number of records. As there are around 1.9 million entries in artist table, so we are seeing a lot of performance differences.

For a fair comparison, we need to fix that performance issue with offset pagination first. We can either store the row count in a separate table and update it whenever a row is added or deleted, or we can cache it somewhere. Some databases maintain row count internally, so refer to their docs for that.

async getBenchmarks() {
    const total = getArtistsCount();
    return {
      'Average Time (Offset)':
        (await this.getOffsetPagingBenchmarks(1, 10, 50, total)) + ' ms',
      'Average Time (Cursor)':
        (await this.getCursorPagingBenchmarks('start', 10, 50)) + ' ms',
    };
}

{
    "Average Time (Offset)": "0.79 ms",
    "Average Time (Cursor)": "0.76 ms"
}

Now, the performance difference is almost negligible after optimisation. For a very small table, you can ignore this optimisation but otherwise, you should always maintain a row count if you ever plan to use offset pagination.

We fetch the last 10 pages with 50 records per page

async getBenchmarks() {
    const total = getArtistsCount();
    return {
      'Average Time (Offset)':
        (await this.getOffsetPagingBenchmarks(38837, 38847, 50, total)) + ' ms',
      'Average Time (Cursor)':
        (await this.getCursorPagingBenchmarks('end', 10, 50)) + ' ms',
    };
}

{
    "Average Time (Offset)": "1734.32 ms",
    "Average Time (Cursor)": "0.78 ms"
}

1. The time taken by cursor pagination remained the same, but the performance of the offset pagination dropped to a point where it became impractical and inefficient to use. As the page number increases, so does the time it takes to fetch that page because the OFFSET clause has to traverse through all the preceding pages to reach the current page. In the above example, when the last page was requested, the offset pagination traversed through all the 1.9 million records to get the last 50 records while the cursor pagination got the last 50 records directly. That’s the main reason why cursor pagination outperformed offset pagination by a huge margin.

Pros and Cons

Offset Pagination

• Pros
  • Easier to implement
  • Can jump to a specific page
  • Can sort data by any column(s)
• Cons
  • Not scalable
  • Not suitable for frequently changing data as page positions may shift when a row is added or deleted from the table

Cursor Pagination

• Pros
  • Scalable to any database size
  • Works very well with frequency changing data
• Cons
  • Can’t jump to a specific page
  • Sorting is very limited as it needs a unique sequential column as a cursor

Conclusion

As a rule – always use cursor pagination. It is scalable and efficient for any database size. But if you must use offset pagination then use it only when the dataset is very small such as search results, blog posts etc otherwise it would lead to degraded performance and slower page loads.

The post The Pagination Dilemma – Offset vs Cursor (Part II) appeared first on Mithle.sh.

Offset

It is the most popular and easy-to-implement paging technique. The offset-pagination is used when we want to give users an option to jump to specific pages.

To implement offset pagination we need to write two queries – one to fetch records and the other to get the total count. We also require two inputs from the user – the number of records per page and the page number.

Suppose, we want to fetch artists from the MusicBrainz database and paginate the data.  To achieve that we would use LIMIT and OFFSET clause as shown in the steps below.

• The first step would be to fetch the specific page using perPage and page parameters provided by the user. For example, with page = 4 and perPage = 10, we need to skip the first 30 records and then fetch the subsequent 10 records to get the fourth page of results.

  
  SELECT * FROM musicbrainz.artist 
  ORDER BY ID asc OFFSET 30 LIMIT 10;
  
  

  const artists = this.knexPg('artist')
    .select('id', 'name')
    .orderBy('id', 'asc')
    .limit(perPage)
    .offset((page - 1) * perPage);

• We then count all of the artists to determine the total number of pages.

  SELECT COUNT(*) FROM musicbrainz.artist

  
  const count = await this.knexPg('artist')
    .count({ total: '*' })
    .first();
  const total = count.total;
  const totalPages = Math.ceil(total / perPage);
  
  

• Here’s the full code for the offset pagination function and a sample response:

  
  async getArtists(perPage, page) {
    const artists = await this.knexPg('artist')
      .select('id', 'name')
      .orderBy('id', 'asc')
      .limit(perPage)
      .offset((page - 1) * perPage);
    const count = await this.knexPg('artist').count({ total: '*'}).first();
    const total = parseInt(count.total);
    const totalPages = Math.ceil(total / perPage);
    return {
      artists: artists,
      pagination: {
        perPage: perPage,
        currentPage: page,
        totalPages: totalPages,
        total: total,
     },
   };
  }

  
  {
    "artists": [...],
    "pagination": {
      "perPage": 10,
      "currentPage": 2,
      "totalPages": 194233,
      "total": 1942322
     }
  }
  
  

Cursor

This paging technique works by making use of WHERE and ORDER BY clauses on a unique, sequential column(s). We fetch data from the table and then return a cursor that the user can send in the next request to fetch the subsequent data.

Unlike offset pagination, we can’t jump to a specific page as there is no such concept in this technique. We can only fetch the first page, the previous page, the next page, and the last page.

In this method, we require the following four inputs from the user:

• no of records per page
• the value of the cursor
• the sort order of the cursor – asc | desc
• direction – next to fetch items after the cursor or prev to fetch items before the cursor

Now let us fetch the artists again from the MusicBrainz database but this time using cursor pagination. To begin with cursor pagination, we need to first choose a cursor – a column that is both sequential and unique. For this example, we would choose ID column as the cursor.

The ID column can sort the data in ascending or descending order and the user can request the next page or the previous page. So there are a total of four scenarios that we need to handle.

• order by is asc and direction is next
• order by is asc and direction is prev
• order by is desc and direction is next
• order by is desc and direction is prev

Let us handle each case one by one:

1. Order – ASC and Direction – NEXT

This is pretty straightforward, for example with cursor = 10, perPage = 10, we would write the following query to fetch the next ten records.

In all the four cases, we set LIMIT = perPage + 1 to check if there is a next/prev page.

SELECT * FROM musicbrainz.artist 
WHERE ID > 10 
ORDER BY ID asc 
LIMIT 11;

If cursor = null, it means the user is requesting the first page, so the query would be:

SELECT * FROM musicbrainz.artist 
ORDER BY ID asc 
LIMIT 11;

• If result.length = perPage or result.length < perPage, it means there is no next page.
• If result.length > perPage, it means there is a next page and there is an extra item in the result.

The ID of the last item from the result will be returned to the user as next_cursor but if there is a next page, we need to first remove that extra last item from the result.

2. Order – ASC and Direction – PREV

You must be thinking that flipping the comparison operator in the previous query is all we need to traverse in the opposite direction. Let us try that to see if it works.

SELECT * FROM musicbrainz.artist 
WHERE ID < 11 
ORDER BY ID asc 
LIMIT 11;

We should the get records from 1-10. It did work but is it right? Now, let us assume the cursor = 21 and then run the query.

SELECT * FROM musicbrainz.artist 
WHERE ID < 21 
ORDER BY ID asc 
LIMIT 11;

The query should have output records 10-20 but instead, we get records 1-11, but why? Because we forgot that the data is in ascending order and begins with ID = 1 and it is also less than 21.

So, to get the correct records, we need to flip the ORDER of the cursor.

SELECT * FROM musicbrainz.artist 
WHERE ID < 21 
ORDER BY ID desc 
LIMIT 11;

results = [
    { id: 20 } ... { id:10 }
]

The data is correct but the order is incorrect. To fix this, just reverse the results array.

• If cursor = null, it means the user is requesting the last page, so the query would be: SELECT * FROM musicbrainz.artist ORDER BY ID desc LIMIT 11;

• If result.length = perPage or result.length < perPage, it means there is no prev page.
• If result.length > perPage, it means there is a prev page and there is an extra item in the result.

The ID of the first item in the result will be returned to the user as the prev_cursor but if there is a prev page, we need to first remove that extra item from the top.

3. Order – DESC and Direction – NEXT

The only difference between this case and the first case is the order. So, we just need to change the comparison operator from > to < and ORDER BY from asc to desc and the rest is exactly the same.

SELECT * FROM musicbrainz.artist 
WHERE ID < 10 
ORDER BY ID desc 
LIMIT 11;

4. Order – DESC and Direction – PREV

Change the comparison operator from < to > and flip the ORDER BY from ‘desc’ to ‘asc’ from the third case and follow the rest.

SELECT * FROM musicbrainz.artist 
WHERE ID > 91 
ORDER BY ID asc 
LIMIT 11;

Here’s the full code for the offset pagination function and a sample response

 public static async cursorPaginate(params: ICursorPaginateParams) {
    const { query, cursor, perPage = 10, dataKey = 'data' } = params;
    const cursorMeta = {
      hasNextPage: false,
      next_cursor: null,
      hasPrevPage: false,
      prev_cursor: null,
    };
    const whereOperator = this.getWhereOperator(cursor.order, cursor.direction);
    // if cursor is null, we need to get the first/last page
    if (cursor.value) {
      query.where(cursor.key, whereOperator, cursor.value);
    }
    // if direction is prev, we need to reverse the order
    const order =
      cursor.direction === 'next'
        ? cursor.order
        : cursor.order === 'asc'
        ? 'desc'
        : 'asc';
    // add +1 to the limit to determine if there is a next/prev page
    const result = await query.orderBy(cursor.key, order).limit(perPage + 1);
    // if direction is prev, we need to reverse the result
    if (order !== cursor.order) {
      result.reverse();
    }
    // if we have more than perPage results, we have a next/prev page
    if (result.length > perPage) {
      if (cursor.direction === 'next') {
        result.pop();
        cursorMeta.hasNextPage = true;
        if (cursor.value) {
          cursorMeta.hasPrevPage = true;
          cursorMeta.prev_cursor = result[0][cursor.key];
        }
        cursorMeta.next_cursor = result[result.length - 1][cursor.key];
      } else {
        result.shift();
        cursorMeta.hasPrevPage = true;
        if (cursor.value) {
          cursorMeta.hasNextPage = true;
          cursorMeta.next_cursor = result[result.length - 1][cursor.key];
        }
        cursorMeta.prev_cursor = result[0][cursor.key];
      }
    } else if (result.length > 0 && result.length <= perPage) {
      if (cursor.direction === 'next') {
        if (cursor.value) {
          cursorMeta.hasPrevPage = true;
          cursorMeta.prev_cursor = result[0][cursor.key];
        }
      } else {
        if (cursor.value) {
          cursorMeta.hasNextPage = true;
          cursorMeta.next_cursor = result[result.length - 1][cursor.key];
        }
      }
    }

    return {
      [dataKey]: result,
      pagination: {
        cursor: cursorMeta,
        per_page: perPage,
      },
    };
  }

  private static getWhereOperator(order, direction) {
    if (order === 'asc') {
      if (direction === 'next') {
        return '>';
      } else {
        return '<';
      }
    } else {
      if (direction === 'prev') {
        return '>';
      } else {
        return '<';
      }
    }
  }

{
    "artists": [...],
    "pagination": {
        "cursor": {
            "hasNextPage": true,
            "next_cursor": 45,
            "hasPrevPage": true,
            "prev_cursor": 41
        },
        "per_page": 10
    }
}

In the next post of this series, we will discuss their pros and cons and benchmark both of these pagination methods to compare their performances using the MusicBrainz database.

The post The Pagination Dilemma – Offset vs Cursor (Part I) appeared first on Mithle.sh.