Changed bigint fields to string + Added null filtering

package.json

@@ -1,6 +1,6 @@
 {
   "name": "kainet-scraper",
-  "version": "1.1.0",
+  "version": "1.1.1",
   "description": "YouTube Music scraper for Kainet Music",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

src/musiclists.ts

@@ -1,5 +1,6 @@
 import request from "./utils/request";
 import * as parse from "./utils/parse";
+import * as filter from "./utils/filter";
 import {
     YtMusicAlbum,
     YtMusicPlaylist,
@@ -76,7 +77,7 @@ const scrape = {
         title: parse.text.header(data?.header, 0)!,
         thumbnails: parse.thumbnails(data?.header?.musicDetailHeaderRenderer?.thumbnail?.croppedSquareThumbnailRenderer?.thumbnail),
         songCount: parse.num.simple(parse.text.header(data?.header, 0, "secondSubtitle")),
-        songs: parsePlaylist.songs(data)
+        songs: parsePlaylist.songs(data).filter(filter.songs)
     }),
     playlistSong: (song?: MusicResponsiveListItemRenderer): YtMusicSong & YtMusicVideo => ({
         id: song?.flexColumns?.[0]?.musicResponsiveListItemFlexColumnRenderer?.text?.runs?.[0]?.navigationEndpoint?.watchEndpoint?.videoId!,
@@ -96,7 +97,7 @@ const scrape = {
             thumbnails: parse.thumbnails(info.thumbnailDetails),
             artist: info.artistDisplayName,
             year: info.releaseDate?.year?.toString(),
-            songs: parseAlbum.songs(data)
+            songs: parseAlbum.songs(data).filter(filter.songs)
         };
     },
     albumAsPlaylist: (data: PlaylistData | undefined, browseId: string): YtMusicAlbum => ({
@@ -106,7 +107,7 @@ const scrape = {
         thumbnails: parse.thumbnails(data?.header?.musicDetailHeaderRenderer?.thumbnail?.croppedSquareThumbnailRenderer?.thumbnail),
         artist: parse.text.header(data?.header, 2, "subtitle"),
         year: parse.text.header(data?.header, -1, "subtitle"),
-        songs: parsePlaylist.songs(data)
+        songs: parsePlaylist.songs(data).filter(filter.songs)
     }),
     albumSong: (song?: MusicTrack): YtMusicSong => ({
         id: song?.videoId!,
@@ -127,6 +128,8 @@ export const getPlaylist = (browseId: string): Promise<YtMusicPlaylist | null> =
     request("browse").with({ browseId })
         .then(res =>
             scrape.playlist(res.data, browseId)
+        ).then(list =>
+            filter.playlists(list) ? list : null
         ).catch(
             () => null
         )
@@ -141,6 +144,8 @@ export const getAlbum = (browseId: string): Promise<YtMusicAlbum | null> => (
     request("browse").with({ browseId })
         .then(res =>
             scrape.album(res.data, browseId) ?? scrape.albumAsPlaylist(res.data, browseId)
+        ).then(list =>
+            filter.albums(list) ? list : null
         ).catch(
             () => null
         )

src/search.ts

@@ -1,5 +1,6 @@
 import request from "./utils/request";
 import * as parse from "./utils/parse";
+import * as filter from "./utils/filter";
 import {
     YtMusicSong,
     YtMusicVideo,
@@ -109,7 +110,7 @@ const scrape: Record<
         duration: parse.duration.fromText(parse.text.columns(data?.flexColumns, 1, -1)),
         durationText: parse.text.columns(data?.flexColumns, 1, -1),
         thumbnails: parse.thumbnails(data?.thumbnail?.musicThumbnailRenderer?.thumbnail),
-        views: parse.num.big(parse.text.columns(data?.flexColumns, 1, -3))
+        views: parse.num.big(parse.text.columns(data?.flexColumns, 1, -3))?.toString()
     }),
     albums: (data?: SearchAlbumData): YtMusicAlbum => ({
         id: parseId.albumOrPlaylist(data)!,
@@ -130,7 +131,7 @@ const scrape: Record<
         id: parse.id.browse(data)!,
         name: parse.text.columns(data?.flexColumns, 0, 0)!,
         thumbnails: parse.thumbnails(data?.thumbnail?.musicThumbnailRenderer?.thumbnail),
-        subCount: parse.num.big(parse.text.columns(data?.flexColumns, 1, 2))
+        subCount: parse.num.big(parse.text.columns(data?.flexColumns, 1, 2))?.toString()
     })
 };
 
@@ -167,15 +168,19 @@ export type SearchTypes = typeof SearchType[keyof typeof SearchType];
  * Retrieves a list of search results based on the queried type and the query text
  * @param type - The type of item to search, as listed in {@link SearchType}
  * @param query - The text to query for
- * @returns An array of of elements of the specified type, or null if something went wrong
+ * @returns An array of elements of the specified type
  */
-export const search = <T extends SearchTypes>(type: T, query: string) => (
+export const search = <T extends SearchTypes>(type: T, query: string): Promise<SearchModels[T][]> => (
     request("search").with({ params: searchParams[type], query })
         .then(res =>
-            parseSearch<SearchResults[T]>(res.data)?.map(
-                el => scrape[type](el) as SearchModels[T]
-            ) ?? null
+            parseSearch<SearchResults[T]>(
+                res.data
+            )?.map(result =>
+                scrape[type](result) as SearchModels[T]
+            )?.filter(item =>
+                (filter[type] as (item: SearchModels[T]) => boolean)(item)
+            ) ?? []
         ).catch(
-            () => null
+            () => []
         )
 );

src/suggestions.ts

@@ -1,5 +1,6 @@
 import request from "./utils/request";
 import * as parse from "./utils/parse";
+import * as filter from "./utils/filter";
 import { YtMusicPlaylist } from "./utils/interfaces";
 import { MusicTwoRowItemRenderer } from "./utils/types";
 
@@ -25,7 +26,7 @@ type SuggestionsData = {
     }
 };
 
-const parseSuggestions = (data: SuggestionsData) => (
+const parseSuggestions = (data?: SuggestionsData) => (
     data?.contents?.singleColumnBrowseResultsRenderer?.tabs?.[0]?.tabRenderer?.content?.sectionListRenderer?.contents
      ?.map(el => el?.musicCarouselShelfRenderer?.contents ?? el?.musicImmersiveCarouselShelfRenderer?.contents)
 );
@@ -39,17 +40,17 @@ const scrapePlaylist = (item?: { musicTwoRowItemRenderer?: MusicTwoRowItemRender
 
 /**
  * Retrieves a list of suggested playlist, as on the YTMusic homepage
- * @returns An array of playlists, or null if something went wrong
+ * @returns An array of playlists
  */
-export const retrieveSuggestions = (): Promise<YtMusicPlaylist[] | null> => (
+export const retrieveSuggestions = (): Promise<YtMusicPlaylist[]> => (
     request("browse").with()
         .then(res =>
             parseSuggestions(res.data)?.flatMap(row => (
                 row?.map(scrapePlaylist)
-            ))?.filter(
-                (el): el is YtMusicPlaylist => !!el
-            ) ?? null
+            ))?.filter((list): list is YtMusicPlaylist =>
+                !!list && filter.playlists(list)
+            ) ?? []
         ).catch(
-            () => null
+            () => []
         )
 );

src/utils/filter.ts

@@ -0,0 +1,13 @@
+import {
+    YtMusicSong,
+    YtMusicVideo,
+    YtMusicAlbum,
+    YtMusicPlaylist,
+    YtMusicArtist
+} from "./interfaces";
+
+export const songs = (item: YtMusicSong) => !!item.id && !!item.title;
+export const videos = (item: YtMusicVideo) => !!item.id && !!item.title;
+export const albums = (item: YtMusicAlbum) => !!item.id && !!item.browseId && !!item.title;
+export const playlists = (item: YtMusicPlaylist) => !!item.id && !!item.browseId && !!item.title;
+export const artists = (item: YtMusicArtist) => !!item.id && !!item.name;

src/utils/interfaces.ts

@@ -15,7 +15,7 @@ export interface YtMusicVideo {
     duration?: number,
     durationText?: string,
     thumbnails: string[],
-    views?: bigint
+    views?: string // bigint is not serializable
 }
 
 export interface YtMusicAlbum {
@@ -41,5 +41,5 @@ export interface YtMusicArtist {
     id: string,
     name: string,
     thumbnails: string[],
-    subCount?: bigint
+    subCount?: string // bigint is not serializable
 }

tests/album.test.ts

@@ -8,8 +8,11 @@ const queries = [
     "MPREb_b3D2y9bdjNa"
 ];
 
-it("returns album item with all data correctly", () => (
-    expectFromQueries(queries, getAlbum)
+it("returns album item with a songs array correctly", () => (
+    expectFromQueries(queries, getAlbum, album => {
+        expect(album).not.toBeNull();
+        expect(album?.songs).not.toStrictEqual([]);
+    })
 ));
 
 it("returns null on no album result", () => (

tests/playlist.test.ts

@@ -8,8 +8,11 @@ const queries = [
     "VLPL5eNRayL5QIRiH9BR6dku6886u_LJmpth"
 ];
 
-it("returns playlist item with all data correctly", () => (
-    expectFromQueries(queries, getPlaylist)
+it("returns playlist item with a songs array correctly", () => (
+    expectFromQueries(queries, getPlaylist, playlist => {
+        expect(playlist).not.toBeNull();
+        expect(playlist?.songs).not.toStrictEqual([]);
+    })
 ));
 
 it("returns null on no playlist result", () => (

tests/search.test.ts

@@ -9,59 +9,61 @@ const queries = [
 ];
 
 const expectFromType = (type: typeof SearchType[keyof typeof SearchType]): Promise<void> => (
-    expectFromQueries(queries, text => search(type, text))
+    expectFromQueries(queries, text => search(type, text), result => (
+        expect(result).not.toStrictEqual([])
+    ))
 );
 
 const expectWrongFromType = (type: typeof SearchType[keyof typeof SearchType]): Promise<void> => (
-    expectFromWrong(text => search(type, text))
+    expectFromWrong(text => search(type, text), [])
 );
 
 describe("songs", () => {
-    it("returns youtube songs with all data correctly", () => (
+    it("returns youtube songs correctly", () => (
         expectFromType(SearchType.SONGS)
     ));
 
-    it("returns null on no songs results", () => (
+    it("returns empty array on no songs results", () => (
         expectWrongFromType(SearchType.SONGS)
     ));
 });
 
 describe("videos", () => {
-    it("returns youtube videos with all data correctly", () => (
+    it("returns youtube videos correctly", () => (
         expectFromType(SearchType.VIDEOS)
     ));
 
-    it("returns null on no videos results", () => (
+    it("returns empty array on no videos results", () => (
         expectWrongFromType(SearchType.VIDEOS)
     ));
 });
 
 describe("albums", () => {
-    it("returns youtube albums with all data correctly", () => (
+    it("returns youtube albums correctly", () => (
         expectFromType(SearchType.ALBUMS)
     ));
 
-    it("returns null on no albums results", () => (
+    it("returns empty array on no albums results", () => (
         expectWrongFromType(SearchType.ALBUMS)
     ));
 });
 
 describe("playlists", () => {
-    it("returns youtube playlists with all data correctly", () => (
+    it("returns youtube playlists correctly", () => (
         expectFromType(SearchType.PLAYLISTS)
     ));
 
-    it("returns null on no playlists results", () => (
+    it("returns empty array on no playlists results", () => (
         expectWrongFromType(SearchType.PLAYLISTS)
     ));
 });
 
 describe("artists", () => {
-    it("returns youtube artists with all data correctly", () => (
+    it("returns youtube artists correctly", () => (
         expectFromType(SearchType.ARTISTS)
     ));
 
-    it("returns null on no artists results", () => (
+    it("returns empty array on no artists results", () => (
         expectWrongFromType(SearchType.ARTISTS)
     ));
 });

tests/suggestions.test.ts

@@ -1,6 +1,6 @@
 import { retrieveSuggestions } from "../src";
 
-it("returns suggested playlists with all data correctly", () => (
+it("returns suggested playlists correctly", () => (
     retrieveSuggestions()
-        .then(suggestions => expect(suggestions).not.toBeNull())
+        .then(suggestions => expect(suggestions).not.toStrictEqual([]))
 ));

tests/testUtils.ts

@@ -1,18 +1,18 @@
-export const expectFromQueries = (
+export const expectFromQueries = <T>(
     queries: string[],
-    callback: (text: string) => Promise<any>
+    runCb: (text: string) => Promise<T>,
+    expectCb: (result: T) => any
 ): Promise<void> => (
-    Promise.all(queries.map(callback))
+    Promise.all(queries.map(runCb))
         .then(results =>
-            results.forEach(result =>
-                expect(result).not.toBeNull()
-            )
+            results.forEach(expectCb)
         )
 );
 
 export const expectFromWrong = async (
-    callback: (text: string) => Promise<any>
+    callback: (text: string) => Promise<any>,
+    errValue: any = null
 ): Promise<void> => (
     callback("ad463b3c3298f77dd6d21c95020feb45baf98d7a")
-        .then(result => expect(result).toBeNull())
+        .then(result => expect(result).toStrictEqual(errValue))
 );