Huge Update

play-dl/YouTube/utils/cipher.ts

@@ -0,0 +1,175 @@
+import { URL } from 'node:url'
+import { url_get } from './request'
+import querystring from 'node:querystring'
+
+interface formatOptions {
+  url? : string;
+  sp? : string;
+  signatureCipher? : string;
+  cipher?: string;
+  s? : string;
+}
+const var_js = '[a-zA-Z_\\$][a-zA-Z_0-9]*';
+const singlequote_js = `'[^'\\\\]*(:?\\\\[\\s\\S][^'\\\\]*)*'`;
+const duoblequote_js = `"[^"\\\\]*(:?\\\\[\\s\\S][^"\\\\]*)*"`;
+const quote_js = `(?:${singlequote_js}|${duoblequote_js})`;
+const key_js = `(?:${var_js}|${quote_js})`;
+const prop_js = `(?:\\.${var_js}|\\[${quote_js}\\])`;
+const empty_js = `(?:''|"")`;
+const reverse_function = ':function\\(a\\)\\{' +
+'(?:return )?a\\.reverse\\(\\)' +
+'\\}';
+const slice_function = ':function\\(a,b\\)\\{' +
+'return a\\.slice\\(b\\)' +
+'\\}';
+const splice_function = ':function\\(a,b\\)\\{' +
+'a\\.splice\\(0,b\\)' +
+'\\}';
+const swap_function = ':function\\(a,b\\)\\{' +
+'var c=a\\[0\\];a\\[0\\]=a\\[b(?:%a\\.length)?\\];a\\[b(?:%a\\.length)?\\]=c(?:;return a)?' +
+'\\}';
+const obj_regexp = new RegExp(
+  `var (${var_js})=\\{((?:(?:${
+    key_js}${reverse_function}|${
+    key_js}${slice_function}|${
+    key_js}${splice_function}|${
+    key_js}${swap_function
+  }),?\\r?\\n?)+)\\};`)
+const function_regexp = new RegExp(`${`function(?: ${var_js})?\\(a\\)\\{` +
+`a=a\\.split\\(${empty_js}\\);\\s*` +
+`((?:(?:a=)?${var_js}`}${
+prop_js
+}\\(a,\\d+\\);)+)` +
+`return a\\.join\\(${empty_js}\\)` +
+`\\}`);
+const reverse_regexp = new RegExp(`(?:^|,)(${key_js})${reverse_function}`, 'm');
+const slice_regexp = new RegExp(`(?:^|,)(${key_js})${slice_function}`, 'm');
+const splice_regexp = new RegExp(`(?:^|,)(${key_js})${splice_function}`, 'm');
+const swap_regexp = new RegExp(`(?:^|,)(${key_js})${swap_function}`, 'm');
+
+export function js_tokens( body:string ) {
+  let function_action = function_regexp.exec(body)
+  let object_action = obj_regexp.exec(body)
+  if(!function_action || !object_action) return null
+
+  let object = object_action[1].replace(/\$/g, '\\$')
+  let object_body = object_action[2].replace(/\$/g, '\\$')
+  let function_body = function_action[1].replace(/\$/g, '\\$')
+
+  let result = reverse_regexp.exec(object_body);
+  const reverseKey = result && result[1]
+  .replace(/\$/g, '\\$')
+  .replace(/\$|^'|^"|'$|"$/g, '');
+  
+  result = slice_regexp.exec(object_body)
+  const sliceKey = result && result[1]
+  .replace(/\$/g, '\\$')
+  .replace(/\$|^'|^"|'$|"$/g, '');
+
+  result = splice_regexp.exec(object_body);
+  const spliceKey = result && result[1]
+  .replace(/\$/g, '\\$')
+  .replace(/\$|^'|^"|'$|"$/g, '');
+
+  result = swap_regexp.exec(object_body);
+  const swapKey = result && result[1]
+  .replace(/\$/g, '\\$')
+  .replace(/\$|^'|^"|'$|"$/g, '');
+
+  const keys = `(${[reverseKey, sliceKey, spliceKey, swapKey].join('|')})`;
+  const myreg = `(?:a=)?${object
+  }(?:\\.${keys}|\\['${keys}'\\]|\\["${keys}"\\])` +
+    `\\(a,(\\d+)\\)`;
+  const tokenizeRegexp = new RegExp(myreg, 'g');
+  const tokens = [];
+  while((result = tokenizeRegexp.exec(function_body)) !== null){
+    let key = result[1] || result[2] || result[3];
+    switch (key) {
+      case swapKey:
+        tokens.push(`sw${result[4]}`);
+        break;
+      case reverseKey:
+        tokens.push('rv');
+        break;
+      case sliceKey:
+        tokens.push(`sl${result[4]}`);
+        break;
+      case spliceKey:
+        tokens.push(`sp${result[4]}`);
+        break;
+    }
+  }
+  return tokens
+}
+
+function deciper_signature(tokens : string[], signature :string){
+  let sig = signature.split('')
+  let len = tokens.length
+  for(let i = 0; i < len; i++ ){
+    let token = tokens[i], pos;
+    switch(token.slice(0,2)){
+      case 'sw':
+        pos = parseInt(token.slice(2))
+        sig = swappositions(sig, pos)
+        break
+      case 'rv':
+        sig = sig.reverse()
+        break
+      case 'sl':
+        pos = parseInt(token.slice(2))
+        sig = sig.slice(pos)
+        break
+      case 'sp':
+        pos = parseInt(token.slice(2))
+        sig.splice(0, pos)
+        break
+    }
+  }
+  return sig.join('')
+}
+
+
+function swappositions(array : string[], position : number){
+  let first = array[0]
+  let pos_args = array[position]
+  array[0] = array[position]
+  array[position] = first
+  return array
+}
+
+function download_url(format: formatOptions, sig : string){
+  let decoded_url;
+  if(!format.url) return;
+  decoded_url = format.url
+
+  decoded_url = decodeURIComponent(decoded_url)
+
+  let parsed_url = new URL(decoded_url)
+  parsed_url.searchParams.set('ratebypass', 'yes');
+
+  if(sig){
+    parsed_url.searchParams.set(format.sp || 'signature', sig)
+  }
+  format.url = parsed_url.toString();
+}
+
+export async function format_decipher(formats: formatOptions[], html5player : string){
+  let body = await url_get(html5player)
+  let tokens = js_tokens(body)
+  formats.forEach((format) => {
+    let cipher = format.signatureCipher || format.cipher;
+    if(cipher){
+      Object.assign(format, querystring.parse(cipher))
+      delete format.signatureCipher;
+      delete format.cipher;
+    }
+    let sig;
+    if(tokens && format.s){
+      sig = deciper_signature(tokens, format.s)
+      download_url(format, sig)
+      delete format.s
+      delete format.sp
+    }
+  });
+  return formats
+}

play-dl/YouTube/utils/extractor.ts

@@ -0,0 +1,174 @@
+import { url_get } from './request'
+import { format_decipher, js_tokens } from './cipher'
+import { Video } from '../classes/Video'
+import { RequestInit } from 'node-fetch'
+import { PlayList } from '../classes/Playlist'
+
+const DEFAULT_API_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8";
+const youtube_url = /https:\/\/www.youtube.com\//g
+const video_pattern = /^((?:https?:)?\/\/)?((?:www|m)\.)?((?:youtube\.com|youtu.be))(\/(?:[\w\-]+\?v=|embed\/|v\/)?)([\w\-]+)(\S+)?$/;
+
+export async function video_basic_info(url : string){
+        if(!url.match(youtube_url) || !url.match(video_pattern)) throw new Error('This is not a YouTube URL')
+        let video_id = url.split('watch?v=')[1].split('&')[0]
+        let new_url = 'https://www.youtube.com/watch?v=' + video_id
+        let body = await url_get(new_url)
+        let player_response = JSON.parse(body.split("var ytInitialPlayerResponse = ")[1].split(";</script>")[0])
+        if(player_response.playabilityStatus.status === 'ERROR') throw new Error(`While getting info from url \n  ${player_response.playabilityStatus.reason}`)
+        let html5player =  'https://www.youtube.com' + body.split('"jsUrl":"')[1].split('"')[0]
+        let format = []
+        format.push(player_response.streamingData.formats[0])
+        format.push(...player_response.streamingData.adaptiveFormats)
+        let vid = player_response.videoDetails
+        let microformat = player_response.microformat.playerMicroformatRenderer
+        let video_details = {
+            id : vid.videoId,
+            url : 'https://www.youtube.com/watch?v=' + vid.videoId,
+            title : vid.title,
+            description : vid.shortDescription,
+            duration : vid.lengthSeconds,
+            uploadedDate : microformat.publishDate,
+            thumbnail : {
+                width : vid.thumbnail.thumbnails[vid.thumbnail.thumbnails.length - 1].width,
+                height : vid.thumbnail.thumbnails[vid.thumbnail.thumbnails.length - 1].height,
+                url : `https://i.ytimg.com/vi/${vid.videoId}/maxresdefault.jpg`
+            },
+            channel : {
+                name : vid.author,
+                id : vid.channelId,
+                url : `https://www.youtube.com/channel/${vid.channelId}`
+            },
+            views : vid.viewCount,
+            tags : vid.keywords,
+            averageRating : vid.averageRating,
+            live : vid.isLiveContent,
+            private : vid.isPrivate
+        }
+        return {
+            html5player,
+            format,
+            video_details
+        }
+}
+
+export async function video_info(url : string) {
+    let data = await video_basic_info(url)
+    if(data.format[0].signatureCipher || data.format[0].cipher){
+        data.format = await format_decipher(data.format, data.html5player)
+        return data
+    }
+    else {
+        return data
+    }
+}
+
+export async function playlist_info(url : string) {
+    if (!url || typeof url !== "string") throw new Error(`Expected playlist url, received ${typeof url}!`);
+    if(url.search('(\\?|\\&)list\\=') === -1) throw new Error('This is not a PlayList URL')
+
+    let Playlist_id = url.split('list=')[1].split('&')[0]
+    let new_url = `https://www.youtube.com/playlist?list=${Playlist_id}`
+    
+    let body = await url_get(new_url)
+    let response = JSON.parse(body.split("var ytInitialData = ")[1].split(";</script>")[0])
+    if(response.alerts && response.alerts[0].alertRenderer.type === 'ERROR') throw new Error(`While parsing playlist url\n   ${response.alerts[0].alertRenderer.text.runs[0].text}`)
+
+    let rawJSON = `${body.split('{"playlistVideoListRenderer":{"contents":')[1].split('}],"playlistId"')[0]}}]`;
+    let parsed = JSON.parse(rawJSON);
+    let playlistDetails = JSON.parse(body.split('{"playlistSidebarRenderer":')[1].split("}};</script>")[0]).items;
+
+    let API_KEY = body.split('INNERTUBE_API_KEY":"')[1]?.split('"')[0] ?? body.split('innertubeApiKey":"')[1]?.split('"')[0] ?? DEFAULT_API_KEY;
+    let videos = getPlaylistVideos(parsed, 100);
+
+    let data = playlistDetails[0].playlistSidebarPrimaryInfoRenderer;
+    if (!data.title.runs || !data.title.runs.length) return undefined;
+
+    let author = playlistDetails[1]?.playlistSidebarSecondaryInfoRenderer.videoOwner;
+    let views = data.stats.length === 3 ? data.stats[1].simpleText.replace(/[^0-9]/g, "") : 0;
+    let lastUpdate = data.stats.find((x: any) => "runs" in x && x["runs"].find((y: any) => y.text.toLowerCase().includes("last update")))?.runs.pop()?.text ?? null;
+    let videosCount = data.stats[0].runs[0].text.replace(/[^0-9]/g, "") || 0;
+
+    let res = new PlayList({
+        continuation: {
+            api: API_KEY,
+            token: getContinuationToken(parsed),
+            clientVersion: body.split('"INNERTUBE_CONTEXT_CLIENT_VERSION":"')[1]?.split('"')[0] ?? body.split('"innertube_context_client_version":"')[1]?.split('"')[0] ?? "<some version>"
+        },
+        id: data.title.runs[0].navigationEndpoint.watchEndpoint.playlistId,
+        title: data.title.runs[0].text,
+        videoCount: parseInt(videosCount) || 0,
+        lastUpdate: lastUpdate,
+        views: parseInt(views) || 0,
+        videos: videos,
+        url: `https://www.youtube.com/playlist?list=${data.title.runs[0].navigationEndpoint.watchEndpoint.playlistId}`,
+        link: `https://www.youtube.com${data.title.runs[0].navigationEndpoint.commandMetadata.webCommandMetadata.url}`,
+        author: author
+            ? {
+                  name: author.videoOwnerRenderer.title.runs[0].text,
+                  id: author.videoOwnerRenderer.title.runs[0].navigationEndpoint.browseEndpoint.browseId,
+                  url: `https://www.youtube.com${author.videoOwnerRenderer.navigationEndpoint.commandMetadata.webCommandMetadata.url || author.videoOwnerRenderer.navigationEndpoint.browseEndpoint.canonicalBaseUrl}`,
+                  icon: author.videoOwnerRenderer.thumbnail.thumbnails.length ? author.videoOwnerRenderer.thumbnail.thumbnails[author.videoOwnerRenderer.thumbnail.thumbnails.length - 1].url : null
+              }
+            : {},
+        thumbnail: data.thumbnailRenderer.playlistVideoThumbnailRenderer?.thumbnail.thumbnails.length ? data.thumbnailRenderer.playlistVideoThumbnailRenderer.thumbnail.thumbnails[data.thumbnailRenderer.playlistVideoThumbnailRenderer.thumbnail.thumbnails.length - 1].url : null
+    });
+    return res;
+}
+
+export function getPlaylistVideos(data:any, limit : number = Infinity) : Video[] {
+    const videos = [];
+
+    for (let i = 0; i < data.length; i++) {
+        if (limit === videos.length) break;
+        const info = data[i].playlistVideoRenderer;
+        if (!info || !info.shortBylineText) continue;
+
+        videos.push(
+            new Video({
+                id: info.videoId,
+                index: parseInt(info.index?.simpleText) || 0,
+                duration: parseDuration(info.lengthText?.simpleText) || 0,
+                duration_raw: info.lengthText?.simpleText ?? "0:00",
+                thumbnail: {
+                    id: info.videoId,
+                    url: info.thumbnail.thumbnails[info.thumbnail.thumbnails.length - 1].url,
+                    height: info.thumbnail.thumbnails[info.thumbnail.thumbnails.length - 1].height,
+                    width: info.thumbnail.thumbnails[info.thumbnail.thumbnails.length - 1].width
+                },
+                title: info.title.runs[0].text,
+                channel: {
+                    id: info.shortBylineText.runs[0].navigationEndpoint.browseEndpoint.browseId || undefined,
+                    name: info.shortBylineText.runs[0].text || undefined,
+                    url: `https://www.youtube.com${info.shortBylineText.runs[0].navigationEndpoint.browseEndpoint.canonicalBaseUrl || info.shortBylineText.runs[0].navigationEndpoint.commandMetadata.webCommandMetadata.url}`,
+                    icon: undefined
+                }
+            })
+        );
+    }
+    return videos   
+}
+
+function parseDuration(duration: string): number {
+    duration ??= "0:00";
+    const args = duration.split(":");
+    let dur = 0;
+
+    switch (args.length) {
+        case 3:
+            dur = parseInt(args[0]) * 60 * 60 + parseInt(args[1]) * 60 + parseInt(args[2]);
+            break;
+        case 2:
+            dur = parseInt(args[0]) * 60 + parseInt(args[1]);
+            break;
+        default:
+            dur = parseInt(args[0]);
+    }
+
+    return dur;
+}
+
+
+export function getContinuationToken(data:any): string {
+    const continuationToken = data.find((x: any) => Object.keys(x)[0] === "continuationItemRenderer")?.continuationItemRenderer.continuationEndpoint?.continuationCommand?.token;
+    return continuationToken;
+}

play-dl/YouTube/utils/index.ts

@@ -0,0 +1 @@
+export { video_basic_info, video_info, playlist_info } from './extractor'

play-dl/YouTube/utils/parser.ts

@@ -0,0 +1,175 @@
+import { Video } from "../classes/Video";
+import { PlayList } from "../classes/Playlist";
+import { Channel } from "../classes/Channel";
+
+export interface ParseSearchInterface {
+    type?: "video" | "playlist" | "channel" | "all";
+    limit?: number;
+}
+
+export function ParseSearchResult(html :string, options? : ParseSearchInterface): (Video | PlayList | Channel)[] {
+    if(!html) throw new Error('Can\'t parse Search result without data')
+    if (!options) options = { type: "video", limit: 0 };
+    if (!options.type) options.type = "video";
+
+    let results = []
+    let details = []
+    let fetched = false;
+
+    try {
+        let data = html.split("ytInitialData = JSON.parse('")[1].split("');</script>")[0];
+        html = data.replace(/\\x([0-9A-F]{2})/gi, (...items) => {
+            return String.fromCharCode(parseInt(items[1], 16));
+        });
+    } catch {
+        /* do nothing */
+    }
+
+    try {
+        details = JSON.parse(html.split('{"itemSectionRenderer":{"contents":')[html.split('{"itemSectionRenderer":{"contents":').length - 1].split(',"continuations":[{')[0]);
+        fetched = true;
+    } catch {
+        /* Do nothing*/
+    }
+
+    if (!fetched) {
+        try {
+            details = JSON.parse(html.split('{"itemSectionRenderer":')[html.split('{"itemSectionRenderer":').length - 1].split('},{"continuationItemRenderer":{')[0]).contents;
+            fetched = true;
+        } catch {
+            /* do nothing */
+        }
+    }
+
+    if (!fetched) throw new Error('Failed to Fetch the data')
+
+    for (let i = 0; i < details.length; i++) {
+        if (typeof options.limit === "number" && options.limit > 0 && results.length >= options.limit) break;
+        let data = details[i];
+        let res;
+        if (options.type === "all") {
+            if (!!data.videoRenderer) options.type = "video";
+            else if (!!data.channelRenderer) options.type = "channel";
+            else if (!!data.playlistRenderer) options.type = "playlist";
+            else continue;
+        }
+
+        if (options.type === "video") {
+            const parsed = parseVideo(data);
+            if (!parsed) continue;
+            res = parsed;
+        } else if (options.type === "channel") {
+            const parsed = parseChannel(data);
+            if (!parsed) continue;
+            res = parsed;
+        } else if (options.type === "playlist") {
+            const parsed = parsePlaylist(data);
+            if (!parsed) continue;
+            res = parsed;
+        }
+
+        results.push(res);
+    }
+
+return results as (Video | Channel | PlayList)[];
+}
+
+function parseDuration(duration: string): number {
+    duration ??= "0:00";
+    const args = duration.split(":");
+    let dur = 0;
+
+    switch (args.length) {
+        case 3:
+            dur = parseInt(args[0]) * 60 * 60 + parseInt(args[1]) * 60 + parseInt(args[2]);
+            break;
+        case 2:
+            dur = parseInt(args[0]) * 60 + parseInt(args[1]);
+            break;
+        default:
+            dur = parseInt(args[0]);
+    }
+
+    return dur;
+}
+
+export function parseChannel(data?: any): Channel | void {
+    if (!data || !data.channelRenderer) return;
+    const badge = data.channelRenderer.ownerBadges && data.channelRenderer.ownerBadges[0];
+    let url = `https://www.youtube.com${data.channelRenderer.navigationEndpoint.browseEndpoint.canonicalBaseUrl || data.channelRenderer.navigationEndpoint.commandMetadata.webCommandMetadata.url}`;
+    let res = new Channel({
+        id: data.channelRenderer.channelId,
+        name: data.channelRenderer.title.simpleText,
+        icon: {
+            url : data.channelRenderer.thumbnail.thumbnails[data.channelRenderer.thumbnail.thumbnails.length - 1].url.replace('//', 'https://'),
+            width : data.channelRenderer.thumbnail.thumbnails[data.channelRenderer.thumbnail.thumbnails.length - 1].width,
+            height: data.channelRenderer.thumbnail.thumbnails[data.channelRenderer.thumbnail.thumbnails.length - 1].height
+        },
+        url: url,
+        verified: Boolean(badge?.metadataBadgeRenderer?.style?.toLowerCase().includes("verified")),
+        subscribers: (data.channelRenderer.subscriberCountText?.simpleText) ? data.channelRenderer.subscriberCountText.simpleText : '0 subscribers'
+    });
+
+    return res;
+}
+
+export function parseVideo(data?: any): Video | void {
+    if (!data || !data.videoRenderer) return;
+
+    const badge = data.videoRenderer.ownerBadges && data.videoRenderer.ownerBadges[0];
+    let res = new Video({
+        id: data.videoRenderer.videoId,
+        url: `https://www.youtube.com/watch?v=${data.videoRenderer.videoId}`,
+        title: data.videoRenderer.title.runs[0].text,
+        description: data.videoRenderer.descriptionSnippet && data.videoRenderer.descriptionSnippet.runs[0] ? data.videoRenderer.descriptionSnippet.runs[0].text : "",
+        duration: data.videoRenderer.lengthText ? parseDuration(data.videoRenderer.lengthText.simpleText) : 0,
+        duration_raw: data.videoRenderer.lengthText ? data.videoRenderer.lengthText.simpleText : null,
+        thumbnail: {
+            id: data.videoRenderer.videoId,
+            url: data.videoRenderer.thumbnail.thumbnails[data.videoRenderer.thumbnail.thumbnails.length - 1].url,
+            height: data.videoRenderer.thumbnail.thumbnails[data.videoRenderer.thumbnail.thumbnails.length - 1].height,
+            width: data.videoRenderer.thumbnail.thumbnails[data.videoRenderer.thumbnail.thumbnails.length - 1].width
+        },
+        channel: {
+            id: data.videoRenderer.ownerText.runs[0].navigationEndpoint.browseEndpoint.browseId || null,
+            name: data.videoRenderer.ownerText.runs[0].text || null,
+            url: `https://www.youtube.com${data.videoRenderer.ownerText.runs[0].navigationEndpoint.browseEndpoint.canonicalBaseUrl || data.videoRenderer.ownerText.runs[0].navigationEndpoint.commandMetadata.webCommandMetadata.url}`,
+            icon: {
+                url: data.videoRenderer.channelThumbnailSupportedRenderers.channelThumbnailWithLinkRenderer.thumbnail.thumbnails[0].url,
+                width: data.videoRenderer.channelThumbnailSupportedRenderers.channelThumbnailWithLinkRenderer.thumbnail.thumbnails[0].width,
+                height: data.videoRenderer.channelThumbnailSupportedRenderers.channelThumbnailWithLinkRenderer.thumbnail.thumbnails[0].height
+            },
+            verified: Boolean(badge?.metadataBadgeRenderer?.style?.toLowerCase().includes("verified"))
+        },
+        uploadedAt: data.videoRenderer.publishedTimeText?.simpleText ?? null,
+        views: data.videoRenderer.viewCountText?.simpleText?.replace(/[^0-9]/g, "") ?? 0
+    });
+
+    return res;
+}
+
+export function parsePlaylist(data?: any): PlayList | void {
+    if (!data.playlistRenderer) return;
+
+    const res = new PlayList(
+        {
+            id: data.playlistRenderer.playlistId,
+            title: data.playlistRenderer.title.simpleText,
+            thumbnail: {
+                id: data.playlistRenderer.playlistId,
+                url: data.playlistRenderer.thumbnails[0].thumbnails[data.playlistRenderer.thumbnails[0].thumbnails.length - 1].url,
+                height: data.playlistRenderer.thumbnails[0].thumbnails[data.playlistRenderer.thumbnails[0].thumbnails.length - 1].height,
+                width: data.playlistRenderer.thumbnails[0].thumbnails[data.playlistRenderer.thumbnails[0].thumbnails.length - 1].width
+            },
+            channel: {
+                id: data.playlistRenderer.shortBylineText.runs[0].navigationEndpoint.browseEndpoint.browseId,
+                name: data.playlistRenderer.shortBylineText.runs[0].text,
+                url: `https://www.youtube.com${data.playlistRenderer.shortBylineText.runs[0].navigationEndpoint.commandMetadata.webCommandMetadata.url}`
+            },
+            videos: parseInt(data.playlistRenderer.videoCount.replace(/[^0-9]/g, ""))
+        },
+        true
+    );
+
+    return res;
+}

play-dl/YouTube/utils/request.ts

@@ -0,0 +1,12 @@
+import fetch, { RequestInit } from 'node-fetch'
+
+export async function url_get (url : string, options? : RequestInit) : Promise<string>{
+    return new Promise(async(resolve, reject) => {
+        let response = await fetch(url, options)
+
+        if(response.status === 200) {
+            resolve(await response.text())
+        }
+        else reject(`Got ${response.status} from ${url}`)
+    })
+}