Added User agents support

play-dl/Request/classes.ts

@@ -1,89 +0,0 @@
-import tls, { TLSSocket } from 'node:tls';
-import { URL } from 'node:url';
-
-interface ProxyOptions extends tls.ConnectionOptions {
-    method: 'GET';
-    headers?: Object;
-}
-
-export class Proxy {
-    parsed_url: URL;
-    statusCode: number;
-    rawHeaders: string;
-    headers: Object;
-    body: string;
-    socket: TLSSocket;
-    sentHeaders: string;
-    private options: ProxyOptions;
-    constructor(parsed_url: URL, options: ProxyOptions) {
-        this.parsed_url = parsed_url;
-        this.sentHeaders = '';
-        this.statusCode = 0;
-        this.rawHeaders = '';
-        this.body = '';
-        this.headers = {};
-        this.options = options;
-        this.socket = tls.connect(
-            {
-                host: this.parsed_url.hostname,
-                port: Number(this.parsed_url.port) || 443,
-                socket: options.socket,
-                rejectUnauthorized: false
-            },
-            () => this.onConnect()
-        );
-        if (options.headers) {
-            for (const [key, value] of Object.entries(options.headers)) {
-                this.sentHeaders += `${key}: ${value}\r\n`;
-            }
-        }
-    }
-
-    private onConnect() {
-        this.socket.write(
-            `${this.options.method} ${this.parsed_url.pathname}${this.parsed_url.search} HTTP/1.1\r\n` +
-                `Host: ${this.parsed_url.hostname}\r\n` +
-                this.sentHeaders +
-                `Connection: close\r\n` +
-                `\r\n`
-        );
-    }
-
-    private parseHeaders() {
-        const head_arr = this.rawHeaders.split('\r\n');
-        this.statusCode = Number(head_arr.shift()?.split(' ')[1]) ?? -1;
-        for (const head of head_arr) {
-            let [key, value] = head.split(': ');
-            if (!value) break;
-            key = key.trim().toLowerCase();
-            value = value.trim();
-            if (Object.keys(this.headers).includes(key)) {
-                let val = (this.headers as any)[key];
-                if (typeof val === 'string') val = [val];
-                Object.assign(this.headers, { [key]: [...val, value] });
-            } else Object.assign(this.headers, { [key]: value });
-        }
-    }
-
-    fetch(): Promise<Proxy> {
-        return new Promise((resolve, reject) => {
-            this.socket.setEncoding('utf-8');
-            this.socket.once('error', (err) => reject(err));
-            const parts: string[] = [];
-            this.socket.on('data', (chunk: string) => {
-                if (this.rawHeaders.length === 0) {
-                    this.rawHeaders = chunk;
-                    this.parseHeaders();
-                } else {
-                    const arr = chunk.split('\r\n');
-                    if (arr.length > 1 && arr[0].length < 5) arr.shift();
-                    parts.push(...arr);
-                }
-            });
-            this.socket.on('end', () => {
-                this.body = parts.join('');
-                resolve(this);
-            });
-        });
-    }
-}

play-dl/Request/index.ts

@@ -1,15 +1,13 @@
-import http, { ClientRequest, IncomingMessage } from 'node:http';
+import { IncomingMessage } from 'node:http';
 import https, { RequestOptions } from 'node:https';
 import { URL } from 'node:url';
+import zlib, { BrotliDecompress, Deflate, Gunzip } from 'node:zlib';
 import { cookieHeaders, getCookies } from '../YouTube/utils/cookie';
-import { Proxy } from './classes';
-
-export type ProxyOptions = ProxyOpts | string;
+import { getRandomUserAgent } from './useragent';
 
 interface RequestOpts extends RequestOptions {
     body?: string;
     method?: 'GET' | 'POST' | 'HEAD';
-    proxies?: ProxyOptions[];
     cookies?: boolean;
 }
 
@@ -48,56 +46,46 @@ export function request_stream(req_url: string, options: RequestOpts = { method:
  */
 export function request(req_url: string, options: RequestOpts = { method: 'GET' }): Promise<string> {
     return new Promise(async (resolve, reject) => {
-        if (!options?.proxies || options.proxies.length === 0) {
-            let cookies_added = false;
-            if (options.cookies) {
-                let cook = getCookies();
-                if (typeof cook === 'string' && options.headers) {
-                    Object.assign(options.headers, { cookie: cook });
-                    cookies_added = true;
-                }
+        let cookies_added = false;
+        if (options.cookies) {
+            let cook = getCookies();
+            if (typeof cook === 'string' && options.headers) {
+                Object.assign(options.headers, { cookie: cook });
+                cookies_added = true;
             }
-            let res = await https_getter(req_url, options).catch((err: Error) => err);
-            if (res instanceof Error) {
-                reject(res);
-                return;
-            }
-            if (res.headers && res.headers['set-cookie'] && cookies_added) {
-                cookieHeaders(res.headers['set-cookie']);
-            }
-            if (Number(res.statusCode) >= 300 && Number(res.statusCode) < 400) {
-                res = await https_getter(res.headers.location as string, options);
-            } else if (Number(res.statusCode) > 400) {
-                reject(new Error(`Got ${res.statusCode} from the request`));
-            }
-            const data: string[] = [];
-            res.setEncoding('utf-8');
-            res.on('data', (c) => data.push(c));
-            res.on('end', () => resolve(data.join('')));
-        } else {
-            let cookies_added = false;
-            if (options.cookies) {
-                let cook = getCookies();
-                if (typeof cook === 'string' && options.headers) {
-                    Object.assign(options.headers, { cookie: cook });
-                    cookies_added = true;
-                }
-            }
-            let res = await proxy_getter(req_url, options.proxies, options.headers).catch((e: Error) => e);
-            if (res instanceof Error) {
-                reject(res);
-                return;
-            }
-            if (res.headers && (res.headers as any)['set-cookie'] && cookies_added) {
-                cookieHeaders((res.headers as any)['set-cookie']);
-            }
-            if (res.statusCode >= 300 && res.statusCode < 400) {
-                res = await proxy_getter((res.headers as any)['location'], options.proxies, options.headers);
-            } else if (res.statusCode > 400) {
-                reject(new Error(`GOT ${res.statusCode} from proxy request`));
-            }
-            resolve(res.body);
         }
+        if (options.headers) {
+            options.headers = {
+                ...options.headers,
+                'accept-encoding': 'gzip, deflate, br',
+                'user-agent': getRandomUserAgent()
+            };
+        }
+        let res = await https_getter(req_url, options).catch((err: Error) => err);
+        if (res instanceof Error) {
+            reject(res);
+            return;
+        }
+        if (res.headers && res.headers['set-cookie'] && cookies_added) {
+            cookieHeaders(res.headers['set-cookie']);
+        }
+        if (Number(res.statusCode) >= 300 && Number(res.statusCode) < 400) {
+            res = await https_getter(res.headers.location as string, options).catch((err) => err);
+            if (res instanceof Error) throw res;
+        } else if (Number(res.statusCode) > 400) {
+            reject(new Error(`Got ${res.statusCode} from the request`));
+        }
+        const data: string[] = [];
+        let decoder: BrotliDecompress | Gunzip | Deflate;
+        const encoding = res.headers['content-encoding'];
+        if (encoding === 'gzip') decoder = zlib.createGunzip();
+        else if (encoding === 'br') decoder = zlib.createBrotliDecompress();
+        else decoder = zlib.createDeflate();
+
+        res.pipe(decoder);
+        decoder.setEncoding('utf-8');
+        decoder.on('data', (c) => data.push(c));
+        decoder.on('end', () => resolve(data.join('')));
     });
 }
 
@@ -126,75 +114,6 @@ export function request_resolve_redirect(url: string): Promise<string> {
     });
 }
 
-/**
- * Chooses one random number between max and min number.
- * @param min Minimum number
- * @param max Maximum number
- * @returns Random Number
- */
-function randomIntFromInterval(min: number, max: number): number {
-    let x = Math.floor(Math.random() * (max - min + 1) + min);
-    if (x === 0) return 0;
-    else return x - 1;
-}
-/**
- * Main module that play-dl uses for proxy.
- * @param req_url URL to make https request to
- * @param req_proxy Proxies array
- * @returns Object with statusCode, head and body
- */
-function proxy_getter(req_url: string, req_proxy: ProxyOptions[], headers?: Object): Promise<Proxy> {
-    return new Promise((resolve, reject) => {
-        const proxy: string | ProxyOpts = req_proxy[randomIntFromInterval(0, req_proxy.length)];
-        const parsed_url = new URL(req_url);
-        let opts: ProxyOpts;
-        if (typeof proxy === 'string') {
-            const parsed = new URL(proxy);
-            opts = {
-                host: parsed.hostname,
-                port: Number(parsed.port),
-                authentication: {
-                    username: parsed.username,
-                    password: parsed.password
-                }
-            };
-        } else
-            opts = {
-                host: proxy.host,
-                port: Number(proxy.port)
-            };
-        let req: ClientRequest;
-        if (!opts.authentication) {
-            req = http.request({
-                host: opts.host,
-                port: opts.port,
-                method: 'CONNECT',
-                path: `${parsed_url.host}:443`
-            });
-        } else {
-            req = http.request({
-                host: opts.host,
-                port: opts.port,
-                method: 'CONNECT',
-                path: `${parsed_url.host}:443`,
-                headers: {
-                    'Proxy-Authorization': `Basic ${Buffer.from(
-                        `${opts.authentication?.username}:${opts.authentication?.password}`
-                    ).toString('base64')}`
-                }
-            });
-        }
-        req.on('connect', async function (res, socket) {
-            const conn_proxy = new Proxy(parsed_url, { method: 'GET', socket: socket, headers: headers });
-            await conn_proxy.fetch();
-            socket.end();
-            resolve(conn_proxy);
-        });
-        req.on('error', (e: Error) => reject(e));
-        req.end();
-    });
-}
-
 /**
  * Main module that play-dl uses for making a https request
  * @param req_url URL to make https request to

play-dl/Request/useragent.ts

@@ -0,0 +1,27 @@
+const useragents: string[] = [
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.93 Safari/537.36 Edg/96.0.1054.43',
+    'Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko',
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.93 Safari/537.36',
+    'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.93 Safari/537.36',
+    'Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.93 Safari/537.36',
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:94.0) Gecko/20100101 Firefox/94.0',
+    'Mozilla/5.0 (Windows NT 6.3) AppleWebKit/537.36.0 (KHTML, like Gecko) Chrome/61.0.0.0 Safari/537.36.0',
+    'Mozilla/5.0 (Windows; U; Windows NT 6.1) AppleWebKit/531.35.5 (KHTML, like Gecko) Version/4.0.3 Safari/531.35.5',
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246',
+    'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:15.0) Gecko/20100101 Firefox/15.0.1'
+];
+
+export function setUserAgent(array: string[]): void {
+    useragents.push(...array);
+}
+
+function getRandomInt(min: number, max: number): number {
+    min = Math.ceil(min);
+    max = Math.floor(max);
+    return Math.floor(Math.random() * (max - min + 1)) + min;
+}
+
+export function getRandomUserAgent() {
+    const random = getRandomInt(0, useragents.length - 1);
+    return useragents[random];
+}