index.js

const puppeteer = require('puppeteer-extra')
const StealthPlugin = require('puppeteer-extra-plugin-stealth')

const flags = [
  `--disable-accelerated-2d-canvas`,
  `--no-first-run`,
  `--disable-setuid-sandbox`,
  `--no-zygote`,
  `--disable-gpu`,
  `--start-maximized`,
  `--disable-notifications`,
  `--disable-translate`,
  `--ignore-certificate-errors`,
  `--disable-popup-blocking`,
  `--no-sandbox`,
  `--disable-dev-shm-usage`,
  `--blink-settings=imagesEnabled=false`,
  `--disable-blink-features=AutomationControlled`,
  `--disable-gl-drawing-for-tests`,
  '--disable-background-timer-throttling',
  '--disable-backgrounding-occluded-windows',
  '--disable-breakpad',
  '--disable-component-extensions-with-background-pages',
  '--disable-extensions',
  '--disable-features=TranslateUI',
  '--disable-sync',
  '--metrics-recording-only',
  '--mute-audio',
  '--safebrowsing-disable-auto-update',
]

const defaultCacheDuration = {
  document: 0,
  script: 600,
  stylesheet: 600
};

const maxTimeout = 10;
const cache = {};
let result  = {}
let options = {}
let page    = null
let api_key = null

async function parseTranscript(page) {
  await page.waitForSelector('#segments-container');
  return page.evaluate(() => {
    return Array.from(document.querySelectorAll('#segments-container yt-formatted-string')).map(element => element.textContent?.trim());
  });
}

function blockedResource(resourceURL, type) {
  const blockedDomains = [
    'www.google-analytics.com',
    'connect.facebook.net',
    'googlevideo.com',
    'googleads.g.doubleclick.net',
    'www.googletagmanager.com',
    'www.googletagservices.com',
    'fonts.googleapis.com',
    'fonts.gstatic.com',
    'adservice.google.co'
  ]

  const blockedTypes = [
    'media',
    'font',
    'image'
  ]

  let resourceUrlObject = ''

  try {
    resourceUrlObject = new URL(resourceURL)
  } catch {
    return true
  }

  const resourceHostName = resourceUrlObject.hostname

  if (resourceURL.match(/\.(woff2|woff|mp3|mp4|jpg|png|gif|avi|flv|mov|wmv|svg)$/i)) {
    return true
  }

  if(blockedTypes.includes(type)) {
    return true
  }

  let isBlocked = false;

  for (const blockedDomain of blockedDomains) {
    if (resourceHostName.includes(blockedDomain)) {
      isBlocked = true;
      break;
    }
  }

  if (isBlocked) {
    return true
  }

  return false
}

exports.run = async(req, res) => {
  options = req.body
  api_key = options.api_key
  result = {
    bytes: 0,
    error: null,
    transcript: null
  }

  if (api_key == process.env.API_KEY) {
    puppeteer.use(StealthPlugin())

    if(options.proxy) {
      flags.push(`--proxy-server=${options.proxy.server}`)
    }

    const browser = await puppeteer.launch({
      headless: "new",
      ignoreDefaultArgs: ["--enable-automation"],
      args: flags
    });

    page = await browser.newPage();

    if(options.proxy) {
      await page.authenticate({
        username: options.proxy.username,
        password: options.proxy.password,
      })
    }

    if (options.user_agent) {
      await page.setUserAgent(options.user_agent.trim())
    }

    await page.setRequestInterception(true)

    page.on('request', async (interceptedRequest) => {
      let type = interceptedRequest.resourceType()
      let url = interceptedRequest.url()

      if (blockedResource(url, type)) {
        interceptedRequest.abort()
      }
      else {
        if (cache[url] && cache[url].expires > Date.now() && !url.includes('www.elon.se')) {
          try {
            await interceptedRequest.respond(cache[url]);
            return;
          } catch {}
        }

        interceptedRequest.continue()
      }
    })

    page.on('response', async (response) => {
      const url = response.url();
      const resourceType = response.request().resourceType();
      const headers = response.headers();

      // Use Cache-Control max-age if available
      const cacheControl = headers['cache-control'] || '';
      const maxAgeMatch = cacheControl.match(/max-age=(\d+)/);
      let maxAge = maxAgeMatch && maxAgeMatch.length > 1 ? parseInt(maxAgeMatch[1], 10) : 0;

      // If max-age is not available, use a default cache duration
      if (!maxAge) {
        maxAge = defaultCacheDuration[resourceType] || 60;
      }

      // Check cache for existing entry
      if (cache[url] && cache[url].expires > Date.now()) {
        // Revalidate with ETag or Last-Modified if present
        const etag = headers['etag'];
        const lastModified = headers['last-modified'];

        if ((etag && cache[url].etag === etag) || (lastModified && cache[url].lastModified === lastModified)) {
          return; // Cached data is still valid
        }
      }

      let buffer;

      try {
        buffer = await response.buffer();
      } catch {
        // Response does not have a buffer
        return;
      }

      // Cache the response
      cache[url] = {
        status: response.status(),
        headers: headers,
        body: buffer,
        expires: Date.now() + (maxAge * 1000),
        etag: headers['etag'], // Store ETag
        lastModified: headers['last-modified'] // Store Last-Modified
      };

      try {
        let length = response.headers()['content-length']
        if (length) {
          result.bytes += Number(length)
        }
      } catch {}
    })


    try {
      await page.goto(options.url, { waitUntil: 'domcontentloaded' });

      await page.evaluate(() => {
        document.querySelector('button[aria-label*=cookies]')?.click()
      });

      await page.waitForSelector("ytd-video-description-transcript-section-renderer button", {
        timeout: maxTimeout * 1000
      })

      await page.evaluate(() => {
        document.querySelector('ytd-video-description-transcript-section-renderer button').click()
      })

      result.transcript = await parseTranscript(page);

      await page.close()
      await browser.close()

      res.status(200).json(
        result
      );

    } catch(error) {
      result.error = error
      console.log(error)

      await page.close()
      await browser.close()
    }

  } else {
    res.status(401).json(
      { error: "Unauthorized" }
    )
  }
}