fix: fallback other yt clients no url found for adaptive formats (#5262)

.ameba.yml

@@ -25,7 +25,7 @@ Lint/NotNil:
 
 Lint/SpecFilename:
   Excluded:
-    - spec/*_helper.cr
+    - spec/parsers_helper.cr
 
 
 #

CHANGELOG.md

@@ -2,6 +2,12 @@
 
 ## vX.Y.0 (future)
 
+## v2.20250504.0
+
+Small release with quick workaround fix for issue #4251 (Nil assertion failed).
+
+PR: https://github.com/iv-org/invidious/issues/5263
+
 ## v2.20250314.0
 
 ### Wrap-up

config/config.example.yml

@@ -54,53 +54,6 @@ db:
 ##
 #signature_server:
 
-##
-## Invidious companion is an external program
-## for loading the video streams from YouTube servers.
-##
-## When this setting is commented out, Invidious companion is not used.
-## Otherwise, Invidious will proxy the requests to Invidious companion.
-## 
-## Note: multiple URL can be configured. In this case, invidious will
-## randomly pick one every time video data needs to be retrieved. This
-## URL is then kept in the video metadata cache to allow video playback
-## to work. Once said cache has expired, requesting that video's data
-## again will cause a new companion URL to be picked.
-##
-## The parameter private_url needs to be configured for the internal
-## communication between the companion and Invidious.
-## And public_url is the public URL from which companion is listening
-## to the requests from the user(s).
-##
-## If you are using a reverse proxy then you will probably need to
-## configure the public_url to be the same as the domain used for Invidious.
-## Also apply when used from an external IP address (without a domain).
-## Examples: https://MYINVIDIOUSDOMAIN or http://192.168.1.100:8282
-##
-## Both parameter can have identical URL when Invidious is hosted in
-## an internal network or at home or locally (localhost).
-##
-## Accepted values: "http(s)://<IP-HOSTNAME>:<Port>"
-## Default: <none>
-##
-#invidious_companion:
-#  - private_url: "http://localhost:8282"
-#    public_url: "http://localhost:8282"
-
-##
-## API key for Invidious companion, used for securing the communication
-## between Invidious and Invidious companion.
-## The size of the key needs to be more or equal to 16.
-##
-## Note: This parameter is mandatory when Invidious companion is enabled
-## and should be a random string.
-## Such random string can be generated on linux with the following
-## command: `pwgen 16 1`
-##
-## Accepted values: a string
-## Default: <none>
-##
-#invidious_companion_key: "CHANGE_ME!!"
 
 #########################################
 #
@@ -206,7 +159,7 @@ https_only: false
 #disable_proxy: false
 
 ##
-## Max size of the HTTP pool used to connect to youtube. Each
+## Size of the HTTP pool used to connect to youtube. Each
 ## domain ('youtube.com', 'ytimg.com', ...) has its own pool.
 ##
 ## Accepted values: a positive integer
@@ -214,16 +167,6 @@ https_only: false
 ##
 #pool_size: 100
 
-##
-## Amount of seconds to wait for a client to be free from the pool
-## before raising an error
-##
-##
-## Accepted values: a positive integer
-## Default: 5
-##
-#pool_checkout_timeout: 5
-
 
 ##
 ## Additional cookies to be sent when requesting the youtube API.
@@ -868,9 +811,9 @@ default_user_preferences:
   ## Default video quality.
   ##
   ## Accepted values: dash, hd720, medium, small
-  ## Default: dash
+  ## Default: hd720
   ##
-  #quality: dash
+  #quality: hd720
 
   ##
   ## Default dash video quality.

shard.yml

@@ -1,5 +1,5 @@
 name: invidious
-version: 2.20250314.0-dev
+version: 2.20250314.0
 
 authors:
   - Invidious team <contact@invidious.io>

spec/helpers/networking/connection_pool_spec.cr

@@ -1,112 +0,0 @@
-# Due to the way that specs are handled this file cannot be run
-# together with everything else without causing a compile time error
-#
-# TODO: Allow running different isolated spec through make
-#
-# For now run this with `crystal spec -p spec/helpers/networking/connection_pool_spec.cr -Drunning_by_self`
-{% skip_file unless flag?(:running_by_self) %}
-
-# Based on https://github.com/jgaskins/http_client/blob/958cf56064c0d31264a117467022b90397eb65d7/spec/http_client_spec.cr
-require "wait_group"
-require "uri"
-require "http"
-require "http/server"
-require "http_proxy"
-
-require "db"
-require "pg"
-require "spectator"
-
-require "../../load_config_helper"
-require "../../../src/invidious/helpers/crystal_class_overrides"
-require "../../../src/invidious/connection/*"
-
-TEST_SERVER_URL = URI.parse("http://localhost:12345")
-
-server = HTTP::Server.new do |context|
-  request = context.request
-  response = context.response
-
-  case {request.method, request.path}
-  when {"GET", "/get"}
-    response << "get"
-  when {"POST", "/post"}
-    response.status = :created
-    response << "post"
-  when {"GET", "/sleep"}
-    duration = request.query_params["duration_sec"].to_i.seconds
-    sleep duration
-  end
-end
-
-spawn server.listen 12345
-
-Fiber.yield
-
-Spectator.describe Invidious::ConnectionPool do
-  describe "Pool" do
-    it "Can make a requests through standard HTTP methods" do
-      pool = Invidious::ConnectionPool::Pool.new(max_capacity: 100) { next make_client(TEST_SERVER_URL) }
-
-      expect(pool.get("/get").body).to eq("get")
-      expect(pool.post("/post").body).to eq("post")
-    end
-
-    it "Can make streaming requests" do
-      pool = Invidious::ConnectionPool::Pool.new(max_capacity: 100) { next make_client(TEST_SERVER_URL) }
-
-      expect(pool.get("/get", &.body_io.gets_to_end)).to eq("get")
-      expect(pool.get("/post", &.body)).to eq("")
-      expect(pool.post("/post", &.body_io.gets_to_end)).to eq("post")
-    end
-
-    it "Allows more than one clients to be checked out (if applicable)" do
-      pool = Invidious::ConnectionPool::Pool.new(max_capacity: 100) { next make_client(TEST_SERVER_URL) }
-
-      pool.checkout do |_|
-        expect(pool.post("/post").body).to eq("post")
-      end
-    end
-
-    it "Can make multiple requests with the same client" do
-      pool = Invidious::ConnectionPool::Pool.new(max_capacity: 100) { next make_client(TEST_SERVER_URL) }
-
-      pool.checkout do |client|
-        expect(client.get("/get").body).to eq("get")
-        expect(client.post("/post").body).to eq("post")
-        expect(client.get("/get").body).to eq("get")
-      end
-    end
-
-    it "Allows concurrent requests" do
-      pool = Invidious::ConnectionPool::Pool.new(max_capacity: 100) { next make_client(TEST_SERVER_URL) }
-      responses = [] of HTTP::Client::Response
-
-      WaitGroup.wait do |wg|
-        100.times do
-          wg.spawn { responses << pool.get("/get") }
-        end
-      end
-
-      expect(responses.map(&.body)).to eq(["get"] * 100)
-    end
-
-    it "Raises on checkout timeout" do
-      pool = Invidious::ConnectionPool::Pool.new(max_capacity: 2, timeout: 0.01) { next make_client(TEST_SERVER_URL) }
-
-      # Long running requests
-      2.times do
-        spawn { pool.get("/sleep?duration_sec=2") }
-      end
-
-      Fiber.yield
-
-      expect { pool.get("/get") }.to raise_error(Invidious::ConnectionPool::Error)
-    end
-
-    it "Raises when an error is encountered" do
-      pool = Invidious::ConnectionPool::Pool.new(max_capacity: 100) { next make_client(TEST_SERVER_URL) }
-      expect { pool.get("/get") { raise IO::Error.new } }.to raise_error(Invidious::ConnectionPool::Error)
-    end
-  end
-end

spec/load_config_helper.cr

@@ -1,15 +0,0 @@
-require "yaml"
-require "log"
-
-abstract class Kemal::BaseLogHandler
-end
-
-require "../src/invidious/config"
-require "../src/invidious/jobs/base_job"
-require "../src/invidious/jobs.cr"
-require "../src/invidious/user/preferences.cr"
-require "../src/invidious/helpers/logger"
-require "../src/invidious/helpers/utils"
-
-CONFIG   = Config.from_yaml(File.open("config/config.example.yml"))
-HMAC_KEY = CONFIG.hmac_key

src/invidious.cr

@@ -35,7 +35,6 @@ require "protodec/utils"
 
 require "./invidious/database/*"
 require "./invidious/database/migrations/*"
-require "./invidious/connection/*"
 require "./invidious/http_server/*"
 require "./invidious/helpers/*"
 require "./invidious/yt_backend/*"
@@ -92,31 +91,11 @@ SOFTWARE = {
   "branch"  => "#{CURRENT_BRANCH}",
 }
 
-YT_POOL = Invidious::ConnectionPool::Pool.new(
+YT_POOL = YoutubeConnectionPool.new(YT_URL, capacity: CONFIG.pool_size)
-  max_capacity: CONFIG.pool_size,
-  timeout: CONFIG.pool_checkout_timeout
-) do
-  next make_client(YT_URL, force_resolve: true)
-end
 
 # Image request pool
 
-GGPHT_URL = URI.parse("https://yt3.ggpht.com")
+GGPHT_POOL = YoutubeConnectionPool.new(URI.parse("https://yt3.ggpht.com"), capacity: CONFIG.pool_size)
-
-GGPHT_POOL = Invidious::ConnectionPool::Pool.new(
-  max_capacity: CONFIG.pool_size,
-  timeout: CONFIG.pool_checkout_timeout
-) do
-  next make_client(GGPHT_URL, force_resolve: true)
-end
-
-COMPANION_POOL = Invidious::ConnectionPool::Pool.new(
-  max_capacity: CONFIG.pool_size,
-  reinitialize_proxy: false
-) do
-  companion = CONFIG.invidious_companion.sample
-  next make_client(companion.private_url, use_http_proxy: false)
-end
 
 # CLI
 Kemal.config.extra_options do |parser|
@@ -188,9 +167,16 @@ DECRYPT_FUNCTION =
   if sig_helper_address = CONFIG.signature_server.presence
     IV::DecryptFunction.new(sig_helper_address)
   else
+    LOGGER.warn("WARNING: inv-sig-helper is required for video playback. For more information see https://docs.invidious.io/installation")
     nil
   end
 
+{% for field in %w(po_token visitor_data) %}
+  if !CONFIG.{{field.id}}
+    LOGGER.warn("WARNING: {{field.id}} is required to view and playback videos. For more information see https://docs.invidious.io/installation")
+  end
+{% end %}
+
 # Start jobs
 
 if CONFIG.channel_threads > 0

src/invidious/channels/channels.cr

@@ -166,7 +166,7 @@ def fetch_channel(ucid, pull_all_videos : Bool)
   }
 
   LOGGER.trace("fetch_channel: #{ucid} : Downloading RSS feed")
-  rss = YT_POOL.get("/feeds/videos.xml?channel_id=#{ucid}").body
+  rss = YT_POOL.client &.get("/feeds/videos.xml?channel_id=#{ucid}").body
   LOGGER.trace("fetch_channel: #{ucid} : Parsing RSS feed")
   rss = XML.parse(rss)
 

src/invidious/config.cr

@@ -35,7 +35,7 @@ struct ConfigPreferences
   property max_results : Int32 = 40
   property notifications_only : Bool = false
   property player_style : String = "invidious"
-  property quality : String = "dash"
+  property quality : String = "hd720"
   property quality_dash : String = "auto"
   property default_home : String? = "Popular"
   property feed_menu : Array(String) = ["Popular", "Trending", "Subscriptions", "Playlists"]
@@ -74,16 +74,6 @@ end
 class Config
   include YAML::Serializable
 
-  class CompanionConfig
-    include YAML::Serializable
-
-    @[YAML::Field(converter: Preferences::URIConverter)]
-    property private_url : URI = URI.parse("")
-
-    @[YAML::Field(converter: Preferences::URIConverter)]
-    property public_url : URI = URI.parse("")
-  end
-
   # Number of threads to use for crawling videos from channels (for updating subscriptions)
   property channel_threads : Int32 = 1
   # Time interval between two executions of the job that crawls channel videos (subscriptions update).
@@ -157,13 +147,8 @@ class Config
   property host_binding : String = "0.0.0.0"
   # Path and permissions to make Invidious listen on a UNIX socket instead of a TCP port
   property socket_binding : SocketBindingConfig? = nil
-
+  # Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`)
-  # Max pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool)
   property pool_size : Int32 = 100
-
-  # Amount of seconds to wait for a client to be free from the pool before rasing an error
-  property pool_checkout_timeout : Float64 = 5
-
   # HTTP Proxy configuration
   property http_proxy : HTTPProxyConfig? = nil
 
@@ -175,12 +160,6 @@ class Config
   # poToken for passing bot attestation
   property po_token : String? = nil
 
-  # Invidious companion
-  property invidious_companion : Array(CompanionConfig) = [] of CompanionConfig
-
-  # Invidious companion API key
-  property invidious_companion_key : String = ""
-
   # Saved cookies in "name1=value1; name2=value2..." format
   @[YAML::Field(converter: Preferences::StringToCookies)]
   property cookies : HTTP::Cookies = HTTP::Cookies.new
@@ -261,27 +240,6 @@ class Config
         end
     {% end %}
 
-    if config.invidious_companion.present?
-      # invidious_companion and signature_server can't work together
-      if config.signature_server
-        puts "Config: You can not run inv_sig_helper and invidious_companion at the same time."
-        exit(1)
-      elsif config.invidious_companion_key.empty?
-        puts "Config: Please configure a key if you are using invidious companion."
-        exit(1)
-      elsif config.invidious_companion_key == "CHANGE_ME!!"
-        puts "Config: The value of 'invidious_companion_key' needs to be changed!!"
-        exit(1)
-      elsif config.invidious_companion_key.size != 16
-        puts "Config: The value of 'invidious_companion_key' needs to be a size of 16 characters."
-        exit(1)
-      end
-    elsif config.signature_server
-      puts("WARNING: inv-sig-helper is deprecated. Please switch to Invidious companion: https://docs.invidious.io/companion-installation/")
-    else
-      puts("WARNING: Invidious companion is required to view and playback videos. For more information see https://docs.invidious.io/companion-installation/")
-    end
-
     # HMAC_key is mandatory
     # See: https://github.com/iv-org/invidious/issues/3854
     if config.hmac_key.empty?

src/invidious/connection/pool.cr

@@ -1,116 +0,0 @@
-module Invidious::ConnectionPool
-  # A connection pool to reuse `HTTP::Client` connections
-  struct Pool
-    getter pool : DB::Pool(HTTP::Client)
-
-    # Creates a connection pool with the provided options, and client factory block.
-    def initialize(
-      *,
-      max_capacity : Int32 = 5,
-      timeout : Float64 = 5.0,
-      @reinitialize_proxy : Bool = true, # Whether or not http-proxy should be reinitialized on checkout
-      &client_factory : -> HTTP::Client
-    )
-      pool_options = DB::Pool::Options.new(
-        initial_pool_size: 0,
-        max_pool_size: max_capacity,
-        max_idle_pool_size: max_capacity,
-        checkout_timeout: timeout
-      )
-
-      @pool = DB::Pool(HTTP::Client).new(pool_options, &client_factory)
-    end
-
-    {% for method in %w[get post put patch delete head options] %}
-      # Streaming API for {{method.id.upcase}} request.
-      # The response will have its body as an `IO` accessed via `HTTP::Client::Response#body_io`.
-      def {{method.id}}(*args, **kwargs, &)
-        self.checkout do | client |
-          client.{{method.id}}(*args, **kwargs) do | response |
-            result = yield response
-            return result
-          ensure
-            response.body_io?.try &.skip_to_end
-          end
-        end
-      end
-
-      # Executes a {{method.id.upcase}} request.
-      # The response will have its body as a `String`, accessed via `HTTP::Client::Response#body`.
-      def {{method.id}}(*args, **kwargs)
-        self.checkout do | client |
-          return client.{{method.id}}(*args, **kwargs)
-        end
-      end
-    {% end %}
-
-    # Checks out a client in the pool
-    def checkout(&)
-      # If a client has been deleted from the pool
-      # we won't try to release it
-      client_exists_in_pool = true
-
-      http_client = pool.checkout
-
-      # When the HTTP::Client connection is closed, the automatic reconnection
-      # feature will create a new IO to connect to the server with
-      #
-      # This new TCP IO will be a direct connection to the server and will not go
-      # through the proxy. As such we'll need to reinitialize the proxy connection
-
-      http_client.proxy = make_configured_http_proxy_client() if @reinitialize_proxy && CONFIG.http_proxy
-
-      response = yield http_client
-    rescue ex : DB::PoolTimeout
-      # Failed to checkout a client
-      raise ConnectionPool::PoolCheckoutError.new(ex.message)
-    rescue ex
-      # An error occurred with the client itself.
-      # Delete the client from the pool and close the connection
-      if http_client
-        client_exists_in_pool = false
-        @pool.delete(http_client)
-        http_client.close
-      end
-
-      # Raise exception for outer methods to handle
-      raise ConnectionPool::Error.new(ex.message, cause: ex)
-    ensure
-      pool.release(http_client) if http_client && client_exists_in_pool
-    end
-  end
-
-  class Error < Exception
-  end
-
-  # Raised when the pool failed to get a client in time
-  class PoolCheckoutError < Error
-  end
-
-  # Mapping of subdomain => Invidious::ConnectionPool::Pool
-  # This is needed as we may need to access arbitrary subdomains of ytimg
-  private YTIMG_POOLS = {} of String => ConnectionPool::Pool
-
-  # Fetches a HTTP pool for the specified subdomain of ytimg.com
-  #
-  # Creates a new one when the specified pool for the subdomain does not exist
-  def self.get_ytimg_pool(subdomain)
-    if pool = YTIMG_POOLS[subdomain]?
-      return pool
-    else
-      LOGGER.info("ytimg_pool: Creating a new HTTP pool for \"https://#{subdomain}.ytimg.com\"")
-      url = URI.parse("https://#{subdomain}.ytimg.com")
-
-      pool = ConnectionPool::Pool.new(
-        max_capacity: CONFIG.pool_size,
-        timeout: CONFIG.pool_checkout_timeout
-      ) do
-        next make_client(url, force_resolve: true)
-      end
-
-      YTIMG_POOLS[subdomain] = pool
-
-      return pool
-    end
-  end
-end

src/invidious/helpers/utils.cr

@@ -383,22 +383,3 @@ def parse_link_endpoint(endpoint : JSON::Any, text : String, video_id : String)
   end
   return text
 end
-
-def encrypt_ecb_without_salt(data, key)
-  cipher = OpenSSL::Cipher.new("aes-128-ecb")
-  cipher.encrypt
-  cipher.key = key
-
-  io = IO::Memory.new
-  io.write(cipher.update(data))
-  io.write(cipher.final)
-  io.rewind
-
-  return io
-end
-
-def invidious_companion_encrypt(data)
-  timestamp = Time.utc.to_unix
-  encrypted_data = encrypt_ecb_without_salt("#{timestamp}|#{data}", CONFIG.invidious_companion_key)
-  return Base64.urlsafe_encode(encrypted_data)
-end

src/invidious/mixes.cr

@@ -26,7 +26,7 @@ def fetch_mix(rdid, video_id, cookies = nil, locale = nil)
   end
 
   video_id = "CvFH_6DNRCY" if rdid.starts_with? "OLAK5uy_"
-  response = YT_POOL.get("/watch?v=#{video_id}&list=#{rdid}&gl=US&hl=en", headers)
+  response = YT_POOL.client &.get("/watch?v=#{video_id}&list=#{rdid}&gl=US&hl=en", headers)
   initial_data = extract_initial_data(response.body)
 
   if !initial_data["contents"]["twoColumnWatchNextResults"]["playlist"]?

src/invidious/routes/api/manifest.cr

@@ -8,11 +8,6 @@ module Invidious::Routes::API::Manifest
     id = env.params.url["id"]
     region = env.params.query["region"]?
 
-    if CONFIG.invidious_companion.present?
-      invidious_companion = CONFIG.invidious_companion.sample
-      return env.redirect "#{invidious_companion.public_url}/api/manifest/dash/id/#{id}?#{env.params.query}"
-    end
-
     # Since some implementations create playlists based on resolution regardless of different codecs,
     # we can opt to only add a source to a representation if it has a unique height within that representation
     unique_res = env.params.query["unique_res"]?.try { |q| (q == "true" || q == "1").to_unsafe }
@@ -26,7 +21,7 @@ module Invidious::Routes::API::Manifest
     end
 
     if dashmpd = video.dash_manifest_url
-      response = YT_POOL.get(URI.parse(dashmpd).request_target)
+      response = YT_POOL.client &.get(URI.parse(dashmpd).request_target)
 
       if response.status_code != 200
         haltf env, status_code: response.status_code
@@ -167,7 +162,7 @@ module Invidious::Routes::API::Manifest
 
   # /api/manifest/hls_playlist/*
   def self.get_hls_playlist(env)
-    response = YT_POOL.get(env.request.path)
+    response = YT_POOL.client &.get(env.request.path)
 
     if response.status_code != 200
       haltf env, status_code: response.status_code
@@ -223,7 +218,7 @@ module Invidious::Routes::API::Manifest
 
   # /api/manifest/hls_variant/*
   def self.get_hls_variant(env)
-    response = YT_POOL.get(env.request.path)
+    response = YT_POOL.client &.get(env.request.path)
 
     if response.status_code != 200
       haltf env, status_code: response.status_code

src/invidious/routes/api/v1/videos.cr

@@ -106,7 +106,7 @@ module Invidious::Routes::API::V1::Videos
       # Auto-generated captions often have cues that aren't aligned properly with the video,
       # as well as some other markup that makes it cumbersome, so we try to fix that here
       if caption.name.includes? "auto-generated"
-        caption_xml = YT_POOL.get(url).body
+        caption_xml = YT_POOL.client &.get(url).body
 
         settings_field = {
           "Kind"     => "captions",
@@ -147,7 +147,7 @@ module Invidious::Routes::API::V1::Videos
         query_params = uri.query_params
         query_params["fmt"] = "vtt"
         uri.query_params = query_params
-        webvtt = YT_POOL.get(uri.request_target).body
+        webvtt = YT_POOL.client &.get(uri.request_target).body
 
         if webvtt.starts_with?("<?xml")
           webvtt = caption.timedtext_to_vtt(webvtt)
@@ -300,7 +300,7 @@ module Invidious::Routes::API::V1::Videos
         cache_annotation(id, annotations)
       end
     else # "youtube"
-      response = YT_POOL.get("/annotations_invideo?video_id=#{id}")
+      response = YT_POOL.client &.get("/annotations_invideo?video_id=#{id}")
 
       if response.status_code != 200
         haltf env, response.status_code

src/invidious/routes/channels.cr

@@ -392,7 +392,7 @@ module Invidious::Routes::Channels
     value = env.request.resource.split("/")[2]
     body = ""
     {"channel", "user", "c"}.each do |type|
-      response = YT_POOL.get("/#{type}/#{value}/live?disable_polymer=1")
+      response = YT_POOL.client &.get("/#{type}/#{value}/live?disable_polymer=1")
       if response.status_code == 200
         body = response.body
       end

src/invidious/routes/embed.cr

@@ -92,7 +92,7 @@ module Invidious::Routes::Embed
 
       return env.redirect url
     when "live_stream"
-      response = YT_POOL.get("/embed/live_stream?channel=#{env.params.query["channel"]? || ""}")
+      response = YT_POOL.client &.get("/embed/live_stream?channel=#{env.params.query["channel"]? || ""}")
       video_id = response.body.match(/"video_id":"(?<video_id>[a-zA-Z0-9_-]{11})"/).try &.["video_id"]
 
       env.params.query.delete_all("channel")
@@ -203,14 +203,6 @@ module Invidious::Routes::Embed
       return env.redirect url
     end
 
-    if CONFIG.invidious_companion.present?
-      invidious_companion = CONFIG.invidious_companion.sample
-      env.response.headers["Content-Security-Policy"] =
-        env.response.headers["Content-Security-Policy"]
-          .gsub("media-src", "media-src #{invidious_companion.public_url}")
-          .gsub("connect-src", "connect-src #{invidious_companion.public_url}")
-    end
-
     rendered "embed"
   end
 end

src/invidious/routes/errors.cr

@@ -9,10 +9,10 @@ module Invidious::Routes::ErrorRoutes
       item = md["id"]
 
       # Check if item is branding URL e.g. https://youtube.com/gaming
-      response = YT_POOL.get("/#{item}")
+      response = YT_POOL.client &.get("/#{item}")
 
       if response.status_code == 301
-        response = YT_POOL.get(URI.parse(response.headers["Location"]).request_target)
+        response = YT_POOL.client &.get(URI.parse(response.headers["Location"]).request_target)
       end
 
       if response.body.empty?
@@ -40,7 +40,7 @@ module Invidious::Routes::ErrorRoutes
       end
 
       # Check if item is video ID
-      if item.match(/^[a-zA-Z0-9_-]{11}$/) && YT_POOL.head("/watch?v=#{item}").status_code != 404
+      if item.match(/^[a-zA-Z0-9_-]{11}$/) && YT_POOL.client &.head("/watch?v=#{item}").status_code != 404
         env.response.headers["Location"] = url
         haltf env, status_code: 302
       end

src/invidious/routes/feeds.cr

@@ -160,9 +160,8 @@ module Invidious::Routes::Feeds
       "default" => "http://www.w3.org/2005/Atom",
     }
 
-    response = YT_POOL.get("/feeds/videos.xml?channel_id=#{ucid}")
+    response = YT_POOL.client &.get("/feeds/videos.xml?channel_id=#{ucid}")
     return error_atom(404, NotFoundException.new("Channel does not exist.")) if response.status_code == 404
-
     rss = XML.parse(response.body)
 
     videos = rss.xpath_nodes("//default:feed/default:entry", namespaces).map do |entry|
@@ -305,7 +304,7 @@ module Invidious::Routes::Feeds
       end
     end
 
-    response = YT_POOL.get("/feeds/videos.xml?playlist_id=#{plid}")
+    response = YT_POOL.client &.get("/feeds/videos.xml?playlist_id=#{plid}")
     return error_atom(404, NotFoundException.new("Playlist does not exist.")) if response.status_code == 404
 
     document = XML.parse(response.body)

src/invidious/routes/images.cr

@@ -12,7 +12,7 @@ module Invidious::Routes::Images
     end
 
     begin
-      GGPHT_POOL.get(url, headers) do |resp|
+      GGPHT_POOL.client &.get(url, headers) do |resp|
         return self.proxy_image(env, resp)
       end
     rescue ex
@@ -42,7 +42,7 @@ module Invidious::Routes::Images
     end
 
     begin
-      ConnectionPool.get_ytimg_pool(authority).get(url, headers) do |resp|
+      get_ytimg_pool(authority).client &.get(url, headers) do |resp|
         env.response.headers["Connection"] = "close"
         return self.proxy_image(env, resp)
       end
@@ -65,7 +65,7 @@ module Invidious::Routes::Images
     end
 
     begin
-      ConnectionPool.get_ytimg_pool("i9").get(url, headers) do |resp|
+      get_ytimg_pool("i9").client &.get(url, headers) do |resp|
         return self.proxy_image(env, resp)
       end
     rescue ex
@@ -81,7 +81,7 @@ module Invidious::Routes::Images
     end
 
     begin
-      YT_POOL.get(env.request.resource, headers) do |response|
+      YT_POOL.client &.get(env.request.resource, headers) do |response|
         env.response.status_code = response.status_code
         response.headers.each do |key, value|
           if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
@@ -111,7 +111,7 @@ module Invidious::Routes::Images
     if name == "maxres.jpg"
       build_thumbnails(id).each do |thumb|
         thumbnail_resource_path = "/vi/#{id}/#{thumb[:url]}.jpg"
-        if ConnectionPool.get_ytimg_pool("i").head(thumbnail_resource_path, headers).status_code == 200
+        if get_ytimg_pool("i").client &.head(thumbnail_resource_path, headers).status_code == 200
           name = thumb[:url] + ".jpg"
           break
         end
@@ -127,7 +127,7 @@ module Invidious::Routes::Images
     end
 
     begin
-      ConnectionPool.get_ytimg_pool("i").get(url, headers) do |resp|
+      get_ytimg_pool("i").client &.get(url, headers) do |resp|
         return self.proxy_image(env, resp)
       end
     rescue ex

src/invidious/routes/playlists.cr

@@ -464,7 +464,7 @@ module Invidious::Routes::Playlists
 
   # Undocumented, creates anonymous playlist with specified 'video_ids', max 50 videos
   def self.watch_videos(env)
-    response = YT_POOL.get(env.request.resource)
+    response = YT_POOL.client &.get(env.request.resource)
     if url = response.headers["Location"]?
       url = URI.parse(url).request_target
       return env.redirect url

src/invidious/routes/video_playback.cr

@@ -256,11 +256,6 @@ module Invidious::Routes::VideoPlayback
   # YouTube /videoplayback links expire after 6 hours,
   # so we have a mechanism here to redirect to the latest version
   def self.latest_version(env)
-    if CONFIG.invidious_companion.present?
-      invidious_companion = CONFIG.invidious_companion.sample
-      return env.redirect "#{invidious_companion.public_url}/latest_version?#{env.params.query}"
-    end
-
     id = env.params.query["id"]?
     itag = env.params.query["itag"]?.try &.to_i?
 

src/invidious/routes/watch.cr

@@ -192,14 +192,6 @@ module Invidious::Routes::Watch
       captions: video.captions
     )
 
-    if CONFIG.invidious_companion.present?
-      invidious_companion = CONFIG.invidious_companion.sample
-      env.response.headers["Content-Security-Policy"] =
-        env.response.headers["Content-Security-Policy"]
-          .gsub("media-src", "media-src #{invidious_companion.public_url}")
-          .gsub("connect-src", "connect-src #{invidious_companion.public_url}")
-    end
-
     templated "watch"
   end
 
@@ -322,19 +314,14 @@ module Invidious::Routes::Watch
       env.params.query["label"] = URI.decode_www_form(label.as_s)
 
       return Invidious::Routes::API::V1::Videos.captions(env)
-    elsif itag = download_widget["itag"]?.try &.as_i.to_s
+    elsif itag = download_widget["itag"]?.try &.as_i
       # URL params specific to /latest_version
       env.params.query["id"] = video_id
+      env.params.query["itag"] = itag.to_s
       env.params.query["title"] = filename
       env.params.query["local"] = "true"
 
-      if (CONFIG.invidious_companion.present?)
+      return Invidious::Routes::VideoPlayback.latest_version(env)
-        video = get_video(video_id)
-        invidious_companion = CONFIG.invidious_companion.sample
-        return env.redirect "#{invidious_companion.public_url}/latest_version?#{env.params.query}"
-      else
-        return Invidious::Routes::VideoPlayback.latest_version(env)
-      end
     else
       return error_template(400, "Invalid label or itag")
     end

src/invidious/search/processors.cr

@@ -16,11 +16,11 @@ module Invidious::Search
     # Search a youtube channel
     # TODO: clean code, and rely more on YoutubeAPI
     def channel(query : Query) : Array(SearchItem)
-      response = YT_POOL.get("/channel/#{query.channel}")
+      response = YT_POOL.client &.get("/channel/#{query.channel}")
 
       if response.status_code == 404
-        response = YT_POOL.get("/user/#{query.channel}")
+        response = YT_POOL.client &.get("/user/#{query.channel}")
-        response = YT_POOL.get("/c/#{query.channel}") if response.status_code == 404
+        response = YT_POOL.client &.get("/c/#{query.channel}") if response.status_code == 404
         initial_data = extract_initial_data(response.body)
         ucid = initial_data.dig?("header", "c4TabbedHeaderRenderer", "channelId").try(&.as_s?)
         raise ChannelSearchException.new(query.channel) if !ucid

src/invidious/videos.cr

@@ -15,7 +15,7 @@ struct Video
   # NOTE: don't forget to bump this number if any change is made to
   # the `params` structure in videos/parser.cr!!!
   #
-  SCHEMA_VERSION = 3
+  SCHEMA_VERSION = 2
 
   property id : String
 

src/invidious/videos/parser.cr

@@ -108,28 +108,19 @@ def extract_video_info(video_id : String)
   params = parse_video_info(video_id, player_response)
   params["reason"] = JSON::Any.new(reason) if reason
 
-  if !CONFIG.invidious_companion.present?
+  if player_response["streamingData"]? && player_response.dig?("streamingData", "adaptiveFormats", 0, "url").nil?
-    new_player_response = nil
+    LOGGER.warn("Missing URLs for adaptive formats, falling back to other YT clients.")
-
+    players_fallback = [YoutubeAPI::ClientType::WebMobile, YoutubeAPI::ClientType::TvHtml5]
-    # Don't use Android test suite client if po_token is passed because po_token doesn't
+    players_fallback.each do |player_fallback|
-    # work for Android test suite client.
+      client_config.client_type = player_fallback
-    if reason.nil? && CONFIG.po_token.nil?
+      player_fallback_response = try_fetch_streaming_data(video_id, client_config)
-      # Fetch the video streams using an Android client in order to get the
+      if player_fallback_response && player_fallback_response["streamingData"]? &&
-      # decrypted URLs and maybe fix throttling issues (#2194). See the
+          player_fallback_response.dig?("streamingData", "adaptiveFormats", 0, "url")
-      # following issue for an explanation about decrypted URLs:
+        streaming_data = player_response["streamingData"].as_h
-      # https://github.com/TeamNewPipe/NewPipeExtractor/issues/562
+        streaming_data["adaptiveFormats"] = player_fallback_response["streamingData"]["adaptiveFormats"]
-      client_config.client_type = YoutubeAPI::ClientType::AndroidTestSuite
+        player_response["streamingData"] = JSON::Any.new(streaming_data)
-      new_player_response = try_fetch_streaming_data(video_id, client_config)
+        break
-    end
+      end
-
-    # Replace player response and reset reason
-    if !new_player_response.nil?
-      # Preserve captions & storyboard data before replacement
-      new_player_response["storyboards"] = player_response["storyboards"] if player_response["storyboards"]?
-      new_player_response["captions"] = player_response["captions"] if player_response["captions"]?
-
-      player_response = new_player_response
-      params.delete("reason")
     end
   end
 

src/invidious/views/components/player.ecr

@@ -22,8 +22,6 @@
                audio_streams.each_with_index do |fmt, i|
                 src_url  = "/latest_version?id=#{video.id}&itag=#{fmt["itag"]}"
                 src_url += "&local=true" if params.local
-                src_url = invidious_companion.public_url.to_s + src_url + 
-                            "&check=#{invidious_companion_encrypt(video.id)}" if (invidious_companion)
 
                 bitrate = fmt["bitrate"]
                 mimetype = HTML.escape(fmt["mimeType"].as_s)
@@ -36,12 +34,8 @@
                 <% end %>
             <% end %>
         <% else %>
-            <% if params.quality == "dash"
+            <% if params.quality == "dash" %>
-               src_url = "/api/manifest/dash/id/" + video.id + "?local=true&unique_res=1"
+                <source src="/api/manifest/dash/id/<%= video.id %>?local=true&unique_res=1" type='application/dash+xml' label="dash">
-               src_url = invidious_companion.public_url.to_s + src_url +
-                            "&check=#{invidious_companion_encrypt(video.id)}" if (invidious_companion)
-            %>
-                <source src="<%= src_url %>" type='application/dash+xml' label="dash">
             <% end %>
 
             <%
@@ -50,8 +44,6 @@
             fmt_stream.each_with_index do |fmt, i|
                 src_url  = "/latest_version?id=#{video.id}&itag=#{fmt["itag"]}"
                 src_url += "&local=true" if params.local
-                src_url = invidious_companion.public_url.to_s + src_url +
-                            "&check=#{invidious_companion_encrypt(video.id)}" if (invidious_companion)
 
                 quality = fmt["quality"]
                 mimetype = HTML.escape(fmt["mimeType"].as_s)

src/invidious/yt_backend/connection_pool.cr

@@ -1,3 +1,51 @@
+# Mapping of subdomain => YoutubeConnectionPool
+# This is needed as we may need to access arbitrary subdomains of ytimg
+private YTIMG_POOLS = {} of String => YoutubeConnectionPool
+
+struct YoutubeConnectionPool
+  property! url : URI
+  property! capacity : Int32
+  property! timeout : Float64
+  property pool : DB::Pool(HTTP::Client)
+
+  def initialize(url : URI, @capacity = 5, @timeout = 5.0)
+    @url = url
+    @pool = build_pool()
+  end
+
+  def client(&)
+    conn = pool.checkout
+    # Proxy needs to be reinstated every time we get a client from the pool
+    conn.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy
+
+    begin
+      response = yield conn
+    rescue ex
+      conn.close
+      conn = make_client(url, force_resolve: true)
+
+      response = yield conn
+    ensure
+      pool.release(conn)
+    end
+
+    response
+  end
+
+  private def build_pool
+    options = DB::Pool::Options.new(
+      initial_pool_size: 0,
+      max_pool_size: capacity,
+      max_idle_pool_size: capacity,
+      checkout_timeout: timeout
+    )
+
+    DB::Pool(HTTP::Client).new(options) do
+      next make_client(url, force_resolve: true)
+    end
+  end
+end
+
 def add_yt_headers(request)
   request.headers.delete("User-Agent") if request.headers["User-Agent"] == "Crystal"
   request.headers["User-Agent"] ||= "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36"
@@ -13,9 +61,9 @@ def add_yt_headers(request)
   end
 end
 
-def make_client(url : URI, region = nil, force_resolve : Bool = false, force_youtube_headers : Bool = false, use_http_proxy : Bool = true)
+def make_client(url : URI, region = nil, force_resolve : Bool = false, force_youtube_headers : Bool = false)
   client = HTTP::Client.new(url)
-  client.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy && use_http_proxy
+  client.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy
 
   # Force the usage of a specific configured IP Family
   if force_resolve
@@ -30,8 +78,8 @@ def make_client(url : URI, region = nil, force_resolve : Bool = false, force_you
   return client
 end
 
-def make_client(url : URI, region = nil, force_resolve : Bool = false, use_http_proxy : Bool = true, &)
+def make_client(url : URI, region = nil, force_resolve : Bool = false, &)
-  client = make_client(url, region, force_resolve: force_resolve, use_http_proxy: use_http_proxy)
+  client = make_client(url, region, force_resolve: force_resolve)
   begin
     yield client
   ensure
@@ -51,3 +99,18 @@ def make_configured_http_proxy_client
     password: config_proxy.password,
   )
 end
+
+# Fetches a HTTP pool for the specified subdomain of ytimg.com
+#
+# Creates a new one when the specified pool for the subdomain does not exist
+def get_ytimg_pool(subdomain)
+  if pool = YTIMG_POOLS[subdomain]?
+    return pool
+  else
+    LOGGER.info("ytimg_pool: Creating a new HTTP pool for \"https://#{subdomain}.ytimg.com\"")
+    pool = YoutubeConnectionPool.new(URI.parse("https://#{subdomain}.ytimg.com"), capacity: CONFIG.pool_size)
+    YTIMG_POOLS[subdomain] = pool
+
+    return pool
+  end
+end

src/invidious/yt_backend/youtube_api.cr

@@ -500,11 +500,7 @@ module YoutubeAPI
       data["params"] = params
     end
 
-    if CONFIG.invidious_companion.present?
+    return self._post_json("/youtubei/v1/player", data, client_config)
-      return self._post_invidious_companion("/youtubei/v1/player", data)
-    else
-      return self._post_json("/youtubei/v1/player", data, client_config)
-    end
   end
 
   ####################################################################
@@ -639,13 +635,15 @@ module YoutubeAPI
     LOGGER.trace("YoutubeAPI: POST data: #{data}")
 
     # Send the POST request
-    body = YT_POOL.post(url, headers: headers, body: data.to_json) do |response|
+    body = YT_POOL.client() do |client|
-      if response.status_code != 200
+      client.post(url, headers: headers, body: data.to_json) do |response|
-        raise InfoException.new("Error: non 200 status code. Youtube API returned \
+        if response.status_code != 200
-          status code #{response.status_code}. See <a href=\"https://docs.invidious.io/youtube-errors-explained/\"> \
+          raise InfoException.new("Error: non 200 status code. Youtube API returned \
-          https://docs.invidious.io/youtube-errors-explained/</a> for troubleshooting.")
+            status code #{response.status_code}. See <a href=\"https://docs.invidious.io/youtube-errors-explained/\"> \
+            https://docs.invidious.io/youtube-errors-explained/</a> for troubleshooting.")
+        end
+        self._decompress(response.body_io, response.headers["Content-Encoding"]?)
       end
-      self._decompress(response.body_io, response.headers["Content-Encoding"]?)
     end
 
     # Convert result to Hash
@@ -668,49 +666,6 @@ module YoutubeAPI
     return initial_data
   end
 
-  ####################################################################
-  # _post_invidious_companion(endpoint, data)
-  #
-  # Internal function that does the actual request to Invidious companion
-  # and handles errors.
-  #
-  # The requested data is an endpoint (URL without the domain part)
-  # and the data as a Hash object.
-  #
-  def _post_invidious_companion(
-    endpoint : String,
-    data : Hash,
-  ) : Hash(String, JSON::Any)
-    headers = HTTP::Headers{
-      "Content-Type"  => "application/json; charset=UTF-8",
-      "Authorization" => "Bearer #{CONFIG.invidious_companion_key}",
-    }
-
-    # Logging
-    LOGGER.debug("Invidious companion: Using endpoint: \"#{endpoint}\"")
-    LOGGER.trace("Invidious companion: POST data: #{data}")
-
-    # Send the POST request
-
-    begin
-      response = COMPANION_POOL.post(endpoint, headers: headers, body: data.to_json)
-      body = response.body
-      if (response.status_code != 200)
-        raise Exception.new(
-          "Error while communicating with Invidious companion: \
-          status code: #{response.status_code} and body: #{body.dump}"
-        )
-      end
-    rescue ex
-      raise InfoException.new("Error while communicating with Invidious companion: " + (ex.message || "no extra info found"))
-    end
-
-    # Convert result to Hash
-    initial_data = JSON.parse(body).as_h
-
-    return initial_data
-  end
-
   ####################################################################
   # _decompress(body_io, headers)
   #