diff --git a/config/config.exs b/config/config.exs
index 044f951f6..a370405e0 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -448,7 +448,7 @@
     Pleroma.Web.RichMedia.Parsers.TwitterCard,
     Pleroma.Web.RichMedia.Parsers.OEmbed
   ],
-  failure_backoff: 60_000,
+  timeout: 5_000,
   ttl_setters: [
     Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl,
     Pleroma.Web.RichMedia.Parser.TTL.Opengraph
diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex
index fba23c657..e2889b351 100644
--- a/lib/pleroma/web/rich_media/helpers.ex
+++ b/lib/pleroma/web/rich_media/helpers.ex
@@ -69,9 +69,12 @@ defp check_content_length(headers) do
   end
 
   defp http_options do
+    timeout = Config.get!([:rich_media, :timeout])
+
     [
       pool: :rich_media,
-      max_body: Config.get([:rich_media, :max_body], 5_000_000)
+      max_body: Config.get([:rich_media, :max_body], 5_000_000),
+      tesla_middleware: [{Tesla.Middleware.Timeout, timeout: timeout}]
     ]
   end
 end
diff --git a/lib/pleroma/workers/rich_media_worker.ex b/lib/pleroma/workers/rich_media_worker.ex
index 30f9d9e9e..0a1c6d58c 100644
--- a/lib/pleroma/workers/rich_media_worker.ex
+++ b/lib/pleroma/workers/rich_media_worker.ex
@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Workers.RichMediaWorker do
+  alias Pleroma.Config
   alias Pleroma.Web.RichMedia.Backfill
   alias Pleroma.Web.RichMedia.Card
 
@@ -31,6 +32,13 @@ def perform(%Job{args: %{"op" => "backfill", "url" => _url} = args}) do
     end
   end
 
+  # There is timeout value enforced by Tesla.Middleware.Timeout
+  # which can be found in the RichMedia.Helpers module to allow us to detect
+  # a slow/infinite data stream and insert a negative cache entry for the URL
+  # We pad it by 2 seconds to be certain a slow connection is detected and we
+  # can inject a negative cache entry for the URL
   @impl Oban.Worker
-  def timeout(_job), do: :timer.seconds(5)
+  def timeout(_job) do
+    Config.get!([:rich_media, :timeout]) + :timer.seconds(2)
+  end
 end