Consider errors during HTTP GET and HEAD to be unrecoverable and insert a negative cache entry

This is for a normal HTTP error response or timeout while receiving the data. A hard error from a process crash, DNS lookup failure, etc should produce a different response than {:ok, %Tesla.Env{}} and the request/job will be retryable.
This commit is contained in:
Mark Felder 2024-07-24 15:43:45 -04:00
parent 731f7b87d2
commit 5a62868106
2 changed files with 2 additions and 10 deletions

View file

@ -36,13 +36,9 @@ def run(%{"url" => url} = args) do
:ok :ok
{:error, type} = error {:error, type} = error
when type in [:invalid_metadata, :body_too_large, :content_type, :validate] -> when type in [:invalid_metadata, :body_too_large, :content_type, :validate, :get, :head] ->
negative_cache(url_hash) negative_cache(url_hash)
error error
{:error, type} = error
when type in [:get, :head] ->
error
end end
end end

View file

@ -20,13 +20,9 @@ def perform(%Job{args: %{"op" => "backfill", "url" => _url} = args}) do
:ok :ok
{:error, type} {:error, type}
when type in [:invalid_metadata, :body_too_large, :content_type, :validate] -> when type in [:invalid_metadata, :body_too_large, :content_type, :validate, :get, :head] ->
{:cancel, type} {:cancel, type}
{:error, type}
when type in [:get, :head] ->
{:error, type}
error -> error ->
{:error, error} {:error, error}
end end