Closed matiasgarciaisaia closed 4 months ago
I was able to test this locally with a small simulated survey and this diff (in order to generate a really large interaction file - it's over 1.5GB and counting):
diff --git a/lib/ask_web/controllers/respondent_controller.ex b/lib/ask_web/controllers/respondent_controller.ex
index 2b585d42..3e5186aa 100644
--- a/lib/ask_web/controllers/respondent_controller.ex
+++ b/lib/ask_web/controllers/respondent_controller.ex
@@ -1125,6 +1125,17 @@ defmodule AskWeb.RespondentController do
|> Repo.all()
end
+ defp results_with_repetitions(results, last_hash, last_id, 0) do
+ case List.last(results) do
+ nil -> {:halt, {last_hash, last_id, 100000}}
+ last_entry -> {results, {last_entry.respondent_hashed_number, last_entry.id, 100000}}
+ end
+ end
+
+ defp results_with_repetitions(results, last_hash, last_id, repetitions) do
+ {results, {last_hash, last_id, repetitions - 1}}
+ end
+
def interactions(conn, %{"project_id" => project_id, "survey_id" => survey_id}) do
project = load_project_for_owner(conn, project_id)
survey = load_survey(project, survey_id)
@@ -1133,8 +1144,8 @@ defmodule AskWeb.RespondentController do
log_entries =
Stream.resource(
- fn -> {"", 0} end,
- fn {last_hash, last_id} ->
+ fn -> {"", 0, 100000} end,
+ fn {last_hash, last_id, repeats} ->
results =
from(e in SurveyLogEntry,
where:
@@ -1146,10 +1157,7 @@ defmodule AskWeb.RespondentController do
)
|> Repo.all()
- case List.last(results) do
- nil -> {:halt, {last_hash, last_id}}
- last_entry -> {results, {last_entry.respondent_hashed_number, last_entry.id}}
- end
+ results_with_repetitions(results, last_hash, last_id, repeats)
end,
fn _ -> [] end
)
I've also tested locally that the endpoint is indeed timing out after one hour (by forcing a slow download speed in curl
):
$ curl --limit-rate 100K http://app.surveda.lvh.me/link/4f22f3f6e56d60ee936dcb9fbde967c6
100 368M 0 368M 0 0 101k 0 --:--:-- 1:02:04 --:--:-- 115k
curl: (18) transfer closed with outstanding read data remaining
368MiB 1:02:04 [ 101KiB/s] [ <=>
Instead of using an environment variable to remove timeouts for every app's endpoint, we now only increase the download timeout for CSV files to 1 hour - just for those endpoints.
See #2146