instedd / surveda

InSTEDD Surveda
https://instedd.org/technologies/surveda-mobile-surveys/
GNU General Public License v3.0
17 stars 6 forks source link

Increase CSV downloads' timeout by default #2361

Closed matiasgarciaisaia closed 4 months ago

matiasgarciaisaia commented 4 months ago

Instead of using an environment variable to remove timeouts for every app's endpoint, we now only increase the download timeout for CSV files to 1 hour - just for those endpoints.

See #2146

matiasgarciaisaia commented 4 months ago

I was able to test this locally with a small simulated survey and this diff (in order to generate a really large interaction file - it's over 1.5GB and counting):

diff --git a/lib/ask_web/controllers/respondent_controller.ex b/lib/ask_web/controllers/respondent_controller.ex
index 2b585d42..3e5186aa 100644
--- a/lib/ask_web/controllers/respondent_controller.ex
+++ b/lib/ask_web/controllers/respondent_controller.ex
@@ -1125,6 +1125,17 @@ defmodule AskWeb.RespondentController do
     |> Repo.all()
   end
+  defp results_with_repetitions(results, last_hash, last_id, 0) do
+    case List.last(results) do
+      nil -> {:halt, {last_hash, last_id, 100000}}
+      last_entry -> {results, {last_entry.respondent_hashed_number, last_entry.id, 100000}}
+    end
+  end
+
+  defp results_with_repetitions(results, last_hash, last_id, repetitions) do
+    {results, {last_hash, last_id, repetitions - 1}}
+  end
+
   def interactions(conn, %{"project_id" => project_id, "survey_id" => survey_id}) do
     project = load_project_for_owner(conn, project_id)
     survey = load_survey(project, survey_id)
@@ -1133,8 +1144,8 @@ defmodule AskWeb.RespondentController do
     log_entries =
       Stream.resource(
-        fn -> {"", 0} end,
-        fn {last_hash, last_id} ->
+        fn -> {"", 0, 100000} end,
+        fn {last_hash, last_id, repeats} ->
           results =
             from(e in SurveyLogEntry,
               where:
@@ -1146,10 +1157,7 @@ defmodule AskWeb.RespondentController do
             )
             |> Repo.all()
-          case List.last(results) do
-            nil -> {:halt, {last_hash, last_id}}
-            last_entry -> {results, {last_entry.respondent_hashed_number, last_entry.id}}
-          end
+          results_with_repetitions(results, last_hash, last_id, repeats)
         end,
         fn _ -> [] end
       )
matiasgarciaisaia commented 4 months ago

I've also tested locally that the endpoint is indeed timing out after one hour (by forcing a slow download speed in curl):

$ curl --limit-rate 100K http://app.surveda.lvh.me/link/4f22f3f6e56d60ee936dcb9fbde967c6
100  368M    0  368M    0     0   101k      0 --:--:--  1:02:04 --:--:--  115k
curl: (18) transfer closed with outstanding read data remaining
 368MiB 1:02:04 [ 101KiB/s] [                                              <=>