search

overthesun / simoc

A scalable model of an interactive, off-world community
https://ngs.simoc.space/
GNU General Public License v3.0
2 stars 2 forks source link

Missing values in last step of get_steps #81

Closed ezio-melotti closed 3 years ago

ezio-melotti commented 5 years ago

I noticed that the last step of a batch received from get_steps is missing values.

Example request:

{
  "game_id": "3579832c",
  "min_step_num": 1,
  "n_steps": 10,
  "total_agent_count": [
    "human_agent"
  ],
  "total_agent_mass": [
    "wheat",
    "cabbage",
    "strawberry",
    "radish",
    "red_beet",
    "lettuce"
  ],
  "total_production": [
    "atmo_co2",
    "atmo_o2",
    "h2o_potb",
    "enrg_kwh"
  ],
  "total_consumption": [
    "atmo_o2",
    "h2o_potb",
    "enrg_kwh"
  ],
  "storage_ratios": {
    "air_storage_1": [
      "atmo_co2",
      "atmo_o2",
      "atmo_ch4",
      "atmo_n2",
      "atmo_h2",
      "atmo_h2o"
    ]
  },
  "parse_filters": [],
  "single_agent": 1
}

Response:

{
  "message": "Step data retrieved.",
  "step_data": {
    "1": {
      "user_id": 1,
      "username": "eziom",
      "start_time": 1565686367,
      "game_id": "3579832c",
      "step_num": 1,
      "hours_per_step": 1,
      "is_terminated": "False",
      "time": 3600,
      "termination_reason": null,
      "total_agent_mass": {
        "wheat": { "value": 5.770949999999997e-12, "unit": "1.0 kg" },
        "cabbage": { "value": 6.035300000000005e-13, "unit": "1.0 kg" },
        "strawberry": { "value": 1.9953350000000022e-12, "unit": "1.0 kg" },
        "radish": { "value": 4.872055000000001e-12, "unit": "1.0 kg" },
        "red_beet": { "value": 1.7469374999999996e-12, "unit": "1.0 kg" },
        "lettuce": { "value": 1.4492228000000006e-12, "unit": "1.0 kg" }
      },
      "total_agent_count": {
        "human_agent": 1
      },
      "total_production": {
        "atmo_co2": { "value": 0.2575, "unit": "1.0 kg" },
        "atmo_o2": { "value": 0.3670000000161751, "unit": "1.0 kg" },
        "h2o_potb": { "value": 4.75, "unit": "1.0 kg" },
        "enrg_kwh": { "value": 0.0015208230000000006, "unit": "1.0 kWh" }
      },
      "total_consumption": {
        "atmo_o2": { "value": 0.19724999999999998, "unit": "1.0 kg" },
        "h2o_potb": { "value": 2.0649280000721206, "unit": "1.0 kg" },
        "enrg_kwh": { "value": 8.27271284891001, "unit": "1.0 kWh" }
      },
      "storage_ratios": {
        "air_storage_1": {
          "atmo_co2": 0.16663666750664316,
          "atmo_o2": 0.16671533197070484,
          "atmo_ch4": 0.166662000130663,
          "atmo_n2": 0.166662000130663,
          "atmo_h2": 0.166662000130663,
          "atmo_h2o": 0.166662000130663
        }
      }
    },
    "2": {
      "user_id": 1,
      "username": "eziom",
      "start_time": 1565686367,
      "game_id": "3579832c",
      "step_num": 2,
      "hours_per_step": 1,
      "is_terminated": "False",
      "time": 7200,
      "termination_reason": null,
      "total_agent_mass": {
        "wheat": { "value": 2.847745000000003e-10, "unit": "1.0 kg" },
        "cabbage": { "value": 2.978189999999998e-11, "unit": "1.0 kg" },
        "strawberry": { "value": 9.846199999999993e-11, "unit": "1.0 kg" },
        "radish": { "value": 2.404165000000003e-10, "unit": "1.0 kg" },
        "red_beet": { "value": 8.620424999999997e-11, "unit": "1.0 kg" },
        "lettuce": { "value": 7.151321999999998e-11, "unit": "1.0 kg" }
      },
      "total_agent_count": {
        "human_agent": 1
      },
      "total_production": {
        "atmo_co2": { "value": 0.2575, "unit": "1.0 kg" },
        "atmo_o2": { "value": 0.3670000007982374, "unit": "1.0 kg" },
        "h2o_potb": { "value": 4.75, "unit": "1.0 kg" },
        "enrg_kwh": { "value": 0.0750465, "unit": "1.0 kWh" }
      },
      "total_consumption": {
        "atmo_o2": { "value": 0.19724999999999998, "unit": "1.0 kg" },
        "h2o_potb": { "value": 2.0649280035581503, "unit": "1.0 kg" },
        "enrg_kwh": { "value": 8.307176198299963, "unit": "1.0 kWh" }
      },
      "storage_ratios": {
        "air_storage_1": {
          "atmo_co2": 0.16660667002647853,
          "atmo_o2": 0.16676232797629997,
          "atmo_ch4": 0.16665900042930928,
          "atmo_n2": 0.16665733385597073,
          "atmo_h2": 0.16665733385597073,
          "atmo_h2o": 0.16665733385597073
        }
      }
    },
    "3": {
      "user_id": 1,
      "username": "eziom",
      "start_time": 1565686367,
      "game_id": "3579832c",
      "step_num": 3,
      "hours_per_step": 1,
      "is_terminated": "False",
      "time": 10800,
      "termination_reason": null,
      "total_agent_mass": {
        "wheat": { "value": 2.847745000000003e-10, "unit": "1.0 kg" },
        "cabbage": { "value": 2.978189999999998e-11, "unit": "1.0 kg" },
        "strawberry": { "value": 9.846199999999993e-11, "unit": "1.0 kg" },
        "radish": { "value": 2.404165000000003e-10, "unit": "1.0 kg" },
        "red_beet": { "value": 8.620424999999997e-11, "unit": "1.0 kg" },
        "lettuce": { "value": 7.151321999999998e-11, "unit": "1.0 kg" }
      },
      "total_agent_count": {
        "human_agent": 1
      },
      "total_production": {
        "atmo_co2": { "value": 0.2575, "unit": "1.0 kg" },
        "atmo_o2": { "value": 0.3670000007982336, "unit": "1.0 kg" },
        "h2o_potb": { "value": 4.75, "unit": "1.0 kg" },
        "enrg_kwh": { "value": 0.0750465, "unit": "1.0 kWh" }
      },
      "total_consumption": {
        "atmo_o2": { "value": 0.19724999999999998, "unit": "1.0 kg" },
        "h2o_potb": { "value": 2.0649280035581414, "unit": "1.0 kg" },
        "enrg_kwh": { "value": 8.307176198299963, "unit": "1.0 kWh" }
      },
      "storage_ratios": {
        "air_storage_1": {
          "atmo_co2": 0.16657667422603167,
          "atmo_o2": 0.16681098787701834,
          "atmo_ch4": 0.16665433436924632,
          "atmo_n2": 0.16665266784256788,
          "atmo_h2": 0.16665266784256788,
          "atmo_h2o": 0.16665266784256788
        }
      }
    },
    "4": {
      "user_id": 1,
      "username": "eziom",
      "start_time": 1565686367,
      "game_id": "3579832c",
      "step_num": 4,
      "hours_per_step": 1,
      "is_terminated": "False",
      "time": 14400,
      "termination_reason": null,
      "total_agent_mass": {
        "wheat": { "value": 5.876649999999995e-9, "unit": "1.0 kg" },
        "cabbage": { "value": 6.145849999999996e-10, "unit": "1.0 kg" },
        "strawberry": { "value": 2.031875e-9, "unit": "1.0 kg" },
        "radish": { "value": 4.961279999999998e-9, "unit": "1.0 kg" },
        "red_beet": { "value": 1.7789276999999988e-9, "unit": "1.0 kg" },
        "lettuce": { "value": 1.475763199999999e-9, "unit": "1.0 kg" }
      },
      "total_agent_count": {
        "human_agent": 1
      },
      "total_production": {
        "atmo_co2": { "value": 0.2575, "unit": "1.0 kg" },
        "atmo_o2": { "value": 0.3670000164725903, "unit": "1.0 kg" },
        "h2o_potb": { "value": 4.75, "unit": "1.0 kg" },
        "enrg_kwh": { "value": 1.5486749999999991, "unit": "1.0 kWh" }
      },
      "total_consumption": {
        "atmo_o2": { "value": 0.19725, "unit": "1.0 kg" },
        "h2o_potb": { "value": 2.0649280734263527, "unit": "1.0 kg" },
        "enrg_kwh": { "value": 8.997902329999999, "unit": "1.0 kWh" }
      },
      "storage_ratios": {
        "air_storage_1": {
          "atmo_co2": 0.16654673561452304,
          "atmo_o2": 0.16685970066675887,
          "atmo_ch4": 0.16664972411414058,
          "atmo_n2": 0.16664805763356424,
          "atmo_h2": 0.16664772433744898,
          "atmo_h2o": 0.16664805763356424
        }
      }
    },
    "5": {
      "user_id": 1,
      "username": "eziom",
      "start_time": 1565686367,
      "game_id": "3579832c",
      "step_num": 5,
      "hours_per_step": 1,
      "is_terminated": "False",
      "time": 18000,
      "termination_reason": null,
      "total_agent_mass": {
        "wheat": { "value": 2.0081749999999994e-8, "unit": "1.0 kg" },
        "cabbage": { "value": 2.1001600000000007e-9, "unit": "1.0 kg" },
        "strawberry": { "value": 6.943350000000004e-9, "unit": "1.0 kg" },
        "radish": { "value": 1.6953699999999995e-8, "unit": "1.0 kg" },
        "red_beet": { "value": 6.078962999999996e-9, "unit": "1.0 kg" },
        "lettuce": { "value": 5.042982000000002e-9, "unit": "1.0 kg" }
      },
      "total_agent_count": {
        "human_agent": 1
      },
      "total_production": {
        "atmo_co2": { "value": 0.2575, "unit": "1.0 kg" },
        "atmo_o2": { "value": 0.36700005629012733, "unit": "1.0 kg" },
        "h2o_potb": { "value": 4.75, "unit": "1.0 kg" },
        "enrg_kwh": { "value": 5.292149999999999, "unit": "1.0 kWh" }
      },
      "total_consumption": {
        "atmo_o2": { "value": 0.19724999999999998, "unit": "1.0 kg" },
        "h2o_potb": { "value": 2.0649282509124522, "unit": "1.0 kg" },
        "enrg_kwh": { "value": 10.752555709999939, "unit": "1.0 kWh" }
      },
      "storage_ratios": {
        "air_storage_1": {
          "atmo_co2": 0.16651646567285158,
          "atmo_o2": 0.16690807699178517,
          "atmo_ch4": 0.16664644730211464,
          "atmo_n2": 0.16664311443982585,
          "atmo_h2": 0.16664278115359696,
          "atmo_h2o": 0.16664311443982585
        }
      }
    },
    "6": {
      "user_id": 1,
      "username": "eziom",
      "start_time": 1565686367,
      "game_id": "3579832c",
      "step_num": 6,
      "hours_per_step": 1,
      "is_terminated": "False",
      "time": 21600,
      "termination_reason": null,
      "total_agent_mass": {
        "wheat": { "value": 2.0081749999999994e-8, "unit": "1.0 kg" },
        "cabbage": { "value": 2.1001600000000007e-9, "unit": "1.0 kg" },
        "strawberry": { "value": 6.943350000000004e-9, "unit": "1.0 kg" },
        "radish": { "value": 1.6953699999999995e-8, "unit": "1.0 kg" },
        "red_beet": { "value": 6.078962999999996e-9, "unit": "1.0 kg" },
        "lettuce": { "value": 5.042982000000002e-9, "unit": "1.0 kg" }
      },
      "total_agent_count": {
        "human_agent": 1
      },
      "total_production": {
        "atmo_co2": { "value": 0.2575, "unit": "1.0 kg" },
        "atmo_o2": { "value": 0.3670000562901294, "unit": "1.0 kg" },
        "h2o_potb": { "value": 4.75, "unit": "1.0 kg" },
        "enrg_kwh": { "value": 5.292149999999999, "unit": "1.0 kWh" }
      },
      "total_consumption": {
        "atmo_o2": { "value": 0.19725, "unit": "1.0 kg" },
        "h2o_potb": { "value": 2.064928250912412, "unit": "1.0 kg" },
        "enrg_kwh": { "value": 10.752555709999939, "unit": "1.0 kWh" }
      },
      "storage_ratios": {
        "air_storage_1": {
          "atmo_co2": 0.1664866969015872,
          "atmo_o2": 0.1669552848454793,
          "atmo_ch4": 0.16664200414330393,
          "atmo_n2": 0.16663867136987653,
          "atmo_h2": 0.16663867136987653,
          "atmo_h2o": 0.16663867136987653
        }
      }
    },
    "7": {
      "user_id": 1,
      "username": "eziom",
      "start_time": 1565686367,
      "game_id": "3579832c",
      "step_num": 7,
      "hours_per_step": 1,
      "is_terminated": "False",
      "time": 25200,
      "termination_reason": null,
      "total_agent_mass": {
        "wheat": { "value": 5.679950000000004e-8, "unit": "1.0 kg" },
        "cabbage": { "value": 5.940150000000005e-9, "unit": "1.0 kg" },
        "strawberry": { "value": 1.963875e-8, "unit": "1.0 kg" },
        "radish": { "value": 4.7952350000000024e-8, "unit": "1.0 kg" },
        "red_beet": { "value": 1.7193924000000014e-8, "unit": "1.0 kg" },
        "lettuce": { "value": 1.426371399999999e-8, "unit": "1.0 kg" }
      },
      "total_agent_count": {
        "human_agent": 1
      },
      "total_production": {
        "atmo_co2": { "value": 0.2575, "unit": "1.0 kg" },
        "atmo_o2": { "value": 0.3670001592122599, "unit": "1.0 kg" },
        "h2o_potb": { "value": 4.75, "unit": "1.0 kg" },
        "enrg_kwh": { "value": 14.968440000000008, "unit": "1.0 kWh" }
      },
      "total_consumption": {
        "atmo_o2": { "value": 0.19724999999999998, "unit": "1.0 kg" },
        "h2o_potb": { "value": 2.064928709689244, "unit": "1.0 kg" },
        "enrg_kwh": { "value": 15.28807950000006, "unit": "1.0 kWh" }
      },
      "storage_ratios": {
        "air_storage_1": {
          "atmo_co2": 0.1664567632933156,
          "atmo_o2": 0.16700398955271087,
          "atmo_ch4": 0.16663739461645338,
          "atmo_n2": 0.16663406193521468,
          "atmo_h2": 0.1666337286670908,
          "atmo_h2o": 0.16663406193521468
        }
      }
    },
    "8": {
      "user_id": 1,
      "username": "eziom",
      "start_time": 1565686367,
      "game_id": "3579832c",
      "step_num": 8,
      "hours_per_step": 1,
      "is_terminated": "False",
      "time": 28800,
      "termination_reason": null,
      "total_agent_mass": {
        "wheat": { "value": 2.5770850000000006e-7, "unit": "1.0 kg" },
        "cabbage": { "value": 2.6951300000000027e-8, "unit": "1.0 kg" },
        "strawberry": { "value": 8.910349999999995e-8, "unit": "1.0 kg" },
        "radish": { "value": 2.1756649999999983e-7, "unit": "1.0 kg" },
        "red_beet": { "value": 7.801134000000001e-8, "unit": "1.0 kg" },
        "lettuce": { "value": 6.471661999999997e-8, "unit": "1.0 kg" }
      },
      "total_agent_count": {
        "human_agent": 1
      },
      "total_production": {
        "atmo_co2": { "value": 0.2575, "unit": "1.0 kg" },
        "atmo_o2": { "value": 0.3670007223704796, "unit": "1.0 kg" },
        "h2o_potb": { "value": 4.75, "unit": "1.0 kg" },
        "enrg_kwh": { "value": 67.91400000000004, "unit": "1.0 kWh" }
      },
      "total_consumption": {
        "atmo_o2": { "value": 0.19725, "unit": "1.0 kg" },
        "h2o_potb": { "value": 2.064931219963342, "unit": "1.0 kg" },
        "enrg_kwh": { "value": 40.10495760000014, "unit": "1.0 kWh" }
      },
      "storage_ratios": {
        "air_storage_1": {
          "atmo_co2": 0.16642644309934057,
          "atmo_o2": 0.16705230186387937,
          "atmo_ch4": 0.16663406291313593,
          "atmo_n2": 0.1666290640412147,
          "atmo_h2": 0.1666290640412147,
          "atmo_h2o": 0.1666290640412147
        }
      }
    },
    "9": {
      "user_id": 1,
      "username": "eziom",
      "start_time": 1565686367,
      "game_id": "3579832c",
      "step_num": 9,
      "hours_per_step": 1,
      "is_terminated": "False",
      "time": 32400,
      "termination_reason": null,
      "total_agent_mass": {
        "wheat": { "value": 4.134019999999997e-7, "unit": "1.0 kg" },
        "cabbage": { "value": 4.323379999999998e-8, "unit": "1.0 kg" },
        "strawberry": { "value": 1.4293550000000007e-7, "unit": "1.0 kg" },
        "radish": { "value": 3.490085000000002e-7, "unit": "1.0 kg" },
        "red_beet": { "value": 1.2514127999999993e-7, "unit": "1.0 kg" },
        "lettuce": { "value": 1.0381458e-7, "unit": "1.0 kg" }
      },
      "total_agent_count": {
        "human_agent": 1
      },
      "total_production": {
        "atmo_co2": { "value": 0.2575, "unit": "1.0 kg" },
        "atmo_o2": { "value": 0.3670011587872482, "unit": "1.0 kg" },
        "h2o_potb": { "value": 4.75, "unit": "1.0 kg" },
        "enrg_kwh": { "value": 108.94380000000002, "unit": "1.0 kWh" }
      },
      "total_consumption": {
        "atmo_o2": { "value": 0.19725, "unit": "1.0 kg" },
        "h2o_potb": { "value": 2.0649331652848213, "unit": "1.0 kg" },
        "enrg_kwh": { "value": 59.336708099999946, "unit": "1.0 kWh" }
      },
      "storage_ratios": {
        "air_storage_1": {
          "atmo_co2": 0.16639645743196474,
          "atmo_o2": 0.16710094539351852,
          "atmo_ch4": 0.16662939834262042,
          "atmo_n2": 0.1666243996106321,
          "atmo_h2": 0.1666243996106321,
          "atmo_h2o": 0.1666243996106321
        }
      }
    },
    "10": {
      "user_id": 1,
      "username": "eziom",
      "start_time": 1565686367,
      "game_id": "3579832c",
      "step_num": 10,
      "hours_per_step": 1,
      "is_terminated": "False",
      "time": 36000,
      "termination_reason": null,
      "total_agent_mass": {
        "wheat": {},
        "cabbage": {},
        "strawberry": {},
        "radish": {},
        "red_beet": {},
        "lettuce": {}
      },
      "total_agent_count": {
        "human_agent": 0
      },
      "total_production": {
        "atmo_co2": {},
        "atmo_o2": {},
        "h2o_potb": {},
        "enrg_kwh": {}
      },
      "total_consumption": {
        "atmo_o2": {},
        "h2o_potb": {},
        "enrg_kwh": {}
      },
      "storage_ratios": {
        "air_storage_1": {
          "atmo_co2": 0.16636675064315323,
          "atmo_o2": 0.16714819850441875,
          "atmo_ch4": 0.1666250116634009,
          "atmo_n2": 0.16662001306300903,
          "atmo_h2": 0.16662001306300903,
          "atmo_h2o": 0.16662001306300903
        }
      }
    }
  }
}

As you can see the data in the last step (step_num 10) are all missing except for the air_storage_1

Another example of a request done just after this that requests step 10 again:

{
  "game_id": "3579832c",
  "min_step_num": 10,
  "n_steps": 10,
  "total_agent_count": [
    "human_agent"
  ],
  "total_agent_mass": [
    "wheat",
    "cabbage",
    "strawberry",
    "radish",
    "red_beet",
    "lettuce"
  ],
  "total_production": [
    "atmo_co2",
    "atmo_o2",
    "h2o_potb",
    "enrg_kwh"
  ],
  "total_consumption": [
    "atmo_o2",
    "h2o_potb",
    "enrg_kwh"
  ],
  "storage_ratios": {
    "air_storage_1": [
      "atmo_co2",
      "atmo_o2",
      "atmo_ch4",
      "atmo_n2",
      "atmo_h2",
      "atmo_h2o"
    ]
  },
  "parse_filters": [],
  "single_agent": 1
}

And the response:

{
  "message": "Step data retrieved.",
  "step_data": {
    "10": {
      "user_id": 1,
      "username": "eziom",
      "start_time": 1565686367,
      "game_id": "3579832c",
      "step_num": 10,
      "hours_per_step": 1,
      "is_terminated": "False",
      "time": 36000,
      "termination_reason": null,
      "total_agent_mass": {
        "wheat": {},
        "cabbage": {},
        "strawberry": {},
        "radish": {},
        "red_beet": {},
        "lettuce": {}
      },
      "total_agent_count": {
        "human_agent": 0
      },
      "total_production": {
        "atmo_co2": {},
        "atmo_o2": {},
        "h2o_potb": {},
        "enrg_kwh": {}
      },
      "total_consumption": {
        "atmo_o2": {},
        "h2o_potb": {},
        "enrg_kwh": {}
      },
      "storage_ratios": {
        "air_storage_1": {
          "atmo_co2": 0.16636675064315323,
          "atmo_o2": 0.16714819850441875,
          "atmo_ch4": 0.1666250116634009,
          "atmo_n2": 0.16662001306300903,
          "atmo_h2": 0.16662001306300903,
          "atmo_h2o": 0.16662001306300903
        }
      }
    }
  }
}

Also in this case all the values except for air_storage_1 are missing.

Is the frontend sending a wrong request or is there something wrong on the server side?

breathnach commented 5 years ago

it can happen that ModelRecord has the data for all steps before StepRecord does. so get_steps will loop through all steps in ModelRecord, and simply fill in all values which depend on StepRecord as empty

solution implemented: don't return any data for step X if both the modelrecord and steprecord are not available.

ezio-melotti commented 5 years ago

This fixed the issue, thanks!

ezio-melotti commented 5 years ago

Even though now all the steps I receive have all values, the last step of the simulation is never received as described in #82.

breathnach commented 5 years ago

I suspect that the database is computing ModelRecord for steps up to and including the step_num passed to step_to, but is computing StepRecord for steps up to but excluding the step_num passed to step_to. I will try to verify that this is the case. If it is, @Pebody will be better able to determine why because I don't know where StepRecord and ModelRecord are being filled.

iurii-milovanov commented 5 years ago

Fixed in https://github.com/kstaats/simoc/commit/d07ad96785eefa56acf19e1eb11e4725dcf0da10