iantei
commented
9 months ago Got the working code: Updated filtering code to identify the right confirmed trip item. RESULT: The charts are identical with this approach too.
def remove_confirmed_trip_labels():
TS_LIMIT = 1703042805
print("Inside test_function")
#first, remove all trips written after a cutoff time
for t in list(edb.get_timeseries_db().find({"metadata.write_ts": {"$gt": TS_LIMIT}, "metadata.key": {"$regex": '^manual/(mode_confirm|purpose_confirm|replaced_mode)$'}})):
# Below code is referenced from function get_confirmed_obj_for_user_input_obj(ts, ui_obj): in
# emission/storage/decorations/trip_queries.py
ts = esta.TimeSeries.get_aggregate_time_series()
ONE_DAY = 24 * 60 * 60
if 'data' in t and 'start_ts' in t['data']:
start_ts_value = t['data']['start_ts']
else:
pass
tq = estt.TimeQuery("data.start_ts", start_ts_value - ONE_DAY,
start_ts_value + ONE_DAY)
# iff the input's key is one of these, the input belongs on a place
# all other keys are only used for trip inputs
place_keys = ["manual/place_user_input", "manual/place_addition_input"]
if t['metadata']['key'] in place_keys:
# if place, we'll query the same time range, but with 'enter_ts'
tq.timeType = "data.enter_ts"
confirmed_trip_list = list(ts.find_entries(["analysis/confirmed_place"], tq))
# print("Hi")
else:
confirmed_trip_list = list(ts.find_entries(["analysis/confirmed_trip"], tq))
if confirmed_trip_list is None:
print("No matching confirmed trip for %s" % t["data"]["start_fmt_time"])
continue
# Assuming t is your current iteration from the loop
current_user_id = t["user_id"]
current_start_ts = t["data"]["start_ts"]
# Filter the confirmed_list based on the specified conditions
matching_items = [item for item in confirmed_trip_list if
item.get("user_id") == current_user_id
and item.get("metadata", {}).get("key") == "analysis/confirmed_trip"
and item.get("data", {}).get("start_ts") == current_start_ts]
# Check if any matching items were found
if matching_items:
# Access the first matching item
confirmed_trip = matching_items[0]
print("Matching item found:")
if confirmed_trip["data"]["user_input"] == {}:
print("Found confirmed trip with matching inferred trip, without user labels")
else:
print("Getting here")
print(confirmed_trip['data']['user_input'])
update_results = edb.get_analysis_timeseries_db().update_one({"user_id": t["user_id"],
"metadata.key": "analysis/confirmed_trip",
"data.start_ts": t["data"]["start_ts"]}, { "$set": { 'data.user_input': {} } })
print("Update results")
print(update_results)
confirmed_trip["data"]["user_input"] = {}
else:
print("No matching item found for the specified conditions.")
print("delete after timestamp")
edb.get_timeseries_db().delete_many({ "metadata.write_ts": { "$gt": TS_LIMIT } })
edb.get_analysis_timeseries_db().delete_many({ "metadata.write_ts": { "$gt": TS_LIMIT } })Steps of execution:
1. Dropped the existing database `openpath_prod_usaid_laos_ev`
2. Re-loaded the database from the original snapshot
3. Executed the following commands:
docker-compose -f docker-compose.yml build
docker-compose -f docker-compose.yml up
4. Executed the Generic Metric notebook loading scriptExecuted the following script for Generic Metrics:
ashrest2-35384s:em-public-dashboard ashrest2$ docker exec -it em-public-dashboard-notebook-server-1 /bin/bash
root@a053cc45b642:/usr/src/app# source setup/activate.sh
(emission) root@a053cc45b642:/usr/src/app# cd saved-notebooks
(emission) root@a053cc45b642:/usr/src/app/saved-notebooks# PYTHONPATH=.. python bin/update_mappings.py mapping_dictionaries.ipynb
(emission) root@a053cc45b642:/usr/src/app/saved-notebooks# PYTHONPATH=.. python bin/generate_plots.py generic_metrics.ipynb default
/usr/src/app/saved-notebooks/bin/generate_plots.py:30: SyntaxWarning: "is not" with a literal. Did you mean "!="?
if r.status_code is not 200:
About to download config from https://raw.githubusercontent.com/e-mission/nrel-openpath-deploy-configs/main/configs/usaid-laos-ev.nrel-op.json
Successfully downloaded config with version 1 for USAID-NREL Support for Electric Vehicle Readiness and data collection URL https://USAID-laos-EV-openpath.nrel.gov/api/
Dynamic labels download was successful for nrel-openpath-deploy-configs: usaid-laos-ev
Running at 2024-01-12T15:30:32.264590+00:00 with args Namespace(plot_notebook='generic_metrics.ipynb', program='default', date=None) for range (<Arrow [2023-05-01T00:00:00+00:00]>, <Arrow [2024-01-01T00:00:00+00:00]>)
Running at 2024-01-12T15:30:32.308029+00:00 with params [Parameter('year', int), Parameter('month', int), Parameter('program', str, value='default'), Parameter('study_type', str, value='study'), Parameter('include_test_users', bool, value=True), Parameter('dynamic_labels', dict, value={'MODE': [{'value': 'walk', 'baseMode': 'WALKING', 'met_equivalent': 'WALKING', 'kgCo2PerKm': 0}, {'value': 'e-auto_rickshaw', 'baseMode': 'MOPED', 'met_equivalent': 'IN_VEHICLE', 'kgCo2PerKm': 0.085416859}, {'value': 'auto_rickshaw', 'baseMode': 'MOPED', 'met_equivalent': 'IN_VEHICLE', 'kgCo2PerKm': 0.231943784}, {'value': 'motorcycle', 'baseMode': 'MOPED', 'met_equivalent': 'IN_VEHICLE', 'kgCo2PerKm': 0.113143309}, {'value': 'e-bike', 'baseMode': 'E_BIKE', 'met': {'ALL': {'range': [0, -1], 'mets': 4.9}}, 'kgCo2PerKm': 0.00728}, {'value': 'bike', 'baseMode': 'BICYCLING', 'met_equivalent': 'BICYCLING', 'kgCo2PerKm': 0}, {'value': 'drove_alone', 'baseMode': 'CAR', 'met_equivalent': 'IN_VEHICLE', 'kgCo2PerKm': 0.22031}, {'value': 'shared_ride', 'baseMode': 'CAR', 'met_equivalent': 'IN_VEHICLE', 'kgCo2PerKm': 0.11015}, {'value': 'e_car_drove_alone', 'baseMode': 'E_CAR', 'met_equivalent': 'IN_VEHICLE', 'kgCo2PerKm': 0.08216}, {'value': 'e_car_shared_ride', 'baseMode': 'E_CAR', 'met_equivalent': 'IN_VEHICLE', 'kgCo2PerKm': 0.04108}, {'value': 'taxi', 'baseMode': 'TAXI', 'met_equivalent': 'IN_VEHICLE', 'kgCo2PerKm': 0.30741}, {'value': 'bus', 'baseMode': 'BUS', 'met_equivalent': 'IN_VEHICLE', 'kgCo2PerKm': 0.20727}, {'value': 'train', 'baseMode': 'TRAIN', 'met_equivalent': 'IN_VEHICLE', 'kgCo2PerKm': 0.12256}, {'value': 'free_shuttle', 'baseMode': 'BUS', 'met_equivalent': 'IN_VEHICLE', 'kgCo2PerKm': 0.20727}, {'value': 'air', 'baseMode': 'AIR', 'met_equivalent': 'IN_VEHICLE', 'kgCo2PerKm': 0.09975}, {'value': 'not_a_trip', 'baseMode': 'UNKNOWN', 'met_equivalent': 'UNKNOWN', 'kgCo2PerKm': 0}, {'value': 'other', 'baseMode': 'OTHER', 'met_equivalent': 'UNKNOWN', 'kgCo2PerKm': 0}], 'PURPOSE': [{'value': 'home'}, {'value': 'work'}, {'value': 'at_work'}, {'value': 'school'}, {'value': 'transit_transfer'}, {'value': 'shopping'}, {'value': 'meal'}, {'value': 'pick_drop_person'}, {'value': 'pick_drop_item'}, {'value': 'personal_med'}, {'value': 'access_recreation'}, {'value': 'exercise'}, {'value': 'entertainment'}, {'value': 'religious'}, {'value': 'other'}], 'translations': {'en': {'walk': 'Walk', 'e-auto_rickshaw': 'E-tuk tuk', 'auto_rickshaw': 'Tuk Tuk', 'motorcycle': 'Motorcycle', 'e-bike': 'E-bike', 'bike': 'Bicycle', 'drove_alone': 'Car Drove Alone', 'shared_ride': 'Car Shared Ride', 'e_car_drove_alone': 'E-Car Drove Alone', 'e_car_shared_ride': 'E-Car Shared Ride', 'taxi': 'Taxi/Loca/inDrive', 'bus': 'Bus', 'train': 'Train', 'free_shuttle': 'Free Shuttle', 'air': 'Airplane', 'not_a_trip': 'Not a trip', 'home': 'Home', 'work': 'To Work', 'at_work': 'At Work', 'school': 'School', 'transit_transfer': 'Transit transfer', 'shopping': 'Shopping', 'meal': 'Meal', 'pick_drop_person': 'Pick-up/ Drop off Person', 'pick_drop_item': 'Pick-up/ Drop off Item', 'personal_med': 'Personal/ Medical', 'access_recreation': 'Access Recreation', 'exercise': 'Recreation/ Exercise', 'entertainment': 'Entertainment/ Social', 'religious': 'Religious', 'other': 'Other'}, 'lo': {'walk': 'ດ້ວຍການຍ່າງ', 'e-auto_rickshaw': 'ລົດ 3 ລໍ້ໄຟຟ້າ ຫລື ຕຸກຕຸກໄຟຟ້າ', 'auto_rickshaw': 'ເດີນທາດ້ວຍ ລົດຕຸກຕຸກ ຫລື ລົດສາມລໍ້', 'motorcycle': 'ລົດຈັກ', 'e-bike': 'ວຍລົດຈັກໄຟຟ້າ', 'bike': 'ລົດຖີບ', 'drove_alone': 'ເດີນທາງ ດ້ວຍລົດໃຫ່ຍ ເຊີ່ງເປັນລົດທີ່ຂັບເອງ', 'shared_ride': 'ເດີນທາງດ້ວຍລົດໃຫ່ຍ ຮ່ວມກັບລົດຄົນອຶ່ນ', 'e_car_drove_alone': 'ດ້ວຍການຂັບລົດໄຟຟ້າໄປເອງ', 'e_car_shared_ride': 'ດ້ວຍການຈ້າງລົດໄຟຟ້າໄປ', 'taxi': 'ແທັກຊີ', 'bus': 'ລົດເມ', 'train': 'ລົດໄຟ', 'free_shuttle': 'ລົດຮັບສົ່ງຟຣີ', 'air': 'ຍົນ', 'not_a_trip': 'ບໍ່ແມ່ນການເດີນທາງ', 'home': 'ບ້ານ', 'work': 'ໄປເຮັດວຽກ', 'at_work': 'ຢູ່ບ່ອນເຮັດວຽກ', 'school': 'ໄປໂຮງຮຽນ', 'transit_transfer': 'ການຖ່າຍໂອນການເດີນທາງ', 'shopping': 'ຊອບປິ້ງ', 'meal': 'ອາຫານ', 'pick_drop_person': 'ໄປຮັບ ຫລື ສົນ ຄົນ', 'pick_drop_item': 'ໄປຮັບ ຫລື ສົ່ງສິນຄ້າ', 'personal_med': 'ໄປຫາໝໍ', 'access_recreation': 'ເຂົ້າເຖິງການພັກຜ່ອນ', 'exercise': 'ພັກຜ່ອນ/ອອກກຳລັງກາຍ', 'entertainment': 'ບັນເທີງ/ສັງຄົມ', 'religious': 'ຈຸດປະສົງທາງສາດສະໜາ', 'other': 'ອື່ນໆ'}}})]Result: Cleared the cache, and accessed: http://localhost:3274/?study_config=usaid-laos-ev Validated the charts are identical.
| Production Snapshot Dec 20th | Docker compose prod |
|---|---|
shankari
Confirmed using https://openpath-stage.nrel.gov/public/?study_config=smart-commute-ebike which does not have the energy impact calculation