Multi-question submission emits only a single xAPI statement

This page in the demo course represents a pretty standard page model where there are multiple questions, but a single button to submit all the problems.

When one submits answers to these three questions, a tracking event with all three responses is created.

When that tracking event is mapped to an xAPI statement, however:

Only one of the responses is included in the statement
The score reflects the overall score not the score for the single response that's captured in the statement

Though they are long, I'm including the json documents here for reference.

Tracking event

{
  "name": "problem_check",
  "context": {
    "course_id": "course-v1:edX+DemoX+Demo_Course",
    "course_user_tags": {
    },
    "user_id": 4,
    "path": "/courses/course-v1:edX+DemoX+Demo_Course/xblock/block-v1:edX+DemoX+Demo_Course+type@problem+block@a0effb954cca4759994f1ac9e9434bf4/handler/xmodule_handler/problem_check",
    "org_id": "edX",
    "enterprise_uuid": "",
    "module": {
      "display_name": "Multiple Choice Questions",
      "usage_key": "block-v1:edX+DemoX+Demo_Course+type@problem+block@a0effb954cca4759994f1ac9e9434bf4"
    },
    "asides": {
    }
  },
  "username": "e0d",
  "session": "97662bef7c463c187b8fd91e0f580468",
  "ip": "172.18.0.1",
  "agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36",
  "host": "local.overhang.io:8000",
  "referer": "http://local.overhang.io:8000/xblock/block-v1:edX+DemoX+Demo_Course+type@vertical+block@54bb9b142c6c4c22afc62bcb628f0e68?show_title=0&show_bookmark_button=0&recheck_access=1&view=student_view&format=Homework",
  "accept_language": "en-US,en;q=0.9",
  "event": {
    "state": {
      "seed": 1,
      "student_answers": {
        "a0effb954cca4759994f1ac9e9434bf4_3_1": "choice_2",
        "a0effb954cca4759994f1ac9e9434bf4_4_1": [
          "choice_0",
          "choice_2"
        ],
        "a0effb954cca4759994f1ac9e9434bf4_2_1": "blue"
      },
      "has_saved_answers": false,
      "correct_map": {
        "a0effb954cca4759994f1ac9e9434bf4_2_1": {
          "correctness": "correct",
          "npoints": null,
          "msg": "",
          "hint": "",
          "hintmode": null,
          "queuestate": null,
          "answervariable": null
        },
        "a0effb954cca4759994f1ac9e9434bf4_3_1": {
          "correctness": "correct",
          "npoints": null,
          "msg": "",
          "hint": "",
          "hintmode": null,
          "queuestate": null,
          "answervariable": null
        },
        "a0effb954cca4759994f1ac9e9434bf4_4_1": {
          "correctness": "correct",
          "npoints": null,
          "msg": "",
          "hint": "",
          "hintmode": null,
          "queuestate": null,
          "answervariable": null
        }
      },
      "input_state": {
        "a0effb954cca4759994f1ac9e9434bf4_2_1": {
        },
        "a0effb954cca4759994f1ac9e9434bf4_3_1": {
        },
        "a0effb954cca4759994f1ac9e9434bf4_4_1": {
        }
      },
      "done": true
    },
    "problem_id": "block-v1:edX+DemoX+Demo_Course+type@problem+block@a0effb954cca4759994f1ac9e9434bf4",
    "answers": {
      "a0effb954cca4759994f1ac9e9434bf4_4_1": [
        "choice_0",
        "choice_2"
      ],
      "a0effb954cca4759994f1ac9e9434bf4_3_1": "choice_2",
      "a0effb954cca4759994f1ac9e9434bf4_2_1": "yellow"
    },
    "grade": 2,
    "max_grade": 3,
    "correct_map": {
      "a0effb954cca4759994f1ac9e9434bf4_2_1": {
        "correctness": "incorrect",
        "npoints": null,
        "msg": "",
        "hint": "",
        "hintmode": null,
        "queuestate": null,
        "answervariable": null
      },
      "a0effb954cca4759994f1ac9e9434bf4_3_1": {
        "correctness": "correct",
        "npoints": null,
        "msg": "",
        "hint": "",
        "hintmode": null,
        "queuestate": null,
        "answervariable": null
      },
      "a0effb954cca4759994f1ac9e9434bf4_4_1": {
        "correctness": "correct",
        "npoints": null,
        "msg": "",
        "hint": "",
        "hintmode": null,
        "queuestate": null,
        "answervariable": null
      }
    },
    "success": "incorrect",
    "attempts": 9,
    "submission": {
      "a0effb954cca4759994f1ac9e9434bf4_4_1": {
        "question": "",
        "answer": [
          "a piano",
          "a guitar"
        ],
        "response_type": "choiceresponse",
        "input_type": "checkboxgroup",
        "correct": true,
        "variant": "",
        "group_label": ""
      },
      "a0effb954cca4759994f1ac9e9434bf4_3_1": {
        "question": "",
        "answer": "a chair",
        "response_type": "multiplechoiceresponse",
        "input_type": "choicegroup",
        "correct": true,
        "variant": "",
        "group_label": ""
      },
      "a0effb954cca4759994f1ac9e9434bf4_2_1": {
        "question": "",
        "answer": "yellow",
        "response_type": "optionresponse",
        "input_type": "optioninput",
        "correct": false,
        "variant": "",
        "group_label": ""
      }
    }
  },
  "time": "2022-07-19T15:29:01.188319+00:00",
  "event_type": "problem_check",
  "event_source": "server",
  "page": "x_module"
}

xAPI Statement

{
  "id": "e903feae-9a7a-44a4-8ec1-ac9f25c417cb",
  "result": {
    "score": {
      "scaled": 1.0,
      "raw": 3.0,
      "min": 0.0,
      "max": 3.0
    },
    "success": true,
    "response": "['a piano', 'a guitar']"
  },
  "version": "1.0.3",
  "actor": {
    "objectType": "Agent",
    "account": {
      "name": "7c27fd76-cef8-452f-8cac-7ddf0c8ec593",
      "homePage": "http://local.overhang.io:8000"
    }
  },
  "verb": {
    "id": "https://w3id.org/xapi/acrossx/verbs/evaluated",
    "display": {
      "en": "evaluated"
    }
  },
  "object": {
    "id": "http://local.overhang.io:8000/xblock/block-v1:edX+DemoX+Demo_Course+type@problem+block@a0effb954cca4759994f1ac9e9434bf4",
    "objectType": "Activity",
    "definition": {
      "description": {
        "en-US": ""
      },
      "type": "http://adlnet.gov/expapi/activities/cmi.interaction",
      "interactionType": "choice",
      "extensions": {
        "http://id.tincanapi.com/extension/attempt-id": 7
      }
    }
  },
  "timestamp": "2022-07-19T15:18:51.610141+00:00",
  "context": {
    "contextActivities": {
      "parent": [
        {
          "id": "http://local.overhang.io:8000/course/course-v1:edX+DemoX+Demo_Course",
          "objectType": "Activity",
          "definition": {
            "name": {
              "en-US": "Demonstration Course"
            },
            "type": "http://adlnet.gov/expapi/activities/course"
          }
        }
      ]
    },
    "extensions": {
      "https://github.com/edx/event-routing-backends/blob/master/docs/xapi-extensions/eventVersion.rst": "1.0"
    }
  }
}

@ayub02 shared that this is a known issue. xAPI doesn't support multiple submissions in a single statement and dis-integrating the score for a multi-submission on the tracking event side is unreliable given weighting and other complexity.

There's another general problem with this model of page -- multiple problems with a single submit -- that was raised in a separate conversation with a team working on learning analytics. The issue is that you cannot effectively measure time-on-task for the questions individually. This prevents performing some important measures of problem efficacy.

@pomegranited while you're pondering on this you may also run across this: https://github.com/openedx/event-routing-backends/issues/311 Just a heads up in case the fixes are adjacent.

@bmtcril My apologies, I won't be able to start this until next week :( So if @Ian2012 would rather take it, you're welcome to :)

@bmtcril @e0d CC @ayub02

I found a recommendation from adlnet for how best to deal with this issue in xAPI:

Parent: an Activity with a direct relation to the Activity which is the Object of the Statement. In almost all cases there is only one sensible parent or none, not multiple. For example: a Statement about a quiz question would have the quiz as its parent Activity.

Grouping: an Activity with an indirect relation to the Activity which is the Object of the Statement. For example: a course that is part of a qualification. The course has several classes. The course relates to a class as the parent, the qualification relates to the class as the grouping.

We can also use StatementRefs to connect the sub-problem "evaluated" xAPI statement to the parent problem's "evaluated" event (see Appendix A: Example Statements (3rd example down)).

This lets us keep the parent problem xAPI statement mostly intact, and issue separate-but-linked xAPI statements for each sub-problem.

I've explained in detail below -- let me know what you think?

Parent multi-part problem xAPI statement

Similar to the original problem xAPI statement, with the following changes:

Removed "result.response": "['a piano', 'a guitar']" -- these responses will be provided by the relevant sub-problem statements.
Changed "object.objectType": "Activity" to `"object.objectType": "Group"
Changed "object.interactionType": "choice" to "object.interactionType": "other".
Added "object.description.en-US": "Multiple Choice Questions", because the display_name of the problem block doesn't seem to be conveyed anywhere in the original xAPI statement.

{
  "id": "e903feae-9a7a-44a4-8ec1-ac9f25c417cb",
  "result": {
    "score": {
      "scaled": 1.0,
      "raw": 3.0,
      "min": 0.0,
      "max": 3.0
    },
    "success": true,
  },
  "version": "1.0.3",
  "actor": {
    "objectType": "Agent",
    "account": {
      "name": "7c27fd76-cef8-452f-8cac-7ddf0c8ec593",
      "homePage": "http://local.overhang.io:8000"
    }
  },
  "verb": {
    "id": "https://w3id.org/xapi/acrossx/verbs/evaluated",
    "display": {
      "en": "evaluated"
    }
  },
  "object": {
    "id": "http://local.overhang.io:8000/xblock/block-v1:edX+DemoX+Demo_Course+type@problem+block@a0effb954cca4759994f1ac9e9434bf4",
    "objectType": "Group",  // was "Activity"
    "definition": {
      "description": {
        "en-US": "Multiple Choice Questions"
      },
      "type": "http://adlnet.gov/expapi/activities/cmi.interaction",
      "interactionType": "other", // was "choice"
      "extensions": {
        "http://id.tincanapi.com/extension/attempt-id": 7
      }
    }
  },
  "timestamp": "2022-07-19T15:18:51.610141+00:00",
  "context": {
    "contextActivities": {
      "parent": [
        {
          "id": "http://local.overhang.io:8000/course/course-v1:edX+DemoX+Demo_Course",
          "objectType": "Activity",
          "definition": {
            "name": {
              "en-US": "Demonstration Course"
            },
            "type": "http://adlnet.gov/expapi/activities/course"
          }
        }
      ]
    },
    "extensions": {
      "https://github.com/edx/event-routing-backends/blob/master/docs/xapi-extensions/eventVersion.rst": "1.0"
    }
  }
}

And issue Activity statements for each of the parts of the problem, which are like our original Activity xAPI statement, with the following changes:

Omits "result.score", because we can't disentangle the scores for each sub-problem.
Includes "result.success", because we do know whether each sub-problem was answered correctly.
Includes "result.response", because we know what the user submitted for each sub-problem. (See Notes below about the response format shown for Question 1.)
Adds "context.statement" so we can reference the parent "evaluated" statement above.
Changes "context.contextActivities.parent" to the Object block from the problem event above.
Adds "context.contextActivities.grouping" to store the course "context.contextActivities.parent" from the problem event above.

Question 1 xAPI statement

{
  "id": "<new uuid>",
  "result": {
    "response": "['blue']",
    "success": true,
  },
  "version": "1.0.3",
  "actor": {
    "objectType": "Agent",
    "account": {
      "name": "7c27fd76-cef8-452f-8cac-7ddf0c8ec593",
      "homePage": "http://local.overhang.io:8000"
    }
  },
  "verb": {
    "id": "https://w3id.org/xapi/acrossx/verbs/evaluated",
    "display": {
      "en": "evaluated"
    }
  },
  "object": {
    "objectType":"Activity",
    "id" :"http://local.overhang.io:8000/xblock/block-v1:edX+DemoX+Demo_Course+type@problem+block@a0effb954cca4759994f1ac9e9434bf4_2_1",
    "definition": {
      "description": {
        "en-US": "Question 1"
      },
      "type": "http://adlnet.gov/expapi/activities/cmi.interaction",
      "interactionType": "choice",
    }
  },
  "timestamp": "2022-07-19T15:18:51.610141+00:00",
  "context": {
    "statement": {
        "id" :"e903feae-9a7a-44a4-8ec1-ac9f25c417cb",  // the Group object's event ID above
        "objectType": "StatementRef",
    },
    "contextActivities": {
      "parent": [
        { // "object" stanza from referenced parent statement
          "id": "http://local.overhang.io:8000/xblock/block-v1:edX+DemoX+Demo_Course+type@problem+block@a0effb954cca4759994f1ac9e9434bf4",
          "objectType": "Group",
          "definition": {
            "description": {
              "en-US": "Multiple Choice Questions"
            },
            "type": "http://adlnet.gov/expapi/activities/cmi.interaction",
            "interactionType": "choice",
            "extensions": {
              "http://id.tincanapi.com/extension/attempt-id": 7
            }
          }
        }
      ],
      "grouping": [
        { // "parent" course stanza from referenced parent statement
          "id": "http://local.overhang.io:8000/course/course-v1:edX+DemoX+Demo_Course",
          "objectType": "Activity",
          "definition": {
            "name": {
              "en-US": "Demonstration Course"
            },
            "type": "http://adlnet.gov/expapi/activities/course"
          }
        }
      ]
    },
    "extensions": {
      "https://github.com/edx/event-routing-backends/blob/master/docs/xapi-extensions/eventVersion.rst": "1.0"
    }
  }
}

Question 3 xAPI statement

Similar to Question 1 above, but with Question 3's "object.id" and submitted "result.response".

{
  "id": "<new uuid>",
  "result": {
    "response": "['a piano', 'a guitar']",
    "success": true,
  },
  "version": "1.0.3",
  "actor": {
    "objectType": "Agent",
    "account": {
      "name": "7c27fd76-cef8-452f-8cac-7ddf0c8ec593",
      "homePage": "http://local.overhang.io:8000"
    }
  },
  "verb": {
    "id": "https://w3id.org/xapi/acrossx/verbs/evaluated",
    "display": {
      "en": "evaluated"
    }
  },
  "object": {
    "objectType":"Activity",
    "id" :"http://local.overhang.io:8000/xblock/block-v1:edX+DemoX+Demo_Course+type@problem+block@a0effb954cca4759994f1ac9e9434bf4_4_1",
    "definition": {
      "description": {
        "en-US": "Question 3"
      },
      "type": "http://adlnet.gov/expapi/activities/cmi.interaction",
      "interactionType": "choice",
    }
  },
  "timestamp": "2022-07-19T15:18:51.610141+00:00",
  "context": {
    "statement": {
        "id" :"e903feae-9a7a-44a4-8ec1-ac9f25c417cb",
        "objectType": "StatementRef",
    },
    "contextActivities": {
      "parent": [
        {
          "id": "http://local.overhang.io:8000/xblock/block-v1:edX+DemoX+Demo_Course+type@problem+block@a0effb954cca4759994f1ac9e9434bf4",
          "objectType": "Group",
          "definition": {
            "description": {
              "en-US": "Multiple Choice Questions"
            },
            "type": "http://adlnet.gov/expapi/activities/cmi.interaction",
            "interactionType": "choice",
            "extensions": {
              "http://id.tincanapi.com/extension/attempt-id": 7
            }
          }
        }
      ],
      "grouping": [
        {
          "id": "http://local.overhang.io:8000/course/course-v1:edX+DemoX+Demo_Course",
          "objectType": "Activity",
          "definition": {
            "name": {
              "en-US": "Demonstration Course"
            },
            "type": "http://adlnet.gov/expapi/activities/course"
          }
        }
      ]
    },
    "extensions": {
      "https://github.com/edx/event-routing-backends/blob/master/docs/xapi-extensions/eventVersion.rst": "1.0"
    }
  }
}

Issues/questions about this approach:

"Group" is only used in the adlnet/xAPI-Spec when describing groups of Agents, so I'm not sure if it's valid to use this for grouping Objects.
I didn't find a good object.interactionType for groups of problems among the definition options for "cmi.interaction". Maybe we should define our own group of problems with a new "object.interactionType": "group" or "object.interactionType": "problem_set", to differentiate these from generic "other" problem types?
The object.id for the sub-problems no longer points to a URL we can visit, because the sub-problem blocks cannot be displayed individually.
Course context for sub-problems needs to be pulled from context.contextActivity.grouping instead of context.contextActivity.parent. Is that going to be difficult?

Notes

Some other things I picked up, which are unrelated to this issue of splitting multi-problem events:

alnet/xAPI-Spec's Response Patterns require that response values for objects with "interactionType": "choice" be represented as an array of choices. It goes on to say if only one item is given, the "," delimiter must be omitted. This test fixture shows that we're not doing this for single responses.
adlnet's xAPI-Spec cmi.interaction Activity object type examples show a "correctnessResponsePattern" to describe correct answers for various problem types. We could add something like this to encode the correct answers in the problem statement at the time of problem scoring.

This makes sense to me from a data model standpoint, and seems really complicated from a query standpoint, but I think it is a good way forward. I think our current suite of reports cares more about the individual questions, but it will be good to be able to model these things in a way that makes sense.

@bmtcril Ok cool -- is there anything I can do to make the querying simpler? These points relate to that question:

I didn't find a good object.interactionType for groups of problems among the definition options for "cmi.interaction". Maybe we should define our own group of problems with a new "object.interactionType": "group" or "object.interactionType": "problem_set", to differentiate these from generic "other" problem types?

Course context for sub-problems needs to be pulled from context.contextActivity.grouping instead of context.contextActivity.parent. Is that going to be difficult?

Ok to proceed with dev here?

I think it makes sense to author our own "object.interactionType": "problem_set"
I don't think it's difficult to look in one more place for the course context, we'll just need to update the MV with a new migration adding a line here: https://github.com/openedx/tutor-contrib-aspects/blob/273a031fbb7273817ce9f343b883172168383d8c/tutoraspects/templates/aspects/apps/aspects/migrations/alembic/versions/0002_raw_xapi_table.py#L66

Yep, go for it unless anyone else has other input!

@bmtcril The first thing I'm trying to do here is to find a place to wedge in the "some events get transformed into multiple events" logic.

Do you have opinions on where this should go?

I'm considering the following approach, but it doesn't feel very clean..

Modify ProblemCheckTransformer to make transform_event return a list of xAPI dict events.
- I'd create an XapiOneToManyTransformerMixin that understands how to set the StatementRef and parent context stanzas on the child events using data from the primary event. So this could be re-used for other one-to-many event types if needed.
Modify events_router to check whether it's got an array of processed_event(s) to send, and if so, appends them all to the route_events lists list.

But:

The BaseTransformerProcessorMixin.transform_event` method says it supports any returned event type, not just dicts, so long as the "router" supports it. But it's not the router that's going to have to deal with this change, it's the main process loop. And that feels icky..
Are there any other process loops that would need to be modified?
Do we know whether people customize their processor chains when configuring event_routing_backends? Any custom xAPI processors in the chain will have to deal with suddenly getting a list of xAPI problem_check events instead of the usual single dict.

The only alternative I came up with was refactoring BaseTransformerProcessorMixin entirely to assume a that transform_event returns a list of events, so that this isn't special case code.. But that seems like overkill for just this one little case, and it would take some effort to keep it backward compatible for custom processors out in the wild.

I see what you're saying, and it's definitely a pain either way. Either way you're suggesting would be ok with me, but overall I have a preference for "doing things one way" instead of having the cognitive load of having one special case. I think you'd have to do most of the plumbing for the BaseTransformerProcessorMixin anyway, so my loosely held preference would be to just do the refactor.

To the best of my knowledge no one is using this in production, let alone writing custom processor chains. @ziafazal might know better. The only other process loop I know if is the management command which is pretty well plumbed for multiple statements already so hopefully not too bad.

Another thing to keep in mind is that Vector can read statements off the logs, so we'd need to make sure the loggers for xAPI and Caliper both emit one statement per line like: https://github.com/openedx/event-routing-backends/blob/46ba27af962362f3bf5c6e6ced1d8492ccdb73e3/event_routing_backends/processors/xapi/transformer_processor.py#L55

@bmtcril

Another thing to keep in mind is that Vector can read statements off the logs, so we'd need to make sure the loggers for xAPI and Caliper both emit one statement per line like:

Ahh... that's a big gotcha. I'll make sure that's still working, and put a giant warning comment around those log statements :)

And from what you said when we talked, I'll see if I can make everything use the bulk_import, and remove the single-event submission code.

I meant the bulk send method in the router vs send fwiw

openedx / event-routing-backends

Multi-question submission emits only a single xAPI statement #219