search

hobbit-project / platform

HOBBIT benchmarking platform
GNU General Public License v2.0
24 stars 9 forks source link

Challenge task config can not be changed for OKE task #67

Closed MichaelRoeder closed 6 years ago

MichaelRoeder commented 7 years ago

Problem

When logged in as gerbil user, it is not possible to change the configuration of "OKE2017 Task 1 Scenario A" of the OKE challenge to use the "GERBIL A2KB" benchmark instead of the "OKE2015 Task1" benchmark. Although the benchmark can be chosen and configured in the front end, the gui sends a PUT request containing the same old configuration.

{
  "type": "challengeBean",
  "description": "The Open Knowledge Extraction (OKE) Challenge invites researchers and practitioners from academia as well as industry to compete to the aim of pushing further the state of the art of knowledge extraction for the Semantic Web.",
  "id": "http://w3id.org/hobbit/challenges#7023e350-9376-4d30-8a1e-2bbf1b639f27",
  "name": "OKE 2017",
  "publishDate": "2017-06-02T00:00Z",
  "executionDate": "2017-05-24T00:00Z",
  "closed": false,
  "organizer": "gerbil",
  "tasks": [
    {
      "description": "This task extends the first task towards 12 additional entity classes in DBpedia. Beside the three types in the first task, a competing system might have to identify other types of entities. The dataset has been automatically created and the system retrieves an increasing amount of requests over time.",
      "id": "http://w3id.org/hobbit/challenges#7023e350-9376-4d30-8a1e-2bbf1b639f27_task5",
      "name": "OKE2017 Task 2 Scenario B",
      "benchmark": {
        "description": "OKE Task 2 Bengal benchmark based on GERBIL",
        "id": "http://w3id.org/gerbil/hobbit/vocab#GerbilBenchmarkBengalT2",
        "name": "GERBIL Benchmark - Bengal - OKE Task 2",
        "configurationParamNames": [
          "Type of experiment",
          "Number of Documents",
          "Number of Bengal Generators",
          "Seed for Bengal Generators",
          "Phases"
        ],
        "configurationParams": [
          {
            "description": "Defines the type of experiment that should be carried out.",
            "id": "http://w3id.org/gerbil/hobbit/vocab#hasExperimentType",
            "name": "Type of experiment",
            "defaultValue": "http://w3id.org/gerbil/vocab#OKE2015_Task1",
            "feature": true,
            "options": [
              {
                "label": "http://w3id.org/gerbil/vocab#OKE2015_Task1",
                "value": "http://w3id.org/gerbil/vocab#OKE2015_Task1"
              }
            ],
            "range": "http://w3id.org/gerbil/vocab#ExperimentTypes",
            "required": false
          },
          {
            "description": "The Number of Documents Bengal should generate.",
            "id": "http://w3id.org/gerbil/hobbit/vocab#hasNumberOfDocuments",
            "name": "Number of Documents",
            "datatype": "xsd:integer",
            "feature": true,
            "range": "http://www.w3.org/2001/XMLSchema#int",
            "required": false
          },
          {
            "description": "The Number of Bengal Generators",
            "id": "http://w3id.org/gerbil/hobbit/vocab#hasNumberOfGenerators",
            "name": "Number of Bengal Generators",
            "datatype": "xsd:integer",
            "feature": true,
            "range": "http://www.w3.org/2001/XMLSchema#int",
            "required": false
          },
          {
            "description": "The seed which should be used to ranomize the Bengal Generators",
            "id": "http://w3id.org/gerbil/hobbit/vocab#hasSeed",
            "name": "Seed for Bengal Generators",
            "datatype": "xsd:string",
            "feature": true,
            "range": "http://www.w3.org/2001/XMLSchema#string",
            "required": false
          },
          {
            "description": "The number of Phases Bengal should run.",
            "id": "http://w3id.org/gerbil/hobbit/vocab#hasPhases",
            "name": "Phases",
            "datatype": "xsd:integer",
            "feature": true,
            "range": "http://www.w3.org/2001/XMLSchema#int",
            "required": false
          }
        ],
        "systems": [
          {
            "description": "This is a system adapter that simply wraps DBpedia Spotlight.",
            "id": "http://w3id.org/hobbit/gerbil/test/SpotWrapNifWS4Test",
            "name": "SpotWrapNifWS4Test"
          },
          {
            "description": "This is a client for the FREME webservice.",
            "id": "http://api.freme-project.eu/current/e-entity/freme-ner/client/OKE-task1",
            "name": "FREME for OKE Task 1"
          }
        ]
      },
      "configurationParams": [
        {
          "description": "The Number of Bengal Generators",
          "id": "http://w3id.org/gerbil/hobbit/vocab#hasNumberOfGenerators",
          "name": "Number of Bengal Generators",
          "datatype": "xsd:integer",
          "range": "http://www.w3.org/2001/XMLSchema#int",
          "value": "4"
        },
        {
          "description": "Defines the type of experiment that should be carried out.",
          "id": "http://w3id.org/gerbil/hobbit/vocab#hasExperimentType",
          "name": "Type of experiment",
          "range": "http://w3id.org/gerbil/vocab#ExperimentTypes",
          "value": "http://w3id.org/gerbil/vocab#OKE2015_Task1"
        },
        {
          "description": "The seed which should be used to ranomize the Bengal Generators",
          "id": "http://w3id.org/gerbil/hobbit/vocab#hasSeed",
          "name": "Seed for Bengal Generators",
          "datatype": "xsd:string",
          "range": "http://www.w3.org/2001/XMLSchema#string",
          "value": "21"
        },
        {
          "description": "The Number of Documents Bengal should generate.",
          "id": "http://w3id.org/gerbil/hobbit/vocab#hasNumberOfDocuments",
          "name": "Number of Documents",
          "datatype": "xsd:integer",
          "range": "http://www.w3.org/2001/XMLSchema#int",
          "value": "20"
        },
        {
          "description": "The number of Phases Bengal should run.",
          "id": "http://w3id.org/gerbil/hobbit/vocab#hasPhases",
          "name": "Phases",
          "datatype": "xsd:integer",
          "range": "http://www.w3.org/2001/XMLSchema#int",
          "value": "7"
        }
      ]
    },
    {
      "description": "The task is composed of two sub tasks focused musical NE recognition and linking. A competing system has to fulfill both tasks. The domain of this task is music, thus the used knowledge base will be MusicBrainz as Linked Data (MBL). The dataset has been automatically created and the system retrieves an increasing amount of requests over time.",
      "id": "http://w3id.org/hobbit/challenges#7023e350-9376-4d30-8a1e-2bbf1b639f27_task6",
      "name": "OKE2017 Task 3 Scenario B",
      "benchmark": {
        "description": "OKE 2015/16 Task 1 benchmark based on GERBIL",
        "id": "http://w3id.org/gerbil/hobbit/vocab#GerbilBenchmarkOKE2015T1",
        "name": "GERBIL Benchmark - OKE 2015/16 Task 1",
        "configurationParamNames": [
          "Type of experiment",
          "Dataset name"
        ],
        "configurationParams": [
          {
            "description": "Defines the type of experiment that should be carried out.",
            "id": "http://w3id.org/gerbil/hobbit/vocab#hasExperimentType",
            "name": "Type of experiment",
            "defaultValue": "http://w3id.org/gerbil/vocab#OKE2015_Task1",
            "feature": true,
            "options": [
              {
                "label": "http://w3id.org/gerbil/vocab#OKE2015_Task1",
                "value": "http://w3id.org/gerbil/vocab#OKE2015_Task1"
              }
            ],
            "range": "http://w3id.org/gerbil/vocab#ExperimentTypes",
            "required": false
          },
          {
            "description": "The dataset which should be tested.",
            "id": "http://w3id.org/gerbil/hobbit/vocab#hasDataset",
            "name": "Dataset name",
            "feature": true,
            "options": [
              {
                "label": "http://w3id.org/gerbil/vocab#OKE2015-1-example",
                "value": "http://w3id.org/gerbil/vocab#OKE2015-1-example"
              },
              {
                "label": "http://w3id.org/gerbil/vocab#OKE2017-1-eval",
                "value": "http://w3id.org/gerbil/vocab#OKE2017-1-eval"
              }
            ],
            "range": "http://w3id.org/gerbil/vocab#Datasets",
            "required": false
          }
        ],
        "systems": [
          {
            "id": "http://ns.aksw.org/foxInstance"
          }
        ]
      },
      "configurationParams": [
        {
          "description": "The dataset which should be tested.",
          "id": "http://w3id.org/gerbil/hobbit/vocab#hasDataset",
          "name": "Dataset name",
          "range": "http://w3id.org/gerbil/vocab#Datasets",
          "value": "http://w3id.org/gerbil/vocab#OKE2015-1-example"
        }
      ]
    },
    {
      "description": "Focused NE Identification and Linking with a manually created dataset. The task comprises the identification of named entities in sentences and the disambiguation of the identified entities to the DBpedia knowledge base. This task is limited to three DBpedia ontology classes (Person, Place, Organisation) and their associated sub classes.",
      "id": "http://w3id.org/hobbit/challenges#7023e350-9376-4d30-8a1e-2bbf1b639f27_task1",
      "name": "OKE2017 Task 1 Scenario A",
      "benchmark": {
        "description": "OKE 2015/16 Task 1 benchmark based on GERBIL",
        "id": "http://w3id.org/gerbil/hobbit/vocab#GerbilBenchmarkOKE2015T1",
        "name": "GERBIL Benchmark - OKE 2015/16 Task 1",
        "configurationParamNames": [
          "Type of experiment",
          "Dataset name"
        ],
        "configurationParams": [
          {
            "description": "Defines the type of experiment that should be carried out.",
            "id": "http://w3id.org/gerbil/hobbit/vocab#hasExperimentType",
            "name": "Type of experiment",
            "defaultValue": "http://w3id.org/gerbil/vocab#OKE2015_Task1",
            "feature": true,
            "options": [
              {
                "label": "http://w3id.org/gerbil/vocab#OKE2015_Task1",
                "value": "http://w3id.org/gerbil/vocab#OKE2015_Task1"
              }
            ],
            "range": "http://w3id.org/gerbil/vocab#ExperimentTypes",
            "required": false
          },
          {
            "description": "The dataset which should be tested.",
            "id": "http://w3id.org/gerbil/hobbit/vocab#hasDataset",
            "name": "Dataset name",
            "feature": true,
            "options": [
              {
                "label": "http://w3id.org/gerbil/vocab#OKE2015-1-example",
                "value": "http://w3id.org/gerbil/vocab#OKE2015-1-example"
              },
              {
                "label": "http://w3id.org/gerbil/vocab#OKE2017-1-eval",
                "value": "http://w3id.org/gerbil/vocab#OKE2017-1-eval"
              }
            ],
            "range": "http://w3id.org/gerbil/vocab#Datasets",
            "required": false
          }
        ],
        "systems": [
          {
            "description": "This is a system adapter that simply wraps DBpedia Spotlight.",
            "id": "http://w3id.org/hobbit/gerbil/test/SpotWrapNifWS4Test",
            "name": "SpotWrapNifWS4Test"
          },
          {
            "description": "This is a client for the FREME webservice.",
            "id": "http://api.freme-project.eu/current/e-entity/freme-ner/client/OKE-task1",
            "name": "FREME for OKE Task 1"
          }
        ]
      },
      "configurationParams": [
        {
          "description": "Defines the type of experiment that should be carried out.",
          "id": "http://w3id.org/gerbil/hobbit/vocab#hasExperimentType",
          "name": "Type of experiment",
          "range": "http://w3id.org/gerbil/vocab#ExperimentTypes",
          "value": "http://w3id.org/gerbil/vocab#OKE2015_Task1"
        },
        {
          "description": "The dataset which should be tested.",
          "id": "http://w3id.org/gerbil/hobbit/vocab#hasDataset",
          "name": "Dataset name",
          "range": "http://w3id.org/gerbil/vocab#Datasets",
          "value": "http://w3id.org/gerbil/vocab#OKE2017-1-eval"
        }
      ]
    },
    {
      "description": "This task extends the first task towards 12 additional entity classes in DBpedia. Beside the three types in the first task, a competing system might have to identify other types of entities. The dataset in this scenario has been manually created and the system receives only one document at once.",
      "id": "http://w3id.org/hobbit/challenges#7023e350-9376-4d30-8a1e-2bbf1b639f27_task2",
      "name": "OKE2017 Task 2 Scenario A",
      "benchmark": {
        "type": "benchmarkBean",
        "description": "RT2KB benchmark based on GERBIL",
        "id": "http://w3id.org/gerbil/hobbit/vocab#GerbilBenchmarkRT2KB",
        "name": "GERBIL Benchmark - RT2KB",
        "configurationParamNames": [
          "Type of experiment",
          "Dataset name"
        ],
        "configurationParams": [
          {
            "description": "Defines the type of experiment that should be carried out.",
            "id": "http://w3id.org/gerbil/hobbit/vocab#hasExperimentType",
            "name": "Type of experiment",
            "defaultValue": "http://w3id.org/gerbil/vocab#RT2KB",
            "feature": true,
            "options": [
              {
                "label": "http://w3id.org/gerbil/vocab#RT2KB",
                "value": "http://w3id.org/gerbil/vocab#RT2KB"
              }
            ],
            "range": "http://w3id.org/gerbil/vocab#ExperimentTypes",
            "required": false
          },
          {
            "description": "The dataset which should be tested.",
            "id": "http://w3id.org/gerbil/hobbit/vocab#hasDataset",
            "name": "Dataset name",
            "feature": true,
            "options": [
              {
                "label": "OKE 2017 Task 2 eval dataset",
                "value": "http://w3id.org/gerbil/vocab#OKE2017-2-eval"
              },
              {
                "label": "OKE 2017 Task 3 training dataset",
                "value": "http://w3id.org/gerbil/vocab#OKE2017-3-training"
              },
              {
                "label": "OKE 2017 Task 2 training dataset",
                "value": "http://w3id.org/gerbil/vocab#OKE2017-2-training"
              },
              {
                "label": "OKE 2015 Task 1 example set",
                "value": "http://w3id.org/gerbil/vocab#OKE2015-1-example"
              },
              {
                "label": "OKE 2016 Task 1 gold standard sample",
                "value": "http://w3id.org/gerbil/vocab#OKE2016-1-gold"
              },
              {
                "label": "OKE 2017 Task 3 eval dataset",
                "value": "http://w3id.org/gerbil/vocab#OKE2017-3-eval"
              },
              {
                "label": "UMBC-Train",
                "value": "http://w3id.org/gerbil/vocab#UMBC-Train"
              },
              {
                "label": "OKE 2015 Task 1 gold standard sample",
                "value": "http://w3id.org/gerbil/vocab#OKE2015-1-gold"
              },
              {
                "label": "UMBC-Test",
                "value": "http://w3id.org/gerbil/vocab#UMBC-Test"
              },
              {
                "label": "OKE 2015 Task 1 evaluation dataset",
                "value": "http://w3id.org/gerbil/vocab#OKE2015-1-eval"
              },
              {
                "label": "OKE 2017 Task 1 training dataset",
                "value": "http://w3id.org/gerbil/vocab#OKE2017-1-training"
              },
              {
                "label": "OKE 2016 Task 1 evaluation dataset",
                "value": "http://w3id.org/gerbil/vocab#OKE2016-1-eval"
              },
              {
                "label": "Ritter",
                "value": "http://w3id.org/gerbil/vocab#Ritter"
              },
              {
                "label": "OKE 2016 Task 1 example set",
                "value": "http://w3id.org/gerbil/vocab#OKE2016-1-example"
              },
              {
                "label": "OKE 2017 Task 1 eval dataset",
                "value": "http://w3id.org/gerbil/vocab#OKE2017-1-eval"
              }
            ],
            "range": "http://w3id.org/gerbil/vocab#Datasets",
            "required": false
          }
        ],
        "systems": []
      },
      "configurationParams": [
        {
          "id": "http://w3id.org/gerbil/hobbit/vocab#hasExperimentType",
          "name": "Type of experiment",
          "value": "http://w3id.org/gerbil/vocab#RT2KB",
          "description": "Defines the type of experiment that should be carried out.",
          "range": "http://w3id.org/gerbil/vocab#ExperimentTypes"
        },
        {
          "id": "http://w3id.org/gerbil/hobbit/vocab#hasDataset",
          "name": "Dataset name",
          "value": "http://w3id.org/gerbil/vocab#OKE2017-2-eval",
          "description": "The dataset which should be tested.",
          "range": "http://w3id.org/gerbil/vocab#Datasets"
        }
      ]
    },
    {
      "description": "The task is composed of two sub tasks focused musical NE recognition and linking. A competing system has to fulfill both tasks. The domain of this task is music, thus the used knowledge base will be MusicBrainz as Linked Data (MBL). The dataset has been automatically created and the system retrieves an increasing amount of requests over time.",
      "id": "http://w3id.org/hobbit/challenges#7023e350-9376-4d30-8a1e-2bbf1b639f27_task3",
      "name": "OKE2017 Task 3 Scenario A",
      "benchmark": {
        "description": "OKE 2015/16 Task 1 benchmark based on GERBIL",
        "id": "http://w3id.org/gerbil/hobbit/vocab#GerbilBenchmarkOKE2015T1",
        "name": "GERBIL Benchmark - OKE 2015/16 Task 1",
        "configurationParamNames": [
          "Type of experiment",
          "Dataset name"
        ],
        "configurationParams": [
          {
            "description": "Defines the type of experiment that should be carried out.",
            "id": "http://w3id.org/gerbil/hobbit/vocab#hasExperimentType",
            "name": "Type of experiment",
            "defaultValue": "http://w3id.org/gerbil/vocab#OKE2015_Task1",
            "feature": true,
            "options": [
              {
                "label": "http://w3id.org/gerbil/vocab#OKE2015_Task1",
                "value": "http://w3id.org/gerbil/vocab#OKE2015_Task1"
              }
            ],
            "range": "http://w3id.org/gerbil/vocab#ExperimentTypes",
            "required": false
          },
          {
            "description": "The dataset which should be tested.",
            "id": "http://w3id.org/gerbil/hobbit/vocab#hasDataset",
            "name": "Dataset name",
            "feature": true,
            "options": [
              {
                "label": "http://w3id.org/gerbil/vocab#OKE2015-1-example",
                "value": "http://w3id.org/gerbil/vocab#OKE2015-1-example"
              },
              {
                "label": "http://w3id.org/gerbil/vocab#OKE2017-1-eval",
                "value": "http://w3id.org/gerbil/vocab#OKE2017-1-eval"
              }
            ],
            "range": "http://w3id.org/gerbil/vocab#Datasets",
            "required": false
          }
        ],
        "systems": [
          {
            "id": "http://ns.aksw.org/foxInstance"
          }
        ]
      },
      "configurationParams": [
        {
          "description": "The dataset which should be tested.",
          "id": "http://w3id.org/gerbil/hobbit/vocab#hasDataset",
          "name": "Dataset name",
          "range": "http://w3id.org/gerbil/vocab#Datasets",
          "value": "http://w3id.org/gerbil/vocab#OKE2015-1-example"
        }
      ]
    },
    {
      "description": "Focused NE Identification and Linking with an automatically created dataset and an increasing amount of requests over time. The task comprises the identification of named entities in sentences and the disambiguation of the identified entities to the DBpedia knowledge base. This task is limited to three DBpedia ontology classes (Person, Place, Organisation) and their associated sub classes.",
      "id": "http://w3id.org/hobbit/challenges#7023e350-9376-4d30-8a1e-2bbf1b639f27_task4",
      "name": "OKE2017 Task 1 Scenario B",
      "benchmark": {
        "description": "OKE Task 1 Bengal benchmark based on GERBIL",
        "id": "http://w3id.org/gerbil/hobbit/vocab#GerbilBenchmarkBengalT1",
        "name": "GERBIL Benchmark - Bengal - OKE Task 1",
        "configurationParamNames": [
          "Type of experiment",
          "Number of Documents",
          "Number of Bengal Generators",
          "Seed for Bengal Generators",
          "Phases"
        ],
        "configurationParams": [
          {
            "description": "Defines the type of experiment that should be carried out.",
            "id": "http://w3id.org/gerbil/hobbit/vocab#hasExperimentType",
            "name": "Type of experiment",
            "defaultValue": "http://w3id.org/gerbil/vocab#OKE2015_Task1",
            "feature": true,
            "options": [
              {
                "label": "http://w3id.org/gerbil/vocab#OKE2015_Task1",
                "value": "http://w3id.org/gerbil/vocab#OKE2015_Task1"
              }
            ],
            "range": "http://w3id.org/gerbil/vocab#ExperimentTypes",
            "required": false
          },
          {
            "description": "The Number of Documents Bengal should generate.",
            "id": "http://w3id.org/gerbil/hobbit/vocab#hasNumberOfDocuments",
            "name": "Number of Documents",
            "datatype": "xsd:integer",
            "feature": true,
            "range": "http://www.w3.org/2001/XMLSchema#int",
            "required": false
          },
          {
            "description": "The Number of Bengal Generators",
            "id": "http://w3id.org/gerbil/hobbit/vocab#hasNumberOfGenerators",
            "name": "Number of Bengal Generators",
            "datatype": "xsd:integer",
            "feature": true,
            "range": "http://www.w3.org/2001/XMLSchema#int",
            "required": false
          },
          {
            "description": "The seed which should be used to ranomize the Bengal Generators",
            "id": "http://w3id.org/gerbil/hobbit/vocab#hasSeed",
            "name": "Seed for Bengal Generators",
            "datatype": "xsd:string",
            "feature": true,
            "range": "http://www.w3.org/2001/XMLSchema#string",
            "required": false
          },
          {
            "description": "The number of Phases Bengal should run.",
            "id": "http://w3id.org/gerbil/hobbit/vocab#hasPhases",
            "name": "Phases",
            "datatype": "xsd:integer",
            "feature": true,
            "range": "http://www.w3.org/2001/XMLSchema#int",
            "required": false
          }
        ],
        "systems": [
          {
            "description": "This is a system adapter that simply wraps DBpedia Spotlight.",
            "id": "http://w3id.org/hobbit/gerbil/test/SpotWrapNifWS4Test",
            "name": "SpotWrapNifWS4Test"
          },
          {
            "description": "This is a client for the FREME webservice.",
            "id": "http://api.freme-project.eu/current/e-entity/freme-ner/client/OKE-task1",
            "name": "FREME for OKE Task 1"
          }
        ]
      },
      "configurationParams": [
        {
          "description": "The Number of Bengal Generators",
          "id": "http://w3id.org/gerbil/hobbit/vocab#hasNumberOfGenerators",
          "name": "Number of Bengal Generators",
          "datatype": "xsd:integer",
          "range": "http://www.w3.org/2001/XMLSchema#int",
          "value": "4"
        },
        {
          "description": "Defines the type of experiment that should be carried out.",
          "id": "http://w3id.org/gerbil/hobbit/vocab#hasExperimentType",
          "name": "Type of experiment",
          "range": "http://w3id.org/gerbil/vocab#ExperimentTypes",
          "value": "http://w3id.org/gerbil/vocab#OKE2015_Task1"
        },
        {
          "description": "The seed which should be used to ranomize the Bengal Generators",
          "id": "http://w3id.org/gerbil/hobbit/vocab#hasSeed",
          "name": "Seed for Bengal Generators",
          "datatype": "xsd:string",
          "range": "http://www.w3.org/2001/XMLSchema#string",
          "value": "21"
        },
        {
          "description": "The Number of Documents Bengal should generate.",
          "id": "http://w3id.org/gerbil/hobbit/vocab#hasNumberOfDocuments",
          "name": "Number of Documents",
          "datatype": "xsd:integer",
          "range": "http://www.w3.org/2001/XMLSchema#int",
          "value": "20"
        },
        {
          "description": "The number of Phases Bengal should run.",
          "id": "http://w3id.org/gerbil/hobbit/vocab#hasPhases",
          "name": "Phases",
          "datatype": "xsd:integer",
          "range": "http://www.w3.org/2001/XMLSchema#int",
          "value": "7"
        }
      ]
    }
  ],
  "visible": true
}
Ennosigaeon commented 6 years ago

@MichaelRoeder Can this issue be closed? That behaviour wasn't reported with the new UI.

MichaelRoeder commented 6 years ago

I couldn't recreate the problem with the new UI.