robinhood / faust

Python Stream Processing
Other
6.7k stars 539 forks source link

SameNode exception raised by fast/app/router.py when reseting application state #193

Open hartmutbehrens opened 5 years ago

hartmutbehrens commented 5 years ago

Checklist

Steps to reproduce

App code:

import faust
import os

kafka_host = 'kafka://localhost'
if 'KAFKA_HOST' in os.environ:
    kafka_host = os.environ['KAFKA_HOST']

app = faust.App(
    'opm',
    autodiscover=True,
    broker=kafka_host,
    origin='opm',
    store='rocksdb://',  
    topic_partitions=4,
    version=71
)

def main() -> None:
    app.main()

Agent code:

import faust
from opm.app import app
from opm.pmdb_service.models import PMObjects, PMObjectsRecord, \
    PMObjectProperties, PMObjectPropertiesRecord, \
    PMObjectPropertyParams, PMObjectPropertyParamsRecord, \
    PMPlants, PMPlantsRecord, \
    PMPropertyTypes, PMPropertyTypesRecord

# Kafka PMDB source topics
plants_topic = app.topic('opm.pmdb.PMPlants', value_type=PMPlantsRecord)
objects_topic = app.topic('opm.pmdb.PMObjects', value_type=PMObjectsRecord)
params_topic = app.topic('opm.pmdb.PMObjectPropertyParams', value_type=PMObjectPropertyParamsRecord)
properties_topic = app.topic('opm.pmdb.PMObjectProperties', value_type=PMObjectPropertiesRecord)
property_types_topic = app.topic('opm.pmdb.PMPropertyTypes', value_type=PMPropertyTypesRecord)

plants = app.Table('plants', default=PMPlants)
# a table of object properties, keyed by object ID
objects = app.Table('objects', default=PMObjects)
# a table of object properties (sensors), keyed by object property (sensor) ID
properties = app.Table('properties', default=PMObjectProperties)
# a table of property types, mostly useful for looking up icons
property_types = app.Table('propertytypes', default=PMPropertyTypes)
# a table of sensor parameters, keyed by parameter ID
parameters = app.Table('params', default=PMObjectPropertyParams)
# a list of directly related child IDs of an object, keyed by object ID
object_children = app.Table('children', default=list)
# a list of sensor IDs, keyed by object ID
sensors = app.Table('sensors', default=list)
# a table of sensor parameter IDs, keyed by sensor ID
sensor_params = app.Table('sensorparams', default=list)

@app.agent(objects_topic)
async def process_objects(stream):
    obj: PMObjectsRecord
    async for obj in stream:
        oid = obj.payload.ID  # PMObjects.ID
        objects[oid] = obj.payload
        # make a list of child objects (NB: not sensors) associated with this object
        pid = obj.payload.ParentObject
        if pid:
            # get a reference to the current child ID list
            # creates a new one if it does not yet exist, due to default=list table argument above
            siblings = object_children[pid]
            siblings.append(oid)
            # important ! write it back
            object_children[pid] = siblings

@app.agent(plants_topic)
async def process_plants(stream):
    plant: PMPlantsRecord
    async for plant in stream:
        plants[plant.payload.ID] = plant.payload

@app.agent(property_types_topic)
async def process_property_types(stream):
    prop_types: PMPropertyTypesRecord
    async for prop_types in stream:
        property_types[prop_types.payload.ID] = prop_types.payload

@app.agent(properties_topic)
async def process_properties(stream):
    props: PMObjectPropertiesRecord
    async for props in stream:
        properties[props.payload.ID] = props.payload
        # make a list of sensors belonging to this object
        oid = props.payload.Object
        if oid:
            siblings = sensors[oid]
            siblings.append(props.payload.ID)
            sensors[oid] = siblings

@app.agent(params_topic)
async def process_params(stream):
    param: PMObjectPropertyParamsRecord
    async for param in stream:
        parameters[param.payload.ID] = param.payload
        # make a list of parameters associated with this sensor
        sensor_id = param.payload.Property
        if sensor_id:
            param_id_list = sensor_params[sensor_id]
            param_id_list.append(param.payload.ID)
            sensor_params[sensor_id] = param_id_list

webview code:

from anytree import AnyNode
from anytree.exporter import DictExporter
from opm.app import app
from opm.pmdb_service.agents import object_children, objects, parameters, plants, \
    properties, property_types, sensors, sensor_params

def add_property_types(prop_type: int, kwargs):
    kwargs['icon'] = property_types[prop_type].ICON
    kwargs['type'] = property_types[prop_type].Description

def get_sensor_parameters(sensor_id: int):
    kwargs = {}
    for prop_id in set(sensor_params[sensor_id]):
        kwargs[parameters[prop_id].Name] = parameters[prop_id].Value
    return kwargs

def get_object_properties(object_id: int):
    obj = objects[object_id]
    kwargs = {'object_id': object_id,
              'label': obj.Description,
              'name': obj.Name}

    add_property_types(obj.PropertyType, kwargs)
    return kwargs

def add_sensors_maybe(node: AnyNode, oid: int):
    for sid in set(sensors[oid]):
        kwargs = {'sensor_id': sid,
                  'label': properties[sid].Description,
                  'name': properties[sid].Name,
                  'units': properties[sid].EU,
                  'source': properties[sid].SourceName,
                  'priority': properties[sid].Priority}
        sensor_kwargs = get_sensor_parameters(sid)
        add_property_types(properties[sid].PropertyType, sensor_kwargs)
        sensor_node = AnyNode(parent=node, **kwargs, **sensor_kwargs)

def build_object_tree(node: AnyNode, oid: int):
    for child in set(object_children[oid]):
        kwargs = get_object_properties(child)

        child_node = AnyNode(parent=node, **kwargs)
        add_sensors_maybe(child_node, child)
        # recurse to add the children of the current child
        build_object_tree(child_node, child)

@app.page('/object/{web_id}')
@app.table_route(table=objects, match_info='web_id')
async def get_object(web, request, web_id):
    oid = int(web_id)  # PMObjects.ID
    if oid in objects:
        kwargs = get_object_properties(oid)
        root = AnyNode(**kwargs)

        if oid in object_children:
            build_object_tree(root, oid)

        exporter = DictExporter()
        return web.json({
            "children": exporter.export(root)
        })
    else:
        return web.json({
            oid: "No data for the object was found."
        })

Expected behavior

Actual behavior

Full traceback

┌ƒaµS† v1.2.1────────────────────────────────────────────────────────────────────┐
│ id        │ opm-v71                                                            │
│ transport │ kafka://s3kafka                                                    │
│ store     │ rocksdb:                                                           │
│ web       │ http://localhost:6066/                                             │
│ log       │ -stderr- (info)                                                    │
│ pid       │ 7317                                                               │
│ hostname  │ hartmutvrbox                                                       │
│ platform  │ CPython 3.6.6 (Linux x86_64)                                       │
│ drivers   │ aiokafka=0.4.18 aiohttp=3.4.4                                      │
│ datadir   │ /home/hartmut/intellisenzo_microservices/services/opm/opm-data     │
│ appdir    │ /home/hartmut/intellisenzo_microservices/services/opm/opm-data/v71 │
└───────────┴────────────────────────────────────────────────────────────────────┘
[2018-10-25 08:03:53,108: INFO]: [^Worker]: Starting...
[2018-10-25 08:03:53,110: INFO]: [^-App]: Starting...
[2018-10-25 08:03:53,110: INFO]: [^--MonitorService]: Starting...
[2018-10-25 08:03:53,110: INFO]: [^CacheBackend]: Starting...
[2018-10-25 08:03:53,110: INFO]: [^--WebService]: Starting...
[2018-10-25 08:03:53,113: INFO]: [^---ServerThread]: Starting...
[2018-10-25 08:03:53,113: INFO]: [^--Web]: Serving on http://localhost:6066/
[2018-10-25 08:03:53,114: INFO]: [^--Producer]: Starting...
[2018-10-25 08:03:53,155: INFO]: [^--Consumer]: Starting...
[2018-10-25 08:03:53,200: INFO]: [^--LeaderAssignor]: Starting...
[2018-10-25 08:03:53,200: INFO]: [^--Producer]: Creating topic opm-v71-__assignor-__leader
[2018-10-25 08:03:53,200: INFO]: [^--ReplyConsumer]: Starting...
[2018-10-25 08:03:53,201: INFO]: [^AgentManagerService]: Starting...
[2018-10-25 08:03:53,201: INFO]: [^--Agent: opm.pmdb_servi[.]process_objects]: Starting...
[2018-10-25 08:03:53,204: INFO]: [^---OneForOneSupervisor]: Starting...
[2018-10-25 08:03:53,205: INFO]: [^--Agent: opm.pmdb_servic[.]process_plants]: Starting...
[2018-10-25 08:03:53,205: INFO]: [^---OneForOneSupervisor]: Starting...
[2018-10-25 08:03:53,205: INFO]: [^--Agent: opm.pmd[.]process_property_types]: Starting...
[2018-10-25 08:03:53,206: INFO]: [^---OneForOneSupervisor]: Starting...
[2018-10-25 08:03:53,206: INFO]: [^--Agent: opm.pmdb_se[.]process_properties]: Starting...
[2018-10-25 08:03:53,207: INFO]: [^---OneForOneSupervisor]: Starting...
[2018-10-25 08:03:53,207: INFO]: [^--Agent: opm.pmdb_servic[.]process_params]: Starting...
[2018-10-25 08:03:53,207: INFO]: [^---OneForOneSupervisor]: Starting...
[2018-10-25 08:03:53,208: INFO]: [^--Conductor]: Starting...
[2018-10-25 08:03:53,208: INFO]: [^--TableManager]: Starting...
[2018-10-25 08:03:55,209: INFO]: [^--Producer]: Creating topic opm-v71-plants-changelog
[2018-10-25 08:03:55,210: INFO]: [^--Producer]: Creating topic opm-v71-propertytypes-changelog
[2018-10-25 08:03:55,210: INFO]: [^--Producer]: Creating topic opm-v71-properties-changelog
[2018-10-25 08:03:55,210: INFO]: [^--Producer]: Creating topic opm-v71-params-changelog
[2018-10-25 08:03:55,211: INFO]: [^--Producer]: Creating topic opm-v71-children-changelog
[2018-10-25 08:03:55,211: INFO]: [^--Producer]: Creating topic opm-v71-sensors-changelog
[2018-10-25 08:03:55,211: INFO]: [^--Producer]: Creating topic opm-v71-sensorparams-changelog
[2018-10-25 08:03:55,211: INFO]: [^--Producer]: Creating topic opm-v71-objects-changelog
[2018-10-25 08:03:55,212: INFO]: [^--Producer]: Creating topic opm-v71-__assignor-__leader
[2018-10-25 08:03:55,212: INFO]: Updating subscribed topics to: frozenset({'opm.pmdb.PMObjectPropertyParams', 'opm.pmdb.PMPlants', 'opm-v71-__assignor-__leader', 'opm-v71-sensors-changelog', 'opm-v71-sensorparams-changelog', 'opm-v71-children-changelog', 'opm.pmdb.PMObjectProperties', 'opm-v71-params-changelog', 'opm.pmdb.PMObjects', 'opm-v71-properties-changelog', 'opm.pmdb.PMPropertyTypes', 'opm-v71-objects-changelog', 'opm-v71-plants-changelog', 'opm-v71-propertytypes-changelog'})
[2018-10-25 08:03:55,214: INFO]: Subscribed to topic(s): {'opm.pmdb.PMObjectPropertyParams', 'opm.pmdb.PMPlants', 'opm-v71-__assignor-__leader', 'opm-v71-sensors-changelog', 'opm-v71-sensorparams-changelog', 'opm-v71-children-changelog', 'opm.pmdb.PMObjectProperties', 'opm-v71-params-changelog', 'opm.pmdb.PMObjects', 'opm-v71-properties-changelog', 'opm.pmdb.PMPropertyTypes', 'opm-v71-objects-changelog', 'opm-v71-plants-changelog', 'opm-v71-propertytypes-changelog'}
[2018-10-25 08:03:55,224: INFO]: Discovered coordinator 1001 for group opm-v71
[2018-10-25 08:03:55,224: INFO]: Revoking previously assigned partitions set() for group opm-v71
[2018-10-25 08:03:55,224: INFO]: [^--Fetcher]: Stopping...
[2018-10-25 08:03:55,225: INFO]: [^--Fetcher]: -Stopped!
[2018-10-25 08:03:55,327: INFO]: (Re-)joining group opm-v71
[2018-10-25 08:04:17,823: INFO]: Joined group 'opm-v71' (generation 6) with member_id faust-1.2.1-f89dc514-804c-4563-b47f-228b8d2b6aca
[2018-10-25 08:04:17,823: INFO]: Elected group leader -- performing partition assignments using faust
[2018-10-25 08:04:17,831: INFO]: Successfully synced group opm-v71 with generation 6
[2018-10-25 08:04:17,831: INFO]: Setting newly assigned partitions 
┌Topic Partition Set──────────────┬──────────────┐
│ topic                           │ partitions   │
├─────────────────────────────────┼──────────────┤
│ opm-v71-params-changelog        │ {0, 1, 2, 3} │
│ opm-v71-children-changelog      │ {0, 1, 2, 3} │
│ opm-v71-sensors-changelog       │ {0, 1, 2, 3} │
│ opm.pmdb.PMObjects              │ {0}          │
│ opm-v71-sensorparams-changelog  │ {0, 1, 2, 3} │
│ opm-v71-objects-changelog       │ {0, 1, 2, 3} │
│ opm-v71-plants-changelog        │ {0, 1, 2, 3} │
│ opm-v71-propertytypes-changelog │ {0, 1, 2, 3} │
│ opm-v71-properties-changelog    │ {0, 1, 2, 3} │
│ opm.pmdb.PMObjectPropertyParams │ {0}          │
│ opm.pmdb.PMObjectProperties     │ {0}          │
│ opm.pmdb.PMPropertyTypes        │ {0}          │
│ opm.pmdb.PMPlants               │ {0}          │
│ opm-v71-__assignor-__leader     │ {0}          │
└─────────────────────────────────┴──────────────┘ for group opm-v71
[2018-10-25 08:04:17,832: INFO]: [^--Fetcher]: Stopping...
[2018-10-25 08:04:17,832: INFO]: [^--Fetcher]: -Stopped!
[2018-10-25 08:04:17,832: INFO]: [^--TableManager]: Triggered recovery in background
[2018-10-25 08:04:17,832: INFO]: [^--TableManager]: New assignments found
[2018-10-25 08:04:19,140: INFO]: [^--TableManager]: Restoring state from changelog topics...
[2018-10-25 08:04:19,144: INFO]: [^---ChangelogReader: plants]: Starting...
[2018-10-25 08:04:19,151: INFO]: [^---ChangelogReader: plants]: Highwater for changelog partitions:
┌Highwater─────────────────┬───────────┬───────────┐
│ topic                    │ partition │ highwater │
├──────────────────────────┼───────────┼───────────┤
│ opm-v71-plants-changelog │ 2         │ -1        │
│ opm-v71-plants-changelog │ 1         │ -1        │
│ opm-v71-plants-changelog │ 3         │ -1        │
│ opm-v71-plants-changelog │ 0         │ 67        │
└──────────────────────────┴───────────┴───────────┘
[2018-10-25 08:04:19,162: INFO]: [^---ChangelogReader: plants]: Updated offsets at start of reading:
┌Reading Starts At─────────┬───────────┬────────┐
│ topic                    │ partition │ offset │
├──────────────────────────┼───────────┼────────┤
│ opm-v71-plants-changelog │ 0         │ 67     │
│ opm-v71-plants-changelog │ 2         │ -1     │
│ opm-v71-plants-changelog │ 1         │ -1     │
│ opm-v71-plants-changelog │ 3         │ -1     │
└──────────────────────────┴───────────┴────────┘
[2018-10-25 08:04:19,162: INFO]: [^---ChangelogReader: plants]: No updates needed
[2018-10-25 08:04:19,162: INFO]: [^---ChangelogReader: plants]: Setting stop event
[2018-10-25 08:04:19,163: INFO]: [^--TableManager]: Started restoring: ChangelogReader: plants
[2018-10-25 08:04:19,163: INFO]: [^---ChangelogReader: objects]: Starting...
[2018-10-25 08:04:19,784: INFO]: [^---ChangelogReader: objects]: Highwater for changelog partitions:
┌Highwater──────────────────┬───────────┬───────────┐
│ topic                     │ partition │ highwater │
├───────────────────────────┼───────────┼───────────┤
│ opm-v71-objects-changelog │ 1         │ -1        │
│ opm-v71-objects-changelog │ 2         │ -1        │
│ opm-v71-objects-changelog │ 0         │ 8911      │
│ opm-v71-objects-changelog │ 3         │ -1        │
└───────────────────────────┴───────────┴───────────┘
[2018-10-25 08:04:19,787: INFO]: [^---ChangelogReader: objects]: Updated offsets at start of reading:
┌Reading Starts At──────────┬───────────┬────────┐
│ topic                     │ partition │ offset │
├───────────────────────────┼───────────┼────────┤
│ opm-v71-objects-changelog │ 0         │ 8911   │
│ opm-v71-objects-changelog │ 1         │ -1     │
│ opm-v71-objects-changelog │ 2         │ -1     │
│ opm-v71-objects-changelog │ 3         │ -1     │
└───────────────────────────┴───────────┴────────┘
[2018-10-25 08:04:19,787: INFO]: [^---ChangelogReader: objects]: No updates needed
[2018-10-25 08:04:19,788: INFO]: [^---ChangelogReader: objects]: Setting stop event
[2018-10-25 08:04:19,788: INFO]: [^--TableManager]: Started restoring: ChangelogReader: objects
[2018-10-25 08:04:19,788: INFO]: [^---ChangelogReader: properties]: Starting...
[2018-10-25 08:04:19,789: INFO]: [^---ChangelogReader: properties]: Highwater for changelog partitions:
┌Highwater─────────────────────┬───────────┬───────────┐
│ topic                        │ partition │ highwater │
├──────────────────────────────┼───────────┼───────────┤
│ opm-v71-properties-changelog │ 0         │ 17607     │
│ opm-v71-properties-changelog │ 2         │ -1        │
│ opm-v71-properties-changelog │ 1         │ -1        │
│ opm-v71-properties-changelog │ 3         │ -1        │
└──────────────────────────────┴───────────┴───────────┘
[2018-10-25 08:04:19,790: INFO]: [^---ChangelogReader: properties]: Updated offsets at start of reading:
┌Reading Starts At─────────────┬───────────┬────────┐
│ topic                        │ partition │ offset │
├──────────────────────────────┼───────────┼────────┤
│ opm-v71-properties-changelog │ 0         │ 17607  │
│ opm-v71-properties-changelog │ 2         │ -1     │
│ opm-v71-properties-changelog │ 1         │ -1     │
│ opm-v71-properties-changelog │ 3         │ -1     │
└──────────────────────────────┴───────────┴────────┘
[2018-10-25 08:04:19,790: INFO]: [^---ChangelogReader: properties]: No updates needed
[2018-10-25 08:04:19,790: INFO]: [^---ChangelogReader: properties]: Setting stop event
[2018-10-25 08:04:19,790: INFO]: [^--TableManager]: Started restoring: ChangelogReader: properties
[2018-10-25 08:04:19,790: INFO]: [^---ChangelogReader: propertytypes]: Starting...
[2018-10-25 08:04:19,791: INFO]: [^---ChangelogReader: propertytypes]: Highwater for changelog partitions:
┌Highwater────────────────────────┬───────────┬───────────┐
│ topic                           │ partition │ highwater │
├─────────────────────────────────┼───────────┼───────────┤
│ opm-v71-propertytypes-changelog │ 1         │ -1        │
│ opm-v71-propertytypes-changelog │ 2         │ -1        │
│ opm-v71-propertytypes-changelog │ 0         │ 445       │
│ opm-v71-propertytypes-changelog │ 3         │ -1        │
└─────────────────────────────────┴───────────┴───────────┘
[2018-10-25 08:04:19,793: INFO]: [^---ChangelogReader: propertytypes]: Updated offsets at start of reading:
┌Reading Starts At────────────────┬───────────┬────────┐
│ topic                           │ partition │ offset │
├─────────────────────────────────┼───────────┼────────┤
│ opm-v71-propertytypes-changelog │ 0         │ 445    │
│ opm-v71-propertytypes-changelog │ 1         │ -1     │
│ opm-v71-propertytypes-changelog │ 2         │ -1     │
│ opm-v71-propertytypes-changelog │ 3         │ -1     │
└─────────────────────────────────┴───────────┴────────┘
[2018-10-25 08:04:19,793: INFO]: [^---ChangelogReader: propertytypes]: No updates needed
[2018-10-25 08:04:19,793: INFO]: [^---ChangelogReader: propertytypes]: Setting stop event
[2018-10-25 08:04:19,793: INFO]: [^--TableManager]: Started restoring: ChangelogReader: propertytypes
[2018-10-25 08:04:19,793: INFO]: [^---ChangelogReader: params]: Starting...
[2018-10-25 08:04:19,794: INFO]: [^---ChangelogReader: params]: Highwater for changelog partitions:
┌Highwater─────────────────┬───────────┬───────────┐
│ topic                    │ partition │ highwater │
├──────────────────────────┼───────────┼───────────┤
│ opm-v71-params-changelog │ 3         │ -1        │
│ opm-v71-params-changelog │ 1         │ -1        │
│ opm-v71-params-changelog │ 2         │ -1        │
│ opm-v71-params-changelog │ 0         │ 31140     │
└──────────────────────────┴───────────┴───────────┘
[2018-10-25 08:04:19,795: INFO]: [^---ChangelogReader: params]: Updated offsets at start of reading:
┌Reading Starts At─────────┬───────────┬────────┐
│ topic                    │ partition │ offset │
├──────────────────────────┼───────────┼────────┤
│ opm-v71-params-changelog │ 0         │ 31083  │
│ opm-v71-params-changelog │ 3         │ -1     │
│ opm-v71-params-changelog │ 1         │ -1     │
│ opm-v71-params-changelog │ 2         │ -1     │
└──────────────────────────┴───────────┴────────┘
[2018-10-25 08:04:19,795: INFO]: [^--TableManager]: Started restoring: ChangelogReader: params
[2018-10-25 08:04:19,795: INFO]: [^---ChangelogReader: children]: Starting...
[2018-10-25 08:04:19,795: INFO]: [^---ChangelogReader: params]: Reading 57 records...
[2018-10-25 08:04:19,796: INFO]: [^---ChangelogReader: params]: Still fetching. Remaining: {TP(topic='opm-v71-params-changelog', partition=0): (31140, 31083, 57)}
[2018-10-25 08:04:19,797: INFO]: [^---ChangelogReader: children]: Highwater for changelog partitions:
┌Highwater───────────────────┬───────────┬───────────┐
│ topic                      │ partition │ highwater │
├────────────────────────────┼───────────┼───────────┤
│ opm-v71-children-changelog │ 1         │ -1        │
│ opm-v71-children-changelog │ 3         │ -1        │
│ opm-v71-children-changelog │ 0         │ 8877      │
│ opm-v71-children-changelog │ 2         │ -1        │
└────────────────────────────┴───────────┴───────────┘
[2018-10-25 08:04:19,798: INFO]: [^---ChangelogReader: children]: Updated offsets at start of reading:
┌Reading Starts At───────────┬───────────┬────────┐
│ topic                      │ partition │ offset │
├────────────────────────────┼───────────┼────────┤
│ opm-v71-children-changelog │ 0         │ 8877   │
│ opm-v71-children-changelog │ 1         │ -1     │
│ opm-v71-children-changelog │ 3         │ -1     │
│ opm-v71-children-changelog │ 2         │ -1     │
└────────────────────────────┴───────────┴────────┘
[2018-10-25 08:04:19,798: INFO]: [^---ChangelogReader: children]: No updates needed
[2018-10-25 08:04:19,798: INFO]: [^---ChangelogReader: children]: Setting stop event
[2018-10-25 08:04:19,798: INFO]: [^--TableManager]: Started restoring: ChangelogReader: children
[2018-10-25 08:04:19,798: INFO]: [^---ChangelogReader: sensors]: Starting...
[2018-10-25 08:04:19,800: INFO]: [^---ChangelogReader: sensors]: Highwater for changelog partitions:
┌Highwater──────────────────┬───────────┬───────────┐
│ topic                     │ partition │ highwater │
├───────────────────────────┼───────────┼───────────┤
│ opm-v71-sensors-changelog │ 3         │ -1        │
│ opm-v71-sensors-changelog │ 1         │ -1        │
│ opm-v71-sensors-changelog │ 2         │ -1        │
│ opm-v71-sensors-changelog │ 0         │ 17559     │
└───────────────────────────┴───────────┴───────────┘
[2018-10-25 08:04:19,802: INFO]: [^---ChangelogReader: sensors]: Updated offsets at start of reading:
┌Reading Starts At──────────┬───────────┬────────┐
│ topic                     │ partition │ offset │
├───────────────────────────┼───────────┼────────┤
│ opm-v71-sensors-changelog │ 0         │ 17559  │
│ opm-v71-sensors-changelog │ 3         │ -1     │
│ opm-v71-sensors-changelog │ 1         │ -1     │
│ opm-v71-sensors-changelog │ 2         │ -1     │
└───────────────────────────┴───────────┴────────┘
[2018-10-25 08:04:19,803: INFO]: [^---ChangelogReader: sensors]: No updates needed
[2018-10-25 08:04:19,803: INFO]: [^---ChangelogReader: sensors]: Setting stop event
[2018-10-25 08:04:19,803: INFO]: [^--TableManager]: Started restoring: ChangelogReader: sensors
[2018-10-25 08:04:19,803: INFO]: [^---ChangelogReader: sensorparams]: Starting...
[2018-10-25 08:04:19,804: INFO]: [^---ChangelogReader: sensorparams]: Highwater for changelog partitions:
┌Highwater───────────────────────┬───────────┬───────────┐
│ topic                          │ partition │ highwater │
├────────────────────────────────┼───────────┼───────────┤
│ opm-v71-sensorparams-changelog │ 3         │ -1        │
│ opm-v71-sensorparams-changelog │ 1         │ -1        │
│ opm-v71-sensorparams-changelog │ 2         │ -1        │
│ opm-v71-sensorparams-changelog │ 0         │ 31140     │
└────────────────────────────────┴───────────┴───────────┘
[2018-10-25 08:04:19,805: INFO]: [^---ChangelogReader: sensorparams]: Updated offsets at start of reading:
┌Reading Starts At───────────────┬───────────┬────────┐
│ topic                          │ partition │ offset │
├────────────────────────────────┼───────────┼────────┤
│ opm-v71-sensorparams-changelog │ 0         │ 31083  │
│ opm-v71-sensorparams-changelog │ 3         │ -1     │
│ opm-v71-sensorparams-changelog │ 1         │ -1     │
│ opm-v71-sensorparams-changelog │ 2         │ -1     │
└────────────────────────────────┴───────────┴────────┘
[2018-10-25 08:04:19,805: INFO]: [^--TableManager]: Started restoring: ChangelogReader: sensorparams
[2018-10-25 08:04:19,805: INFO]: [^--Fetcher]: Starting...
[2018-10-25 08:04:19,805: INFO]: [^--TableManager]: Waiting for restore to finish...
[2018-10-25 08:04:19,806: INFO]: [^---ChangelogReader: sensorparams]: Reading 57 records...
[2018-10-25 08:04:19,806: INFO]: [^---ChangelogReader: sensorparams]: Still fetching. Remaining: {TP(topic='opm-v71-sensorparams-changelog', partition=0): (31140, 31083, 57)}
[2018-10-25 08:04:19,909: INFO]: [^---ChangelogReader: params]: Waiting for 56 records...
[2018-10-25 08:04:19,912: INFO]: [^---ChangelogReader: params]: Stopped reading!
[2018-10-25 08:04:19,928: INFO]: [^---ChangelogReader: params]: Setting stop event
[2018-10-25 08:04:19,928: INFO]: [^---ChangelogReader: sensorparams]: Waiting for 56 records...
[2018-10-25 08:04:19,932: INFO]: [^---ChangelogReader: sensorparams]: Stopped reading!
[2018-10-25 08:04:19,934: INFO]: [^---ChangelogReader: sensorparams]: Setting stop event
[2018-10-25 08:04:19,935: INFO]: [^--TableManager]: Done reading all changelogs
[2018-10-25 08:04:19,936: INFO]: [^--TableManager]: Syncing offsets:
┌Sync Offset───────────────┬───────────┬────────┐
│ topic                    │ partition │ offset │
├──────────────────────────┼───────────┼────────┤
│ opm-v71-plants-changelog │ 0         │ 67     │
│ opm-v71-plants-changelog │ 2         │ -1     │
│ opm-v71-plants-changelog │ 1         │ -1     │
│ opm-v71-plants-changelog │ 3         │ -1     │
└──────────────────────────┴───────────┴────────┘
[2018-10-25 08:04:19,937: INFO]: [^--TableManager]: After syncing:
┌Table Offsets────────────────────┬───────────┬────────┐
│ topic                           │ partition │ offset │
├─────────────────────────────────┼───────────┼────────┤
│ opm-v71-plants-changelog        │ 0         │ 67     │
│ opm-v71-objects-changelog       │ 0         │ 8911   │
│ opm-v71-properties-changelog    │ 0         │ 17607  │
│ opm-v71-propertytypes-changelog │ 0         │ 445    │
│ opm-v71-params-changelog        │ 0         │ 31083  │
│ opm-v71-children-changelog      │ 0         │ 8877   │
│ opm-v71-sensors-changelog       │ 0         │ 17559  │
│ opm-v71-sensorparams-changelog  │ 0         │ 31083  │
└─────────────────────────────────┴───────────┴────────┘
[2018-10-25 08:04:19,937: INFO]: [^--TableManager]: Syncing offsets:
┌Sync Offset────────────────┬───────────┬────────┐
│ topic                     │ partition │ offset │
├───────────────────────────┼───────────┼────────┤
│ opm-v71-objects-changelog │ 0         │ 8911   │
│ opm-v71-objects-changelog │ 1         │ -1     │
│ opm-v71-objects-changelog │ 2         │ -1     │
│ opm-v71-objects-changelog │ 3         │ -1     │
└───────────────────────────┴───────────┴────────┘
[2018-10-25 08:04:19,937: INFO]: [^--TableManager]: After syncing:
┌Table Offsets────────────────────┬───────────┬────────┐
│ topic                           │ partition │ offset │
├─────────────────────────────────┼───────────┼────────┤
│ opm-v71-plants-changelog        │ 0         │ 67     │
│ opm-v71-objects-changelog       │ 0         │ 8911   │
│ opm-v71-properties-changelog    │ 0         │ 17607  │
│ opm-v71-propertytypes-changelog │ 0         │ 445    │
│ opm-v71-params-changelog        │ 0         │ 31083  │
│ opm-v71-children-changelog      │ 0         │ 8877   │
│ opm-v71-sensors-changelog       │ 0         │ 17559  │
│ opm-v71-sensorparams-changelog  │ 0         │ 31083  │
└─────────────────────────────────┴───────────┴────────┘
[2018-10-25 08:04:19,938: INFO]: [^--TableManager]: Syncing offsets:
┌Sync Offset───────────────────┬───────────┬────────┐
│ topic                        │ partition │ offset │
├──────────────────────────────┼───────────┼────────┤
│ opm-v71-properties-changelog │ 0         │ 17607  │
│ opm-v71-properties-changelog │ 2         │ -1     │
│ opm-v71-properties-changelog │ 1         │ -1     │
│ opm-v71-properties-changelog │ 3         │ -1     │
└──────────────────────────────┴───────────┴────────┘
[2018-10-25 08:04:19,938: INFO]: [^--TableManager]: After syncing:
┌Table Offsets────────────────────┬───────────┬────────┐
│ topic                           │ partition │ offset │
├─────────────────────────────────┼───────────┼────────┤
│ opm-v71-plants-changelog        │ 0         │ 67     │
│ opm-v71-objects-changelog       │ 0         │ 8911   │
│ opm-v71-properties-changelog    │ 0         │ 17607  │
│ opm-v71-propertytypes-changelog │ 0         │ 445    │
│ opm-v71-params-changelog        │ 0         │ 31083  │
│ opm-v71-children-changelog      │ 0         │ 8877   │
│ opm-v71-sensors-changelog       │ 0         │ 17559  │
│ opm-v71-sensorparams-changelog  │ 0         │ 31083  │
└─────────────────────────────────┴───────────┴────────┘
[2018-10-25 08:04:19,939: INFO]: [^--TableManager]: Syncing offsets:
┌Sync Offset──────────────────────┬───────────┬────────┐
│ topic                           │ partition │ offset │
├─────────────────────────────────┼───────────┼────────┤
│ opm-v71-propertytypes-changelog │ 0         │ 445    │
│ opm-v71-propertytypes-changelog │ 1         │ -1     │
│ opm-v71-propertytypes-changelog │ 2         │ -1     │
│ opm-v71-propertytypes-changelog │ 3         │ -1     │
└─────────────────────────────────┴───────────┴────────┘
[2018-10-25 08:04:19,939: INFO]: [^--TableManager]: After syncing:
┌Table Offsets────────────────────┬───────────┬────────┐
│ topic                           │ partition │ offset │
├─────────────────────────────────┼───────────┼────────┤
│ opm-v71-plants-changelog        │ 0         │ 67     │
│ opm-v71-objects-changelog       │ 0         │ 8911   │
│ opm-v71-properties-changelog    │ 0         │ 17607  │
│ opm-v71-propertytypes-changelog │ 0         │ 445    │
│ opm-v71-params-changelog        │ 0         │ 31083  │
│ opm-v71-children-changelog      │ 0         │ 8877   │
│ opm-v71-sensors-changelog       │ 0         │ 17559  │
│ opm-v71-sensorparams-changelog  │ 0         │ 31083  │
└─────────────────────────────────┴───────────┴────────┘
[2018-10-25 08:04:19,940: INFO]: [^--TableManager]: Syncing offsets:
┌Sync Offset───────────────┬───────────┬────────┐
│ topic                    │ partition │ offset │
├──────────────────────────┼───────────┼────────┤
│ opm-v71-params-changelog │ 0         │ 31140  │
│ opm-v71-params-changelog │ 3         │ -1     │
│ opm-v71-params-changelog │ 1         │ -1     │
│ opm-v71-params-changelog │ 2         │ -1     │
└──────────────────────────┴───────────┴────────┘
[2018-10-25 08:04:19,940: INFO]: [^--TableManager]: After syncing:
┌Table Offsets────────────────────┬───────────┬────────┐
│ topic                           │ partition │ offset │
├─────────────────────────────────┼───────────┼────────┤
│ opm-v71-plants-changelog        │ 0         │ 67     │
│ opm-v71-objects-changelog       │ 0         │ 8911   │
│ opm-v71-properties-changelog    │ 0         │ 17607  │
│ opm-v71-propertytypes-changelog │ 0         │ 445    │
│ opm-v71-params-changelog        │ 0         │ 31140  │
│ opm-v71-children-changelog      │ 0         │ 8877   │
│ opm-v71-sensors-changelog       │ 0         │ 17559  │
│ opm-v71-sensorparams-changelog  │ 0         │ 31083  │
└─────────────────────────────────┴───────────┴────────┘
[2018-10-25 08:04:19,941: INFO]: [^--TableManager]: Syncing offsets:
┌Sync Offset─────────────────┬───────────┬────────┐
│ topic                      │ partition │ offset │
├────────────────────────────┼───────────┼────────┤
│ opm-v71-children-changelog │ 0         │ 8877   │
│ opm-v71-children-changelog │ 1         │ -1     │
│ opm-v71-children-changelog │ 3         │ -1     │
│ opm-v71-children-changelog │ 2         │ -1     │
└────────────────────────────┴───────────┴────────┘
[2018-10-25 08:04:19,942: INFO]: [^--TableManager]: After syncing:
┌Table Offsets────────────────────┬───────────┬────────┐
│ topic                           │ partition │ offset │
├─────────────────────────────────┼───────────┼────────┤
│ opm-v71-plants-changelog        │ 0         │ 67     │
│ opm-v71-objects-changelog       │ 0         │ 8911   │
│ opm-v71-properties-changelog    │ 0         │ 17607  │
│ opm-v71-propertytypes-changelog │ 0         │ 445    │
│ opm-v71-params-changelog        │ 0         │ 31140  │
│ opm-v71-children-changelog      │ 0         │ 8877   │
│ opm-v71-sensors-changelog       │ 0         │ 17559  │
│ opm-v71-sensorparams-changelog  │ 0         │ 31083  │
└─────────────────────────────────┴───────────┴────────┘
[2018-10-25 08:04:19,942: INFO]: [^--TableManager]: Syncing offsets:
┌Sync Offset────────────────┬───────────┬────────┐
│ topic                     │ partition │ offset │
├───────────────────────────┼───────────┼────────┤
│ opm-v71-sensors-changelog │ 0         │ 17559  │
│ opm-v71-sensors-changelog │ 3         │ -1     │
│ opm-v71-sensors-changelog │ 1         │ -1     │
│ opm-v71-sensors-changelog │ 2         │ -1     │
└───────────────────────────┴───────────┴────────┘
[2018-10-25 08:04:19,942: INFO]: [^--TableManager]: After syncing:
┌Table Offsets────────────────────┬───────────┬────────┐
│ topic                           │ partition │ offset │
├─────────────────────────────────┼───────────┼────────┤
│ opm-v71-plants-changelog        │ 0         │ 67     │
│ opm-v71-objects-changelog       │ 0         │ 8911   │
│ opm-v71-properties-changelog    │ 0         │ 17607  │
│ opm-v71-propertytypes-changelog │ 0         │ 445    │
│ opm-v71-params-changelog        │ 0         │ 31140  │
│ opm-v71-children-changelog      │ 0         │ 8877   │
│ opm-v71-sensors-changelog       │ 0         │ 17559  │
│ opm-v71-sensorparams-changelog  │ 0         │ 31083  │
└─────────────────────────────────┴───────────┴────────┘
[2018-10-25 08:04:19,943: INFO]: [^--TableManager]: Syncing offsets:
┌Sync Offset─────────────────────┬───────────┬────────┐
│ topic                          │ partition │ offset │
├────────────────────────────────┼───────────┼────────┤
│ opm-v71-sensorparams-changelog │ 0         │ 31140  │
│ opm-v71-sensorparams-changelog │ 3         │ -1     │
│ opm-v71-sensorparams-changelog │ 1         │ -1     │
│ opm-v71-sensorparams-changelog │ 2         │ -1     │
└────────────────────────────────┴───────────┴────────┘
[2018-10-25 08:04:19,943: INFO]: [^--TableManager]: After syncing:
┌Table Offsets────────────────────┬───────────┬────────┐
│ topic                           │ partition │ offset │
├─────────────────────────────────┼───────────┼────────┤
│ opm-v71-plants-changelog        │ 0         │ 67     │
│ opm-v71-objects-changelog       │ 0         │ 8911   │
│ opm-v71-properties-changelog    │ 0         │ 17607  │
│ opm-v71-propertytypes-changelog │ 0         │ 445    │
│ opm-v71-params-changelog        │ 0         │ 31140  │
│ opm-v71-children-changelog      │ 0         │ 8877   │
│ opm-v71-sensors-changelog       │ 0         │ 17559  │
│ opm-v71-sensorparams-changelog  │ 0         │ 31140  │
└─────────────────────────────────┴───────────┴────────┘
[2018-10-25 08:04:19,943: INFO]: [^--TableManager]: Done reading from changelog topics
[2018-10-25 08:04:19,943: INFO]: [^--Fetcher]: Stopping...
[2018-10-25 08:04:19,944: INFO]: [^--Fetcher]: -Stopped!
[2018-10-25 08:04:19,944: INFO]: [^---ChangelogReader: plants]: Stopping...
[2018-10-25 08:04:19,944: INFO]: [^---ChangelogReader: plants]: -Stopped!
[2018-10-25 08:04:19,944: INFO]: [^--TableManager]: Stopped restoring: ChangelogReader: plants
[2018-10-25 08:04:19,944: INFO]: [^---ChangelogReader: objects]: Stopping...
[2018-10-25 08:04:19,944: INFO]: [^---ChangelogReader: objects]: -Stopped!
[2018-10-25 08:04:19,944: INFO]: [^--TableManager]: Stopped restoring: ChangelogReader: objects
[2018-10-25 08:04:19,944: INFO]: [^---ChangelogReader: properties]: Stopping...
[2018-10-25 08:04:19,944: INFO]: [^---ChangelogReader: properties]: -Stopped!
[2018-10-25 08:04:19,944: INFO]: [^--TableManager]: Stopped restoring: ChangelogReader: properties
[2018-10-25 08:04:19,944: INFO]: [^---ChangelogReader: propertytypes]: Stopping...
[2018-10-25 08:04:19,945: INFO]: [^---ChangelogReader: propertytypes]: -Stopped!
[2018-10-25 08:04:19,945: INFO]: [^--TableManager]: Stopped restoring: ChangelogReader: propertytypes
[2018-10-25 08:04:19,945: INFO]: [^---ChangelogReader: params]: Stopping...
[2018-10-25 08:04:19,945: INFO]: [^---ChangelogReader: params]: -Stopped!
[2018-10-25 08:04:19,945: INFO]: [^--TableManager]: Stopped restoring: ChangelogReader: params
[2018-10-25 08:04:19,945: INFO]: [^---ChangelogReader: children]: Stopping...
[2018-10-25 08:04:19,945: INFO]: [^---ChangelogReader: children]: -Stopped!
[2018-10-25 08:04:19,945: INFO]: [^--TableManager]: Stopped restoring: ChangelogReader: children
[2018-10-25 08:04:19,945: INFO]: [^---ChangelogReader: sensors]: Stopping...
[2018-10-25 08:04:19,946: INFO]: [^---ChangelogReader: sensors]: -Stopped!
[2018-10-25 08:04:19,946: INFO]: [^--TableManager]: Stopped restoring: ChangelogReader: sensors
[2018-10-25 08:04:19,946: INFO]: [^---ChangelogReader: sensorparams]: Stopping...
[2018-10-25 08:04:19,946: INFO]: [^---ChangelogReader: sensorparams]: -Stopped!
[2018-10-25 08:04:19,946: INFO]: [^--TableManager]: Stopped restoring: ChangelogReader: sensorparams
[2018-10-25 08:04:19,946: INFO]: [^--TableManager]: Stopped restoring
[2018-10-25 08:04:19,946: INFO]: [^--TableManager]: Restore complete!
[2018-10-25 08:04:19,947: INFO]: [^--TableManager]: Attempting to start standbys
[2018-10-25 08:04:19,947: INFO]: [^--TableManager]: New assignments handled
[2018-10-25 08:04:19,947: INFO]: [^--Table: plants]: Starting...
[2018-10-25 08:04:19,947: INFO]: [^--Producer]: Creating topic opm-v71-plants-changelog
[2018-10-25 08:04:19,947: INFO]: [^---Store: plants]: Starting...
[2018-10-25 08:04:19,947: INFO]: [^--Table: objects]: Starting...
[2018-10-25 08:04:19,947: INFO]: [^--Producer]: Creating topic opm-v71-objects-changelog
[2018-10-25 08:04:19,948: INFO]: [^---Store: objects]: Starting...
[2018-10-25 08:04:19,948: INFO]: [^--Table: properties]: Starting...
[2018-10-25 08:04:19,948: INFO]: [^--Producer]: Creating topic opm-v71-properties-changelog
[2018-10-25 08:04:19,948: INFO]: [^---Store: properties]: Starting...
[2018-10-25 08:04:19,948: INFO]: [^--Table: propertytypes]: Starting...
[2018-10-25 08:04:19,948: INFO]: [^--Producer]: Creating topic opm-v71-propertytypes-changelog
[2018-10-25 08:04:19,948: INFO]: [^---Store: propertytypes]: Starting...
[2018-10-25 08:04:19,948: INFO]: [^--Table: params]: Starting...
[2018-10-25 08:04:19,948: INFO]: [^--Producer]: Creating topic opm-v71-params-changelog
[2018-10-25 08:04:19,948: INFO]: [^---Store: params]: Starting...
[2018-10-25 08:04:19,948: INFO]: [^--Table: children]: Starting...
[2018-10-25 08:04:19,949: INFO]: [^--Producer]: Creating topic opm-v71-children-changelog
[2018-10-25 08:04:19,949: INFO]: [^---Store: children]: Starting...
[2018-10-25 08:04:19,949: INFO]: [^--Table: sensors]: Starting...
[2018-10-25 08:04:19,949: INFO]: [^--Producer]: Creating topic opm-v71-sensors-changelog
[2018-10-25 08:04:19,949: INFO]: [^---Store: sensors]: Starting...
[2018-10-25 08:04:19,949: INFO]: [^--Table: sensorparams]: Starting...
[2018-10-25 08:04:19,949: INFO]: [^--Producer]: Creating topic opm-v71-sensorparams-changelog
[2018-10-25 08:04:19,949: INFO]: [^---Store: sensorparams]: Starting...
[2018-10-25 08:04:19,949: INFO]: [^--Fetcher]: Starting...
[2018-10-25 08:04:19,949: INFO]: [^--TableManager]: Worker ready
[2018-10-25 08:04:19,950: INFO]: [^Worker]: Ready
[2018-10-25 08:08:17,622: ERROR]: Error handling request
Traceback (most recent call last):
  File "/home/hartmut/.local/share/virtualenvs/opm-JvQpWqh_/lib/python3.6/site-packages/faust/app/base.py", line 791, in get
    view.web, request)
  File "/home/hartmut/.local/share/virtualenvs/opm-JvQpWqh_/lib/python3.6/site-packages/faust/app/router.py", line 53, in route_req
    raise SameNode()
faust.exceptions.SameNode

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/hartmut/.local/share/virtualenvs/opm-JvQpWqh_/lib/python3.6/site-packages/aiohttp/web_protocol.py", line 390, in start
    resp = await self._request_handler(request)
  File "/home/hartmut/.local/share/virtualenvs/opm-JvQpWqh_/lib/python3.6/site-packages/aiohttp/web_app.py", line 366, in _handle
    resp = await handler(request)
  File "/home/hartmut/.local/share/virtualenvs/opm-JvQpWqh_/lib/python3.6/site-packages/aiohttp/web_urldispatcher.py", line 120, in handler_wrapper
    result = await result
  File "/home/hartmut/.local/share/virtualenvs/opm-JvQpWqh_/lib/python3.6/site-packages/faust/web/views.py", line 66, in __call__
    return await self.dispatch(request)
  File "/home/hartmut/.local/share/virtualenvs/opm-JvQpWqh_/lib/python3.6/site-packages/faust/web/views.py", line 77, in dispatch
    return await method(cast(Request, request), **kwargs)
  File "/home/hartmut/.local/share/virtualenvs/opm-JvQpWqh_/lib/python3.6/site-packages/faust/app/base.py", line 793, in get
    return await fun(view, request, *args, **kwargs)
  File "/home/hartmut/intellisenzo_microservices/services/opm/opm/pmdb_service/views.py", line 64, in get_object
    build_object_tree(root, oid)
  File "/home/hartmut/intellisenzo_microservices/services/opm/opm/pmdb_service/views.py", line 47, in build_object_tree
    kwargs = get_object_properties(child)
  File "/home/hartmut/intellisenzo_microservices/services/opm/opm/pmdb_service/views.py", line 26, in get_object_properties
    add_property_types(obj.PropertyType, kwargs)
  File "/home/hartmut/intellisenzo_microservices/services/opm/opm/pmdb_service/views.py", line 9, in add_property_types
    kwargs['icon'] = property_types[prop_type].ICON
  File "/home/hartmut/.local/share/virtualenvs/opm-JvQpWqh_/lib/python3.6/site-packages/mode/utils/collections.py", line 337, in __getitem__
    return super().__getitem__(key)
  File "/home/hartmut/.local/share/virtualenvs/opm-JvQpWqh_/lib/python3.6/site-packages/mode/utils/collections.py", line 64, in __getitem__
    return self.__missing__(key)  # type: ignore
  File "/home/hartmut/.local/share/virtualenvs/opm-JvQpWqh_/lib/python3.6/site-packages/faust/tables/table.py", line 42, in __missing__
    return self.default()
TypeError: __init__() missing 5 required positional arguments: 'ID', 'Revision', 'Description', 'G2Name', and 'ICON'
[2018-10-25 08:08:17,623: INFO]: 127.0.0.1 [25/Oct/2018:06:08:17 +0000] "GET /object/1480 HTTP/1.1" 500 330 "-" "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:62.0) Gecko/20100101 Firefox/62.0"
[2018-10-25 08:09:22,417: INFO]: 127.0.0.1 [25/Oct/2018:06:09:22 +0000] "GET /favicon.ico HTTP/1.1" 404 172 "-" "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:63.0) Gecko/20100101 Firefox/63.0"

The TypeError on the last line referers to the type of property_types table (referenced in views.py) and is show below:

class PMPropertyTypes(faust.Record, serializer='json'):
    ID: int
    Revision: bytes
    Description: str
    G2Name: str
    ICON: str

Versions

GumpSun commented 2 years ago

How did you solve this problem?