Cannot parse getEntries data buffer

mor-cloudwize commented 3 years ago

We're trying to get entries for some logs, when we fetch entries it seems that the data is represented by some buffer that we cant convert, we guess that it is related somehow to the proto files or it might be that the buffer is corrupt?

async function listLogEntries(logName) {
    const { Logging } = require('@google-cloud/logging');
    const logging = new Logging({ projectId: projectId, credentials: cred });
    const log = logging.log(logName);

    async function printEntryMetadata() {
        const [entries] = await log.getEntries({
            filter: 'logName="SOME_LOG_NAME"', pageSize: 5
        });
        console.log('Logs:');
        entries.forEach(entry => {
           console.log(entry.data.value;) // Buffer
        });
    }
    printEntryMetadata();
}

0xSage commented 3 years ago

Hi @mor-cloudwize can you try the syntax from this sample instead?

// to get data property (its a vector)
logging.getEntries().then(data => {
  const entries = data[0];
});

// to get metadata payload (its another map)
entries.forEach(entry => {
    const metadata = entry.metadata;
    console.log(`${metadata.timestamp}:`, metadata[metadata.payload]);
  });

If not, try to stringify the value and let me know what you get.

And lastly, if the above do not work, please share your entire snippet including writing the log, so that I can reproduce and help you debug.

mor-cloudwize commented 3 years ago

Hi, thanks for the reply, I still can't get a valid value from the payload:

const logName = 'cloudaudit.googleapis.com%2Factivity';
async function main(scope, query, assetTypes, pageSize, pageToken, orderBy) {
    // Imports the Google Cloud client library
    const { Logging } = require('@google-cloud/logging');

    // Creates a client
    const logging = new Logging({ projectId, credentials: cred });
    const logName = 'cloudaudit.googleapis.com%2Factivity';
    const log = logging.log(logName);

    async function printEntryMetadata() {
        log.getEntries().then(data => {
            return data[0]
        }).then((entries) => {
            console.log('Logs:');
            entries.forEach(entry => {
                const metadata = entry.metadata;
                console.log(`${metadata.timestamp}:`, metadata[metadata.payload].value.toString());
            });
        });
    }
    printEntryMetadata();
}

main(...process.argv.slice(2)).catch(err => {
    console.error(err.message);
    process.exitCode = 1;
})

Im adding one of the results printed in console:


2billing-export-bigquery@system.gserviceaccount.com":B:bigquery.googleapis.comB1google.cloud.bigquery.v2.TableService.UpdateTableJ�
fprojects/<<project-id>>/datasets/<<company-name>>_billing/tables/gcp_billing_export_v1_0170A2_15340F_FDCB33bigquery.tables.update*Zfprojects/<<project-id>>/datasets/<<company-name>>_billing/tables/gcp_billing_export_v1_0170A2_15340F_FDCB33��)
G
@type><type.googleapis.com/google.cloud.audit.BigQueryAuditMetadata
�(

tableChange�(*�(
�(
table�'*�'
�&

schemaJson�&�&{
  "fields": [{
    "name": "billing_account_id",
    "type": "STRING",
    "mode": "NULLABLE"
  }, {
    "name": "service",
    "type": "RECORD",
    "mode": "NULLABLE",
    "schema": {
      "fields": [{
        "name": "id",
        "type": "STRING",
        "mode": "NULLABLE"
      }, {
        "name": "description",
        "type": "STRING",
        "mode": "NULLABLE"
      }]
    }
  }, {
    "name": "sku",
    "type": "RECORD",
    "mode": "NULLABLE",
    "schema": {
      "fields": [{
        "name": "id",
        "type": "STRING",
        "mode": "NULLABLE"
      }, {
        "name": "description",
        "type": "STRING",
        "mode": "NULLABLE"
      }]
    }
  }, {
    "name": "usage_start_time",
    "type": "TIMESTAMP",
    "mode": "NULLABLE"
  }, {
    "name": "usage_end_time",
    "type": "TIMESTAMP",
    "mode": "NULLABLE"
  }, {
    "name": "project",
    "type": "RECORD",
    "mode": "NULLABLE",
    "schema": {
      "fields": [{
        "name": "id",
        "type": "STRING",
        "mode": "NULLABLE"
      }, {
        "name": "number",
        "type": "STRING",
        "mode": "NULLABLE"
      }, {
        "name": "name",
        "type": "STRING",
        "mode": "NULLABLE"
      }, {
        "name": "labels",
        "type": "RECORD",
        "mode": "REPEATED",
        "schema": {
          "fields": [{
            "name": "key",
            "type": "STRING",
            "mode": "NULLABLE"
          }, {
            "name": "value",
            "type": "STRING",
            "mode": "NULLABLE"
          }]
        }
      }, {
        "name": "ancestry_numbers",
        "type": "STRING",
        "mode": "NULLABLE"
      }]
    }
  }, {
    "name": "labels",
    "type": "RECORD",
    "mode": "REPEATED",
    "schema": {
      "fields": [{
        "name": "key",
        "type": "STRING",
        "mode": "NULLABLE"
      }, {
        "name": "value",
        "type": "STRING",
        "mode": "NULLABLE"
      }]
    }
  }, {
    "name": "system_labels",
    "type": "RECORD",
    "mode": "REPEATED",
    "schema": {
      "fields": [{
        "name": "key",
        "type": "STRING",
        "mode": "NULLABLE"
      }, {
        "name": "value",
        "type": "STRING",
        "mode": "NULLABLE"
      }]
    }
  }, {
    "name": "location",
    "type": "RECORD",
    "mode": "NULLABLE",
    "schema": {
      "fields": [{
        "name": "location",
        "type": "STRING",
        "mode": "NULLABLE"
      }, {
        "name": "country",
        "type": "STRING",
        "mode": "NULLABLE"
      }, {
        "name": "region",
        "type": "STRING",
        "mode": "NULLABLE"
      }, {
        "name": "zone",
        "type": "STRING",
        "mode": "NULLABLE"
      }]
    }
  }, {
    "name": "export_time",
    "type": "TIMESTAMP",
    "mode": "NULLABLE"
  }, {
    "name": "cost",
    "type": "FLOAT",
    "mode": "NULLABLE"
  }, {
    "name": "currency",
    "type": "STRING",
    "mode": "NULLABLE"
  }, {
    "name": "currency_conversion_rate",
    "type": "FLOAT",
    "mode": "NULLABLE"
  }, {
    "name": "usage",
    "type": "RECORD",
    "mode": "NULLABLE",
    "schema": {
      "fields": [{
        "name": "amount",
        "type": "FLOAT",
        "mode": "NULLABLE"
      }, {
        "name": "unit",
        "type": "STRING",
        "mode": "NULLABLE"
      }, {
        "name": "amount_in_pricing_units",
        "type": "FLOAT",
        "mode": "NULLABLE"
      }, {
        "name": "pricing_unit",
        "type": "STRING",
        "mode": "NULLABLE"
      }]
    }
  }, {
    "name": "credits",
    "type": "RECORD",
    "mode": "REPEATED",
    "schema": {
      "fields": [{
        "name": "name",
        "type": "STRING",
        "mode": "NULLABLE"
      }, {
        "name": "amount",
        "type": "FLOAT",
        "mode": "NULLABLE"
      }, {
        "name": "full_name",
        "type": "STRING",
        "mode": "NULLABLE"
      }, {
        "name": "id",
        "type": "STRING",
        "mode": "NULLABLE"
      }, {
        "name": "type",
        "type": "STRING",
        "mode": "NULLABLE"
      }]
    }
  }, {
    "name": "invoice",
    "type": "RECORD",
    "mode": "NULLABLE",
    "schema": {
      "fields": [{
        "name": "month",
        "type": "STRING",
        "mode": "NULLABLE"
      }]
    }
  }, {
    "name": "cost_type",
    "type": "STRING",
    "mode": "NULLABLE"
  }, {
    "name": "adjustment_info",
    "type": "RECORD",
    "mode": "NULLABLE",
    "schema": {
      "fields": [{
        "name": "id",
        "type": "STRING",
        "mode": "NULLABLE"
      }, {
        "name": "description",
        "type": "STRING",
        "mode": "NULLABLE"
      }, {
        "name": "mode",
        "type": "STRING",
        "mode": "NULLABLE"
      }, {
        "name": "type",
        "type": "STRING",
        "mode": "NULLABLE"
      }]
    }
  }]
}
(

updateTime2020-11-09T20:02:55.628Z
(

createTime2019-03-24T18:47:10.559Z
u
        tableNamehfprojects/<<project-id>>/datasets/<<company-name>>_billing/tables/gcp_billing_export_v1_0170A2_15340F_FDCB33

reasonTABLE_UPDATE_REQUEST

You can see the value is not valid, am i missing something or doing something wrong?

0xSage commented 3 years ago

Hi, ah thanks for the clarification.

This client library does not provide custom proto to json obj conversions for audit log payloads. Without this customer wrapper, there does seem to be known issues with converting auditLog protoPayloads into a formatted result.

You can see some alternative solutions in this issue here: https://github.com/googleapis/nodejs-logging/issues/785

I will try other solutions and update here shortly.

Update: There isn't another idiomatic library that helps with the specific type conversion you need.

The recommended approach is to route your logs into a sink instead. If not, you will have to handle the BigQueryAuditMetadata proto to json type conversions yourself:

See schema for the BigQueryAuditMetadata here.

This protobufjs library should be able to help you specify the conversion options.

0xSage commented 3 years ago

Closing this issue due to lack of response. let me know if the above approached worked, or if this issue needs to be reopened - thanks!

mynameisnyke commented 3 years ago

Resurfacing this, i've tried the suggestions in this thread, but also the suggestions in #785.

For my project, I need to get the authenticationInfo.principalEmail which is nested inside of protoPayload key, however as others have found this property comes out as a buffer. I understand that there's the option to leverage a sink, but this is something we want to do across of all of the projects in our organization and would prefer to not have to build a sink every-time if at all possible.

googleapis / nodejs-logging

Cannot parse getEntries data buffer #984