I am running a big query upload with JSON data using AppEngine. The code I am
running looks like this:
#Field Processing and Calls to Client
while CurrentPosition < TotalIssues:
IssuesList = IssuesPage['issues']
CurrentPosition += (IssuesPage['maxResults'])
TransformedIssues = []
while len(IssuesList) > 0:
Issue = IssuesList.pop()
Issue['fields']['key'] = Issue['key']
TransformedIssues.append(self._TransformIssue(Issue['fields']))
InsertResponse = self.BQClient.InsertRows(Constants.DATASETNAME, InsertTableName, TransformedIssues)
IssuesPage = self._GetPagedIssues(Opener, startAt=CurrentPosition)
#BQClient.InsertRows()
def InsertRows(self, dataset, table, rows, insertIDKey=None):
TableData = self.Service.tabledata()
RowsData=[]
for row in rows:
RowDict = {}
RowDict['json'] = row
if insertIDKey in row: RowDict["insertId"] = row[insertIDKey]
RowsData.append(RowDict)
Data = {
"kind": "bigquery#tableDataInsertAllRequest",
"rows": RowsData
}
try:
Response = TableData.insertAll(
projectId=self.ProjectID,
datasetId=dataset,
tableId=table,
body=Data
).execute()
sleep(1)
if Response.get('insertErrors'):
for error in Response['insertErrors']:
self.Log.AddLog("ERROR", "BigQueryClient", "FailedToInsertRow. {0}".format(str(error)))
raise
return Response
except HttpError, e:
self.Log.AddLog("ERROR","BigQueryClient","Unable to insert rows in table: Error:{0}".format(repr(e)))
return {
'insertErrors': [{
'errors': [{
'reason': 'httperror',
'message': e
}]
}]
}
The Response returns with no insertErrors in the dictionary, however when
errors are purposefully introduced BigQuery returns the appropriate errors.
Also the app is able to create a table earlier in the same execution so I am
sure the connection is good.
However once execution is completed, when I go to query the table, there are
no rows. I am unable to locate any administrative dashboards or logs to debug
this issue myself.
Original issue reported on code.google.com by nmarsh...@arkatechture.com on 18 Jun 2015 at 8:17
Original issue reported on code.google.com by
nmarsh...@arkatechture.com
on 18 Jun 2015 at 8:17