search

SAP / PyRFC

Asynchronous, non-blocking SAP NW RFC SDK bindings for Python
http://sap.github.io/PyRFC
Apache License 2.0
509 stars 137 forks source link

UnicodeDecodeError: 'utf8' codec can't decode byte 0xc2 #4

Closed martinduffy closed 9 years ago

martinduffy commented 10 years ago

I am developing a query for the HRP1000 table using RFC_READ_TABLE. The code is below. I have used this same code for a number of other tables with no issue. I am querying for the Position Codes and Position Descriptions in HRP1000.

I get the error below before getting all of the rows in the table that satisfy the query. The error is happening during the call for the RFC_READ_TABLE by PyRFC.

I am not sure if there is anything I could do to trap this error. Any suggestion would be appreciated. I am thinking that the fix is going to require some modification to PyRFC. I tried using the unicodecsv module for python but that will not help I think.

Let me mention that though the code that I have included here is in development it works really well to get data extracts from SAP and written to a csv file and could be pretty easily changed for other formats of files and includes using all or almost all of the parameters and options for RFC_READ_TABLE.

============= Error =================================== Traceback (most recent call last): File "I:\Apps\PyRFC-master\examples\sap-hrp1000-p.py", line 167, in main() File "I:\Apps\PyRFC-master\examples\sap-hrp1000-p.py", line 65, in main tables = conn.call('RFC_READ_TABLE', QUERY_TABLE=table, OPTIONS = options, DELIMITER='|', FIELDS = fields2, ROWSKIPS = rowskips,ROWCOUNT = ROWS_AT_A_TIME) File "_pyrfc.pyx", line 368, in pyrfc._pyrfc.Connection.call (pyrfc_pyrfc.c:4006) File "_pyrfc.pyx", line 1837, in pyrfc._pyrfc.wrapResult (pyrfc_pyrfc.c:19400) File "_pyrfc.pyx", line 1910, in pyrfc._pyrfc.wrapVariable (pyrfc_pyrfc.c:19954) File "_pyrfc.pyx", line 1882, in pyrfc._pyrfc.wrapTable (pyrfc_pyrfc.c:19732) File "_pyrfc.pyx", line 1855, in pyrfc._pyrfc.wrapStructure (pyrfc_pyrfc.c:19609) File "_pyrfc.pyx", line 1917, in pyrfc._pyrfc.wrapVariable (pyrfc_pyrfc.c:20034) File "_pyrfc.pyx", line 2049, in pyrfc._pyrfc.wrapString (pyrfc_pyrfc.c:21861) File "\ms240xxxx.na.xxxxxx.net\ctxredir$\mdufXXXX\Apps\python2.7.8\lib\encodings\utf_8.py", line 16, in decode return codecs.utf_8_decode(input, errors, True) UnicodeDecodeError: 'utf8' codec can't decode byte 0xc2 in position 46: unexpected end of data

========================== End of Error ===================

================== Code ================================

from pyrfc import Connection, ABAPApplicationError, LogonError from datetime import date from ConfigParser import ConfigParser import unicodecsv

conn = ""

NOW = date.today() # get today's date DATESTR = NOW.strftime("%m%d%Y") # Assign formatted date to variable FILENAME = ("HRP000_Positionsnapshot" + DATESTR + ".csv")

f = open(FILENAME,'w')

def main(): try: config = ConfigParser() config.read('sapnwrfc.cfg')

params_connection = config._sections['connection']

    params_connection = config._sections['pe1']

params_connection["user"] = user

params_connection["passwd"] = passwd

    conn = Connection(**params_connection)
except LogonError:
    print "Invalid username or password"

fields = []
fields_name = []
counter = 0
table="HRP1000"
#print table

#fields2 =[{'FIELDNAME':'PLVAR'}, {'FIELDNAME':'OTYPE'}, {'FIELDNAME':'OBJID'}, {'FIELDNAME':'BEGDA'}, {'FIELDNAME':'ENDDA'}, {'FIELDNAME':'OTJID'}, {'FIELDNAME':'SHORT'},{'FIELDNAME':'STEXT'}]
fields2 =[{'FIELDNAME':'OBJID'}, {'FIELDNAME':'BEGDA'}, {'FIELDNAME':'ENDDA'},{'FIELDNAME':'STEXT'}]

options = [{ 'TEXT': "PLVAR = '01' AND OTYPE = 'S' AND ENDDA = '99991231'"}]

ROWS_AT_A_TIME = 100
rowskips = 0
count = 0
while True:
    #print rowskips
    data_fields = []
    data_names = []
    #tables = conn.call('RFC_READ_TABLE', OPTIONS = options, FIELDS = fields2, QUERY_TABLE=table, DELIMITER='|', ROWSKIPS = rowskips,ROWCOUNT = ROWS_AT_A_TIME)
    #tables = conn.call("RFC_READ_TABLE", OPTIONS = options, QUERY_TABLE= table, DELIMITER='|')
    print "ok before tables"
    tables = conn.call('RFC_READ_TABLE', QUERY_TABLE=table, OPTIONS = options, DELIMITER='|', FIELDS = fields2, ROWSKIPS = rowskips,ROWCOUNT = ROWS_AT_A_TIME)

    print tables

    print "ok after tables"
    data_fields = tables["DATA"]
    data_names = tables["FIELDS"]
    #data_fields = data_fields.encode("utf-8")

print data_fields

print ""

print ""

print ""

print data_names

print ""

print ""

print ""

    long_fields = len(data_fields)
    long_names = len(data_names)

    rowskips = rowskips + ROWS_AT_A_TIME

    #print rowskips
    fields = []
    field_name = []

    for line in range(0, long_fields):
        #fields.append(unicodecsv.reader(data_fields[line]["WA"].strip(), encoding='utf-8'))
        #fields = (unicodecsv.writer(fields, encoding='utf-8')

        fields.append(data_fields[line]["WA"].strip())
    #print fields
    #print ""
    #print ""

    for line in range(0, long_names):
        fields_name.append(data_names[line]["FIELDNAME"].strip())

    #print fields_name
    #print ""
    #print ""
    #print ""

    output=""    

    for line in range(0, long_names):
        field_name = fields_name[line]
        output = output + field_name + ","
    output = output.rstrip(",")
    output = output + "\n"
    if count == 0:
        print output
        f.write(str(output))
        count = 1
    output = ""

    for line in range(0, long_fields):
        data_split = fields[line].split("|")

        for line in range(0, long_names):
            #print data_split[line]
            #print " before data_split ok"

            output = output + data_split[line].encode("utf-8") + ","
            #print "after data_split ok" 
            #output = unicodecsv.reader(line, encoding='utf-8') 
            #output = output + data_split[line].encode("utf-8") + ","
            #output = output + data_split[line] + ","
            #r = unicodecsv.reader(f, encoding='utf-8')
        output = output.rstrip(",")
        output = output + "\n"
    print output
    print "output ok before write"
    f.write(str(output))
    print "output ok after write"
    output = ""

    if len(tables['DATA']) < ROWS_AT_A_TIME:
        break
conn.close()

f.close()

f2 = open(FILENAME,'r+') for line in f2: if line.strip(): f2.write(line)

#except ABAPApplicationError:
#    output = "Table %s was not found" % table

output = "

Table %s was not found

" % table

#    return output

##return output

f2.close()

if name == 'main': main()

eyesonly commented 10 years ago

Hi Martin,

Two possibilites come to mind:

    • Try this exception handling: ... from pyrfc import Connection,\ ABAPApplicationError, ABAPRuntimeError, LogonError, CommunicationError ... except CommunicationError: print u"Could not connect to server." raise except LogonError: print u"Could not log in. Wrong credentials?" raise except (ABAPApplicationError, ABAPRuntimeError): print u"An error occurred." raise
    • BAPI_ORGUNITEXT_DATA_GET can also give you positions and descriptions (call with PLVAR = your plan version, OTYPE = O, OBJID = root org unit, KEYDATE = today or other key date, SCENARIO = MDT1, evalpath = O-S-P (or ORGCHART, amongst other possibilities)

Best Regards, Jonathan Groll

On Wed, 24 Sep 2014 11:21:18 -0700, martinduffy notifications@github.com wrote:

I am developing a query for the HRP1000 table using RFC_READ_TABLE. The code is below. I have used this same code for a number of other tables with no issue. I am querying for the Position Codes and Position Descriptions in HRP1000.

I get the error below before getting all of the rows in the table that satisfy the query. The error is happening during the call for the RFC_READ_TABLE by PyRFC.

I am not sure if there is anything I could do to trap this error. Any suggestion would be appreciated. I am thinking that the fix is going to require some modification to PyRFC. I tried using the unicodecsv module for python but that will not help I think.

Let me mention that though the code that I have included here is in development it works really well to get data extracts from SAP and written to a csv file and could be pretty easily changed for other formats of files and includes using all or almost all of the parameters and options for RFC_READ_TABLE.

============= Error =================================== Traceback (most recent call last): File "I:\Apps\PyRFC-master\examples\sap-hrp1000-p.py", line 167, in main() File "I:\Apps\PyRFC-master\examples\sap-hrp1000-p.py", line 65, in main tables = conn.call('RFC_READ_TABLE', QUERY_TABLE=table, OPTIONS = options, DELIMITER='|', FIELDS = fields2, ROWSKIPS = rowskips,ROWCOUNT = ROWS_AT_A_TIME) File "_pyrfc.pyx", line 368, in pyrfc._pyrfc.Connection.call (pyrfc_pyrfc.c:4006) File "_pyrfc.pyx", line 1837, in pyrfc._pyrfc.wrapResult (pyrfc_pyrfc.c:19400) File "_pyrfc.pyx", line 1910, in pyrfc._pyrfc.wrapVariable (pyrfc_pyrfc.c:19954) File "_pyrfc.pyx", line 1882, in pyrfc._pyrfc.wrapTable (pyrfc_pyrfc.c:19732) File "_pyrfc.pyx", line 1855, in pyrfc._pyrfc.wrapStructure (pyrfc_pyrfc.c:19609) File "_pyrfc.pyx", line 1917, in pyrfc._pyrfc.wrapVariable (pyrfc_pyrfc.c:20034) File "_pyrfc.pyx", line 2049, in pyrfc._pyrfc.wrapString (pyrfc_pyrfc.c:21861) File "\ms240xxxx.na.xxxxxx.net\ctxredir$\mdufXXXX\Apps\python2.7.8\lib\ encodings\utf_8.py", line 16, in decode return codecs.utf_8_decode(input, errors, True) UnicodeDecodeError: 'utf8' codec can't decode byte 0xc2 in position 46: unexpected end of data

========================== End of Error ===================

================== Code ================================

from pyrfc import Connection, ABAPApplicationError, LogonError from datetime import date from ConfigParser import ConfigParser import unicodecsv

conn = ""

NOW = date.today() # get today's date DATESTR = NOW.strftime("%m%d%Y") # Assign formatted date to variable FILENAME = ("HRP000_Positionsnapshot" + DATESTR + ".csv")

f = open(FILENAME,'w')

def main(): try: config = ConfigParser() config.read('sapnwrfc.cfg')

params_connection = config._sections['connection']

params_connection = config._sections['pe1']

params_connection["user"] = user

params_connection["passwd"] = passwd

conn = Connection(**params_connection)

except LogonError: print "Invalid username or password"

fields = [] fields_name = [] counter = 0 table="HRP1000"

print table

fields2 =[{'FIELDNAME':'PLVAR'}, {'FIELDNAME':'OTYPE'}, {'FIELDNAME':'OBJID'}, {'FIELDNAME':'BEGDA'}, {'FIELDNAME':'ENDDA'}, {'FIELDNAME':'OTJID'}, {'FIELDNAME':'SHORT'},{'FIELDNAME':'STEXT'}]

fields2 =[{'FIELDNAME':'OBJID'}, {'FIELDNAME':'BEGDA'}, {'FIELDNAME':'ENDDA'},{'FIELDNAME':'STEXT'}]

options = [{ 'TEXT': "PLVAR = '01' AND OTYPE = 'S' AND ENDDA = '99991231'"}]

ROWS_AT_A_TIME = 100 rowskips = 0 count = 0 while True:

print rowskips

data_fields = []
data_names = []
#tables = conn.call('RFC_READ_TABLE', OPTIONS = options, FIELDS = fields2, QUERY_TABLE=table, DELIMITER='|', ROWSKIPS = rowskips,ROWCOUNT = ROWS_AT_A_TIME)
#tables = conn.call("RFC_READ_TABLE", OPTIONS = options, QUERY_TABLE= table, DELIMITER='|')
print "ok before tables"
tables = conn.call('RFC_READ_TABLE', QUERY_TABLE=table, OPTIONS = options, DELIMITER='|', FIELDS = fields2, ROWSKIPS = rowskips,ROWCOUNT = ROWS_AT_A_TIME)
print tables

print "ok after tables"
data_fields = tables["DATA"]
data_names = tables["FIELDS"]
#data_fields = data_fields.encode("utf-8")

print data_fields

print ""

print ""

print ""

print data_names

print ""

print ""

print ""

long_fields = len(data_fields)
long_names = len(data_names)

rowskips = rowskips + ROWS_AT_A_TIME

#print rowskips
fields = []
field_name = []

for line in range(0, long_fields):
    #fields.append(unicodecsv.reader(data_fields[line]["WA"].strip(), encoding='utf-8'))
    #fields = (unicodecsv.writer(fields, encoding='utf-8')

    fields.append(data_fields[line]["WA"].strip())
#print fields
#print ""
#print ""

for line in range(0, long_names):
    fields_name.append(data_names[line]["FIELDNAME"].strip())

#print fields_name
#print ""
#print ""
#print ""

output=""

for line in range(0, long_names):
    field_name = fields_name[line]
    output = output + field_name + ","
output = output.rstrip(",")
output = output + "\n"
if count == 0:
    print output
    f.write(str(output))
    count = 1
output = ""

for line in range(0, long_fields):
    data_split = fields[line].split("|")

    for line in range(0, long_names):
        #print data_split[line]
        #print " before data_split ok"

        output = output + data_split[line].encode("utf-8") + ","
        #print "after data_split ok"
        #output = unicodecsv.reader(line, encoding='utf-8')
        #output = output + data_split[line].encode("utf-8") + ","
        #output = output + data_split[line] + ","
        #r = unicodecsv.reader(f, encoding='utf-8')
    output = output.rstrip(",")
    output = output + "\n"
print output
print "output ok before write"
f.write(str(output))
print "output ok after write"
output = ""

if len(tables['DATA']) < ROWS_AT_A_TIME:
    break

conn.close()

f.close()

f2 = open(FILENAME,'r+') for line in f2: if line.strip(): f2.write(line)

except ABAPApplicationError:

output = "Table %s was not found" % table

output = "

                        Table %s was not found

" % table

return output

return output

f2.close()

if name == 'main': main()

— Reply to this email directly or view it on GitHub.*

bsrdjan commented 9 years ago

As no comments, looks closed.