Index: /issm/trunk-jpl/src/py3/archive/arch.py =================================================================== --- /issm/trunk-jpl/src/py3/archive/arch.py (revision 23688) +++ /issm/trunk-jpl/src/py3/archive/arch.py (revision 23689) @@ -1,7 +1,5 @@ import numpy as np -import math import struct -import sys -import os +from os import path from collections import OrderedDict @@ -18,5 +16,5 @@ # open file try: - if not os.path.isfile(filename): + if not path.isfile(filename): fid=open(filename,'wb') else: @@ -31,5 +29,5 @@ name=args[2*i] write_field_name(fid,name) - + # write data associated with field name data=args[2*i+1] @@ -43,5 +41,5 @@ else: raise ValueError("archwrite : error writing data, invalid code entered '%d'" % code) - + fid.close() @@ -55,5 +53,5 @@ """ try: - if os.path.isfile(filename): + if path.isfile(filename): fid=open(filename,'rb') else: @@ -61,9 +59,10 @@ except IOError as e: raise IOError("archread error : could not open file '%s' to read from" % filename) - + archive_results=[] # read first result result=read_field(fid) + while result: if fieldname == result['field_name']: @@ -71,11 +70,11 @@ archive_results=result['data']; # we only want the data break - + # read next result result=read_field(fid) - + # close file fid.close() - + return archive_results # }}} @@ -88,5 +87,5 @@ """ try: - if os.path.isfile(filename): + if path.isfile(filename): fid=open(filename,'rb') else: @@ -94,5 +93,5 @@ except IOError as e: raise IOError("archread error : could not open file '%s' to read from" % filename) - + print('Source file: ') print('\t{0}'.format(filename)) @@ -106,5 +105,5 @@ # go to next result result=read_field(fid) - + # close file fid.close() @@ -112,5 +111,5 @@ # }}} -# Helper functions +# Helper functions def write_field_name(fid,data): # {{{ """ @@ -121,5 +120,5 @@ reclen=len(data)+4+4 fid.write(struct.pack('>i',reclen)) - + # write format code code=format_archive_code(data); @@ -130,5 +129,5 @@ # write string length, and then the string fid.write(struct.pack('>i',len(data))) - fid.write(struct.pack('>%ds' % len(data),data)) + fid.write(struct.pack('>{}s'.format(len(data)),data.encode('utf8'))) # }}} def write_scalar(fid,data): # {{{ @@ -143,5 +142,5 @@ # write the format code (2 for scalar) fid.write(struct.pack('>i',2)) - + # write the double fid.write(struct.pack('>d',data)) @@ -158,5 +157,5 @@ elif isinstance(data,(list,tuple)): data=np.array(data).reshape(-1,) - + if np.ndim(data) == 1: if np.size(data): @@ -164,5 +163,5 @@ else: data=data.reshape(0,0) - + # get size of data sz=data.shape @@ -175,5 +174,5 @@ raise ValueError("archwrite error : can not write vector to binary file because it is too large") fid.write(struct.pack('>i',reclen)) - + # write format code fid.write(struct.pack('>i',3)) @@ -204,6 +203,6 @@ raise ValueError('archread error : a string was not present at the start of the arch file') namelen=struct.unpack('>i',fid.read(struct.calcsize('>i')))[0] - fieldname=struct.unpack('>%ds' % namelen,fid.read(namelen))[0] - + fieldname=struct.unpack('>{}s'.format(namelen),fid.read(namelen))[0] + # then, read the data datalen=struct.unpack('>i',fid.read(struct.calcsize('>i')))[0] @@ -211,5 +210,5 @@ if data_type==2: - # unpack scalar + # struct.upack scalar data=struct.unpack('>d',fid.read(struct.calcsize('>d')))[0] elif data_type==3: @@ -218,11 +217,11 @@ raw_data=np.zeros(shape=(rows,cols),dtype=float) for i in range(rows): - raw_data[i,:]=struct.unpack('>%dd' % cols,fid.read(cols*struct.calcsize('>d'))) - # The matrix will be unpacked in order and will be filled left -> right by column + raw_data[i,:]=struct.unpack('>{}d'.format(cols),fid.read(cols*struct.calcsize('>d'))) + # The matrix will be struct.upacked in order and will be filled left -> right by column # We need to reshape and transpose the matrix so it can be read correctly data=raw_data.reshape(raw_data.shape[::-1]).T else: raise TypeError("Cannot read data type %d" % data_type) - + # give additional data to user if data_type==2: @@ -234,5 +233,5 @@ result=OrderedDict() - result['field_name']=fieldname + result['field_name']=fieldname.decode('utf8') result['size']=data_size result['data_type']=data_type_str Index: /issm/trunk-jpl/src/py3/classes/pairoptions.py =================================================================== --- /issm/trunk-jpl/src/py3/classes/pairoptions.py (revision 23688) +++ /issm/trunk-jpl/src/py3/classes/pairoptions.py (revision 23689) @@ -5,5 +5,5 @@ """ PAIROPTIONS class definition - + Usage: pairoptions=pairoptions(); @@ -27,14 +27,14 @@ # }}} def __repr__(self): # {{{ - s=" functionname: '%s'\n" % self.functionname + s=" functionname: '{}'\n".format(self.functionname) if self.list: - s+=" list: (%ix%i)\n\n" % (len(self.list),2) + s+=" list: ({}x{}) \n\n".format(len(self.list),2) for item in self.list.items(): - if isinstance(item[1],str): - s+=" field: %-10s value: '%s'\n" % (item[0],item[1]) - elif isinstance(item[1],(bool,int,float)): - s+=" field: %-10s value: %g\n" % (item[0],item[1]) - else: - s+=" field: %-10s value: %s\n" % (item[0],type(item[1])) + #if isinstance(item[1],str): + s+=" field: {} value: '{}'\n".format((item[0],item[1])) + # elif isinstance(item[1],(bool,int,float)): + # s+=" field: %-10s value: %g\n" % (item[0],item[1]) + # else: + # s+=" field: %-10s value: %s\n" % (item[0],type(item[1])) else: s+=" list: empty\n" @@ -46,6 +46,6 @@ #check length of input if len(arg) % 2: - raise TypeError('Invalid parameter/value pair arguments') - numoptions = len(arg)/2 + raise TypeError('Invalid parameter/value pair arguments') + numoptions = int(len(arg)/2) #go through arg and build list of objects @@ -55,5 +55,5 @@ else: #option is not a string, ignore it - print("WARNING: option number %d is not a string and will be ignored." % (i+1)) + print("WARNING: option number {} is not a string and will be ignored.".format(i+1)) # }}} def addfield(self,field,value): # {{{ @@ -61,5 +61,5 @@ if isinstance(field,str): if field in self.list: - print("WARNING: field '%s' with value=%s exists and will be overwritten with value=%s." % (field,str(self.list[field]),str(value))) + print("WARNING: field '{}' with value={} exists and will be overwritten with value={}.".format(field,str(self.list[field]),str(value))) self.list[field] = value # }}} @@ -87,5 +87,5 @@ """EXIST - check if the option exist""" - #some argument checking: + #some argument checking: if field == None or field == '': raise ValueError('exist error message: bad usage'); @@ -102,11 +102,11 @@ """ GETOPTION - get the value of an option - + Usage: value=options.getfieldvalue(field,default) - + Find an option value from a field. A default option can be given in input if the field does not exist - + Examples: value=options.getfieldvalue(options,'caxis') @@ -114,5 +114,5 @@ """ - #some argument checking: + #some argument checking: if field == None or field == '': raise ValueError('getfieldvalue error message: bad usage'); @@ -134,8 +134,8 @@ """ REMOVEFIELD - delete a field in an option list - + Usage: obj=removefield(self,field,warn) - + if warn==1 display an info message to warn user that some of his options have been removed. Index: /issm/trunk-jpl/src/py3/consistency/checkfield.py =================================================================== --- /issm/trunk-jpl/src/py3/consistency/checkfield.py (revision 23688) +++ /issm/trunk-jpl/src/py3/consistency/checkfield.py (revision 23689) @@ -2,4 +2,5 @@ import os from pairoptions import pairoptions +from operator import attrgetter import MatlabFuncs as m @@ -41,5 +42,7 @@ else: fieldname=options.getfieldvalue('fieldname') - exec("field=md.{}".format(fieldname)) + + field=attrgetter(fieldname)(md) +# exec("field=md.{}".format(fieldname),namespace) if isinstance(field,(bool,int,float)): Index: /issm/trunk-jpl/src/py3/solve/WriteData.py =================================================================== --- /issm/trunk-jpl/src/py3/solve/WriteData.py (revision 23688) +++ /issm/trunk-jpl/src/py3/solve/WriteData.py (revision 23689) @@ -1,6 +1,5 @@ import numpy as np -import struct -import pairoptions -import MatlabFuncs as m +from struct import pack,unpack +from pairoptions import pairoptions def WriteData(fid,prefix,*args): @@ -13,5 +12,5 @@ #process options - options=pairoptions.pairoptions(*args) + options=pairoptions(*args) #Get data properties @@ -31,5 +30,5 @@ name = options.getfieldvalue('name') - format = options.getfieldvalue('format') + datatype = options.getfieldvalue('format') mattype = options.getfieldvalue('mattype',0) #only required for matrices timeserieslength = options.getfieldvalue('timeserieslength',-1) @@ -56,65 +55,67 @@ #Step 1: write the enum to identify this record uniquely - fid.write(struct.pack('i',len(name))) - fid.write(struct.pack('%ds' % len(name),name)) + fid.write(pack('>i',len(name))) + fid.write(pack('>{}s'.format(len(name)),name.encode())) + # print(name) + # print(pack('>{}s'.format(len(name)),name) #Step 2: write the data itself. - if m.strcmpi(format,'Boolean'): # {{{ + if datatype=='Boolean': # {{{ # if len(data) !=1: # raise ValueError('field %s cannot be marshalled as it has more than one element!' % name[0]) #first write length of record - fid.write(struct.pack('i',4+4)) #1 bool (disguised as an int)+code - - #write data code: - fid.write(struct.pack('i',FormatToCode(format))) + fid.write(pack('>i',4+4)) #1 bool (disguised as an int)+code + + #write data code: + fid.write(pack('>i',FormatToCode(datatype))) #now write integer - fid.write(struct.pack('i',int(data))) #send an int, not easy to send a bool - # }}} - - elif m.strcmpi(format,'Integer'): # {{{ + fid.write(pack('>i',int(data))) #send an int, not easy to send a bool + # }}} + + elif datatype=='Integer': # {{{ # if len(data) !=1: # raise ValueError('field %s cannot be marshalled as it has more than one element!' % name[0]) #first write length of record - fid.write(struct.pack('i',4+4)) #1 integer + code - - #write data code: - fid.write(struct.pack('i',FormatToCode(format))) + fid.write(pack('>i',4+4)) #1 integer + code + + #write data code: + fid.write(pack('>i',FormatToCode(datatype))) #now write integer - fid.write(struct.pack('i',data)) - # }}} - - elif m.strcmpi(format,'Double'): # {{{ + fid.write(pack('>i',int(data))) #force an int, + # }}} + + elif datatype=='Double': # {{{ # if len(data) !=1: # raise ValueError('field %s cannot be marshalled as it has more than one element!' % name[0]) #first write length of record - fid.write(struct.pack('i',8+4)) #1 double+code - - #write data code: - fid.write(struct.pack('i',FormatToCode(format))) + fid.write(pack('>i',8+4)) #1 double+code + + #write data code: + fid.write(pack('>i',FormatToCode(datatype))) #now write double - fid.write(struct.pack('d',data)) - # }}} - - elif m.strcmpi(format,'String'): # {{{ - #first write length of record - fid.write(struct.pack('i',len(data)+4+4)) #string + string size + code - - #write data code: - fid.write(struct.pack('i',FormatToCode(format))) + fid.write(pack('>d',data)) + # }}} + + elif datatype=='String': # {{{ + #first write length of record + fid.write(pack('>i',len(data)+4+4)) #string + string size + code + + #write data code: + fid.write(pack('>i',FormatToCode(datatype))) #now write string - fid.write(struct.pack('i',len(data))) - fid.write(struct.pack('%ds' % len(data),data)) - # }}} - - elif m.strcmpi(format,'BooleanMat'): # {{{ - - if isinstance(data,bool): + fid.write(pack('>i',len(data))) + fid.write(pack('>{}s'.format(len(data)),data.encode())) + # }}} + + elif datatype in ['IntMat','BooleanMat']: # {{{ + + if isinstance(data,(int,bool)): data=np.array([data]) elif isinstance(data,(list,tuple)): @@ -133,64 +134,25 @@ #first write length of record - fid.write(struct.pack('i',4+4+8*np.product(s)+4+4)) #2 integers (32 bits) + the double matrix + code + matrix type + fid.write(pack('>i',4+4+8*np.product(s)+4+4)) #2 integers (32 bits) + the double matrix + code + matrix type #write data code and matrix type: - fid.write(struct.pack('i',FormatToCode(format))) - fid.write(struct.pack('i',mattype)) + fid.write(pack('>i',FormatToCode(datatype))) + fid.write(pack('>i',mattype)) #now write matrix - if np.ndim(data)==1: - fid.write(struct.pack('i',s[0])) - fid.write(struct.pack('i',1)) + if np.ndim(data) == 1: + fid.write(pack('>i',s[0])) + fid.write(pack('>i',1)) for i in range(s[0]): - fid.write(struct.pack('d',float(data[i]))) #get to the "c" convention, hence the transpose - else: - fid.write(struct.pack('i',s[0])) - fid.write(struct.pack('i',s[1])) + fid.write(pack('>d',float(data[i]))) #get to the "c" convention, hence the transpose + else: + fid.write(pack('>i',s[0])) + fid.write(pack('>i',s[1])) for i in range(s[0]): for j in range(s[1]): - fid.write(struct.pack('d',float(data[i][j]))) #get to the "c" convention, hence the transpose - # }}} - - elif m.strcmpi(format,'IntMat'): # {{{ - - if isinstance(data,int): - data=np.array([data]) - elif isinstance(data,(list,tuple)): - data=np.array(data).reshape(-1,) - if np.ndim(data) == 1: - if np.size(data): - data=data.reshape(np.size(data),) - else: - data=data.reshape(0,0) - - #Get size - s=data.shape - #if matrix = NaN, then do not write anything - if np.ndim(data)==2 and np.product(s)==1 and np.all(np.isnan(data)): - s=(0,0) - - #first write length of record - fid.write(struct.pack('i',4+4+8*np.product(s)+4+4)) #2 integers (32 bits) + the double matrix + code + matrix type - - #write data code and matrix type: - fid.write(struct.pack('i',FormatToCode(format))) - fid.write(struct.pack('i',mattype)) - - #now write matrix - if np.ndim(data) == 1: - fid.write(struct.pack('i',s[0])) - fid.write(struct.pack('i',1)) - for i in range(s[0]): - fid.write(struct.pack('d',float(data[i]))) #get to the "c" convention, hence the transpose - else: - fid.write(struct.pack('i',s[0])) - fid.write(struct.pack('i',s[1])) - for i in range(s[0]): - for j in range(s[1]): - fid.write(struct.pack('d',float(data[i][j]))) #get to the "c" convention, hence the transpose - # }}} - - elif m.strcmpi(format,'DoubleMat'): # {{{ + fid.write(pack('>d',float(data[i][j]))) #get to the "c" convention, hence the transpose + # }}} + + elif datatype=='DoubleMat': # {{{ if isinstance(data,(bool,int,float)): @@ -215,25 +177,25 @@ raise ValueError('field %s cannot be marshalled because it is larger than 4^31 bytes!' % enum) - fid.write(struct.pack('i',recordlength)) #2 integers (32 bits) + the double matrix + code + matrix type + fid.write(pack('>i',recordlength)) #2 integers (32 bits) + the double matrix + code + matrix type #write data code and matrix type: - fid.write(struct.pack('i',FormatToCode(format))) - fid.write(struct.pack('i',mattype)) + fid.write(pack('>i',FormatToCode(datatype))) + fid.write(pack('>i',mattype)) #now write matrix if np.ndim(data) == 1: - fid.write(struct.pack('i',s[0])) - fid.write(struct.pack('i',1)) + fid.write(pack('>i',s[0])) + fid.write(pack('>i',1)) for i in range(s[0]): - fid.write(struct.pack('d',float(data[i]))) #get to the "c" convention, hence the transpose - else: - fid.write(struct.pack('i',s[0])) - fid.write(struct.pack('i',s[1])) + fid.write(pack('>d',float(data[i]))) #get to the "c" convention, hence the transpose + else: + fid.write(pack('>i',s[0])) + fid.write(pack('>i',s[1])) for i in range(s[0]): for j in range(s[1]): - fid.write(struct.pack('d',float(data[i][j]))) #get to the "c" convention, hence the transpose - # }}} - - elif m.strcmpi(format,'CompressedMat'): # {{{ + fid.write(pack('>d',float(data[i][j]))) #get to the "c" convention, hence the transpose + # }}} + + elif datatype=='CompressedMat': # {{{ if isinstance(data,(bool,int,float)): @@ -264,9 +226,9 @@ raise ValueError('field %s cannot be marshalled because it is larger than 4^31 bytes!' % enum) - fid.write(struct.pack('i',recordlength)) #2 integers (32 bits) + the matrix + code + matrix type + fid.write(pack('>i',recordlength)) #2 integers (32 bits) + the matrix + code + matrix type #write data code and matrix type: - fid.write(struct.pack('i',FormatToCode(format))) - fid.write(struct.pack('i',mattype)) + fid.write(pack('>i',FormatToCode(datatype))) + fid.write(pack('>i',mattype)) #Write offset and range @@ -282,28 +244,28 @@ #now write matrix if np.ndim(data) == 1: - fid.write(struct.pack('i',s[0])) - fid.write(struct.pack('i',1)) - fid.write(struct.pack('d',float(offsetA))) - fid.write(struct.pack('d',float(rangeA))) + fid.write(pack('>i',s[0])) + fid.write(pack('>i',1)) + fid.write(pack('>d',float(offsetA))) + fid.write(pack('>d',float(rangeA))) for i in range(s[0]-1): - fid.write(struct.pack('B',int(A[i]))) - - fid.write(struct.pack('d',float(data[s[0]-1]))) #get to the "c" convention, hence the transpose + fid.write(pack('>B',int(A[i]))) + + fid.write(pack('>d',float(data[s[0]-1]))) #get to the "c" convention, hence the transpose elif np.product(s) > 0: - fid.write(struct.pack('i',s[0])) - fid.write(struct.pack('i',s[1])) - fid.write(struct.pack('d',float(offsetA))) - fid.write(struct.pack('d',float(rangeA))) + fid.write(pack('>i',s[0])) + fid.write(pack('>i',s[1])) + fid.write(pack('>d',float(offsetA))) + fid.write(pack('>d',float(rangeA))) for i in range(s[0]-1): for j in range(s[1]): - fid.write(struct.pack('B',int(A[i][j]))) #get to the "c" convention, hence the transpose + fid.write(pack('>B',int(A[i][j]))) #get to the "c" convention, hence the transpose for j in range(s[1]): - fid.write(struct.pack('d',float(data[s[0]-1][j]))) - - # }}} - - elif m.strcmpi(format,'MatArray'): # {{{ + fid.write(pack('>d',float(data[s[0]-1][j]))) + + # }}} + + elif datatype=='MatArray': # {{{ #first get length of record @@ -324,11 +286,11 @@ #write length of record - fid.write(struct.pack('i',recordlength)) - - #write data code: - fid.write(struct.pack('i',FormatToCode(format))) + fid.write(pack('>i',recordlength)) + + #write data code: + fid.write(pack('>i',FormatToCode(datatype))) #write data, first number of records - fid.write(struct.pack('i',len(data))) + fid.write(pack('>i',len(data))) for matrix in data: @@ -343,17 +305,17 @@ if np.ndim(matrix) == 1: - fid.write(struct.pack('i',s[0])) - fid.write(struct.pack('i',1)) + fid.write(pack('>i',s[0])) + fid.write(pack('>i',1)) for i in range(s[0]): - fid.write(struct.pack('d',float(matrix[i]))) #get to the "c" convention, hence the transpose + fid.write(pack('>d',float(matrix[i]))) #get to the "c" convention, hence the transpose else: - fid.write(struct.pack('i',s[0])) - fid.write(struct.pack('i',s[1])) + fid.write(pack('>i',s[0])) + fid.write(pack('>i',s[1])) for i in range(s[0]): for j in range(s[1]): - fid.write(struct.pack('d',float(matrix[i][j]))) - # }}} - - elif m.strcmpi(format,'StringArray'): # {{{ + fid.write(pack('>d',float(matrix[i][j]))) + # }}} + + elif datatype=='StringArray': # {{{ #first get length of record @@ -363,48 +325,48 @@ #write length of record - fid.write(struct.pack('i',recordlength)) - - #write data code: - fid.write(struct.pack('i',FormatToCode(format))) + fid.write(pack('>i',recordlength)) + + #write data code: + fid.write(pack('>i',FormatToCode(datatype))) #now write length of string array - fid.write(struct.pack('i',len(data))) + fid.write(pack('>i',len(data))) #now write the strings for string in data: - fid.write(struct.pack('i',len(string))) - fid.write(struct.pack('%ds' % len(string),string)) + fid.write(pack('>i',len(string))) + fid.write(pack('>{}s'.format(len(string)),string.encode())) # }}} else: # {{{ - raise TypeError('WriteData error message: data type: %d not supported yet! (%s)' % (format,enum)) + raise TypeError('WriteData error message: data type: {} not supported yet! ({})'.format(datatype,enum)) # }}} -def FormatToCode(format): # {{{ +def FormatToCode(datatype): # {{{ """ - This routine takes the format string, and hardcodes it into an integer, which + This routine takes the datatype string, and hardcodes it into an integer, which is passed along the record, in order to identify the nature of the dataset being sent. """ - if m.strcmpi(format,'Boolean'): + if datatype=='Boolean': code=1 - elif m.strcmpi(format,'Integer'): + elif datatype=='Integer': code=2 - elif m.strcmpi(format,'Double'): + elif datatype=='Double': code=3 - elif m.strcmpi(format,'String'): + elif datatype=='String': code=4 - elif m.strcmpi(format,'BooleanMat'): + elif datatype=='BooleanMat': code=5 - elif m.strcmpi(format,'IntMat'): + elif datatype=='IntMat': code=6 - elif m.strcmpi(format,'DoubleMat'): + elif datatype=='DoubleMat': code=7 - elif m.strcmpi(format,'MatArray'): + elif datatype=='MatArray': code=8 - elif m.strcmpi(format,'StringArray'): + elif datatype=='StringArray': code=9 - elif m.strcmpi(format,'CompressedMat'): + elif datatype=='CompressedMat': code=10 else: