import time
import sys
try:
import cPickle as pickle
except ImportError:
import pickle
try:
import cdecimal as decimal
except ImportError:
import decimal
NODECIMAL = decimal.Decimal(1)
try:
import cElementTree as ET
except ImportError:
try:
import elementtree.ElementTree as ET
except ImportError:
try:
from xml.etree import cElementTree as ET
except ImportError:
from xml.etree import ElementTree as ET
try:
import elementtree.ElementInclude as ETI
except ImportError:
from xml.etree import ElementInclude as ETI
try:
import json as simplejson
except ImportError:
import simplejson
try:
from collections import OrderedDict
except:
from .bots_ordereddict import OrderedDict #python2.6
from django.utils.translation import ugettext as _
#bots-modules
import botslib
import botsglobal
import message
import grammar
import node
from botsconfig import *
def outmessage_init(**ta_info):
''' dispatch function class Outmessage or subclass
ta_info: needed is editype, messagetype, filename, charset, merge
'''
try:
classtocall = globals()[ta_info['editype']]
except KeyError:
raise botslib.OutMessageError(_(u'Unknown editype for outgoing message: %(editype)s'),ta_info)
return classtocall(ta_info)
class Outmessage(message.Message):
''' abstract class; represents a outgoing edi message.
subclassing is necessary for the editype (csv, edi, x12, etc)
A tree of nodes is build form the mpaths received from put()or putloop(). tree starts at self.root.
Put() recieves mpaths from mappingscript
The next algorithm is used to 'map' a mpath into the tree:
For each part of a mpath: search node in 'current' level of tree
If part already as a node:
recursively search node-children
If part not as a node:
append new node to tree;
recursively append next parts to tree
After the mappingscript is finished, the resulting tree is converted to self.lex_records.
These lex_records are written to file.
'''
def __init__(self,ta_info):
super(Outmessage,self).__init__(ta_info)
self.root = node.Node(record={}) #message tree; build via put()-interface in mappingscript. Initialise with empty dict
def messagegrammarread(self,typeofgrammarfile='grammars'):
''' read grammar for a message/envelope.
(try to) read the topartner dependent grammar syntax.
'''
super(Outmessage,self).messagegrammarread(typeofgrammarfile)
#read partner-syntax. Use this to always overrule values in self.ta_info
if self.ta_info.get('frompartner'):
try:
partnersyntax = grammar.grammarread(self.ta_info['editype'],self.ta_info['frompartner'],typeofgrammarfile='partners')
self.ta_info.update(partnersyntax.syntax) #partner syntax overrules!
except botslib.BotsImportError:
pass #No partner specific syntax found (is not an error).
if self.ta_info.get('topartner'):
try:
partnersyntax = grammar.grammarread(self.ta_info['editype'],self.ta_info['topartner'],typeofgrammarfile='partners')
self.ta_info.update(partnersyntax.syntax) #partner syntax overrules!
except botslib.BotsImportError:
pass #No partner specific syntax found (is not an error).
def writeall(self):
''' writeall is called for writing all 'real' outmessage objects; but not for envelopes.
writeall is call from transform.translate()
'''
self.messagegrammarread()
self.checkmessage(self.root,self.defmessage)
self.checkforerrorlist()
self.nrmessagewritten = 0
if self.root.record: #root record contains information; write whole tree in one time
self.multiplewrite = False
self._initwrite()
self._write(self.root)
self.nrmessagewritten = 1
self.ta_info['nrmessages'] = self.nrmessagewritten
self._closewrite()
elif not self.root.children:
raise botslib.OutMessageError(_(u'No outgoing message')) #then there is nothing to write...
else:
self.multiplewrite = True
self._initwrite()
for childnode in self.root.children:
self._write(childnode)
self.nrmessagewritten += 1
#'write back' the number of messages. Tricky thing here is that sometimes such a structure is indeed one message: eg csv without BOTS iD.
#in general: when only one type of record in recorddefs (mind: for xml this is not useful) no not writeback the count as nrofmessages
#for now: always write back unless csv of fixed.
if not isinstance(self,(csv,fixed)):
self.ta_info['nrmessages'] = self.nrmessagewritten
self._closewrite()
def _initwrite(self):
botsglobal.logger.debug(u'Start writing to file "%(filename)s".',self.ta_info)
self._outstream = botslib.opendata(self.ta_info['filename'],'wb',charset=self.ta_info['charset'],errors=self.ta_info['checkcharsetout'])
def _closewrite(self):
botsglobal.logger.debug(u'End writing to file "%(filename)s".',self.ta_info)
self._outstream.close()
def _write(self,node_instance):
''' the write method for most classes.
tree is serialised to lex_records; lex_records are written to file.
Classses that write using other libraries (xml, json, template, db) use specific write methods.
'''
self.tree2records(node_instance)
value = self.record2string(self.lex_records)
wrap_length = int(self.ta_info.get('wrap_length', 0))
if wrap_length:
try:
for i in range(0,len(value),wrap_length): #split in fixed lengths
self._outstream.write(value[i:i+wrap_length] + '\r\n')
except UnicodeError as msg:
content = botslib.get_relevant_text_for_UnicodeError(msg)
raise botslib.OutMessageError(_(u'[F50]: Characters not in character-set "%(char)s": %(content)s'),
{'char':self.ta_info['charset'],'content':content})
else:
try:
self._outstream.write(value)
except UnicodeError as msg:
content = botslib.get_relevant_text_for_UnicodeError(msg)
raise botslib.OutMessageError(_(u'[F50]: Characters not in character-set "%(char)s": %(content)s'),
{'char':self.ta_info['charset'],'content':content})
def tree2records(self,node_instance):
self.lex_records = [] #tree of nodes is flattened to these lex_records
self._tree2recordscore(node_instance,self.defmessage.structure[0])
def _tree2recordscore(self,node_instance,structure):
''' Write tree of nodes to flat lex_records.
The nodes are already sorted
'''
self._tree2recordfields(node_instance.record,structure) #write node->lex_record
for childnode in node_instance.children:
botsid_childnode = childnode.record['BOTSID'].strip() #speed up: use local var
botsidnr_childnode = childnode.record['BOTSIDnr'] #speed up: use local var
for structure_record in structure[LEVEL]: #for structure_record of this level in grammar
if botsid_childnode == structure_record[ID] and botsidnr_childnode == structure_record[BOTSIDNR]: #check if is is the right node
self._tree2recordscore(childnode,structure_record) #use rest of index in deeper level
break #childnode was found and used; break to go to next child node
def _tree2recordfields(self,noderecord,structure_record):
''' from noderecord->lex_record; use structure_record as guide.
complex because is is used for: editypes that have compression rules (edifact), var editypes without compression, fixed protocols
'''
lex_record = [] #the record build; list (=record) of dicts (=fields).
recordbuffer = []
for field_definition in structure_record[FIELDS]: #loop all fields in grammar-definition
if field_definition[ISFIELD]: #if field (no composite)
if field_definition[MAXREPEAT] == 1: #if non-repeating
field_has_data = False
if field_definition[ID] in noderecord and noderecord[field_definition[ID]]:
#field exists in outgoing message and has data
field_has_data = True
recordbuffer.append({VALUE:noderecord[field_definition[ID]],SFIELD:0,FORMATFROMGRAMMAR:field_definition[FORMAT]})
elif self.ta_info['stripfield_sep']:
#no data and field not needed: write new empty field to recordbuffer;
recordbuffer.append({VALUE:'',SFIELD:0,FORMATFROMGRAMMAR:field_definition[FORMAT]})
else:
#no data but field is needed: initialise empty field. For eg fixed and csv: all fields have to be present
field_has_data = True
value = self._initfield(field_definition)
recordbuffer.append({VALUE:value,SFIELD:0,FORMATFROMGRAMMAR:field_definition[FORMAT]})
if field_has_data:
lex_record += recordbuffer #write recordbuffer to lex_record
recordbuffer = [] #clear recordbuffer
else: #repeating field
field_has_data = False
if field_definition[ID] in noderecord: #field exists in outgoing message
type_of_field = 0 #first field in repeat is marked as a field (not as repeat).
fieldbuffer = [] #buffer for this repeating field.
for field in noderecord[field_definition[ID]]:
if field:
field_has_data = True
fieldbuffer.append({VALUE:field,SFIELD:type_of_field,FORMATFROMGRAMMAR:field_definition[FORMAT]})
recordbuffer += fieldbuffer
fieldbuffer = []
else:
fieldbuffer.append({VALUE:'',SFIELD:type_of_field,FORMATFROMGRAMMAR:field_definition[FORMAT]})
type_of_field = 2 #mark rest of repeats as repeat.
if field_has_data:
lex_record += recordbuffer #write recordbuffer to lex_record
recordbuffer = [] #clear recordbuffer
else:
recordbuffer.append({VALUE:'',SFIELD:0,FORMATFROMGRAMMAR:field_definition[FORMAT]})
else: #if composite
if field_definition[MAXREPEAT] == 1: #if non-repeating
field_has_data = False
type_of_field = 0 #first subfield in composite is marked as a field (not a subfield).
fieldbuffer = [] #buffer for this composite.
for grammarsubfield in field_definition[SUBFIELDS]: #loop subfields
if grammarsubfield[ID] in noderecord and noderecord[grammarsubfield[ID]]: #field exists in outgoing message and has data
field_has_data = True
fieldbuffer.append({VALUE:noderecord[grammarsubfield[ID]],SFIELD:type_of_field}) #append field
recordbuffer += fieldbuffer
fieldbuffer = []
else:
fieldbuffer.append({VALUE:'',SFIELD:type_of_field}) #append new empty to buffer;
type_of_field = 1
if field_has_data:
lex_record += recordbuffer #write recordbuffer to lex_record
recordbuffer = [] #clear recordbuffer
else:
#composite has no data: write empty field
recordbuffer.append({VALUE:'',SFIELD:0})
else: #repeating composite
#receive list, including empty members
field_has_data = False
if field_definition[ID] in noderecord: #field exists in outgoing message
type_of_field = 0 #first subfield in composite is marked as a field (not a subfield).
fieldbuffer = [] #buffer for this composite.
for comp_dict in noderecord[field_definition[ID]]:
composite_has_data = False #comp_dict can be empty
compositebuffer = [] #buffer for this composite.
if comp_dict:
for grammarsubfield in field_definition[SUBFIELDS]: #loop subfields
if grammarsubfield[ID] in comp_dict and comp_dict[grammarsubfield[ID]]: #field exists in outgoing message and has data
composite_has_data = True
compositebuffer.append({VALUE:comp_dict[grammarsubfield[ID]],SFIELD:type_of_field,FORMATFROMGRAMMAR:grammarsubfield[FORMAT]})
fieldbuffer += compositebuffer
compositebuffer = []
else:
compositebuffer.append({VALUE:'',SFIELD:type_of_field,FORMATFROMGRAMMAR:grammarsubfield[FORMAT]})
type_of_field = 1
if composite_has_data:
field_has_data = True
recordbuffer += fieldbuffer
fieldbuffer = []
else:
fieldbuffer.append({VALUE:'',SFIELD:type_of_field})
type_of_field = 2
if field_has_data:
lex_record += recordbuffer #write recordbuffer to lex_record
recordbuffer = [] #clear recordbuffer
else:
#no data: write placeholder to recordbuffer;
recordbuffer.append({VALUE:'',SFIELD:0})
self.lex_records.append(lex_record)
def _formatfield(self,value, field_definition,structure_record,node_instance):
''' Input: value (as a string) and field definition.
Some parameters of self.syntax are used, eg decimaal
Format is checked and converted (if needed).
return the formatted value
'''
if field_definition[BFORMAT] == 'A':
if isinstance(self,fixed): #check length fields in variable records
if field_definition[FORMAT] == 'AR': #if field format is alfanumeric right aligned
value = value.rjust(field_definition[MINLENGTH])
else:
value = value.ljust(field_definition[MINLENGTH]) #add spaces (left, because A-field is right aligned)
if len(value) > field_definition[LENGTH]:
self.add2errorlist(_(u'[F20]: Record "%(record)s" field "%(field)s" too big (max %(max)s): "%(content)s".\n')%
{'record':self.mpathformat(structure_record[MPATH]),'field':field_definition[ID],'content':value,'max':field_definition[LENGTH]})
if len(value) < field_definition[MINLENGTH]:
self.add2errorlist(_(u'[F21]: Record "%(record)s" field "%(field)s" too small (min %(min)s): "%(content)s".\n')%
{'record':self.mpathformat(structure_record[MPATH]),'field':field_definition[ID],'content':value,'min':field_definition[MINLENGTH]})
elif field_definition[BFORMAT] in 'DT':
lenght = len(value)
if field_definition[BFORMAT] == 'D':
try:
if lenght == 6:
time.strptime(value,'%y%m%d')
elif lenght == 8:
time.strptime(value,'%Y%m%d')
else:
raise ValueError(u'To be catched')
except ValueError:
self.add2errorlist(_(u'[F22]: Record "%(record)s" date field "%(field)s" not a valid date: "%(content)s".\n')%
{'record':self.mpathformat(structure_record[MPATH]),'field':field_definition[ID],'content':value})
if lenght > field_definition[LENGTH]:
self.add2errorlist(_(u'[F31]: Record "%(record)s" date field "%(field)s" too big (max %(max)s): "%(content)s".\n')%
{'record':self.mpathformat(structure_record[MPATH]),'field':field_definition[ID],'content':value,'max':field_definition[LENGTH]})
if lenght < field_definition[MINLENGTH]:
self.add2errorlist(_(u'[F32]: Record "%(record)s" date field "%(field)s" too small (min %(min)s): "%(content)s".\n')%
{'record':self.mpathformat(structure_record[MPATH]),'field':field_definition[ID],'content':value,'min':field_definition[MINLENGTH]})
else: #if field_definition[BFORMAT] == 'T':
try:
if lenght == 4:
time.strptime(value,'%H%M')
elif lenght == 6:
time.strptime(value,'%H%M%S')
else:
raise ValueError(u'To be catched')
except ValueError:
self.add2errorlist(_(u'[F23]: Record "%(record)s" time field "%(field)s" not a valid time: "%(content)s".\n')%
{'record':self.mpathformat(structure_record[MPATH]),'field':field_definition[ID],'content':value})
if lenght > field_definition[LENGTH]:
self.add2errorlist(_(u'[F33]: Record "%(record)s" time field "%(field)s" too big (max %(max)s): "%(content)s".\n')%
{'record':self.mpathformat(structure_record[MPATH]),'field':field_definition[ID],'content':value,'max':field_definition[LENGTH]})
if lenght < field_definition[MINLENGTH]:
self.add2errorlist(_(u'[F34]: Record "%(record)s" time field "%(field)s" too small (min %(min)s): "%(content)s".\n')%
{'record':self.mpathformat(structure_record[MPATH]),'field':field_definition[ID],'content':value,'min':field_definition[MINLENGTH]})
else: #numerics
if value[0] == '-':
minussign = '-'
absvalue = value[1:]
else:
minussign = ''
absvalue = value
digits,decimalsign,decimals = absvalue.partition('.')
if not digits and not decimals:# and decimalsign:
self.add2errorlist(_(u'[F24]: Record "%(record)s" field "%(field)s" numerical format not valid: "%(content)s".\n')%
{'field':field_definition[ID],'content':value,'record':self.mpathformat(structure_record[MPATH])})
if not digits:
digits = '0'
lengthcorrection = 0 #for some formats (if self.ta_info['lengthnumericbare']=True; eg edifact) length is calculated without decimal sing and/or minus sign.
if field_definition[BFORMAT] == 'R': #floating point: use all decimals received
if self.ta_info['lengthnumericbare']:
if minussign:
lengthcorrection += 1
if decimalsign:
lengthcorrection += 1
try:
value = unicode(decimal.Decimal(minussign + digits + decimalsign + decimals).quantize(decimal.Decimal(10) ** -len(decimals)))
except:
self.add2errorlist(_(u'[F25]: Record "%(record)s" field "%(field)s" numerical format not valid: "%(content)s".\n')%
{'field':field_definition[ID],'content':value,'record':self.mpathformat(structure_record[MPATH])})
if field_definition[FORMAT] == 'RL': #if field format is numeric right aligned
value = value.ljust(field_definition[MINLENGTH] + lengthcorrection)
elif field_definition[FORMAT] == 'RR': #if field format is numeric right aligned
value = value.rjust(field_definition[MINLENGTH] + lengthcorrection)
else:
value = value.zfill(field_definition[MINLENGTH] + lengthcorrection)
value = value.replace('.',self.ta_info['decimaal'],1) #replace '.' by required decimal sep.
elif field_definition[BFORMAT] == 'N': #fixed decimals; round
if self.ta_info['lengthnumericbare']:
if minussign:
lengthcorrection += 1
if field_definition[DECIMALS]:
lengthcorrection += 1
try:
value = unicode(decimal.Decimal(minussign + digits + decimalsign + decimals).quantize(decimal.Decimal(10) ** -field_definition[DECIMALS]))
except:
self.add2errorlist(_(u'[F26]: Record "%(record)s" field "%(field)s" numerical format not valid: "%(content)s".\n')%
{'field':field_definition[ID],'content':value,'record':self.mpathformat(structure_record[MPATH])})
if field_definition[FORMAT] == 'NL': #if field format is numeric right aligned
value = value.ljust(field_definition[MINLENGTH] + lengthcorrection)
elif field_definition[FORMAT] == 'NR': #if field format is numeric right aligned
value = value.rjust(field_definition[MINLENGTH] + lengthcorrection)
else:
value = value.zfill(field_definition[MINLENGTH] + lengthcorrection)
value = value.replace('.',self.ta_info['decimaal'],1) #replace '.' by required decimal sep.
elif field_definition[BFORMAT] == 'I': #implicit decimals
if self.ta_info['lengthnumericbare']:
if minussign:
lengthcorrection += 1
try:
dec_value = decimal.Decimal(minussign + digits + decimalsign + decimals) * 10**field_definition[DECIMALS]
value = unicode(dec_value.quantize(NODECIMAL ))
except:
self.add2errorlist(_(u'[F27]: Record "%(record)s" field "%(field)s" numerical format not valid: "%(content)s".\n')%
{'field':field_definition[ID],'content':value,'record':self.mpathformat(structure_record[MPATH])})
value = value.zfill(field_definition[MINLENGTH] + lengthcorrection)
if len(value)-lengthcorrection > field_definition[LENGTH]:
self.add2errorlist(_(u'[F28]: Record "%(record)s" field "%(field)s" too big: "%(content)s".\n')%
{'record':self.mpathformat(structure_record[MPATH]),'field':field_definition[ID],'content':value})
return value
def _initfield(self,field_definition):
''' for some editypes like fixed fields without date have specific initalisation.
this is controlled by the 'stripfield_sep' parameter in grammar.
'''
if field_definition[BFORMAT] in 'ADT':
value = ''
else: #numerics
value = '0'
if field_definition[BFORMAT] == 'R': #floating point: use all decimals received
value = value.zfill(field_definition[MINLENGTH] )
elif field_definition[BFORMAT] == 'N': #fixed decimals; round
value = unicode(decimal.Decimal(value).quantize(decimal.Decimal(10) ** -field_definition[DECIMALS]))
value = value.zfill(field_definition[MINLENGTH])
value = value.replace('.',self.ta_info['decimaal'],1) #replace '.' by required decimal sep.
elif field_definition[BFORMAT] == 'I': #implicit decimals
dec_value = decimal.Decimal(value) * 10**field_definition[DECIMALS]
value = unicode(dec_value.quantize(NODECIMAL ))
value = value.zfill(field_definition[MINLENGTH])
return value
def record2string(self,lex_records):
''' write lex_records to a file.
using the right editype (edifact, x12, etc) and charset.
write (all fields of) each record using the right separators, escape etc
'''
sfield_sep = self.ta_info['sfield_sep']
if self.ta_info['record_tag_sep']:
record_tag_sep = self.ta_info['record_tag_sep']
else:
record_tag_sep = self.ta_info['field_sep']
field_sep = self.ta_info['field_sep']
quote_char = self.ta_info['quote_char']
escape = self.ta_info['escape']
record_sep = self.ta_info['record_sep'] + self.ta_info['add_crlfafterrecord_sep']
forcequote = self.ta_info['forcequote']
escapechars = self._getescapechars()
noBOTSID = self.ta_info.get('noBOTSID',False)
rep_sep = self.ta_info['reserve']
lijst = []
for lex_record in lex_records:
if noBOTSID: #for csv/fixed: do not write BOTSID so remove it
del lex_record[0]
fieldcount = 0
mode_quote = False
value = u'' #to collect the formatted record-string.
for field in lex_record: #loop all fields in lex_record
if not field[SFIELD]: #is a field:
if fieldcount == 0: #do nothing because first field in lex_record is not preceded by a separator
fieldcount = 1
elif fieldcount == 1:
value += record_tag_sep
fieldcount = 2
else:
value += field_sep
elif field[SFIELD] == 1: #is a subfield:
value += sfield_sep
else: #repeat
value += rep_sep
if quote_char: #quote char only used for csv
start_to__quote = False
if forcequote == 2:
if field[FORMATFROMGRAMMAR] in ['AN','A','AR']:
start_to__quote = True
elif forcequote: #always quote; this catches values 1, '1', '0'
start_to__quote = True
else:
if field_sep in field[VALUE] or quote_char in field[VALUE] or record_sep in field[VALUE]:
start_to__quote = True
if start_to__quote:
value += quote_char
mode_quote = True
for char in field[VALUE]: #use escape (edifact, tradacom). For x12 is warned if content contains separator
if char in escapechars:
if isinstance(self,x12):
if self.ta_info['replacechar']:
char = self.ta_info['replacechar']
else:
raise botslib.OutMessageError(_(u'[F51]: Character "%(char)s" is used as separator in this x12 file, so it can not be used in content. Field: "%(content)s".'),
{'char':char,'content':field[VALUE]})
else:
value +=escape
elif mode_quote and char == quote_char:
value += quote_char
value += char
if mode_quote:
value += quote_char
mode_quote = False
value += record_sep
lijst.append(value)
return ''.join(lijst)
def _getescapechars(self):
return ''
class fixed(Outmessage):
def _initfield(self,field_definition):
if field_definition[BFORMAT] == 'A':
if field_definition[FORMAT] == 'AR': #if field format is alfanumeric right aligned
value = ''.rjust(field_definition[MINLENGTH])
else:
value = ''.ljust(field_definition[MINLENGTH]) #add spaces (left, because A-field is right aligned)
elif field_definition[BFORMAT] == 'D':
value = ''.ljust(field_definition[MINLENGTH]) #add spaces
elif field_definition[BFORMAT] == 'T':
value = ''.ljust(field_definition[MINLENGTH]) #add spaces
else: #numerics
if field_definition[BFORMAT] == 'R': #floating point: use all decimals received
if field_definition[FORMAT] == 'RL': #if field format is numeric right aligned
value = '0'.ljust(field_definition[MINLENGTH] )
elif field_definition[FORMAT] == 'RR': #if field format is numeric right aligned
value = '0'.rjust(field_definition[MINLENGTH] )
else:
value = '0'.zfill(field_definition[MINLENGTH] )
elif field_definition[BFORMAT] == 'N': #fixed decimals; round
value = unicode(decimal.Decimal('0').quantize(decimal.Decimal(10) ** -field_definition[DECIMALS]))
if field_definition[FORMAT] == 'NL': #if field format is numeric right aligned
value = value.ljust(field_definition[MINLENGTH])
elif field_definition[FORMAT] == 'NR': #if field format is numeric right aligned
value = value.rjust(field_definition[MINLENGTH])
else:
value = value.zfill(field_definition[MINLENGTH])
value = value.replace('.',self.ta_info['decimaal'],1) #replace '.' by required decimal sep.
elif field_definition[BFORMAT] == 'I': #implicit decimals
dec_value = decimal.Decimal('0') * 10**field_definition[DECIMALS]
value = unicode(dec_value.quantize(NODECIMAL ))
value = value.zfill(field_definition[MINLENGTH])
return value
class idoc(fixed):
def __init__(self,ta_info):
super(idoc,self).__init__(ta_info)
self.recordnumber = 0 #segment counter. For sequential recordnumbering in records.
def _canonicaltree(self,node_instance,structure):
self.headerrecordnumber = self.recordnumber
super(idoc,self)._canonicaltree(node_instance,structure)
def _canonicalfields(self,node_instance,record_definition):
if self.ta_info['automaticcount']:
node_instance.record.update({'MANDT':self.ta_info['MANDT'],'DOCNUM':self.ta_info['DOCNUM'],'SEGNUM':unicode(self.recordnumber),'PSGNUM':unicode(self.headerrecordnumber),'HLEVEL':unicode(len(record_definition[MPATH]))})
else:
node_instance.record.update({'MANDT':self.ta_info['MANDT'],'DOCNUM':self.ta_info['DOCNUM']})
super(idoc,self)._canonicalfields(node_instance,record_definition)
self.recordnumber += 1 #tricky. EDI_DC is not counted, so I count after writing.
class var(Outmessage):
pass
class csv(var):
def _getescapechars(self):
return self.ta_info['escape']
class edifact(var):
def _getescapechars(self):
terug = self.ta_info['record_sep']+self.ta_info['field_sep']+self.ta_info['sfield_sep']+self.ta_info['escape']
if self.ta_info['version'] >= '4':
terug += self.ta_info['reserve']
return terug
class tradacoms(var):
def _getescapechars(self):
terug = self.ta_info['record_sep']+self.ta_info['field_sep']+self.ta_info['sfield_sep']+self.ta_info['escape']+self.ta_info['record_tag_sep']
return terug
def writeall(self):
''' writeall is called for writing all 'real' outmessage objects; but not for enveloping.
writeall is call from transform.translate()
'''
self.nrmessagewritten = 0
if not self.root.children:
raise botslib.OutMessageError(_(u'No outgoing message')) #then there is nothing to write...
messagetype = self.ta_info['messagetype']
for tradacomsmessage in self.root.getloop({'BOTSID':'STX'},{'BOTSID':'MHD'}):
self.ta_info['messagetype'] = tradacomsmessage.get({'BOTSID':'MHD','TYPE.01':None}) + tradacomsmessage.get({'BOTSID':'MHD','TYPE.02':None})
self.messagegrammarread()
if not self.nrmessagewritten:
self._initwrite()
self.checkmessage(tradacomsmessage,self.defmessage)
self.checkforerrorlist()
self._write(tradacomsmessage)
self.nrmessagewritten += 1
self.ta_info['messagetype'] = messagetype
self._closewrite()
self.ta_info['nrmessages'] = self.nrmessagewritten
class x12(var):
def _getescapechars(self):
terug = self.ta_info['record_sep']+self.ta_info['field_sep']+self.ta_info['sfield_sep']
if self.ta_info['version'] >= '00403':
terug += self.ta_info['reserve']
return terug
class xml(Outmessage):
''' Some problems with right xml prolog, standalone, DOCTYPE, processing instructons: Different ET versions give different results.
Things work OK for python 2.7
celementtree in 2.7 is version 1.0.6, but different implementation in 2.6??
For python <2.7: do not generate standalone, DOCTYPE, processing instructions for encoding !=utf-8,ascii OR if elementtree package is installed (version 1.3.0 or bigger)
'''
def _write(self,node_instance):
''' write normal XML messages (no envelope)'''
xmltree = ET.ElementTree(self._node2xml(node_instance))
root = xmltree.getroot()
self._xmlcorewrite(xmltree,root)
def envelopewrite(self,node_instance):
''' write envelope for XML messages'''
self._initwrite()
self.checkmessage(node_instance,self.defmessage)
self.checkforerrorlist()
xmltree = ET.ElementTree(self._node2xml(node_instance))
root = xmltree.getroot()
ETI.include(root)
self._xmlcorewrite(xmltree,root)
self._closewrite()
def _xmlcorewrite(self,xmltree,root):
if sys.version_info[1] >= 7 and self.ta_info['namespace_prefixes']: # Register any namespace prefixes specified in syntax
for eachns in self.ta_info['namespace_prefixes']:
ET.register_namespace(eachns[0], eachns[1])
#xml prolog: always use.*********************************
#standalone, DOCTYPE, processing instructions: only possible in python >= 2.7 or if encoding is utf-8/ascii
if sys.version_info[1] >= 7 or self.ta_info['charset'] in ['us-ascii','utf-8'] or ET.VERSION >= '1.3.0':
if self.ta_info['indented']:
indentstring = '\n'
else:
indentstring = ''
if self.ta_info['standalone']:
standalonestring = 'standalone="%s" '%(self.ta_info['standalone'])
else:
standalonestring = ''
processing_instruction = ET.ProcessingInstruction('xml', 'version="%s" encoding="%s" %s'%(self.ta_info['version'],self.ta_info['charset'], standalonestring))
self._outstream.write(ET.tostring(processing_instruction) + indentstring) #do not use encoding here. gives double xml prolog; possibly because ET.ElementTree.write i used again by write()
#doctype /DTD **************************************
if self.ta_info['DOCTYPE']:
self._outstream.write(''%(self.ta_info['DOCTYPE']) + indentstring)
#processing instructions (other than prolog) ************
if self.ta_info['processing_instructions']:
for eachpi in self.ta_info['processing_instructions']:
processing_instruction = ET.ProcessingInstruction(eachpi[0], eachpi[1])
self._outstream.write(ET.tostring(processing_instruction) + indentstring) #do not use encoding here. gives double xml prolog; possibly because ET.ElementTree.write i used again by write()
#indent the xml elements
if self.ta_info['indented']:
botslib.indent_xml(root)
#write tree to file; this is differnt for different python/elementtree versions
if sys.version_info[1] < 7 and ET.VERSION < '1.3.0':
xmltree.write(self._outstream,encoding=self.ta_info['charset'])
else:
xmltree.write(self._outstream,encoding=self.ta_info['charset'],xml_declaration=False)
def _node2xml(self,node_instance):
''' recursive method.
'''
newnode = self._node2xmlfields(node_instance.record)
for childnode in node_instance.children:
newnode.append(self._node2xml(childnode))
return newnode
def _node2xmlfields(self,noderecord):
''' write record as xml-record-entity plus xml-field-entities within the xml-record-entity.
output is sorted according to grammar, attributes alfabetically.
'''
#***generate the xml-record-entity***************************
recordtag = noderecord['BOTSID']
del noderecord['BOTSID']
del noderecord['BOTSIDnr']
#pick out the attributes for the xml-record-entity (if fieldnames start with attribute-marker these are xml-attribute for the xml-'record'; store these in attributedict)
keyattributemarker = recordtag + self.ta_info['attributemarker'] #attributemarker is a marker in the fieldname used to find out if field is an attribute of either record or field
attributedict = {}
for key,value in noderecord.items():
if key.startswith(keyattributemarker):
attributedict[key[len(keyattributemarker):]] = value
del noderecord[key]
xmlrecord = ET.Element(recordtag,attributedict) #make the xml-record-entity
#add the content/text xml-record-entity (in BOTSCONTENT)
if 'BOTSCONTENT' in noderecord:
xmlrecord.text = noderecord['BOTSCONTENT']
del noderecord['BOTSCONTENT']
#***generate the xml-field-entities within the xml-record-entity***************************
for field_def in self.defmessage.recorddefs[recordtag]: #loop over remaining fields in 'record': write these as subelements
if self.ta_info['attributemarker'] in field_def[ID][1:-1]: #skip fields that are marked as xml attributes
continue
text = noderecord.get(field_def[ID],None)
keyattributemarker = field_def[ID] + self.ta_info['attributemarker']
attributedict = {}
for key,value in noderecord.items():
if key.startswith(keyattributemarker):
attributedict[key[len(keyattributemarker):]] = value
del noderecord[key]
if text is not None or attributedict:
ET.SubElement(xmlrecord, field_def[ID],attributedict).text=text #add xml element to xml record
#~ del noderecord[field_def[ID]] #remove xml entity tag
return xmlrecord
def _initwrite(self):
botsglobal.logger.debug(u'Start writing to file "%(filename)s".',self.ta_info)
self._outstream = botslib.opendata(self.ta_info['filename'],"wb")
class xmlnocheck(xml):
def _node2xmlfields(self,noderecord):
''' write record as xml-record-entity plus xml-field-entities within the xml-record-entity.
output is sorted alfabetically, attributes alfabetically. Empty xml-entities comes as last.
'''
if 'BOTSID' not in noderecord:
raise botslib.OutMessageError(_(u'[X52]: No field "BOTSID" in xml-output in: "%(record)s"'),{'record':noderecord})
#***generate the xml-record-entity***************************
recordtag = noderecord['BOTSID']
del noderecord['BOTSID'] #remove 'record' tag
if 'BOTSIDnr' in noderecord: #BOTSIDnr does never go to the output; only internally used
del noderecord['BOTSIDnr']
#first generate the xml-'record'
attributemarker = self.ta_info['attributemarker']
keyattributemarker = recordtag + attributemarker
attributedict = {}
for key,value in noderecord.items(): #find the attributes for the xml-record, put these in attributedict
if key.startswith(keyattributemarker):
attributedict[key[len(keyattributemarker):]] = value
del noderecord[key]
xmlrecord = ET.Element(recordtag,attributedict) #make the xml ET node
if 'BOTSCONTENT' in noderecord:
xmlrecord.text = noderecord['BOTSCONTENT']
del noderecord['BOTSCONTENT']
#***generate the xml-field-entities within the xml-record-entity***************************
for key in sorted(noderecord.keys()):
if key not in noderecord or attributemarker in key[1:-1]: #if field not in outmessage: skip
continue
keyattributemarker = key + attributemarker
attributedict = {}
for key2,value2 in noderecord.items():
if key2.startswith(keyattributemarker):
attributedict[key2[len(keyattributemarker):]] = value2
del noderecord[key2]
ET.SubElement(xmlrecord, key,attributedict).text=noderecord[key] #add xml element to xml record
del noderecord[key] #remove xml entity tag
#***problem: empty xml-fields-entities with attribute are not written*************************
if noderecord:
fielddict = {}
for key,value in noderecord.iteritems():
field,nep,attribute = key.partition(attributemarker)
if not field in fielddict:
fielddict[field] = {}
fielddict[field][attribute] = value
for key,attributedict in fielddict.iteritems():
ET.SubElement(xmlrecord, key,attributedict).text=None #add xml element to xml record
return xmlrecord
class json(Outmessage):
def _initwrite(self):
super(json,self)._initwrite()
if self.multiplewrite:
self._outstream.write(u'[')
def _write(self,node_instance):
''' convert node tree to appropriate python object.
python objects are written to json by simplejson.
'''
if self.nrmessagewritten:
self._outstream.write(u',')
jsonobject = {node_instance.record['BOTSID']:self._node2json(node_instance)}
if self.ta_info['indented']:
indent = 2
else:
indent = None
simplejson.dump(jsonobject, self._outstream, skipkeys=False, ensure_ascii=False, check_circular=False, indent=indent)
def _closewrite(self):
if self.multiplewrite:
self._outstream.write(u']')
super(json,self)._closewrite()
def _node2json(self,node_instance):
''' recursive method.
'''
#newjsonobject is the json object assembled in the function.
newjsonobject = node_instance.record.copy() #init newjsonobject with record fields from node
for childnode in node_instance.children: #fill newjsonobject with the lex_records from childnodes.
key = childnode.record['BOTSID']
if childnode.linpos_info == 'OK':
newjsonobject[key] = self._node2json(childnode)
else:
if key in newjsonobject:
newjsonobject[key].append(self._node2json(childnode))
else:
newjsonobject[key] = [self._node2json(childnode)]
del newjsonobject['BOTSID']
del newjsonobject['BOTSIDnr']
return newjsonobject
def _canonicaltree(self,node_instance,structure):
''' some specific handling: if max one occurence of record: not as a list, but as a record.
'''
super(json, self)._canonicaltree(node_instance,structure) #verify as usual
self.correct_max_one_occurence(node_instance,structure)
def correct_max_one_occurence(self,node_instance,structure):
if node_instance.structure is None:
node_instance.structure = structure
if LEVEL in structure:
for record_definition in structure[LEVEL]: #for every record_definition (in grammar) of this level
for childnode in node_instance.children: #for every node in mpathtree; SPEED: delete nodes from list when found
#~ #print 'xxxxxxxxxx',record_definition[ID]
if childnode.record['BOTSID'] != record_definition[ID] or childnode.record['BOTSIDnr'] != record_definition[BOTSIDNR]: #if it is not the right NODE":
continue
if record_definition[MAX] == 1:
print 'yes',record_definition[ID]
childnode.linpos_info = 'OK'
self.correct_max_one_occurence(childnode,record_definition) #use rest of index in deeper level
#~ def _canonicalfields(self,node_instance,record_definition):
#~ ''' subclassed method; sorts using OrderedDict
#~ For all fields: check M/C, format.
#~ Fields are sorted according to grammar.
#~ Fields are never added.
#~ '''
#~ noderecord = node_instance.record
#~ new_noderecord = OrderedDict()
#~ for field_definition in record_definition[FIELDS]: #loop over fields in grammar
#~ value = noderecord.get(field_definition[ID])
#~ if not value:
#~ if field_definition[MANDATORY]:
#~ self.add2errorlist(_(u'[F02]%(linpos)s: Record "%(mpath)s" field "%(field)s" is mandatory.\n')%
#~ {'linpos':node_instance.linpos(),'mpath':self.mpathformat(record_definition[MPATH]),'field':field_definition[ID]})
#~ if value is None: #None-values are not used
#~ continue
#~ new_noderecord[field_definition[ID]] = self._formatfield(value,field_definition,record_definition,node_instance)
#~ node_instance.record = new_noderecord
def _canonicalfields(self,node_instance,record_definition):
''' For all fields: check M/C, format.
Fields are not sorted (a dict can not be sorted).
Fields are never added.
'''
noderecord = node_instance.record
new_noderecord = OrderedDict()
for field_definition in record_definition[FIELDS]: #loop over fields in grammar
if field_definition[ISFIELD]: #if field (no composite)
if field_definition[MAXREPEAT] == 1: #if non-repeating
value = noderecord.get(field_definition[ID])
if not value:
if field_definition[MANDATORY]:
self.add2errorlist(_(u'[F02]%(linpos)s: Record "%(mpath)s" field "%(field)s" is mandatory.\n')%
{'linpos':node_instance.linpos(),'mpath':self.mpathformat(record_definition[MPATH]),'field':field_definition[ID]})
continue
new_noderecord[field_definition[ID]] = self._formatfield(value,field_definition,record_definition,node_instance)
else: #repeating field;
#a list of values; values can be empty or None; at least one field should have value, else dropped
valuelist = noderecord.get(field_definition[ID])
if valuelist is None: #empty lists are already catched in node.put()
if field_definition[MANDATORY]:
self.add2errorlist(_(u'[F41]%(linpos)s: Record "%(mpath)s" repeating field "%(field)s" is mandatory.\n')%
{'linpos':node_instance.linpos(),'mpath':self.mpathformat(record_definition[MPATH]),'field':field_definition[ID]})
continue
if not isinstance(valuelist,list):
raise botslib.MappingFormatError(_(u'Repeating field: must be a list: put(%(valuelist)s)'),{'mpath':valuelist})
if len(valuelist) > field_definition[MAXREPEAT]:
self.add2errorlist(_(u'[F42]%(linpos)s: Record "%(mpath)s" repeating field "%(field)s" occurs %(occurs)s times, max is %(max)s.\n')%
{'linpos':node_instance.linpos(),'mpath':self.mpathformat(record_definition[MPATH]),'field':field_definition[ID],
'occurs':len(valuelist),'max':field_definition[MAXREPEAT]})
newlist = []
repeating_field_has_data = False
for value in valuelist:
if value is None:
value = ''
else:
value = unicode(value).strip()
if value:
repeating_field_has_data = True
newlist.append(self._formatfield(value,field_definition,record_definition,node_instance))
if not repeating_field_has_data:
if field_definition[MANDATORY]:
self.add2errorlist(_(u'[F43]%(linpos)s: Record "%(mpath)s" repeating field "%(field)s" is mandatory.\n')%
{'linpos':node_instance.linpos(),'mpath':self.mpathformat(record_definition[MPATH]),'field':field_definition[ID]})
del noderecord[field_definition[ID]]
continue
new_noderecord[field_definition[ID]] = newlist
new_noderecord['BOTSIDnr'] = node_instance.record['BOTSIDnr']
node_instance.record = new_noderecord
class jsonnocheck(json):
pass
class templatehtml(Outmessage):
''' uses Genshi library for templating. Genshi is very similar to Kid, and is the fork/follow-up of Kid.
Kid is not being developed further; in time Kid will not be in repositories etc.
Templates for Genshi are like Kid templates. Changes:
- other namespace: xmlns:py="http://genshi.edgewall.org/" instead of xmlns:py="http://purl.org/kid/ns#"
- enveloping is different: instead of
2 modes:
1. use self.data, a class that can contain any python object (older way of working)
2. use structure, recordedefs, write node tree. This is more like normal way of working; output is checked etc.
the procided template can handle msot things, change only css of envelope.
'''
class TemplateData(object):
pass
def __init__(self,ta_info):
try:
self.template = botslib.botsbaseimport('genshi.template')
except ImportError:
raise ImportError(_(u'Dependency failure: editype "templatehtml" requires python library "genshi".'))
super(templatehtml,self).__init__(ta_info)
self.data = templatehtml.TemplateData() #self.data can be used by mappingscript as container for content
def _write(self,node_instance):
templatefile = botslib.abspath(self.__class__.__name__,self.ta_info['template'])
try:
botsglobal.logger.debug(u'Start writing to file "%(filename)s".',self.ta_info)
loader = self.template.TemplateLoader(auto_reload=False)
tmpl = loader.load(templatefile)
except:
txt = botslib.txtexc()
raise botslib.OutMessageError(_(u'While templating "%(editype)s.%(messagetype)s", error:\n%(txt)s'),
{'editype':self.ta_info['editype'],'messagetype':self.ta_info['messagetype'],'txt':txt})
try:
filehandler = botslib.opendata(self.ta_info['filename'],'wb')
if self.ta_info['has_structure']: #new way of working
if self.ta_info['print_as_row']:
node_instance.collectlines(self.ta_info['print_as_row'])
stream = tmpl.generate(node=node_instance)
else:
stream = tmpl.generate(data=self.data)
stream.render(method='xhtml',encoding=self.ta_info['charset'],out=filehandler)
except:
txt = botslib.txtexc()
raise botslib.OutMessageError(_(u'While templating "%(editype)s.%(messagetype)s", error:\n%(txt)s'),
{'editype':self.ta_info['editype'],'messagetype':self.ta_info['messagetype'],'txt':txt})
botsglobal.logger.debug(_(u'End writing to file "%(filename)s".'),self.ta_info)
def writeall(self):
if not self.root.record:
self.root.record = {'BOTSID':'dummy'} #dummy, is not used but needed for writeall of base class
super(templatehtml,self).writeall()
class db(Outmessage):
''' For database connector.
out.root is pickled, and saved.
'''
def __init__(self,ta_info):
super(db,self).__init__(ta_info)
self.root = None #make root None; root is not a Node-object anyway; None can easy be tested when writing.
def writeall(self):
if self.root is None:
raise botslib.OutMessageError(_(u'No outgoing message')) #then there is nothing to write...
botsglobal.logger.debug(u'Start writing to file "%(filename)s".',self.ta_info)
self._outstream = botslib.opendata(self.ta_info['filename'],'wb')
pickle.dump(self.root,self._outstream)
self._outstream.close()
botsglobal.logger.debug(u'End writing to file "%(filename)s".',self.ta_info)
self.ta_info['envelope'] = 'db'
self.ta_info['merge'] = False
class raw(Outmessage):
''' out.root is just saved.
'''
def __init__(self,ta_info):
super(raw,self).__init__(ta_info)
self.root = None #make root None; root is not a Node-object anyway; None can easy be tested when writing.
def writeall(self):
if self.root is None:
raise botslib.OutMessageError(_(u'No outgoing message')) #then there is nothing to write...
botsglobal.logger.debug(u'Start writing to file "%(filename)s".',self.ta_info)
self._outstream = botslib.opendata(self.ta_info['filename'],'wb')
self._outstream.write(self.root)
self._outstream.close()
botsglobal.logger.debug(u'End writing to file "%(filename)s".',self.ta_info)
self.ta_info['envelope'] = 'raw'
self.ta_info['merge'] = False