from __future__ import absolute_import, division, print_function from fable \ import unsigned_integer_scan, \ identifier_scan, \ floating_point_scan_after_exponent_char, \ floating_point_scan_after_dot class token(object): __slots__ = ["typeid", "ssl", "i_code", "value"] def __init__(O, typeid, ssl, i_code, value): O.typeid = typeid O.ssl = ssl O.i_code = i_code O.value = value def type(O): return [ "identifier", "integer", "hexadecimal", "real", "double_precision", "logical", "string", "op", "complex", "seq", "parentheses", "implied_do", "power", "format"][O.typeid] def is_identifier(O): return O.typeid == 0 def is_integer(O): return O.typeid == 1 def is_hexadecimal(O): return O.typeid == 2 def is_real(O): return O.typeid == 3 def is_double_precision(O): return O.typeid == 4 def is_logical(O): return O.typeid == 5 def is_string(O): return O.typeid == 6 def is_op(O): return O.typeid == 7 def is_complex(O): return O.typeid == 8 def is_seq(O): return O.typeid == 9 def is_parentheses(O): return O.typeid == 10 def is_implied_do(O): return O.typeid == 11 def is_power(O): return O.typeid == 12 def is_format(O): return O.typeid == 13 def is_identifier_or_scalar_number(O): return O.typeid <= 4 def is_op_with(O, value): return O.typeid == 7 and O.value == value def is_unary_plus_or_minus(O): return O.typeid == 7 and O.value in ["+", "-"] def is_seq_or_parentheses(O): return 9 <= O.typeid <= 10 def is_seq_or_parentheses_or_implied_do(O): return 9 <= O.typeid <= 11 def stmt_location(O): return O.ssl.stmt_location(i=O.i_code) def format_error(O, msg, prefix=""): return O.ssl.format_error(msg=msg, i=O.i_code, prefix=prefix) def raise_error(O, msg, ErrorType=None): O.ssl.raise_error(msg=msg, i=O.i_code, ErrorType=ErrorType) def raise_syntax_error(O): O.ssl.raise_syntax_error(i=O.i_code) def raise_semantic_error(O, msg=None): O.ssl.raise_semantic_error(msg=msg, i=O.i_code) def raise_not_supported(O): O.raise_error( msg="Sorry: not supported", ErrorType=RuntimeError) def raise_internal_error(O): O.ssl.raise_internal_error(i=O.i_code) def raise_if_not_identifier(O): i = identifier_scan(code=O.value) if (i < 0 or i != len(O.value)): O.raise_error(msg="Not an identifier: %s" % repr(O.value)) def raise_missing_closing(O): O.raise_error(msg='Missing a closing ")"') def raise_missing_opening(O): O.raise_error(msg='Closing ")" without a matching opening parenthesis') def fquote(O): return "'" + O.value.replace("'", "''") + "'" def tk_identifier(ssl, i_code, value): return token(0, ssl, i_code, value) def tk_integer(ssl, i_code, value): return token(1, ssl, i_code, value) def tk_hexadecimal(ssl, i_code, value): return token(2, ssl, i_code, value) def tk_real(ssl, i_code, value): return token(3, ssl, i_code, value) def tk_double_precision(ssl, i_code, value): return token(4, ssl, i_code, value) def tk_logical(ssl, i_code, value): return token(5, ssl, i_code, value) def tk_string(ssl, i_code, value): return token(6, ssl, i_code, value) def tk_op(ssl, i_code, value): return token(7, ssl, i_code, value) def tk_complex(ssl, i_code, value): return token(8, ssl, i_code, value) def tk_seq(ssl, i_code, value): return token(9, ssl, i_code, value) def tk_parentheses(ssl, i_code, value): return token(10, ssl, i_code, value) def tk_implied_do(ssl, i_code, value): return token(11, ssl, i_code, value) def tk_power(ssl, i_code, value): return token(12, ssl, i_code, value) def tk_format(ssl, i_code, value): return token(13, ssl, i_code, value) class ssl_iterator(object): "stripped source line iterator" __slots__ = ["ssl", "start", "stop", "i", "buffer"] def __init__(O, ssl, start, stop=None): O.ssl = ssl O.start = start if (stop is None): O.stop = len(ssl.code) else: O.stop = stop O.i = start O.buffer = [] def __iter__(O): return O def __next__(self): tok = self.get(optional=True) if (tok is None): raise StopIteration return tok def next(self): return self.__next__() def get(O, optional=False): ssl = O.ssl code = ssl.code stop = O.stop def return_none(): if (not optional): if (stop == 0): ssl.raise_syntax_error() else: ssl.raise_syntax_error(i=stop-1) O.i = None return None if (len(O.buffer) != 0): O.i, result = O.buffer.pop(0) if (result is None): return_none() return result if (O.i is None): if (optional): return None ssl.raise_internal_error(i=stop-1) while (O.i < stop): i_code = O.i c = code[i_code] if ("=<>".find(c) >= 0): if (code.startswith("=", i_code+1)): O.i += 2 return tk_op(ssl=ssl, i_code=i_code, value=code[i_code:i_code+2]) O.i += 1 return tk_op(ssl=ssl, i_code=i_code, value=c) if ("(),:+-".find(c) >= 0): O.i += 1 return tk_op(ssl=ssl, i_code=i_code, value=c) if (c == "'"): O.i += 1 return tk_string( ssl=ssl, i_code=i_code, value=ssl.strings[ssl.string_indices.index(i_code)]) if (c == "x" and code.startswith("'", i_code+1)): O.i += 2 return tk_hexadecimal( ssl=ssl, i_code=i_code, value=ssl.strings[ssl.string_indices.index(i_code+1)]) j = identifier_scan(code=code, start=i_code) if (j > 0): O.i = j return tk_identifier(ssl=ssl, i_code=i_code, value=code[i_code:j]) if (c == "*"): if (code.startswith("*", i_code+1)): O.i += 2 return tk_op(ssl=ssl, i_code=i_code, value="**") O.i += 1 return tk_op(ssl=ssl, i_code=i_code, value="*") if (c == "/"): if (code.startswith("/", i_code+1)): O.i += 2 return tk_op(ssl=ssl, i_code=i_code, value="//") if (code.startswith("=", i_code+1)): O.i += 2 return tk_op(ssl=ssl, i_code=i_code, value="/=") O.i += 1 return tk_op(ssl=ssl, i_code=i_code, value="/") if (c == "."): return O.__after_dot(i_fld=i_code, i_dot=i_code) j = unsigned_integer_scan(code=code, start=i_code, stop=stop) if (j > 0): if (j == stop): O.i = j return tk_integer(ssl=ssl, i_code=i_code, value=code[i_code:j]) cj = code[j] if (cj == "."): if (j + 1 == stop): O.i = stop return tk_real(ssl=ssl, i_code=i_code, value=code[i_code:stop]) return O.__after_dot(i_fld=i_code, i_dot=j) if (cj == "e" or cj == "d"): k = floating_point_scan_after_exponent_char( code=code, start=j+1, stop=stop) if (k < 0): ssl.raise_error( msg="Invalid floating-point literal", i=j+1) O.i = k if (cj == "d"): return tk_double_precision( ssl=ssl, i_code=i_code, value=code[i_code:k]) return tk_real(ssl=ssl, i_code=i_code, value=code[i_code:k]) O.i = j return tk_integer(ssl=ssl, i_code=i_code, value=code[i_code:j]) ssl.raise_syntax_error(i=i_code) return_none() def __after_dot(O, i_fld, i_dot): ssl = O.ssl stop = O.stop if (i_dot + 1 == stop): ssl.raise_error( msg="Expression unexpectedly ends with a dot", i=i_dot) code = ssl.code csw = code.startswith c = code[i_dot+1] if (c == "a"): if (not csw("nd.", i_dot+2)): ssl.raise_syntax_error(i=i_dot+2) tok = tk_op(ssl=ssl, i_code=i_dot, value=".and.") if (i_dot == i_fld): O.i = i_dot+5 return tok O.buffer.append((i_dot+5, tok)) O.i = i_dot return tk_integer(ssl=ssl, i_code=i_fld, value=code[i_fld:i_dot]) if (c == "o"): if (not csw("r.", i_dot+2)): ssl.raise_syntax_error(i=i_dot+2) tok = tk_op(ssl=ssl, i_code=i_dot, value=".or.") if (i_dot == i_fld): O.i = i_dot+4 return tok O.buffer.append((i_dot+4, tok)) O.i = i_dot return tk_integer(ssl=ssl, i_code=i_fld, value=code[i_fld:i_dot]) if (c == "e" or c == "d"): if (c == "e"): if (csw("q.", i_dot+2)): tok = tk_op(ssl=ssl, i_code=i_dot, value=".eq.") if (i_dot == i_fld): O.i = i_dot+4 return tok O.buffer.append((i_dot+4, tok)) O.i = i_dot return tk_integer(ssl=ssl, i_code=i_fld, value=code[i_fld:i_dot]) if (csw("qv.", i_dot+2)): tok = tk_op(ssl=ssl, i_code=i_dot, value=".eqv.") if (i_dot == i_fld): O.i = i_dot+5 return tok O.buffer.append((i_dot+5, tok)) return tk_integer(ssl=ssl, i_code=i_fld, value=code[i_fld:i_dot]) if (i_dot == i_fld): ssl.raise_syntax_error(i=i_dot+1) j = floating_point_scan_after_exponent_char( code=code, start=i_dot+2, stop=stop) if (j < 0): ssl.raise_syntax_error(i=i_dot+1) O.i = j if (c == "d"): return tk_double_precision( ssl=ssl, i_code=i_fld, value=code[i_fld:j]) return tk_real(ssl=ssl, i_code=i_fld, value=code[i_fld:j]) if (c == "f"): if (not csw("alse.", i_dot+2) or i_dot != i_fld): ssl.raise_syntax_error(i=i_dot+1) O.i = i_dot+7 return tk_logical(ssl=ssl, i_code=i_dot, value=".false.") if (c == "t"): if (not csw("rue.", i_dot+2) or i_dot != i_fld): ssl.raise_syntax_error(i=i_dot+1) O.i = i_dot+6 return tk_logical(ssl=ssl, i_code=i_dot, value=".true.") if (c == "n"): if (csw("ot.", i_dot+2)): if (i_dot != i_fld): ssl.raise_syntax_error(i=i_dot+1) O.i = i_dot+5 return tk_op(ssl=ssl, i_code=i_dot, value=".not.") if (csw("e.", i_dot+2)): tok = tk_op(ssl=ssl, i_code=i_dot, value=".ne.") if (i_dot == i_fld): O.i = i_dot+4 return tok O.buffer.append((i_dot+4, tok)) O.i = i_dot return tk_integer(ssl=ssl, i_code=i_fld, value=code[i_fld:i_dot]) if (csw("eqv.", i_dot+2)): tok = tk_op(ssl=ssl, i_code=i_dot, value=".neqv.") if (i_dot == i_fld): O.i = i_dot+6 return tok O.buffer.append((i_dot+6, tok)) O.i = i_dot return tk_integer(ssl=ssl, i_code=i_fld, value=code[i_fld:i_dot]) ssl.raise_syntax_error(i=i_dot+1) if (c == "g" or c == "l"): if (not csw("t.", i_dot+2) and not csw("e.", i_dot+2)): ssl.raise_syntax_error(i=i_dot+1) tok = tk_op(ssl=ssl, i_code=i_dot, value=code[i_dot:i_dot+4]) if (i_dot == i_fld): O.i = i_dot+4 return tok O.buffer.append((i_dot+4, tok)) O.i = i_dot return tk_integer(ssl=ssl, i_code=i_fld, value=code[i_fld:i_dot]) j = floating_point_scan_after_dot(code=code, start=i_dot+1, stop=stop) if (j < 0): ssl.raise_syntax_error(i=i_dot+1) O.i = j if (code.find("d", i_dot+1, O.i) < 0): tk_type = tk_real else: tk_type = tk_double_precision return tk_type(ssl=ssl, i_code=i_fld, value=code[i_fld:j]) def look_ahead(O, optional=False): if (len(O.buffer) != 0): return O.buffer[0][1] prev_i = O.i tok = O.get(optional=optional) O.buffer.append((O.i, tok)) O.i = prev_i return tok def get_complex_literal(O, opening_token): assert opening_token.ssl is O.ssl invalid_message = "Invalid complex number literal" result = [opening_token] def get_part(): tok = O.get() sign_tok = None if (tok.is_op()): if (tok.value not in ["+", "-"]): tok.raise_error(msg=invalid_message) sign_tok = tok tok = O.get() if (not tok.is_identifier_or_scalar_number()): tok.raise_error(msg=invalid_message) return (sign_tok, tok) result.append(get_part()) tok = O.get() if (not tok.is_op_with(value=",")): tok.raise_error(msg=invalid_message) result.append(get_part()) tok = O.get() if (not tok.is_op_with(value=")")): tok.raise_error(msg=invalid_message) result.append(tok) return tk_complex(ssl=O.ssl, i_code=opening_token.i_code, value=result) def get_inside_parentheses(O, opening_token): tok = O.get(optional=True) if (tok is None): opening_token.raise_missing_closing() return tok def collect_comma_separated_identifiers(O, callback, one_required=False, enable_common=False): n_list = 0 while True: tok = O.get(optional=not one_required) if (tok is None): break if (tok.is_identifier()): callback(tok) elif (not enable_common): tok.raise_syntax_error() else: if (not tok.is_op_with("/")): tok.raise_syntax_error() tok = O.get() if (not tok.is_identifier()): tok.raise_syntax_error() tok = O.get() if (not tok.is_op_with("/")): tok.raise_syntax_error() # SAVE of COMMON are ignored (considered redundant) n_list += 1 tok = O.get(optional=True) if (tok is None): break if (not tok.is_op_with(value=",")): tok.raise_syntax_error() one_required = True return n_list def collect_comma_separated_expressions(O, callback, opening_token=None, first_get_optional=True, stop_after_given_number_of_commas=None, enable_implied_do=0): tokens = tk_seq(ssl=O.ssl, i_code=O.i, value=[]) tapp = tokens.value.append last_comma = None n_callbacks = 0 i_assignment_op = None get_optional = first_get_optional while True: tok = O.get(optional=get_optional) get_optional = True if (tok is None): if (opening_token is not None): opening_token.raise_missing_closing() if (len(tokens.value) == 0): if (last_comma is None): return None last_comma.raise_syntax_error() callback(tokens) return None if (tok.is_op()): tv = tok.value if (tv ==")"): if (opening_token is None): tok.raise_missing_opening() if (i_assignment_op is None): if (enable_implied_do == 2): tok.raise_syntax_error() elif (i_assignment_op == n_callbacks): tok.raise_syntax_error() if (len(tokens.value) == 0): if (last_comma is None): return i_assignment_op last_comma.raise_syntax_error() callback(tokens) return i_assignment_op if (tv == ","): if (len(tokens.value) == 0): tok.raise_syntax_error() callback(tokens) n_callbacks += 1 if (stop_after_given_number_of_commas is not None and n_callbacks == stop_after_given_number_of_commas): return tokens = tk_seq(ssl=O.ssl, i_code=O.i, value=[]) tapp = tokens.value.append last_comma = tok elif (tv == "("): nested_tokens = [] if (O.collect_comma_separated_expressions( callback=nested_tokens.append, opening_token=tok, enable_implied_do=int(enable_implied_do!=0)) is None): tk_type = tk_parentheses else: tk_type = tk_implied_do tapp(tk_type( ssl=tok.ssl, i_code=tok.i_code, value=nested_tokens)) elif (tv == "="): if ( opening_token is None or enable_implied_do == 0 or n_callbacks == 0 or i_assignment_op is not None or len(tokens.value) != 1 or not tokens.value[0].is_identifier()): tok.raise_syntax_error() i_assignment_op = n_callbacks tapp(tok) else: tapp(tok) else: tapp(tok) def collect_to_matching_parenthesis(O, callback, opening_token): nested_tokens = [] O.collect_comma_separated_expressions( callback=nested_tokens.append, opening_token=opening_token) callback(tk_parentheses( ssl=opening_token.ssl, i_code=opening_token.i_code, value=nested_tokens)) def get_implied_do(O, opening_token): result = [] O.collect_comma_separated_expressions( callback=result.append, opening_token=opening_token, enable_implied_do=2) return tk_implied_do(ssl=O.ssl, i_code=opening_token.i_code, value=result) def remove_redundant_parentheses(tokens): result = [] for tok in tokens: def handle_seq_with_one_parentheses(): if (tok.is_seq()): tv = tok.value if (len(tv) == 1): inner_tok = tv[0] if (inner_tok.is_parentheses()): result.extend( remove_redundant_parentheses(tokens=inner_tok.value)) return return result.append(tok) handle_seq_with_one_parentheses() return result def group_power(tokens): result = [] i_tok = 0 n_toks = len(tokens) while (i_tok < n_toks): tok = tokens[i_tok]; i_tok += 1 if (tok.is_op_with(value="**")): if (len(result) == 0): tok.raise_syntax_error() if (i_tok == len(tokens)): tok.raise_syntax_error() if ( result[-1].is_parentheses() and len(result) != 1 and result[-2].is_identifier()): base_tok = tk_seq( ssl=result[-2].ssl, i_code=result[-2].i_code, value=result[-2:]) result.pop() else: base_tok = result[-1] result.pop() exponent_tok = tokens[i_tok]; i_tok += 1 if ( exponent_tok.is_identifier() and i_tok < len(tokens) and tokens[i_tok].is_parentheses()): next_tok = tokens[i_tok]; i_tok += 1 exponent_tok = tk_seq( ssl=exponent_tok.ssl, i_code=exponent_tok.i_code, value=[exponent_tok, next_tok]) result.append(tk_power( ssl=exponent_tok.ssl, i_code=exponent_tok.i_code, value=[base_tok, exponent_tok])) else: result.append(tok) return result class implied_do_info(object): __slots__ = ["dlist_size", "id_tok", "fls_tokens"] def __init__(O, tokens): assert len(tokens) >= 3 def get_first_tokens(i): tok = tokens[i] if (not tok.is_seq()): return False tv = tok.value if (len(tv) < 3): return False if (not tv[0].is_identifier()): return False if (not tv[1].is_op_with(value="=")): return False O.dlist_size = len(tokens) + i O.id_tok = tv[0] O.fls_tokens = [tk_seq(ssl=tv[2].ssl, i_code=tv[2].i_code, value=tv[2:])] return True if (get_first_tokens(i=-2)): O.fls_tokens.append(tokens[-1]) else: assert get_first_tokens(i=-3) O.fls_tokens.extend(tokens[-2:]) def tok_seq_is_star(tok_seq): return (len(tok_seq.value) == 1 and tok_seq.value[0].is_op_with(value="*")) def tokens_as_strings(tokens, result=None): if (result is None): result = [] for tok in tokens: if (tok.is_seq()): result.append("[") tokens_as_strings(tokens=tok.value, result=result) result.append("]") elif (tok.is_parentheses()): result.append("(") tokens_as_strings(tokens=tok.value, result=result) result.append(")") elif (tok.is_implied_do()): result.append("{") tokens_as_strings(tokens=tok.value, result=result) result.append("}") elif (tok.is_string()): result.append(tok.fquote()) elif (tok.is_power()): result.append("power(") tokens_as_strings(tokens=tok.value, result=result) result.append(")") elif (tok.is_format()): result.append("format(" + tok.fquote() + ")") else: result.append(tok.value) return result def tokens_as_string(tokens): return " ".join(tokens_as_strings(tokens=tokens)) def tokens_as_python_code(tokens, result=None, allow_power=True): join_result = (result is None) if (join_result): result = [] for tok in tokens: if (tok.is_seq()): if (tokens_as_python_code(tokens=tok.value, result=result) is None): return None elif (tok.is_parentheses()): result.append("(") if (tokens_as_python_code(tokens=tok.value, result=result) is None): return None result.append(")") elif (tok.is_implied_do()): return None elif (tok.is_string()): return None elif (tok.is_power()): return None elif (tok.is_format()): return None elif (not allow_power and tok.is_op_with(value="**")): return None else: result.append(tok.value) if (join_result): return " ".join(result) return result def search_for_id_tokens(callback, tokens, with_next_token): for i_tok,tok in enumerate(tokens): if (tok.is_seq_or_parentheses_or_implied_do()): search_for_id_tokens( callback=callback, tokens=tok.value, with_next_token=with_next_token) elif (tok.is_identifier()): if (not with_next_token): callback(tok) else: next_tok = None j_tok = i_tok + 1 if (j_tok != len(tokens)): next_tok = tokens[j_tok] callback(tok, next_tok) def extract_identifiers(tokens, result=None): if (result is None): result = [] search_for_id_tokens( callback=result.append, tokens=tokens, with_next_token=False) return result def search_for_data_or_read_target_tokens(callback, tokens): for tok in tokens: assert tok.is_seq() if (len(tok.value) == 0): continue first_target_tok = tok.value[0] if (first_target_tok.is_identifier()): callback(tok=first_target_tok) elif (first_target_tok.is_implied_do()): idi = implied_do_info(tokens=first_target_tok.value) search_for_data_or_read_target_tokens( callback=callback, tokens=first_target_tok.value[:idi.dlist_size]) class fss_iterator(object): "format string stripped iterator" __slots__ = ["fss", "i"] def __init__(O, fss): O.fss = fss O.i = 0 def __iter__(O): return O def __next__(self): tok = self.get() if (tok is None): raise StopIteration return tok def next(self): return self.__next__() def get(O): fss = O.fss code = fss.code stop = len(code) assert O.i is not None def raise_invalid(): fss.raise_error(msg="Invalid FORMAT specification", i=min(stop-1, O.i)) while (O.i < stop): i_code = O.i c = code[i_code] if (c == ","): O.i += 1 continue if (c == "x"): O.i += 1 return tk_format(ssl=fss, i_code=i_code, value="1x") if ("():/$".find(c) >= 0): O.i += 1 return tk_op(ssl=fss, i_code=i_code, value=c) if (c == "'"): O.i += 1 return tk_string( ssl=fss, i_code=i_code, value=fss.strings[fss.string_indices.index(i_code)]) if (c == "+" or c == "-"): j = unsigned_integer_scan(code=code, start=i_code+1, stop=stop) if (j < 0 or not code.startswith("p", j)): raise_invalid() O.i = j + 1 return tk_format( ssl=fss, i_code=i_code, value=code[i_code:O.i]) j = unsigned_integer_scan(code=code, start=i_code, stop=stop) if (j > 0): O.i = j if (code.startswith("h", O.i)): # extract Hollerith edit descriptor if it did not confuse the # previous ignorant parsing algorithms sl, i_text = fss.text_location(i=O.i) assert sl.text[i_text].lower() == "h" i_text += 1 nh = int(code[i_code:j]) j_text = i_text + nh tv = sl.text[i_text:j_text] if (len(tv) != nh): raise RuntimeError(fss.format_error( i=i_code, msg="FATAL: Not supported: FORMAT Hollerith edit descriptor" " spanning continuation lines")) if (tv.find("'") >= 0 or tv.find('"') >= 0): raise RuntimeError(fss.format_error( i=i_code, msg="FATAL: Not supported:" " FORMAT Hollerith edit descriptor with quotes")) while (O.i < stop): sl, i_text = fss.text_location(i=O.i) if (i_text >= j_text): break O.i += 1 return tk_string(ssl=fss, i_code=i_code, value=tv) if (code.startswith("x", O.i) or code.startswith("p", O.i)): O.i += 1 return tk_format(ssl=fss, i_code=i_code, value=code[i_code:O.i]) return tk_integer(ssl=fss, i_code=i_code, value=code[i_code:j]) if ("defgiz".find(c) >= 0): j = unsigned_integer_scan(code=code, start=i_code+1, stop=stop) if (j > 0): O.i = j if (code.startswith(".", j)): j = unsigned_integer_scan(code=code, start=j+1, stop=stop) if (j < 0): raise_invalid() O.i = j return tk_format(ssl=fss, i_code=i_code, value=code[i_code:O.i]) if (c == "a" or c == "l"): O.i += 1 j = unsigned_integer_scan(code=code, start=i_code+1, stop=stop) if (j > 0): O.i = j return tk_format(ssl=fss, i_code=i_code, value=code[i_code:O.i]) if (code.startswith("bn", i_code) or code.startswith("bz", i_code)): O.i += 2 return tk_format(ssl=fss, i_code=i_code, value=code[i_code:O.i]) if (c == "s"): O.i += 1 if (code.startswith("p", O.i) or code.startswith("s", O.i)): O.i += 1 return tk_format(ssl=fss, i_code=i_code, value=code[i_code:O.i]) if (c == "t"): O.i += 1 if (code.startswith("l", O.i) or code.startswith("r", O.i)): O.i += 1 j = unsigned_integer_scan(code=code, start=O.i, stop=stop) if (j < 0): raise_invalid() O.i = j return tk_format(ssl=fss, i_code=i_code, value=code[i_code:O.i]) raise_invalid() O.i = None return None def get_statement_label_token(tokens): assert len(tokens) != 0 tok = tokens[0] if (len(tokens) != 1 or not tok.is_integer()): tok.raise_error(msg="Invalid statement label") return tok def fmt_tokens_as_string(tokens, comma=","): result = [] def result_ends_with_comma(): return (len(result) != 0 and result[-1] == comma) for tok in tokens: if (tok.is_integer()): result.append(tok.value) elif (tok.is_string()): result.append("'" + tok.value.replace("'","''") + "'") result.append(comma) elif (tok.is_format()): result.append(tok.value) result.append(comma) elif (tok.is_op()): tv = tok.value if (tv == "(" or tv == ")"): if (result_ends_with_comma()): result.pop() result.append(tv) if (tv == ")"): result.append(comma) else: result.append(tv) result.append(comma) else: raise AssertionError if (result_ends_with_comma()): result.pop() return "".join(result)