defmodule SnmpKit.SnmpLib.MIB.SnmpTokenizer do @moduledoc """ True 1:1 Elixir port of Erlang SNMP tokenizer (snmpc_tok.erl). This is a direct translation of the official Erlang SNMP tokenizer from OTP lib/snmp/src/compile/snmpc_tok.erl Original copyright: Ericsson AB 1996-2025 (Apache License 2.0) """ use GenServer # State record equivalent defstruct line: 1, chars: [], get_line_fun: nil # Reserved words from SNMP/SMI - complete list from Erlang tokenizer # Using a Map for O(1) lookup and reduced cyclomatic complexity @reserved_words %{ "DEFINITIONS" => :DEFINITIONS, "BEGIN" => :BEGIN, "END" => :END, "IMPORTS" => :IMPORTS, "FROM" => :FROM, "EXPORTS" => :EXPORTS, "OBJECT" => :OBJECT, "IDENTIFIER" => :IDENTIFIER, "OBJECT-TYPE" => :"OBJECT-TYPE", "SYNTAX" => :SYNTAX, "ACCESS" => :ACCESS, "MAX-ACCESS" => :"MAX-ACCESS", "STATUS" => :STATUS, "DESCRIPTION" => :DESCRIPTION, "REFERENCE" => :REFERENCE, "INDEX" => :INDEX, "AUGMENTS" => :AUGMENTS, "DEFVAL" => :DEFVAL, "UNITS" => :UNITS, "SEQUENCE" => :SEQUENCE, "OF" => :OF, "CHOICE" => :CHOICE, "SIZE" => :SIZE, "INTEGER" => :INTEGER, "OCTET" => :OCTET, "STRING" => :STRING, "NULL" => :NULL, "IpAddress" => :IpAddress, "Counter" => :Counter, "Counter32" => :Counter32, "Counter64" => :Counter64, "Gauge" => :Gauge, "Gauge32" => :Gauge32, "TimeTicks" => :TimeTicks, "Unsigned32" => :Unsigned32, "Integer32" => :Integer32, "Opaque" => :Opaque, "BITS" => :BITS, "MODULE-IDENTITY" => :"MODULE-IDENTITY", "OBJECT-IDENTITY" => :"OBJECT-IDENTITY", "TEXTUAL-CONVENTION" => :"TEXTUAL-CONVENTION", "OBJECT-GROUP" => :"OBJECT-GROUP", "NOTIFICATION-GROUP" => :"NOTIFICATION-GROUP", "MODULE-COMPLIANCE" => :"MODULE-COMPLIANCE", "AGENT-CAPABILITIES" => :"AGENT-CAPABILITIES", "NOTIFICATION-TYPE" => :"NOTIFICATION-TYPE", "TRAP-TYPE" => :"TRAP-TYPE", "LAST-UPDATED" => :"LAST-UPDATED", "ORGANIZATION" => :ORGANIZATION, "CONTACT-INFO" => :"CONTACT-INFO", "REVISION" => :REVISION, "DISPLAY-HINT" => :"DISPLAY-HINT", "IMPLIED" => :IMPLIED, "OBJECTS" => :OBJECTS, "NOTIFICATIONS" => :NOTIFICATIONS, "MANDATORY-GROUPS" => :"MANDATORY-GROUPS", "GROUP" => :GROUP, "MODULE" => :MODULE, "WRITE-SYNTAX" => :"WRITE-SYNTAX", "MIN-ACCESS" => :"MIN-ACCESS", "PRODUCT-RELEASE" => :"PRODUCT-RELEASE", "SUPPORTS" => :SUPPORTS, "INCLUDES" => :INCLUDES, "VARIATION" => :VARIATION, "CREATION-REQUIRES" => :"CREATION-REQUIRES", "ENTERPRISE" => :ENTERPRISE, "VARIABLES" => :VARIABLES, "APPLICATION" => :APPLICATION, "IMPLICIT" => :IMPLICIT, "EXPLICIT" => :EXPLICIT, "UNIVERSAL" => :UNIVERSAL, "PRIVATE" => :PRIVATE, "MACRO" => :MACRO, "TYPE" => :TYPE, "NOTATION" => :NOTATION, "VALUE" => :VALUE, # Access values "read-only" => :"read-only", "read-write" => :"read-write", "write-only" => :"write-only", "not-accessible" => :"not-accessible", "accessible-for-notify" => :"accessible-for-notify", "read-create" => :"read-create" } @type state() :: %__MODULE__{ line: pos_integer(), chars: charlist(), get_line_fun: function() | nil } @type token() :: {atom(), any(), pos_integer()} # API Functions - exact equivalents from Erlang @doc """ Start tokenizer gen_server. Equivalent to snmpc_tok:start_link/2 """ @spec start_link(charlist(), pid()) :: {:ok, pid()} | {:error, term()} def start_link(chars, get_line_pid) when is_list(chars) and is_pid(get_line_pid) do GenServer.start_link(__MODULE__, {chars, get_line_pid}, []) end @doc """ Get next token from tokenizer. Equivalent to snmpc_tok:get_token/1 """ @spec get_token(pid()) :: {:ok, token()} | {:error, term()} def get_token(pid) do GenServer.call(pid, :get_token) end @doc """ Get all remaining tokens. Equivalent to snmpc_tok:get_all_tokens/1 """ @spec get_all_tokens(pid()) :: {:ok, [token()]} | {:error, term()} def get_all_tokens(pid) do GenServer.call(pid, :get_all_tokens) end @doc """ Tokenize a string directly. Equivalent to snmpc_tok:tokenize/2 """ @spec tokenize(charlist(), function()) :: {:ok, [token()]} | {:error, term()} def tokenize(chars, _get_line_fun) when is_list(chars) do # For direct tokenization, we don't need the gen_server complexity # Just tokenize the input directly state = %__MODULE__{ line: 1, chars: chars, get_line_fun: &null_get_line/0 } case tokenize_all_direct(state, []) do {:ok, tokens} -> {:ok, Enum.reverse(tokens)} {:error, reason} -> {:error, reason} end end # Direct tokenization without gen_server defp tokenize_all_direct(state, acc) do case tokenise(state) do {{:"$end", _line}, _new_state} -> {:ok, [{:"$end", state.line} | acc]} {{:eof, _line}, _new_state} -> {:ok, [{:"$end", state.line} | acc]} {token, new_state} -> tokenize_all_direct(new_state, [token | acc]) {:error, reason, _new_state} -> {:error, reason} end end @doc """ Stop tokenizer. Equivalent to snmpc_tok:stop/1 """ @spec stop(pid()) :: :ok def stop(pid) do GenServer.call(pid, :stop) end @doc """ Format error message. Equivalent to snmpc_tok:format_error/1 """ @spec format_error(term()) :: charlist() def format_error(error) do case error do {:illegal, char} -> :io_lib.format(~c"illegal character '~c'", [char]) {:unterminated_string, line} -> :io_lib.format(~c"unterminated string starting at line ~p", [line]) {:unterminated_quote, line} -> :io_lib.format(~c"unterminated quote starting at line ~p", [line]) other -> :io_lib.format(~c"~p", [other]) end end @doc """ Null get_line function. Equivalent to snmpc_tok:null_get_line/0 """ @spec null_get_line() :: :eof def null_get_line, do: :eof @doc """ Test function. Equivalent to snmpc_tok:test/0 """ def test do test_string = ~c"TEST-MIB DEFINITIONS ::= BEGIN testObject OBJECT IDENTIFIER ::= { test 1 } END" case tokenize(test_string, &null_get_line/0) do {:ok, tokens} -> :io.format(~c"Tokens: ~p~n", [tokens]) :ok {:error, reason} -> :io.format(~c"Error: ~p~n", [reason]) :error end end # GenServer callbacks @impl true def init({chars, get_line_pid}) do get_line_fun = fn -> case GenServer.call(get_line_pid, :get_line) do line when is_list(line) -> line :eof -> :eof other -> other end end state = %__MODULE__{ line: 1, chars: chars, get_line_fun: get_line_fun } {:ok, state} end @impl true def handle_call(:get_token, _from, state) do case tokenise(state) do {token, new_state} -> {:reply, {:ok, token}, new_state} {:error, reason, new_state} -> {:reply, {:error, reason}, new_state} end end @impl true def handle_call(:get_all_tokens, _from, state) do case get_all_tokens_loop(state, []) do {:ok, tokens, new_state} -> {:reply, {:ok, Enum.reverse(tokens)}, new_state} {:error, reason, new_state} -> {:reply, {:error, reason}, new_state} end end @impl true def handle_call(:stop, _from, state) do {:stop, :normal, :ok, state} end @impl true def terminate(_reason, _state) do :ok end # Private tokenization functions - direct ports from Erlang # Get all tokens loop defp get_all_tokens_loop(state, acc) do case tokenise(state) do {{:eof, _line}, new_state} -> {:ok, [{:eof, state.line} | acc], new_state} {token, new_state} -> get_all_tokens_loop(new_state, [token | acc]) {:error, reason, new_state} -> {:error, reason, new_state} end end # Main tokenization function - direct port from tokenise/1 defp tokenise(%__MODULE__{chars: []} = state) do # For direct tokenization, we've reached the end {{:"$end", state.line}, state} end defp tokenise(%__MODULE__{chars: [?\s | chars]} = state) do # Skip whitespace new_state = %{state | chars: chars} tokenise(new_state) end defp tokenise(%__MODULE__{chars: [?\t | chars]} = state) do # Skip tab new_state = %{state | chars: chars} tokenise(new_state) end defp tokenise(%__MODULE__{chars: [?\n | chars]} = state) do # Handle newline new_state = %{state | chars: chars, line: state.line + 1} tokenise(new_state) end defp tokenise(%__MODULE__{chars: [?\r | chars]} = state) do # Skip carriage return new_state = %{state | chars: chars} tokenise(new_state) end defp tokenise(%__MODULE__{chars: [?-, ?- | chars]} = state) do # Handle comments -- new_state = %{state | chars: chars} skip_comment(new_state) end defp tokenise(%__MODULE__{chars: [?" | chars]} = state) do # Handle string literals scan_string(chars, [], state.line, %{state | chars: chars}) end defp tokenise(%__MODULE__{chars: [?' | chars]} = state) do # Handle quoted atoms scan_quote(chars, [], state.line, %{state | chars: chars}) end defp tokenise(%__MODULE__{chars: [?{ | chars]} = state) do # Left brace new_state = %{state | chars: chars} {{:"{", state.line}, new_state} end defp tokenise(%__MODULE__{chars: [?} | chars]} = state) do # Right brace new_state = %{state | chars: chars} {{:"}", state.line}, new_state} end defp tokenise(%__MODULE__{chars: [?( | chars]} = state) do # Left parenthesis new_state = %{state | chars: chars} {{:"(", state.line}, new_state} end defp tokenise(%__MODULE__{chars: [?) | chars]} = state) do # Right parenthesis new_state = %{state | chars: chars} {{:")", state.line}, new_state} end defp tokenise(%__MODULE__{chars: [?[ | chars]} = state) do # Left bracket new_state = %{state | chars: chars} {{:"[", state.line}, new_state} end defp tokenise(%__MODULE__{chars: [?] | chars]} = state) do # Right bracket new_state = %{state | chars: chars} {{:"]", state.line}, new_state} end defp tokenise(%__MODULE__{chars: [?, | chars]} = state) do # Comma new_state = %{state | chars: chars} {{:",", state.line}, new_state} end defp tokenise(%__MODULE__{chars: [?; | chars]} = state) do # Semicolon new_state = %{state | chars: chars} {{:";", state.line}, new_state} end defp tokenise(%__MODULE__{chars: [?| | chars]} = state) do # Pipe new_state = %{state | chars: chars} {{:|, state.line}, new_state} end defp tokenise(%__MODULE__{chars: [?: | chars]} = state) do # Handle :: and ::= case chars do [?: | more_chars] -> case more_chars do [?= | remaining] -> # ::= new_state = %{state | chars: remaining} {{:"::=", state.line}, new_state} _ -> # :: new_state = %{state | chars: more_chars} {{:"::", state.line}, new_state} end _ -> # : new_state = %{state | chars: chars} {{:":", state.line}, new_state} end end defp tokenise(%__MODULE__{chars: [?. | chars]} = state) do # Handle .. (range) case chars do [?. | more_chars] -> # .. new_state = %{state | chars: more_chars} {{:.., state.line}, new_state} _ -> # . new_state = %{state | chars: chars} {{:., state.line}, new_state} end end defp tokenise(%__MODULE__{chars: [?- | chars]} = state) do # Handle negative numbers or minus case chars do [digit | _] when digit >= ?0 and digit <= ?9 -> scan_integer(state.chars, [], state.line, state) _ -> # Single minus new_state = %{state | chars: chars} {{:-, state.line}, new_state} end end defp tokenise(%__MODULE__{chars: [digit | _chars]} = state) when digit >= ?0 and digit <= ?9 do # Handle positive integers scan_integer(state.chars, [], state.line, state) end defp tokenise(%__MODULE__{chars: [char | _chars]} = state) when (char >= ?a and char <= ?z) or (char >= ?A and char <= ?Z) or char == ?_ do # Handle identifiers and atoms scan_name(state.chars, [], state.line, state) end defp tokenise(%__MODULE__{chars: [char | chars]} = state) do # Illegal character new_state = %{state | chars: chars} {:error, {:illegal, char}, new_state} end # Skip comment until end of line defp skip_comment(%__MODULE__{chars: []} = state) do tokenise(state) end defp skip_comment(%__MODULE__{chars: [?\n | chars]} = state) do new_state = %{state | chars: chars, line: state.line + 1} tokenise(new_state) end defp skip_comment(%__MODULE__{chars: [_char | chars]} = state) do new_state = %{state | chars: chars} skip_comment(new_state) end # Scan string literal defp scan_string([], _acc, start_line, state) do {:error, {:unterminated_string, start_line}, state} end defp scan_string([?" | chars], acc, _start_line, state) do # End of string - convert to Elixir string string_value = acc |> Enum.reverse() |> List.to_string() new_state = %{state | chars: chars} {{:string, state.line, string_value}, new_state} end defp scan_string([?\\ | chars], acc, start_line, state) do # Handle escape sequences case chars do [escaped_char | rest] -> new_acc = [escaped_char | acc] scan_string(rest, new_acc, start_line, state) [] -> {:error, {:unterminated_string, start_line}, state} end end defp scan_string([?\n | chars], acc, start_line, state) do # Newline in string new_state = %{state | chars: chars, line: state.line + 1} scan_string(chars, [?\n | acc], start_line, new_state) end defp scan_string([char | chars], acc, start_line, state) do # Regular character scan_string(chars, [char | acc], start_line, state) end # Scan quoted atom defp scan_quote([], _acc, start_line, state) do {:error, {:unterminated_quote, start_line}, state} end defp scan_quote([?' | chars], acc, _start_line, state) do # End of quote - check for hex string suffix case chars do [?H | remaining_chars] -> # Hex string with uppercase H suffix: 'FF'H - treat as special atom hex_chars = Enum.reverse(acc) # Create a special atom that the grammar can recognize hex_atom = case hex_chars do # Empty hex string [] -> :"" _ -> List.to_atom(hex_chars) end new_state = %{state | chars: remaining_chars} {{:atom, state.line, hex_atom}, new_state} [?h | remaining_chars] -> # Hex string with lowercase h suffix: 'FF'h - treat as special atom hex_chars = Enum.reverse(acc) hex_atom = case hex_chars do # Empty hex string [] -> :"" _ -> List.to_atom(hex_chars) end new_state = %{state | chars: remaining_chars} {{:atom, state.line, hex_atom}, new_state} _ -> # Regular quoted atom atom_chars = Enum.reverse(acc) atom_value = List.to_atom(atom_chars) new_state = %{state | chars: chars} {{:atom, state.line, atom_value}, new_state} end end defp scan_quote([?\\ | chars], acc, start_line, state) do # Handle escape sequences in quotes case chars do [escaped_char | rest] -> new_acc = [escaped_char | acc] scan_quote(rest, new_acc, start_line, state) [] -> {:error, {:unterminated_quote, start_line}, state} end end defp scan_quote([char | chars], acc, start_line, state) do # Regular character in quote scan_quote(chars, [char | acc], start_line, state) end # Scan integer defp scan_integer([?- | chars], [], line, state) do # Negative number scan_integer_digits(chars, [?-], line, state) end defp scan_integer(chars, [], line, state) do # Positive number scan_integer_digits(chars, [], line, state) end defp scan_integer_digits([digit | chars], acc, line, state) when (digit >= ?0 and digit <= ?9) or (digit >= ?a and digit <= ?f) or (digit >= ?A and digit <= ?F) do # Include hex digits in integer scanning for large hex numbers like '7FFFFFFF' scan_integer_digits(chars, [digit | acc], line, state) end defp scan_integer_digits(chars, acc, line, state) do # End of integer - determine if it's hex or decimal integer_chars = Enum.reverse(acc) # Check if it contains hex digits has_hex_digits = Enum.any?(integer_chars, fn char -> (char >= ?a and char <= ?f) or (char >= ?A and char <= ?F) end) if has_hex_digits do # It's a hex number - convert from hex to decimal hex_string = List.to_string(integer_chars) try do integer_value = String.to_integer(hex_string, 16) new_state = %{state | chars: chars} {{:integer, line, integer_value}, new_state} rescue _ -> # Fall back to treating as atom if hex conversion fails atom_value = String.to_atom(hex_string) new_state = %{state | chars: chars} {{:atom, line, atom_value}, new_state} end else # Regular decimal integer integer_value = List.to_integer(integer_chars) new_state = %{state | chars: chars} {{:integer, line, integer_value}, new_state} end end # Scan identifier/atom name defp scan_name([char | chars], acc, line, state) when (char >= ?a and char <= ?z) or (char >= ?A and char <= ?Z) or (char >= ?0 and char <= ?9) or char == ?_ or char == ?- do scan_name(chars, [char | acc], line, state) end defp scan_name(chars, acc, line, state) do # End of name name_chars = Enum.reverse(acc) name_string = List.to_string(name_chars) # Determine if it's a reserved word, variable, or atom token = case classify_name(name_string, name_chars) do {:reserved, atom} -> {atom, line} {:variable, _} -> {:variable, line, name_string} {:atom, _} -> {:atom, line, String.to_atom(name_string)} end new_state = %{state | chars: chars} {token, new_state} end # Classify name as reserved word, variable, or atom defp classify_name(name_string, name_chars) do # Check if it's a reserved word first case reserved_word(name_string) do nil -> # Not a reserved word, check if variable or atom case name_chars do [first_char | _] when first_char >= ?A and first_char <= ?Z -> {:variable, name_string} _ -> {:atom, name_string} end reserved_atom -> {:reserved, reserved_atom} end end defp reserved_word(word), do: Map.get(@reserved_words, word) end